initial telemetry setup (#69330)

This commit is contained in:
Michael Olorunnisola 2020-07-13 20:52:25 -04:00 committed by GitHub
parent 2340f8a59b
commit 8325222c0a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 546 additions and 25 deletions

View file

@ -114,6 +114,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
initSavedObjects(core.savedObjects);
initUiSettings(core.uiSettings);
initUsageCollectors({
core,
kibanaIndex: globalConfig.kibana.index,
ml: plugins.ml,
usageCollection: plugins.usageCollection,

View file

@ -4,20 +4,32 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { LegacyAPICaller } from '../../../../../src/core/server';
import { LegacyAPICaller, CoreSetup } from '../../../../../src/core/server';
import { CollectorDependencies } from './types';
import { DetectionsUsage, fetchDetectionsUsage } from './detections';
import { EndpointUsage, getEndpointTelemetryFromFleet } from './endpoints';
export type RegisterCollector = (deps: CollectorDependencies) => void;
export interface UsageData {
detections: DetectionsUsage;
endpoints: EndpointUsage;
}
export const registerCollector: RegisterCollector = ({ kibanaIndex, ml, usageCollection }) => {
export async function getInternalSavedObjectsClient(core: CoreSetup) {
return core.getStartServices().then(async ([coreStart]) => {
return coreStart.savedObjects.createInternalRepository();
});
}
export const registerCollector: RegisterCollector = ({
core,
kibanaIndex,
ml,
usageCollection,
}) => {
if (!usageCollection) {
return;
}
const collector = usageCollection.makeUsageCollector<UsageData>({
type: 'security_solution',
schema: {
@ -43,11 +55,32 @@ export const registerCollector: RegisterCollector = ({ kibanaIndex, ml, usageCol
},
},
},
endpoints: {
total_installed: { type: 'long' },
active_within_last_24_hours: { type: 'long' },
os: {
full_name: { type: 'keyword' },
platform: { type: 'keyword' },
version: { type: 'keyword' },
count: { type: 'long' },
},
policies: {
malware: {
success: { type: 'long' },
warning: { type: 'long' },
failure: { type: 'long' },
},
},
},
},
isReady: () => kibanaIndex.length > 0,
fetch: async (callCluster: LegacyAPICaller): Promise<UsageData> => ({
detections: await fetchDetectionsUsage(kibanaIndex, callCluster, ml),
}),
fetch: async (callCluster: LegacyAPICaller): Promise<UsageData> => {
const savedObjectsClient = await getInternalSavedObjectsClient(core);
return {
detections: await fetchDetectionsUsage(kibanaIndex, callCluster, ml),
endpoints: await getEndpointTelemetryFromFleet(savedObjectsClient),
};
},
});
usageCollection.registerCollector(collector);

View file

@ -3,7 +3,7 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { INTERNAL_IMMUTABLE_KEY } from '../../common/constants';
import { INTERNAL_IMMUTABLE_KEY } from '../../../common/constants';
export const getMockJobSummaryResponse = () => [
{

View file

@ -4,20 +4,20 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { LegacyAPICaller } from '../../../../../src/core/server';
import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks';
import { jobServiceProvider } from '../../../ml/server/models/job_service';
import { DataRecognizer } from '../../../ml/server/models/data_recognizer';
import { mlServicesMock } from '../lib/machine_learning/mocks';
import { LegacyAPICaller } from '../../../../../../src/core/server';
import { elasticsearchServiceMock } from '../../../../../../src/core/server/mocks';
import { jobServiceProvider } from '../../../../ml/server/models/job_service';
import { DataRecognizer } from '../../../../ml/server/models/data_recognizer';
import { mlServicesMock } from '../../lib/machine_learning/mocks';
import {
getMockJobSummaryResponse,
getMockListModulesResponse,
getMockRulesResponse,
} from './detections.mocks';
import { fetchDetectionsUsage } from './detections';
import { fetchDetectionsUsage } from './index';
jest.mock('../../../ml/server/models/job_service');
jest.mock('../../../ml/server/models/data_recognizer');
jest.mock('../../../../ml/server/models/job_service');
jest.mock('../../../../ml/server/models/data_recognizer');
describe('Detections Usage', () => {
describe('fetchDetectionsUsage()', () => {

View file

@ -6,15 +6,15 @@
import { SearchParams } from 'elasticsearch';
import { LegacyAPICaller, SavedObjectsClient } from '../../../../../src/core/server';
import { LegacyAPICaller, SavedObjectsClient } from '../../../../../../src/core/server';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { jobServiceProvider } from '../../../ml/server/models/job_service';
import { jobServiceProvider } from '../../../../ml/server/models/job_service';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { DataRecognizer } from '../../../ml/server/models/data_recognizer';
import { MlPluginSetup } from '../../../ml/server';
import { SIGNALS_ID, INTERNAL_IMMUTABLE_KEY } from '../../common/constants';
import { DetectionRulesUsage, MlJobsUsage } from './detections';
import { isJobStarted } from '../../common/machine_learning/helpers';
import { DataRecognizer } from '../../../../ml/server/models/data_recognizer';
import { MlPluginSetup } from '../../../../ml/server';
import { SIGNALS_ID, INTERNAL_IMMUTABLE_KEY } from '../../../common/constants';
import { DetectionRulesUsage, MlJobsUsage } from './index';
import { isJobStarted } from '../../../common/machine_learning/helpers';
interface DetectionsMetric {
isElastic: boolean;

View file

@ -4,9 +4,9 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { LegacyAPICaller } from '../../../../../src/core/server';
import { LegacyAPICaller } from '../../../../../../src/core/server';
import { getMlJobsUsage, getRulesUsage } from './detections_helpers';
import { MlPluginSetup } from '../../../ml/server';
import { MlPluginSetup } from '../../../../ml/server';
interface FeatureUsage {
enabled: number;

View file

@ -0,0 +1,131 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { SavedObjectsFindResponse } from 'src/core/server';
import { AgentEventSOAttributes } from './../../../../ingest_manager/common/types/models/agent';
import {
AGENT_SAVED_OBJECT_TYPE,
AGENT_EVENT_SAVED_OBJECT_TYPE,
} from '../../../../ingest_manager/common/constants/agent';
import { Agent } from '../../../../ingest_manager/common';
import { FLEET_ENDPOINT_PACKAGE_CONSTANT } from './fleet_saved_objects';
const testAgentId = 'testAgentId';
const testConfigId = 'testConfigId';
/** Mock OS Platform for endpoint telemetry */
export const MockOSPlatform = 'somePlatform';
/** Mock OS Name for endpoint telemetry */
export const MockOSName = 'somePlatformName';
/** Mock OS Version for endpoint telemetry */
export const MockOSVersion = '1';
/** Mock OS Full Name for endpoint telemetry */
export const MockOSFullName = 'somePlatformFullName';
/**
*
* @param lastCheckIn - the last time the agent checked in. Defaults to current ISO time.
* @description We request the install and OS related telemetry information from the 'fleet-agents' saved objects in ingest_manager. This mocks that response
*/
export const mockFleetObjectsResponse = (
lastCheckIn = new Date().toISOString()
): SavedObjectsFindResponse<Agent> => ({
page: 1,
per_page: 20,
total: 1,
saved_objects: [
{
type: AGENT_SAVED_OBJECT_TYPE,
id: testAgentId,
attributes: {
active: true,
id: testAgentId,
config_id: 'randoConfigId',
type: 'PERMANENT',
user_provided_metadata: {},
enrolled_at: lastCheckIn,
current_error_events: [],
local_metadata: {
elastic: {
agent: {
id: testAgentId,
},
},
host: {
hostname: 'testDesktop',
name: 'testDesktop',
id: 'randoHostId',
},
os: {
platform: MockOSPlatform,
version: MockOSVersion,
name: MockOSName,
full: MockOSFullName,
},
},
packages: [FLEET_ENDPOINT_PACKAGE_CONSTANT, 'system'],
last_checkin: lastCheckIn,
},
references: [],
updated_at: lastCheckIn,
version: 'WzI4MSwxXQ==',
score: 0,
},
],
});
/**
*
* @param running - allows us to set whether the mocked endpoint is in an active or disabled/failed state
* @param updatedDate - the last time the endpoint was updated. Defaults to current ISO time.
* @description We request the events triggered by the agent and get the most recent endpoint event to confirm it is still running. This allows us to mock both scenarios
*/
export const mockFleetEventsObjectsResponse = (
running?: boolean,
updatedDate = new Date().toISOString()
): SavedObjectsFindResponse<AgentEventSOAttributes> => {
return {
page: 1,
per_page: 20,
total: 2,
saved_objects: [
{
type: AGENT_EVENT_SAVED_OBJECT_TYPE,
id: 'id1',
attributes: {
agent_id: testAgentId,
type: running ? 'STATE' : 'ERROR',
timestamp: updatedDate,
subtype: running ? 'RUNNING' : 'FAILED',
message: `Application: endpoint-security--8.0.0[d8f7f6e8-9375-483c-b456-b479f1d7a4f2]: State changed to ${
running ? 'RUNNING' : 'FAILED'
}: `,
config_id: testConfigId,
},
references: [],
updated_at: updatedDate,
version: 'WzExOCwxXQ==',
score: 0,
},
{
type: AGENT_EVENT_SAVED_OBJECT_TYPE,
id: 'id2',
attributes: {
agent_id: testAgentId,
type: 'STATE',
timestamp: updatedDate,
subtype: 'STARTING',
message:
'Application: endpoint-security--8.0.0[d8f7f6e8-9375-483c-b456-b479f1d7a4f2]: State changed to STARTING: Starting',
config_id: testConfigId,
},
references: [],
updated_at: updatedDate,
version: 'WzExNywxXQ==',
score: 0,
},
],
};
};

View file

@ -0,0 +1,116 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { savedObjectsRepositoryMock } from 'src/core/server/mocks';
import {
mockFleetObjectsResponse,
mockFleetEventsObjectsResponse,
MockOSFullName,
MockOSPlatform,
MockOSVersion,
} from './endpoint.mocks';
import { ISavedObjectsRepository, SavedObjectsFindResponse } from 'src/core/server';
import { AgentEventSOAttributes } from '../../../../ingest_manager/common/types/models/agent';
import { Agent } from '../../../../ingest_manager/common';
import * as endpointTelemetry from './index';
import * as fleetSavedObjects from './fleet_saved_objects';
describe('test security solution endpoint telemetry', () => {
let mockSavedObjectsRepository: jest.Mocked<ISavedObjectsRepository>;
let getFleetSavedObjectsMetadataSpy: jest.SpyInstance<Promise<SavedObjectsFindResponse<Agent>>>;
let getFleetEventsSavedObjectsSpy: jest.SpyInstance<Promise<
SavedObjectsFindResponse<AgentEventSOAttributes>
>>;
beforeAll(() => {
getFleetEventsSavedObjectsSpy = jest.spyOn(fleetSavedObjects, 'getFleetEventsSavedObjects');
getFleetSavedObjectsMetadataSpy = jest.spyOn(fleetSavedObjects, 'getFleetSavedObjectsMetadata');
mockSavedObjectsRepository = savedObjectsRepositoryMock.create();
});
afterAll(() => {
jest.resetAllMocks();
});
it('should have a default shape', () => {
expect(endpointTelemetry.getDefaultEndpointTelemetry()).toMatchInlineSnapshot(`
Object {
"active_within_last_24_hours": 0,
"os": Array [],
"total_installed": 0,
}
`);
});
describe('when an agent has not been installed', () => {
it('should return the default shape if no agents are found', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve({ saved_objects: [], total: 0, per_page: 0, page: 0 })
);
const emptyEndpointTelemetryData = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(getFleetSavedObjectsMetadataSpy).toHaveBeenCalled();
expect(emptyEndpointTelemetryData).toEqual({
total_installed: 0,
active_within_last_24_hours: 0,
os: [],
});
});
});
describe('when an agent has been installed', () => {
it('should show one enpoint installed but it is inactive', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getFleetEventsSavedObjectsSpy.mockImplementation(() =>
Promise.resolve(mockFleetEventsObjectsResponse())
);
const emptyEndpointTelemetryData = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(emptyEndpointTelemetryData).toEqual({
total_installed: 1,
active_within_last_24_hours: 0,
os: [
{
full_name: MockOSFullName,
platform: MockOSPlatform,
version: MockOSVersion,
count: 1,
},
],
});
});
it('should show one endpoint installed and it is active', async () => {
getFleetSavedObjectsMetadataSpy.mockImplementation(() =>
Promise.resolve(mockFleetObjectsResponse())
);
getFleetEventsSavedObjectsSpy.mockImplementation(() =>
Promise.resolve(mockFleetEventsObjectsResponse(true))
);
const emptyEndpointTelemetryData = await endpointTelemetry.getEndpointTelemetryFromFleet(
mockSavedObjectsRepository
);
expect(emptyEndpointTelemetryData).toEqual({
total_installed: 1,
active_within_last_24_hours: 1,
os: [
{
full_name: MockOSFullName,
platform: MockOSPlatform,
version: MockOSVersion,
count: 1,
},
],
});
});
});
});

View file

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { ISavedObjectsRepository } from 'src/core/server';
import { AgentEventSOAttributes } from './../../../../ingest_manager/common/types/models/agent';
import {
AGENT_SAVED_OBJECT_TYPE,
AGENT_EVENT_SAVED_OBJECT_TYPE,
} from './../../../../ingest_manager/common/constants/agent';
import { Agent, DefaultPackages as FleetDefaultPackages } from '../../../../ingest_manager/common';
export const FLEET_ENDPOINT_PACKAGE_CONSTANT = FleetDefaultPackages.endpoint;
export const getFleetSavedObjectsMetadata = async (savedObjectsClient: ISavedObjectsRepository) =>
savedObjectsClient.find<Agent>({
type: AGENT_SAVED_OBJECT_TYPE,
fields: ['packages', 'last_checkin', 'local_metadata'],
filter: `${AGENT_SAVED_OBJECT_TYPE}.attributes.packages: ${FLEET_ENDPOINT_PACKAGE_CONSTANT}`,
sortField: 'enrolled_at',
sortOrder: 'desc',
});
export const getFleetEventsSavedObjects = async (
savedObjectsClient: ISavedObjectsRepository,
agentId: string
) =>
savedObjectsClient.find<AgentEventSOAttributes>({
type: AGENT_EVENT_SAVED_OBJECT_TYPE,
filter: `${AGENT_EVENT_SAVED_OBJECT_TYPE}.attributes.agent_id: ${agentId} and ${AGENT_EVENT_SAVED_OBJECT_TYPE}.attributes.message: "${FLEET_ENDPOINT_PACKAGE_CONSTANT}"`,
sortField: 'timestamp',
sortOrder: 'desc',
search: agentId,
searchFields: ['agent_id'],
});

View file

@ -0,0 +1,159 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { ISavedObjectsRepository } from 'src/core/server';
import { AgentMetadata } from '../../../../ingest_manager/common/types/models/agent';
import {
getFleetSavedObjectsMetadata,
getFleetEventsSavedObjects,
FLEET_ENDPOINT_PACKAGE_CONSTANT,
} from './fleet_saved_objects';
export interface AgentOSMetadataTelemetry {
full_name: string;
platform: string;
version: string;
count: number;
}
export interface PoliciesTelemetry {
malware: {
success: number;
warning: number;
failure: number;
};
}
export interface EndpointUsage {
total_installed: number;
active_within_last_24_hours: number;
os: AgentOSMetadataTelemetry[];
policies?: PoliciesTelemetry; // TODO: make required when able to enable policy information
}
export interface AgentLocalMetadata extends AgentMetadata {
elastic: {
agent: {
id: string;
};
};
host: {
id: string;
};
os: {
name: string;
platform: string;
version: string;
full: string;
};
}
export type OSTracker = Record<string, AgentOSMetadataTelemetry>;
/**
* @description returns an empty telemetry object to be incrmented and updated within the `getEndpointTelemetryFromFleet` fn
*/
export const getDefaultEndpointTelemetry = (): EndpointUsage => ({
total_installed: 0,
active_within_last_24_hours: 0,
os: [],
});
export const trackEndpointOSTelemetry = (
os: AgentLocalMetadata['os'],
osTracker: OSTracker
): OSTracker => {
const updatedOSTracker = { ...osTracker };
const { version: osVersion, platform: osPlatform, full: osFullName } = os;
if (osFullName && osVersion) {
if (updatedOSTracker[osFullName]) updatedOSTracker[osFullName].count += 1;
else {
updatedOSTracker[osFullName] = {
full_name: osFullName,
platform: osPlatform,
version: osVersion,
count: 1,
};
}
}
return updatedOSTracker;
};
/**
* @description This aggregates the telemetry details from the two fleet savedObject sources, `fleet-agents` and `fleet-agent-events` to populate
* the telemetry details for endpoint. Since we cannot access our own indices due to `kibana_system` not having access, this is the best alternative.
* Once the data is requested, we iterate over all agents with endpoints registered, and then request the events for each active agent (within last 24 hours)
* to confirm whether or not the endpoint is still active
*/
export const getEndpointTelemetryFromFleet = async (
savedObjectsClient: ISavedObjectsRepository
): Promise<EndpointUsage> => {
// Retrieve every agent that references the endpoint as an installed package. It will not be listed if it was never installed
const { saved_objects: endpointAgents } = await getFleetSavedObjectsMetadata(savedObjectsClient);
const endpointTelemetry = getDefaultEndpointTelemetry();
// If there are no installed endpoints return the default telemetry object
if (!endpointAgents || endpointAgents.length < 1) return endpointTelemetry;
// Use unique hosts to prevent any potential duplicates
const uniqueHostIds: Set<string> = new Set();
// Need unique agents to get events data for those that have run in last 24 hours
const uniqueAgentIds: Set<string> = new Set();
const aDayAgo = new Date();
aDayAgo.setDate(aDayAgo.getDate() - 1);
let osTracker: OSTracker = {};
const endpointMetadataTelemetry = endpointAgents.reduce(
(metadataTelemetry, { attributes: metadataAttributes }) => {
const { last_checkin: lastCheckin, local_metadata: localMetadata } = metadataAttributes;
// The extended AgentMetadata is just an empty blob, so cast to account for our specific use case
const { host, os, elastic } = localMetadata as AgentLocalMetadata;
if (lastCheckin && new Date(lastCheckin) > aDayAgo) {
// Get agents that have checked in within the last 24 hours to later see if their endpoints are running
uniqueAgentIds.add(elastic.agent.id);
}
if (host && uniqueHostIds.has(host.id)) {
return metadataTelemetry;
} else {
uniqueHostIds.add(host.id);
osTracker = trackEndpointOSTelemetry(os, osTracker);
return metadataTelemetry;
}
},
endpointTelemetry
);
// All unique agents with an endpoint installed. You can technically install a new agent on a host, so relying on most recently installed.
endpointTelemetry.total_installed = uniqueHostIds.size;
// Get the objects to populate our OS Telemetry
endpointMetadataTelemetry.os = Object.values(osTracker);
// Check for agents running in the last 24 hours whose endpoints are still active
for (const agentId of uniqueAgentIds) {
const { saved_objects: agentEvents } = await getFleetEventsSavedObjects(
savedObjectsClient,
agentId
);
const lastEndpointStatus = agentEvents.find((agentEvent) =>
agentEvent.attributes.message.includes(FLEET_ENDPOINT_PACKAGE_CONSTANT)
);
/*
We can assume that if the last status of the endpoint is RUNNING and the agent has checked in within the last 24 hours
then the endpoint has still been running within the last 24 hours. If / when we get the policy response, then we can use that
instead
*/
const endpointIsActive = lastEndpointStatus?.attributes.subtype === 'RUNNING';
if (endpointIsActive) {
endpointMetadataTelemetry.active_within_last_24_hours += 1;
}
}
return endpointMetadataTelemetry;
};

View file

@ -4,9 +4,10 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { CoreSetup } from 'src/core/server';
import { SetupPlugins } from '../plugin';
export type CollectorDependencies = { kibanaIndex: string } & Pick<
export type CollectorDependencies = { kibanaIndex: string; core: CoreSetup } & Pick<
SetupPlugins,
'ml' | 'usageCollection'
>;

View file

@ -217,6 +217,49 @@
}
}
}
},
"endpoints": {
"properties": {
"total_installed": {
"type": "long"
},
"active_within_last_24_hours": {
"type": "long"
},
"os": {
"properties": {
"full_name": {
"type": "keyword"
},
"platform": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"count": {
"type": "long"
}
}
},
"policies": {
"properties": {
"malware": {
"properties": {
"success": {
"type": "long"
},
"warning": {
"type": "long"
},
"failure": {
"type": "long"
}
}
}
}
}
}
}
}
},