maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Adding related event category stats for resolver nodes (#67909)

Browse source 
* Allowing the categories to be specified for related events

* Adding checks in the api tests for the stats

* Adding more comments

* Allow array or number of cateogires generation and fix up comment

* Fixing type error

* Renaming to byCategory

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Jonathan Buttner 2020-06-04 16:10:42 -04:00 committed by GitHub
parent c8e6855281
commit 8fe1eb1f78
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 356 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts
View file

@ -3,7 +3,14 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { EndpointDocGenerator, Event, Tree, TreeNode } from './generate_data';
import {
EndpointDocGenerator,
Event,
Tree,
TreeNode,
RelatedEventCategory,
ECSCategory,
} from './generate_data';
interface Node {
events: Event[];
@ -106,7 +113,11 @@ describe('data generator', () => {
generations,
percentTerminated: 100,
percentWithRelated: 100,
relatedEvents: 4,
relatedEvents: [
{ category: RelatedEventCategory.Driver, count: 1 },
{ category: RelatedEventCategory.File, count: 2 },
{ category: RelatedEventCategory.Network, count: 1 },
],
});
});
@ -117,6 +128,36 @@ describe('data generator', () => {
return (inRelated || inLifecycle) && event.process.entity_id === node.id;
};
it('has the right related events for each node', () => {
const checkRelatedEvents = (node: TreeNode) => {
expect(node.relatedEvents.length).toEqual(4);
const counts: Record<string, number> = {};
for (const event of node.relatedEvents) {
if (Array.isArray(event.event.category)) {
for (const cat of event.event.category) {
counts[cat] = counts[cat] + 1 || 1;
}
} else {
counts[event.event.category] = counts[event.event.category] + 1 || 1;
}
}
expect(counts[ECSCategory.Driver]).toEqual(1);
expect(counts[ECSCategory.File]).toEqual(2);
expect(counts[ECSCategory.Network]).toEqual(1);
};
for (const node of tree.ancestry.values()) {
checkRelatedEvents(node);
}
for (const node of tree.children.values()) {
checkRelatedEvents(node);
}
checkRelatedEvents(tree.origin);
});
it('has the right number of ancestors', () => {
expect(tree.ancestry.size).toEqual(ancestors);
});

151
x-pack/plugins/security_solution/common/endpoint/generate_data.ts
View file

@ -24,7 +24,7 @@ interface EventOptions {
entityID?: string;
parentEntityID?: string;
eventType?: string;
eventCategory?: string;
eventCategory?: string | string[];
processName?: string;
}
@ -75,21 +75,98 @@ const POLICIES: Array<{ name: string; id: string }> = [
const FILE_OPERATIONS: string[] = ['creation', 'open', 'rename', 'execution', 'deletion'];
interface EventInfo {
category: string;
category: string | string[];
/**
* This denotes the `event.type` field for when an event is created, this can be `start` or `creation`
*/
creationType: string;
}
/**
* The valid ecs categories.
*/
export enum ECSCategory {
Driver = 'driver',
File = 'file',
Network = 'network',
/**
* Registry has not been added to ecs yet.
*/
Registry = 'registry',
Authentication = 'authentication',
Session = 'session',
}
/**
* High level categories for related events. These specify the type of related events that should be generated.
*/
export enum RelatedEventCategory {
/**
* The Random category allows the related event categories to be chosen randomly
*/
Random = 'random',
Driver = 'driver',
File = 'file',
Network = 'network',
Registry = 'registry',
/**
* Security isn't an actual category but defines a type of related event to be created.
*/
Security = 'security',
}
/**
* This map defines the relationship between a higher level event type defined by the RelatedEventCategory enums and
* the ECS categories that is should map to. This should only be used for tests that need to determine the exact
* ecs categories that were created based on the related event information passed to the generator.
*/
export const categoryMapping: Record<RelatedEventCategory, ECSCategory | ECSCategory[] | ''> = {
[RelatedEventCategory.Security]: [ECSCategory.Authentication, ECSCategory.Session],
[RelatedEventCategory.Driver]: ECSCategory.Driver,
[RelatedEventCategory.File]: ECSCategory.File,
[RelatedEventCategory.Network]: ECSCategory.Network,
[RelatedEventCategory.Registry]: ECSCategory.Registry,
/**
* Random is only used by the generator to indicate that it should randomly choose the event information when generating
* related events. It does not map to a specific ecs category.
*/
[RelatedEventCategory.Random]: '',
};
/**
* The related event category and number of events that should be generated.
*/
export interface RelatedEventInfo {
category: RelatedEventCategory;
count: number;
}
// These are from the v1 schemas and aren't all valid ECS event categories, still in flux
const OTHER_EVENT_CATEGORIES: EventInfo[] = [
{ category: 'driver', creationType: 'start' },
{ category: 'file', creationType: 'creation' },
{ category: 'library', creationType: 'start' },
{ category: 'network', creationType: 'start' },
{ category: 'registry', creationType: 'creation' },
];
const OTHER_EVENT_CATEGORIES: Record<
Exclude<RelatedEventCategory, RelatedEventCategory.Random>,
EventInfo
> = {
[RelatedEventCategory.Security]: {
category: categoryMapping[RelatedEventCategory.Security],
creationType: 'start',
},
[RelatedEventCategory.Driver]: {
category: categoryMapping[RelatedEventCategory.Driver],
creationType: 'start',
},
[RelatedEventCategory.File]: {
category: categoryMapping[RelatedEventCategory.File],
creationType: 'creation',
},
[RelatedEventCategory.Network]: {
category: categoryMapping[RelatedEventCategory.Network],
creationType: 'start',
},
[RelatedEventCategory.Registry]: {
category: categoryMapping[RelatedEventCategory.Registry],
creationType: 'creation',
},
};
interface HostInfo {
elastic: {
@ -164,7 +241,7 @@ export interface TreeOptions {
ancestors?: number;
generations?: number;
children?: number;
relatedEvents?: number;
relatedEvents?: RelatedEventInfo[];
percentWithRelated?: number;
percentTerminated?: number;
alwaysGenMaxChildrenPerNode?: boolean;
@ -487,7 +564,8 @@ export class EndpointDocGenerator {
* @param alertAncestors - number of ancestor generations to create relative to the alert
* @param childGenerations - number of child generations to create relative to the alert
* @param maxChildrenPerNode - maximum number of children for any given node in the tree
* @param relatedEventsPerNode - number of related events (file, registry, etc) to create for each process event in the tree
* @param relatedEventsPerNode - can be an array of RelatedEventInfo objects describing the related events that should be generated for each process node
* or a number which defines the number of related events and will default to random categories
* @param percentNodesWithRelated - percent of nodes which should have related events
* @param percentTerminated - percent of nodes which will have process termination events
* @param alwaysGenMaxChildrenPerNode - flag to always return the max children per node instead of it being a random number of children
@ -496,7 +574,7 @@ export class EndpointDocGenerator {
alertAncestors?: number,
childGenerations?: number,
maxChildrenPerNode?: number,
relatedEventsPerNode?: number,
relatedEventsPerNode?: RelatedEventInfo[] | number,
percentNodesWithRelated?: number,
percentTerminated?: number,
alwaysGenMaxChildrenPerNode?: boolean
@ -525,13 +603,14 @@ export class EndpointDocGenerator {
/**
* Creates an alert event and associated process ancestry. The alert event will always be the last event in the return array.
* @param alertAncestors - number of ancestor generations to create
* @param relatedEventsPerNode - number of related events to add to each process node being created
* @param relatedEventsPerNode - can be an array of RelatedEventInfo objects describing the related events that should be generated for each process node
* or a number which defines the number of related events and will default to random categories
* @param pctWithRelated - percent of ancestors that will have related events
* @param pctWithTerminated - percent of ancestors that will have termination events
*/
public createAlertEventAncestry(
alertAncestors = 3,
relatedEventsPerNode = 5,
relatedEventsPerNode: RelatedEventInfo[] | number = 5,
pctWithRelated = 30,
pctWithTerminated = 100
): Event[] {
@ -611,7 +690,8 @@ export class EndpointDocGenerator {
* @param root - The process event to use as the root node of the tree
* @param generations - number of child generations to create. The root node is not counted as a generation.
* @param maxChildrenPerNode - maximum number of children for any given node in the tree
* @param relatedEventsPerNode - number of related events (file, registry, etc) to create for each process event in the tree
* @param relatedEventsPerNode - can be an array of RelatedEventInfo objects describing the related events that should be generated for each process node
* or a number which defines the number of related events and will default to random categories
* @param percentNodesWithRelated - percent of nodes which should have related events
* @param percentChildrenTerminated - percent of nodes which will have process termination events
* @param alwaysGenMaxChildrenPerNode - flag to always return the max children per node instead of it being a random number of children
@ -620,7 +700,7 @@ export class EndpointDocGenerator {
root: Event,
generations = 2,
maxChildrenPerNode = 2,
relatedEventsPerNode = 3,
relatedEventsPerNode: RelatedEventInfo[] | number = 3,
percentNodesWithRelated = 100,
percentChildrenTerminated = 100,
alwaysGenMaxChildrenPerNode = false
@ -686,25 +766,40 @@ export class EndpointDocGenerator {
/**
* Creates related events for a process event
* @param node - process event to relate events to by entityID
* @param numRelatedEvents - number of related events to generate
* @param relatedEvents - can be an array of RelatedEventInfo objects describing the related events that should be generated for each process node
* or a number which defines the number of related events and will default to random categories
* @param processDuration - maximum number of seconds after process event that related event timestamp can be
*/
public *relatedEventsGenerator(
node: Event,
numRelatedEvents = 10,
relatedEvents: RelatedEventInfo[] | number = 10,
processDuration: number = 6 * 3600
) {
for (let i = 0; i < numRelatedEvents; i++) {
const eventInfo = this.randomChoice(OTHER_EVENT_CATEGORIES);
let relatedEventsInfo: RelatedEventInfo[];
if (typeof relatedEvents === 'number') {
relatedEventsInfo = [{ category: RelatedEventCategory.Random, count: relatedEvents }];
} else {
relatedEventsInfo = relatedEvents;
}
for (const event of relatedEventsInfo) {
let eventInfo: EventInfo;
const ts = node['@timestamp'] + this.randomN(processDuration) * 1000;
yield this.generateEvent({
timestamp: ts,
entityID: node.process.entity_id,
parentEntityID: node.process.parent?.entity_id,
eventCategory: eventInfo.category,
eventType: eventInfo.creationType,
});
for (let i = 0; i < event.count; i++) {
if (event.category === RelatedEventCategory.Random) {
eventInfo = this.randomChoice(Object.values(OTHER_EVENT_CATEGORIES));
} else {
eventInfo = OTHER_EVENT_CATEGORIES[event.category];
}
const ts = node['@timestamp'] + this.randomN(processDuration) * 1000;
yield this.generateEvent({
timestamp: ts,
entityID: node.process.entity_id,
parentEntityID: node.process.parent?.entity_id,
eventCategory: eventInfo.category,
eventType: eventInfo.creationType,
});
}
}
}

21
x-pack/plugins/security_solution/common/endpoint/types.ts
View file

@ -41,14 +41,30 @@ type ImmutableMap<K, V> = ReadonlyMap<Immutable<K>, Immutable<V>>;
type ImmutableSet<T> = ReadonlySet<Immutable<T>>;
type ImmutableObject<T> = { readonly [K in keyof T]: Immutable<T[K]> };
export interface EventStats {
/**
* The total number of related events (all events except process and alerts) that exist for a node.
*/
total: number;
/**
* A mapping of ECS event.category to the number of related events are marked with that category
* For example:
* {
* network: 5,
* file: 2
* }
*/
byCategory: Record<string, number>;
}
/**
* Statistical information for a node in a resolver tree.
*/
export interface ResolverNodeStats {
/**
* The total number of related events (all events except process and alerts) that exist for a node.
* The stats for related events (excludes alerts and process events) for a particular node in the resolver tree.
*/
totalEvents: number;
events: EventStats;
/**
* The total number of alerts that exist for a node.
*/
@ -379,6 +395,7 @@ export interface LegacyEndpointEvent {
event?: {
action?: string;
type?: string;
category?: string | string[];
};
}

124
x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts
View file

@ -5,13 +5,23 @@
*/
import { SearchResponse } from 'elasticsearch';
import { ResolverQuery } from './base';
import { ResolverEvent } from '../../../../../common/endpoint/types';
import { ResolverEvent, EventStats } from '../../../../../common/endpoint/types';
import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/public';
import { AggBucket } from '../utils/pagination';
export interface StatsResult {
alerts: Record<string, number>;
events: Record<string, number>;
events: Record<string, EventStats>;
}
interface CategoriesAgg extends AggBucket {
/**
* The reason categories is optional here is because if no data was returned in the query the categories aggregation
* will not be defined on the response (because it's a sub aggregation).
*/
categories?: {
buckets?: AggBucket[];
};
}
export class StatsQuery extends ResolverQuery<StatsResult> {
@ -64,13 +74,25 @@ export class StatsQuery extends ResolverQuery<StatsResult> {
alerts: {
filter: { term: { 'event.kind': 'alert' } },
aggs: {
ids: { terms: { field: 'endgame.data.alert_details.acting_process.unique_pid' } },
ids: {
terms: {
field: 'endgame.data.alert_details.acting_process.unique_pid',
size: uniquePIDs.length,
},
},
},
},
events: {
filter: { term: { 'event.kind': 'event' } },
aggs: {
ids: { terms: { field: 'endgame.unique_pid' } },
ids: {
terms: { field: 'endgame.unique_pid', size: uniquePIDs.length },
aggs: {
categories: {
terms: { field: 'event.category', size: 1000 },
},
},
},
},
},
},
@ -112,34 +134,106 @@ export class StatsQuery extends ResolverQuery<StatsResult> {
alerts: {
filter: { term: { 'event.kind': 'alert' } },
aggs: {
ids: { terms: { field: 'process.entity_id' } },
ids: { terms: { field: 'process.entity_id', size: entityIDs.length } },
},
},
events: {
filter: { term: { 'event.kind': 'event' } },
aggs: {
ids: { terms: { field: 'process.entity_id' } },
ids: {
// The entityIDs array will be made up of alert and event entity_ids, so we're guaranteed that there
// won't be anymore unique process.entity_ids than the size of the array passed in
terms: { field: 'process.entity_id', size: entityIDs.length },
aggs: {
categories: {
// Currently ECS defines a small number of valid categories (under 10 right now), as ECS grows it's possible that the
// valid categories could exceed this hardcoded limit. If that happens we might want to revisit this
// and transition it to a composite aggregation so that we can paginate through all the possible response
terms: { field: 'event.category', size: 1000 },
},
},
},
},
},
},
};
}
public formatResponse(response: SearchResponse<ResolverEvent>): StatsResult {
const alerts = response.aggregations.alerts.ids.buckets.reduce(
(cummulative: Record<string, number>, bucket: AggBucket) => ({
...cummulative,
[bucket.key]: bucket.doc_count,
}),
{}
);
const events = response.aggregations.events.ids.buckets.reduce(
private static getEventStats(catAgg: CategoriesAgg): EventStats {
const total = catAgg.doc_count;
if (!catAgg.categories?.buckets) {
return {
total,
byCategory: {},
};
}
const byCategory: Record<string, number> = catAgg.categories.buckets.reduce(
(cummulative: Record<string, number>, bucket: AggBucket) => ({
...cummulative,
[bucket.key]: bucket.doc_count,
}),
{}
);
return {
total,
byCategory,
};
}
public formatResponse(response: SearchResponse<ResolverEvent>): StatsResult {
let alerts: Record<string, number> = {};
if (response.aggregations?.alerts?.ids?.buckets) {
alerts = response.aggregations.alerts.ids.buckets.reduce(
(cummulative: Record<string, number>, bucket: AggBucket) => ({
...cummulative,
[bucket.key]: bucket.doc_count,
}),
{}
);
}
/**
* The response for the events ids aggregation should look like this:
* "aggregations" : {
* "ids" : {
* "doc_count_error_upper_bound" : 0,
* "sum_other_doc_count" : 0,
* "buckets" : [
* {
* "key" : "entity_id1",
* "doc_count" : 3,
* "categories" : {
* "doc_count_error_upper_bound" : 0,
* "sum_other_doc_count" : 0,
* "buckets" : [
* {
* "key" : "session",
* "doc_count" : 3
* },
* {
* "key" : "authentication",
* "doc_count" : 2
* }
* ]
* }
* },
*
* Which would indicate that entity_id1 had 3 related events. 3 of the related events had category session,
* and 2 had authentication
*/
let events: Record<string, EventStats> = {};
if (response.aggregations?.events?.ids?.buckets) {
events = response.aggregations.events.ids.buckets.reduce(
(cummulative: Record<string, number>, bucket: CategoriesAgg) => ({
...cummulative,
[bucket.key]: StatsQuery.getEventStats(bucket),
}),
{}
);
}
return {
alerts,
events,

9
x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts
View file

@ -173,10 +173,13 @@ export class Fetcher {
const statsQuery = new StatsQuery(this.indexPattern, this.endpointID);
const ids = tree.ids();
const res = await statsQuery.search(this.client, ids);
const alerts = res?.alerts || {};
const events = res?.events || {};
const alerts = res.alerts;
const events = res.events;
ids.forEach((id) => {
tree.addStats(id, { totalAlerts: alerts[id] || 0, totalEvents: events[id] || 0 });
tree.addStats(id, {
totalAlerts: alerts[id] || 0,
events: events[id] || { total: 0, byCategory: {} },
});
});
}
}

5
x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts
View file

@ -81,7 +81,10 @@ export function createTree(entityID: string): ResolverTree {
},
stats: {
totalAlerts: 0,
totalEvents: 0,
events: {
total: 0,
byCategory: {},
},
},
};
}

53
x-pack/test/api_integration/apis/endpoint/resolver.ts
View file

@ -14,6 +14,7 @@ import {
ResolverChildren,
ResolverTree,
LegacyEndpointEvent,
ResolverNodeStats,
} from '../../../../plugins/security_solution/common/endpoint/types';
import { parentEntityId } from '../../../../plugins/security_solution/common/endpoint/models/event';
import { FtrProviderContext } from '../../ftr_provider_context';
@ -21,6 +22,9 @@ import {
Event,
Tree,
TreeNode,
RelatedEventCategory,
RelatedEventInfo,
categoryMapping,
} from '../../../../plugins/security_solution/common/endpoint/generate_data';
import { Options, GeneratedTrees } from '../../services/resolver';
@ -141,16 +145,60 @@ const compareArrays = (
});
};
/**
* Verifies that the stats received from ES for a node reflect the categories of events that the generator created.
*
* @param relatedEvents the related events received for a particular node
* @param categories the related event info used when generating the resolver tree
*/
const verifyStats = (stats: ResolverNodeStats | undefined, categories: RelatedEventInfo[]) => {
expect(stats).to.not.be(undefined);
let totalExpEvents = 0;
for (const cat of categories) {
const ecsCategories = categoryMapping[cat.category];
if (Array.isArray(ecsCategories)) {
// if there are multiple ecs categories used to define a related event, the count for all of them should be the same
// and they should equal what is defined in the categories used to generate the related events
for (const ecsCat of ecsCategories) {
expect(stats?.events.byCategory[ecsCat]).to.be(cat.count);
}
} else {
expect(stats?.events.byCategory[ecsCategories]).to.be(cat.count);
}
totalExpEvents += cat.count;
}
expect(stats?.events.total).to.be(totalExpEvents);
};
/**
* A helper function for verifying the stats information an array of nodes.
*
* @param nodes an array of lifecycle nodes that should have a stats field defined
* @param categories the related event info used when generating the resolver tree
*/
const verifyLifecycleStats = (nodes: LifecycleNode[], categories: RelatedEventInfo[]) => {
for (const node of nodes) {
verifyStats(node.stats, categories);
}
};
export default function resolverAPIIntegrationTests({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
const esArchiver = getService('esArchiver');
const resolver = getService('resolverGenerator');
const relatedEventsToGen = [
{ category: RelatedEventCategory.Driver, count: 2 },
{ category: RelatedEventCategory.File, count: 1 },
{ category: RelatedEventCategory.Registry, count: 1 },
];
let resolverTrees: GeneratedTrees;
let tree: Tree;
const treeOptions: Options = {
ancestors: 5,
relatedEvents: 4,
relatedEvents: relatedEventsToGen,
children: 3,
generations: 2,
percentTerminated: 100,
@ -563,14 +611,17 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC
expect(body.children.nextChild).to.equal(null);
expect(body.children.childNodes.length).to.equal(12);
verifyChildren(body.children.childNodes, tree, 4, 3);
verifyLifecycleStats(body.children.childNodes, relatedEventsToGen);
expect(body.ancestry.nextAncestor).to.equal(null);
verifyAncestry(body.ancestry.ancestors, tree, true);
verifyLifecycleStats(body.ancestry.ancestors, relatedEventsToGen);
expect(body.relatedEvents.nextEvent).to.equal(null);
compareArrays(tree.origin.relatedEvents, body.relatedEvents.events, true);
compareArrays(tree.origin.lifecycle, body.lifecycle, true);
verifyStats(body.stats, relatedEventsToGen);
});
});
});