maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

EMT-339: add policy response index and documents (#65004)

Browse source 
* EMT-339: add policy response index and documents

* EMT-339: add routes, unit and integration tests

* EMT-339: review comments, change types, url, update tests
This commit is contained in:
nnamdifrankie 2020-05-04 14:58:12 -04:00 committed by GitHub
parent 6c1f5ec81b
commit 99a5db6aab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 1469 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
x-pack/plugins/endpoint/common/generate_data.test.ts
View file

@ -45,6 +45,17 @@ describe('data generator', () => {
expect(metadata.host).not.toBeNull();
});
it('creates policy response documents', () => {
const timestamp = new Date().getTime();
const hostPolicyResponse = generator.generatePolicyResponse(timestamp);
expect(hostPolicyResponse['@timestamp']).toEqual(timestamp);
expect(hostPolicyResponse.event.created).toEqual(timestamp);
expect(hostPolicyResponse.endpoint).not.toBeNull();
expect(hostPolicyResponse.agent).not.toBeNull();
expect(hostPolicyResponse.host).not.toBeNull();
expect(hostPolicyResponse.endpoint.policy.applied).not.toBeNull();
});
it('creates alert event documents', () => {
const timestamp = new Date().getTime();
const alert = generator.generateAlert(timestamp);

229
x-pack/plugins/endpoint/common/generate_data.ts
View file

@ -12,9 +12,10 @@ import {
Host,
HostMetadata,
HostOS,
PolicyData,
HostPolicyResponse,
HostPolicyResponseActions,
HostPolicyResponseActionStatus,
PolicyData,
} from './types';
import { factory as policyFactory } from './models/policy_config';
@ -136,6 +137,13 @@ export class EndpointDocGenerator {
this.commonInfo.host.ip = this.randomArray(3, () => this.randomIP());
}
/**
* Creates new random policy id for the host to simulate new policy application
*/
public updatePolicyId() {
this.commonInfo.endpoint.policy.id = this.randomChoice(POLICIES).id;
}
private createHostData(): HostInfo {
return {
agent: {
@ -498,106 +506,145 @@ export class EndpointDocGenerator {
/**
* Generates a Host Policy response message
*/
generatePolicyResponse(): HostPolicyResponse {
public generatePolicyResponse(ts = new Date().getTime()): HostPolicyResponse {
const policyVersion = this.seededUUIDv4();
return {
'@timestamp': new Date().toISOString(),
'@timestamp': ts,
agent: {
id: this.commonInfo.agent.id,
version: '1.0.0-local.20200416.0',
},
elastic: {
agent: {
id: 'c2a9093e-e289-4c0a-aa44-8c32a414fa7a',
id: this.commonInfo.elastic.agent.id,
},
},
ecs: {
version: '1.0.0',
version: '1.4.0',
},
event: {
created: '2015-01-01T12:10:30Z',
kind: 'policy_response',
},
agent: {
version: '6.0.0-rc2',
id: '8a4f500d',
host: {
id: this.commonInfo.host.id,
},
endpoint: {
artifacts: {
'global-manifest': {
version: '1.2.3',
sha256: 'abcdef',
},
'endpointpe-v4-windows': {
version: '1.2.3',
sha256: 'abcdef',
},
'user-whitelist-windows': {
version: '1.2.3',
sha256: 'abcdef',
},
'global-whitelist-windows': {
version: '1.2.3',
sha256: 'abcdef',
},
},
policy: {
applied: {
version: '1.0.0',
id: '17d4b81d-9940-4b64-9de5-3e03ef1fb5cf',
status: HostPolicyResponseActionStatus.success,
actions: {
configure_elasticsearch_connection: {
message: 'elasticsearch comms configured successfully',
status: HostPolicyResponseActionStatus.success,
},
configure_kernel: {
message: 'Failed to configure kernel',
status: HostPolicyResponseActionStatus.failure,
},
configure_logging: {
message: 'Successfully configured logging',
status: HostPolicyResponseActionStatus.success,
},
configure_malware: {
message: 'Unexpected error configuring malware',
status: HostPolicyResponseActionStatus.failure,
},
connect_kernel: {
message: 'Successfully initialized minifilter',
status: HostPolicyResponseActionStatus.success,
},
detect_file_open_events: {
message: 'Successfully stopped file open event reporting',
status: HostPolicyResponseActionStatus.success,
},
detect_file_write_events: {
message: 'Failed to stop file write event reporting',
status: HostPolicyResponseActionStatus.success,
},
detect_image_load_events: {
message: 'Successfuly started image load event reporting',
status: HostPolicyResponseActionStatus.success,
},
detect_process_events: {
message: 'Successfully started process event reporting',
status: HostPolicyResponseActionStatus.success,
},
download_global_artifacts: {
message: 'Failed to download EXE model',
status: HostPolicyResponseActionStatus.success,
},
load_config: {
message: 'successfully parsed configuration',
status: HostPolicyResponseActionStatus.success,
},
load_malware_model: {
message: 'Error deserializing EXE model; no valid malware model installed',
status: HostPolicyResponseActionStatus.success,
},
read_elasticsearch_config: {
message: 'Successfully read Elasticsearch configuration',
status: HostPolicyResponseActionStatus.success,
},
read_events_config: {
message: 'Successfully read events configuration',
status: HostPolicyResponseActionStatus.success,
},
read_kernel_config: {
message: 'Succesfully read kernel configuration',
status: HostPolicyResponseActionStatus.success,
},
read_logging_config: {
message: 'field (logging.debugview) not found in config',
status: HostPolicyResponseActionStatus.success,
},
read_malware_config: {
message: 'Successfully read malware detect configuration',
status: HostPolicyResponseActionStatus.success,
},
workflow: {
message: 'Failed to apply a portion of the configuration (kernel)',
status: HostPolicyResponseActionStatus.success,
},
download_model: {
message: 'Failed to apply a portion of the configuration (kernel)',
status: HostPolicyResponseActionStatus.success,
},
ingest_events_config: {
message: 'Failed to apply a portion of the configuration (kernel)',
status: HostPolicyResponseActionStatus.success,
},
},
id: this.commonInfo.endpoint.policy.id,
policy: {
id: this.commonInfo.endpoint.policy.id,
version: policyVersion,
},
response: {
configurations: {
malware: {
status: HostPolicyResponseActionStatus.success,
concerned_actions: ['download_model', 'workflow', 'a_custom_future_action'],
},
events: {
status: HostPolicyResponseActionStatus.success,
concerned_actions: ['ingest_events_config', 'workflow'],
concerned_actions: this.randomHostPolicyResponseActions(),
status: this.randomHostPolicyResponseActionStatus(),
},
logging: {
status: HostPolicyResponseActionStatus.success,
concerned_actions: ['configure_elasticsearch_connection'],
concerned_actions: this.randomHostPolicyResponseActions(),
status: this.randomHostPolicyResponseActionStatus(),
},
malware: {
concerned_actions: this.randomHostPolicyResponseActions(),
status: this.randomHostPolicyResponseActionStatus(),
},
streaming: {
status: HostPolicyResponseActionStatus.success,
concerned_actions: [
'detect_file_open_events',
'download_global_artifacts',
'a_custom_future_action',
],
},
},
actions: {
download_model: {
status: HostPolicyResponseActionStatus.success,
message: 'model downloaded',
},
ingest_events_config: {
status: HostPolicyResponseActionStatus.success,
message: 'no action taken',
},
workflow: {
status: HostPolicyResponseActionStatus.success,
message: 'the flow worked well',
},
a_custom_future_action: {
status: HostPolicyResponseActionStatus.success,
message: 'future message',
},
configure_elasticsearch_connection: {
status: HostPolicyResponseActionStatus.success,
message: 'some message',
},
detect_file_open_events: {
status: HostPolicyResponseActionStatus.success,
message: 'some message',
},
download_global_artifacts: {
status: HostPolicyResponseActionStatus.success,
message: 'some message',
concerned_actions: this.randomHostPolicyResponseActions(),
status: this.randomHostPolicyResponseActionStatus(),
},
},
},
status: this.randomHostPolicyResponseActionStatus(),
version: policyVersion,
},
},
},
event: {
created: ts,
id: this.seededUUIDv4(),
kind: 'policy_response',
},
};
}
@ -644,6 +691,34 @@ export class EndpointDocGenerator {
private seededUUIDv4(): string {
return uuid.v4({ random: [...this.randomNGenerator(255, 16)] });
}
private randomHostPolicyResponseActions(): Array<keyof HostPolicyResponseActions> {
return this.randomArray(this.randomN(8), () =>
this.randomChoice([
'load_config',
'workflow',
'download_global_artifacts',
'configure_malware',
'read_malware_config',
'load_malware_model',
'read_kernel_config',
'configure_kernel',
'detect_process_events',
'detect_file_write_events',
'detect_file_open_events',
'detect_image_load_events',
'connect_kernel',
])
);
}
private randomHostPolicyResponseActionStatus(): HostPolicyResponseActionStatus {
return this.randomChoice([
HostPolicyResponseActionStatus.failure,
HostPolicyResponseActionStatus.success,
HostPolicyResponseActionStatus.warning,
]);
}
}
const fakeProcessNames = [

12
x-pack/plugins/endpoint/common/schema/policy.ts Normal file
View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { schema } from '@kbn/config-schema';
export const GetPolicyResponseSchema = {
query: schema.object({
hostId: schema.string(),
}),
};

22
x-pack/plugins/endpoint/common/types.ts
View file

@ -613,7 +613,7 @@ export enum HostPolicyResponseActionStatus {
/**
* The details of a given action
*/
interface HostPolicyResponseActionDetails {
export interface HostPolicyResponseActionDetails {
status: HostPolicyResponseActionStatus;
message: string;
}
@ -621,7 +621,7 @@ interface HostPolicyResponseActionDetails {
/**
* A known list of possible Endpoint actions
*/
interface HostPolicyResponseActions {
export interface HostPolicyResponseActions {
download_model: HostPolicyResponseActionDetails;
ingest_events_config: HostPolicyResponseActionDetails;
workflow: HostPolicyResponseActionDetails;
@ -642,9 +642,6 @@ interface HostPolicyResponseActions {
read_kernel_config: HostPolicyResponseActionDetails;
read_logging_config: HostPolicyResponseActionDetails;
read_malware_config: HostPolicyResponseActionDetails;
// The list of possible Actions will change rapidly, so the below entry will allow
// them without us defining them here statically
[key: string]: HostPolicyResponseActionDetails;
}
interface HostPolicyResponseConfigurationStatus {
@ -656,7 +653,7 @@ interface HostPolicyResponseConfigurationStatus {
* Information about the applying of a policy to a given host
*/
export interface HostPolicyResponse {
'@timestamp': string;
'@timestamp': number;
elastic: {
agent: {
id: string;
@ -665,21 +662,29 @@ export interface HostPolicyResponse {
ecs: {
version: string;
};
host: {
id: string;
};
event: {
created: string;
created: number;
kind: string;
id: string;
};
agent: {
version: string;
id: string;
};
endpoint: {
artifacts: {};
policy: {
applied: {
version: string;
id: string;
status: HostPolicyResponseActionStatus;
actions: Partial<HostPolicyResponseActions>;
policy: {
id: string;
version: string;
};
response: {
configurations: {
malware: HostPolicyResponseConfigurationStatus;
@ -687,7 +692,6 @@ export interface HostPolicyResponse {
logging: HostPolicyResponseConfigurationStatus;
streaming: HostPolicyResponseConfigurationStatus;
};
actions: Partial<HostPolicyResponseActions>;
};
};
};

398
x-pack/plugins/endpoint/scripts/policy_mapping.json Normal file
View file

@ -0,0 +1,398 @@
{
"mappings": {
"_meta": {
"version": "1.6.0-dev"
},
"date_detection": false,
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"properties": {
"@timestamp": {
"type": "date"
},
"ecs": {
"properties": {
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"endpoint": {
"properties": {
"policy": {
"properties": {
"applied": {
"properties": {
"actions": {
"properties": {
"configure_elasticsearch_connection": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"configure_kernel": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"configure_logging": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"configure_malware": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"connect_kernel": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"detect_file_open_events": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"detect_file_write_events": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"detect_image_load_events": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"detect_process_events": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"download_global_artifacts": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"load_config": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"load_malware_model": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"read_elasticsearch_config": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"read_events_config": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"read_kernel_config": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"read_logging_config": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"read_malware_config": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"workflow": {
"properties": {
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
},
"type": "object"
},
"configurations": {
"properties": {
"events": {
"properties": {
"concerned_actions": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"logging": {
"properties": {
"concerned_actions": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"malware": {
"properties": {
"concerned_actions": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"streaming": {
"properties": {
"concerned_actions": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
}
},
"type": "object"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"policy": {
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
},
"response": {
"type": "object"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
},
"type": "object"
}
},
"type": "object"
}
}
},
"event": {
"properties": {
"created": {
"type": "date"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"kind": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"host": {
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": 10000
}
},
"refresh_interval": "5s"
}
}
}

29
x-pack/plugins/endpoint/scripts/resolver_generator.ts
View file

@ -10,6 +10,7 @@ import { ResponseError } from '@elastic/elasticsearch/lib/errors';
import { EndpointDocGenerator, Event } from '../common/generate_data';
import { default as eventMapping } from './event_mapping.json';
import { default as alertMapping } from './alert_mapping.json';
import { default as policyMapping } from './policy_mapping.json';
main();
@ -44,6 +45,12 @@ async function main() {
default: 'metrics-endpoint-default-1',
type: 'string',
},
policyIndex: {
alias: 'pi',
describe: 'index to store host policy in',
default: 'metrics-endpoint.policy-default-1',
type: 'string',
},
auth: {
describe: 'elasticsearch username and password, separated by a colon',
type: 'string',
@ -90,6 +97,12 @@ async function main() {
type: 'number',
default: 1,
},
numDocs: {
alias: 'nd',
describe: 'number of metadata and policy response doc to generate per host',
type: 'number',
default: 5,
},
alertsPerHost: {
alias: 'ape',
describe: 'number of resolver trees to make for each host',
@ -123,7 +136,7 @@ async function main() {
if (argv.delete) {
try {
await client.indices.delete({
index: [argv.eventIndex, argv.metadataIndex, argv.alertIndex],
index: [argv.eventIndex, argv.metadataIndex, argv.alertIndex, argv.policyIndex],
});
} catch (err) {
if (err instanceof ResponseError && err.statusCode !== 404) {
@ -165,6 +178,7 @@ async function main() {
await createIndex(client, argv.alertIndex, alertMapping);
await createIndex(client, argv.eventIndex, eventMapping);
await createIndex(client, argv.policyIndex, policyMapping);
if (argv.setupOnly) {
process.exit(0);
}
@ -179,14 +193,19 @@ async function main() {
for (let i = 0; i < argv.numHosts; i++) {
const generator = new EndpointDocGenerator(random);
const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents
const numMetadataDocs = 5;
const timestamp = new Date().getTime();
for (let j = 0; j < numMetadataDocs; j++) {
for (let j = 0; j < argv.numDocs; j++) {
generator.updateHostData();
generator.updatePolicyId();
await client.index({
index: argv.metadataIndex,
body: generator.generateHostMetadata(
timestamp - timeBetweenDocs * (numMetadataDocs - j - 1)
body: generator.generateHostMetadata(timestamp - timeBetweenDocs * (argv.numDocs - j - 1)),
});
await client.index({
index: argv.policyIndex,
body: generator.generatePolicyResponse(
timestamp - timeBetweenDocs * (argv.numDocs - j - 1)
),
});
}

5
x-pack/plugins/endpoint/server/index_pattern.ts
View file

@ -11,6 +11,7 @@ export interface IndexPatternRetriever {
getIndexPattern(ctx: RequestHandlerContext, datasetPath: string): Promise<string>;
getEventIndexPattern(ctx: RequestHandlerContext): Promise<string>;
getMetadataIndexPattern(ctx: RequestHandlerContext): Promise<string>;
getPolicyResponseIndexPattern(ctx: RequestHandlerContext): Promise<string>;
}
/**
@ -74,4 +75,8 @@ export class IngestIndexPatternRetriever implements IndexPatternRetriever {
throw new Error(errMsg);
}
}
getPolicyResponseIndexPattern(ctx: RequestHandlerContext): Promise<string> {
return Promise.resolve('metrics-endpoint.policy-default-1');
}
}

30
x-pack/plugins/endpoint/server/mocks.ts
View file

@ -4,7 +4,13 @@
* you may not use this file except in compliance with the Elastic License.
*/
import {
IScopedClusterClient,
RequestHandlerContext,
SavedObjectsClientContract,
} from 'kibana/server';
import { AgentService, IngestManagerStartContract } from '../../ingest_manager/server';
import { IndexPatternRetriever } from './index_pattern';
/**
* Creates a mock IndexPatternRetriever for use in tests.
@ -12,12 +18,13 @@ import { AgentService, IngestManagerStartContract } from '../../ingest_manager/s
* @param indexPattern a string index pattern to return when any of the mock's public methods are called.
* @returns the same string passed in via `indexPattern`
*/
export const createMockIndexPatternRetriever = (indexPattern: string) => {
export const createMockIndexPatternRetriever = (indexPattern: string): IndexPatternRetriever => {
const mockGetFunc = jest.fn().mockResolvedValue(indexPattern);
return {
getIndexPattern: mockGetFunc,
getEventIndexPattern: mockGetFunc,
getMetadataIndexPattern: mockGetFunc,
getPolicyResponseIndexPattern: mockGetFunc,
};
};
@ -56,3 +63,24 @@ export const createMockIngestManagerStartContract = (
agentService: createMockAgentService(),
};
};
export function createRouteHandlerContext(
dataClient: jest.Mocked<IScopedClusterClient>,
savedObjectsClient: jest.Mocked<SavedObjectsClientContract>
) {
return ({
core: {
elasticsearch: {
dataClient,
},
savedObjects: {
client: savedObjectsClient,
},
},
/**
* Using unknown here because the object defined is not a full `RequestHandlerContext`. We don't
* need all of the fields required to run the tests, but the `routeHandler` function requires a
* `RequestHandlerContext`.
*/
} as unknown) as RequestHandlerContext;
}

2
x-pack/plugins/endpoint/server/plugin.ts
View file

@ -16,6 +16,7 @@ import { registerEndpointRoutes } from './routes/metadata';
import { IngestIndexPatternRetriever } from './index_pattern';
import { IngestManagerStartContract } from '../../ingest_manager/server';
import { EndpointAppContextService } from './endpoint_app_context_services';
import { registerPolicyRoutes } from './routes/policy';
export type EndpointPluginStart = void;
export type EndpointPluginSetup = void;
@ -87,6 +88,7 @@ export class EndpointPlugin
registerResolverRoutes(router, endpointContext);
registerAlertRoutes(router, endpointContext);
registerIndexPatternRoute(router, endpointContext);
registerPolicyRoutes(router, endpointContext);
}
public start(core: CoreStart, plugins: EndpointPluginStartDependencies) {

28
x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts
View file

@ -9,7 +9,6 @@ import {
IScopedClusterClient,
KibanaResponseFactory,
RequestHandler,
RequestHandlerContext,
RouteConfig,
SavedObjectsClientContract,
} from 'kibana/server';
@ -25,7 +24,11 @@ import { SearchResponse } from 'elasticsearch';
import { registerEndpointRoutes } from './index';
import { EndpointConfigSchema } from '../../config';
import * as data from '../../test_data/all_metadata_data.json';
import { createMockAgentService, createMockMetadataIndexPatternRetriever } from '../../mocks';
import {
createMockAgentService,
createMockMetadataIndexPatternRetriever,
createRouteHandlerContext,
} from '../../mocks';
import { AgentService } from '../../../../ingest_manager/server';
import Boom from 'boom';
import { EndpointAppContextService } from '../../endpoint_app_context_services';
@ -66,27 +69,6 @@ describe('test endpoint route', () => {
afterEach(() => endpointAppContextService.stop());
function createRouteHandlerContext(
dataClient: jest.Mocked<IScopedClusterClient>,
savedObjectsClient: jest.Mocked<SavedObjectsClientContract>
) {
return ({
core: {
elasticsearch: {
dataClient,
},
savedObjects: {
client: savedObjectsClient,
},
},
/**
* Using unknown here because the object defined is not a full `RequestHandlerContext`. We don't
* need all of the fields required to run the tests, but the `routeHandler` function requires a
* `RequestHandlerContext`.
*/
} as unknown) as RequestHandlerContext;
}
it('test find the latest of all endpoints', async () => {
const mockRequest = httpServerMock.createKibanaRequest({});

138
x-pack/plugins/endpoint/server/routes/policy/handlers.test.ts Normal file
View file

@ -0,0 +1,138 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { EndpointAppContextService } from '../../endpoint_app_context_services';
import {
createMockAgentService,
createMockIndexPatternRetriever,
createRouteHandlerContext,
} from '../../mocks';
import { getHostPolicyResponseHandler } from './handlers';
import { EndpointConfigSchema } from '../../config';
import {
IScopedClusterClient,
KibanaResponseFactory,
SavedObjectsClientContract,
} from 'kibana/server';
import {
elasticsearchServiceMock,
httpServerMock,
loggingServiceMock,
savedObjectsClientMock,
} from '../../../../../../src/core/server/mocks';
import { AgentService } from '../../../../ingest_manager/server/services';
import { SearchResponse } from 'elasticsearch';
import { GetHostPolicyResponse, HostPolicyResponse } from '../../../common/types';
import { EndpointDocGenerator } from '../../../common/generate_data';
describe('test policy response handler', () => {
let endpointAppContextService: EndpointAppContextService;
let mockScopedClient: jest.Mocked<IScopedClusterClient>;
let mockSavedObjectClient: jest.Mocked<SavedObjectsClientContract>;
let mockResponse: jest.Mocked<KibanaResponseFactory>;
let mockAgentService: jest.Mocked<AgentService>;
beforeEach(() => {
mockScopedClient = elasticsearchServiceMock.createScopedClusterClient();
mockSavedObjectClient = savedObjectsClientMock.create();
mockResponse = httpServerMock.createResponseFactory();
endpointAppContextService = new EndpointAppContextService();
mockAgentService = createMockAgentService();
endpointAppContextService.start({
indexPatternRetriever: createMockIndexPatternRetriever('metrics-endpoint-policy-*'),
agentService: mockAgentService,
});
});
afterEach(() => endpointAppContextService.stop());
it('should return the latest policy response for a host', async () => {
const response = createSearchResponse(new EndpointDocGenerator().generatePolicyResponse());
const hostPolicyResponseHandler = getHostPolicyResponseHandler({
logFactory: loggingServiceMock.create(),
service: endpointAppContextService,
config: () => Promise.resolve(EndpointConfigSchema.validate({})),
});
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
const mockRequest = httpServerMock.createKibanaRequest({
params: { hostId: 'id' },
});
await hostPolicyResponseHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockResponse.ok).toBeCalled();
const result = mockResponse.ok.mock.calls[0][0]?.body as GetHostPolicyResponse;
expect(result.policy_response.host.id).toEqual(response.hits.hits[0]._source.host.id);
});
it('should return not found when there is no response policy for host', async () => {
const hostPolicyResponseHandler = getHostPolicyResponseHandler({
logFactory: loggingServiceMock.create(),
service: endpointAppContextService,
config: () => Promise.resolve(EndpointConfigSchema.validate({})),
});
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() =>
Promise.resolve(createSearchResponse())
);
const mockRequest = httpServerMock.createKibanaRequest({
params: { hostId: 'id' },
});
await hostPolicyResponseHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockResponse.notFound).toBeCalled();
const message = mockResponse.notFound.mock.calls[0][0]?.body;
expect(message).toEqual('Policy Response Not Found');
});
});
/**
* Create a SearchResponse with the hostPolicyResponse provided, else return an empty
* SearchResponse
* @param hostPolicyResponse
*/
function createSearchResponse(
hostPolicyResponse?: HostPolicyResponse
): SearchResponse<HostPolicyResponse> {
return ({
took: 15,
timed_out: false,
_shards: {
total: 1,
successful: 1,
skipped: 0,
failed: 0,
},
hits: {
total: {
value: 5,
relation: 'eq',
},
max_score: null,
hits: hostPolicyResponse
? [
{
_index: 'metrics-endpoint.policy-default-1',
_id: '8FhM0HEBYyRTvb6lOQnw',
_score: null,
_source: hostPolicyResponse,
sort: [1588337587997],
},
]
: [],
},
} as unknown) as SearchResponse<HostPolicyResponse>;
}

36
x-pack/plugins/endpoint/server/routes/policy/handlers.ts Normal file
View file

@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { RequestHandler } from 'kibana/server';
import { TypeOf } from '@kbn/config-schema';
import { GetPolicyResponseSchema } from '../../../common/schema/policy';
import { EndpointAppContext } from '../../types';
import { getPolicyResponseByHostId } from './service';
export const getHostPolicyResponseHandler = function(
endpointAppContext: EndpointAppContext
): RequestHandler<undefined, TypeOf<typeof GetPolicyResponseSchema.query>, undefined> {
return async (context, request, response) => {
try {
const index = await endpointAppContext.service
.getIndexPatternRetriever()
.getPolicyResponseIndexPattern(context);
const doc = await getPolicyResponseByHostId(
index,
request.query.hostId,
context.core.elasticsearch.dataClient
);
if (doc) {
return response.ok({ body: doc });
}
return response.notFound({ body: 'Policy Response Not Found' });
} catch (err) {
return response.internalError({ body: err });
}
};
};

23
x-pack/plugins/endpoint/server/routes/policy/index.ts Normal file
View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { IRouter } from 'kibana/server';
import { EndpointAppContext } from '../../types';
import { GetPolicyResponseSchema } from '../../../common/schema/policy';
import { getHostPolicyResponseHandler } from './handlers';
export const BASE_POLICY_RESPONSE_ROUTE = `/api/endpoint/policy_response`;
export function registerPolicyRoutes(router: IRouter, endpointAppContext: EndpointAppContext) {
router.get(
{
path: BASE_POLICY_RESPONSE_ROUTE,
validate: GetPolicyResponseSchema,
options: { authRequired: true },
},
getHostPolicyResponseHandler(endpointAppContext)
);
}

19
x-pack/plugins/endpoint/server/routes/policy/service.test.ts Normal file
View file

@ -0,0 +1,19 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { GetPolicyResponseSchema } from '../../../common/schema/policy';
describe('test policy handlers schema', () => {
it('validate that get policy response query schema', async () => {
expect(
GetPolicyResponseSchema.query.validate({
hostId: 'id',
})
).toBeTruthy();
expect(() => GetPolicyResponseSchema.query.validate({})).toThrowError();
});
});

49
x-pack/plugins/endpoint/server/routes/policy/service.ts Normal file
View file

@ -0,0 +1,49 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { SearchResponse } from 'elasticsearch';
import { IScopedClusterClient } from 'kibana/server';
import { GetHostPolicyResponse, HostPolicyResponse } from '../../../common/types';
export function getESQueryPolicyResponseByHostID(hostID: string, index: string) {
return {
body: {
query: {
match: {
'host.id': hostID,
},
},
sort: [
{
'event.created': {
order: 'desc',
},
},
],
size: 1,
},
index,
};
}
export async function getPolicyResponseByHostId(
index: string,
hostId: string,
dataClient: IScopedClusterClient
): Promise<GetHostPolicyResponse | undefined> {
const query = getESQueryPolicyResponseByHostID(hostId, index);
const response = (await dataClient.callAsCurrentUser('search', query)) as SearchResponse<
HostPolicyResponse
>;
if (response.hits.hits.length === 0) {
return undefined;
}
return {
policy_response: response.hits.hits[0]._source,
};
}

1
x-pack/test/api_integration/apis/endpoint/index.ts
View file

@ -20,5 +20,6 @@ export default function endpointAPIIntegrationTests({
loadTestFile(require.resolve('./resolver'));
loadTestFile(require.resolve('./metadata'));
loadTestFile(require.resolve('./alerts'));
loadTestFile(require.resolve('./policy'));
});
}

40
x-pack/test/api_integration/apis/endpoint/policy.ts Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import expect from '@kbn/expect/expect.js';
import { FtrProviderContext } from '../../ftr_provider_context';
export default function({ getService }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const supertest = getService('supertest');
describe('Endpoint policy api', () => {
describe('GET /api/endpoint/policy_response', () => {
before(async () => await esArchiver.load('endpoint/policy'));
after(async () => await esArchiver.unload('endpoint/policy'));
it('should return one policy response for host', async () => {
const expectedHostId = '4f3b9858-a96d-49d8-a326-230d7763d767';
const { body } = await supertest
.get(`/api/endpoint/policy_response?hostId=${expectedHostId}`)
.send()
.expect(200);
expect(body.policy_response.host.id).to.eql(expectedHostId);
expect(body.policy_response.endpoint.policy).to.not.be(undefined);
});
it('should return not found if host has no policy response', async () => {
const { body } = await supertest
.get(`/api/endpoint/policy_response?hostId=bad_host_id`)
.send()
.expect(404);
expect(body.message).to.contain('Policy Response Not Found');
});
});
});
}

BIN
x-pack/test/functional/es_archives/endpoint/policy/data.json.gz Normal file
View file

Binary file not shown.

512
x-pack/test/functional/es_archives/endpoint/policy/mappings.json Normal file
View file

@ -0,0 +1,512 @@
{
"type": "index",
"value": {
"aliases": {
},
"index": "metrics-endpoint.policy-default-1",
"mappings": {
"_meta": {
"version": "1.6.0-dev"
},
"date_detection": false,
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"properties": {
"@timestamp": {
"type": "date"
},
"agent": {
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"ecs": {
"properties": {
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"elastic": {
"properties": {
"agent": {
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"endpoint": {
"properties": {
"policy": {
"properties": {
"applied": {
"properties": {
"actions": {
"properties": {
"configure_elasticsearch_connection": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"configure_kernel": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"configure_logging": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"configure_malware": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"connect_kernel": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"detect_file_open_events": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"detect_file_write_events": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"detect_image_load_events": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"detect_process_events": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"download_global_artifacts": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"download_model": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"ingest_events_config": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"load_config": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"load_malware_model": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"read_elasticsearch_config": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"read_events_config": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"read_kernel_config": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"read_logging_config": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"read_malware_config": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"workflow": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"configurations": {
"properties": {
"events": {
"properties": {
"concerned_actions": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"logging": {
"properties": {
"concerned_actions": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"malware": {
"properties": {
"concerned_actions": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"streaming": {
"properties": {
"concerned_actions": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"policy": {
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"response": {
"properties": {
"configurations": {
"properties": {
"events": {
"properties": {
"concerned_actions": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"logging": {
"properties": {
"concerned_actions": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"malware": {
"properties": {
"concerned_actions": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"streaming": {
"properties": {
"concerned_actions": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
}
}
},
"status": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
}
}
},
"event": {
"properties": {
"created": {
"type": "date"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"kind": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"host": {
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"stream": {
"properties": {
"dataset": {
"type": "constant_keyword"
},
"namespace": {
"type": "constant_keyword"
},
"type": {
"type": "constant_keyword"
}
}
}
}
},
"settings": {
"index": {
"codec": "best_compression",
"lifecycle": {
"name": "metrics-default"
},
"mapping": {
"total_fields": {
"limit": "10000"
}
},
"number_of_replicas": "1",
"number_of_shards": "1",
"prefer_v2_templates": "true",
"query": {
"default_field": [
"message"
]
},
"refresh_interval": "5s"
}
}
}
}