maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Security Solution][Roles] Add Read-Only Endpoint indexes to the (Detections) role creation scripts + FTR improvements (#107086)

Browse source 
* Add needed indexes to the role scripts
* Moved/renamed detection engine roles and user utility to `x-pack/test/common/security_solution`
* removed duplicate code in rule_registry and instead exported same methods from `common/services/security_solution`
* new endpoint FTR service that includes methods for loading and unloading data (uses existing data indexer methods)
* Transforms: Added additional methods to the FTR Test service
This commit is contained in:
Paul Tavares 2021-08-09 16:24:58 -04:00 committed by GitHub
parent 2230c032c6
commit a3119a5541
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
47 changed files with 1708 additions and 695 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
x-pack/plugins/security_solution/common/endpoint/constants.ts
View file

@ -7,10 +7,19 @@
export const eventsIndexPattern = 'logs-endpoint.events.*';
export const alertsIndexPattern = 'logs-endpoint.alerts-*';
/** index pattern for the data source index (data stream) that the Endpoint streams documents to */
export const metadataIndexPattern = 'metrics-endpoint.metadata-*';
/** index that the metadata transform writes to (destination) and that is used by endpoint APIs */
export const metadataCurrentIndexPattern = 'metrics-endpoint.metadata_current_*';
/** The metadata Transform Name prefix with NO (package) version) */
export const metadataTransformPrefix = 'endpoint.metadata_current-default';
/** The metadata Transform Name prefix with NO namespace and NO (package) version) */
export const metadataTransformPattern = 'endpoint.metadata_current-*';
export const policyIndexPattern = 'metrics-endpoint.policy-*';
export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';
export const LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG = 'endpoint:limited-concurrency';

74
x-pack/plugins/security_solution/common/endpoint/data_loaders/index_alerts.ts Normal file
View file

@ -0,0 +1,74 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Client } from '@elastic/elasticsearch';
import { EndpointDocGenerator, Event, TreeOptions } from '../generate_data';
import { firstNonNullValue } from '../models/ecs_safety_helpers';
function delay(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/**
* Indexes Alerts/Events into elasticsarch
*
* @param client
* @param eventIndex
* @param alertIndex
* @param generator
* @param numAlerts
* @param options
*/
export async function indexAlerts({
client,
eventIndex,
alertIndex,
generator,
numAlerts,
options = {},
}: {
client: Client;
eventIndex: string;
alertIndex: string;
generator: EndpointDocGenerator;
numAlerts: number;
options: TreeOptions;
}) {
const alertGenerator = generator.alertsGenerator(numAlerts, options);
let result = alertGenerator.next();
while (!result.done) {
let k = 0;
const resolverDocs: Event[] = [];
while (k < 1000 && !result.done) {
resolverDocs.push(result.value);
result = alertGenerator.next();
k++;
}
const body = resolverDocs.reduce(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(array: Array<Record<string, any>>, doc) => {
let index = eventIndex;
if (firstNonNullValue(doc.event?.kind) === 'alert') {
index = alertIndex;
}
array.push({ create: { _index: index } }, doc);
return array;
},
[]
);
await client.bulk({ body, refresh: true });
}
await client.indices.refresh({
index: eventIndex,
});
// TODO: Unclear why the documents are not showing up after the call to refresh.
// Waiting 5 seconds allows the indices to refresh automatically and
// the documents become available in API/integration tests.
await delay(5000);
}

310
x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_hosts.ts Normal file
View file

@ -0,0 +1,310 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Client } from '@elastic/elasticsearch';
import { cloneDeep, merge } from 'lodash';
import { AxiosResponse } from 'axios';
// eslint-disable-next-line import/no-extraneous-dependencies
import { KbnClient } from '@kbn/test';
import { DeleteByQueryResponse } from '@elastic/elasticsearch/api/types';
import { Agent, CreatePackagePolicyResponse, GetPackagesResponse } from '../../../../fleet/common';
import { EndpointDocGenerator } from '../generate_data';
import { HostMetadata, HostPolicyResponse } from '../types';
import {
deleteIndexedFleetAgents,
DeleteIndexedFleetAgentsResponse,
IndexedFleetAgentResponse,
indexFleetAgentForHost,
} from './index_fleet_agent';
import {
deleteIndexedFleetActions,
DeleteIndexedFleetActionsResponse,
IndexedFleetActionsForHostResponse,
indexFleetActionsForHost,
} from './index_fleet_actions';
import {
deleteIndexedFleetEndpointPolicies,
DeleteIndexedFleetEndpointPoliciesResponse,
IndexedFleetEndpointPolicyResponse,
indexFleetEndpointPolicy,
} from './index_fleet_endpoint_policy';
import { metadataCurrentIndexPattern } from '../constants';
import { EndpointDataLoadingError, wrapErrorAndRejectPromise } from './utils';
export interface IndexedHostsResponse
extends IndexedFleetAgentResponse,
IndexedFleetActionsForHostResponse,
IndexedFleetEndpointPolicyResponse {
/**
* The documents (1 or more) that were generated for the (single) endpoint host.
* If consuming this data and wanting only the last one created, just access the
* last item in the array
*/
hosts: HostMetadata[];
/**
* The list of Endpoint Policy Response documents (1 or more) that was created.
*/
policyResponses: HostPolicyResponse[];
metadataIndex: string;
policyResponseIndex: string;
}
/**
* Indexes the requested number of documents for the endpoint host metadata currently being output by the generator.
* Endpoint Host metadata documents are added to an index that is set as "append only", thus one Endpoint host could
* have multiple documents in that index.
*
*
*
* @param numDocs
* @param client
* @param kbnClient
* @param realPolicies
* @param epmEndpointPackage
* @param metadataIndex
* @param policyResponseIndex
* @param enrollFleet
* @param generator
*/
export async function indexEndpointHostDocs({
numDocs,
client,
kbnClient,
realPolicies,
epmEndpointPackage,
metadataIndex,
policyResponseIndex,
enrollFleet,
generator,
}: {
numDocs: number;
client: Client;
kbnClient: KbnClient;
realPolicies: Record<string, CreatePackagePolicyResponse['item']>;
epmEndpointPackage: GetPackagesResponse['response'][0];
metadataIndex: string;
policyResponseIndex: string;
enrollFleet: boolean;
generator: EndpointDocGenerator;
}): Promise<IndexedHostsResponse> {
const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents
const timestamp = new Date().getTime();
const kibanaVersion = await fetchKibanaVersion(kbnClient);
const response: IndexedHostsResponse = {
hosts: [],
agents: [],
policyResponses: [],
metadataIndex,
policyResponseIndex,
fleetAgentsIndex: '',
actionResponses: [],
responsesIndex: '',
actions: [],
actionsIndex: '',
integrationPolicies: [],
agentPolicies: [],
};
let hostMetadata: HostMetadata;
let wasAgentEnrolled = false;
let enrolledAgent: undefined | Agent;
for (let j = 0; j < numDocs; j++) {
generator.updateHostData();
generator.updateHostPolicyData();
hostMetadata = generator.generateHostMetadata(
timestamp - timeBetweenDocs * (numDocs - j - 1),
EndpointDocGenerator.createDataStreamFromIndex(metadataIndex)
);
if (enrollFleet) {
const { id: appliedPolicyId, name: appliedPolicyName } = hostMetadata.Endpoint.policy.applied;
// If we don't yet have a "real" policy record, then create it now in ingest (package config)
if (!realPolicies[appliedPolicyId]) {
const createdPolicies = await indexFleetEndpointPolicy(
kbnClient,
appliedPolicyName,
epmEndpointPackage.version
);
merge(response, createdPolicies);
// eslint-disable-next-line require-atomic-updates
realPolicies[appliedPolicyId] = createdPolicies.integrationPolicies[0];
}
// If we did not yet enroll an agent for this Host, do it now that we have good policy id
if (!wasAgentEnrolled) {
wasAgentEnrolled = true;
const indexedAgentResponse = await indexFleetAgentForHost(
client,
kbnClient,
hostMetadata!,
realPolicies[appliedPolicyId].policy_id,
kibanaVersion
);
enrolledAgent = indexedAgentResponse.agents[0];
merge(response, indexedAgentResponse);
}
// Update the Host metadata record with the ID of the "real" policy along with the enrolled agent id
hostMetadata = {
...hostMetadata,
elastic: {
...hostMetadata.elastic,
agent: {
...hostMetadata.elastic.agent,
id: enrolledAgent?.id ?? hostMetadata.elastic.agent.id,
},
},
Endpoint: {
...hostMetadata.Endpoint,
policy: {
...hostMetadata.Endpoint.policy,
applied: {
...hostMetadata.Endpoint.policy.applied,
id: realPolicies[appliedPolicyId].id,
},
},
},
};
// Create some actions for this Host
await indexFleetActionsForHost(client, hostMetadata);
}
await client
.index({
index: metadataIndex,
body: hostMetadata,
op_type: 'create',
})
.catch(wrapErrorAndRejectPromise);
const hostPolicyResponse = generator.generatePolicyResponse({
ts: timestamp - timeBetweenDocs * (numDocs - j - 1),
policyDataStream: EndpointDocGenerator.createDataStreamFromIndex(policyResponseIndex),
});
await client
.index({
index: policyResponseIndex,
body: hostPolicyResponse,
op_type: 'create',
})
.catch(wrapErrorAndRejectPromise);
// Clone the hostMetadata and policyResponse document to ensure that no shared state
// (as a result of using the generator) is returned across docs.
response.hosts.push(cloneDeep(hostMetadata));
response.policyResponses.push(cloneDeep(hostPolicyResponse));
}
return response;
}
const fetchKibanaVersion = async (kbnClient: KbnClient) => {
const version = ((await kbnClient.request({
path: '/api/status',
method: 'GET',
})) as AxiosResponse).data.version.number;
if (!version) {
throw new EndpointDataLoadingError('failed to get kibana version via `/api/status` api');
}
return version;
};
export interface DeleteIndexedEndpointHostsResponse
extends DeleteIndexedFleetAgentsResponse,
DeleteIndexedFleetActionsResponse,
DeleteIndexedFleetEndpointPoliciesResponse {
hosts: DeleteByQueryResponse | undefined;
policyResponses: DeleteByQueryResponse | undefined;
}
export const deleteIndexedEndpointHosts = async (
esClient: Client,
kbnClient: KbnClient,
indexedData: IndexedHostsResponse
): Promise<DeleteIndexedEndpointHostsResponse> => {
const response: DeleteIndexedEndpointHostsResponse = {
hosts: undefined,
policyResponses: undefined,
agents: undefined,
responses: undefined,
actions: undefined,
integrationPolicies: undefined,
agentPolicies: undefined,
};
if (indexedData.hosts.length) {
const body = {
query: {
bool: {
filter: [{ terms: { 'agent.id': indexedData.hosts.map((host) => host.agent.id) } }],
},
},
};
response.hosts = (
await esClient
.deleteByQuery({
index: indexedData.metadataIndex,
wait_for_completion: true,
body,
})
.catch(wrapErrorAndRejectPromise)
).body;
// Delete from the transform destination index
await esClient
.deleteByQuery({
index: metadataCurrentIndexPattern,
wait_for_completion: true,
body,
})
.catch(wrapErrorAndRejectPromise);
}
if (indexedData.policyResponses.length) {
response.policyResponses = (
await esClient
.deleteByQuery({
index: indexedData.policyResponseIndex,
wait_for_completion: true,
body: {
query: {
bool: {
filter: [
{
terms: {
'agent.id': indexedData.policyResponses.map(
(policyResponse) => policyResponse.agent.id
),
},
},
],
},
},
},
})
.catch(wrapErrorAndRejectPromise)
).body;
}
merge(response, await deleteIndexedFleetAgents(esClient, indexedData));
merge(response, await deleteIndexedFleetActions(esClient, indexedData));
merge(response, await deleteIndexedFleetEndpointPolicies(kbnClient, indexedData));
return response;
};

218
x-pack/plugins/security_solution/common/endpoint/data_loaders/index_fleet_actions.ts Normal file
View file

@ -0,0 +1,218 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Client } from '@elastic/elasticsearch';
import { DeleteByQueryResponse } from '@elastic/elasticsearch/api/types';
import { EndpointAction, EndpointActionResponse, HostMetadata } from '../types';
import { AGENT_ACTIONS_INDEX, AGENT_ACTIONS_RESULTS_INDEX } from '../../../../fleet/common';
import { FleetActionGenerator } from '../data_generators/fleet_action_generator';
import { wrapErrorAndRejectPromise } from './utils';
const defaultFleetActionGenerator = new FleetActionGenerator();
export interface IndexedFleetActionsForHostResponse {
actions: EndpointAction[];
actionResponses: EndpointActionResponse[];
actionsIndex: string;
responsesIndex: string;
}
/**
* Indexes a randome number of Endpoint (via Fleet) Actions for a given host
* (NOTE: ensure that fleet is setup first before calling this loading function)
*
* @param esClient
* @param endpointHost
* @param [fleetActionGenerator]
*/
export const indexFleetActionsForHost = async (
esClient: Client,
endpointHost: HostMetadata,
fleetActionGenerator: FleetActionGenerator = defaultFleetActionGenerator
): Promise<IndexedFleetActionsForHostResponse> => {
const ES_INDEX_OPTIONS = { headers: { 'X-elastic-product-origin': 'fleet' } };
const agentId = endpointHost.elastic.agent.id;
const total = fleetActionGenerator.randomN(5);
const response: IndexedFleetActionsForHostResponse = {
actions: [],
actionResponses: [],
actionsIndex: AGENT_ACTIONS_INDEX,
responsesIndex: AGENT_ACTIONS_RESULTS_INDEX,
};
for (let i = 0; i < total; i++) {
// create an action
const action = fleetActionGenerator.generate({
data: { comment: 'data generator: this host is bad' },
});
action.agents = [agentId];
esClient
.index(
{
index: AGENT_ACTIONS_INDEX,
body: action,
},
ES_INDEX_OPTIONS
)
.catch(wrapErrorAndRejectPromise);
// Create an action response for the above
const actionResponse = fleetActionGenerator.generateResponse({
action_id: action.action_id,
agent_id: agentId,
action_data: action.data,
});
esClient
.index(
{
index: AGENT_ACTIONS_RESULTS_INDEX,
body: actionResponse,
},
ES_INDEX_OPTIONS
)
.catch(wrapErrorAndRejectPromise);
response.actions.push(action);
response.actionResponses.push(actionResponse);
}
// Add edge cases (maybe)
if (fleetActionGenerator.randomFloat() < 0.3) {
const randomFloat = fleetActionGenerator.randomFloat();
// 60% of the time just add either an Isolate -OR- an UnIsolate action
if (randomFloat < 0.6) {
let action: EndpointAction;
if (randomFloat < 0.3) {
// add a pending isolation
action = fleetActionGenerator.generateIsolateAction({
'@timestamp': new Date().toISOString(),
});
} else {
// add a pending UN-isolation
action = fleetActionGenerator.generateUnIsolateAction({
'@timestamp': new Date().toISOString(),
});
}
action.agents = [agentId];
await esClient
.index(
{
index: AGENT_ACTIONS_INDEX,
body: action,
},
ES_INDEX_OPTIONS
)
.catch(wrapErrorAndRejectPromise);
response.actions.push(action);
} else {
// Else (40% of the time) add a pending isolate AND pending un-isolate
const action1 = fleetActionGenerator.generateIsolateAction({
'@timestamp': new Date().toISOString(),
});
const action2 = fleetActionGenerator.generateUnIsolateAction({
'@timestamp': new Date().toISOString(),
});
action1.agents = [agentId];
action2.agents = [agentId];
await Promise.all([
esClient
.index(
{
index: AGENT_ACTIONS_INDEX,
body: action1,
},
ES_INDEX_OPTIONS
)
.catch(wrapErrorAndRejectPromise),
esClient
.index(
{
index: AGENT_ACTIONS_INDEX,
body: action2,
},
ES_INDEX_OPTIONS
)
.catch(wrapErrorAndRejectPromise),
]);
response.actions.push(action1, action2);
}
}
return response;
};
export interface DeleteIndexedFleetActionsResponse {
actions: DeleteByQueryResponse | undefined;
responses: DeleteByQueryResponse | undefined;
}
export const deleteIndexedFleetActions = async (
esClient: Client,
indexedData: IndexedFleetActionsForHostResponse
): Promise<DeleteIndexedFleetActionsResponse> => {
const response: DeleteIndexedFleetActionsResponse = {
actions: undefined,
responses: undefined,
};
if (indexedData.actions.length) {
response.actions = (
await esClient
.deleteByQuery({
index: `${indexedData.actionsIndex}-*`,
wait_for_completion: true,
body: {
query: {
bool: {
filter: [
{ terms: { action_id: indexedData.actions.map((action) => action.action_id) } },
],
},
},
},
})
.catch(wrapErrorAndRejectPromise)
).body;
}
if (indexedData.actionResponses) {
response.responses = (
await esClient
.deleteByQuery({
index: `${indexedData.responsesIndex}-*`,
wait_for_completion: true,
body: {
query: {
bool: {
filter: [
{
terms: {
action_id: indexedData.actionResponses.map((action) => action.action_id),
},
},
],
},
},
},
})
.catch(wrapErrorAndRejectPromise)
).body;
}
return response;
};

133
x-pack/plugins/security_solution/common/endpoint/data_loaders/index_fleet_agent.ts Normal file
View file

@ -0,0 +1,133 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Client } from '@elastic/elasticsearch';
import { AxiosResponse } from 'axios';
import { DeleteByQueryResponse } from '@elastic/elasticsearch/api/types';
// eslint-disable-next-line import/no-extraneous-dependencies
import { KbnClient } from '@kbn/test';
import { HostMetadata } from '../types';
import {
Agent,
AGENT_API_ROUTES,
FleetServerAgent,
GetOneAgentResponse,
} from '../../../../fleet/common';
import { FleetAgentGenerator } from '../data_generators/fleet_agent_generator';
import { wrapErrorAndRejectPromise } from './utils';
const defaultFleetAgentGenerator = new FleetAgentGenerator();
export interface IndexedFleetAgentResponse {
agents: Agent[];
fleetAgentsIndex: string;
}
/**
* Indexes a Fleet Agent
* (NOTE: ensure that fleet is setup first before calling this loading function)
*
* @param esClient
* @param kbnClient
* @param endpointHost
* @param agentPolicyId
* @param [kibanaVersion]
* @param [fleetAgentGenerator]
*/
export const indexFleetAgentForHost = async (
esClient: Client,
kbnClient: KbnClient,
endpointHost: HostMetadata,
agentPolicyId: string,
kibanaVersion: string = '8.0.0',
fleetAgentGenerator: FleetAgentGenerator = defaultFleetAgentGenerator
): Promise<IndexedFleetAgentResponse> => {
const agentDoc = fleetAgentGenerator.generateEsHit({
_source: {
local_metadata: {
elastic: {
agent: {
version: kibanaVersion,
},
},
host: {
...endpointHost.host,
},
os: {
...endpointHost.host.os,
},
},
policy_id: agentPolicyId,
},
});
const createdFleetAgent = await esClient
.index<FleetServerAgent>({
index: agentDoc._index,
id: agentDoc._id,
body: agentDoc._source!,
op_type: 'create',
})
.catch(wrapErrorAndRejectPromise);
return {
fleetAgentsIndex: agentDoc._index,
agents: [
await fetchFleetAgent(kbnClient, createdFleetAgent.body._id).catch(wrapErrorAndRejectPromise),
],
};
};
const fetchFleetAgent = async (kbnClient: KbnClient, agentId: string): Promise<Agent> => {
return ((await kbnClient
.request({
path: AGENT_API_ROUTES.INFO_PATTERN.replace('{agentId}', agentId),
method: 'GET',
})
.catch(wrapErrorAndRejectPromise)) as AxiosResponse<GetOneAgentResponse>).data.item;
};
export interface DeleteIndexedFleetAgentsResponse {
agents: DeleteByQueryResponse | undefined;
}
export const deleteIndexedFleetAgents = async (
esClient: Client,
indexedData: IndexedFleetAgentResponse
): Promise<DeleteIndexedFleetAgentsResponse> => {
const response: DeleteIndexedFleetAgentsResponse = {
agents: undefined,
};
if (indexedData.agents.length) {
response.agents = (
await esClient
.deleteByQuery({
index: `${indexedData.fleetAgentsIndex}-*`,
wait_for_completion: true,
body: {
query: {
bool: {
filter: [
{
terms: {
'local_metadata.elastic.agent.id': indexedData.agents.map(
(agent) => agent.local_metadata.elastic.agent.id
),
},
},
],
},
},
},
})
.catch(wrapErrorAndRejectPromise)
).body;
}
return response;
};

159
x-pack/plugins/security_solution/common/endpoint/data_loaders/index_fleet_endpoint_policy.ts Normal file
View file

@ -0,0 +1,159 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
// eslint-disable-next-line import/no-extraneous-dependencies
import { KbnClient } from '@kbn/test';
import { AxiosResponse } from 'axios';
import {
AGENT_POLICY_API_ROUTES,
AgentPolicy,
CreateAgentPolicyRequest,
CreateAgentPolicyResponse,
CreatePackagePolicyRequest,
CreatePackagePolicyResponse,
DeleteAgentPolicyResponse,
DeletePackagePoliciesResponse,
PACKAGE_POLICY_API_ROUTES,
} from '../../../../fleet/common';
import { PolicyData } from '../types';
import { policyFactory as policyConfigFactory } from '../models/policy_config';
import { wrapErrorAndRejectPromise } from './utils';
export interface IndexedFleetEndpointPolicyResponse {
integrationPolicies: PolicyData[];
agentPolicies: AgentPolicy[];
}
/**
* Create an endpoint Integration Policy (and associated Agent Policy) via Fleet
* (NOTE: ensure that fleet is setup first before calling this loading function)
*/
export const indexFleetEndpointPolicy = async (
kbnClient: KbnClient,
policyName: string,
endpointPackageVersion: string = '8.0.0'
): Promise<IndexedFleetEndpointPolicyResponse> => {
const response: IndexedFleetEndpointPolicyResponse = {
integrationPolicies: [],
agentPolicies: [],
};
// Create Agent Policy first
const newAgentPolicyData: CreateAgentPolicyRequest['body'] = {
name: `Policy for ${policyName} (${Math.random().toString(36).substr(2, 5)})`,
description: `Policy created with endpoint data generator (${policyName})`,
namespace: 'default',
};
let agentPolicy: AxiosResponse<CreateAgentPolicyResponse>;
try {
agentPolicy = (await kbnClient
.request({
path: AGENT_POLICY_API_ROUTES.CREATE_PATTERN,
method: 'POST',
body: newAgentPolicyData,
})
.catch(wrapErrorAndRejectPromise)) as AxiosResponse<CreateAgentPolicyResponse>;
} catch (error) {
throw new Error(`create fleet agent policy failed ${error}`);
}
response.agentPolicies.push(agentPolicy.data.item);
// Create integration (package) policy
const newPackagePolicyData: CreatePackagePolicyRequest['body'] = {
name: policyName,
description: 'Protect the worlds data',
policy_id: agentPolicy.data.item.id,
enabled: true,
output_id: '',
inputs: [
{
type: 'endpoint',
enabled: true,
streams: [],
config: {
policy: {
value: policyConfigFactory(),
},
},
},
],
namespace: 'default',
package: {
name: 'endpoint',
title: 'endpoint',
version: endpointPackageVersion,
},
};
const packagePolicy = (await kbnClient
.request({
path: PACKAGE_POLICY_API_ROUTES.CREATE_PATTERN,
method: 'POST',
body: newPackagePolicyData,
})
.catch(wrapErrorAndRejectPromise)) as AxiosResponse<CreatePackagePolicyResponse>;
response.integrationPolicies.push(packagePolicy.data.item as PolicyData);
return response;
};
export interface DeleteIndexedFleetEndpointPoliciesResponse {
integrationPolicies: DeletePackagePoliciesResponse | undefined;
agentPolicies: DeleteAgentPolicyResponse[] | undefined;
}
/**
* Delete indexed Fleet Endpoint integration policies along with their respective Agent Policies.
* Prior to calling this function, ensure that no agents are associated with the Agent Policy.
* (NOTE: ensure that fleet is setup first before calling this loading function)
* @param kbnClient
* @param indexData
*/
export const deleteIndexedFleetEndpointPolicies = async (
kbnClient: KbnClient,
indexData: IndexedFleetEndpointPolicyResponse
): Promise<DeleteIndexedFleetEndpointPoliciesResponse> => {
const response: DeleteIndexedFleetEndpointPoliciesResponse = {
integrationPolicies: undefined,
agentPolicies: undefined,
};
if (indexData.integrationPolicies.length) {
response.integrationPolicies = ((await kbnClient
.request({
path: PACKAGE_POLICY_API_ROUTES.DELETE_PATTERN,
method: 'POST',
body: {
packagePolicyIds: indexData.integrationPolicies.map((policy) => policy.id),
},
})
.catch(wrapErrorAndRejectPromise)) as AxiosResponse<DeletePackagePoliciesResponse>).data;
}
if (indexData.agentPolicies.length) {
response.agentPolicies = [];
for (const agentPolicy of indexData.agentPolicies) {
response.agentPolicies.push(
((await kbnClient
.request({
path: AGENT_POLICY_API_ROUTES.DELETE_PATTERN,
method: 'POST',
body: {
agentPolicyId: agentPolicy.id,
},
})
.catch(wrapErrorAndRejectPromise)) as AxiosResponse<DeleteAgentPolicyResponse>).data
);
}
}
return response;
};

53
x-pack/plugins/security_solution/common/endpoint/data_loaders/index_fleet_server.ts Normal file
View file

@ -0,0 +1,53 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Client } from '@elastic/elasticsearch';
import { FLEET_SERVER_SERVERS_INDEX } from '../../../../fleet/common';
import { wrapErrorAndRejectPromise } from './utils';
/**
* Will ensure that at least one fleet server is present in the `.fleet-servers` index. This will
* enable the `Agent` section of kibana Fleet to be displayed
*
* @param esClient
* @param version
*/
export const enableFleetServerIfNecessary = async (esClient: Client, version: string = '8.0.0') => {
const res = await esClient.search<{}, {}>({
index: FLEET_SERVER_SERVERS_INDEX,
ignore_unavailable: true,
rest_total_hits_as_int: true,
});
if (res.body.hits.total > 0) {
return;
}
// Create a Fake fleet-server in this kibana instance
await esClient
.index({
index: FLEET_SERVER_SERVERS_INDEX,
body: {
agent: {
id: '12988155-475c-430d-ac89-84dc84b67cd1',
version,
},
host: {
architecture: 'linux',
id: 'c3e5f4f690b4a3ff23e54900701a9513',
ip: ['127.0.0.1', '::1', '10.201.0.213', 'fe80::4001:aff:fec9:d5'],
name: 'endpoint-data-generator',
},
server: {
id: '12988155-475c-430d-ac89-84dc84b67cd1',
version,
},
'@timestamp': '2021-05-12T18:42:52.009482058Z',
},
})
.catch(wrapErrorAndRejectPromise);
};

117
x-pack/plugins/security_solution/common/endpoint/data_loaders/setup_fleet_for_endpoint.ts Normal file
View file

@ -0,0 +1,117 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { AxiosResponse } from 'axios';
// eslint-disable-next-line import/no-extraneous-dependencies
import { KbnClient } from '@kbn/test';
import {
AGENTS_SETUP_API_ROUTES,
BulkInstallPackageInfo,
BulkInstallPackagesResponse,
EPM_API_ROUTES,
IBulkInstallPackageHTTPError,
PostFleetSetupResponse,
SETUP_API_ROUTE,
} from '../../../../fleet/common';
import { EndpointDataLoadingError, wrapErrorAndRejectPromise } from './utils';
/**
* Calls the fleet setup APIs and then installs the latest Endpoint package
* @param kbnClient
*/
export const setupFleetForEndpoint = async (kbnClient: KbnClient) => {
// We try to use the kbnClient **private** logger, bug if unable to access it, then just use console
// @ts-ignore
const log = kbnClient.log ? kbnClient.log : console;
// Setup Fleet
try {
const setupResponse = (await kbnClient
.request({
path: SETUP_API_ROUTE,
method: 'POST',
})
.catch(wrapErrorAndRejectPromise)) as AxiosResponse<PostFleetSetupResponse>;
if (!setupResponse.data.isInitialized) {
log.error(setupResponse.data);
throw new Error('Initializing the ingest manager failed, existing');
}
} catch (error) {
log.error(error);
throw error;
}
// Setup Agents
try {
const setupResponse = (await kbnClient
.request({
path: AGENTS_SETUP_API_ROUTES.CREATE_PATTERN,
method: 'POST',
})
.catch(wrapErrorAndRejectPromise)) as AxiosResponse<PostFleetSetupResponse>;
if (!setupResponse.data.isInitialized) {
log.error(setupResponse.data);
throw new Error('Initializing Fleet failed, existing');
}
} catch (error) {
log.error(error);
throw error;
}
// Install/upgrade the endpoint package
try {
await installOrUpgradeEndpointFleetPackage(kbnClient);
} catch (error) {
log.error(error);
throw error;
}
};
/**
* Installs the Endpoint package (or upgrades it) in Fleet to the latest available in the registry
*
* @param kbnClient
*/
export const installOrUpgradeEndpointFleetPackage = async (kbnClient: KbnClient): Promise<void> => {
const installEndpointPackageResp = (await kbnClient
.request({
path: EPM_API_ROUTES.BULK_INSTALL_PATTERN,
method: 'POST',
body: {
packages: ['endpoint'],
},
})
.catch(wrapErrorAndRejectPromise)) as AxiosResponse<BulkInstallPackagesResponse>;
const bulkResp = installEndpointPackageResp.data.response;
if (bulkResp.length <= 0) {
throw new EndpointDataLoadingError(
'Installing the Endpoint package failed, response was empty, existing',
bulkResp
);
}
if (isFleetBulkInstallError(bulkResp[0])) {
if (bulkResp[0].error instanceof Error) {
throw new EndpointDataLoadingError(
`Installing the Endpoint package failed: ${bulkResp[0].error.message}, exiting`,
bulkResp
);
}
throw new EndpointDataLoadingError(bulkResp[0].error, bulkResp);
}
};
function isFleetBulkInstallError(
installResponse: BulkInstallPackageInfo | IBulkInstallPackageHTTPError
): installResponse is IBulkInstallPackageHTTPError {
return 'error' in installResponse && installResponse.error !== undefined;
}

20
x-pack/plugins/security_solution/common/endpoint/data_loaders/utils.ts Normal file
View file

@ -0,0 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export class EndpointDataLoadingError extends Error {
constructor(message: string, public meta?: unknown) {
super(message);
}
}
export const wrapErrorIfNeeded = (error: Error): EndpointDataLoadingError =>
error instanceof EndpointDataLoadingError
? error
: new EndpointDataLoadingError(error.message, error);
// Use it in Promise's `.catch()` as `.catch(wrapErrorAndRejectPromise)`
export const wrapErrorAndRejectPromise = (error: Error) => Promise.reject(wrapErrorIfNeeded(error));

483
x-pack/plugins/security_solution/common/endpoint/index_data.ts
View file

@ -5,39 +5,49 @@
* 2.0.
*/
import { Client, estypes } from '@elastic/elasticsearch';
import { Client } from '@elastic/elasticsearch';
import seedrandom from 'seedrandom';
// eslint-disable-next-line import/no-extraneous-dependencies
import { KbnClient } from '@kbn/test';
import { AxiosResponse } from 'axios';
import { EndpointDocGenerator, Event, TreeOptions } from './generate_data';
import { firstNonNullValue } from './models/ecs_safety_helpers';
import { merge } from 'lodash';
import { EndpointDocGenerator, TreeOptions } from './generate_data';
import {
AGENT_ACTIONS_INDEX,
AGENT_ACTIONS_RESULTS_INDEX,
AGENT_POLICY_API_ROUTES,
CreateAgentPolicyRequest,
CreateAgentPolicyResponse,
CreatePackagePolicyRequest,
CreatePackagePolicyResponse,
EPM_API_ROUTES,
FLEET_SERVER_SERVERS_INDEX,
FleetServerAgent,
GetPackagesResponse,
PACKAGE_POLICY_API_ROUTES,
} from '../../../fleet/common';
import { policyFactory as policyConfigFactory } from './models/policy_config';
import { EndpointAction, HostMetadata } from './types';
import { KbnClientWithApiKeySupport } from '../../scripts/endpoint/kbn_client_with_api_key_support';
import { FleetAgentGenerator } from './data_generators/fleet_agent_generator';
import { FleetActionGenerator } from './data_generators/fleet_action_generator';
import {
deleteIndexedEndpointHosts,
DeleteIndexedEndpointHostsResponse,
IndexedHostsResponse,
indexEndpointHostDocs,
} from './data_loaders/index_endpoint_hosts';
import { enableFleetServerIfNecessary } from './data_loaders/index_fleet_server';
import { indexAlerts } from './data_loaders/index_alerts';
import { setupFleetForEndpoint } from './data_loaders/setup_fleet_for_endpoint';
const fleetAgentGenerator = new FleetAgentGenerator();
const fleetActionGenerator = new FleetActionGenerator();
export type IndexedHostsAndAlertsResponse = IndexedHostsResponse;
/**
* Indexes Endpoint Hosts (with optional Fleet counterparts) along with alerts
*
* @param client
* @param kbnClient
* @param seed
* @param numHosts
* @param numDocs
* @param metadataIndex
* @param policyResponseIndex
* @param eventIndex
* @param alertIndex
* @param alertsPerHost
* @param fleet
* @param options
*/
export async function indexHostsAndAlerts(
client: Client,
kbnClient: KbnClientWithApiKeySupport,
kbnClient: KbnClient,
seed: string,
numHosts: number,
numDocs: number,
@ -48,9 +58,26 @@ export async function indexHostsAndAlerts(
alertsPerHost: number,
fleet: boolean,
options: TreeOptions = {}
) {
): Promise<IndexedHostsAndAlertsResponse> {
const random = seedrandom(seed);
const epmEndpointPackage = await getEndpointPackageInfo(kbnClient);
const response: IndexedHostsAndAlertsResponse = {
hosts: [],
policyResponses: [],
agents: [],
fleetAgentsIndex: '',
metadataIndex,
policyResponseIndex,
actionResponses: [],
responsesIndex: '',
actions: [],
actionsIndex: '',
integrationPolicies: [],
agentPolicies: [],
};
// Ensure fleet is setup and endpint package installed
await setupFleetForEndpoint(kbnClient);
// If `fleet` integration is true, then ensure a (fake) fleet-server is connected
if (fleet) {
@ -62,7 +89,7 @@ export async function indexHostsAndAlerts(
for (let i = 0; i < numHosts; i++) {
const generator = new EndpointDocGenerator(random);
await indexHostDocs({
const indexedHosts = await indexEndpointHostDocs({
numDocs,
client,
kbnClient,
@ -73,6 +100,9 @@ export async function indexHostsAndAlerts(
enrollFleet: fleet,
generator,
});
merge(response, indexedHosts);
await indexAlerts({
client,
eventIndex,
@ -83,220 +113,9 @@ export async function indexHostsAndAlerts(
});
}
await client.indices.refresh({
index: eventIndex,
});
// TODO: Unclear why the documents are not showing up after the call to refresh.
// Waiting 5 seconds allows the indices to refresh automatically and
// the documents become available in API/integration tests.
await delay(5000);
return response;
}
function delay(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function indexHostDocs({
numDocs,
client,
kbnClient,
realPolicies,
epmEndpointPackage,
metadataIndex,
policyResponseIndex,
enrollFleet,
generator,
}: {
numDocs: number;
client: Client;
kbnClient: KbnClientWithApiKeySupport;
realPolicies: Record<string, CreatePackagePolicyResponse['item']>;
epmEndpointPackage: GetPackagesResponse['response'][0];
metadataIndex: string;
policyResponseIndex: string;
enrollFleet: boolean;
generator: EndpointDocGenerator;
}) {
const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents
const timestamp = new Date().getTime();
const kibanaVersion = await fetchKibanaVersion(kbnClient);
let hostMetadata: HostMetadata;
let wasAgentEnrolled = false;
let enrolledAgent: undefined | estypes.SearchHit<FleetServerAgent>;
for (let j = 0; j < numDocs; j++) {
generator.updateHostData();
generator.updateHostPolicyData();
hostMetadata = generator.generateHostMetadata(
timestamp - timeBetweenDocs * (numDocs - j - 1),
EndpointDocGenerator.createDataStreamFromIndex(metadataIndex)
);
if (enrollFleet) {
const { id: appliedPolicyId, name: appliedPolicyName } = hostMetadata.Endpoint.policy.applied;
// If we don't yet have a "real" policy record, then create it now in ingest (package config)
if (!realPolicies[appliedPolicyId]) {
// eslint-disable-next-line require-atomic-updates
realPolicies[appliedPolicyId] = await createPolicy(
kbnClient,
appliedPolicyName,
epmEndpointPackage.version
);
}
// If we did not yet enroll an agent for this Host, do it now that we have good policy id
if (!wasAgentEnrolled) {
wasAgentEnrolled = true;
enrolledAgent = await indexFleetAgentForHost(
client,
hostMetadata!,
realPolicies[appliedPolicyId].policy_id,
kibanaVersion
);
}
// Update the Host metadata record with the ID of the "real" policy along with the enrolled agent id
hostMetadata = {
...hostMetadata,
elastic: {
...hostMetadata.elastic,
agent: {
...hostMetadata.elastic.agent,
id: enrolledAgent?._id ?? hostMetadata.elastic.agent.id,
},
},
Endpoint: {
...hostMetadata.Endpoint,
policy: {
...hostMetadata.Endpoint.policy,
applied: {
...hostMetadata.Endpoint.policy.applied,
id: realPolicies[appliedPolicyId].id,
},
},
},
};
// Create some actions for this Host
await indexFleetActionsForHost(client, hostMetadata);
}
await client.index({
index: metadataIndex,
body: hostMetadata,
op_type: 'create',
});
await client.index({
index: policyResponseIndex,
body: generator.generatePolicyResponse({
ts: timestamp - timeBetweenDocs * (numDocs - j - 1),
policyDataStream: EndpointDocGenerator.createDataStreamFromIndex(policyResponseIndex),
}),
op_type: 'create',
});
}
}
async function indexAlerts({
client,
eventIndex,
alertIndex,
generator,
numAlerts,
options = {},
}: {
client: Client;
eventIndex: string;
alertIndex: string;
generator: EndpointDocGenerator;
numAlerts: number;
options: TreeOptions;
}) {
const alertGenerator = generator.alertsGenerator(numAlerts, options);
let result = alertGenerator.next();
while (!result.done) {
let k = 0;
const resolverDocs: Event[] = [];
while (k < 1000 && !result.done) {
resolverDocs.push(result.value);
result = alertGenerator.next();
k++;
}
const body = resolverDocs.reduce(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(array: Array<Record<string, any>>, doc) => {
let index = eventIndex;
if (firstNonNullValue(doc.event?.kind) === 'alert') {
index = alertIndex;
}
array.push({ create: { _index: index } }, doc);
return array;
},
[]
);
await client.bulk({ body, refresh: true });
}
}
const createPolicy = async (
kbnClient: KbnClient,
policyName: string,
endpointPackageVersion: string
): Promise<CreatePackagePolicyResponse['item']> => {
// Create Agent Policy first
const newAgentPolicyData: CreateAgentPolicyRequest['body'] = {
name: `Policy for ${policyName} (${Math.random().toString(36).substr(2, 5)})`,
description: `Policy created with endpoint data generator (${policyName})`,
namespace: 'default',
};
let agentPolicy;
try {
agentPolicy = (await kbnClient.request({
path: AGENT_POLICY_API_ROUTES.CREATE_PATTERN,
method: 'POST',
body: newAgentPolicyData,
})) as AxiosResponse<CreateAgentPolicyResponse>;
} catch (error) {
throw new Error(`create policy ${error}`);
}
// Create Package Configuration
const newPackagePolicyData: CreatePackagePolicyRequest['body'] = {
name: policyName,
description: 'Protect the worlds data',
policy_id: agentPolicy.data.item.id,
enabled: true,
output_id: '',
inputs: [
{
type: 'endpoint',
enabled: true,
streams: [],
config: {
policy: {
value: policyConfigFactory(),
},
},
},
],
namespace: 'default',
package: {
name: 'endpoint',
title: 'endpoint',
version: endpointPackageVersion,
},
};
const packagePolicy = (await kbnClient.request({
path: PACKAGE_POLICY_API_ROUTES.CREATE_PATTERN,
method: 'POST',
body: newPackagePolicyData,
})) as AxiosResponse<CreatePackagePolicyResponse>;
return packagePolicy.data.item;
};
const getEndpointPackageInfo = async (
kbnClient: KbnClient
): Promise<GetPackagesResponse['response'][0]> => {
@ -314,194 +133,14 @@ const getEndpointPackageInfo = async (
return endpointPackage;
};
const fetchKibanaVersion = async (kbnClient: KbnClientWithApiKeySupport) => {
const version = ((await kbnClient.request({
path: '/api/status',
method: 'GET',
})) as AxiosResponse).data.version.number;
export type DeleteIndexedHostsAndAlertsResponse = DeleteIndexedEndpointHostsResponse;
if (!version) {
// eslint-disable-next-line no-console
console.log('failed to retrieve kibana version');
return '8.0.0';
}
return version;
};
/**
* Will ensure that at least one fleet server is present in the `.fleet-servers` index. This will
* enable the `Agent` section of kibana Fleet to be displayed
*
* @param esClient
* @param version
*/
const enableFleetServerIfNecessary = async (esClient: Client, version: string = '8.0.0') => {
const res = await esClient.search<{}, {}>({
index: FLEET_SERVER_SERVERS_INDEX,
ignore_unavailable: true,
});
// @ts-expect-error value is number | TotalHits
if (res.body.hits.total.value > 0) {
return;
}
// Create a Fake fleet-server in this kibana instance
await esClient.index({
index: FLEET_SERVER_SERVERS_INDEX,
body: {
agent: {
id: '12988155-475c-430d-ac89-84dc84b67cd1',
version: '',
},
host: {
architecture: 'linux',
id: 'c3e5f4f690b4a3ff23e54900701a9513',
ip: ['127.0.0.1', '::1', '10.201.0.213', 'fe80::4001:aff:fec9:d5'],
name: 'endpoint-data-generator',
},
server: {
id: '12988155-475c-430d-ac89-84dc84b67cd1',
version: '8.0.0-SNAPSHOT',
},
'@timestamp': '2021-05-12T18:42:52.009482058Z',
},
});
};
const indexFleetAgentForHost = async (
export const deleteIndexedHostsAndAlerts = async (
esClient: Client,
endpointHost: HostMetadata,
agentPolicyId: string,
kibanaVersion: string = '8.0.0'
): Promise<estypes.SearchHit<FleetServerAgent>> => {
const agentDoc = fleetAgentGenerator.generateEsHit({
_source: {
local_metadata: {
elastic: {
agent: {
version: kibanaVersion,
},
},
host: {
...endpointHost.host,
},
os: {
...endpointHost.host.os,
},
},
policy_id: agentPolicyId,
},
});
await esClient.index<FleetServerAgent>({
index: agentDoc._index,
id: agentDoc._id,
body: agentDoc._source!,
op_type: 'create',
});
return agentDoc;
};
const indexFleetActionsForHost = async (
esClient: Client,
endpointHost: HostMetadata
): Promise<void> => {
const ES_INDEX_OPTIONS = { headers: { 'X-elastic-product-origin': 'fleet' } };
const agentId = endpointHost.elastic.agent.id;
const total = fleetActionGenerator.randomN(5);
for (let i = 0; i < total; i++) {
// create an action
const action = fleetActionGenerator.generate({
data: { comment: 'data generator: this host is bad' },
});
action.agents = [agentId];
esClient.index(
{
index: AGENT_ACTIONS_INDEX,
body: action,
},
ES_INDEX_OPTIONS
);
// Create an action response for the above
const actionResponse = fleetActionGenerator.generateResponse({
action_id: action.action_id,
agent_id: agentId,
action_data: action.data,
});
esClient.index(
{
index: AGENT_ACTIONS_RESULTS_INDEX,
body: actionResponse,
},
ES_INDEX_OPTIONS
);
}
// Add edge cases (maybe)
if (fleetActionGenerator.randomFloat() < 0.3) {
const randomFloat = fleetActionGenerator.randomFloat();
// 60% of the time just add either an Isolate -OR- an UnIsolate action
if (randomFloat < 0.6) {
let action: EndpointAction;
if (randomFloat < 0.3) {
// add a pending isolation
action = fleetActionGenerator.generateIsolateAction({
'@timestamp': new Date().toISOString(),
});
} else {
// add a pending UN-isolation
action = fleetActionGenerator.generateUnIsolateAction({
'@timestamp': new Date().toISOString(),
});
}
action.agents = [agentId];
await esClient.index(
{
index: AGENT_ACTIONS_INDEX,
body: action,
},
ES_INDEX_OPTIONS
);
} else {
// Else (40% of the time) add a pending isolate AND pending un-isolate
const action1 = fleetActionGenerator.generateIsolateAction({
'@timestamp': new Date().toISOString(),
});
const action2 = fleetActionGenerator.generateUnIsolateAction({
'@timestamp': new Date().toISOString(),
});
action1.agents = [agentId];
action2.agents = [agentId];
await Promise.all([
esClient.index(
{
index: AGENT_ACTIONS_INDEX,
body: action1,
},
ES_INDEX_OPTIONS
),
esClient.index(
{
index: AGENT_ACTIONS_INDEX,
body: action2,
},
ES_INDEX_OPTIONS
),
]);
}
}
kbnClient: KbnClient,
indexedData: IndexedHostsAndAlertsResponse
): Promise<DeleteIndexedHostsAndAlertsResponse> => {
return {
...(await deleteIndexedEndpointHosts(esClient, kbnClient, indexedData)),
};
};

5
x-pack/plugins/security_solution/public/detections/pages/detection_engine/detection_engine.tsx
View file

@ -227,7 +227,10 @@ const DetectionEnginePageComponent = () => {
<SiemSearchBar id="global" indexPattern={indexPattern} />
</FiltersGlobal>
<SecuritySolutionPageWrapper noPadding={globalFullScreen}>
<SecuritySolutionPageWrapper
noPadding={globalFullScreen}
data-test-subj="detectionsAlertsPage"
>
<Display show={!globalFullScreen}>
<DetectionEngineHeaderPage
subtitle={

5
x-pack/plugins/security_solution/public/hosts/pages/details/index.tsx
View file

@ -126,7 +126,10 @@ const HostDetailsComponent: React.FC<HostDetailsProps> = ({ detailName, hostDeta
<SiemSearchBar indexPattern={indexPattern} id="global" />
</FiltersGlobal>
<SecuritySolutionPageWrapper noPadding={globalFullScreen}>
<SecuritySolutionPageWrapper
noPadding={globalFullScreen}
data-test-subj="hostDetailsPage"
>
<Display show={!globalFullScreen}>
<HeaderPage
border

36
x-pack/plugins/security_solution/scripts/endpoint/kbn_client_with_api_key_support.ts
View file

@ -1,36 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { URL } from 'url';
import { KbnClient, KbnClientOptions } from '@kbn/test';
import fetch, { RequestInit } from 'node-fetch';
export class KbnClientWithApiKeySupport extends KbnClient {
private kibanaUrlNoAuth: URL;
constructor(options: KbnClientOptions) {
super(options);
// strip auth from url
const url = new URL(this.resolveUrl('/'));
url.username = '';
url.password = '';
this.kibanaUrlNoAuth = url;
}
/**
* The fleet api to enroll and agent requires an api key when you make
* the request, however KbnClient currently does not support sending
* an api key with the request. This function allows you to send an
* api key with a request.
*/
requestWithApiKey(path: string, init?: RequestInit) {
return fetch(new URL(path, this.kibanaUrlNoAuth), init);
}
}

95
x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts
View file

@ -12,21 +12,8 @@ import { Client, ClientOptions } from '@elastic/elasticsearch';
import { ResponseError } from '@elastic/elasticsearch/lib/errors';
import { ToolingLog, CA_CERT_PATH } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
import { AxiosResponse } from 'axios';
import { indexHostsAndAlerts } from '../../common/endpoint/index_data';
import { ANCESTRY_LIMIT, EndpointDocGenerator } from '../../common/endpoint/generate_data';
import {
AGENTS_SETUP_API_ROUTES,
EPM_API_ROUTES,
SETUP_API_ROUTE,
} from '../../../fleet/common/constants';
import {
BulkInstallPackageInfo,
BulkInstallPackagesResponse,
IBulkInstallPackageHTTPError,
PostFleetSetupResponse,
} from '../../../fleet/common/types/rest_spec';
import { KbnClientWithApiKeySupport } from './kbn_client_with_api_key_support';
main();
@ -50,75 +37,6 @@ async function deleteIndices(indices: string[], client: Client) {
}
}
function isFleetBulkInstallError(
installResponse: BulkInstallPackageInfo | IBulkInstallPackageHTTPError
): installResponse is IBulkInstallPackageHTTPError {
return 'error' in installResponse && installResponse.error !== undefined;
}
async function doIngestSetup(kbnClient: KbnClient) {
// Setup Ingest
try {
const setupResponse = (await kbnClient.request({
path: SETUP_API_ROUTE,
method: 'POST',
})) as AxiosResponse<PostFleetSetupResponse>;
if (!setupResponse.data.isInitialized) {
console.error(setupResponse.data);
throw new Error('Initializing the ingest manager failed, existing');
}
} catch (error) {
console.error(error);
throw error;
}
// Setup Fleet
try {
const setupResponse = (await kbnClient.request({
path: AGENTS_SETUP_API_ROUTES.CREATE_PATTERN,
method: 'POST',
})) as AxiosResponse<PostFleetSetupResponse>;
if (!setupResponse.data.isInitialized) {
console.error(setupResponse.data);
throw new Error('Initializing Fleet failed, existing');
}
} catch (error) {
console.error(error);
throw error;
}
// Install/upgrade the endpoint package
try {
const installEndpointPackageResp = (await kbnClient.request({
path: EPM_API_ROUTES.BULK_INSTALL_PATTERN,
method: 'POST',
body: {
packages: ['endpoint'],
},
})) as AxiosResponse<BulkInstallPackagesResponse>;
const bulkResp = installEndpointPackageResp.data.response;
if (bulkResp.length <= 0) {
throw new Error('Installing the Endpoint package failed, response was empty, existing');
}
if (isFleetBulkInstallError(bulkResp[0])) {
if (bulkResp[0].error instanceof Error) {
throw new Error(
`Installing the Endpoint package failed: ${bulkResp[0].error.message}, exiting`
);
}
throw new Error(bulkResp[0].error);
}
} catch (error) {
console.error(error);
throw error;
}
}
async function main() {
const argv = yargs.help().options({
seed: {
@ -255,14 +173,14 @@ async function main() {
},
}).argv;
let ca: Buffer;
let kbnClient: KbnClientWithApiKeySupport;
let kbnClient: KbnClient;
let clientOptions: ClientOptions;
if (argv.ssl) {
ca = fs.readFileSync(CA_CERT_PATH);
const url = argv.kibana.replace('http:', 'https:');
const node = argv.node.replace('http:', 'https:');
kbnClient = new KbnClientWithApiKeySupport({
kbnClient = new KbnClient({
log: new ToolingLog({
level: 'info',
writeTo: process.stdout,
@ -272,7 +190,7 @@ async function main() {
});
clientOptions = { node, ssl: { ca: [ca] } };
} else {
kbnClient = new KbnClientWithApiKeySupport({
kbnClient = new KbnClient({
log: new ToolingLog({
level: 'info',
writeTo: process.stdout,
@ -283,13 +201,6 @@ async function main() {
}
const client = new Client(clientOptions);
try {
await doIngestSetup(kbnClient);
} catch (error) {
// eslint-disable-next-line no-process-exit
process.exit(1);
}
if (argv.delete) {
await deleteIndices(
[argv.eventIndex, argv.metadataIndex, argv.policyIndex, argv.alertIndex],

16
x-pack/plugins/security_solution/server/endpoint/services/metadata/endpoint_metadata_service.ts
View file

@ -26,7 +26,12 @@ import {
} from './errors';
import { getESQueryHostMetadataByID } from '../../routes/metadata/query_builders';
import { queryResponseToHostResult } from '../../routes/metadata/support/query_strategies';
import { DEFAULT_ENDPOINT_HOST_STATUS, fleetAgentStatusToEndpointHostStatus } from '../../utils';
import {
catchAndWrapError,
DEFAULT_ENDPOINT_HOST_STATUS,
fleetAgentStatusToEndpointHostStatus,
wrapErrorIfNeeded,
} from '../../utils';
import { EndpointError } from '../../errors';
import { createInternalReadonlySoClient } from '../../utils/create_internal_readonly_so_client';
@ -34,15 +39,6 @@ type AgentPolicyWithPackagePolicies = Omit<AgentPolicy, 'package_policies'> & {
package_policies: PackagePolicy[];
};
// Will wrap the given Error with `EndpointError`, which will
// help getting a good picture of where in our code the error originated from.
const wrapErrorIfNeeded = (error: Error): EndpointError =>
error instanceof EndpointError ? error : new EndpointError(error.message, error);
// used as the callback to `Promise#catch()` to ensure errors
// (especially those from kibana/elasticsearch clients) are wrapped
const catchAndWrapError = <E extends Error>(error: E) => Promise.reject(wrapErrorIfNeeded(error));
export class EndpointMetadataService {
/**
* For internal use only by the `this.DANGEROUS_INTERNAL_SO_CLIENT`

1
x-pack/plugins/security_solution/server/endpoint/utils/index.ts
View file

@ -6,3 +6,4 @@
*/
export * from './fleet_agent_status_to_endpoint_host_status';
export * from './wrap_errors';

24
x-pack/plugins/security_solution/server/endpoint/utils/wrap_errors.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EndpointError } from '../errors';
/**
* Will wrap the given Error with `EndpointError`, which will help getting a good picture of where in
* our code the error originated (better stack trace).
*/
export const wrapErrorIfNeeded = (error: Error): EndpointError =>
error instanceof EndpointError ? error : new EndpointError(error.message, error);
/**
* used as the callback to `Promise#catch()` to ensure errors
* (especially those from kibana/elasticsearch clients) are wrapped
*
* @param error
*/
export const catchAndWrapError = <E extends Error>(error: E) =>
Promise.reject(wrapErrorIfNeeded(error));

2
x-pack/plugins/security_solution/server/index.ts
View file

@ -47,5 +47,5 @@ export const config: PluginConfigDescriptor<ConfigType> = {
export { ConfigType, Plugin, PluginSetup, PluginStart };
export { AppClient };
export type { AppRequestContext } from './types';
export { EndpointError } from './endpoint/errors';

8
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/detections_admin/detections_role.json
View file

@ -17,6 +17,14 @@
"winlogbeat-*"
],
"privileges": ["manage", "write", "read"]
},
{
"names": [
"metrics-endpoint.metadata_current_*",
".fleet-agents*",
".fleet-actions*"
],
"privileges": ["read"]
}
]
},

8
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/hunter/detections_role.json
View file

@ -22,6 +22,14 @@
{
"names": [".lists*", ".items*"],
"privileges": ["read", "write"]
},
{
"names": [
"metrics-endpoint.metadata_current_*",
".fleet-agents*",
".fleet-actions*"
],
"privileges": ["read"]
}
]
},

5
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/platform_engineer/detections_role.json
View file

@ -15,7 +15,10 @@
"filebeat-*",
"logs-*",
"packetbeat-*",
"winlogbeat-*"
"winlogbeat-*",
"metrics-endpoint.metadata_current_*",
".fleet-agents*",
".fleet-actions*"
],
"privileges": ["all"]
},

5
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/reader/detections_role.json
View file

@ -6,7 +6,10 @@
"names" : [
".siem-signals*",
".lists*",
".items*"
".items*",
"metrics-endpoint.metadata_current_*",
".fleet-agents*",
".fleet-actions*"
],
"privileges" : ["read"]
},

8
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/rule_author/detections_role.json
View file

@ -20,6 +20,14 @@
{
"names": [".siem-signals-*"],
"privileges": ["read", "write", "maintenance", "view_index_metadata"]
},
{
"names": [
"metrics-endpoint.metadata_current_*",
".fleet-agents*",
".fleet-actions*"
],
"privileges": ["read"]
}
]
},

8
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/soc_manager/detections_role.json
View file

@ -20,6 +20,14 @@
{
"names": [".siem-signals-*"],
"privileges": ["read", "write", "manage"]
},
{
"names": [
"metrics-endpoint.metadata_current_*",
".fleet-agents*",
".fleet-actions*"
],
"privileges": ["read"]
}
]
},

5
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t1_analyst/detections_role.json
View file

@ -12,7 +12,10 @@
"filebeat-*",
"logs-*",
"packetbeat-*",
"winlogbeat-*"
"winlogbeat-*",
"metrics-endpoint.metadata_current_*",
".fleet-agents*",
".fleet-actions*"
],
"privileges": ["read"]
}

5
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t2_analyst/detections_role.json
View file

@ -14,7 +14,10 @@
"filebeat-*",
"logs-*",
"packetbeat-*",
"winlogbeat-*"
"winlogbeat-*",
"metrics-endpoint.metadata_current_*",
".fleet-agents*",
".fleet-actions*"
],
"privileges": ["read"]
}

8
x-pack/test/common/services/security_solution/index.ts Normal file
View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export * from './roles_users_utils';

11
x-pack/test/detection_engine_api_integration/security_and_spaces/roles_users_utils/index.ts → x-pack/test/common/services/security_solution/roles_users_utils.ts
View file

@ -5,8 +5,8 @@
* 2.0.
*/
import { assertUnreachable } from '../../../../plugins/security_solution/common/utility_types';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { assertUnreachable } from '../../../../plugins/security_solution/common';
import { FtrProviderContext } from '../../ftr_provider_context';
import {
t1AnalystUser,
t2AnalystUser,
@ -28,6 +28,13 @@ import {
import { ROLES } from '../../../../plugins/security_solution/common/test';
export { ROLES };
/**
* creates a security solution centric role and a user (both having the same name)
* @param getService
* @param role
*/
export const createUserAndRole = async (
getService: FtrProviderContext['getService'],
role: ROLES

2
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts
View file

@ -53,7 +53,7 @@ import {
getThresholdRuleForSignalTesting,
} from '../../utils';
import { ROLES } from '../../../../plugins/security_solution/common/test';
import { createUserAndRole, deleteUserAndRole } from '../roles_users_utils';
import { createUserAndRole, deleteUserAndRole } from '../../../common/services/security_solution';
// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext) => {

2
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts
View file

@ -14,7 +14,7 @@ import {
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { deleteSignalsIndex } from '../../utils';
import { ROLES } from '../../../../plugins/security_solution/common/test';
import { createUserAndRole, deleteUserAndRole } from '../roles_users_utils';
import { createUserAndRole, deleteUserAndRole } from '../../../common/services/security_solution';
// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext) => {

2
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts
View file

@ -32,7 +32,7 @@ import {
getRuleForSignalTestingWithTimestampOverride,
} from '../../utils';
import { ROLES } from '../../../../plugins/security_solution/common/test';
import { createUserAndRole, deleteUserAndRole } from '../roles_users_utils';
import { createUserAndRole, deleteUserAndRole } from '../../../common/services/security_solution';
import { RuleStatusResponse } from '../../../../plugins/security_solution/server/lib/detection_engine/rules/types';
// eslint-disable-next-line import/no-default-export

2
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_signals_migrations.ts
View file

@ -22,7 +22,7 @@ import {
getIndexNameFromLoad,
waitForIndexToPopulate,
} from '../../utils';
import { createUserAndRole, deleteUserAndRole } from '../roles_users_utils';
import { createUserAndRole, deleteUserAndRole } from '../../../common/services/security_solution';
interface CreateResponse {
index: string;

2
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/delete_signals_migrations.ts
View file

@ -15,7 +15,7 @@ import {
import { ROLES } from '../../../../plugins/security_solution/common/test';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { createSignalsIndex, deleteSignalsIndex, getIndexNameFromLoad, waitFor } from '../../utils';
import { createUserAndRole } from '../roles_users_utils';
import { createUserAndRole } from '../../../common/services/security_solution';
interface CreateResponse {
index: string;

2
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/finalize_signals_migrations.ts
View file

@ -21,7 +21,7 @@ import {
getIndexNameFromLoad,
waitFor,
} from '../../utils';
import { createUserAndRole, deleteUserAndRole } from '../roles_users_utils';
import { createUserAndRole, deleteUserAndRole } from '../../../common/services/security_solution';
interface StatusResponse {
index: string;

2
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/get_signals_migration_status.ts
View file

@ -11,7 +11,7 @@ import { DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL } from '../../../../plugi
import { ROLES } from '../../../../plugins/security_solution/common/test';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { createSignalsIndex, deleteSignalsIndex, getIndexNameFromLoad } from '../../utils';
import { createUserAndRole, deleteUserAndRole } from '../roles_users_utils';
import { createUserAndRole, deleteUserAndRole } from '../../../common/services/security_solution';
// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext): void => {

2
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/open_close_signals.ts
View file

@ -27,7 +27,7 @@ import {
waitForRuleSuccessOrStatus,
getRuleForSignalTesting,
} from '../../utils';
import { createUserAndRole, deleteUserAndRole } from '../roles_users_utils';
import { createUserAndRole, deleteUserAndRole } from '../../../common/services/security_solution';
import { ROLES } from '../../../../plugins/security_solution/common/test';
// eslint-disable-next-line import/no-default-export

2
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/read_privileges.ts
View file

@ -10,7 +10,7 @@ import { DETECTION_ENGINE_PRIVILEGES_URL } from '../../../../plugins/security_so
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { ROLES } from '../../../../plugins/security_solution/common/test';
import { createUserAndRole, deleteUserAndRole } from '../roles_users_utils';
import { createUserAndRole, deleteUserAndRole } from '../../../common/services/security_solution';
// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext) => {

21
x-pack/test/functional/services/transform/api.ts
View file

@ -12,6 +12,9 @@ import { TransformState, TRANSFORM_STATE } from '../../../../plugins/transform/c
import type { TransformStats } from '../../../../plugins/transform/common/types/transform_stats';
import { FtrProviderContext } from '../../ftr_provider_context';
import { GetTransformsResponseSchema } from '../../../../plugins/transform/common/api_schemas/transforms';
import { PostTransformsUpdateRequestSchema } from '../../../../plugins/transform/common/api_schemas/update_transforms';
import { TransformPivotConfig } from '../../../../plugins/transform/common/types/transform';
export async function asyncForEach(array: any[], callback: Function) {
for (let index = 0; index < array.length; index++) {
@ -174,10 +177,28 @@ export function TransformAPIProvider({ getService }: FtrProviderContext) {
});
},
async getTransformList(size: number = 10): Promise<GetTransformsResponseSchema> {
return (await esSupertest
.get(`/_transform`)
.expect(200)
.then((response) => response.body)) as GetTransformsResponseSchema;
},
async getTransform(transformId: string, expectedCode = 200) {
return await esSupertest.get(`/_transform/${transformId}`).expect(expectedCode);
},
async updateTransform(
transformId: string,
updates: Partial<PostTransformsUpdateRequestSchema>
): Promise<TransformPivotConfig> {
return await esSupertest
.post(`/_transform/${transformId}/_update`)
.send(updates)
.expect(200)
.then((response: { body: TransformPivotConfig }) => response.body);
},
async createTransform(transformId: string, transformConfig: PutTransformsRequestSchema) {
log.debug(`Creating transform with id '${transformId}'...`);
await esSupertest.put(`/_transform/${transformId}`).send(transformConfig).expect(200);

118
x-pack/test/rule_registry/security_and_spaces/roles_users_utils/index.ts
View file

@ -5,120 +5,4 @@
* 2.0.
*/
import { assertUnreachable } from '../../../../plugins/security_solution/common/utility_types';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import {
t1AnalystUser,
t2AnalystUser,
hunterUser,
ruleAuthorUser,
socManagerUser,
platformEngineerUser,
detectionsAdminUser,
readerUser,
t1AnalystRole,
t2AnalystRole,
hunterRole,
ruleAuthorRole,
socManagerRole,
platformEngineerRole,
detectionsAdminRole,
readerRole,
} from '../../../../plugins/security_solution/server/lib/detection_engine/scripts/roles_users';
import { ROLES } from '../../../../plugins/security_solution/common/test';
export const createUserAndRole = async (
getService: FtrProviderContext['getService'],
role: ROLES
): Promise<void> => {
switch (role) {
case ROLES.detections_admin:
return postRoleAndUser(
ROLES.detections_admin,
detectionsAdminRole,
detectionsAdminUser,
getService
);
case ROLES.t1_analyst:
return postRoleAndUser(ROLES.t1_analyst, t1AnalystRole, t1AnalystUser, getService);
case ROLES.t2_analyst:
return postRoleAndUser(ROLES.t2_analyst, t2AnalystRole, t2AnalystUser, getService);
case ROLES.hunter:
return postRoleAndUser(ROLES.hunter, hunterRole, hunterUser, getService);
case ROLES.rule_author:
return postRoleAndUser(ROLES.rule_author, ruleAuthorRole, ruleAuthorUser, getService);
case ROLES.soc_manager:
return postRoleAndUser(ROLES.soc_manager, socManagerRole, socManagerUser, getService);
case ROLES.platform_engineer:
return postRoleAndUser(
ROLES.platform_engineer,
platformEngineerRole,
platformEngineerUser,
getService
);
case ROLES.reader:
return postRoleAndUser(ROLES.reader, readerRole, readerUser, getService);
default:
return assertUnreachable(role);
}
};
/**
* Given a roleName and security service this will delete the roleName
* and user
* @param roleName The user and role to delete with the same name
* @param securityService The security service
*/
export const deleteUserAndRole = async (
getService: FtrProviderContext['getService'],
roleName: ROLES
): Promise<void> => {
const securityService = getService('security');
await securityService.user.delete(roleName);
await securityService.role.delete(roleName);
};
interface UserInterface {
password: string;
roles: string[];
full_name: string;
email: string;
}
interface RoleInterface {
elasticsearch: {
cluster: string[];
indices: Array<{
names: string[];
privileges: string[];
}>;
};
kibana: Array<{
feature: {
ml: string[];
siem: string[];
actions: string[];
builtInAlerts: string[];
};
spaces: string[];
}>;
}
export const postRoleAndUser = async (
roleName: string,
role: RoleInterface,
user: UserInterface,
getService: FtrProviderContext['getService']
): Promise<void> => {
const securityService = getService('security');
await securityService.role.create(roleName, {
kibana: role.kibana,
elasticsearch: role.elasticsearch,
});
await securityService.user.create(roleName, {
password: 'changeme',
full_name: user.full_name,
roles: user.roles,
});
};
export * from '../../../common/services/security_solution/roles_users_utils';

95
x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_permissions.ts Normal file
View file

@ -0,0 +1,95 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import {
createUserAndRole,
deleteUserAndRole,
ROLES,
} from '../../../common/services/security_solution';
import { IndexedHostsAndAlertsResponse } from '../../../../plugins/security_solution/common/endpoint/index_data';
export default ({ getPageObjects, getService }: FtrProviderContext) => {
const PageObjects = getPageObjects(['security', 'endpoint', 'detections', 'hosts']);
const testSubjects = getService('testSubjects');
const endpointTestResources = getService('endpointTestResources');
const policyTestResources = getService('policyTestResources');
describe('Endpoint permissions:', () => {
let indexedData: IndexedHostsAndAlertsResponse;
before(async () => {
// todo: way to force an endpoint to be created in isolated mode so we can check that state in the UI
const endpointPackage = await policyTestResources.getEndpointPackage();
await endpointTestResources.setMetadataTransformFrequency('1s', endpointPackage.version);
indexedData = await endpointTestResources.loadEndpointData();
// Force a logout so that we start from the login page
await PageObjects.security.forceLogout();
});
after(async () => {
await endpointTestResources.unloadEndpointData(indexedData);
});
// Run the same set of tests against all of the Security Solution roles
for (const role of Object.keys(ROLES) as Array<keyof typeof ROLES>) {
describe(`when running with user/role [${role}]`, () => {
before(async () => {
// create role/user
await createUserAndRole(getService, ROLES[role]);
// log back in with new uer
await PageObjects.security.login(role, 'changeme');
});
after(async () => {
// Log the user back out
await PageObjects.security.forceLogout();
// delete role/user
await deleteUserAndRole(getService, ROLES[role]);
});
it('should NOT allow access to endpoint management pages', async () => {
await PageObjects.endpoint.navigateToEndpointList();
await testSubjects.existOrFail('noIngestPermissions');
});
it('should display endpoint data on Host Details', async () => {
const endpoint = indexedData.hosts[0];
await PageObjects.hosts.navigateToHostDetails(endpoint.host.name);
const endpointSummary = await PageObjects.hosts.hostDetailsEndpointOverviewData();
expect(endpointSummary['Endpoint integration policy']).to.be(
endpoint.Endpoint.policy.applied.name
);
expect(endpointSummary['Endpoint version']).to.be(endpoint.agent.version);
// The values for these are calculated, so let's just make sure its not teh default when no data is returned
expect(endpointSummary['Policy status']).not.be('—');
expect(endpointSummary['Agent status']).not.to.be('—');
});
// FIXME: this area (detections) is unstable and due to time, skipping it.
// The page does not always (its intermittent) display with the created roles. Sometimes you get a
// "not enought priviliges" and others the data shows up.
it.skip('should display endpoint data on Alert Details', async () => {
await PageObjects.detections.navigateToAlerts();
await PageObjects.detections.openFirstAlertDetailsForHostName(
indexedData.hosts[0].host.name
);
const hostAgentStatus = await testSubjects.getVisibleText('rowHostStatus');
expect(hostAgentStatus).to.eql('Healthy');
});
});
}
});
};

6
x-pack/test/security_solution_endpoint/apps/endpoint/index.ts
View file

@ -18,6 +18,7 @@ export default function (providerContext: FtrProviderContext) {
describe('endpoint', function () {
const ingestManager = getService('ingestManager');
const log = getService('log');
const endpointTestResources = getService('endpointTestResources');
if (!isRegistryEnabled()) {
log.warning('These tests are being run with an external package registry');
@ -27,12 +28,17 @@ export default function (providerContext: FtrProviderContext) {
log.info(`Package registry URL for tests: ${registryUrl}`);
before(async () => {
log.info('calling Fleet setup');
await ingestManager.setup();
log.info('installing/upgrading Endpoint fleet package');
await endpointTestResources.installOrUpgradeEndpointFleetPackage();
});
loadTestFile(require.resolve('./endpoint_list'));
loadTestFile(require.resolve('./policy_details'));
loadTestFile(require.resolve('./endpoint_telemetry'));
loadTestFile(require.resolve('./trusted_apps_list'));
loadTestFile(require.resolve('./fleet_integrations'));
loadTestFile(require.resolve('./endpoint_permissions'));
});
}

4
x-pack/test/security_solution_endpoint/page_objects/index.ts
View file

@ -12,6 +12,8 @@ import { TrustedAppsPageProvider } from './trusted_apps_page';
import { EndpointPageUtils } from './page_utils';
import { IngestManagerCreatePackagePolicy } from './ingest_manager_create_package_policy_page';
import { FleetIntegrations } from './fleet_integrations_page';
import { DetectionsPageObject } from '../../security_solution_ftr/page_objects/detections';
import { HostsPageObject } from '../../security_solution_ftr/page_objects/hosts';
export const pageObjects = {
...xpackFunctionalPageObjects,
@ -21,4 +23,6 @@ export const pageObjects = {
endpointPageUtils: EndpointPageUtils,
ingestManagerCreatePackagePolicy: IngestManagerCreatePackagePolicy,
fleetIntegrations: FleetIntegrations,
detections: DetectionsPageObject,
hosts: HostsPageObject,
};

195
x-pack/test/security_solution_endpoint/services/endpoint.ts Normal file
View file

@ -0,0 +1,195 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ResponseError } from '@elastic/elasticsearch/lib/errors';
import { Client } from '@elastic/elasticsearch';
import { FtrService } from '../../functional/ftr_provider_context';
import {
metadataCurrentIndexPattern,
metadataTransformPrefix,
} from '../../../plugins/security_solution/common/endpoint/constants';
import { EndpointError } from '../../../plugins/security_solution/server';
import {
deleteIndexedHostsAndAlerts,
IndexedHostsAndAlertsResponse,
indexHostsAndAlerts,
} from '../../../plugins/security_solution/common/endpoint/index_data';
import { TransformPivotConfig } from '../../../plugins/transform/common/types/transform';
import { GetTransformsResponseSchema } from '../../../plugins/transform/common/api_schemas/transforms';
import { catchAndWrapError } from '../../../plugins/security_solution/server/endpoint/utils';
import { installOrUpgradeEndpointFleetPackage } from '../../../plugins/security_solution/common/endpoint/data_loaders/setup_fleet_for_endpoint';
export class EndpointTestResources extends FtrService {
private readonly esClient = this.ctx.getService('es');
private readonly retry = this.ctx.getService('retry');
private readonly kbnClient = this.ctx.getService('kibanaServer');
private readonly transform = this.ctx.getService('transform');
private generateTransformId(endpointPackageVersion?: string): string {
return `${metadataTransformPrefix}-${endpointPackageVersion ?? ''}`;
}
/**
* Fetches the information for the endpoint transform
*
* @param [endpointPackageVersion] if set, it will be used to get the specific transform this this package version. Else just returns first one found
*/
async getTransform(endpointPackageVersion?: string): Promise<TransformPivotConfig> {
const transformId = this.generateTransformId(endpointPackageVersion);
let transform: TransformPivotConfig | undefined;
if (endpointPackageVersion) {
await this.transform.api.waitForTransformToExist(transformId);
transform = ((
await this.transform.api
.getTransform(transformId)
.catch(catchAndWrapError)
.then((response: { body: GetTransformsResponseSchema }) => response)
).body as GetTransformsResponseSchema).transforms[0];
} else {
transform = (
await this.transform.api.getTransformList(100).catch(catchAndWrapError)
).transforms.find((t) => t.id.startsWith(transformId));
}
if (!transform) {
throw new EndpointError('Endpoint metadata transform not found');
}
return transform;
}
async setMetadataTransformFrequency(
frequency: string,
/** Used to update the transform installed with the given package version */
endpointPackageVersion?: string
): Promise<void> {
const transform = await this.getTransform(endpointPackageVersion).catch(catchAndWrapError);
await this.transform.api.updateTransform(transform.id, { frequency }).catch(catchAndWrapError);
}
/**
* Loads endpoint host/alert/event data into elasticsearch
* @param [options]
* @param [options.numHosts=1] Number of Endpoint Hosts to be loaded
* @param [options.numHostDocs=1] Number of Document to be loaded per Endpoint Host (Endpoint hosts index uses a append-only index)
* @param [options.alertsPerHost=1] Number of Alerts and Events to be loaded per Endpoint Host
* @param [options.enableFleetIntegration=true] When set to `true`, Fleet data will also be loaded (ex. Integration Policies, Agent Policies, "fake" Agents)
* @param [options.generatorSeed='seed`] The seed to be used by the data generator. Important in order to ensure the same data is generated on very run.
* @param [options.waitUntilTransformed=true] If set to `true`, the data loading process will wait until the endpoint hosts metadata is processd by the transform
*/
async loadEndpointData(
options: Partial<{
numHosts: number;
numHostDocs: number;
alertsPerHost: number;
enableFleetIntegration: boolean;
generatorSeed: string;
waitUntilTransformed: boolean;
}> = {}
): Promise<IndexedHostsAndAlertsResponse> {
const {
numHosts = 1,
numHostDocs = 1,
alertsPerHost = 1,
enableFleetIntegration = true,
generatorSeed = 'seed',
waitUntilTransformed = true,
} = options;
// load data into the system
const indexedData = await indexHostsAndAlerts(
this.esClient as Client,
this.kbnClient,
generatorSeed,
numHosts,
numHostDocs,
'metrics-endpoint.metadata-default',
'metrics-endpoint.policy-default',
'logs-endpoint.events.process-default',
'logs-endpoint.alerts-default',
alertsPerHost,
enableFleetIntegration
);
if (waitUntilTransformed) {
await this.waitForEndpoints(indexedData.hosts.map((host) => host.agent.id));
}
return indexedData;
}
/**
* Deletes the loaded data created via `loadEndpointData()`
* @param indexedData
*/
async unloadEndpointData(indexedData: IndexedHostsAndAlertsResponse) {
return deleteIndexedHostsAndAlerts(this.esClient as Client, this.kbnClient, indexedData);
}
/**
* Waits for endpoints to show up on the `metadata-current` index.
* Optionally, specific endpoint IDs (agent.id) can be provided to ensure those specific ones show up.
*
* @param [ids] optional list of ids to check for. If empty, it will just check if data exists in the index
*/
async waitForEndpoints(ids: string[] = []) {
const body = ids.length
? {
query: {
bool: {
filter: [
{
terms: {
'agent.id': ids,
},
},
],
},
},
}
: {
query: {
match_all: {},
},
};
// If we have a specific number of endpoint hosts to check for, then use that number,
// else we just want to make sure the index has data, thus just having one in the index will do
const size = ids.length || 1;
await this.retry.waitFor('wait for endpoints hosts', async () => {
try {
const searchResponse = await this.esClient.search({
index: metadataCurrentIndexPattern,
size,
body,
rest_total_hits_as_int: true,
});
return searchResponse.body.hits.total === size;
} catch (error) {
// We ignore 404's (index might not exist)
if (error instanceof ResponseError && error.statusCode === 404) {
return false;
}
// Wrap the ES error so that we get a good stack trace
throw new EndpointError(error.message, error);
}
});
}
/**
* installs (or upgrades) the Endpoint Fleet package
* (NOTE: ensure that fleet is setup first before calling this function)
*/
async installOrUpgradeEndpointFleetPackage(): Promise<void> {
return installOrUpgradeEndpointFleetPackage(this.kbnClient);
}
}

14
x-pack/test/security_solution_endpoint/services/endpoint_policy.ts
View file

@ -23,6 +23,7 @@ import { Immutable } from '../../../plugins/security_solution/common/endpoint/ty
// NOTE: import path below should be the deep path to the actual module - else we get CI errors
import { pkgKeyFromPackageInfo } from '../../../plugins/fleet/public/services/pkg_key_from_package_info';
import { EndpointError } from '../../../plugins/security_solution/server';
const INGEST_API_ROOT = '/api/fleet';
const INGEST_API_AGENT_POLICIES = `${INGEST_API_ROOT}/agent_policies`;
@ -119,6 +120,19 @@ export function EndpointPolicyTestResourcesProvider({ getService }: FtrProviderC
return pkgKeyFromPackageInfo((await retrieveEndpointPackageInfo())!);
},
/**
* Retrieves the currently installed endpoint package
*/
async getEndpointPackage(): Promise<Immutable<GetPackagesResponse['response'][0]>> {
const endpointPackage = await retrieveEndpointPackageInfo();
if (!endpointPackage) {
throw new EndpointError(`endpoint package not instealled`);
}
return endpointPackage;
},
/**
* Retrieves the full Agent policy, which mirrors what the Elastic Agent would get
* once they checkin.

2
x-pack/test/security_solution_endpoint/services/index.ts
View file

@ -9,9 +9,11 @@ import { services as xPackFunctionalServices } from '../../functional/services';
import { EndpointPolicyTestResourcesProvider } from './endpoint_policy';
import { IngestManagerProvider } from '../../common/services/ingest_manager';
import { EndpointTelemetryTestResourcesProvider } from './endpoint_telemetry';
import { EndpointTestResources } from './endpoint';
export const services = {
...xPackFunctionalServices,
endpointTestResources: EndpointTestResources,
policyTestResources: EndpointPolicyTestResourcesProvider,
telemetryTestResources: EndpointTelemetryTestResourcesProvider,
ingestManager: IngestManagerProvider,

39
x-pack/test/security_solution_ftr/page_objects/detections/index.ts
View file

@ -12,11 +12,17 @@ export class DetectionsPageObject extends FtrService {
private readonly find = this.ctx.getService('find');
private readonly common = this.ctx.getPageObject('common');
private readonly testSubjects = this.ctx.getService('testSubjects');
private readonly headerPageObjects = this.ctx.getPageObject('header');
async navigateHome(): Promise<void> {
await this.navigateToDetectionsPage();
}
async navigateToAlerts(): Promise<void> {
await this.navigateToDetectionsPage('alerts');
await this.headerPageObjects.waitUntilLoadingHasFinished();
}
async navigateToRules(): Promise<void> {
await this.navigateToDetectionsPage('rules');
}
@ -139,6 +145,39 @@ export class DetectionsPageObject extends FtrService {
await this.common.clickAndValidate('thresholdRuleType', 'input');
}
async ensureOnAlertsPage(): Promise<void> {
await this.testSubjects.existOrFail('detectionsAlertsPage');
}
async openFirstAlertDetailsForHostName(hostName: string): Promise<void> {
await this.ensureOnAlertsPage();
let foundAndHandled = false;
// Get all event rows
const allEvents = await this.testSubjects.findAll('event');
for (const eventRow of allEvents) {
const hostNameButton = await this.testSubjects.findDescendant(
'host-details-button',
eventRow
);
const eventRowHostName = (await hostNameButton.getVisibleText()).trim();
if (eventRowHostName === hostName) {
const expandAlertButton = await this.testSubjects.findDescendant('expand-event', eventRow);
await expandAlertButton.click();
await this.testSubjects.existOrFail('eventDetails');
foundAndHandled = true;
break;
}
}
if (!foundAndHandled) {
throw new Error(`no alerts found for host: ${hostName}`);
}
}
private async navigateToDetectionsPage(path: string = ''): Promise<void> {
const subUrl = `detections${path ? `/${path}` : ''}`;
await this.common.navigateToUrl('securitySolution', subUrl, {

60
x-pack/test/security_solution_ftr/page_objects/hosts/index.ts Normal file
View file

@ -0,0 +1,60 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrService } from '../../../functional/ftr_provider_context';
import { WebElementWrapper } from '../../../../../test/functional/services/lib/web_element_wrapper';
export class HostsPageObject extends FtrService {
private readonly pageObjects = this.ctx.getPageObjects(['common', 'header']);
private readonly testSubjects = this.ctx.getService('testSubjects');
async navigateToHostDetails(hostName: string): Promise<void> {
await this.pageObjects.common.navigateToUrl('securitySolution', `hosts/${hostName}`, {
shouldUseHashForSubUrl: false,
});
await this.pageObjects.header.waitUntilLoadingHasFinished();
}
async ensureOnHostDetails(): Promise<void> {
await this.testSubjects.existOrFail('hostDetailsPage');
}
/**
* Returns an object with the Endpoint overview data, where the keys are the visible labels in the UI.
* Must be on the Host details in order for this page object to work
*/
async hostDetailsEndpointOverviewData(): Promise<Record<string, string>> {
await this.ensureOnHostDetails();
const endpointDescriptionLists: WebElementWrapper[] = await this.testSubjects.findAll(
'endpoint-overview'
);
const data: Record<string, string> = {};
for (const dlElement of endpointDescriptionLists) {
const $ = await dlElement.parseDomContent();
const title = $('dt')
.text()
.replace(/&nbsp;/g, '')
.trim();
// The value could be draggable, in which case we need to grab the value displayed from a deeper element
const $ddElement = $('dd');
const $valueContainer = $ddElement.find('.draggable-keyboard-wrapper .euiToolTipAnchor');
const value = ($valueContainer.length > 0 ? $valueContainer : $ddElement)
.text()
.replace(/&nbsp;/g, '')
.trim();
data[title] = value;
}
return data;
}
}