maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

EMT-179: implement metadata query versioning based on ingest manager installed ES assets (#77252)

Browse source 
* EMT-179: initial refactor for versioning

* EMT-179: move things before pulling from master

* EMT-179: fix build

* EMT-179: clean up

* EMT-179: add ingest hook, and improve all tests

* EMT-179: fix build

* EMT-179: clean up

* EMT-179: fix build

* EMT-179: fix build

* EMT-179: clean up

* EMT-179: more clean up

* EMT-179: clean up

* EMT-179: fix build

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
nnamdifrankie 2020-09-16 21:27:06 -04:00 committed by GitHub
parent 3e3f9c7ee9
commit 8bfdefe4e0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 2232 additions and 662 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/ingest_manager/server/index.ts
View file

@ -11,7 +11,7 @@ import {
AGENT_POLICY_ROLLOUT_RATE_LIMIT_REQUEST_PER_INTERVAL,
AGENT_POLLING_REQUEST_TIMEOUT_MS,
} from '../common';
export { AgentService, ESIndexPatternService, getRegistryUrl } from './services';
export { AgentService, ESIndexPatternService, getRegistryUrl, PackageService } from './services';
export {
IngestManagerSetupContract,
IngestManagerSetupDeps,

15
x-pack/plugins/ingest_manager/server/plugin.ts
View file

@ -13,6 +13,7 @@ import {
PluginInitializerContext,
SavedObjectsServiceStart,
HttpServiceSetup,
SavedObjectsClientContract,
} from 'kibana/server';
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { LicensingPluginSetup, ILicense } from '../../licensing/server';
@ -47,7 +48,7 @@ import {
registerSettingsRoutes,
registerAppRoutes,
} from './routes';
import { IngestManagerConfigType, NewPackagePolicy } from '../common';
import { EsAssetReference, IngestManagerConfigType, NewPackagePolicy } from '../common';
import {
appContextService,
licenseService,
@ -55,6 +56,7 @@ import {
ESIndexPatternService,
AgentService,
packagePolicyService,
PackageService,
} from './services';
import {
getAgentStatusById,
@ -65,6 +67,7 @@ import {
import { CloudSetup } from '../../cloud/server';
import { agentCheckinState } from './services/agents/checkin/state';
import { registerIngestManagerUsageCollector } from './collectors/register';
import { getInstallation } from './services/epm/packages';
export interface IngestManagerSetupDeps {
licensing: LicensingPluginSetup;
@ -118,6 +121,7 @@ export type ExternalCallbacksStorage = Map<ExternalCallback[0], Set<ExternalCall
*/
export interface IngestManagerStartContract {
esIndexPatternService: ESIndexPatternService;
packageService: PackageService;
agentService: AgentService;
/**
* Services for Ingest's package policies
@ -273,6 +277,15 @@ export class IngestManagerPlugin
return {
esIndexPatternService: new ESIndexPatternSavedObjectService(),
packageService: {
getInstalledEsAssetReferences: async (
savedObjectsClient: SavedObjectsClientContract,
pkgName: string
): Promise<EsAssetReference[]> => {
const installation = await getInstallation({ savedObjectsClient, pkgName });
return installation?.installed_es || [];
},
},
agentService: {
getAgent,
listAgents,

13
x-pack/plugins/ingest_manager/server/services/index.ts
View file

@ -5,7 +5,7 @@
*/
import { SavedObjectsClientContract, KibanaRequest } from 'kibana/server';
import { AgentStatus, Agent } from '../types';
import { AgentStatus, Agent, EsAssetReference } from '../types';
import * as settingsService from './settings';
export { ESIndexPatternSavedObjectService } from './es_index_pattern';
@ -22,6 +22,17 @@ export interface ESIndexPatternService {
): Promise<string | undefined>;
}
/**
* Service that provides exported function that return information about EPM packages
*/
export interface PackageService {
getInstalledEsAssetReferences(
savedObjectsClient: SavedObjectsClientContract,
pkgName: string
): Promise<EsAssetReference[]>;
}
/**
* A service that provides exported functions that return information about an Agent
*/

1
x-pack/plugins/security_solution/common/endpoint/constants.ts
View file

@ -8,6 +8,7 @@ export const eventsIndexPattern = 'logs-endpoint.events.*';
export const alertsIndexPattern = 'logs-endpoint.alerts-*';
export const metadataIndexPattern = 'metrics-endpoint.metadata-*';
export const metadataCurrentIndexPattern = 'metrics-endpoint.metadata_current-*';
export const metadataTransformPrefix = 'metrics-endpoint.metadata-current-default';
export const policyIndexPattern = 'metrics-endpoint.policy-*';
export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';
export const LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG = 'endpoint:limited-concurrency';

9
x-pack/plugins/security_solution/common/endpoint/types/index.ts
View file

@ -297,6 +297,8 @@ export interface HostResultList {
request_page_size: number;
/* the page index requested */
request_page_index: number;
/* the version of the query strategy */
query_strategy_version: MetadataQueryStrategyVersions;
}
/**
@ -504,9 +506,16 @@ export enum HostStatus {
UNENROLLING = 'unenrolling',
}
export enum MetadataQueryStrategyVersions {
VERSION_1 = 'v1',
VERSION_2 = 'v2',
}
export type HostInfo = Immutable<{
metadata: HostMetadata;
host_status: HostStatus;
/* the version of the query strategy */
query_strategy_version: MetadataQueryStrategyVersions;
}>;
export type HostMetadataDetails = Immutable<{

5
x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/mock_endpoint_result_list.ts
View file

@ -11,6 +11,7 @@ import {
HostPolicyResponse,
HostResultList,
HostStatus,
MetadataQueryStrategyVersions,
} from '../../../../../common/endpoint/types';
import { EndpointDocGenerator } from '../../../../../common/endpoint/generate_data';
import {
@ -49,6 +50,7 @@ export const mockEndpointResultList: (options?: {
hosts.push({
metadata: generator.generateHostMetadata(),
host_status: HostStatus.ERROR,
query_strategy_version: MetadataQueryStrategyVersions.VERSION_2,
});
}
const mock: HostResultList = {
@ -56,6 +58,7 @@ export const mockEndpointResultList: (options?: {
total,
request_page_size: requestPageSize,
request_page_index: requestPageIndex,
query_strategy_version: MetadataQueryStrategyVersions.VERSION_2,
};
return mock;
};
@ -67,6 +70,7 @@ export const mockEndpointDetailsApiResult = (): HostInfo => {
return {
metadata: generator.generateHostMetadata(),
host_status: HostStatus.ERROR,
query_strategy_version: MetadataQueryStrategyVersions.VERSION_2,
};
};
@ -103,6 +107,7 @@ const endpointListApiPathHandlerMocks = ({
request_page_size: 10,
request_page_index: 0,
total: endpointsResults?.length || 0,
query_strategy_version: MetadataQueryStrategyVersions.VERSION_2,
};
},

6
x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx
View file

@ -20,6 +20,7 @@ import {
HostPolicyResponseActionStatus,
HostPolicyResponseAppliedAction,
HostStatus,
MetadataQueryStrategyVersions,
} from '../../../../../common/endpoint/types';
import { EndpointDocGenerator } from '../../../../../common/endpoint/generate_data';
import { POLICY_STATUS_TO_HEALTH_COLOR, POLICY_STATUS_TO_TEXT } from './host_constants';
@ -131,6 +132,7 @@ describe('when on the list page', () => {
hostListData[index] = {
metadata: hostListData[index].metadata,
host_status: status,
query_strategy_version: MetadataQueryStrategyVersions.VERSION_2,
};
}
);
@ -301,6 +303,8 @@ describe('when on the list page', () => {
// eslint-disable-next-line @typescript-eslint/naming-convention
host_status,
metadata: { host, ...details },
// eslint-disable-next-line @typescript-eslint/naming-convention
query_strategy_version,
} = mockEndpointDetailsApiResult();
hostDetails = {
@ -312,6 +316,7 @@ describe('when on the list page', () => {
id: '1',
},
},
query_strategy_version,
};
agentId = hostDetails.metadata.elastic.agent.id;
@ -681,6 +686,7 @@ describe('when on the list page', () => {
hostInfo = {
host_status: hosts[0].host_status,
metadata: hosts[0].metadata,
query_strategy_version: MetadataQueryStrategyVersions.VERSION_2,
};
const packagePolicy = docGenerator.generatePolicyPackagePolicy();
packagePolicy.id = hosts[0].metadata.Endpoint.policy.applied.id;

62
x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts
View file

@ -9,12 +9,64 @@ import {
SavedObjectsServiceStart,
SavedObjectsClientContract,
} from 'src/core/server';
import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server';
import {
AgentService,
IngestManagerStartContract,
PackageService,
} from '../../../ingest_manager/server';
import { getPackagePolicyCreateCallback } from './ingest_integration';
import { ManifestManager } from './services/artifacts';
import { MetadataQueryStrategy } from './types';
import { MetadataQueryStrategyVersions } from '../../common/endpoint/types';
import {
metadataQueryStrategyV1,
metadataQueryStrategyV2,
} from './routes/metadata/support/query_strategies';
import { ElasticsearchAssetType } from '../../../ingest_manager/common/types/models';
import { metadataTransformPrefix } from '../../common/endpoint/constants';
export interface MetadataService {
queryStrategy(
savedObjectsClient: SavedObjectsClientContract,
version?: MetadataQueryStrategyVersions
): Promise<MetadataQueryStrategy>;
}
export const createMetadataService = (packageService: PackageService): MetadataService => {
return {
async queryStrategy(
savedObjectsClient: SavedObjectsClientContract,
version?: MetadataQueryStrategyVersions
): Promise<MetadataQueryStrategy> {
if (version === MetadataQueryStrategyVersions.VERSION_1) {
return metadataQueryStrategyV1();
}
if (!packageService) {
throw new Error('package service is uninitialized');
}
if (version === MetadataQueryStrategyVersions.VERSION_2 || !version) {
const assets = await packageService.getInstalledEsAssetReferences(
savedObjectsClient,
'endpoint'
);
const expectedTransformAssets = assets.filter(
(ref) =>
ref.type === ElasticsearchAssetType.transform &&
ref.id.startsWith(metadataTransformPrefix)
);
if (expectedTransformAssets && expectedTransformAssets.length === 1) {
return metadataQueryStrategyV2();
}
return metadataQueryStrategyV1();
}
return metadataQueryStrategyV1();
},
};
};
export type EndpointAppContextServiceStartContract = Partial<
Pick<IngestManagerStartContract, 'agentService'>
Pick<IngestManagerStartContract, 'agentService' | 'packageService'>
> & {
logger: Logger;
manifestManager?: ManifestManager;
@ -30,11 +82,13 @@ export class EndpointAppContextService {
private agentService: AgentService | undefined;
private manifestManager: ManifestManager | undefined;
private savedObjectsStart: SavedObjectsServiceStart | undefined;
private metadataService: MetadataService | undefined;
public start(dependencies: EndpointAppContextServiceStartContract) {
this.agentService = dependencies.agentService;
this.manifestManager = dependencies.manifestManager;
this.savedObjectsStart = dependencies.savedObjectsStart;
this.metadataService = createMetadataService(dependencies.packageService!);
if (this.manifestManager && dependencies.registerIngestCallback) {
dependencies.registerIngestCallback(
@ -50,6 +104,10 @@ export class EndpointAppContextService {
return this.agentService;
}
public getMetadataService(): MetadataService | undefined {
return this.metadataService;
}
public getManifestManager(): ManifestManager | undefined {
return this.manifestManager;
}

13
x-pack/plugins/security_solution/server/endpoint/mocks.ts
View file

@ -11,6 +11,7 @@ import {
AgentService,
IngestManagerStartContract,
ExternalCallback,
PackageService,
} from '../../../ingest_manager/server';
import { createPackagePolicyServiceMock } from '../../../ingest_manager/server/mocks';
import { createMockConfig } from '../lib/detection_engine/routes/__mocks__';
@ -58,6 +59,7 @@ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked<
> => {
return {
agentService: createMockAgentService(),
packageService: createMockPackageService(),
logger: loggingSystemMock.create().get('mock_endpoint_app_context'),
savedObjectsStart: savedObjectsServiceMock.createStartContract(),
manifestManager: getManifestManagerMock(),
@ -68,6 +70,16 @@ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked<
};
};
/**
* Create mock PackageService
*/
export const createMockPackageService = (): jest.Mocked<PackageService> => {
return {
getInstalledEsAssetReferences: jest.fn(),
};
};
/**
* Creates a mock AgentService
*/
@ -95,6 +107,7 @@ export const createMockIngestManagerStartContract = (
getESIndexPattern: jest.fn().mockResolvedValue(indexPattern),
},
agentService: createMockAgentService(),
packageService: createMockPackageService(),
registerExternalCallback: jest.fn((...args: ExternalCallback) => {}),
packagePolicyService: createPackagePolicyServiceMock(),
};

290
x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts Normal file
View file

@ -0,0 +1,290 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import Boom from 'boom';
import { RequestHandlerContext, Logger, RequestHandler } from 'kibana/server';
import { TypeOf } from '@kbn/config-schema';
import {
HostInfo,
HostMetadata,
HostResultList,
HostStatus,
MetadataQueryStrategyVersions,
} from '../../../../common/endpoint/types';
import { getESQueryHostMetadataByID, kibanaRequestToMetadataListESQuery } from './query_builders';
import { Agent, AgentStatus } from '../../../../../ingest_manager/common/types/models';
import { EndpointAppContext, HostListQueryResult } from '../../types';
import { GetMetadataListRequestSchema, GetMetadataRequestSchema } from './index';
import { findAllUnenrolledAgentIds } from './support/unenroll';
import { findAgentIDsByStatus } from './support/agent_status';
import { EndpointAppContextService } from '../../endpoint_app_context_services';
export interface MetadataRequestContext {
endpointAppContextService: EndpointAppContextService;
logger: Logger;
requestHandlerContext: RequestHandlerContext;
}
const HOST_STATUS_MAPPING = new Map<AgentStatus, HostStatus>([
['online', HostStatus.ONLINE],
['offline', HostStatus.OFFLINE],
['unenrolling', HostStatus.UNENROLLING],
]);
/**
* 00000000-0000-0000-0000-000000000000 is initial Elastic Agent id sent by Endpoint before policy is configured
* 11111111-1111-1111-1111-111111111111 is Elastic Agent id sent by Endpoint when policy does not contain an id
*/
const IGNORED_ELASTIC_AGENT_IDS = [
'00000000-0000-0000-0000-000000000000',
'11111111-1111-1111-1111-111111111111',
];
export const getLogger = (endpointAppContext: EndpointAppContext): Logger => {
return endpointAppContext.logFactory.get('metadata');
};
export const getMetadataListRequestHandler = function (
endpointAppContext: EndpointAppContext,
logger: Logger,
queryStrategyVersion?: MetadataQueryStrategyVersions
): RequestHandler<undefined, undefined, TypeOf<typeof GetMetadataListRequestSchema.body>> {
return async (context, request, response) => {
try {
const agentService = endpointAppContext.service.getAgentService();
if (agentService === undefined) {
throw new Error('agentService not available');
}
const metadataRequestContext: MetadataRequestContext = {
endpointAppContextService: endpointAppContext.service,
logger,
requestHandlerContext: context,
};
const unenrolledAgentIds = await findAllUnenrolledAgentIds(
agentService,
context.core.savedObjects.client
);
const statusIDs = request?.body?.filters?.host_status?.length
? await findAgentIDsByStatus(
agentService,
context.core.savedObjects.client,
request.body?.filters?.host_status
)
: undefined;
const queryStrategy = await endpointAppContext.service
?.getMetadataService()
?.queryStrategy(context.core.savedObjects.client, queryStrategyVersion);
const queryParams = await kibanaRequestToMetadataListESQuery(
request,
endpointAppContext,
queryStrategy!,
{
unenrolledAgentIds: unenrolledAgentIds.concat(IGNORED_ELASTIC_AGENT_IDS),
statusAgentIDs: statusIDs,
}
);
const hostListQueryResult = queryStrategy!.queryResponseToHostListResult(
await context.core.elasticsearch.legacy.client.callAsCurrentUser('search', queryParams)
);
return response.ok({
body: await mapToHostResultList(queryParams, hostListQueryResult, metadataRequestContext),
});
} catch (err) {
logger.warn(JSON.stringify(err, null, 2));
return response.internalError({ body: err });
}
};
};
export const getMetadataRequestHandler = function (
endpointAppContext: EndpointAppContext,
logger: Logger,
queryStrategyVersion?: MetadataQueryStrategyVersions
): RequestHandler<TypeOf<typeof GetMetadataRequestSchema.params>, undefined, undefined> {
return async (context, request, response) => {
const agentService = endpointAppContext.service.getAgentService();
if (agentService === undefined) {
return response.internalError({ body: 'agentService not available' });
}
const metadataRequestContext: MetadataRequestContext = {
endpointAppContextService: endpointAppContext.service,
logger,
requestHandlerContext: context,
};
try {
const doc = await getHostData(
metadataRequestContext,
request?.params?.id,
queryStrategyVersion
);
if (doc) {
return response.ok({ body: doc });
}
return response.notFound({ body: 'Endpoint Not Found' });
} catch (err) {
logger.warn(JSON.stringify(err, null, 2));
if (err.isBoom) {
return response.customError({
statusCode: err.output.statusCode,
body: { message: err.message },
});
}
return response.internalError({ body: err });
}
};
};
export async function getHostData(
metadataRequestContext: MetadataRequestContext,
id: string,
queryStrategyVersion?: MetadataQueryStrategyVersions
): Promise<HostInfo | undefined> {
const queryStrategy = await metadataRequestContext.endpointAppContextService
?.getMetadataService()
?.queryStrategy(
metadataRequestContext.requestHandlerContext.core.savedObjects.client,
queryStrategyVersion
);
const query = getESQueryHostMetadataByID(id, queryStrategy!);
const hostResult = queryStrategy!.queryResponseToHostResult(
await metadataRequestContext.requestHandlerContext.core.elasticsearch.legacy.client.callAsCurrentUser(
'search',
query
)
);
const hostMetadata = hostResult.result;
if (!hostMetadata) {
return undefined;
}
const agent = await findAgent(metadataRequestContext, hostMetadata);
if (agent && !agent.active) {
throw Boom.badRequest('the requested endpoint is unenrolled');
}
const metadata = await enrichHostMetadata(
hostMetadata,
metadataRequestContext,
hostResult.queryStrategyVersion
);
return { ...metadata, query_strategy_version: hostResult.queryStrategyVersion };
}
async function findAgent(
metadataRequestContext: MetadataRequestContext,
hostMetadata: HostMetadata
): Promise<Agent | undefined> {
try {
return await metadataRequestContext.endpointAppContextService
?.getAgentService()
?.getAgent(
metadataRequestContext.requestHandlerContext.core.savedObjects.client,
hostMetadata.elastic.agent.id
);
} catch (e) {
if (
metadataRequestContext.requestHandlerContext.core.savedObjects.client.errors.isNotFoundError(
e
)
) {
metadataRequestContext.logger.warn(
`agent with id ${hostMetadata.elastic.agent.id} not found`
);
return undefined;
} else {
throw e;
}
}
}
export async function mapToHostResultList(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
queryParams: Record<string, any>,
hostListQueryResult: HostListQueryResult,
metadataRequestContext: MetadataRequestContext
): Promise<HostResultList> {
const totalNumberOfHosts = hostListQueryResult.resultLength;
if (hostListQueryResult.resultList.length > 0) {
return {
request_page_size: queryParams.size,
request_page_index: queryParams.from,
hosts: await Promise.all(
hostListQueryResult.resultList.map(async (entry) =>
enrichHostMetadata(
entry,
metadataRequestContext,
hostListQueryResult.queryStrategyVersion
)
)
),
total: totalNumberOfHosts,
query_strategy_version: hostListQueryResult.queryStrategyVersion,
};
} else {
return {
request_page_size: queryParams.size,
request_page_index: queryParams.from,
total: totalNumberOfHosts,
hosts: [],
query_strategy_version: hostListQueryResult.queryStrategyVersion,
};
}
}
async function enrichHostMetadata(
hostMetadata: HostMetadata,
metadataRequestContext: MetadataRequestContext,
metadataQueryStrategyVersion: MetadataQueryStrategyVersions
): Promise<HostInfo> {
let hostStatus = HostStatus.ERROR;
let elasticAgentId = hostMetadata?.elastic?.agent?.id;
const log = metadataRequestContext.logger;
try {
/**
* Get agent status by elastic agent id if available or use the host id.
*/
if (!elasticAgentId) {
elasticAgentId = hostMetadata.host.id;
log.warn(`Missing elastic agent id, using host id instead ${elasticAgentId}`);
}
const status = await metadataRequestContext.endpointAppContextService
?.getAgentService()
?.getAgentStatusById(
metadataRequestContext.requestHandlerContext.core.savedObjects.client,
elasticAgentId
);
hostStatus = HOST_STATUS_MAPPING.get(status!) || HostStatus.ERROR;
} catch (e) {
if (
metadataRequestContext.requestHandlerContext.core.savedObjects.client.errors.isNotFoundError(
e
)
) {
log.warn(`agent with id ${elasticAgentId} not found`);
} else {
log.error(e);
throw e;
}
}
return {
metadata: hostMetadata,
host_status: hostStatus,
query_strategy_version: metadataQueryStrategyVersion,
};
}

325
x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts
View file

@ -4,51 +4,18 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { IRouter, Logger, RequestHandlerContext } from 'kibana/server';
import { SearchResponse } from 'elasticsearch';
import { IRouter } from 'kibana/server';
import { schema } from '@kbn/config-schema';
import Boom from 'boom';
import { metadataCurrentIndexPattern } from '../../../../common/endpoint/constants';
import { getESQueryHostMetadataByID, kibanaRequestToMetadataListESQuery } from './query_builders';
import {
HostInfo,
HostMetadata,
HostMetadataDetails,
HostResultList,
HostStatus,
} from '../../../../common/endpoint/types';
import { HostStatus, MetadataQueryStrategyVersions } from '../../../../common/endpoint/types';
import { EndpointAppContext } from '../../types';
import { AgentService } from '../../../../../ingest_manager/server';
import { Agent, AgentStatus } from '../../../../../ingest_manager/common/types/models';
import { findAllUnenrolledAgentIds } from './support/unenroll';
import { findAgentIDsByStatus } from './support/agent_status';
import { getLogger, getMetadataListRequestHandler, getMetadataRequestHandler } from './handlers';
interface MetadataRequestContext {
agentService: AgentService;
logger: Logger;
requestHandlerContext: RequestHandlerContext;
}
const HOST_STATUS_MAPPING = new Map<AgentStatus, HostStatus>([
['online', HostStatus.ONLINE],
['offline', HostStatus.OFFLINE],
['unenrolling', HostStatus.UNENROLLING],
]);
/**
* 00000000-0000-0000-0000-000000000000 is initial Elastic Agent id sent by Endpoint before policy is configured
* 11111111-1111-1111-1111-111111111111 is Elastic Agent id sent by Endpoint when policy does not contain an id
*/
const IGNORED_ELASTIC_AGENT_IDS = [
'00000000-0000-0000-0000-000000000000',
'11111111-1111-1111-1111-111111111111',
];
const getLogger = (endpointAppContext: EndpointAppContext): Logger => {
return endpointAppContext.logFactory.get('metadata');
};
export const BASE_ENDPOINT_ROUTE = '/api/endpoint';
export const METADATA_REQUEST_V1_ROUTE = `${BASE_ENDPOINT_ROUTE}/v1/metadata`;
export const GET_METADATA_REQUEST_V1_ROUTE = `${METADATA_REQUEST_V1_ROUTE}/{id}`;
export const METADATA_REQUEST_ROUTE = `${BASE_ENDPOINT_ROUTE}/metadata`;
export const GET_METADATA_REQUEST_ROUTE = `${METADATA_REQUEST_ROUTE}/{id}`;
/* Filters that can be applied to the endpoint fetch route */
export const endpointFilters = schema.object({
@ -65,241 +32,73 @@ export const endpointFilters = schema.object({
),
});
export const GetMetadataRequestSchema = {
params: schema.object({ id: schema.string() }),
};
export const GetMetadataListRequestSchema = {
body: schema.nullable(
schema.object({
paging_properties: schema.nullable(
schema.arrayOf(
schema.oneOf([
/**
* the number of results to return for this request per page
*/
schema.object({
page_size: schema.number({ defaultValue: 10, min: 1, max: 10000 }),
}),
/**
* the zero based page index of the the total number of pages of page size
*/
schema.object({ page_index: schema.number({ defaultValue: 0, min: 0 }) }),
])
)
),
filters: endpointFilters,
})
),
};
export function registerEndpointRoutes(router: IRouter, endpointAppContext: EndpointAppContext) {
const logger = getLogger(endpointAppContext);
router.post(
{
path: '/api/endpoint/metadata',
validate: {
body: schema.nullable(
schema.object({
paging_properties: schema.nullable(
schema.arrayOf(
schema.oneOf([
/**
* the number of results to return for this request per page
*/
schema.object({
page_size: schema.number({ defaultValue: 10, min: 1, max: 10000 }),
}),
/**
* the zero based page index of the the total number of pages of page size
*/
schema.object({ page_index: schema.number({ defaultValue: 0, min: 0 }) }),
])
)
),
filters: endpointFilters,
})
),
},
path: `${METADATA_REQUEST_V1_ROUTE}`,
validate: GetMetadataListRequestSchema,
options: { authRequired: true, tags: ['access:securitySolution'] },
},
async (context, req, res) => {
try {
const agentService = endpointAppContext.service.getAgentService();
if (agentService === undefined) {
throw new Error('agentService not available');
}
getMetadataListRequestHandler(
endpointAppContext,
logger,
MetadataQueryStrategyVersions.VERSION_1
)
);
const metadataRequestContext: MetadataRequestContext = {
agentService,
logger,
requestHandlerContext: context,
};
const unenrolledAgentIds = await findAllUnenrolledAgentIds(
agentService,
context.core.savedObjects.client
);
const statusIDs = req.body?.filters?.host_status?.length
? await findAgentIDsByStatus(
agentService,
context.core.savedObjects.client,
req.body?.filters?.host_status
)
: undefined;
const queryParams = await kibanaRequestToMetadataListESQuery(
req,
endpointAppContext,
metadataCurrentIndexPattern,
{
unenrolledAgentIds: unenrolledAgentIds.concat(IGNORED_ELASTIC_AGENT_IDS),
statusAgentIDs: statusIDs,
}
);
const response = (await context.core.elasticsearch.legacy.client.callAsCurrentUser(
'search',
queryParams
)) as SearchResponse<HostMetadataDetails>;
return res.ok({
body: await mapToHostResultList(queryParams, response, metadataRequestContext),
});
} catch (err) {
logger.warn(JSON.stringify(err, null, 2));
return res.internalError({ body: err });
}
}
router.post(
{
path: `${METADATA_REQUEST_ROUTE}`,
validate: GetMetadataListRequestSchema,
options: { authRequired: true, tags: ['access:securitySolution'] },
},
getMetadataListRequestHandler(endpointAppContext, logger)
);
router.get(
{
path: '/api/endpoint/metadata/{id}',
validate: {
params: schema.object({ id: schema.string() }),
},
path: `${GET_METADATA_REQUEST_V1_ROUTE}`,
validate: GetMetadataRequestSchema,
options: { authRequired: true, tags: ['access:securitySolution'] },
},
async (context, req, res) => {
const agentService = endpointAppContext.service.getAgentService();
if (agentService === undefined) {
return res.internalError({ body: 'agentService not available' });
}
getMetadataRequestHandler(endpointAppContext, logger, MetadataQueryStrategyVersions.VERSION_1)
);
const metadataRequestContext: MetadataRequestContext = {
agentService,
logger,
requestHandlerContext: context,
};
try {
const doc = await getHostData(metadataRequestContext, req.params.id);
if (doc) {
return res.ok({ body: doc });
}
return res.notFound({ body: 'Endpoint Not Found' });
} catch (err) {
logger.warn(JSON.stringify(err, null, 2));
if (err.isBoom) {
return res.customError({
statusCode: err.output.statusCode,
body: { message: err.message },
});
}
return res.internalError({ body: err });
}
}
router.get(
{
path: `${GET_METADATA_REQUEST_ROUTE}`,
validate: GetMetadataRequestSchema,
options: { authRequired: true, tags: ['access:securitySolution'] },
},
getMetadataRequestHandler(endpointAppContext, logger)
);
}
export async function getHostData(
metadataRequestContext: MetadataRequestContext,
id: string
): Promise<HostInfo | undefined> {
const query = getESQueryHostMetadataByID(id, metadataCurrentIndexPattern);
const response = (await metadataRequestContext.requestHandlerContext.core.elasticsearch.legacy.client.callAsCurrentUser(
'search',
query
)) as SearchResponse<HostMetadataDetails>;
if (response.hits.hits.length === 0) {
return undefined;
}
const hostMetadata: HostMetadata = response.hits.hits[0]._source.HostDetails;
const agent = await findAgent(metadataRequestContext, hostMetadata);
if (agent && !agent.active) {
throw Boom.badRequest('the requested endpoint is unenrolled');
}
return enrichHostMetadata(hostMetadata, metadataRequestContext);
}
async function findAgent(
metadataRequestContext: MetadataRequestContext,
hostMetadata: HostMetadata
): Promise<Agent | undefined> {
try {
return await metadataRequestContext.agentService.getAgent(
metadataRequestContext.requestHandlerContext.core.savedObjects.client,
hostMetadata.elastic.agent.id
);
} catch (e) {
if (
metadataRequestContext.requestHandlerContext.core.savedObjects.client.errors.isNotFoundError(
e
)
) {
metadataRequestContext.logger.warn(
`agent with id ${hostMetadata.elastic.agent.id} not found`
);
return undefined;
} else {
throw e;
}
}
}
async function mapToHostResultList(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
queryParams: Record<string, any>,
searchResponse: SearchResponse<HostMetadataDetails>,
metadataRequestContext: MetadataRequestContext
): Promise<HostResultList> {
const totalNumberOfHosts =
((searchResponse.hits?.total as unknown) as { value: number; relation: string }).value || 0;
if (searchResponse.hits.hits.length > 0) {
return {
request_page_size: queryParams.size,
request_page_index: queryParams.from,
hosts: await Promise.all(
searchResponse.hits.hits.map(async (entry) =>
enrichHostMetadata(entry._source.HostDetails, metadataRequestContext)
)
),
total: totalNumberOfHosts,
};
} else {
return {
request_page_size: queryParams.size,
request_page_index: queryParams.from,
total: totalNumberOfHosts,
hosts: [],
};
}
}
async function enrichHostMetadata(
hostMetadata: HostMetadata,
metadataRequestContext: MetadataRequestContext
): Promise<HostInfo> {
let hostStatus = HostStatus.ERROR;
let elasticAgentId = hostMetadata?.elastic?.agent?.id;
const log = metadataRequestContext.logger;
try {
/**
* Get agent status by elastic agent id if available or use the host id.
*/
if (!elasticAgentId) {
elasticAgentId = hostMetadata.host.id;
log.warn(`Missing elastic agent id, using host id instead ${elasticAgentId}`);
}
const status = await metadataRequestContext.agentService.getAgentStatusById(
metadataRequestContext.requestHandlerContext.core.savedObjects.client,
elasticAgentId
);
hostStatus = HOST_STATUS_MAPPING.get(status) || HostStatus.ERROR;
} catch (e) {
if (
metadataRequestContext.requestHandlerContext.core.savedObjects.client.errors.isNotFoundError(
e
)
) {
log.warn(`agent with id ${elasticAgentId} not found`);
} else {
log.error(e);
throw e;
}
}
return {
metadata: hostMetadata,
host_status: hostStatus,
};
}

732
x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts
View file

@ -22,21 +22,26 @@ import {
} from '../../../../../../../src/core/server/mocks';
import {
HostInfo,
HostMetadata,
HostMetadataDetails,
HostResultList,
HostStatus,
MetadataQueryStrategyVersions,
} from '../../../../common/endpoint/types';
import { SearchResponse } from 'elasticsearch';
import { registerEndpointRoutes, endpointFilters } from './index';
import { registerEndpointRoutes, METADATA_REQUEST_ROUTE } from './index';
import {
createMockEndpointAppContextServiceStartContract,
createMockPackageService,
createRouteHandlerContext,
} from '../../mocks';
import { EndpointAppContextService } from '../../endpoint_app_context_services';
import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__';
import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data';
import { Agent } from '../../../../../ingest_manager/common/types/models';
import {
Agent,
ElasticsearchAssetType,
EsAssetReference,
} from '../../../../../ingest_manager/common/types/models';
import { createV1SearchResponse, createV2SearchResponse } from './support/test_support';
import { PackageService } from '../../../../../ingest_manager/server/services';
describe('test endpoint route', () => {
let routerMock: jest.Mocked<IRouter>;
@ -44,6 +49,7 @@ describe('test endpoint route', () => {
let mockClusterClient: jest.Mocked<ILegacyClusterClient>;
let mockScopedClient: jest.Mocked<ILegacyScopedClusterClient>;
let mockSavedObjectClient: jest.Mocked<SavedObjectsClientContract>;
let mockPackageService: jest.Mocked<PackageService>;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let routeHandler: RequestHandler<any, any, any>;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@ -61,195 +67,45 @@ describe('test endpoint route', () => {
};
beforeEach(() => {
mockScopedClient = elasticsearchServiceMock.createLegacyScopedClusterClient();
mockClusterClient = elasticsearchServiceMock.createLegacyClusterClient() as jest.Mocked<
ILegacyClusterClient
>;
mockScopedClient = elasticsearchServiceMock.createLegacyScopedClusterClient();
mockSavedObjectClient = savedObjectsClientMock.create();
mockClusterClient.asScoped.mockReturnValue(mockScopedClient);
routerMock = httpServiceMock.createRouter();
mockResponse = httpServerMock.createResponseFactory();
endpointAppContextService = new EndpointAppContextService();
const startContract = createMockEndpointAppContextServiceStartContract();
endpointAppContextService.start(startContract);
mockAgentService = startContract.agentService!;
registerEndpointRoutes(routerMock, {
logFactory: loggingSystemMock.create(),
service: endpointAppContextService,
config: () => Promise.resolve(createMockConfig()),
});
});
afterEach(() => endpointAppContextService.stop());
it('test find the latest of all endpoints', async () => {
const mockRequest = httpServerMock.createKibanaRequest({});
const response = createSearchResponse(new EndpointDocGenerator().generateHostMetadata());
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith('/api/endpoint/metadata')
)!;
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'] });
expect(mockResponse.ok).toBeCalled();
const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList;
expect(endpointResultList.hosts.length).toEqual(1);
expect(endpointResultList.total).toEqual(1);
expect(endpointResultList.request_page_index).toEqual(0);
expect(endpointResultList.request_page_size).toEqual(10);
});
it('test find the latest of all endpoints with paging properties', async () => {
const mockRequest = httpServerMock.createKibanaRequest({
body: {
paging_properties: [
{
page_size: 10,
},
{
page_index: 1,
},
],
},
});
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() =>
Promise.resolve(createSearchResponse(new EndpointDocGenerator().generateHostMetadata()))
);
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith('/api/endpoint/metadata')
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query).toEqual({
bool: {
must_not: {
terms: {
'HostDetails.elastic.agent.id': [
'00000000-0000-0000-0000-000000000000',
'11111111-1111-1111-1111-111111111111',
],
},
},
},
});
expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'] });
expect(mockResponse.ok).toBeCalled();
const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList;
expect(endpointResultList.hosts.length).toEqual(1);
expect(endpointResultList.total).toEqual(1);
expect(endpointResultList.request_page_index).toEqual(10);
expect(endpointResultList.request_page_size).toEqual(10);
});
it('test find the latest of all endpoints with paging and filters properties', async () => {
const mockRequest = httpServerMock.createKibanaRequest({
body: {
paging_properties: [
{
page_size: 10,
},
{
page_index: 1,
},
],
filters: { kql: 'not host.ip:10.140.73.246' },
},
});
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() =>
Promise.resolve(createSearchResponse(new EndpointDocGenerator().generateHostMetadata()))
);
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith('/api/endpoint/metadata')
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toBeCalled();
expect(mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query).toEqual({
bool: {
must: [
{
bool: {
must_not: {
terms: {
'HostDetails.elastic.agent.id': [
'00000000-0000-0000-0000-000000000000',
'11111111-1111-1111-1111-111111111111',
],
},
},
},
},
{
bool: {
must_not: {
bool: {
should: [
{
match: {
'host.ip': '10.140.73.246',
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
});
expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'] });
expect(mockResponse.ok).toBeCalled();
const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList;
expect(endpointResultList.hosts.length).toEqual(1);
expect(endpointResultList.total).toEqual(1);
expect(endpointResultList.request_page_index).toEqual(10);
expect(endpointResultList.request_page_size).toEqual(10);
});
describe('Endpoint Details route', () => {
it('should return 404 on no results', async () => {
const mockRequest = httpServerMock.createKibanaRequest({ params: { id: 'BADID' } });
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() =>
Promise.resolve(createSearchResponse())
describe('with no transform package', () => {
beforeEach(() => {
endpointAppContextService = new EndpointAppContextService();
const startContract = createMockEndpointAppContextServiceStartContract();
mockPackageService = createMockPackageService();
mockPackageService.getInstalledEsAssetReferences.mockReturnValue(
Promise.resolve(([] as unknown) as EsAssetReference[])
);
endpointAppContextService.start({ ...startContract, packageService: mockPackageService });
mockAgentService = startContract.agentService!;
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.getAgent = jest.fn().mockReturnValue(({
active: true,
} as unknown) as Agent);
registerEndpointRoutes(routerMock, {
logFactory: loggingSystemMock.create(),
service: endpointAppContextService,
config: () => Promise.resolve(createMockConfig()),
});
});
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith('/api/endpoint/metadata')
afterEach(() => endpointAppContextService.stop());
it('test find the latest of all endpoints', async () => {
const mockRequest = httpServerMock.createKibanaRequest({});
const response = createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata());
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_ROUTE}`)
)!;
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
@ -261,13 +117,19 @@ describe('test endpoint route', () => {
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.notFound).toBeCalled();
const message = mockResponse.notFound.mock.calls[0][0]?.body;
expect(message).toEqual('Endpoint Not Found');
expect(mockResponse.ok).toBeCalled();
const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList;
expect(endpointResultList.hosts.length).toEqual(1);
expect(endpointResultList.total).toEqual(1);
expect(endpointResultList.request_page_index).toEqual(0);
expect(endpointResultList.request_page_size).toEqual(10);
expect(endpointResultList.query_strategy_version).toEqual(
MetadataQueryStrategyVersions.VERSION_1
);
});
it('should return a single endpoint with status online', async () => {
const response = createSearchResponse(new EndpointDocGenerator().generateHostMetadata());
const response = createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata());
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
});
@ -279,7 +141,7 @@ describe('test endpoint route', () => {
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith('/api/endpoint/metadata')
path.startsWith(`${METADATA_REQUEST_ROUTE}`)
)!;
await routeHandler(
@ -297,29 +159,48 @@ describe('test endpoint route', () => {
const result = mockResponse.ok.mock.calls[0][0]?.body as HostInfo;
expect(result).toHaveProperty('metadata.Endpoint');
expect(result.host_status).toEqual(HostStatus.ONLINE);
expect(result.query_strategy_version).toEqual(MetadataQueryStrategyVersions.VERSION_1);
});
});
describe('with new transform package', () => {
beforeEach(() => {
endpointAppContextService = new EndpointAppContextService();
const startContract = createMockEndpointAppContextServiceStartContract();
mockPackageService = createMockPackageService();
mockPackageService.getInstalledEsAssetReferences.mockReturnValue(
Promise.resolve([
{
id: 'logs-endpoint.events.security',
type: ElasticsearchAssetType.indexTemplate,
},
{
id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0',
type: ElasticsearchAssetType.transform,
},
])
);
endpointAppContextService.start({ ...startContract, packageService: mockPackageService });
mockAgentService = startContract.agentService!;
registerEndpointRoutes(routerMock, {
logFactory: loggingSystemMock.create(),
service: endpointAppContextService,
config: () => Promise.resolve(createMockConfig()),
});
});
it('should return a single endpoint with status error when AgentService throw 404', async () => {
const response = createSearchResponse(new EndpointDocGenerator().generateHostMetadata());
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
});
mockAgentService.getAgentStatusById = jest.fn().mockImplementation(() => {
SavedObjectsErrorHelpers.createGenericNotFoundError();
});
mockAgentService.getAgent = jest.fn().mockImplementation(() => {
SavedObjectsErrorHelpers.createGenericNotFoundError();
});
afterEach(() => endpointAppContextService.stop());
it('test find the latest of all endpoints', async () => {
const mockRequest = httpServerMock.createKibanaRequest({});
const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata());
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith('/api/endpoint/metadata')
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_ROUTE}`)
)!;
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
@ -332,151 +213,314 @@ describe('test endpoint route', () => {
tags: ['access:securitySolution'],
});
expect(mockResponse.ok).toBeCalled();
const result = mockResponse.ok.mock.calls[0][0]?.body as HostInfo;
expect(result.host_status).toEqual(HostStatus.ERROR);
});
it('should return a single endpoint with status error when status is not offline, online or enrolling', async () => {
const response = createSearchResponse(new EndpointDocGenerator().generateHostMetadata());
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
});
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('warning');
mockAgentService.getAgent = jest.fn().mockReturnValue(({
active: true,
} as unknown) as Agent);
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith('/api/endpoint/metadata')
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList;
expect(endpointResultList.hosts.length).toEqual(1);
expect(endpointResultList.total).toEqual(1);
expect(endpointResultList.request_page_index).toEqual(0);
expect(endpointResultList.request_page_size).toEqual(10);
expect(endpointResultList.query_strategy_version).toEqual(
MetadataQueryStrategyVersions.VERSION_2
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.ok).toBeCalled();
const result = mockResponse.ok.mock.calls[0][0]?.body as HostInfo;
expect(result.host_status).toEqual(HostStatus.ERROR);
});
it('should throw error when endpoint agent is not active', async () => {
const response = createSearchResponse(new EndpointDocGenerator().generateHostMetadata());
it('test find the latest of all endpoints with paging properties', async () => {
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
});
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
mockAgentService.getAgent = jest.fn().mockReturnValue(({
active: false,
} as unknown) as Agent);
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith('/api/endpoint/metadata')
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(mockResponse.customError).toBeCalled();
});
});
});
describe('Filters Schema Test', () => {
it('accepts a single host status', () => {
expect(
endpointFilters.validate({
host_status: ['error'],
})
).toBeTruthy();
});
it('accepts multiple host status filters', () => {
expect(
endpointFilters.validate({
host_status: ['offline', 'unenrolling'],
})
).toBeTruthy();
});
it('rejects invalid statuses', () => {
expect(() =>
endpointFilters.validate({
host_status: ['foobar'],
})
).toThrowError();
});
it('accepts a KQL string', () => {
expect(
endpointFilters.validate({
kql: 'whatever.field',
})
).toBeTruthy();
});
it('accepts KQL + status', () => {
expect(
endpointFilters.validate({
kql: 'thing.var',
host_status: ['online'],
})
).toBeTruthy();
});
it('accepts no filters', () => {
expect(endpointFilters.validate({})).toBeTruthy();
});
});
function createSearchResponse(hostMetadata?: HostMetadata): SearchResponse<HostMetadataDetails> {
return ({
took: 15,
timed_out: false,
_shards: {
total: 1,
successful: 1,
skipped: 0,
failed: 0,
},
hits: {
total: {
value: 1,
relation: 'eq',
},
max_score: null,
hits: hostMetadata
? [
body: {
paging_properties: [
{
_index: 'metrics-endpoint.metadata-default',
_id: '8FhM0HEBYyRTvb6lOQnw',
_score: null,
_source: {
agent: {
id: '1e3472bb-5c20-4946-b469-b5af1a809e4f',
},
HostDetails: {
...hostMetadata,
page_size: 10,
},
{
page_index: 1,
},
],
},
});
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() =>
Promise.resolve(createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()))
);
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query).toEqual({
bool: {
must_not: {
terms: {
'HostDetails.elastic.agent.id': [
'00000000-0000-0000-0000-000000000000',
'11111111-1111-1111-1111-111111111111',
],
},
},
},
});
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.ok).toBeCalled();
const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList;
expect(endpointResultList.hosts.length).toEqual(1);
expect(endpointResultList.total).toEqual(1);
expect(endpointResultList.request_page_index).toEqual(10);
expect(endpointResultList.request_page_size).toEqual(10);
expect(endpointResultList.query_strategy_version).toEqual(
MetadataQueryStrategyVersions.VERSION_2
);
});
it('test find the latest of all endpoints with paging and filters properties', async () => {
const mockRequest = httpServerMock.createKibanaRequest({
body: {
paging_properties: [
{
page_size: 10,
},
{
page_index: 1,
},
],
filters: { kql: 'not host.ip:10.140.73.246' },
},
});
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() =>
Promise.resolve(createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()))
);
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toBeCalled();
expect(mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query).toEqual({
bool: {
must: [
{
bool: {
must_not: {
terms: {
'HostDetails.elastic.agent.id': [
'00000000-0000-0000-0000-000000000000',
'11111111-1111-1111-1111-111111111111',
],
},
},
},
sort: [1588337587997],
},
]
: [],
},
} as unknown) as SearchResponse<HostMetadataDetails>;
}
{
bool: {
must_not: {
bool: {
should: [
{
match: {
'host.ip': '10.140.73.246',
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
});
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.ok).toBeCalled();
const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList;
expect(endpointResultList.hosts.length).toEqual(1);
expect(endpointResultList.total).toEqual(1);
expect(endpointResultList.request_page_index).toEqual(10);
expect(endpointResultList.request_page_size).toEqual(10);
expect(endpointResultList.query_strategy_version).toEqual(
MetadataQueryStrategyVersions.VERSION_2
);
});
describe('Endpoint Details route', () => {
it('should return 404 on no results', async () => {
const mockRequest = httpServerMock.createKibanaRequest({ params: { id: 'BADID' } });
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() =>
Promise.resolve(createV2SearchResponse())
);
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.getAgent = jest.fn().mockReturnValue(({
active: true,
} as unknown) as Agent);
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.notFound).toBeCalled();
const message = mockResponse.notFound.mock.calls[0][0]?.body;
expect(message).toEqual('Endpoint Not Found');
});
it('should return a single endpoint with status online', async () => {
const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata());
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
});
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('online');
mockAgentService.getAgent = jest.fn().mockReturnValue(({
active: true,
} as unknown) as Agent);
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.ok).toBeCalled();
const result = mockResponse.ok.mock.calls[0][0]?.body as HostInfo;
expect(result).toHaveProperty('metadata.Endpoint');
expect(result.host_status).toEqual(HostStatus.ONLINE);
expect(result.query_strategy_version).toEqual(MetadataQueryStrategyVersions.VERSION_2);
});
it('should return a single endpoint with status error when AgentService throw 404', async () => {
const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata());
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
});
mockAgentService.getAgentStatusById = jest.fn().mockImplementation(() => {
SavedObjectsErrorHelpers.createGenericNotFoundError();
});
mockAgentService.getAgent = jest.fn().mockImplementation(() => {
SavedObjectsErrorHelpers.createGenericNotFoundError();
});
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.ok).toBeCalled();
const result = mockResponse.ok.mock.calls[0][0]?.body as HostInfo;
expect(result.host_status).toEqual(HostStatus.ERROR);
});
it('should return a single endpoint with status error when status is not offline, online or enrolling', async () => {
const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata());
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
});
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('warning');
mockAgentService.getAgent = jest.fn().mockReturnValue(({
active: true,
} as unknown) as Agent);
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.ok).toBeCalled();
const result = mockResponse.ok.mock.calls[0][0]?.body as HostInfo;
expect(result.host_status).toEqual(HostStatus.ERROR);
});
it('should throw error when endpoint agent is not active', async () => {
const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata());
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
});
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
mockAgentService.getAgent = jest.fn().mockReturnValue(({
active: false,
} as unknown) as Agent);
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(mockResponse.customError).toBeCalled();
});
});
});
});

412
x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata_v1.test.ts Normal file
View file

@ -0,0 +1,412 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import {
ILegacyClusterClient,
IRouter,
ILegacyScopedClusterClient,
KibanaResponseFactory,
RequestHandler,
RouteConfig,
SavedObjectsClientContract,
} from 'kibana/server';
import { SavedObjectsErrorHelpers } from '../../../../../../../src/core/server/';
import {
elasticsearchServiceMock,
httpServerMock,
httpServiceMock,
loggingSystemMock,
savedObjectsClientMock,
} from '../../../../../../../src/core/server/mocks';
import {
HostInfo,
HostResultList,
HostStatus,
MetadataQueryStrategyVersions,
} from '../../../../common/endpoint/types';
import { registerEndpointRoutes, METADATA_REQUEST_V1_ROUTE } from './index';
import {
createMockEndpointAppContextServiceStartContract,
createMockPackageService,
createRouteHandlerContext,
} from '../../mocks';
import { EndpointAppContextService } from '../../endpoint_app_context_services';
import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__';
import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data';
import { Agent, EsAssetReference } from '../../../../../ingest_manager/common/types/models';
import { createV1SearchResponse } from './support/test_support';
import { PackageService } from '../../../../../ingest_manager/server/services';
describe('test endpoint route v1', () => {
let routerMock: jest.Mocked<IRouter>;
let mockResponse: jest.Mocked<KibanaResponseFactory>;
let mockClusterClient: jest.Mocked<ILegacyClusterClient>;
let mockScopedClient: jest.Mocked<ILegacyScopedClusterClient>;
let mockSavedObjectClient: jest.Mocked<SavedObjectsClientContract>;
let mockPackageService: jest.Mocked<PackageService>;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let routeHandler: RequestHandler<any, any, any>;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let routeConfig: RouteConfig<any, any, any, any>;
// tests assume that ingestManager is enabled, and thus agentService is available
let mockAgentService: Required<
ReturnType<typeof createMockEndpointAppContextServiceStartContract>
>['agentService'];
let endpointAppContextService: EndpointAppContextService;
const noUnenrolledAgent = {
agents: [],
total: 0,
page: 1,
perPage: 1,
};
beforeEach(() => {
mockClusterClient = elasticsearchServiceMock.createLegacyClusterClient() as jest.Mocked<
ILegacyClusterClient
>;
mockScopedClient = elasticsearchServiceMock.createLegacyScopedClusterClient();
mockSavedObjectClient = savedObjectsClientMock.create();
mockClusterClient.asScoped.mockReturnValue(mockScopedClient);
routerMock = httpServiceMock.createRouter();
mockResponse = httpServerMock.createResponseFactory();
endpointAppContextService = new EndpointAppContextService();
mockPackageService = createMockPackageService();
mockPackageService.getInstalledEsAssetReferences.mockReturnValue(
Promise.resolve(([] as unknown) as EsAssetReference[])
);
const startContract = createMockEndpointAppContextServiceStartContract();
endpointAppContextService.start({ ...startContract, packageService: mockPackageService });
mockAgentService = startContract.agentService!;
registerEndpointRoutes(routerMock, {
logFactory: loggingSystemMock.create(),
service: endpointAppContextService,
config: () => Promise.resolve(createMockConfig()),
});
});
afterEach(() => endpointAppContextService.stop());
it('test find the latest of all endpoints', async () => {
const mockRequest = httpServerMock.createKibanaRequest({});
const response = createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata());
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`)
)!;
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'] });
expect(mockResponse.ok).toBeCalled();
const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList;
expect(endpointResultList.hosts.length).toEqual(1);
expect(endpointResultList.total).toEqual(1);
expect(endpointResultList.request_page_index).toEqual(0);
expect(endpointResultList.request_page_size).toEqual(10);
expect(endpointResultList.query_strategy_version).toEqual(
MetadataQueryStrategyVersions.VERSION_1
);
});
it('test find the latest of all endpoints with paging properties', async () => {
const mockRequest = httpServerMock.createKibanaRequest({
body: {
paging_properties: [
{
page_size: 10,
},
{
page_index: 1,
},
],
},
});
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() =>
Promise.resolve(createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata()))
);
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query).toEqual({
bool: {
must_not: {
terms: {
'elastic.agent.id': [
'00000000-0000-0000-0000-000000000000',
'11111111-1111-1111-1111-111111111111',
],
},
},
},
});
expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'] });
expect(mockResponse.ok).toBeCalled();
const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList;
expect(endpointResultList.hosts.length).toEqual(1);
expect(endpointResultList.total).toEqual(1);
expect(endpointResultList.request_page_index).toEqual(10);
expect(endpointResultList.request_page_size).toEqual(10);
expect(endpointResultList.query_strategy_version).toEqual(
MetadataQueryStrategyVersions.VERSION_1
);
});
it('test find the latest of all endpoints with paging and filters properties', async () => {
const mockRequest = httpServerMock.createKibanaRequest({
body: {
paging_properties: [
{
page_size: 10,
},
{
page_index: 1,
},
],
filters: { kql: 'not host.ip:10.140.73.246' },
},
});
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() =>
Promise.resolve(createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata()))
);
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toBeCalled();
expect(mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query).toEqual({
bool: {
must: [
{
bool: {
must_not: {
terms: {
'elastic.agent.id': [
'00000000-0000-0000-0000-000000000000',
'11111111-1111-1111-1111-111111111111',
],
},
},
},
},
{
bool: {
must_not: {
bool: {
should: [
{
match: {
'host.ip': '10.140.73.246',
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
});
expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'] });
expect(mockResponse.ok).toBeCalled();
const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList;
expect(endpointResultList.hosts.length).toEqual(1);
expect(endpointResultList.total).toEqual(1);
expect(endpointResultList.request_page_index).toEqual(10);
expect(endpointResultList.request_page_size).toEqual(10);
expect(endpointResultList.query_strategy_version).toEqual(
MetadataQueryStrategyVersions.VERSION_1
);
});
describe('Endpoint Details route', () => {
it('should return 404 on no results', async () => {
const mockRequest = httpServerMock.createKibanaRequest({ params: { id: 'BADID' } });
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() =>
Promise.resolve(createV1SearchResponse())
);
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.getAgent = jest.fn().mockReturnValue(({
active: true,
} as unknown) as Agent);
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.notFound).toBeCalled();
const message = mockResponse.notFound.mock.calls[0][0]?.body;
expect(message).toEqual('Endpoint Not Found');
});
it('should return a single endpoint with status online', async () => {
const response = createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata());
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
});
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('online');
mockAgentService.getAgent = jest.fn().mockReturnValue(({
active: true,
} as unknown) as Agent);
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.ok).toBeCalled();
const result = mockResponse.ok.mock.calls[0][0]?.body as HostInfo;
expect(result).toHaveProperty('metadata.Endpoint');
expect(result.host_status).toEqual(HostStatus.ONLINE);
});
it('should return a single endpoint with status error when AgentService throw 404', async () => {
const response = createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata());
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
});
mockAgentService.getAgentStatusById = jest.fn().mockImplementation(() => {
SavedObjectsErrorHelpers.createGenericNotFoundError();
});
mockAgentService.getAgent = jest.fn().mockImplementation(() => {
SavedObjectsErrorHelpers.createGenericNotFoundError();
});
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.ok).toBeCalled();
const result = mockResponse.ok.mock.calls[0][0]?.body as HostInfo;
expect(result.host_status).toEqual(HostStatus.ERROR);
});
it('should return a single endpoint with status error when status is not offline, online or enrolling', async () => {
const response = createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata());
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
});
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('warning');
mockAgentService.getAgent = jest.fn().mockReturnValue(({
active: true,
} as unknown) as Agent);
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.ok).toBeCalled();
const result = mockResponse.ok.mock.calls[0][0]?.body as HostInfo;
expect(result.host_status).toEqual(HostStatus.ERROR);
});
it('should throw error when endpoint agent is not active', async () => {
const response = createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata());
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
});
mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response));
mockAgentService.getAgent = jest.fn().mockReturnValue(({
active: false,
} as unknown) as Agent);
[routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1);
expect(mockResponse.customError).toBeCalled();
});
});
});

11
x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts
View file

@ -8,6 +8,7 @@ import { kibanaRequestToMetadataListESQuery, getESQueryHostMetadataByID } from '
import { EndpointAppContextService } from '../../endpoint_app_context_services';
import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__';
import { metadataCurrentIndexPattern } from '../../../../common/endpoint/constants';
import { metadataQueryStrategyV2 } from './support/query_strategies';
describe('query builder', () => {
describe('MetadataListESQuery', () => {
@ -22,7 +23,7 @@ describe('query builder', () => {
service: new EndpointAppContextService(),
config: () => Promise.resolve(createMockConfig()),
},
metadataCurrentIndexPattern
metadataQueryStrategyV2()
);
expect(query).toEqual({
body: {
@ -59,7 +60,7 @@ describe('query builder', () => {
service: new EndpointAppContextService(),
config: () => Promise.resolve(createMockConfig()),
},
metadataCurrentIndexPattern,
metadataQueryStrategyV2(),
{
unenrolledAgentIds: [unenrolledElasticAgentId],
}
@ -107,7 +108,7 @@ describe('query builder', () => {
service: new EndpointAppContextService(),
config: () => Promise.resolve(createMockConfig()),
},
metadataCurrentIndexPattern
metadataQueryStrategyV2()
);
expect(query).toEqual({
@ -166,7 +167,7 @@ describe('query builder', () => {
service: new EndpointAppContextService(),
config: () => Promise.resolve(createMockConfig()),
},
metadataCurrentIndexPattern,
metadataQueryStrategyV2(),
{
unenrolledAgentIds: [unenrolledElasticAgentId],
}
@ -225,7 +226,7 @@ describe('query builder', () => {
describe('MetadataGetQuery', () => {
it('searches for the correct ID', () => {
const mockID = 'AABBCCDD-0011-2233-AA44-DEADBEEF8899';
const query = getESQueryHostMetadataByID(mockID, metadataCurrentIndexPattern);
const query = getESQueryHostMetadataByID(mockID, metadataQueryStrategyV2());
expect(query).toEqual({
body: {

39
x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.ts
View file

@ -5,7 +5,7 @@
*/
import { KibanaRequest } from 'kibana/server';
import { esKuery } from '../../../../../../../src/plugins/data/server';
import { EndpointAppContext } from '../../types';
import { EndpointAppContext, MetadataQueryStrategy } from '../../types';
export interface QueryBuilderOptions {
unenrolledAgentIds?: string[];
@ -16,29 +16,26 @@ export async function kibanaRequestToMetadataListESQuery(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
request: KibanaRequest<any, any, any>,
endpointAppContext: EndpointAppContext,
index: string,
metadataQueryStrategy: MetadataQueryStrategy,
queryBuilderOptions?: QueryBuilderOptions
// eslint-disable-next-line @typescript-eslint/no-explicit-any
): Promise<Record<string, any>> {
const pagingProperties = await getPagingProperties(request, endpointAppContext);
return {
body: {
query: buildQueryBody(
request,
metadataQueryStrategy,
queryBuilderOptions?.unenrolledAgentIds!,
queryBuilderOptions?.statusAgentIDs!
),
sort: [
{
'HostDetails.event.created': {
order: 'desc',
},
},
],
...metadataQueryStrategy.extraBodyProperties,
sort: metadataQueryStrategy.sortProperty,
},
from: pagingProperties.pageIndex * pagingProperties.pageSize,
size: pagingProperties.pageSize,
index,
index: metadataQueryStrategy.index,
};
}
@ -66,6 +63,7 @@ async function getPagingProperties(
function buildQueryBody(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
request: KibanaRequest<any, any, any>,
metadataQueryStrategy: MetadataQueryStrategy,
unerolledAgentIds: string[] | undefined,
statusAgentIDs: string[] | undefined
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@ -75,7 +73,7 @@ function buildQueryBody(
? {
must_not: {
terms: {
'HostDetails.elastic.agent.id': unerolledAgentIds,
[metadataQueryStrategy.elasticAgentIdProperty]: unerolledAgentIds,
},
},
}
@ -84,7 +82,7 @@ function buildQueryBody(
? {
must: {
terms: {
'HostDetails.elastic.agent.id': statusAgentIDs,
[metadataQueryStrategy.elasticAgentIdProperty]: statusAgentIDs,
},
},
}
@ -117,23 +115,20 @@ function buildQueryBody(
};
}
export function getESQueryHostMetadataByID(hostID: string, index: string) {
export function getESQueryHostMetadataByID(
hostID: string,
metadataQueryStrategy: MetadataQueryStrategy
) {
return {
body: {
query: {
match: {
'HostDetails.host.id': hostID,
[metadataQueryStrategy.hostIdProperty]: hostID,
},
},
sort: [
{
'HostDetails.event.created': {
order: 'desc',
},
},
],
sort: metadataQueryStrategy.sortProperty,
size: 1,
},
index,
index: metadataQueryStrategy.index,
};
}

300
x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders_v1.test.ts Normal file
View file

@ -0,0 +1,300 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { httpServerMock, loggingSystemMock } from '../../../../../../../src/core/server/mocks';
import { kibanaRequestToMetadataListESQuery, getESQueryHostMetadataByID } from './query_builders';
import { EndpointAppContextService } from '../../endpoint_app_context_services';
import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__';
import { metadataIndexPattern } from '../../../../common/endpoint/constants';
import { metadataQueryStrategyV1 } from './support/query_strategies';
describe('query builder v1', () => {
describe('MetadataListESQuery', () => {
it('test default query params for all endpoints metadata when no params or body is provided', async () => {
const mockRequest = httpServerMock.createKibanaRequest({
body: {},
});
const query = await kibanaRequestToMetadataListESQuery(
mockRequest,
{
logFactory: loggingSystemMock.create(),
service: new EndpointAppContextService(),
config: () => Promise.resolve(createMockConfig()),
},
metadataQueryStrategyV1()
);
expect(query).toEqual({
body: {
query: {
match_all: {},
},
collapse: {
field: 'host.id',
inner_hits: {
name: 'most_recent',
size: 1,
sort: [{ 'event.created': 'desc' }],
},
},
aggs: {
total: {
cardinality: {
field: 'host.id',
},
},
},
sort: [
{
'event.created': {
order: 'desc',
},
},
],
},
from: 0,
size: 10,
index: metadataIndexPattern,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as Record<string, any>);
});
it(
'test default query params for all endpoints metadata when no params or body is provided ' +
'with unenrolled host ids excluded',
async () => {
const unenrolledElasticAgentId = '1fdca33f-799f-49f4-939c-ea4383c77672';
const mockRequest = httpServerMock.createKibanaRequest({
body: {},
});
const query = await kibanaRequestToMetadataListESQuery(
mockRequest,
{
logFactory: loggingSystemMock.create(),
service: new EndpointAppContextService(),
config: () => Promise.resolve(createMockConfig()),
},
metadataQueryStrategyV1(),
{
unenrolledAgentIds: [unenrolledElasticAgentId],
}
);
expect(query).toEqual({
body: {
query: {
bool: {
must_not: {
terms: {
'elastic.agent.id': [unenrolledElasticAgentId],
},
},
},
},
collapse: {
field: 'host.id',
inner_hits: {
name: 'most_recent',
size: 1,
sort: [{ 'event.created': 'desc' }],
},
},
aggs: {
total: {
cardinality: {
field: 'host.id',
},
},
},
sort: [
{
'event.created': {
order: 'desc',
},
},
],
},
from: 0,
size: 10,
index: metadataIndexPattern,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as Record<string, any>);
}
);
});
describe('test query builder with kql filter', () => {
it('test default query params for all endpoints metadata when body filter is provided', async () => {
const mockRequest = httpServerMock.createKibanaRequest({
body: {
filters: { kql: 'not host.ip:10.140.73.246' },
},
});
const query = await kibanaRequestToMetadataListESQuery(
mockRequest,
{
logFactory: loggingSystemMock.create(),
service: new EndpointAppContextService(),
config: () => Promise.resolve(createMockConfig()),
},
metadataQueryStrategyV1()
);
expect(query).toEqual({
body: {
query: {
bool: {
must: [
{
bool: {
must_not: {
bool: {
should: [
{
match: {
'host.ip': '10.140.73.246',
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
},
collapse: {
field: 'host.id',
inner_hits: {
name: 'most_recent',
size: 1,
sort: [{ 'event.created': 'desc' }],
},
},
aggs: {
total: {
cardinality: {
field: 'host.id',
},
},
},
sort: [
{
'event.created': {
order: 'desc',
},
},
],
},
from: 0,
size: 10,
index: metadataIndexPattern,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as Record<string, any>);
});
it(
'test default query params for all endpoints endpoint metadata excluding unerolled endpoint ' +
'and when body filter is provided',
async () => {
const unenrolledElasticAgentId = '1fdca33f-799f-49f4-939c-ea4383c77672';
const mockRequest = httpServerMock.createKibanaRequest({
body: {
filters: { kql: 'not host.ip:10.140.73.246' },
},
});
const query = await kibanaRequestToMetadataListESQuery(
mockRequest,
{
logFactory: loggingSystemMock.create(),
service: new EndpointAppContextService(),
config: () => Promise.resolve(createMockConfig()),
},
metadataQueryStrategyV1(),
{
unenrolledAgentIds: [unenrolledElasticAgentId],
}
);
expect(query).toEqual({
body: {
query: {
bool: {
must: [
{
bool: {
must_not: {
terms: {
'elastic.agent.id': [unenrolledElasticAgentId],
},
},
},
},
{
bool: {
must_not: {
bool: {
should: [
{
match: {
'host.ip': '10.140.73.246',
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
},
collapse: {
field: 'host.id',
inner_hits: {
name: 'most_recent',
size: 1,
sort: [{ 'event.created': 'desc' }],
},
},
aggs: {
total: {
cardinality: {
field: 'host.id',
},
},
},
sort: [
{
'event.created': {
order: 'desc',
},
},
],
},
from: 0,
size: 10,
index: metadataIndexPattern,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as Record<string, any>);
}
);
});
describe('MetadataGetQuery', () => {
it('searches for the correct ID', () => {
const mockID = 'AABBCCDD-0011-2233-AA44-DEADBEEF8899';
const query = getESQueryHostMetadataByID(mockID, metadataQueryStrategyV1());
expect(query).toEqual({
body: {
query: { match: { 'host.id': mockID } },
sort: [{ 'event.created': { order: 'desc' } }],
size: 1,
},
index: metadataIndexPattern,
});
});
});
});

53
x-pack/plugins/security_solution/server/endpoint/routes/metadata/route_schema_test.ts Normal file
View file

@ -0,0 +1,53 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { endpointFilters } from './index';
describe('Filters Schema Test', () => {
it('accepts a single host status', () => {
expect(
endpointFilters.validate({
host_status: ['error'],
})
).toBeTruthy();
});
it('accepts multiple host status filters', () => {
expect(
endpointFilters.validate({
host_status: ['offline', 'unenrolling'],
})
).toBeTruthy();
});
it('rejects invalid statuses', () => {
expect(() =>
endpointFilters.validate({
host_status: ['foobar'],
})
).toThrowError();
});
it('accepts a KQL string', () => {
expect(
endpointFilters.validate({
kql: 'whatever.field',
})
).toBeTruthy();
});
it('accepts KQL + status', () => {
expect(
endpointFilters.validate({
kql: 'thing.var',
host_status: ['online'],
})
).toBeTruthy();
});
it('accepts no filters', () => {
expect(endpointFilters.validate({})).toBeTruthy();
});
});

115
x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/query_strategies.ts Normal file
View file

@ -0,0 +1,115 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { SearchResponse } from 'elasticsearch';
import {
metadataCurrentIndexPattern,
metadataIndexPattern,
} from '../../../../../common/endpoint/constants';
import {
HostMetadata,
HostMetadataDetails,
MetadataQueryStrategyVersions,
} from '../../../../../common/endpoint/types';
import { HostListQueryResult, HostQueryResult, MetadataQueryStrategy } from '../../../types';
interface HitSource {
_source: HostMetadata;
}
export function metadataQueryStrategyV1(): MetadataQueryStrategy {
return {
index: metadataIndexPattern,
elasticAgentIdProperty: 'elastic.agent.id',
hostIdProperty: 'host.id',
sortProperty: [
{
'event.created': {
order: 'desc',
},
},
],
extraBodyProperties: {
collapse: {
field: 'host.id',
inner_hits: {
name: 'most_recent',
size: 1,
sort: [{ 'event.created': 'desc' }],
},
},
aggs: {
total: {
cardinality: {
field: 'host.id',
},
},
},
},
queryResponseToHostListResult: (
searchResponse: SearchResponse<HostMetadata | HostMetadataDetails>
): HostListQueryResult => {
const response = searchResponse as SearchResponse<HostMetadata>;
return {
resultLength: response?.aggregations?.total?.value || 0,
resultList: response.hits.hits
.map((hit) => hit.inner_hits.most_recent.hits.hits)
.flatMap((data) => data as HitSource)
.map((entry) => entry._source),
queryStrategyVersion: MetadataQueryStrategyVersions.VERSION_1,
};
},
queryResponseToHostResult: (
searchResponse: SearchResponse<HostMetadata | HostMetadataDetails>
): HostQueryResult => {
const response = searchResponse as SearchResponse<HostMetadata>;
return {
resultLength: response.hits.hits.length,
result: response.hits.hits.length > 0 ? response.hits.hits[0]._source : undefined,
queryStrategyVersion: MetadataQueryStrategyVersions.VERSION_1,
};
},
};
}
export function metadataQueryStrategyV2(): MetadataQueryStrategy {
return {
index: metadataCurrentIndexPattern,
elasticAgentIdProperty: 'HostDetails.elastic.agent.id',
hostIdProperty: 'HostDetails.host.id',
sortProperty: [
{
'HostDetails.event.created': {
order: 'desc',
},
},
],
queryResponseToHostListResult: (
searchResponse: SearchResponse<HostMetadata | HostMetadataDetails>
): HostListQueryResult => {
const response = searchResponse as SearchResponse<HostMetadataDetails>;
return {
resultLength:
((response.hits?.total as unknown) as { value: number; relation: string }).value || 0,
resultList:
response.hits.hits.length > 0
? response.hits.hits.map((entry) => entry._source.HostDetails)
: [],
queryStrategyVersion: MetadataQueryStrategyVersions.VERSION_2,
};
},
queryResponseToHostResult: (
searchResponse: SearchResponse<HostMetadata | HostMetadataDetails>
): HostQueryResult => {
const response = searchResponse as SearchResponse<HostMetadataDetails>;
return {
resultLength: response.hits.hits.length,
result:
response.hits.hits.length > 0 ? response.hits.hits[0]._source.HostDetails : undefined,
queryStrategyVersion: MetadataQueryStrategyVersions.VERSION_2,
};
},
};
}

103
x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/test_support.ts Normal file
View file

@ -0,0 +1,103 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { SearchResponse } from 'elasticsearch';
import { HostMetadata, HostMetadataDetails } from '../../../../../common/endpoint/types';
export function createV1SearchResponse(hostMetadata?: HostMetadata): SearchResponse<HostMetadata> {
return ({
took: 15,
timed_out: false,
_shards: {
total: 1,
successful: 1,
skipped: 0,
failed: 0,
},
hits: {
total: {
value: 5,
relation: 'eq',
},
max_score: null,
hits: hostMetadata
? [
{
_index: 'metrics-endpoint.metadata-default',
_id: '8FhM0HEBYyRTvb6lOQnw',
_score: null,
_source: hostMetadata,
sort: [1588337587997],
inner_hits: {
most_recent: {
hits: {
total: {
value: 2,
relation: 'eq',
},
max_score: null,
hits: [
{
_index: 'metrics-endpoint.metadata-default',
_id: 'W6Vo1G8BYQH1gtPUgYkC',
_score: null,
_source: hostMetadata,
sort: [1579816615336],
},
],
},
},
},
},
]
: [],
},
aggregations: {
total: {
value: 1,
},
},
} as unknown) as SearchResponse<HostMetadata>;
}
export function createV2SearchResponse(
hostMetadata?: HostMetadata
): SearchResponse<HostMetadataDetails> {
return ({
took: 15,
timed_out: false,
_shards: {
total: 1,
successful: 1,
skipped: 0,
failed: 0,
},
hits: {
total: {
value: 1,
relation: 'eq',
},
max_score: null,
hits: hostMetadata
? [
{
_index: 'metrics-endpoint.metadata-default',
_id: '8FhM0HEBYyRTvb6lOQnw',
_score: null,
_source: {
agent: {
id: '1e3472bb-5c20-4946-b469-b5af1a809e4f',
},
HostDetails: {
...hostMetadata,
},
},
sort: [1588337587997],
},
]
: [],
},
} as unknown) as SearchResponse<HostMetadataDetails>;
}

33
x-pack/plugins/security_solution/server/endpoint/types.ts
View file

@ -4,8 +4,15 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { LoggerFactory } from 'kibana/server';
import { SearchResponse } from 'elasticsearch';
import { ConfigType } from '../config';
import { EndpointAppContextService } from './endpoint_app_context_services';
import { JsonObject } from '../../../infra/common/typed_json';
import {
HostMetadata,
HostMetadataDetails,
MetadataQueryStrategyVersions,
} from '../../common/endpoint/types';
/**
* The context for Endpoint apps.
@ -19,3 +26,29 @@ export interface EndpointAppContext {
*/
service: EndpointAppContextService;
}
export interface HostListQueryResult {
resultLength: number;
resultList: HostMetadata[];
queryStrategyVersion: MetadataQueryStrategyVersions;
}
export interface HostQueryResult {
resultLength: number;
result: HostMetadata | undefined;
queryStrategyVersion: MetadataQueryStrategyVersions;
}
export interface MetadataQueryStrategy {
index: string;
elasticAgentIdProperty: string;
hostIdProperty: string;
sortProperty: JsonObject[];
extraBodyProperties?: JsonObject;
queryResponseToHostListResult: (
searchResponse: SearchResponse<HostMetadata | HostMetadataDetails>
) => HostListQueryResult;
queryResponseToHostResult: (
searchResponse: SearchResponse<HostMetadata | HostMetadataDetails>
) => HostQueryResult;
}

21
x-pack/plugins/security_solution/server/lib/hosts/elasticsearch_adapter.test.ts
View file

@ -30,7 +30,12 @@ import { HostAggEsItem } from './types';
import { EndpointAppContext } from '../../endpoint/types';
import { mockLogger } from '../detection_engine/signals/__mocks__/es_results';
import { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services';
import { createMockEndpointAppContextServiceStartContract } from '../../endpoint/mocks';
import {
createMockEndpointAppContextServiceStartContract,
createMockPackageService,
} from '../../endpoint/mocks';
import { PackageService } from '../../../../ingest_manager/server/services';
import { ElasticsearchAssetType } from '../../../../ingest_manager/common/types/models';
jest.mock('./query.hosts.dsl', () => {
return {
@ -49,7 +54,7 @@ jest.mock('./query.last_first_seen_host.dsl', () => {
buildLastFirstSeenHostQuery: jest.fn(() => mockGetHostLastFirstSeenDsl),
};
});
jest.mock('../../endpoint/routes/metadata', () => {
jest.mock('../../endpoint/routes/metadata/handlers', () => {
return {
getHostData: jest.fn(() => mockEndpointMetadata),
};
@ -167,8 +172,16 @@ describe('hosts elasticsearch_adapter', () => {
const endpointAppContextService = new EndpointAppContextService();
const startContract = createMockEndpointAppContextServiceStartContract();
endpointAppContextService.start(startContract);
const mockPackageService: jest.Mocked<PackageService> = createMockPackageService();
mockPackageService.getInstalledEsAssetReferences.mockReturnValue(
Promise.resolve([
{
id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0',
type: ElasticsearchAssetType.transform,
},
])
);
endpointAppContextService.start({ ...startContract, packageService: mockPackageService });
const endpointContext: EndpointAppContext = {
logFactory: mockLogger,
service: endpointAppContextService,

10
x-pack/plugins/security_solution/server/lib/hosts/elasticsearch_adapter.ts
View file

@ -8,11 +8,11 @@ import { set } from '@elastic/safer-lodash-set/fp';
import { get, getOr, has, head } from 'lodash/fp';
import {
EndpointFields,
FirstLastSeenHost,
HostItem,
HostsData,
HostsEdges,
EndpointFields,
} from '../../graphql/types';
import { inspectStringifyObject } from '../../utils/build_query';
import { hostFieldsMap } from '../ecs_fields';
@ -25,16 +25,16 @@ import {
HostAggEsData,
HostAggEsItem,
HostBuckets,
HostOverviewRequestOptions,
HostEsData,
HostLastFirstSeenRequestOptions,
HostOverviewRequestOptions,
HostsAdapter,
HostsRequestOptions,
HostValue,
} from './types';
import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../common/constants';
import { EndpointAppContext } from '../../endpoint/types';
import { getHostData } from '../../endpoint/routes/metadata';
import { getHostData } from '../../endpoint/routes/metadata/handlers';
export class ElasticsearchHostsAdapter implements HostsAdapter {
constructor(
@ -116,12 +116,12 @@ export class ElasticsearchHostsAdapter implements HostsAdapter {
throw new Error('agentService not available');
}
const metadataRequestContext = {
agentService,
endpointAppContextService: this.endpointContext.service,
logger,
requestHandlerContext: request.context,
};
const endpointData =
hostId != null && metadataRequestContext.agentService != null
hostId != null && metadataRequestContext.endpointAppContextService.getAgentService() != null
? await getHostData(metadataRequestContext, hostId)
: null;
return endpointData != null && endpointData.metadata

1
x-pack/plugins/security_solution/server/plugin.ts
View file

@ -315,6 +315,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
this.endpointAppContextService.start({
agentService: plugins.ingestManager?.agentService,
packageService: plugins.ingestManager?.packageService,
logger: this.logger,
manifestManager,
registerIngestCallback,

1
x-pack/test/security_solution_endpoint_api_int/apis/index.ts
View file

@ -28,6 +28,7 @@ export default function endpointAPIIntegrationTests(providerContext: FtrProvider
});
loadTestFile(require.resolve('./resolver/index'));
loadTestFile(require.resolve('./metadata'));
loadTestFile(require.resolve('./metadata_v1'));
loadTestFile(require.resolve('./policy'));
loadTestFile(require.resolve('./artifacts'));
loadTestFile(require.resolve('./package'));

38
x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts
View file

@ -11,6 +11,8 @@ import {
deleteAllDocsFromMetadataIndex,
deleteMetadataStream,
} from './data_stream_helper';
import { METADATA_REQUEST_ROUTE } from '../../../plugins/security_solution/server/endpoint/routes/metadata';
import { MetadataQueryStrategyVersions } from '../../../plugins/security_solution/common/endpoint/types';
/**
* The number of host documents in the es archive.
@ -22,14 +24,14 @@ export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
describe('test metadata api', () => {
describe('POST /api/endpoint/metadata when index is empty', () => {
describe(`POST ${METADATA_REQUEST_ROUTE} when index is empty`, () => {
it('metadata api should return empty result when index is empty', async () => {
await deleteMetadataStream(getService);
await deleteAllDocsFromMetadataIndex(getService);
await deleteMetadataCurrentStream(getService);
await deleteAllDocsFromMetadataCurrentIndex(getService);
const { body } = await supertest
.post('/api/endpoint/metadata')
.post(`${METADATA_REQUEST_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send()
.expect(200);
@ -40,7 +42,7 @@ export default function ({ getService }: FtrProviderContext) {
});
});
describe('POST /api/endpoint/metadata when index is not empty', () => {
describe(`POST ${METADATA_REQUEST_ROUTE} when index is not empty`, () => {
before(async () => {
await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true });
// wait for transform
@ -56,7 +58,7 @@ export default function ({ getService }: FtrProviderContext) {
});
it('metadata api should return one entry for each host with default paging', async () => {
const { body } = await supertest
.post('/api/endpoint/metadata')
.post(`${METADATA_REQUEST_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send()
.expect(200);
@ -68,7 +70,7 @@ export default function ({ getService }: FtrProviderContext) {
it('metadata api should return page based on paging properties passed.', async () => {
const { body } = await supertest
.post('/api/endpoint/metadata')
.post(`${METADATA_REQUEST_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
paging_properties: [
@ -85,6 +87,7 @@ export default function ({ getService }: FtrProviderContext) {
expect(body.hosts.length).to.eql(1);
expect(body.request_page_size).to.eql(1);
expect(body.request_page_index).to.eql(1);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_2);
});
/* test that when paging properties produces no result, the total should reflect the actual number of metadata
@ -92,7 +95,7 @@ export default function ({ getService }: FtrProviderContext) {
*/
it('metadata api should return accurate total metadata if page index produces no result', async () => {
const { body } = await supertest
.post('/api/endpoint/metadata')
.post(`${METADATA_REQUEST_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
paging_properties: [
@ -109,11 +112,12 @@ export default function ({ getService }: FtrProviderContext) {
expect(body.hosts.length).to.eql(0);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(30);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_2);
});
it('metadata api should return 400 when pagingProperties is below boundaries.', async () => {
const { body } = await supertest
.post('/api/endpoint/metadata')
.post(`${METADATA_REQUEST_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
paging_properties: [
@ -131,7 +135,7 @@ export default function ({ getService }: FtrProviderContext) {
it('metadata api should return page based on filters passed.', async () => {
const { body } = await supertest
.post('/api/endpoint/metadata')
.post(`${METADATA_REQUEST_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
filters: {
@ -143,12 +147,13 @@ export default function ({ getService }: FtrProviderContext) {
expect(body.hosts.length).to.eql(2);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_2);
});
it('metadata api should return page based on filters and paging passed.', async () => {
const notIncludedIp = '10.46.229.234';
const { body } = await supertest
.post('/api/endpoint/metadata')
.post(`${METADATA_REQUEST_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
paging_properties: [
@ -180,12 +185,13 @@ export default function ({ getService }: FtrProviderContext) {
expect(body.hosts.length).to.eql(2);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_2);
});
it('metadata api should return page based on host.os.Ext.variant filter.', async () => {
const variantValue = 'Windows Pro';
const { body } = await supertest
.post('/api/endpoint/metadata')
.post(`${METADATA_REQUEST_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
filters: {
@ -201,12 +207,13 @@ export default function ({ getService }: FtrProviderContext) {
expect(body.hosts.length).to.eql(2);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_2);
});
it('metadata api should return the latest event for all the events for an endpoint', async () => {
const targetEndpointIp = '10.46.229.234';
const { body } = await supertest
.post('/api/endpoint/metadata')
.post(`${METADATA_REQUEST_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
filters: {
@ -223,11 +230,12 @@ export default function ({ getService }: FtrProviderContext) {
expect(body.hosts.length).to.eql(1);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_2);
});
it('metadata api should return the latest event for all the events where policy status is not success', async () => {
const { body } = await supertest
.post('/api/endpoint/metadata')
.post(`${METADATA_REQUEST_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
filters: {
@ -248,7 +256,7 @@ export default function ({ getService }: FtrProviderContext) {
const targetEndpointId = 'fc0ff548-feba-41b6-8367-65e8790d0eaf';
const targetElasticAgentId = '023fa40c-411d-4188-a941-4147bfadd095';
const { body } = await supertest
.post('/api/endpoint/metadata')
.post(`${METADATA_REQUEST_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
filters: {
@ -266,11 +274,12 @@ export default function ({ getService }: FtrProviderContext) {
expect(body.hosts.length).to.eql(1);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_2);
});
it('metadata api should return all hosts when filter is empty string', async () => {
const { body } = await supertest
.post('/api/endpoint/metadata')
.post(`${METADATA_REQUEST_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
filters: {
@ -282,6 +291,7 @@ export default function ({ getService }: FtrProviderContext) {
expect(body.hosts.length).to.eql(numberOfHostsInFixture);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_2);
});
});
});

284
x-pack/test/security_solution_endpoint_api_int/apis/metadata_v1.ts Normal file
View file

@ -0,0 +1,284 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import expect from '@kbn/expect/expect.js';
import { FtrProviderContext } from '../ftr_provider_context';
import { deleteMetadataStream } from './data_stream_helper';
import { METADATA_REQUEST_V1_ROUTE } from '../../../plugins/security_solution/server/endpoint/routes/metadata';
import { MetadataQueryStrategyVersions } from '../../../plugins/security_solution/common/endpoint/types';
/**
* The number of host documents in the es archive.
*/
const numberOfHostsInFixture = 3;
export default function ({ getService }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const supertest = getService('supertest');
describe('test metadata api v1', () => {
describe(`POST ${METADATA_REQUEST_V1_ROUTE} when index is empty`, () => {
it('metadata api should return empty result when index is empty', async () => {
// the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
// to do it manually
await deleteMetadataStream(getService);
const { body } = await supertest
.post(`${METADATA_REQUEST_V1_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send()
.expect(200);
expect(body.total).to.eql(0);
expect(body.hosts.length).to.eql(0);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_1);
});
});
describe(`POST ${METADATA_REQUEST_V1_ROUTE} when index is not empty`, () => {
before(
async () => await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true })
);
// the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
// to do it manually
after(async () => await deleteMetadataStream(getService));
it('metadata api should return one entry for each host with default paging', async () => {
const { body } = await supertest
.post(`${METADATA_REQUEST_V1_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send()
.expect(200);
expect(body.total).to.eql(numberOfHostsInFixture);
expect(body.hosts.length).to.eql(numberOfHostsInFixture);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_1);
});
it('metadata api should return page based on paging properties passed.', async () => {
const { body } = await supertest
.post(`${METADATA_REQUEST_V1_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
paging_properties: [
{
page_size: 1,
},
{
page_index: 1,
},
],
})
.expect(200);
expect(body.total).to.eql(numberOfHostsInFixture);
expect(body.hosts.length).to.eql(1);
expect(body.request_page_size).to.eql(1);
expect(body.request_page_index).to.eql(1);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_1);
});
/* test that when paging properties produces no result, the total should reflect the actual number of metadata
in the index.
*/
it('metadata api should return accurate total metadata if page index produces no result', async () => {
const { body } = await supertest
.post(`${METADATA_REQUEST_V1_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
paging_properties: [
{
page_size: 10,
},
{
page_index: 3,
},
],
})
.expect(200);
expect(body.total).to.eql(numberOfHostsInFixture);
expect(body.hosts.length).to.eql(0);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(30);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_1);
});
it('metadata api should return 400 when pagingProperties is below boundaries.', async () => {
const { body } = await supertest
.post(`${METADATA_REQUEST_V1_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
paging_properties: [
{
page_size: 0,
},
{
page_index: 1,
},
],
})
.expect(400);
expect(body.message).to.contain('Value must be equal to or greater than [1]');
});
it('metadata api should return page based on filters passed.', async () => {
const { body } = await supertest
.post(`${METADATA_REQUEST_V1_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
filters: {
kql: 'not host.ip:10.46.229.234',
},
})
.expect(200);
expect(body.total).to.eql(2);
expect(body.hosts.length).to.eql(2);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_1);
});
it('metadata api should return page based on filters and paging passed.', async () => {
const notIncludedIp = '10.46.229.234';
const { body } = await supertest
.post(`${METADATA_REQUEST_V1_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
paging_properties: [
{
page_size: 10,
},
{
page_index: 0,
},
],
filters: {
kql: `not host.ip:${notIncludedIp}`,
},
})
.expect(200);
expect(body.total).to.eql(2);
const resultIps: string[] = [].concat(
...body.hosts.map((hostInfo: Record<string, any>) => hostInfo.metadata.host.ip)
);
expect(resultIps).to.eql([
'10.192.213.130',
'10.70.28.129',
'10.101.149.26',
'2606:a000:ffc0:39:11ef:37b9:3371:578c',
]);
expect(resultIps).not.include.eql(notIncludedIp);
expect(body.hosts.length).to.eql(2);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_1);
});
it('metadata api should return page based on host.os.Ext.variant filter.', async () => {
const variantValue = 'Windows Pro';
const { body } = await supertest
.post(`${METADATA_REQUEST_V1_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
filters: {
kql: `host.os.Ext.variant:${variantValue}`,
},
})
.expect(200);
expect(body.total).to.eql(2);
const resultOsVariantValue: Set<string> = new Set(
body.hosts.map((hostInfo: Record<string, any>) => hostInfo.metadata.host.os.Ext.variant)
);
expect(Array.from(resultOsVariantValue)).to.eql([variantValue]);
expect(body.hosts.length).to.eql(2);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_1);
});
it('metadata api should return the latest event for all the events for an endpoint', async () => {
const targetEndpointIp = '10.46.229.234';
const { body } = await supertest
.post(`${METADATA_REQUEST_V1_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
filters: {
kql: `host.ip:${targetEndpointIp}`,
},
})
.expect(200);
expect(body.total).to.eql(1);
const resultIp: string = body.hosts[0].metadata.host.ip.filter(
(ip: string) => ip === targetEndpointIp
);
expect(resultIp).to.eql([targetEndpointIp]);
expect(body.hosts[0].metadata.event.created).to.eql(1579881969541);
expect(body.hosts.length).to.eql(1);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_1);
});
it('metadata api should return the latest event for all the events where policy status is not success', async () => {
const { body } = await supertest
.post(`${METADATA_REQUEST_V1_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
filters: {
kql: `not Endpoint.policy.applied.status:success`,
},
})
.expect(200);
const statuses: Set<string> = new Set(
body.hosts.map(
(hostInfo: Record<string, any>) => hostInfo.metadata.Endpoint.policy.applied.status
)
);
expect(statuses.size).to.eql(1);
expect(Array.from(statuses)).to.eql(['failure']);
});
it('metadata api should return the endpoint based on the elastic agent id, and status should be error', async () => {
const targetEndpointId = 'fc0ff548-feba-41b6-8367-65e8790d0eaf';
const targetElasticAgentId = '023fa40c-411d-4188-a941-4147bfadd095';
const { body } = await supertest
.post(`${METADATA_REQUEST_V1_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
filters: {
kql: `elastic.agent.id:${targetElasticAgentId}`,
},
})
.expect(200);
expect(body.total).to.eql(1);
const resultHostId: string = body.hosts[0].metadata.host.id;
const resultElasticAgentId: string = body.hosts[0].metadata.elastic.agent.id;
expect(resultHostId).to.eql(targetEndpointId);
expect(resultElasticAgentId).to.eql(targetElasticAgentId);
expect(body.hosts[0].metadata.event.created).to.eql(1579881969541);
expect(body.hosts[0].host_status).to.eql('error');
expect(body.hosts.length).to.eql(1);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_1);
});
it('metadata api should return all hosts when filter is empty string', async () => {
const { body } = await supertest
.post(`${METADATA_REQUEST_V1_ROUTE}`)
.set('kbn-xsrf', 'xxx')
.send({
filters: {
kql: '',
},
})
.expect(200);
expect(body.total).to.eql(numberOfHostsInFixture);
expect(body.hosts.length).to.eql(numberOfHostsInFixture);
expect(body.request_page_size).to.eql(10);
expect(body.request_page_index).to.eql(0);
expect(body.query_strategy_version).to.eql(MetadataQueryStrategyVersions.VERSION_1);
});
});
});
}