maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[RAC][Security Solution] Register Security Detection Rules with Rule Registry (#96015) (#100940)

Browse source 
## Summary

This PR starts the migration of the Security Solution rules to use the rule-registry introduced in https://github.com/elastic/kibana/pull/95903. This is a pathfinding effort in porting over the existing Security Solution rules, and may include some temporary reference rules for testing out different paradigms as we move the rules over. See https://github.com/elastic/kibana/issues/95735 for details

Enable via the following feature flags in your `kibana.dev.yml`:

```
# Security Solution Rules on Rule Registry
xpack.ruleRegistry.index: '.kibana-[USERNAME]-alerts' # Only necessary to scope from other devs testing, if not specified defaults to `.alerts-security-solution`
xpack.securitySolution.enableExperimental: ['ruleRegistryEnabled']
```

> Note: if setting a custom `xpack.ruleRegistry.index`, for the time being you must also update the [DEFAULT_ALERTS_INDEX](9e213fb7a5/x-pack/plugins/security_solution/common/constants.ts (L28)) in order for the UI to display alerts within the alerts table.

---

Three reference rule types have been added (`query`, `eql`, `threshold`), along with scripts for creating them located in:

```
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/scripts/
```

Main Detection page TGrid queries have been short-circuited to query `.alerts-security-solution*` for displaying alerts from the new alerts as data indices.

To test, checkout, enable the above feature flag(s), and run one of the scripts from the above directory, e.g.  `./create_reference_rule_query.sh` (ensure your ENV vars as set! :)

Alerts as data within the main Detection Page 🎉
<p align="center">
  <img width="500" src="https://user-images.githubusercontent.com/2946766/119911768-39cfba00-bf17-11eb-8996-63c0b813fdcc.png" />
</p>

cc @madirey @dgieselaar @pmuellr @yctercero @dhurley14 @marshallmain

# Conflicts:
#	x-pack/plugins/security_solution/server/plugin.ts
This commit is contained in:
Garrett Spong 2021-05-28 14:46:51 -06:00 committed by GitHub
parent f759046189
commit 8ca9d3ffc8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
46 changed files with 1606 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
x-pack/plugins/rule_registry/README.md
View file

@ -145,3 +145,6 @@ The following fields are defined in the technical field component template and s
- `kibana.rac.alert.severity.value`: the severity of the alert, as a numerical value, which allows sorting.
- `kibana.rac.alert.evaluation.value`: The measured (numerical value).
- `kibana.rac.alert.threshold.value`: The threshold that was defined (or, in case of multiple thresholds, the one that was exceeded).
- `kibana.rac.alert.ancestors`: the array of ancestors (if any) for the alert.
- `kibana.rac.alert.depth`: the depth of the alert in the ancestral tree (default 0).
- `kibana.rac.alert.building_block_type`: the building block type of the alert (default undefined).

1
x-pack/plugins/rule_registry/server/index.ts
View file

@ -14,6 +14,7 @@ export { RuleDataClient } from './rule_data_client';
export { IRuleDataClient } from './rule_data_client/types';
export { getRuleExecutorData, RuleExecutorData } from './utils/get_rule_executor_data';
export { createLifecycleRuleTypeFactory } from './utils/create_lifecycle_rule_type_factory';
export { createPersistenceRuleTypeFactory } from './utils/create_persistence_rule_type_factory';
export const plugin = (initContext: PluginInitializerContext) =>
new RuleRegistryPlugin(initContext);

4
x-pack/plugins/rule_registry/server/rule_data_client/index.ts
View file

@ -73,8 +73,8 @@ export class RuleDataClient implements IRuleDataClient {
return clusterClient.bulk(requestWithDefaultParameters).then((response) => {
if (response.body.errors) {
if (
response.body.items.length === 1 &&
response.body.items[0]?.index?.error?.type === 'index_not_found_exception'
response.body.items.length > 0 &&
response.body.items?.[0]?.index?.error?.type === 'index_not_found_exception'
) {
return this.createOrUpdateWriteTarget({ namespace }).then(() => {
return clusterClient.bulk(requestWithDefaultParameters);

112
x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_factory.ts Normal file
View file

@ -0,0 +1,112 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ESSearchRequest } from 'typings/elasticsearch';
import v4 from 'uuid/v4';
import { Logger } from '@kbn/logging';
import { AlertInstance } from '../../../alerting/server';
import {
AlertInstanceContext,
AlertInstanceState,
AlertTypeParams,
} from '../../../alerting/common';
import { RuleDataClient } from '../rule_data_client';
import { AlertTypeWithExecutor } from '../types';
type PersistenceAlertService<TAlertInstanceContext extends Record<string, unknown>> = (
alerts: Array<Record<string, unknown>>
) => Array<AlertInstance<AlertInstanceState, TAlertInstanceContext, string>>;
type PersistenceAlertQueryService = (
query: ESSearchRequest
) => Promise<Array<Record<string, unknown>>>;
type CreatePersistenceRuleTypeFactory = (options: {
ruleDataClient: RuleDataClient;
logger: Logger;
}) => <
TParams extends AlertTypeParams,
TAlertInstanceContext extends AlertInstanceContext,
TServices extends {
alertWithPersistence: PersistenceAlertService<TAlertInstanceContext>;
findAlerts: PersistenceAlertQueryService;
}
>(
type: AlertTypeWithExecutor<TParams, TAlertInstanceContext, TServices>
) => AlertTypeWithExecutor<TParams, TAlertInstanceContext, any>;
export const createPersistenceRuleTypeFactory: CreatePersistenceRuleTypeFactory = ({
logger,
ruleDataClient,
}) => (type) => {
return {
...type,
executor: async (options) => {
const {
services: { alertInstanceFactory, scopedClusterClient },
} = options;
const currentAlerts: Array<Record<string, unknown>> = [];
const timestamp = options.startedAt.toISOString();
const state = await type.executor({
...options,
services: {
...options.services,
alertWithPersistence: (alerts) => {
alerts.forEach((alert) => currentAlerts.push(alert));
return alerts.map((alert) =>
alertInstanceFactory(alert['kibana.rac.alert.uuid']! as string)
);
},
findAlerts: async (query) => {
const { body } = await scopedClusterClient.asCurrentUser.search({
...query,
body: {
...query.body,
},
ignore_unavailable: true,
});
return body.hits.hits
.map((event: { _source: any }) => event._source!)
.map((event: { [x: string]: any }) => {
const alertUuid = event['kibana.rac.alert.uuid'];
const isAlert = alertUuid != null;
return {
...event,
'event.kind': 'signal',
'kibana.rac.alert.id': '???',
'kibana.rac.alert.status': 'open',
'kibana.rac.alert.uuid': v4(),
'kibana.rac.alert.ancestors': isAlert
? ((event['kibana.rac.alert.ancestors'] as string[]) ?? []).concat([
alertUuid!,
] as string[])
: [],
'kibana.rac.alert.depth': isAlert
? ((event['kibana.rac.alert.depth'] as number) ?? 0) + 1
: 0,
'@timestamp': timestamp,
};
});
},
},
});
const numAlerts = currentAlerts.length;
logger.debug(`Found ${numAlerts} alerts.`);
if (ruleDataClient && numAlerts) {
await ruleDataClient.getWriter().bulk({
body: currentAlerts.flatMap((event) => [{ index: {} }, event]),
});
}
return state;
},
};
};

13
x-pack/plugins/security_solution/common/constants.ts
View file

@ -25,6 +25,7 @@ export const DEFAULT_TIME_RANGE = 'timepicker:timeDefaults';
export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults';
export const DEFAULT_APP_TIME_RANGE = 'securitySolution:timeDefaults';
export const DEFAULT_APP_REFRESH_INTERVAL = 'securitySolution:refreshIntervalDefaults';
export const DEFAULT_ALERTS_INDEX = '.alerts-security-solution';
export const DEFAULT_SIGNALS_INDEX = '.siem-signals';
export const DEFAULT_LISTS_INDEX = '.lists';
export const DEFAULT_ITEMS_INDEX = '.items';
@ -148,6 +149,18 @@ export const DEFAULT_TRANSFORMS_SETTING = JSON.stringify(defaultTransformsSettin
*/
export const SIGNALS_ID = `siem.signals`;
/**
* Id's for reference rule types
*/
export const REFERENCE_RULE_ALERT_TYPE_ID = `siem.referenceRule`;
export const REFERENCE_RULE_PERSISTENCE_ALERT_TYPE_ID = `siem.referenceRulePersistence`;
export const CUSTOM_ALERT_TYPE_ID = `siem.customRule`;
export const EQL_ALERT_TYPE_ID = `siem.eqlRule`;
export const INDICATOR_ALERT_TYPE_ID = `siem.indicatorRule`;
export const ML_ALERT_TYPE_ID = `siem.mlRule`;
export const THRESHOLD_ALERT_TYPE_ID = `siem.thresholdRule`;
/**
* Id for the notifications alerting type
*/

1
x-pack/plugins/security_solution/common/experimental_features.ts
View file

@ -15,6 +15,7 @@ const allowedExperimentalValues = Object.freeze({
trustedAppsByPolicyEnabled: false,
metricsEntitiesEnabled: false,
hostIsolationEnabled: false,
ruleRegistryEnabled: false,
});
type ExperimentalConfigKeys = Array<keyof ExperimentalFeatures>;

1
x-pack/plugins/security_solution/kibana.json
View file

@ -8,6 +8,7 @@
"actions",
"alerting",
"cases",
"ruleRegistry",
"data",
"dataEnhanced",
"embeddable",

18
x-pack/plugins/security_solution/public/common/components/exceptions/use_add_exception.test.tsx
View file

@ -21,7 +21,7 @@ import type {
CreateExceptionListItemSchema,
UpdateExceptionListItemSchema,
} from '@kbn/securitysolution-io-ts-list-types';
import { TestProviders } from '../../mock';
import {
useAddOrUpdateException,
UseAddOrUpdateExceptionProps,
@ -134,12 +134,16 @@ describe('useAddOrUpdateException', () => {
addOrUpdateItemsArgs = [ruleId, itemsToAddOrUpdate];
render = () =>
renderHook<UseAddOrUpdateExceptionProps, ReturnUseAddOrUpdateException>(() =>
useAddOrUpdateException({
http: mockKibanaHttpService,
onError,
onSuccess,
})
renderHook<UseAddOrUpdateExceptionProps, ReturnUseAddOrUpdateException>(
() =>
useAddOrUpdateException({
http: mockKibanaHttpService,
onError,
onSuccess,
}),
{
wrapper: TestProviders,
}
);
});

20
x-pack/plugins/security_solution/public/common/components/exceptions/use_add_exception.tsx
View file

@ -19,9 +19,11 @@ import { getUpdateAlertsQuery } from '../../../detections/components/alerts_tabl
import {
buildAlertStatusFilter,
buildAlertsRuleIdFilter,
buildAlertStatusFilterRuleRegistry,
} from '../../../detections/components/alerts_table/default_config';
import { getQueryFilter } from '../../../../common/detection_engine/get_query_filter';
import { Index } from '../../../../common/detection_engine/schemas/common/schemas';
import { useIsExperimentalFeatureEnabled } from '../../hooks/use_experimental_features';
import { formatExceptionItemForUpdate, prepareExceptionItemsForBulkClose } from './helpers';
import { useKibana } from '../../lib/kibana';
@ -82,6 +84,8 @@ export const useAddOrUpdateException = ({
},
[]
);
// TODO: Once we are past experimental phase this code should be removed
const ruleRegistryEnabled = useIsExperimentalFeatureEnabled('ruleRegistryEnabled');
useEffect(() => {
let isSubscribed = true;
@ -127,10 +131,15 @@ export const useAddOrUpdateException = ({
}
if (bulkCloseIndex != null) {
// TODO: Once we are past experimental phase this code should be removed
const alertStatusFilter = ruleRegistryEnabled
? buildAlertStatusFilterRuleRegistry('open')
: buildAlertStatusFilter('open');
const filter = getQueryFilter(
'',
'kuery',
[...buildAlertsRuleIdFilter(ruleId), ...buildAlertStatusFilter('open')],
[...buildAlertsRuleIdFilter(ruleId), ...alertStatusFilter],
bulkCloseIndex,
prepareExceptionItemsForBulkClose(exceptionItemsToAddOrUpdate),
false
@ -176,7 +185,14 @@ export const useAddOrUpdateException = ({
isSubscribed = false;
abortCtrl.abort();
};
}, [http, onSuccess, onError, updateExceptionListItem, addExceptionListItem]);
}, [
addExceptionListItem,
http,
onSuccess,
onError,
ruleRegistryEnabled,
updateExceptionListItem,
]);
return [{ isLoading }, addOrUpdateException];
};

1
x-pack/plugins/security_solution/public/common/mock/global_state.ts
View file

@ -43,6 +43,7 @@ export const mockGlobalState: State = {
trustedAppsByPolicyEnabled: false,
metricsEntitiesEnabled: false,
hostIsolationEnabled: false,
ruleRegistryEnabled: false,
},
},
hosts: {

26
x-pack/plugins/security_solution/public/common/mock/test_providers.tsx
View file

@ -24,11 +24,12 @@ import {
import { FieldHook } from '../../shared_imports';
import { SUB_PLUGINS_REDUCER } from './utils';
import { createSecuritySolutionStorageMock, localStorageMock } from './mock_local_storage';
import { UserPrivilegesProvider } from '../../detections/components/user_privileges';
const state: State = mockGlobalState;
interface Props {
children: React.ReactNode;
children?: React.ReactNode;
store?: Store;
onDragEnd?: (result: DropResult, provided: ResponderProvided) => void;
}
@ -59,7 +60,30 @@ const TestProvidersComponent: React.FC<Props> = ({
</I18nProvider>
);
/**
* A utility for wrapping children in the providers required to run most tests
* WITH user privileges provider.
*/
const TestProvidersWithPrivilegesComponent: React.FC<Props> = ({
children,
store = createStore(state, SUB_PLUGINS_REDUCER, kibanaObservable, storage),
onDragEnd = jest.fn(),
}) => (
<I18nProvider>
<MockKibanaContextProvider>
<ReduxStoreProvider store={store}>
<ThemeProvider theme={() => ({ eui: euiDarkVars, darkMode: true })}>
<UserPrivilegesProvider>
<DragDropContext onDragEnd={onDragEnd}>{children}</DragDropContext>
</UserPrivilegesProvider>
</ThemeProvider>
</ReduxStoreProvider>
</MockKibanaContextProvider>
</I18nProvider>
);
export const TestProviders = React.memo(TestProvidersComponent);
export const TestProvidersWithPrivileges = React.memo(TestProvidersWithPrivilegesComponent);
export const useFormFieldMock = <T,>(options?: Partial<FieldHook<T>>): FieldHook<T> => {
return {

76
x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx
View file

@ -5,11 +5,12 @@
* 2.0.
*/
import { defaultColumnHeaderType } from '../../../timelines/components/timeline/body/column_headers/default_headers';
import { RowRendererId } from '../../../../common/types/timeline';
import { Status } from '../../../../common/detection_engine/schemas/common/schemas';
import { Filter } from '../../../../../../../src/plugins/data/common/es_query';
import { SubsetTimelineModel } from '../../../timelines/store/timeline/model';
import { ColumnHeaderOptions, SubsetTimelineModel } from '../../../timelines/store/timeline/model';
import { timelineDefaults } from '../../../timelines/store/timeline/defaults';
import { columns } from '../../configurations/security_solution_detections/columns';
@ -124,3 +125,76 @@ export const requiredFieldsForActions = [
'host.os.family',
'event.code',
];
// TODO: Once we are past experimental phase this code should be removed
export const buildAlertStatusFilterRuleRegistry = (status: Status): Filter[] => [
{
meta: {
alias: null,
negate: false,
disabled: false,
type: 'phrase',
key: 'kibana.rac.alert.status',
params: {
query: status,
},
},
query: {
term: {
'kibana.rac.alert.status': status,
},
},
},
];
export const buildShowBuildingBlockFilterRuleRegistry = (
showBuildingBlockAlerts: boolean
): Filter[] =>
showBuildingBlockAlerts
? []
: [
{
meta: {
alias: null,
negate: true,
disabled: false,
type: 'exists',
key: 'kibana.rac.rule.building_block_type',
value: 'exists',
},
// @ts-expect-error TODO: Rework parent typings to support ExistsFilter[]
exists: { field: 'kibana.rac.rule.building_block_type' },
},
];
export const requiredFieldMappingsForActionsRuleRegistry = {
'@timestamp': '@timestamp',
'alert.id': 'kibana.rac.alert.id',
'event.kind': 'event.kind',
'alert.start': 'kibana.rac.alert.start',
'alert.uuid': 'kibana.rac.alert.uuid',
'event.action': 'event.action',
'alert.status': 'kibana.rac.alert.status',
'alert.duration.us': 'kibana.rac.alert.duration.us',
'rule.uuid': 'rule.uuid',
'rule.id': 'rule.id',
'rule.name': 'rule.name',
'rule.category': 'rule.category',
producer: 'kibana.rac.alert.producer',
tags: 'tags',
};
export const alertsHeadersRuleRegistry: ColumnHeaderOptions[] = Object.entries(
requiredFieldMappingsForActionsRuleRegistry
).map<ColumnHeaderOptions>(([alias, field]) => ({
columnHeaderType: defaultColumnHeaderType,
displayAsText: alias,
id: field,
}));
export const alertsDefaultModelRuleRegistry: SubsetTimelineModel = {
...timelineDefaults,
columns: alertsHeadersRuleRegistry,
showCheckboxes: true,
excludedRowRendererIds: Object.values(RowRendererId),
};

32
x-pack/plugins/security_solution/public/detections/components/alerts_table/index.tsx
View file

@ -16,6 +16,7 @@ import { TimelineIdLiteral } from '../../../../common/types/timeline';
import { useAppToasts } from '../../../common/hooks/use_app_toasts';
import { StatefulEventsViewer } from '../../../common/components/events_viewer';
import { HeaderSection } from '../../../common/components/header_section';
import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features';
import { combineQueries } from '../../../timelines/components/timeline/helpers';
import { useKibana } from '../../../common/lib/kibana';
import { inputsSelectors, State, inputsModel } from '../../../common/store';
@ -29,6 +30,8 @@ import {
requiredFieldsForActions,
alertsDefaultModel,
buildAlertStatusFilter,
alertsDefaultModelRuleRegistry,
buildAlertStatusFilterRuleRegistry,
} from './default_config';
import { FILTER_OPEN, AlertsTableFilterGroup } from './alerts_filter_group';
import { AlertsUtilityBar } from './alerts_utility_bar';
@ -104,6 +107,8 @@ export const AlertsTableComponent: React.FC<AlertsTableComponentProps> = ({
const [, dispatchToaster] = useStateToaster();
const { addWarning } = useAppToasts();
const { initializeTimeline, setSelectAll } = useManageTimeline();
// TODO: Once we are past experimental phase this code should be removed
const ruleRegistryEnabled = useIsExperimentalFeatureEnabled('ruleRegistryEnabled');
const getGlobalQuery = useCallback(
(customFilters: Filter[]) => {
@ -236,7 +241,11 @@ export const AlertsTableComponent: React.FC<AlertsTableComponentProps> = ({
refetchQuery: inputsModel.Refetch,
{ status, selectedStatus }: UpdateAlertsStatusProps
) => {
const currentStatusFilter = buildAlertStatusFilter(status);
// TODO: Once we are past experimental phase this code should be removed
const currentStatusFilter = ruleRegistryEnabled
? buildAlertStatusFilterRuleRegistry(status)
: buildAlertStatusFilter(status);
await updateAlertStatusAction({
query: showClearSelectionAction
? getGlobalQuery(currentStatusFilter)?.filterQuery
@ -258,6 +267,7 @@ export const AlertsTableComponent: React.FC<AlertsTableComponentProps> = ({
showClearSelectionAction,
onAlertStatusUpdateSuccess,
onAlertStatusUpdateFailure,
ruleRegistryEnabled,
]
);
@ -301,18 +311,28 @@ export const AlertsTableComponent: React.FC<AlertsTableComponentProps> = ({
);
const defaultFiltersMemo = useMemo(() => {
// TODO: Once we are past experimental phase this code should be removed
const alertStatusFilter = ruleRegistryEnabled
? buildAlertStatusFilterRuleRegistry(filterGroup)
: buildAlertStatusFilter(filterGroup);
if (isEmpty(defaultFilters)) {
return buildAlertStatusFilter(filterGroup);
return alertStatusFilter;
} else if (defaultFilters != null && !isEmpty(defaultFilters)) {
return [...defaultFilters, ...buildAlertStatusFilter(filterGroup)];
return [...defaultFilters, ...alertStatusFilter];
}
}, [defaultFilters, filterGroup]);
}, [defaultFilters, filterGroup, ruleRegistryEnabled]);
const { filterManager } = useKibana().services.data.query;
// TODO: Once we are past experimental phase this code should be removed
const defaultTimelineModel = ruleRegistryEnabled
? alertsDefaultModelRuleRegistry
: alertsDefaultModel;
useEffect(() => {
initializeTimeline({
defaultModel: {
...alertsDefaultModel,
...defaultTimelineModel,
columns,
},
documentType: i18n.ALERTS_DOCUMENT_TYPE,
@ -344,7 +364,7 @@ export const AlertsTableComponent: React.FC<AlertsTableComponentProps> = ({
return (
<StatefulEventsViewer
pageFilters={defaultFiltersMemo}
defaultModel={alertsDefaultModel}
defaultModel={defaultTimelineModel}
end={to}
headerFilterGroup={headerFilterGroup}
id={timelineId}

31
x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.test.tsx
View file

@ -4,21 +4,16 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React from 'react';
import { renderHook, act } from '@testing-library/react-hooks';
import { TestProvidersWithPrivileges } from '../../../../common/mock';
import { useSignalIndex, ReturnSignalIndex } from './use_signal_index';
import * as api from './api';
import { useAppToastsMock } from '../../../../common/hooks/use_app_toasts.mock';
import { useAppToasts } from '../../../../common/hooks/use_app_toasts';
import { UserPrivilegesProvider } from '../../../components/user_privileges';
jest.mock('./api');
jest.mock('../../../../common/hooks/use_app_toasts');
const Wrapper = ({ children }: { children?: React.ReactNode }) => (
<UserPrivilegesProvider>{children}</UserPrivilegesProvider>
);
describe('useSignalIndex', () => {
let appToastsMock: jest.Mocked<ReturnType<typeof useAppToastsMock.create>>;
@ -33,7 +28,9 @@ describe('useSignalIndex', () => {
await act(async () => {
const { result, waitForNextUpdate } = renderHook<void, ReturnSignalIndex>(
() => useSignalIndex(),
{ wrapper: Wrapper }
{
wrapper: TestProvidersWithPrivileges,
}
);
await waitForNextUpdate();
expect(result.current).toEqual({
@ -50,7 +47,9 @@ describe('useSignalIndex', () => {
await act(async () => {
const { result, waitForNextUpdate } = renderHook<void, ReturnSignalIndex>(
() => useSignalIndex(),
{ wrapper: Wrapper }
{
wrapper: TestProvidersWithPrivileges,
}
);
await waitForNextUpdate();
await waitForNextUpdate();
@ -69,7 +68,9 @@ describe('useSignalIndex', () => {
await act(async () => {
const { result, waitForNextUpdate } = renderHook<void, ReturnSignalIndex>(
() => useSignalIndex(),
{ wrapper: Wrapper }
{
wrapper: TestProvidersWithPrivileges,
}
);
await waitForNextUpdate();
await waitForNextUpdate();
@ -93,7 +94,9 @@ describe('useSignalIndex', () => {
await act(async () => {
const { result, waitForNextUpdate } = renderHook<void, ReturnSignalIndex>(
() => useSignalIndex(),
{ wrapper: Wrapper }
{
wrapper: TestProvidersWithPrivileges,
}
);
await waitForNextUpdate();
await waitForNextUpdate();
@ -114,7 +117,9 @@ describe('useSignalIndex', () => {
await act(async () => {
const { result, waitForNextUpdate } = renderHook<void, ReturnSignalIndex>(
() => useSignalIndex(),
{ wrapper: Wrapper }
{
wrapper: TestProvidersWithPrivileges,
}
);
await waitForNextUpdate();
await waitForNextUpdate();
@ -140,7 +145,9 @@ describe('useSignalIndex', () => {
await act(async () => {
const { result, waitForNextUpdate } = renderHook<void, ReturnSignalIndex>(
() => useSignalIndex(),
{ wrapper: Wrapper }
{
wrapper: TestProvidersWithPrivileges,
}
);
await waitForNextUpdate();
await waitForNextUpdate();

13
x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.tsx
View file

@ -6,8 +6,10 @@
*/
import { useEffect, useState } from 'react';
import { DEFAULT_ALERTS_INDEX } from '../../../../../common/constants';
import { useAppToasts } from '../../../../common/hooks/use_app_toasts';
import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features';
import { createSignalIndex, getSignalIndex } from './api';
import * as i18n from './translations';
import { isSecurityAppError } from '../../../../common/utils/api';
@ -38,6 +40,8 @@ export const useSignalIndex = (): ReturnSignalIndex => {
});
const { addError } = useAppToasts();
const { hasIndexRead } = useAlertsPrivileges();
// TODO: Once we are past experimental phase this code should be removed
const ruleRegistryEnabled = useIsExperimentalFeatureEnabled('ruleRegistryEnabled');
useEffect(() => {
let isSubscribed = true;
@ -48,10 +52,15 @@ export const useSignalIndex = (): ReturnSignalIndex => {
setLoading(true);
const signal = await getSignalIndex({ signal: abortCtrl.signal });
// TODO: Once we are past experimental phase we can update `getSignalIndex` to return the space-aware DEFAULT_ALERTS_INDEX
const signalIndices = ruleRegistryEnabled
? `${DEFAULT_ALERTS_INDEX},${signal.name}`
: signal.name;
if (isSubscribed && signal != null) {
setSignalIndex({
signalIndexExists: true,
signalIndexName: signal.name,
signalIndexName: signalIndices,
signalIndexMappingOutdated: signal.index_mapping_outdated,
createDeSignalIndex: createIndex,
});
@ -115,7 +124,7 @@ export const useSignalIndex = (): ReturnSignalIndex => {
isSubscribed = false;
abortCtrl.abort();
};
}, [addError, hasIndexRead]);
}, [addError, hasIndexRead, ruleRegistryEnabled]);
return { loading, ...signalIndex };
};

16
x-pack/plugins/security_solution/public/detections/pages/detection_engine/detection_engine.tsx
View file

@ -11,6 +11,7 @@ import { noop } from 'lodash/fp';
import React, { useCallback, useMemo, useRef, useState } from 'react';
import { useDispatch } from 'react-redux';
import { useHistory } from 'react-router-dom';
import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features';
import { useDeepEqualSelector, useShallowEqualSelector } from '../../../common/hooks/use_selector';
import { SecurityPageName } from '../../../app/types';
@ -51,6 +52,7 @@ import { timelineSelectors } from '../../../timelines/store/timeline';
import { timelineDefaults } from '../../../timelines/store/timeline/defaults';
import {
buildShowBuildingBlockFilter,
buildShowBuildingBlockFilterRuleRegistry,
buildThreatMatchFilter,
} from '../../components/alerts_table/default_config';
import { useSourcererScope } from '../../../common/containers/sourcerer';
@ -81,6 +83,8 @@ const DetectionEnginePageComponent = () => {
const getGlobalQuerySelector = useMemo(() => inputsSelectors.globalQuerySelector(), []);
const query = useDeepEqualSelector(getGlobalQuerySelector);
const filters = useDeepEqualSelector(getGlobalFiltersQuerySelector);
// TODO: Once we are past experimental phase this code should be removed
const ruleRegistryEnabled = useIsExperimentalFeatureEnabled('ruleRegistryEnabled');
const { to, from, deleteQuery, setQuery } = useGlobalTime();
const { globalFullScreen } = useGlobalFullScreen();
@ -134,19 +138,23 @@ const DetectionEnginePageComponent = () => {
const alertsHistogramDefaultFilters = useMemo(
() => [
...filters,
...buildShowBuildingBlockFilter(showBuildingBlockAlerts),
...(ruleRegistryEnabled
? buildShowBuildingBlockFilterRuleRegistry(showBuildingBlockAlerts) // TODO: Once we are past experimental phase this code should be removed
: buildShowBuildingBlockFilter(showBuildingBlockAlerts)),
...buildThreatMatchFilter(showOnlyThreatIndicatorAlerts),
],
[filters, showBuildingBlockAlerts, showOnlyThreatIndicatorAlerts]
[filters, ruleRegistryEnabled, showBuildingBlockAlerts, showOnlyThreatIndicatorAlerts]
);
// AlertsTable manages global filters itself, so not including `filters`
const alertsTableDefaultFilters = useMemo(
() => [
...buildShowBuildingBlockFilter(showBuildingBlockAlerts),
...(ruleRegistryEnabled
? buildShowBuildingBlockFilterRuleRegistry(showBuildingBlockAlerts) // TODO: Once we are past experimental phase this code should be removed
: buildShowBuildingBlockFilter(showBuildingBlockAlerts)),
...buildThreatMatchFilter(showOnlyThreatIndicatorAlerts),
],
[showBuildingBlockAlerts, showOnlyThreatIndicatorAlerts]
[ruleRegistryEnabled, showBuildingBlockAlerts, showOnlyThreatIndicatorAlerts]
);
const onShowBuildingBlockAlertsChangedCallback = useCallback(

11
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx
View file

@ -36,6 +36,7 @@ import {
useDeepEqualSelector,
useShallowEqualSelector,
} from '../../../../../common/hooks/use_selector';
import { useIsExperimentalFeatureEnabled } from '../../../../../common/hooks/use_experimental_features';
import { useKibana } from '../../../../../common/lib/kibana';
import { TimelineId } from '../../../../../../common/types/timeline';
import { UpdateDateRange } from '../../../../../common/components/charts/common';
@ -64,6 +65,7 @@ import { StepScheduleRule } from '../../../../components/rules/step_schedule_rul
import {
buildAlertsRuleIdFilter,
buildShowBuildingBlockFilter,
buildShowBuildingBlockFilterRuleRegistry,
buildThreatMatchFilter,
} from '../../../../components/alerts_table/default_config';
import { RuleSwitch } from '../../../../components/rules/rule_switch';
@ -222,6 +224,9 @@ const RuleDetailsPageComponent = () => {
const { formatUrl } = useFormatUrl(SecurityPageName.detections);
const { globalFullScreen } = useGlobalFullScreen();
// TODO: Once we are past experimental phase this code should be removed
const ruleRegistryEnabled = useIsExperimentalFeatureEnabled('ruleRegistryEnabled');
// TODO: Refactor license check + hasMlAdminPermissions to common check
const hasMlPermissions = hasMlLicense(mlCapabilities) && hasMlAdminPermissions(mlCapabilities);
const {
@ -307,10 +312,12 @@ const RuleDetailsPageComponent = () => {
const alertDefaultFilters = useMemo(
() => [
...buildAlertsRuleIdFilter(ruleId),
...buildShowBuildingBlockFilter(showBuildingBlockAlerts),
...(ruleRegistryEnabled
? buildShowBuildingBlockFilterRuleRegistry(showBuildingBlockAlerts) // TODO: Once we are past experimental phase this code should be removed
: buildShowBuildingBlockFilter(showBuildingBlockAlerts)),
...buildThreatMatchFilter(showOnlyThreatIndicatorAlerts),
],
[ruleId, showBuildingBlockAlerts, showOnlyThreatIndicatorAlerts]
[ruleId, ruleRegistryEnabled, showBuildingBlockAlerts, showOnlyThreatIndicatorAlerts]
);
const alertMergedFilters = useMemo(() => [...alertDefaultFilters, ...filters], [

18
x-pack/plugins/security_solution/public/plugin.tsx
View file

@ -44,6 +44,7 @@ import {
APP_PATH,
DEFAULT_INDEX_KEY,
DETECTION_ENGINE_INDEX_URL,
DEFAULT_ALERTS_INDEX,
} from '../common/constants';
import { SecurityPageName } from './app/types';
@ -446,6 +447,9 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
*/
private async store(coreStart: CoreStart, startPlugins: StartPlugins): Promise<SecurityAppStore> {
if (!this._store) {
const experimentalFeatures = parseExperimentalConfigValue(
this.config.enableExperimental || []
);
const defaultIndicesName = coreStart.uiSettings.get(DEFAULT_INDEX_KEY);
const [
{ createStore, createInitialState },
@ -474,9 +478,15 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
let signal: { name: string | null } = { name: null };
try {
signal = await coreStart.http.fetch(DETECTION_ENGINE_INDEX_URL, {
method: 'GET',
});
// TODO: Once we are past experimental phase this code should be removed
// TODO: This currently prevents TGrid from refreshing
if (experimentalFeatures.ruleRegistryEnabled) {
signal = { name: DEFAULT_ALERTS_INDEX };
} else {
signal = await coreStart.http.fetch(DETECTION_ENGINE_INDEX_URL, {
method: 'GET',
});
}
} catch {
signal = { name: null };
}
@ -514,7 +524,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
kibanaIndexPatterns,
configIndexPatterns: configIndexPatterns.indicesExist,
signalIndexName: signal.name,
enableExperimental: parseExperimentalConfigValue(this.config.enableExperimental || []),
enableExperimental: experimentalFeatures,
}
),
{

12
x-pack/plugins/security_solution/public/timelines/containers/helpers.ts
View file

@ -5,6 +5,7 @@
* 2.0.
*/
import { DEFAULT_ALERTS_INDEX } from '../../../common/constants';
import { TimelineId } from '../../../common/types/timeline';
export const detectionsTimelineIds = [
@ -12,7 +13,14 @@ export const detectionsTimelineIds = [
TimelineId.detectionsRulesDetailsPage,
];
export const skipQueryForDetectionsPage = (id: string, defaultIndex: string[]) =>
// TODO: Once we are past experimental phase `useRuleRegistry` should be removed
export const skipQueryForDetectionsPage = (
id: string,
defaultIndex: string[],
useRuleRegistry = false
) =>
id != null &&
detectionsTimelineIds.some((timelineId) => timelineId === id) &&
!defaultIndex.some((di) => di.toLowerCase().startsWith('.siem-signals'));
!defaultIndex.some((di) =>
di.toLowerCase().startsWith(useRuleRegistry ? DEFAULT_ALERTS_INDEX : '.siem-signals')
);

5
x-pack/plugins/security_solution/public/timelines/containers/index.test.tsx
View file

@ -9,6 +9,7 @@ import { renderHook, act } from '@testing-library/react-hooks';
import { initSortDefault, TimelineArgs, useTimelineEvents, UseTimelineEventsProps } from '.';
import { SecurityPageName } from '../../../common/constants';
import { TimelineId } from '../../../common/types/timeline';
import { useIsExperimentalFeatureEnabled } from '../../common/hooks/use_experimental_features';
import { mockTimelineData } from '../../common/mock';
import { useRouteSpy } from '../../common/utils/route/use_route_spy';
@ -26,6 +27,9 @@ const mockEvents = mockTimelineData.filter((i, index) => index <= 11);
const mockSearch = jest.fn();
jest.mock('../../common/hooks/use_experimental_features');
const useIsExperimentalFeatureEnabledMock = useIsExperimentalFeatureEnabled as jest.Mock;
jest.mock('../../common/lib/kibana', () => ({
useToasts: jest.fn().mockReturnValue({
addError: jest.fn(),
@ -93,6 +97,7 @@ mockUseRouteSpy.mockReturnValue([
]);
describe('useTimelineEvents', () => {
useIsExperimentalFeatureEnabledMock.mockReturnValue(false);
beforeEach(() => {
mockSearch.mockReset();
});

15
x-pack/plugins/security_solution/public/timelines/containers/index.tsx
View file

@ -13,6 +13,7 @@ import { Subscription } from 'rxjs';
import { ESQuery } from '../../../common/typed_json';
import { isCompleteResponse, isErrorResponse } from '../../../../../../src/plugins/data/public';
import { useIsExperimentalFeatureEnabled } from '../../common/hooks/use_experimental_features';
import { inputsModel, KueryFilterQueryKind } from '../../common/store';
import { useKibana } from '../../common/lib/kibana';
import { createFilter } from '../../common/containers/helpers';
@ -197,6 +198,9 @@ export const useTimelineEvents = ({
});
const { addError, addWarning } = useAppToasts();
// TODO: Once we are past experimental phase this code should be removed
const ruleRegistryEnabled = useIsExperimentalFeatureEnabled('ruleRegistryEnabled');
const timelineSearch = useCallback(
(request: TimelineRequest<typeof language> | null) => {
if (request == null || pageName === '' || skip) {
@ -305,7 +309,10 @@ export const useTimelineEvents = ({
);
useEffect(() => {
if (skipQueryForDetectionsPage(id, indexNames) || indexNames.length === 0) {
if (
skipQueryForDetectionsPage(id, indexNames, ruleRegistryEnabled) ||
indexNames.length === 0
) {
return;
}
@ -364,7 +371,10 @@ export const useTimelineEvents = ({
activeTimeline.setActivePage(newActivePage);
}
}
if (!skipQueryForDetectionsPage(id, indexNames) && !deepEqual(prevRequest, currentRequest)) {
if (
!skipQueryForDetectionsPage(id, indexNames, ruleRegistryEnabled) &&
!deepEqual(prevRequest, currentRequest)
) {
return currentRequest;
}
return prevRequest;
@ -380,6 +390,7 @@ export const useTimelineEvents = ({
id,
language,
limit,
ruleRegistryEnabled,
startDate,
sort,
fields,

76
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/__mocks__/rule_type.ts Normal file
View file

@ -0,0 +1,76 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { of } from 'rxjs';
import { v4 } from 'uuid';
import { Logger } from 'kibana/server';
import { elasticsearchServiceMock } from 'src/core/server/mocks';
import type { RuleDataClient } from '../../../../../../rule_registry/server';
import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../../../alerting/server';
import { ConfigType } from '../../../../config';
export const createRuleTypeMocks = () => {
/* eslint-disable @typescript-eslint/no-explicit-any */
let alertExecutor: (...args: any[]) => Promise<any>;
const mockedConfig$ = of({} as ConfigType);
const loggerMock = ({
debug: jest.fn(),
info: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
} as unknown) as Logger;
const alerting = {
registerType: ({ executor }) => {
alertExecutor = executor;
},
} as AlertingPluginSetupContract;
const scheduleActions = jest.fn();
const services = {
scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(),
alertInstanceFactory: jest.fn(() => ({ scheduleActions })),
findAlerts: jest.fn(), // TODO: does this stay?
alertWithPersistence: jest.fn(),
logger: loggerMock,
};
return {
dependencies: {
alerting,
config$: mockedConfig$,
logger: loggerMock,
ruleDataClient: ({
getReader: () => {
return {
search: jest.fn(),
};
},
getWriter: () => {
return {
bulk: jest.fn(),
};
},
} as unknown) as RuleDataClient,
},
services,
scheduleActions,
executor: async ({ params }: { params: Record<string, unknown> }) => {
return alertExecutor({
services,
params,
alertId: v4(),
startedAt: new Date(),
});
},
};
};

61
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/__mocks__/threshold.ts Normal file
View file

@ -0,0 +1,61 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { sampleDocNoSortId } from '../../signals/__mocks__/es_results';
export const mockThresholdResults = {
rawResponse: {
body: {
is_partial: false,
is_running: false,
took: 527,
timed_out: false,
hits: {
total: {
value: 0,
relation: 'eq',
},
hits: [],
},
aggregations: {
'threshold_0:source.ip': {
buckets: [
{
key: '127.0.0.1',
doc_count: 5,
'threshold_1:host.name': {
buckets: [
{
key: 'tardigrade',
doc_count: 3,
top_threshold_hits: {
hits: {
total: {
value: 1,
relation: 'eq',
},
hits: [
{
...sampleDocNoSortId(),
'host.name': 'tardigrade',
},
],
},
},
cardinality_count: {
value: 3,
},
},
],
},
},
],
},
},
},
},
};

92
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/eql.test.ts Normal file
View file

@ -0,0 +1,92 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks';
import { sequenceResponse } from '../../../search_strategy/timeline/eql/__mocks__';
import { createEqlAlertType } from './eql';
import { createRuleTypeMocks } from './__mocks__/rule_type';
describe('EQL alerts', () => {
it('does not send an alert when sequence not found', async () => {
const { services, dependencies, executor } = createRuleTypeMocks();
const eqlAlertType = createEqlAlertType(dependencies.ruleDataClient, dependencies.logger);
dependencies.alerting.registerType(eqlAlertType);
const params = {
eqlQuery: 'sequence by host.name↵[any where true]↵[any where true]↵[any where true]',
indexPatterns: ['*'],
};
services.scopedClusterClient.asCurrentUser.transport.request.mockReturnValue(
elasticsearchClientMock.createSuccessTransportRequestPromise({
hits: {
hits: [],
sequences: [],
events: [],
total: {
relation: 'eq',
value: 0,
},
},
took: 0,
timed_out: false,
_shards: {
failed: 0,
skipped: 0,
successful: 1,
total: 1,
},
})
);
await executor({ params });
expect(services.alertInstanceFactory).not.toBeCalled();
});
it('sends a properly formatted alert when sequence is found', async () => {
const { services, dependencies, executor } = createRuleTypeMocks();
const eqlAlertType = createEqlAlertType(dependencies.ruleDataClient, dependencies.logger);
dependencies.alerting.registerType(eqlAlertType);
const params = {
eqlQuery: 'sequence by host.name↵[any where true]↵[any where true]↵[any where true]',
indexPatterns: ['*'],
};
services.scopedClusterClient.asCurrentUser.transport.request.mockReturnValue(
elasticsearchClientMock.createSuccessTransportRequestPromise({
hits: sequenceResponse.rawResponse.body.hits,
took: 0,
timed_out: false,
_shards: {
failed: 0,
skipped: 0,
successful: 1,
total: 1,
},
})
);
await executor({ params });
expect(services.alertInstanceFactory).toBeCalled();
/*
expect(services.alertWithPersistence).toBeCalledWith(
expect.arrayContaining([
expect.objectContaining({
'event.kind': 'signal',
'kibana.rac.alert.building_block_type': 'default',
}),
])
);
*/
});
});

121
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/eql.ts Normal file
View file

@ -0,0 +1,121 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import moment from 'moment';
import v4 from 'uuid/v4';
import { ApiResponse } from '@elastic/elasticsearch';
import { schema } from '@kbn/config-schema';
import { Logger } from '@kbn/logging';
import {
RuleDataClient,
createPersistenceRuleTypeFactory,
} from '../../../../../rule_registry/server';
import { EQL_ALERT_TYPE_ID } from '../../../../common/constants';
import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_query_filter';
import { BaseSignalHit, EqlSignalSearchResponse } from '../signals/types';
export const createEqlAlertType = (ruleDataClient: RuleDataClient, logger: Logger) => {
const createPersistenceRuleType = createPersistenceRuleTypeFactory({
ruleDataClient,
logger,
});
return createPersistenceRuleType({
id: EQL_ALERT_TYPE_ID,
name: 'EQL Rule',
validate: {
params: schema.object({
eqlQuery: schema.string(),
indexPatterns: schema.arrayOf(schema.string()),
}),
},
actionGroups: [
{
id: 'default',
name: 'Default',
},
],
defaultActionGroupId: 'default',
actionVariables: {
context: [{ name: 'server', description: 'the server' }],
},
minimumLicenseRequired: 'basic',
producer: 'security-solution',
async executor({
startedAt,
services: { alertWithPersistence, findAlerts, scopedClusterClient },
params: { indexPatterns, eqlQuery },
}) {
const from = moment(startedAt).subtract(moment.duration(5, 'm')).toISOString(); // hardcoded 5-minute rule interval
const to = startedAt.toISOString();
const request = buildEqlSearchRequest(
eqlQuery,
indexPatterns,
from,
to,
10,
undefined,
[],
undefined
);
const { body: response } = (await scopedClusterClient.asCurrentUser.transport.request(
request
)) as ApiResponse<EqlSignalSearchResponse>;
const buildSignalFromEvent = (event: BaseSignalHit) => {
return {
...event,
'event.kind': 'signal',
'kibana.rac.alert.id': '???',
'kibana.rac.alert.uuid': v4(),
'@timestamp': new Date().toISOString(),
};
};
/* eslint-disable @typescript-eslint/no-explicit-any */
let alerts: any[] = [];
if (response.hits.sequences !== undefined) {
alerts = response.hits.sequences.reduce((allAlerts: any[], sequence) => {
let previousAlertUuid: string | undefined;
return [
...allAlerts,
...sequence.events.map((event, idx) => {
const alert = {
...buildSignalFromEvent(event),
'kibana.rac.alert.ancestors': previousAlertUuid != null ? [previousAlertUuid] : [],
'kibana.rac.alert.building_block_type': 'default',
'kibana.rac.alert.depth': idx,
};
previousAlertUuid = alert['kibana.rac.alert.uuid'];
return alert;
}),
];
}, []);
} else if (response.hits.events !== undefined) {
alerts = response.hits.events.map((event) => {
return buildSignalFromEvent(event);
}, []);
} else {
throw new Error(
'eql query response should have either `sequences` or `events` but had neither'
);
}
if (alerts.length > 0) {
alertWithPersistence(alerts).forEach((alert) => {
alert.scheduleActions('default', { server: 'server-test' });
});
}
return {
lastChecked: new Date(),
};
},
});
};

70
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/ml.ts Normal file
View file

@ -0,0 +1,70 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
/*
import { schema } from '@kbn/config-schema';
import { KibanaRequest, Logger } from 'src/core/server';
import { SavedObject } from 'src/core/types';
import { buildEsQuery, IIndexPattern } from '../../../../../../../src/plugins/data/common';
import { createPersistenceRuleTypeFactory } from '../../../../../rule_registry/server';
import { ML_ALERT_TYPE_ID } from '../../../../common/constants';
import { SecurityRuleRegistry } from '../../../plugin';
const createSecurityMlRuleType = createPersistenceRuleTypeFactory<SecurityRuleRegistry>();
import {
AlertInstanceContext,
AlertInstanceState,
AlertServices,
} from '../../../../../alerting/server';
import { ListClient } from '../../../../../lists/server';
import { isJobStarted } from '../../../../common/machine_learning/helpers';
import { ExceptionListItemSchema } from '../../../../common/shared_imports';
import { SetupPlugins } from '../../../plugin';
import { RefreshTypes } from '../types';
import { bulkCreateMlSignals } from '../signals/bulk_create_ml_signals';
import { filterEventsAgainstList } from '../signals/filters/filter_events_against_list';
import { findMlSignals } from '../signals/find_ml_signals';
import { BuildRuleMessage } from '../signals/rule_messages';
import { RuleStatusService } from '../signals/rule_status_service';
import { MachineLearningRuleAttributes } from '../signals/types';
import { createErrorsFromShard, createSearchAfterReturnType, mergeReturns } from '../signals/utils';
export const mlAlertType = createSecurityMlRuleType({
id: ML_ALERT_TYPE_ID,
name: 'Machine Learning Rule',
validate: {
params: schema.object({
indexPatterns: schema.arrayOf(schema.string()),
customQuery: schema.string(),
}),
},
actionGroups: [
{
id: 'default',
name: 'Default',
},
],
defaultActionGroupId: 'default',
actionVariables: {
context: [{ name: 'server', description: 'the server' }],
},
minimumLicenseRequired: 'basic',
producer: 'security-solution',
async executor({
services: { alertWithPersistence, findAlerts },
params: { indexPatterns, customQuery },
}) {
return {
lastChecked: new Date(),
};
},
});
*/

99
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.test.ts Normal file
View file

@ -0,0 +1,99 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { v4 } from 'uuid';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks';
import { sampleDocNoSortId } from '../signals/__mocks__/es_results';
import { createQueryAlertType } from './query';
import { createRuleTypeMocks } from './__mocks__/rule_type';
describe('Custom query alerts', () => {
it('does not send an alert when no events found', async () => {
const { services, dependencies, executor } = createRuleTypeMocks();
const queryAlertType = createQueryAlertType(dependencies.ruleDataClient, dependencies.logger);
dependencies.alerting.registerType(queryAlertType);
const params = {
customQuery: 'dne:42',
indexPatterns: ['*'],
};
services.scopedClusterClient.asCurrentUser.search.mockReturnValue(
elasticsearchClientMock.createSuccessTransportRequestPromise({
hits: {
hits: [],
sequences: [],
events: [],
total: {
relation: 'eq',
value: 0,
},
},
took: 0,
timed_out: false,
_shards: {
failed: 0,
skipped: 0,
successful: 1,
total: 1,
},
})
);
await executor({ params });
expect(services.alertInstanceFactory).not.toBeCalled();
});
it('sends a properly formatted alert when events are found', async () => {
const { services, dependencies, executor } = createRuleTypeMocks();
const queryAlertType = createQueryAlertType(dependencies.ruleDataClient, dependencies.logger);
dependencies.alerting.registerType(queryAlertType);
const params = {
customQuery: '*:*',
indexPatterns: ['*'],
};
services.scopedClusterClient.asCurrentUser.search.mockReturnValue(
elasticsearchClientMock.createSuccessTransportRequestPromise({
hits: {
hits: [sampleDocNoSortId(v4()), sampleDocNoSortId(v4()), sampleDocNoSortId(v4())],
total: {
relation: 'eq',
value: 3,
},
},
took: 0,
timed_out: false,
_shards: {
failed: 0,
skipped: 0,
successful: 1,
total: 1,
},
})
);
await executor({ params });
expect(services.alertInstanceFactory).toBeCalled();
/*
expect(services.alertWithPersistence).toBeCalledWith(
expect.arrayContaining([
expect.objectContaining({
'event.kind': 'signal',
}),
])
);
*/
});
});

88
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.ts Normal file
View file

@ -0,0 +1,88 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { QueryContainer } from '@elastic/elasticsearch/api/types';
import { schema } from '@kbn/config-schema';
import { Logger } from '@kbn/logging';
import { ESSearchRequest } from 'typings/elasticsearch';
import { buildEsQuery, IIndexPattern } from '../../../../../../../src/plugins/data/common';
import {
RuleDataClient,
createPersistenceRuleTypeFactory,
} from '../../../../../rule_registry/server';
import { CUSTOM_ALERT_TYPE_ID } from '../../../../common/constants';
export const createQueryAlertType = (ruleDataClient: RuleDataClient, logger: Logger) => {
const createPersistenceRuleType = createPersistenceRuleTypeFactory({
ruleDataClient,
logger,
});
return createPersistenceRuleType({
id: CUSTOM_ALERT_TYPE_ID,
name: 'Custom Query Rule',
validate: {
params: schema.object({
indexPatterns: schema.arrayOf(schema.string()),
customQuery: schema.string(),
}),
},
actionGroups: [
{
id: 'default',
name: 'Default',
},
],
defaultActionGroupId: 'default',
actionVariables: {
context: [{ name: 'server', description: 'the server' }],
},
minimumLicenseRequired: 'basic',
producer: 'security-solution',
async executor({
services: { alertWithPersistence, findAlerts },
params: { indexPatterns, customQuery },
}) {
try {
const indexPattern: IIndexPattern = {
fields: [],
title: indexPatterns.join(),
};
// TODO: kql or lucene?
const esQuery = buildEsQuery(
indexPattern,
{ query: customQuery, language: 'kuery' },
[]
) as QueryContainer;
const query: ESSearchRequest = {
body: {
query: esQuery,
fields: ['*'],
sort: {
'@timestamp': 'asc' as const,
},
},
};
const alerts = await findAlerts(query);
// console.log('alerts', alerts);
alertWithPersistence(alerts).forEach((alert) => {
alert.scheduleActions('default', { server: 'server-test' });
});
return {
lastChecked: new Date(),
};
} catch (error) {
logger.error(error);
}
},
});
};

34
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/scripts/create_reference_rule_eql.sh Executable file
View file

@ -0,0 +1,34 @@
#!/bin/sh
#
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License
# 2.0; you may not use this file except in compliance with the Elastic License
# 2.0.
#
curl -X POST http://localhost:5601/${BASE_PATH}/api/alerts/alert \
-u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
-H 'kbn-xsrf: true' \
-H 'Content-Type: application/json' \
--verbose \
-d '
{
"params":{
"indexPatterns": ["*"],
"eqlQuery": "sequence by host.name↵[any where true]↵[any where true]↵[any where true]"
},
"consumer":"alerts",
"alertTypeId":"siem.eqlRule",
"schedule":{
"interval":"1m"
},
"actions":[],
"tags":[
"eql",
"persistence"
],
"notifyWhen":"onActionGroupChange",
"name":"Basic EQL rule"
}'

34
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/scripts/create_reference_rule_query.sh Executable file
View file

@ -0,0 +1,34 @@
#!/bin/sh
#
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License
# 2.0; you may not use this file except in compliance with the Elastic License
# 2.0.
#
curl -X POST http://localhost:5601/${BASE_PATH}/api/alerts/alert \
-u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
-H 'kbn-xsrf: true' \
-H 'Content-Type: application/json' \
--verbose \
-d '
{
"params":{
"indexPatterns": ["*"],
"customQuery": "*:*"
},
"consumer":"alerts",
"alertTypeId":"siem.customRule",
"schedule":{
"interval":"1m"
},
"actions":[],
"tags":[
"custom",
"persistence"
],
"notifyWhen":"onActionGroupChange",
"name":"Basic custom query rule"
}'

37
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/scripts/create_reference_rule_threshold.sh Executable file
View file

@ -0,0 +1,37 @@
#!/bin/sh
#
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License
# 2.0; you may not use this file except in compliance with the Elastic License
# 2.0.
#
curl -X POST http://localhost:5601/${BASE_PATH}/api/alerts/alert \
-u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
-H 'kbn-xsrf: true' \
-H 'Content-Type: application/json' \
--verbose \
-d '
{
"params":{
"indexPatterns": ["*"],
"customQuery": "*:*",
"thresholdFields": ["source.ip", "destination.ip"],
"thresholdValue": 50,
"thresholdCardinality": []
},
"consumer":"alerts",
"alertTypeId":"siem.thresholdRule",
"schedule":{
"interval":"1m"
},
"actions":[],
"tags":[
"persistence",
"threshold"
],
"notifyWhen":"onActionGroupChange",
"name":"Basic Threshold rule"
}'

132
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/threshold.test.ts Normal file
View file

@ -0,0 +1,132 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks';
import { createRuleTypeMocks } from './__mocks__/rule_type';
import { mockThresholdResults } from './__mocks__/threshold';
import { createThresholdAlertType } from './threshold';
describe('Threshold alerts', () => {
it('does not send an alert when threshold is not met', async () => {
const { services, dependencies, executor } = createRuleTypeMocks();
const thresholdAlertType = createThresholdAlertType(
dependencies.ruleDataClient,
dependencies.logger
);
dependencies.alerting.registerType(thresholdAlertType);
const params = {
indexPatterns: ['*'],
customQuery: '*:*',
thresholdFields: ['source.ip', 'host.name'],
thresholdValue: 4,
};
services.scopedClusterClient.asCurrentUser.search.mockReturnValue(
elasticsearchClientMock.createSuccessTransportRequestPromise({
hits: {
hits: [],
sequences: [],
events: [],
total: {
relation: 'eq',
value: 0,
},
},
aggregations: {
'threshold_0:source.ip': {
buckets: [],
},
},
took: 0,
timed_out: false,
_shards: {
failed: 0,
skipped: 0,
successful: 1,
total: 1,
},
})
);
await executor({ params });
expect(services.alertInstanceFactory).not.toBeCalled();
});
it('sends a properly formatted alert when threshold is met', async () => {
const { services, dependencies, executor } = createRuleTypeMocks();
const thresholdAlertType = createThresholdAlertType(
dependencies.ruleDataClient,
dependencies.logger
);
dependencies.alerting.registerType(thresholdAlertType);
const params = {
indexPatterns: ['*'],
customQuery: '*:*',
thresholdFields: ['source.ip', 'host.name'],
thresholdValue: 4,
};
services.scopedClusterClient.asCurrentUser.search
.mockReturnValueOnce(
elasticsearchClientMock.createSuccessTransportRequestPromise({
hits: {
hits: [],
total: {
relation: 'eq',
value: 0,
},
},
took: 0,
timed_out: false,
_shards: {
failed: 0,
skipped: 0,
successful: 1,
total: 1,
},
})
)
.mockReturnValueOnce(
elasticsearchClientMock.createSuccessTransportRequestPromise({
hits: {
hits: [],
total: {
relation: 'eq',
value: 0,
},
},
aggregations: mockThresholdResults.rawResponse.body.aggregations,
took: 0,
timed_out: false,
_shards: {
failed: 0,
skipped: 0,
successful: 1,
total: 1,
},
})
);
await executor({ params });
expect(services.alertInstanceFactory).toBeCalled();
/*
expect(services.alertWithPersistence).toBeCalledWith(
expect.arrayContaining([
expect.objectContaining({
'event.kind': 'signal',
}),
])
);
*/
});
});

206
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/threshold.ts Normal file
View file

@ -0,0 +1,206 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import moment from 'moment';
import v4 from 'uuid/v4';
import { schema } from '@kbn/config-schema';
import { Logger } from '@kbn/logging';
import { AlertServices } from '../../../../../alerting/server';
import {
RuleDataClient,
createPersistenceRuleTypeFactory,
} from '../../../../../rule_registry/server';
import { THRESHOLD_ALERT_TYPE_ID } from '../../../../common/constants';
import { SignalSearchResponse, ThresholdSignalHistory } from '../signals/types';
import {
findThresholdSignals,
getThresholdBucketFilters,
getThresholdSignalHistory,
transformThresholdResultsToEcs,
} from '../signals/threshold';
import { getFilter } from '../signals/get_filter';
import { BuildRuleMessage } from '../signals/rule_messages';
interface RuleParams {
indexPatterns: string[];
customQuery: string;
thresholdFields: string[];
thresholdValue: number;
thresholdCardinality: Array<{
field: string;
value: number;
}>;
}
interface BulkCreateThresholdSignalParams {
results: SignalSearchResponse;
ruleParams: RuleParams;
services: AlertServices & { logger: Logger };
inputIndexPattern: string[];
ruleId: string;
startedAt: Date;
from: Date;
thresholdSignalHistory: ThresholdSignalHistory;
buildRuleMessage: BuildRuleMessage;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const formatThresholdSignals = (params: BulkCreateThresholdSignalParams): any[] => {
const thresholdResults = params.results;
const threshold = {
field: params.ruleParams.thresholdFields,
value: params.ruleParams.thresholdValue,
};
const results = transformThresholdResultsToEcs(
thresholdResults,
params.ruleParams.indexPatterns.join(','),
params.startedAt,
params.from,
undefined,
params.services.logger,
threshold,
params.ruleId,
undefined,
params.thresholdSignalHistory
);
return results.hits.hits.map((hit) => {
return {
...hit,
'event.kind': 'signal',
'kibana.rac.alert.id': '???',
'kibana.rac.alert.uuid': v4(),
'@timestamp': new Date().toISOString(),
};
});
};
export const createThresholdAlertType = (ruleDataClient: RuleDataClient, logger: Logger) => {
const createPersistenceRuleType = createPersistenceRuleTypeFactory({
ruleDataClient,
logger,
});
return createPersistenceRuleType({
id: THRESHOLD_ALERT_TYPE_ID,
name: 'Threshold Rule',
validate: {
params: schema.object({
indexPatterns: schema.arrayOf(schema.string()),
customQuery: schema.string(),
thresholdFields: schema.arrayOf(schema.string()),
thresholdValue: schema.number(),
thresholdCardinality: schema.arrayOf(
schema.object({
field: schema.string(),
value: schema.number(),
})
),
}),
},
actionGroups: [
{
id: 'default',
name: 'Default',
},
],
defaultActionGroupId: 'default',
actionVariables: {
context: [{ name: 'server', description: 'the server' }],
},
minimumLicenseRequired: 'basic',
producer: 'security-solution',
async executor({ startedAt, services, params, alertId }) {
const fromDate = moment(startedAt).subtract(moment.duration(5, 'm')); // hardcoded 5-minute rule interval
const from = fromDate.toISOString();
const to = startedAt.toISOString();
// TODO: how to get the output index?
const outputIndex = ['.kibana-madi-8-alerts-security-solution-8.0.0-000001'];
const buildRuleMessage = (...messages: string[]) => messages.join();
const timestampOverride = undefined;
const {
thresholdSignalHistory,
searchErrors: previousSearchErrors,
} = await getThresholdSignalHistory({
indexPattern: outputIndex,
from,
to,
services: (services as unknown) as AlertServices,
logger,
ruleId: alertId,
bucketByFields: params.thresholdFields,
timestampOverride,
buildRuleMessage,
});
const bucketFilters = await getThresholdBucketFilters({
thresholdSignalHistory,
timestampOverride,
});
const esFilter = await getFilter({
type: 'threshold',
filters: bucketFilters,
language: 'kuery',
query: params.customQuery,
savedId: undefined,
services: (services as unknown) as AlertServices,
index: params.indexPatterns,
lists: [],
});
const {
searchResult: thresholdResults,
searchErrors,
searchDuration: thresholdSearchDuration,
} = await findThresholdSignals({
inputIndexPattern: params.indexPatterns,
from,
to,
services: (services as unknown) as AlertServices,
logger,
filter: esFilter,
threshold: {
field: params.thresholdFields,
value: params.thresholdValue,
cardinality: params.thresholdCardinality,
},
timestampOverride,
buildRuleMessage,
});
logger.info(`Threshold search took ${thresholdSearchDuration}ms`); // TODO: rule status service
const alerts = formatThresholdSignals({
results: thresholdResults,
ruleParams: params,
services: (services as unknown) as AlertServices & { logger: Logger },
inputIndexPattern: ['TODO'],
ruleId: alertId,
startedAt,
from: fromDate.toDate(),
thresholdSignalHistory,
buildRuleMessage,
});
const errors = searchErrors.concat(previousSearchErrors);
if (errors.length === 0) {
services.alertWithPersistence(alerts).forEach((alert) => {
alert.scheduleActions('default', { server: 'server-test' });
});
} else {
throw new Error(errors.join('\n'));
}
return {
lastChecked: new Date(),
};
},
});
};

16
x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts
View file

@ -6,15 +6,17 @@
*/
import { transformError, getIndexExists } from '@kbn/securitysolution-es-utils';
import { parseExperimentalConfigValue } from '../../../../../common/experimental_features';
import { ConfigType } from '../../../../config';
import type { SecuritySolutionPluginRouter } from '../../../../types';
import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants';
import { DEFAULT_ALERTS_INDEX, DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants';
import { buildSiemResponse } from '../utils';
import { SIGNALS_TEMPLATE_VERSION } from './get_signals_template';
import { getIndexVersion } from './get_index_version';
import { isOutdated } from '../../migrations/helpers';
export const readIndexRoute = (router: SecuritySolutionPluginRouter) => {
export const readIndexRoute = (router: SecuritySolutionPluginRouter, config: ConfigType) => {
router.get(
{
path: DETECTION_ENGINE_INDEX_URL,
@ -34,8 +36,16 @@ export const readIndexRoute = (router: SecuritySolutionPluginRouter) => {
return siemResponse.error({ statusCode: 404 });
}
// TODO: Once we are past experimental phase this code should be removed
const { ruleRegistryEnabled } = parseExperimentalConfigValue(config.enableExperimental);
if (ruleRegistryEnabled) {
return response.ok({
body: { name: DEFAULT_ALERTS_INDEX, index_mapping_outdated: false },
});
}
const index = siemClient.getSignalsIndex();
const indexExists = await getIndexExists(esClient, index);
const indexExists = ruleRegistryEnabled ? true : await getIndexExists(esClient, index);
if (indexExists) {
let mappingOutdated: boolean | null = null;

4
x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts
View file

@ -6,6 +6,7 @@
*/
import { transformError, getIndexExists } from '@kbn/securitysolution-es-utils';
import { RuleDataClient } from '../../../../../../rule_registry/server';
import { buildRouteValidation } from '../../../../utils/build_validation/route_validation';
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
import { SetupPlugins } from '../../../../plugin';
@ -24,7 +25,8 @@ import { convertCreateAPIToInternalSchema } from '../../schemas/rule_converters'
export const createRulesRoute = (
router: SecuritySolutionPluginRouter,
ml: SetupPlugins['ml']
ml: SetupPlugins['ml'],
ruleDataClient?: RuleDataClient | null
): void => {
router.post(
{

6
x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_route.ts
View file

@ -6,6 +6,7 @@
*/
import { transformError } from '@kbn/securitysolution-es-utils';
import { RuleDataClient } from '../../../../../../rule_registry/server';
import { queryRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/query_rules_type_dependents';
import {
queryRulesSchema,
@ -22,7 +23,10 @@ import { deleteNotifications } from '../../notifications/delete_notifications';
import { deleteRuleActionsSavedObject } from '../../rule_actions/delete_rule_actions_saved_object';
import { ruleStatusSavedObjectsClientFactory } from '../../signals/rule_status_saved_objects_client';
export const deleteRulesRoute = (router: SecuritySolutionPluginRouter) => {
export const deleteRulesRoute = (
router: SecuritySolutionPluginRouter,
ruleDataClient?: RuleDataClient | null
) => {
router.delete(
{
path: DETECTION_ENGINE_RULES_URL,

6
x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts
View file

@ -6,6 +6,7 @@
*/
import { transformError } from '@kbn/securitysolution-es-utils';
import { RuleDataClient } from '../../../../../../rule_registry/server';
import { findRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/find_rules_type_dependents';
import {
findRulesSchema,
@ -20,7 +21,10 @@ import { buildRouteValidation } from '../../../../utils/build_validation/route_v
import { transformFindAlerts } from './utils';
import { getBulkRuleActionsSavedObject } from '../../rule_actions/get_bulk_rule_actions_saved_object';
export const findRulesRoute = (router: SecuritySolutionPluginRouter) => {
export const findRulesRoute = (
router: SecuritySolutionPluginRouter,
ruleDataClient?: RuleDataClient | null
) => {
router.get(
{
path: `${DETECTION_ENGINE_RULES_URL}/_find`,

7
x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts
View file

@ -6,6 +6,7 @@
*/
import { transformError } from '@kbn/securitysolution-es-utils';
import { RuleDataClient } from '../../../../../../rule_registry/server';
import { RuleAlertAction } from '../../../../../common/detection_engine/types';
import { patchRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/patch_rules_type_dependents';
import { buildRouteValidation } from '../../../../utils/build_validation/route_validation';
@ -28,7 +29,11 @@ import { ruleStatusSavedObjectsClientFactory } from '../../signals/rule_status_s
import { readRules } from '../../rules/read_rules';
import { PartialFilter } from '../../types';
export const patchRulesRoute = (router: SecuritySolutionPluginRouter, ml: SetupPlugins['ml']) => {
export const patchRulesRoute = (
router: SecuritySolutionPluginRouter,
ml: SetupPlugins['ml'],
ruleDataClient?: RuleDataClient | null
) => {
router.patch(
{
path: DETECTION_ENGINE_RULES_URL,

6
x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/read_rules_route.ts
View file

@ -6,6 +6,7 @@
*/
import { transformError } from '@kbn/securitysolution-es-utils';
import { RuleDataClient } from '../../../../../../rule_registry/server';
import { queryRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/query_rules_type_dependents';
import {
queryRulesSchema,
@ -21,7 +22,10 @@ import { readRules } from '../../rules/read_rules';
import { getRuleActionsSavedObject } from '../../rule_actions/get_rule_actions_saved_object';
import { ruleStatusSavedObjectsClientFactory } from '../../signals/rule_status_saved_objects_client';
export const readRulesRoute = (router: SecuritySolutionPluginRouter) => {
export const readRulesRoute = (
router: SecuritySolutionPluginRouter,
ruleDataClient?: RuleDataClient | null
) => {
router.get(
{
path: DETECTION_ENGINE_RULES_URL,

7
x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts
View file

@ -6,6 +6,7 @@
*/
import { transformError } from '@kbn/securitysolution-es-utils';
import { RuleDataClient } from '../../../../../../rule_registry/server';
import { updateRulesSchema } from '../../../../../common/detection_engine/schemas/request';
import { updateRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/update_rules_type_dependents';
import type { SecuritySolutionPluginRouter } from '../../../../types';
@ -22,7 +23,11 @@ import { updateRulesNotifications } from '../../rules/update_rules_notifications
import { ruleStatusSavedObjectsClientFactory } from '../../signals/rule_status_saved_objects_client';
import { buildRouteValidation } from '../../../../utils/build_validation/route_validation';
export const updateRulesRoute = (router: SecuritySolutionPluginRouter, ml: SetupPlugins['ml']) => {
export const updateRulesRoute = (
router: SecuritySolutionPluginRouter,
ml: SetupPlugins['ml'],
ruleDataClient?: RuleDataClient | null
) => {
router.put(
{
path: DETECTION_ENGINE_RULES_URL,

4
x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/query_signals_route.test.ts
View file

@ -14,7 +14,7 @@ import {
getSignalsAggsAndQueryRequest,
getEmptySignalsResponse,
} from '../__mocks__/request_responses';
import { requestContextMock, serverMock, requestMock } from '../__mocks__';
import { requestContextMock, serverMock, requestMock, createMockConfig } from '../__mocks__';
import { querySignalsRoute } from './query_signals_route';
describe('query for signal', () => {
@ -27,7 +27,7 @@ describe('query for signal', () => {
clients.clusterClient.callAsCurrentUser.mockResolvedValue(getEmptySignalsResponse());
querySignalsRoute(server.router);
querySignalsRoute(server.router, createMockConfig());
});
describe('query and agg on signals index', () => {

14
x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/query_signals_route.ts
View file

@ -6,8 +6,13 @@
*/
import { transformError } from '@kbn/securitysolution-es-utils';
import { parseExperimentalConfigValue } from '../../../../../common/experimental_features';
import { ConfigType } from '../../../../config';
import type { SecuritySolutionPluginRouter } from '../../../../types';
import { DETECTION_ENGINE_QUERY_SIGNALS_URL } from '../../../../../common/constants';
import {
DEFAULT_ALERTS_INDEX,
DETECTION_ENGINE_QUERY_SIGNALS_URL,
} from '../../../../../common/constants';
import { buildSiemResponse } from '../utils';
import { buildRouteValidation } from '../../../../utils/build_validation/route_validation';
@ -16,7 +21,7 @@ import {
QuerySignalsSchemaDecoded,
} from '../../../../../common/detection_engine/schemas/request/query_signals_index_schema';
export const querySignalsRoute = (router: SecuritySolutionPluginRouter) => {
export const querySignalsRoute = (router: SecuritySolutionPluginRouter, config: ConfigType) => {
router.post(
{
path: DETECTION_ENGINE_QUERY_SIGNALS_URL,
@ -48,9 +53,12 @@ export const querySignalsRoute = (router: SecuritySolutionPluginRouter) => {
const clusterClient = context.core.elasticsearch.legacy.client;
const siemClient = context.securitySolution!.getAppClient();
// TODO: Once we are past experimental phase this code should be removed
const { ruleRegistryEnabled } = parseExperimentalConfigValue(config.enableExperimental);
try {
const result = await clusterClient.callAsCurrentUser('search', {
index: siemClient.getSignalsIndex(),
index: ruleRegistryEnabled ? DEFAULT_ALERTS_INDEX : siemClient.getSignalsIndex(),
body: { query, aggs, _source, track_total_hits, size },
ignoreUnavailable: true,
});

2
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts
View file

@ -138,7 +138,7 @@ export const findThresholdSignals = async ({
logger,
// @ts-expect-error refactor to pass type explicitly instead of unknown
filter,
pageSize: 1,
pageSize: 0,
sortOrder: 'desc',
buildRuleMessage,
});

111
x-pack/plugins/security_solution/server/plugin.ts
View file

@ -5,6 +5,7 @@
* 2.0.
*/
import { once } from 'lodash';
import { Observable } from 'rxjs';
import { i18n } from '@kbn/i18n';
import LRU from 'lru-cache';
@ -27,7 +28,17 @@ import {
PluginSetupContract as AlertingSetup,
PluginStartContract as AlertPluginStartContract,
} from '../../alerting/server';
import {
ECS_COMPONENT_TEMPLATE_NAME,
TECHNICAL_COMPONENT_TEMPLATE_NAME,
} from '../../rule_registry/common/assets';
import { SecurityPluginSetup as SecuritySetup, SecurityPluginStart } from '../../security/server';
import {
RuleDataClient,
RuleRegistryPluginSetupContract,
RuleRegistryPluginStartContract,
} from '../../rule_registry/server';
import { PluginSetupContract as FeaturesSetup } from '../../features/server';
import { MlPluginSetup as MlSetup } from '../../ml/server';
import { ListPluginSetup } from '../../lists/server';
@ -37,6 +48,9 @@ import { ILicense, LicensingPluginStart } from '../../licensing/server';
import { FleetStartContract } from '../../fleet/server';
import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server';
import { compose } from './lib/compose/kibana';
import { createQueryAlertType } from './lib/detection_engine/reference_rules/query';
import { createEqlAlertType } from './lib/detection_engine/reference_rules/eql';
import { createThresholdAlertType } from './lib/detection_engine/reference_rules/threshold';
import { initRoutes } from './routes';
import { isAlertExecutor } from './lib/detection_engine/signals/types';
import { signalRulesAlertType } from './lib/detection_engine/signals/signal_rule_alert_type';
@ -53,6 +67,8 @@ import {
SecurityPageName,
SIGNALS_ID,
NOTIFICATIONS_ID,
REFERENCE_RULE_ALERT_TYPE_ID,
REFERENCE_RULE_PERSISTENCE_ALERT_TYPE_ID,
} from '../common/constants';
import { registerEndpointRoutes } from './endpoint/routes/metadata';
import { registerLimitedConcurrencyRoutes } from './endpoint/routes/limited_concurrency';
@ -86,6 +102,7 @@ export interface SetupPlugins {
features: FeaturesSetup;
lists?: ListPluginSetup;
ml?: MlSetup;
ruleRegistry: RuleRegistryPluginSetupContract;
security?: SecuritySetup;
spaces?: SpacesSetup;
taskManager?: TaskManagerSetupContract;
@ -98,6 +115,7 @@ export interface StartPlugins {
data: DataPluginStart;
fleet?: FleetStartContract;
licensing: LicensingPluginStart;
ruleRegistry: RuleRegistryPluginStartContract;
taskManager?: TaskManagerStartContract;
telemetry?: TelemetryPluginStart;
security: SecurityPluginStart;
@ -133,6 +151,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
private readonly config: ConfigType;
private context: PluginInitializerContext;
private appClientFactory: AppClientFactory;
private setupPlugins?: SetupPlugins;
private readonly endpointAppContextService = new EndpointAppContextService();
private readonly telemetryEventsSender: TelemetryEventsSender;
@ -157,6 +176,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
public setup(core: CoreSetup<StartPlugins, PluginStart>, plugins: SetupPlugins) {
this.logger.debug('plugin setup');
this.setupPlugins = plugins;
const config = this.config;
const globalConfig = this.context.config.legacy.get();
@ -193,13 +213,75 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
config,
});
// TODO: Once we are past experimental phase this check can be removed along with legacy registration of rules
let ruleDataClient: RuleDataClient | null = null;
if (experimentalFeatures.ruleRegistryEnabled) {
const { ruleDataService } = plugins.ruleRegistry;
const start = () => core.getStartServices().then(([coreStart]) => coreStart);
const ready = once(async () => {
const componentTemplateName = ruleDataService.getFullAssetName(
'security-solution-mappings'
);
if (!ruleDataService.isWriteEnabled()) {
return;
}
await ruleDataService.createOrUpdateComponentTemplate({
name: componentTemplateName,
body: {
template: {
settings: {
number_of_shards: 1,
},
mappings: {}, // TODO: Add mappings here via `mappingFromFieldMap()`
},
},
});
await ruleDataService.createOrUpdateIndexTemplate({
name: ruleDataService.getFullAssetName('security-solution-index-template'),
body: {
index_patterns: [ruleDataService.getFullAssetName('security-solution*')],
composed_of: [
ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME),
ruleDataService.getFullAssetName(ECS_COMPONENT_TEMPLATE_NAME),
componentTemplateName,
],
},
});
});
ready().catch((err) => {
this.logger!.error(err);
});
ruleDataClient = new RuleDataClient({
alias: plugins.ruleRegistry.ruleDataService.getFullAssetName('security-solution'),
getClusterClient: async () => {
const coreStart = await start();
return coreStart.elasticsearch.client.asInternalUser;
},
ready,
});
// Register reference rule types via rule-registry
this.setupPlugins.alerting.registerType(createQueryAlertType(ruleDataClient, this.logger));
this.setupPlugins.alerting.registerType(createEqlAlertType(ruleDataClient, this.logger));
this.setupPlugins.alerting.registerType(
createThresholdAlertType(ruleDataClient, this.logger)
);
}
// TO DO We need to get the endpoint routes inside of initRoutes
initRoutes(
router,
config,
plugins.encryptedSavedObjects?.canEncrypt === true,
plugins.security,
plugins.ml
plugins.ml,
ruleDataClient
);
registerEndpointRoutes(router, endpointContext);
registerLimitedConcurrencyRoutes(core);
@ -208,6 +290,16 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
registerTrustedAppsRoutes(router, endpointContext);
registerHostIsolationRoutes(router, endpointContext);
const referenceRuleTypes = [
REFERENCE_RULE_ALERT_TYPE_ID,
REFERENCE_RULE_PERSISTENCE_ALERT_TYPE_ID,
];
const ruleTypes = [
SIGNALS_ID,
NOTIFICATIONS_ID,
...(experimentalFeatures.ruleRegistryEnabled ? referenceRuleTypes : []),
];
plugins.features.registerKibanaFeature({
id: SERVER_APP_ID,
name: i18n.translate('xpack.securitySolution.featureRegistry.linkSecuritySolutionTitle', {
@ -220,7 +312,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
management: {
insightsAndAlerting: ['triggersActions'],
},
alerting: [SIGNALS_ID, NOTIFICATIONS_ID],
alerting: ruleTypes,
privileges: {
all: {
app: [...securitySubPlugins, 'kibana'],
@ -238,10 +330,10 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
},
alerting: {
rule: {
all: [SIGNALS_ID, NOTIFICATIONS_ID],
all: ruleTypes,
},
alert: {
all: [SIGNALS_ID, NOTIFICATIONS_ID],
all: ruleTypes,
},
},
management: {
@ -265,10 +357,10 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
},
alerting: {
rule: {
read: [SIGNALS_ID, NOTIFICATIONS_ID],
read: ruleTypes,
},
alert: {
read: [SIGNALS_ID, NOTIFICATIONS_ID],
read: ruleTypes,
},
},
management: {
@ -279,7 +371,8 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
},
});
if (plugins.alerting != null) {
// Continue to register legacy rules against alerting client exposed through rule-registry
if (this.setupPlugins.alerting != null) {
const signalRuleType = signalRulesAlertType({
logger: this.logger,
eventsTelemetry: this.telemetryEventsSender,
@ -292,11 +385,11 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
});
if (isAlertExecutor(signalRuleType)) {
plugins.alerting.registerType(signalRuleType);
this.setupPlugins.alerting.registerType(signalRuleType);
}
if (isNotificationAlertExecutor(ruleNotificationType)) {
plugins.alerting.registerType(ruleNotificationType);
this.setupPlugins.alerting.registerType(ruleNotificationType);
}
}

23
x-pack/plugins/security_solution/server/routes/index.ts
View file

@ -5,6 +5,8 @@
* 2.0.
*/
import { RuleDataClient } from '../../../rule_registry/server';
import { SecuritySolutionPluginRouter } from '../types';
import { createRulesRoute } from '../lib/detection_engine/routes/rules/create_rules_route';
@ -59,16 +61,19 @@ export const initRoutes = (
config: ConfigType,
hasEncryptionKey: boolean,
security: SetupPlugins['security'],
ml: SetupPlugins['ml']
ml: SetupPlugins['ml'],
ruleDataClient: RuleDataClient | null
) => {
// Detection Engine Rule routes that have the REST endpoints of /api/detection_engine/rules
// All REST rule creation, deletion, updating, etc......
createRulesRoute(router, ml);
readRulesRoute(router);
updateRulesRoute(router, ml);
patchRulesRoute(router, ml);
deleteRulesRoute(router);
findRulesRoute(router);
createRulesRoute(router, ml, ruleDataClient);
readRulesRoute(router, ruleDataClient);
updateRulesRoute(router, ml, ruleDataClient);
patchRulesRoute(router, ml, ruleDataClient);
deleteRulesRoute(router, ruleDataClient);
findRulesRoute(router, ruleDataClient);
// TODO: pass ruleDataClient to all relevant routes
addPrepackedRulesRoute(router, config, security);
getPrepackagedRulesStatusRoute(router, config, security);
@ -102,7 +107,7 @@ export const initRoutes = (
// POST /api/detection_engine/signals/status
// Example usage can be found in security_solution/server/lib/detection_engine/scripts/signals
setSignalsStatusRoute(router);
querySignalsRoute(router);
querySignalsRoute(router, config);
getSignalsMigrationStatusRoute(router);
createSignalsMigrationRoute(router, security);
finalizeSignalsMigrationRoute(router, security);
@ -111,7 +116,7 @@ export const initRoutes = (
// Detection Engine index routes that have the REST endpoints of /api/detection_engine/index
// All REST index creation, policy management for spaces
createIndexRoute(router);
readIndexRoute(router);
readIndexRoute(router, config);
deleteIndexRoute(router);
// Detection Engine tags routes that have the REST endpoints of /api/detection_engine/tags

1
x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/index.test.tsx
View file

@ -33,6 +33,7 @@ const mockDeps = {
trustedAppsByPolicyEnabled: false,
metricsEntitiesEnabled: false,
hostIsolationEnabled: false,
ruleRegistryEnabled: false,
},
service: {} as EndpointAppContextService,
} as EndpointAppContext,