maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SIEM] Dns histogram enhancement (#54902)

Browse source 
* update DNS histogram

* fix indent

* hide dropdown if only one option provided

* update DNS histogram

* fix types
This commit is contained in:
Angela Chuang 2020-01-16 18:46:22 +08:00 committed by GitHub
parent ea9351aaaa
commit c2f3c977eb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 531 additions and 189 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
x-pack/legacy/plugins/siem/public/components/matrix_histogram/index.tsx
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import React, { useState, useEffect, useCallback, useMemo } from 'react';
import React, { useState, useEffect, useCallback } from 'react';
import { ScaleType } from '@elastic/charts';
import darkTheme from '@elastic/eui/dist/eui_theme_dark.json';
@ -26,14 +26,12 @@ import {
HistogramAggregation,
MatrixHistogramQueryProps,
} from './types';
import { generateTablePaginationOptions } from '../paginated_table/helpers';
import { ChartSeriesData } from '../charts/common';
import { InspectButtonContainer } from '../inspect';
export const MatrixHistogramComponent: React.FC<MatrixHistogramProps &
MatrixHistogramQueryProps> = ({
activePage,
dataKey,
defaultStackByOption,
endDate,
@ -45,12 +43,10 @@ export const MatrixHistogramComponent: React.FC<MatrixHistogramProps &
isAlertsHistogram,
isAnomaliesHistogram,
isAuthenticationsHistogram,
isDNSHistogram,
isEventsType,
isPtrIncluded,
isDnsHistogram,
isEventsHistogram,
isInspected,
legendPosition,
limit,
mapping,
query,
scaleType = ScaleType.Time,
@ -104,10 +100,6 @@ export const MatrixHistogramComponent: React.FC<MatrixHistogramProps &
},
[]
);
const getPagination = () =>
activePage != null && limit != null
? generateTablePaginationOptions(activePage, limit)
: undefined;
const { data, loading, inspect, totalCount, refetch = noop } = useQuery<{}, HistogramAggregation>(
{
@ -118,16 +110,13 @@ export const MatrixHistogramComponent: React.FC<MatrixHistogramProps &
query,
skip,
startDate,
sort,
title,
isAlertsHistogram,
isAnomaliesHistogram,
isAuthenticationsHistogram,
isDNSHistogram,
isEventsType,
isDnsHistogram,
isEventsHistogram,
isInspected,
isPtrIncluded,
pagination: useMemo(() => getPagination(), [activePage, limit]),
stackByField: selectedStackByOption.value,
}
);
@ -179,7 +168,7 @@ export const MatrixHistogramComponent: React.FC<MatrixHistogramProps &
>
<EuiFlexGroup alignItems="center" gutterSize="none">
<EuiFlexItem grow={false}>
{stackByOptions && (
{stackByOptions?.length > 1 && (
<EuiSelect
onChange={setSelectedChartOptionCallback}
options={stackByOptions}

4
x-pack/legacy/plugins/siem/public/components/matrix_histogram/types.ts
View file

@ -64,8 +64,8 @@ export interface MatrixHistogramQueryProps {
isAlertsHistogram?: boolean;
isAnomaliesHistogram?: boolean;
isAuthenticationsHistogram?: boolean;
isDNSHistogram?: boolean;
isEventsType?: boolean;
isDnsHistogram?: boolean;
isEventsHistogram?: boolean;
isInspected: boolean;
isPtrIncluded?: boolean;
pagination?: PaginationInputPaginated;

2
x-pack/legacy/plugins/siem/public/components/matrix_histogram/utils.ts
View file

@ -42,7 +42,7 @@ export const getBarchartConfigs = ({
settings: {
legendPosition: legendPosition ?? Position.Bottom,
onBrushEnd,
showLegend: showLegend || true,
showLegend: showLegend ?? true,
theme: {
scales: {
barsPadding: 0.08,

22
x-pack/legacy/plugins/siem/public/containers/matrix_histogram/index.gql_query.ts
View file

@ -11,8 +11,9 @@ export const MatrixHistogramGqlQuery = gql`
$isAlertsHistogram: Boolean!
$isAnomaliesHistogram: Boolean!
$isAuthenticationsHistogram: Boolean!
$isDnsHistogram: Boolean!
$defaultIndex: [String!]!
$isEventsType: Boolean!
$isEventsHistogram: Boolean!
$filterQuery: String
$inspect: Boolean!
$sourceId: ID!
@ -77,7 +78,24 @@ export const MatrixHistogramGqlQuery = gql`
filterQuery: $filterQuery
defaultIndex: $defaultIndex
stackByField: $stackByField
) @include(if: $isEventsType) {
) @include(if: $isEventsHistogram) {
matrixHistogramData {
x
y
g
}
totalCount
inspect @include(if: $inspect) {
dsl
response
}
}
NetworkDnsHistogram(
timerange: $timerange
filterQuery: $filterQuery
defaultIndex: $defaultIndex
stackByField: $stackByField
) @include(if: $isDnsHistogram) {
matrixHistogramData {
x
y

10
x-pack/legacy/plugins/siem/public/containers/matrix_histogram/index.tsx
View file

@ -24,21 +24,23 @@ import { UpdateDateRange } from '../../components/charts/common';
import { SetQuery } from '../../pages/hosts/navigation/types';
export interface OwnProps extends QueryTemplateProps {
isAlertsHistogram?: boolean;
isAnomaliesHistogram?: boolean;
isAuthenticationsHistogram?: boolean;
dataKey: string | string[];
defaultStackByOption: MatrixHistogramOption;
deleteQuery?: ({ id }: { id: string }) => void;
isEventsType?: boolean;
errorMessage: string;
headerChildren?: React.ReactNode;
hideHistogramIfEmpty?: boolean;
isAlertsHistogram?: boolean;
isAnomaliesHistogram?: boolean;
isAuthenticationsHistogram?: boolean;
id: string;
isDnsHistogram?: boolean;
isEventsHistogram?: boolean;
legendPosition?: Position;
mapping?: MatrixHistogramMappingTypes;
query: Maybe<string>;
setQuery: SetQuery;
showLegend?: boolean;
sourceId: string;
stackByOptions: MatrixHistogramOption[];
subtitle?: string | GetSubTitle;

52
x-pack/legacy/plugins/siem/public/containers/matrix_histogram/utils.ts
View file

@ -16,7 +16,7 @@ import { useUiSetting$ } from '../../lib/kibana';
import { createFilter } from '../helpers';
import { useApolloClient } from '../../utils/apollo_context';
import { inputsModel } from '../../store';
import { GetMatrixHistogramQuery, GetNetworkDnsQuery } from '../../graphql/types';
import { GetMatrixHistogramQuery } from '../../graphql/types';
export const useQuery = <Hit, Aggs, TCache = object>({
dataKey,
@ -26,15 +26,12 @@ export const useQuery = <Hit, Aggs, TCache = object>({
isAlertsHistogram = false,
isAnomaliesHistogram = false,
isAuthenticationsHistogram = false,
isEventsType = false,
isDNSHistogram,
isPtrIncluded,
isEventsHistogram = false,
isDnsHistogram = false,
isInspected,
query,
stackByField,
startDate,
sort,
pagination,
}: MatrixHistogramQueryProps) => {
const [defaultIndex] = useUiSetting$<string[]>(DEFAULT_INDEX_KEY);
const [, dispatchToaster] = useStateToaster();
@ -45,21 +42,7 @@ export const useQuery = <Hit, Aggs, TCache = object>({
const [totalCount, setTotalCount] = useState(-1);
const apolloClient = useApolloClient();
const isDNSQuery = (
variable: Pick<
MatrixHistogramQueryProps,
'isDNSHistogram' | 'isPtrIncluded' | 'sort' | 'pagination'
>
): variable is GetNetworkDnsQuery.Variables => {
return (
!!isDNSHistogram &&
variable.isDNSHistogram !== undefined &&
variable.isPtrIncluded !== undefined &&
variable.sort !== undefined &&
variable.pagination !== undefined
);
};
const basicVariables = {
const matrixHistogramVariables: GetMatrixHistogramQuery.Variables = {
filterQuery: createFilter(filterQuery),
sourceId: 'default',
timerange: {
@ -70,20 +53,11 @@ export const useQuery = <Hit, Aggs, TCache = object>({
defaultIndex,
inspect: isInspected,
stackByField,
};
const dnsVariables = {
...basicVariables,
isDNSHistogram,
isPtrIncluded,
sort,
pagination,
};
const matrixHistogramVariables: GetMatrixHistogramQuery.Variables = {
...basicVariables,
isAlertsHistogram,
isAnomaliesHistogram,
isAuthenticationsHistogram,
isEventsType,
isDnsHistogram,
isEventsHistogram,
};
useEffect(() => {
@ -92,16 +66,13 @@ export const useQuery = <Hit, Aggs, TCache = object>({
const abortSignal = abortCtrl.signal;
async function fetchData() {
if (!apolloClient || (pagination != null && pagination.querySize < 0)) return null;
if (!apolloClient) return null;
setLoading(true);
return apolloClient
.query<
GetMatrixHistogramQuery.Query | GetNetworkDnsQuery.Query,
GetMatrixHistogramQuery.Variables | GetNetworkDnsQuery.Variables
>({
.query<GetMatrixHistogramQuery.Query, GetMatrixHistogramQuery.Variables>({
query,
fetchPolicy: 'cache-first',
variables: isDNSQuery(dnsVariables) ? dnsVariables : matrixHistogramVariables,
variables: matrixHistogramVariables,
context: {
fetchOptions: {
abortSignal,
@ -145,11 +116,8 @@ export const useQuery = <Hit, Aggs, TCache = object>({
query,
filterQuery,
isInspected,
isDNSHistogram,
isDnsHistogram,
stackByField,
sort,
isPtrIncluded,
pagination,
startDate,
endDate,
]);

10
x-pack/legacy/plugins/siem/public/containers/network_dns/index.gql_query.ts
View file

@ -11,7 +11,6 @@ export const networkDnsQuery = gql`
$defaultIndex: [String!]!
$filterQuery: String
$inspect: Boolean!
$isDNSHistogram: Boolean!
$isPtrIncluded: Boolean!
$pagination: PaginationInputPaginated!
$sort: NetworkDnsSortField!
@ -31,7 +30,7 @@ export const networkDnsQuery = gql`
stackByField: $stackByField
) {
totalCount
edges @skip(if: $isDNSHistogram) {
edges {
node {
_id
dnsBytesIn
@ -44,7 +43,7 @@ export const networkDnsQuery = gql`
value
}
}
pageInfo @skip(if: $isDNSHistogram) {
pageInfo {
activePage
fakeTotalCount
showMorePagesIndicator
@ -53,11 +52,6 @@ export const networkDnsQuery = gql`
dsl
response
}
histogram @include(if: $isDNSHistogram) {
x
y
g
}
}
}
}

10
x-pack/legacy/plugins/siem/public/containers/network_dns/index.tsx
View file

@ -26,7 +26,7 @@ import { generateTablePaginationOptions } from '../../components/paginated_table
import { createFilter, getDefaultFetchPolicy } from '../helpers';
import { QueryTemplatePaginated, QueryTemplatePaginatedProps } from '../query_template_paginated';
import { networkDnsQuery } from './index.gql_query';
import { DEFAULT_TABLE_ACTIVE_PAGE } from '../../store/constants';
import { DEFAULT_TABLE_ACTIVE_PAGE, DEFAULT_TABLE_LIMIT } from '../../store/constants';
import { MatrixHistogram } from '../../components/matrix_histogram';
import { MatrixHistogramOption, GetSubTitle } from '../../components/matrix_histogram/types';
import { UpdateDateRange } from '../../components/charts/common';
@ -57,8 +57,7 @@ interface DnsHistogramOwnProps extends QueryTemplatePaginatedProps {
dataKey: string | string[];
defaultStackByOption: MatrixHistogramOption;
errorMessage: string;
isDNSHistogram?: boolean;
limit: number;
isDnsHistogram?: boolean;
query: DocumentNode;
scaleType: ScaleType;
setQuery: SetQuery;
@ -105,7 +104,6 @@ export class NetworkDnsComponentQuery extends QueryTemplatePaginated<
const variables: GetNetworkDnsQuery.Variables = {
defaultIndex: kibana.services.uiSettings.get<string[]>(DEFAULT_INDEX_KEY),
filterQuery: createFilter(filterQuery),
isDNSHistogram: false,
inspect: isInspected,
isPtrIncluded,
pagination: generateTablePaginationOptions(activePage, limit),
@ -186,12 +184,12 @@ const makeMapStateToProps = () => {
const makeMapHistogramStateToProps = () => {
const getNetworkDnsSelector = networkSelectors.dnsSelector();
const getQuery = inputsSelectors.globalQueryByIdSelector();
const mapStateToProps = (state: State, { id = HISTOGRAM_ID, limit }: DnsHistogramOwnProps) => {
const mapStateToProps = (state: State, { id = HISTOGRAM_ID }: DnsHistogramOwnProps) => {
const { isInspected } = getQuery(state, id);
return {
...getNetworkDnsSelector(state),
activePage: DEFAULT_TABLE_ACTIVE_PAGE,
limit,
limit: DEFAULT_TABLE_LIMIT,
isInspected,
id,
};

108
x-pack/legacy/plugins/siem/public/graphql/introspection.json
View file

@ -1901,6 +1901,59 @@
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "NetworkDnsHistogram",
"description": "",
"args": [
{
"name": "filterQuery",
"description": "",
"type": { "kind": "SCALAR", "name": "String", "ofType": null },
"defaultValue": null
},
{
"name": "defaultIndex",
"description": "",
"type": {
"kind": "NON_NULL",
"name": null,
"ofType": {
"kind": "LIST",
"name": null,
"ofType": {
"kind": "NON_NULL",
"name": null,
"ofType": { "kind": "SCALAR", "name": "String", "ofType": null }
}
}
},
"defaultValue": null
},
{
"name": "timerange",
"description": "",
"type": {
"kind": "NON_NULL",
"name": null,
"ofType": { "kind": "INPUT_OBJECT", "name": "TimerangeInput", "ofType": null }
},
"defaultValue": null
},
{
"name": "stackByField",
"description": "",
"type": { "kind": "SCALAR", "name": "String", "ofType": null },
"defaultValue": null
}
],
"type": {
"kind": "NON_NULL",
"name": null,
"ofType": { "kind": "OBJECT", "name": "NetworkDsOverTimeData", "ofType": null }
},
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "NetworkHttp",
"description": "",
@ -8744,6 +8797,61 @@
"enumValues": null,
"possibleTypes": null
},
{
"kind": "OBJECT",
"name": "NetworkDsOverTimeData",
"description": "",
"fields": [
{
"name": "inspect",
"description": "",
"args": [],
"type": { "kind": "OBJECT", "name": "Inspect", "ofType": null },
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "matrixHistogramData",
"description": "",
"args": [],
"type": {
"kind": "NON_NULL",
"name": null,
"ofType": {
"kind": "LIST",
"name": null,
"ofType": {
"kind": "NON_NULL",
"name": null,
"ofType": {
"kind": "OBJECT",
"name": "MatrixOverTimeHistogramData",
"ofType": null
}
}
}
},
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "totalCount",
"description": "",
"args": [],
"type": {
"kind": "NON_NULL",
"name": null,
"ofType": { "kind": "SCALAR", "name": "Float", "ofType": null }
},
"isDeprecated": false,
"deprecationReason": null
}
],
"inputFields": null,
"interfaces": [],
"enumValues": null,
"possibleTypes": null
},
{
"kind": "INPUT_OBJECT",
"name": "NetworkHttpSortField",

65
x-pack/legacy/plugins/siem/public/graphql/types.ts
View file

@ -499,6 +499,8 @@ export interface Source {
NetworkDns: NetworkDnsData;
NetworkDnsHistogram: NetworkDsOverTimeData;
NetworkHttp: NetworkHttpData;
OverviewNetwork?: Maybe<OverviewNetworkData>;
@ -1752,6 +1754,14 @@ export interface MatrixOverOrdinalHistogramData {
g: string;
}
export interface NetworkDsOverTimeData {
inspect?: Maybe<Inspect>;
matrixHistogramData: MatrixOverTimeHistogramData[];
totalCount: number;
}
export interface NetworkHttpData {
edges: NetworkHttpEdges[];
@ -2430,6 +2440,15 @@ export interface NetworkDnsSourceArgs {
defaultIndex: string[];
}
export interface NetworkDnsHistogramSourceArgs {
filterQuery?: Maybe<string>;
defaultIndex: string[];
timerange: TimerangeInput;
stackByField?: Maybe<string>;
}
export interface NetworkHttpSourceArgs {
id?: Maybe<string>;
@ -3306,8 +3325,9 @@ export namespace GetMatrixHistogramQuery {
isAlertsHistogram: boolean;
isAnomaliesHistogram: boolean;
isAuthenticationsHistogram: boolean;
isDnsHistogram: boolean;
defaultIndex: string[];
isEventsType: boolean;
isEventsHistogram: boolean;
filterQuery?: Maybe<string>;
inspect: boolean;
sourceId: string;
@ -3333,6 +3353,8 @@ export namespace GetMatrixHistogramQuery {
AuthenticationsHistogram: AuthenticationsHistogram;
EventsHistogram: EventsHistogram;
NetworkDnsHistogram: NetworkDnsHistogram;
};
export type AlertsHistogram = {
@ -3446,6 +3468,34 @@ export namespace GetMatrixHistogramQuery {
response: string[];
};
export type NetworkDnsHistogram = {
__typename?: 'NetworkDsOverTimeData';
matrixHistogramData: ____MatrixHistogramData[];
totalCount: number;
inspect: Maybe<____Inspect>;
};
export type ____MatrixHistogramData = {
__typename?: 'MatrixOverTimeHistogramData';
x: number;
y: number;
g: string;
};
export type ____Inspect = {
__typename?: 'Inspect';
dsl: string[];
response: string[];
};
}
export namespace GetNetworkDnsQuery {
@ -3453,7 +3503,6 @@ export namespace GetNetworkDnsQuery {
defaultIndex: string[];
filterQuery?: Maybe<string>;
inspect: boolean;
isDNSHistogram: boolean;
isPtrIncluded: boolean;
pagination: PaginationInputPaginated;
sort: NetworkDnsSortField;
@ -3486,8 +3535,6 @@ export namespace GetNetworkDnsQuery {
pageInfo: PageInfo;
inspect: Maybe<Inspect>;
histogram: Maybe<Histogram[]>;
};
export type Edges = {
@ -3537,16 +3584,6 @@ export namespace GetNetworkDnsQuery {
response: string[];
};
export type Histogram = {
__typename?: 'MatrixOverOrdinalHistogramData';
x: string;
y: number;
g: string;
};
}
export namespace GetNetworkHttpQuery {

2
x-pack/legacy/plugins/siem/public/pages/hosts/navigation/events_query_tab_body.tsx
View file

@ -52,7 +52,7 @@ export const EventsQueryTabBody = ({
defaultStackByOption={eventsStackByOptions[0]}
deleteQuery={deleteQuery}
endDate={endDate}
isEventsType={true}
isEventsHistogram={true}
errorMessage={i18n.ERROR_FETCHING_EVENTS_DATA}
filterQuery={filterQuery}
query={MatrixHistogramGqlQuery}

108
x-pack/legacy/plugins/siem/public/pages/network/navigation/dns_query_tab_body.tsx
View file

@ -4,31 +4,27 @@
* you may not use this file except in compliance with the Elastic License.
*/
import React, { useEffect } from 'react';
import React, { useEffect, useCallback } from 'react';
import { getOr } from 'lodash/fp';
import { EuiSpacer } from '@elastic/eui';
import { ScaleType } from '@elastic/charts';
import { NetworkDnsTable } from '../../../components/page/network/network_dns_table';
import {
NetworkDnsQuery,
NetworkDnsHistogramQuery,
HISTOGRAM_ID,
} from '../../../containers/network_dns';
import { NetworkDnsQuery, HISTOGRAM_ID } from '../../../containers/network_dns';
import { manageQuery } from '../../../components/page/manage_query';
import { NetworkComponentQueryProps } from './types';
import { networkModel } from '../../../store';
import { MatrixHistogramOption } from '../../../components/matrix_histogram/types';
import { networkDnsQuery } from '../../../containers/network_dns/index.gql_query';
import * as i18n from '../translations';
import { useFormatBytes } from '../../../components/formatted_bytes';
import { MatrixHistogramGqlQuery } from '../../../containers/matrix_histogram/index.gql_query';
import { MatrixHistogramContainer } from '../../../containers/matrix_histogram';
const NetworkDnsTableManage = manageQuery(NetworkDnsTable);
const dnsStackByOptions: MatrixHistogramOption[] = [
{
text: i18n.NAVIGATION_DNS_STACK_BY_DOMAIN,
text: i18n.STACK_BY_DOMAIN,
value: 'dns.question.registered_domain',
},
];
@ -50,50 +46,52 @@ export const DnsQueryTabBody = ({
}
};
}, [deleteQuery]);
const formatBytes = useFormatBytes();
const getTitle = useCallback(
(option: MatrixHistogramOption) => i18n.DOMAINS_COUNT_BY(option.text),
[]
);
return (
<NetworkDnsQuery
endDate={endDate}
filterQuery={filterQuery}
skip={skip}
sourceId="default"
startDate={startDate}
type={type}
>
{({
totalCount,
loading,
networkDns,
pageInfo,
loadPage,
id,
inspect,
isInspected,
refetch,
}) => (
<>
<NetworkDnsHistogramQuery
dataKey={['NetworkDns', 'histogram']}
defaultStackByOption={dnsStackByOptions[0]}
endDate={endDate}
errorMessage={i18n.ERROR_FETCHING_DNS_DATA}
filterQuery={filterQuery}
isDNSHistogram={true}
limit={totalCount}
query={networkDnsQuery}
scaleType={ScaleType.Ordinal}
setQuery={setQuery}
sourceId="default"
startDate={startDate}
stackByOptions={dnsStackByOptions}
title={i18n.NAVIGATION_DNS_TITLE}
type={networkModel.NetworkType.page}
updateDateRange={updateDateRange}
yTickFormatter={formatBytes}
showLegend={false}
/>
<EuiSpacer />
<>
<MatrixHistogramContainer
dataKey={['NetworkDnsHistogram', 'matrixHistogramData']}
defaultStackByOption={dnsStackByOptions[0]}
endDate={endDate}
errorMessage={i18n.ERROR_FETCHING_DNS_DATA}
filterQuery={filterQuery}
id={HISTOGRAM_ID}
isDnsHistogram={true}
query={MatrixHistogramGqlQuery}
setQuery={setQuery}
sourceId="default"
startDate={startDate}
stackByOptions={dnsStackByOptions}
title={getTitle}
type={networkModel.NetworkType.page}
updateDateRange={updateDateRange}
showLegend={false}
/>
<EuiSpacer />
<NetworkDnsQuery
endDate={endDate}
filterQuery={filterQuery}
skip={skip}
sourceId="default"
startDate={startDate}
type={type}
>
{({
totalCount,
loading,
networkDns,
pageInfo,
loadPage,
id,
inspect,
isInspected,
refetch,
}) => (
<NetworkDnsTableManage
data={networkDns}
fakeTotalCount={getOr(50, 'fakeTotalCount', pageInfo)}
@ -108,9 +106,9 @@ export const DnsQueryTabBody = ({
totalCount={totalCount}
type={type}
/>
</>
)}
</NetworkDnsQuery>
)}
</NetworkDnsQuery>
</>
);
};

15
x-pack/legacy/plugins/siem/public/pages/network/translations.ts
View file

@ -22,12 +22,9 @@ export const NAVIGATION_DNS_TITLE = i18n.translate('xpack.siem.network.navigatio
defaultMessage: 'DNS',
});
export const NAVIGATION_DNS_STACK_BY_DOMAIN = i18n.translate(
'xpack.siem.hosts.navigation.dns.stackByDomain',
{
defaultMessage: 'domain',
}
);
export const STACK_BY_DOMAIN = i18n.translate('xpack.siem.hosts.dns.stackByDomain', {
defaultMessage: 'unique domains',
});
export const ERROR_FETCHING_DNS_DATA = i18n.translate(
'xpack.siem.hosts.navigation.dns.histogram.errorFetchingDnsData',
@ -54,3 +51,9 @@ export const NAVIGATION_ANOMALIES_TITLE = i18n.translate(
export const NAVIGATION_ALERTS_TITLE = i18n.translate('xpack.siem.network.navigation.alertsTitle', {
defaultMessage: 'Alerts',
});
export const DOMAINS_COUNT_BY = (groupByField: string) =>
i18n.translate('xpack.siem.network.dns.stackByUniqueSubdomain', {
values: { groupByField },
defaultMessage: 'Top domains by {groupByField}',
});

2
x-pack/legacy/plugins/siem/public/pages/overview/events_by_dataset/index.tsx
View file

@ -108,7 +108,7 @@ export const EventsByDataset = React.memo<Props>(
})}
headerChildren={eventsCountViewEventsButton}
id={ID}
isEventsType={true}
isEventsHistogram={true}
legendPosition={'right'}
query={MatrixHistogramGqlQuery}
setQuery={setQuery}

15
x-pack/legacy/plugins/siem/server/graphql/network/resolvers.ts
View file

@ -7,7 +7,7 @@
import { SourceResolvers } from '../../graphql/types';
import { AppResolverOf, ChildResolverOf } from '../../lib/framework';
import { Network } from '../../lib/network';
import { createOptionsPaginated } from '../../utils/build_query/create_options';
import { createOptionsPaginated, createOptions } from '../../utils/build_query/create_options';
import { QuerySourceResolver } from '../sources/resolvers';
type QueryNetworkTopCountriesResolver = ChildResolverOf<
@ -30,6 +30,10 @@ type QueryDnsResolver = ChildResolverOf<
QuerySourceResolver
>;
type QueryDnsHistogramResolver = ChildResolverOf<
AppResolverOf<SourceResolvers.NetworkDnsHistogramResolver>,
QuerySourceResolver
>;
export interface NetworkResolversDeps {
network: Network;
}
@ -42,6 +46,7 @@ export const createNetworkResolvers = (
NetworkTopCountries: QueryNetworkTopCountriesResolver;
NetworkTopNFlow: QueryNetworkTopNFlowResolver;
NetworkDns: QueryDnsResolver;
NetworkDnsHistogram: QueryDnsHistogramResolver;
};
} => ({
Source: {
@ -76,9 +81,15 @@ export const createNetworkResolvers = (
...createOptionsPaginated(source, args, info),
networkDnsSortField: args.sort,
isPtrIncluded: args.isPtrIncluded,
stackByField: args.stackByField,
};
return libs.network.getNetworkDns(req, options);
},
async NetworkDnsHistogram(source, args, { req }, info) {
const options = {
...createOptions(source, args, info),
stackByField: args.stackByField,
};
return libs.network.getNetworkDnsHistogramData(req, options);
},
},
});

12
x-pack/legacy/plugins/siem/server/graphql/network/schema.gql.ts
View file

@ -196,6 +196,12 @@ export const networkSchema = gql`
inspect: Inspect
}
type NetworkDsOverTimeData {
inspect: Inspect
matrixHistogramData: [MatrixOverTimeHistogramData!]!
totalCount: Float!
}
extend type Source {
NetworkTopCountries(
id: String
@ -227,6 +233,12 @@ export const networkSchema = gql`
timerange: TimerangeInput!
defaultIndex: [String!]!
): NetworkDnsData!
NetworkDnsHistogram(
filterQuery: String
defaultIndex: [String!]!
timerange: TimerangeInput!
stackByField: String
): NetworkDsOverTimeData!
NetworkHttp(
id: String
filterQuery: String

67
x-pack/legacy/plugins/siem/server/graphql/types.ts
View file

@ -501,6 +501,8 @@ export interface Source {
NetworkDns: NetworkDnsData;
NetworkDnsHistogram: NetworkDsOverTimeData;
NetworkHttp: NetworkHttpData;
OverviewNetwork?: Maybe<OverviewNetworkData>;
@ -1754,6 +1756,14 @@ export interface MatrixOverOrdinalHistogramData {
g: string;
}
export interface NetworkDsOverTimeData {
inspect?: Maybe<Inspect>;
matrixHistogramData: MatrixOverTimeHistogramData[];
totalCount: number;
}
export interface NetworkHttpData {
edges: NetworkHttpEdges[];
@ -2432,6 +2442,15 @@ export interface NetworkDnsSourceArgs {
defaultIndex: string[];
}
export interface NetworkDnsHistogramSourceArgs {
filterQuery?: Maybe<string>;
defaultIndex: string[];
timerange: TimerangeInput;
stackByField?: Maybe<string>;
}
export interface NetworkHttpSourceArgs {
id?: Maybe<string>;
@ -2930,6 +2949,8 @@ export namespace SourceResolvers {
NetworkDns?: NetworkDnsResolver<NetworkDnsData, TypeParent, TContext>;
NetworkDnsHistogram?: NetworkDnsHistogramResolver<NetworkDsOverTimeData, TypeParent, TContext>;
NetworkHttp?: NetworkHttpResolver<NetworkHttpData, TypeParent, TContext>;
OverviewNetwork?: OverviewNetworkResolver<Maybe<OverviewNetworkData>, TypeParent, TContext>;
@ -3281,6 +3302,21 @@ export namespace SourceResolvers {
defaultIndex: string[];
}
export type NetworkDnsHistogramResolver<
R = NetworkDsOverTimeData,
Parent = Source,
TContext = SiemContext
> = Resolver<R, Parent, TContext, NetworkDnsHistogramArgs>;
export interface NetworkDnsHistogramArgs {
filterQuery?: Maybe<string>;
defaultIndex: string[];
timerange: TimerangeInput;
stackByField?: Maybe<string>;
}
export type NetworkHttpResolver<
R = NetworkHttpData,
Parent = Source,
@ -7547,6 +7583,36 @@ export namespace MatrixOverOrdinalHistogramDataResolvers {
> = Resolver<R, Parent, TContext>;
}
export namespace NetworkDsOverTimeDataResolvers {
export interface Resolvers<TContext = SiemContext, TypeParent = NetworkDsOverTimeData> {
inspect?: InspectResolver<Maybe<Inspect>, TypeParent, TContext>;
matrixHistogramData?: MatrixHistogramDataResolver<
MatrixOverTimeHistogramData[],
TypeParent,
TContext
>;
totalCount?: TotalCountResolver<number, TypeParent, TContext>;
}
export type InspectResolver<
R = Maybe<Inspect>,
Parent = NetworkDsOverTimeData,
TContext = SiemContext
> = Resolver<R, Parent, TContext>;
export type MatrixHistogramDataResolver<
R = MatrixOverTimeHistogramData[],
Parent = NetworkDsOverTimeData,
TContext = SiemContext
> = Resolver<R, Parent, TContext>;
export type TotalCountResolver<
R = number,
Parent = NetworkDsOverTimeData,
TContext = SiemContext
> = Resolver<R, Parent, TContext>;
}
export namespace NetworkHttpDataResolvers {
export interface Resolvers<TContext = SiemContext, TypeParent = NetworkHttpData> {
edges?: EdgesResolver<NetworkHttpEdges[], TypeParent, TContext>;
@ -9227,6 +9293,7 @@ export type IResolvers<TContext = SiemContext> = {
NetworkDnsEdges?: NetworkDnsEdgesResolvers.Resolvers<TContext>;
NetworkDnsItem?: NetworkDnsItemResolvers.Resolvers<TContext>;
MatrixOverOrdinalHistogramData?: MatrixOverOrdinalHistogramDataResolvers.Resolvers<TContext>;
NetworkDsOverTimeData?: NetworkDsOverTimeDataResolvers.Resolvers<TContext>;
NetworkHttpData?: NetworkHttpDataResolvers.Resolvers<TContext>;
NetworkHttpEdges?: NetworkHttpEdgesResolvers.Resolvers<TContext>;
NetworkHttpItem?: NetworkHttpItemResolvers.Resolvers<TContext>;

62
x-pack/legacy/plugins/siem/server/lib/network/elasticsearch_adapter.ts
View file

@ -18,10 +18,16 @@ import {
NetworkHttpData,
NetworkHttpEdges,
NetworkTopNFlowEdges,
MatrixOverOrdinalHistogramData,
NetworkDsOverTimeData,
MatrixOverTimeHistogramData,
} from '../../graphql/types';
import { inspectStringifyObject } from '../../utils/build_query';
import { DatabaseSearchResponse, FrameworkAdapter, FrameworkRequest } from '../framework';
import {
DatabaseSearchResponse,
FrameworkAdapter,
FrameworkRequest,
MatrixHistogramRequestOptions,
} from '../framework';
import { TermAggregation } from '../types';
import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../common/constants';
@ -32,6 +38,7 @@ import {
NetworkTopNFlowRequestOptions,
} from './index';
import { buildDnsQuery } from './query_dns.dsl';
import { buildDnsHistogramQuery } from './query_dns_histogram.dsl';
import { buildTopNFlowQuery, getOppositeField } from './query_top_n_flow.dsl';
import { buildHttpQuery } from './query_http.dsl';
import { buildTopCountriesQuery } from './query_top_countries.dsl';
@ -41,7 +48,9 @@ import {
NetworkTopCountriesBuckets,
NetworkHttpBuckets,
NetworkTopNFlowBuckets,
DnsHistogramGroupData,
} from './types';
import { EventHit } from '../events/types';
export class ElasticsearchNetworkAdapter implements NetworkAdapter {
constructor(private readonly framework: FrameworkAdapter) {}
@ -141,7 +150,6 @@ export class ElasticsearchNetworkAdapter implements NetworkAdapter {
);
const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount;
const edges = networkDnsEdges.splice(cursorStart, querySize - cursorStart);
const histogram = getHistogramData(edges);
const inspect = {
dsl: [inspectStringifyObject(dsl)],
response: [inspectStringifyObject(response)],
@ -156,7 +164,6 @@ export class ElasticsearchNetworkAdapter implements NetworkAdapter {
showMorePagesIndicator,
},
totalCount,
histogram,
};
}
@ -195,29 +202,36 @@ export class ElasticsearchNetworkAdapter implements NetworkAdapter {
totalCount,
};
}
public async getNetworkDnsHistogramData(
request: FrameworkRequest,
options: MatrixHistogramRequestOptions
): Promise<NetworkDsOverTimeData> {
const dsl = buildDnsHistogramQuery(options);
const response = await this.framework.callWithRequest<EventHit, TermAggregation>(
request,
'search',
dsl
);
const totalCount = getOr(0, 'hits.total.value', response);
const matrixHistogramData = getOr([], 'aggregations.NetworkDns.buckets', response);
const inspect = {
dsl: [inspectStringifyObject(dsl)],
response: [inspectStringifyObject(response)],
};
return {
inspect,
matrixHistogramData: getHistogramData(matrixHistogramData),
totalCount,
};
}
}
const getHistogramData = (
data: NetworkDnsEdges[]
): MatrixOverOrdinalHistogramData[] | undefined => {
if (!Array.isArray(data)) return undefined;
const getHistogramData = (data: DnsHistogramGroupData[]): MatrixOverTimeHistogramData[] => {
return data.reduce(
(acc: MatrixOverOrdinalHistogramData[], { node: { dnsBytesOut, dnsBytesIn, _id } }) => {
if (_id != null && dnsBytesOut != null && dnsBytesIn != null)
return [
...acc,
{
x: _id,
y: dnsBytesOut,
g: 'DNS Bytes Out',
},
{
x: _id,
y: dnsBytesIn,
g: 'DNS Bytes In',
},
];
return acc;
(acc: MatrixOverTimeHistogramData[], { key: time, histogram: { buckets } }) => {
const temp = buckets.map(({ key, doc_count }) => ({ x: time, y: doc_count, g: key }));
return [...acc, ...temp];
},
[]
);

14
x-pack/legacy/plugins/siem/server/lib/network/index.ts
View file

@ -14,8 +14,13 @@ import {
NetworkTopCountriesData,
NetworkTopNFlowData,
NetworkTopTablesSortField,
NetworkDsOverTimeData,
} from '../../graphql/types';
import { FrameworkRequest, RequestOptionsPaginated } from '../framework';
import {
FrameworkRequest,
RequestOptionsPaginated,
MatrixHistogramRequestOptions,
} from '../framework';
export * from './elasticsearch_adapter';
import { NetworkAdapter } from './types';
@ -68,6 +73,13 @@ export class Network {
return this.adapter.getNetworkDns(req, options);
}
public async getNetworkDnsHistogramData(
req: FrameworkRequest,
options: MatrixHistogramRequestOptions
): Promise<NetworkDsOverTimeData> {
return this.adapter.getNetworkDnsHistogramData(req, options);
}
public async getNetworkHttp(
req: FrameworkRequest,
options: NetworkHttpRequestOptions

83
x-pack/legacy/plugins/siem/server/lib/network/query_dns_histogram.dsl.ts Normal file
View file

@ -0,0 +1,83 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { createQueryFilterClauses, calculateTimeseriesInterval } from '../../utils/build_query';
import { MatrixHistogramRequestOptions } from '../framework';
export const buildDnsHistogramQuery = ({
filterQuery,
timerange: { from, to },
defaultIndex,
sourceConfiguration: {
fields: { timestamp },
},
stackByField,
}: MatrixHistogramRequestOptions) => {
const filter = [
...createQueryFilterClauses(filterQuery),
{
range: {
[timestamp]: {
gte: from,
lte: to,
},
},
},
];
const getHistogramAggregation = () => {
const interval = calculateTimeseriesInterval(from, to);
const histogramTimestampField = '@timestamp';
const dateHistogram = {
date_histogram: {
field: histogramTimestampField,
fixed_interval: `${interval}s`,
},
};
return {
NetworkDns: {
...dateHistogram,
aggs: {
histogram: {
terms: {
field: stackByField,
order: {
orderAgg: 'desc',
},
size: 10,
},
aggs: {
orderAgg: {
cardinality: {
field: 'dns.question.name',
},
},
},
},
},
},
};
};
const dslQuery = {
index: defaultIndex,
allowNoIndices: true,
ignoreUnavailable: true,
body: {
aggregations: getHistogramAggregation(),
query: {
bool: {
filter,
},
},
size: 0,
track_total_hits: true,
},
};
return dslQuery;
};

31
x-pack/legacy/plugins/siem/server/lib/network/types.ts
View file

@ -9,8 +9,13 @@ import {
NetworkHttpData,
NetworkTopCountriesData,
NetworkTopNFlowData,
NetworkDsOverTimeData,
} from '../../graphql/types';
import { FrameworkRequest, RequestOptionsPaginated } from '../framework';
import {
FrameworkRequest,
RequestOptionsPaginated,
MatrixHistogramRequestOptions,
} from '../framework';
import { TotalValue } from '../types';
import { NetworkDnsRequestOptions } from '.';
@ -24,6 +29,10 @@ export interface NetworkAdapter {
options: RequestOptionsPaginated
): Promise<NetworkTopNFlowData>;
getNetworkDns(req: FrameworkRequest, options: NetworkDnsRequestOptions): Promise<NetworkDnsData>;
getNetworkDnsHistogramData(
request: FrameworkRequest,
options: MatrixHistogramRequestOptions
): Promise<NetworkDsOverTimeData>;
getNetworkHttp(req: FrameworkRequest, options: RequestOptionsPaginated): Promise<NetworkHttpData>;
}
@ -143,3 +152,23 @@ export interface NetworkHttpBuckets {
buckets: GenericBuckets[];
};
}
interface DnsHistogramSubBucket {
key: string;
doc_count: number;
orderAgg: {
value: number;
};
}
interface DnsHistogramBucket {
doc_count_error_upper_bound: number;
sum_other_doc_count: number;
buckets: DnsHistogramSubBucket[];
}
export interface DnsHistogramGroupData {
key: number;
doc_count: number;
key_as_string: string;
histogram: DnsHistogramBucket;
}

3
x-pack/test/api_integration/apis/siem/network_dns.ts
View file

@ -30,7 +30,6 @@ export default function({ getService }: FtrProviderContext) {
query: networkDnsQuery,
variables: {
defaultIndex: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
isDNSHistogram: false,
inspect: false,
isPtrIncluded: false,
pagination: {
@ -66,7 +65,7 @@ export default function({ getService }: FtrProviderContext) {
query: networkDnsQuery,
variables: {
defaultIndex: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
isDNSHistogram: false,
isDnsHistogram: false,
inspect: false,
isPtrIncluded: false,
pagination: {