maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Logs UI] Transmit and render array field values in log entries (#81385)

Browse source 
Co-authored-by: Alejandro Fernández Gómez <antarticonorte@gmail.com>
This commit is contained in:
Felix Stürmer 2020-10-28 12:53:02 +01:00 committed by GitHub
parent 7e0b9ffad0
commit 756c3f16e3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
40 changed files with 3967 additions and 2969 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
x-pack/plugins/infra/common/http_api/log_entries/entries.ts
View file

@ -5,6 +5,7 @@
*/ */
import * as rt from 'io-ts'; import * as rt from 'io-ts';
import { jsonArrayRT } from '../../typed_json';
import { logEntriesCursorRT } from './common'; import { logEntriesCursorRT } from './common';
export const LOG_ENTRIES_PATH = '/api/log_entries/entries'; export const LOG_ENTRIES_PATH = '/api/log_entries/entries';
@ -54,7 +55,7 @@ export const logMessageConstantPartRT = rt.type({
}); });
export const logMessageFieldPartRT = rt.type({ export const logMessageFieldPartRT = rt.type({
field: rt.string, field: rt.string,
value: rt.unknown, value: jsonArrayRT,
highlights: rt.array(rt.string), highlights: rt.array(rt.string),
}); });
@ -64,7 +65,7 @@ export const logTimestampColumnRT = rt.type({ columnId: rt.string, timestamp: rt
export const logFieldColumnRT = rt.type({ export const logFieldColumnRT = rt.type({
columnId: rt.string, columnId: rt.string,
field: rt.string, field: rt.string,
value: rt.unknown, value: jsonArrayRT,
highlights: rt.array(rt.string), highlights: rt.array(rt.string),
}); });
export const logMessageColumnRT = rt.type({ export const logMessageColumnRT = rt.type({

2
x-pack/plugins/infra/common/http_api/log_entries/item.ts
View file

@ -16,7 +16,7 @@ export const logEntriesItemRequestRT = rt.type({
export type LogEntriesItemRequest = rt.TypeOf<typeof logEntriesItemRequestRT>; export type LogEntriesItemRequest = rt.TypeOf<typeof logEntriesItemRequestRT>;
const logEntriesItemFieldRT = rt.type({ field: rt.string, value: rt.string }); const logEntriesItemFieldRT = rt.type({ field: rt.string, value: rt.array(rt.string) });
const logEntriesItemRT = rt.type({ const logEntriesItemRT = rt.type({
id: rt.string, id: rt.string,
index: rt.string, index: rt.string,

22
x-pack/plugins/infra/common/typed_json.ts
View file

@ -4,11 +4,21 @@
* you may not use this file except in compliance with the Elastic License. * you may not use this file except in compliance with the Elastic License.
*/ */
export type JsonValue = null | boolean | number | string | JsonObject | JsonArray; import * as rt from 'io-ts';
import { JsonArray, JsonObject, JsonValue } from '../../../../src/plugins/kibana_utils/common';
// eslint-disable-next-line @typescript-eslint/no-empty-interface export const jsonScalarRT = rt.union([rt.null, rt.boolean, rt.number, rt.string]);
export interface JsonArray extends Array<JsonValue> {}
export interface JsonObject { export const jsonValueRT: rt.Type<JsonValue> = rt.recursion('JsonValue', () =>
[key: string]: JsonValue; rt.union([jsonScalarRT, jsonArrayRT, jsonObjectRT])
} );
export const jsonArrayRT: rt.Type<JsonArray> = rt.recursion('JsonArray', () =>
rt.array(jsonValueRT)
);
export const jsonObjectRT: rt.Type<JsonObject> = rt.recursion('JsonObject', () =>
rt.record(rt.string, jsonValueRT)
);
export { JsonValue, JsonArray, JsonObject };

18
x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx
View file

@ -29,7 +29,7 @@ describe('LogEntryActionsMenu component', () => {
<ProviderWrapper> <ProviderWrapper>
<LogEntryActionsMenu <LogEntryActionsMenu
logItem={{ logItem={{
fields: [{ field: 'host.ip', value: 'HOST_IP' }], fields: [{ field: 'host.ip', value: ['HOST_IP'] }],
id: 'ITEM_ID', id: 'ITEM_ID',
index: 'INDEX', index: 'INDEX',
key: { key: {
@ -59,7 +59,7 @@ describe('LogEntryActionsMenu component', () => {
<ProviderWrapper> <ProviderWrapper>
<LogEntryActionsMenu <LogEntryActionsMenu
logItem={{ logItem={{
fields: [{ field: 'container.id', value: 'CONTAINER_ID' }], fields: [{ field: 'container.id', value: ['CONTAINER_ID'] }],
id: 'ITEM_ID', id: 'ITEM_ID',
index: 'INDEX', index: 'INDEX',
key: { key: {
@ -89,7 +89,7 @@ describe('LogEntryActionsMenu component', () => {
<ProviderWrapper> <ProviderWrapper>
<LogEntryActionsMenu <LogEntryActionsMenu
logItem={{ logItem={{
fields: [{ field: 'kubernetes.pod.uid', value: 'POD_UID' }], fields: [{ field: 'kubernetes.pod.uid', value: ['POD_UID'] }],
id: 'ITEM_ID', id: 'ITEM_ID',
index: 'INDEX', index: 'INDEX',
key: { key: {
@ -120,9 +120,9 @@ describe('LogEntryActionsMenu component', () => {
<LogEntryActionsMenu <LogEntryActionsMenu
logItem={{ logItem={{
fields: [ fields: [
{ field: 'container.id', value: 'CONTAINER_ID' }, { field: 'container.id', value: ['CONTAINER_ID'] },
{ field: 'host.ip', value: 'HOST_IP' }, { field: 'host.ip', value: ['HOST_IP'] },
{ field: 'kubernetes.pod.uid', value: 'POD_UID' }, { field: 'kubernetes.pod.uid', value: ['POD_UID'] },
], ],
id: 'ITEM_ID', id: 'ITEM_ID',
index: 'INDEX', index: 'INDEX',
@ -189,7 +189,7 @@ describe('LogEntryActionsMenu component', () => {
<ProviderWrapper> <ProviderWrapper>
<LogEntryActionsMenu <LogEntryActionsMenu
logItem={{ logItem={{
fields: [{ field: 'trace.id', value: '1234567' }], fields: [{ field: 'trace.id', value: ['1234567'] }],
id: 'ITEM_ID', id: 'ITEM_ID',
index: 'INDEX', index: 'INDEX',
key: { key: {
@ -221,8 +221,8 @@ describe('LogEntryActionsMenu component', () => {
<LogEntryActionsMenu <LogEntryActionsMenu
logItem={{ logItem={{
fields: [ fields: [
{ field: 'trace.id', value: '1234567' }, { field: 'trace.id', value: ['1234567'] },
{ field: '@timestamp', value: timestamp }, { field: '@timestamp', value: [timestamp] },
], ],
id: 'ITEM_ID', id: 'ITEM_ID',
index: 'INDEX', index: 'INDEX',

15
x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.tsx
View file

@ -4,7 +4,6 @@
* you may not use this file except in compliance with the Elastic License. * you may not use this file except in compliance with the Elastic License.
*/ */
import * as rt from 'io-ts';
import { EuiButtonEmpty, EuiContextMenuItem, EuiContextMenuPanel, EuiPopover } from '@elastic/eui'; import { EuiButtonEmpty, EuiContextMenuItem, EuiContextMenuPanel, EuiPopover } from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react'; import { FormattedMessage } from '@kbn/i18n/react';
import React, { useMemo } from 'react'; import React, { useMemo } from 'react';
@ -12,7 +11,6 @@ import { useVisibilityState } from '../../../utils/use_visibility_state';
import { getTraceUrl } from '../../../../../apm/public'; import { getTraceUrl } from '../../../../../apm/public';
import { LogEntriesItem } from '../../../../common/http_api'; import { LogEntriesItem } from '../../../../common/http_api';
import { useLinkProps, LinkDescriptor } from '../../../hooks/use_link_props'; import { useLinkProps, LinkDescriptor } from '../../../hooks/use_link_props';
import { decodeOrThrow } from '../../../../common/runtime_types';
const UPTIME_FIELDS = ['container.id', 'host.ip', 'kubernetes.pod.uid']; const UPTIME_FIELDS = ['container.id', 'host.ip', 'kubernetes.pod.uid'];
@ -97,12 +95,7 @@ const getUptimeLink = (logItem: LogEntriesItem): LinkDescriptor | undefined => {
.filter(({ field, value }) => value != null && UPTIME_FIELDS.includes(field)) .filter(({ field, value }) => value != null && UPTIME_FIELDS.includes(field))
.reduce<string[]>((acc, fieldItem) => { .reduce<string[]>((acc, fieldItem) => {
const { field, value } = fieldItem; const { field, value } = fieldItem;
try { return acc.concat(value.map((val) => `${field}:${val}`));
const parsedValue = decodeOrThrow(rt.array(rt.string))(JSON.parse(value));
return acc.concat(parsedValue.map((val) => `${field}:${val}`));
} catch (e) {
return acc.concat([`${field}:${value}`]);
}
}, []); }, []);
if (searchExpressions.length === 0) { if (searchExpressions.length === 0) {
@ -119,7 +112,7 @@ const getUptimeLink = (logItem: LogEntriesItem): LinkDescriptor | undefined => {
const getAPMLink = (logItem: LogEntriesItem): LinkDescriptor | undefined => { const getAPMLink = (logItem: LogEntriesItem): LinkDescriptor | undefined => {
const traceIdEntry = logItem.fields.find( const traceIdEntry = logItem.fields.find(
({ field, value }) => value != null && field === 'trace.id' ({ field, value }) => value[0] != null && field === 'trace.id'
); );
if (!traceIdEntry) { if (!traceIdEntry) {
@ -127,7 +120,7 @@ const getAPMLink = (logItem: LogEntriesItem): LinkDescriptor | undefined => {
} }
const timestampField = logItem.fields.find(({ field }) => field === '@timestamp'); const timestampField = logItem.fields.find(({ field }) => field === '@timestamp');
const timestamp = timestampField ? timestampField.value : null; const timestamp = timestampField ? timestampField.value[0] : null;
const { rangeFrom, rangeTo } = timestamp const { rangeFrom, rangeTo } = timestamp
? (() => { ? (() => {
const from = new Date(timestamp); const from = new Date(timestamp);
@ -142,6 +135,6 @@ const getAPMLink = (logItem: LogEntriesItem): LinkDescriptor | undefined => {
return { return {
app: 'apm', app: 'apm',
hash: getTraceUrl({ traceId: traceIdEntry.value, rangeFrom, rangeTo }), hash: getTraceUrl({ traceId: traceIdEntry.value[0], rangeFrom, rangeTo }),
}; };
}; };

6
x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_flyout.tsx
View file

@ -94,7 +94,7 @@ export const LogEntryFlyout = ({
onClick={createFilterHandler(item)} onClick={createFilterHandler(item)}
/> />
</EuiToolTip> </EuiToolTip>
{item.value} {formatValue(item.value)}
</span> </span>
), ),
}, },
@ -147,3 +147,7 @@ export const InfraFlyoutLoadingPanel = euiStyled.div`
bottom: 0; bottom: 0;
left: 0; left: 0;
`; `;
function formatValue(value: string[]) {
return value.length > 1 ? value.join(', ') : value[0];
}

66
x-pack/plugins/infra/public/components/logging/log_text_stream/field_value.tsx Normal file
View file

@ -0,0 +1,66 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import stringify from 'json-stable-stringify';
import React from 'react';
import { euiStyled } from '../../../../../observability/public';
import { JsonArray, JsonValue } from '../../../../common/typed_json';
import { ActiveHighlightMarker, highlightFieldValue, HighlightMarker } from './highlighting';
export const FieldValue: React.FC<{
highlightTerms: string[];
isActiveHighlight: boolean;
value: JsonArray;
}> = React.memo(({ highlightTerms, isActiveHighlight, value }) => {
if (value.length === 1) {
return (
<>
{highlightFieldValue(
formatValue(value[0]),
highlightTerms,
isActiveHighlight ? ActiveHighlightMarker : HighlightMarker
)}
</>
);
} else if (value.length > 1) {
return (
<ul data-test-subj="LogEntryFieldValues">
{value.map((entry, i) => (
<CommaSeparatedLi
key={`LogEntryFieldValue-${i}`}
data-test-subj={`LogEntryFieldValue-${i}`}
>
{highlightFieldValue(
formatValue(entry),
highlightTerms,
isActiveHighlight ? ActiveHighlightMarker : HighlightMarker
)}
</CommaSeparatedLi>
))}
</ul>
);
}
return null;
});
const formatValue = (value: JsonValue): string => {
if (typeof value === 'string') {
return value;
}
return stringify(value);
};
const CommaSeparatedLi = euiStyled.li`
display: inline;
&:not(:last-child) {
margin-right: 1ex;
&::after {
content: ',';
}
}
`;

4
x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_column.tsx
View file

@ -39,7 +39,9 @@ export const LogEntryColumn = euiStyled.div.attrs(() => ({
overflow: hidden; overflow: hidden;
`; `;
export const LogEntryColumnContent = euiStyled.div` export const LogEntryColumnContent = euiStyled.div.attrs({
'data-test-subj': 'LogEntryColumnContent',
})`
flex: 1 0 0%; flex: 1 0 0%;
padding: 2px ${COLUMN_PADDING}px; padding: 2px ${COLUMN_PADDING}px;
`; `;

90
x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.test.tsx
View file

@ -4,85 +4,113 @@
* you may not use this file except in compliance with the Elastic License. * you may not use this file except in compliance with the Elastic License.
*/ */
import { mount } from 'enzyme'; import { render } from '@testing-library/react';
import React from 'react'; import React from 'react';
import { EuiThemeProvider } from '../../../../../observability/public'; import { EuiThemeProvider } from '../../../../../observability/public';
import { LogFieldColumn } from '../../../../common/http_api';
import { LogEntryFieldColumn } from './log_entry_field_column'; import { LogEntryFieldColumn } from './log_entry_field_column';
import { LogColumn } from '../../../../common/http_api';
describe('LogEntryFieldColumn', () => { describe('LogEntryFieldColumn', () => {
it('should output a <ul> when displaying an Array of values', () => { it('renders a single value without a wrapping list', () => {
const column: LogColumn = { const column: LogFieldColumn = {
columnId: 'TEST_COLUMN',
field: 'TEST_FIELD',
value: ['a'],
highlights: [],
};
const renderResult = render(
<LogEntryFieldColumn
columnValue={column}
highlights={[]}
isActiveHighlight={false}
wrapMode="pre-wrapped"
/>,
{ wrapper: EuiThemeProvider }
);
expect(renderResult.getByTestId('LogEntryColumnContent')).toHaveTextContent(/^a$/);
expect(renderResult.queryByTestId('LogEntryFieldValues')).toBe(null);
});
it('renders an array of values as a list', () => {
const column: LogFieldColumn = {
columnId: 'TEST_COLUMN', columnId: 'TEST_COLUMN',
field: 'TEST_FIELD', field: 'TEST_FIELD',
value: ['a', 'b', 'c'], value: ['a', 'b', 'c'],
highlights: [], highlights: [],
}; };
const component = mount( const renderResult = render(
<LogEntryFieldColumn <LogEntryFieldColumn
columnValue={column} columnValue={column}
highlights={[]} highlights={[]}
isActiveHighlight={false} isActiveHighlight={false}
wrapMode="pre-wrapped" wrapMode="pre-wrapped"
/>, />,
{ wrappingComponent: EuiThemeProvider } as any // https://github.com/DefinitelyTyped/DefinitelyTyped/issues/36075 { wrapper: EuiThemeProvider }
); );
expect(component.exists('ul')).toBe(true); expect(renderResult.getByTestId('LogEntryFieldValues')).not.toBeEmptyDOMElement();
expect( expect(renderResult.getByTestId('LogEntryFieldValue-0')).toHaveTextContent('a');
component.containsAllMatchingElements([ expect(renderResult.getByTestId('LogEntryFieldValue-1')).toHaveTextContent('b');
<li key="LogEntryFieldColumn-a-0">a</li>, expect(renderResult.getByTestId('LogEntryFieldValue-2')).toHaveTextContent('c');
<li key="LogEntryFieldColumn-b-1">b</li>,
<li key="LogEntryFieldColumn-c-2">c</li>,
])
).toBe(true);
}); });
it('should output a text representation of a passed complex value', () => { it('renders a text representation of a single complex object', () => {
const column: LogColumn = { const column: LogFieldColumn = {
columnId: 'TEST_COLUMN', columnId: 'TEST_COLUMN',
field: 'TEST_FIELD', field: 'TEST_FIELD',
value: { value: [
lat: 1, {
lon: 2, lat: 1,
}, lon: 2,
},
],
highlights: [], highlights: [],
}; };
const component = mount( const renderResult = render(
<LogEntryFieldColumn <LogEntryFieldColumn
columnValue={column} columnValue={column}
highlights={[]} highlights={[]}
isActiveHighlight={false} isActiveHighlight={false}
wrapMode="pre-wrapped" wrapMode="pre-wrapped"
/>, />,
{ wrappingComponent: EuiThemeProvider } as any // https://github.com/DefinitelyTyped/DefinitelyTyped/issues/36075 { wrapper: EuiThemeProvider }
); );
expect(component.text()).toEqual('{"lat":1,"lon":2}'); expect(renderResult.getByTestId('LogEntryColumnContent')).toHaveTextContent(
'{"lat":1,"lon":2}'
);
}); });
it('should output just text when passed a non-Array', () => { it('renders text representations of a multiple complex objects', () => {
const column: LogColumn = { const column: LogFieldColumn = {
columnId: 'TEST_COLUMN', columnId: 'TEST_COLUMN',
field: 'TEST_FIELD', field: 'TEST_FIELD',
value: 'foo', value: [
{
lat: 1,
lon: 2,
},
[3, 4],
],
highlights: [], highlights: [],
}; };
const component = mount( const renderResult = render(
<LogEntryFieldColumn <LogEntryFieldColumn
columnValue={column} columnValue={column}
highlights={[]} highlights={[]}
isActiveHighlight={false} isActiveHighlight={false}
wrapMode="pre-wrapped" wrapMode="pre-wrapped"
/>, />,
{ wrappingComponent: EuiThemeProvider } as any // https://github.com/DefinitelyTyped/DefinitelyTyped/issues/36075 { wrapper: EuiThemeProvider }
); );
expect(component.exists('ul')).toBe(false); expect(renderResult.getByTestId('LogEntryFieldValues')).not.toBeEmptyDOMElement();
expect(component.text()).toEqual('foo'); expect(renderResult.getByTestId('LogEntryFieldValue-0')).toHaveTextContent('{"lat":1,"lon":2}');
expect(renderResult.getByTestId('LogEntryFieldValue-1')).toHaveTextContent('[3,4]');
}); });
}); });

58
x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.tsx
View file

@ -4,14 +4,12 @@
* you may not use this file except in compliance with the Elastic License. * you may not use this file except in compliance with the Elastic License.
*/ */
import stringify from 'json-stable-stringify'; import React from 'react';
import React, { useMemo } from 'react';
import { euiStyled } from '../../../../../observability/public'; import { euiStyled } from '../../../../../observability/public';
import { isFieldColumn, isHighlightFieldColumn } from '../../../utils/log_entry';
import { ActiveHighlightMarker, highlightFieldValue, HighlightMarker } from './highlighting';
import { LogEntryColumnContent } from './log_entry_column';
import { LogColumn } from '../../../../common/http_api'; import { LogColumn } from '../../../../common/http_api';
import { isFieldColumn, isHighlightFieldColumn } from '../../../utils/log_entry';
import { FieldValue } from './field_value';
import { LogEntryColumnContent } from './log_entry_column';
import { import {
longWrappedContentStyle, longWrappedContentStyle,
preWrappedContentStyle, preWrappedContentStyle,
@ -32,44 +30,20 @@ export const LogEntryFieldColumn: React.FunctionComponent<LogEntryFieldColumnPro
isActiveHighlight, isActiveHighlight,
wrapMode, wrapMode,
}) => { }) => {
const value = useMemo(() => { if (isFieldColumn(columnValue)) {
if (isFieldColumn(columnValue)) { return (
return columnValue.value; <FieldColumnContent wrapMode={wrapMode}>
} <FieldValue
highlightTerms={isHighlightFieldColumn(firstHighlight) ? firstHighlight.highlights : []}
isActiveHighlight={isActiveHighlight}
value={columnValue.value}
/>
</FieldColumnContent>
);
} else {
return null; return null;
}, [columnValue]);
const formattedValue = Array.isArray(value) ? (
<ul>
{value.map((entry, i) => (
<CommaSeparatedLi key={`LogEntryFieldColumn-${i}`}>
{highlightFieldValue(
entry,
isHighlightFieldColumn(firstHighlight) ? firstHighlight.highlights : [],
isActiveHighlight ? ActiveHighlightMarker : HighlightMarker
)}
</CommaSeparatedLi>
))}
</ul>
) : (
highlightFieldValue(
typeof value === 'string' ? value : stringify(value),
isHighlightFieldColumn(firstHighlight) ? firstHighlight.highlights : [],
isActiveHighlight ? ActiveHighlightMarker : HighlightMarker
)
);
return <FieldColumnContent wrapMode={wrapMode}>{formattedValue}</FieldColumnContent>;
};
const CommaSeparatedLi = euiStyled.li`
display: inline;
&:not(:last-child) {
margin-right: 1ex;
&::after {
content: ',';
}
} }
`; };
interface LogEntryColumnContentProps { interface LogEntryColumnContentProps {
wrapMode: WrapMode; wrapMode: WrapMode;

82
x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.test.tsx Normal file
View file

@ -0,0 +1,82 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { render } from '@testing-library/react';
import React from 'react';
import { EuiThemeProvider } from '../../../../../observability/public';
import { LogMessageColumn } from '../../../../common/http_api';
import { LogEntryMessageColumn } from './log_entry_message_column';
describe('LogEntryMessageColumn', () => {
it('renders a single scalar field value without a wrapping list', () => {
const column: LogMessageColumn = {
columnId: 'TEST_COLUMN',
message: [{ field: 'TEST_FIELD', value: ['VALUE'], highlights: [] }],
};
const renderResult = render(
<LogEntryMessageColumn
columnValue={column}
highlights={[]}
isActiveHighlight={false}
wrapMode="pre-wrapped"
/>,
{ wrapper: EuiThemeProvider }
);
expect(renderResult.getByTestId('LogEntryColumnContent')).toHaveTextContent(/^VALUE$/);
expect(renderResult.queryByTestId('LogEntryFieldValues')).toBe(null);
});
it('renders a single array of scalar field values as a list', () => {
const column: LogMessageColumn = {
columnId: 'TEST_COLUMN',
message: [{ field: 'TEST_FIELD', value: ['VALUE_1', 'VALUE_2'], highlights: [] }],
};
const renderResult = render(
<LogEntryMessageColumn
columnValue={column}
highlights={[]}
isActiveHighlight={false}
wrapMode="pre-wrapped"
/>,
{ wrapper: EuiThemeProvider }
);
expect(renderResult.getByTestId('LogEntryFieldValues')).not.toBeEmptyDOMElement();
expect(renderResult.getByTestId('LogEntryFieldValue-0')).toHaveTextContent('VALUE_1');
expect(renderResult.getByTestId('LogEntryFieldValue-1')).toHaveTextContent('VALUE_2');
});
it('renders a complex message with an array of complex field values', () => {
const column: LogMessageColumn = {
columnId: 'TEST_COLUMN',
message: [
{ constant: 'CONSTANT_1' },
{ field: 'TEST_FIELD', value: [{ lat: 1, lon: 2 }, 'VALUE_2'], highlights: [] },
{ constant: 'CONSTANT_2' },
],
};
const renderResult = render(
<LogEntryMessageColumn
columnValue={column}
highlights={[]}
isActiveHighlight={false}
wrapMode="pre-wrapped"
/>,
{ wrapper: EuiThemeProvider }
);
expect(renderResult.getByTestId('LogEntryColumnContent')).toHaveTextContent(
/^CONSTANT_1.*{"lat":1,"lon":2}.*VALUE_2.*CONSTANT_2$/
);
expect(renderResult.getByTestId('LogEntryFieldValues')).not.toBeEmptyDOMElement();
expect(renderResult.getByTestId('LogEntryFieldValue-0')).toHaveTextContent('{"lat":1,"lon":2}');
expect(renderResult.getByTestId('LogEntryFieldValue-1')).toHaveTextContent('VALUE_2');
});
});

55
x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.tsx
View file

@ -5,17 +5,16 @@
*/ */
import React, { memo, useMemo } from 'react'; import React, { memo, useMemo } from 'react';
import stringify from 'json-stable-stringify';
import { euiStyled } from '../../../../../observability/public'; import { euiStyled } from '../../../../../observability/public';
import { LogColumn, LogMessagePart } from '../../../../common/http_api';
import { import {
isConstantSegment, isConstantSegment,
isFieldSegment, isFieldSegment,
isHighlightFieldSegment,
isHighlightMessageColumn, isHighlightMessageColumn,
isMessageColumn, isMessageColumn,
isHighlightFieldSegment,
} from '../../../utils/log_entry'; } from '../../../utils/log_entry';
import { ActiveHighlightMarker, highlightFieldValue, HighlightMarker } from './highlighting'; import { FieldValue } from './field_value';
import { LogEntryColumnContent } from './log_entry_column'; import { LogEntryColumnContent } from './log_entry_column';
import { import {
longWrappedContentStyle, longWrappedContentStyle,
@ -23,7 +22,6 @@ import {
unwrappedContentStyle, unwrappedContentStyle,
WrapMode, WrapMode,
} from './text_styles'; } from './text_styles';
import { LogColumn, LogMessagePart } from '../../../../common/http_api';
interface LogEntryMessageColumnProps { interface LogEntryMessageColumnProps {
columnValue: LogColumn; columnValue: LogColumn;
@ -65,10 +63,10 @@ const formatMessageSegments = (
highlights: LogColumn[], highlights: LogColumn[],
isActiveHighlight: boolean isActiveHighlight: boolean
) => ) =>
messageSegments.map((messageSegment, index) => messageSegments.map((messageSegment, index) => {
formatMessageSegment( if (isFieldSegment(messageSegment)) {
messageSegment, // we only support one highlight for now
highlights.map((highlight) => { const [firstHighlight = []] = highlights.map((highlight) => {
if (isHighlightMessageColumn(highlight)) { if (isHighlightMessageColumn(highlight)) {
const segment = highlight.message[index]; const segment = highlight.message[index];
if (isHighlightFieldSegment(segment)) { if (isHighlightFieldSegment(segment)) {
@ -76,30 +74,19 @@ const formatMessageSegments = (
} }
} }
return []; return [];
}), });
isActiveHighlight
)
);
const formatMessageSegment = ( return (
messageSegment: LogMessagePart, <FieldValue
[firstHighlight = []]: string[][], // we only support one highlight for now highlightTerms={firstHighlight}
isActiveHighlight: boolean isActiveHighlight={isActiveHighlight}
): React.ReactNode => { key={`MessageSegment-${index}`}
if (isFieldSegment(messageSegment)) { value={messageSegment.value}
const value = />
typeof messageSegment.value === 'string' );
? messageSegment.value } else if (isConstantSegment(messageSegment)) {
: stringify(messageSegment.value); return messageSegment.constant;
}
return highlightFieldValue( return 'failed to format message';
value, });
firstHighlight,
isActiveHighlight ? ActiveHighlightMarker : HighlightMarker
);
} else if (isConstantSegment(messageSegment)) {
return messageSegment.constant;
}
return 'failed to format message';
};

4
x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx
View file

@ -86,7 +86,7 @@ export const CategoryExampleMessage: React.FunctionComponent<{
<LogEntryMessageColumn <LogEntryMessageColumn
columnValue={{ columnValue={{
columnId: messageColumnId, columnId: messageColumnId,
message: [{ field: 'message', value: message, highlights: [] }], message: [{ field: 'message', value: [message], highlights: [] }],
}} }}
highlights={noHighlights} highlights={noHighlights}
isActiveHighlight={false} isActiveHighlight={false}
@ -98,7 +98,7 @@ export const CategoryExampleMessage: React.FunctionComponent<{
columnValue={{ columnValue={{
columnId: datasetColumnId, columnId: datasetColumnId,
field: 'event.dataset', field: 'event.dataset',
value: humanFriendlyDataset, value: [humanFriendlyDataset],
highlights: [], highlights: [],
}} }}
highlights={noHighlights} highlights={noHighlights}

4
x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/log_entry_example.tsx
View file

@ -165,7 +165,7 @@ export const LogEntryExampleMessage: React.FunctionComponent<Props> = ({
<LogEntryMessageColumn <LogEntryMessageColumn
columnValue={{ columnValue={{
columnId: messageColumnId, columnId: messageColumnId,
message: [{ field: 'message', value: message, highlights: [] }], message: [{ field: 'message', value: [message], highlights: [] }],
}} }}
highlights={noHighlights} highlights={noHighlights}
isActiveHighlight={false} isActiveHighlight={false}
@ -177,7 +177,7 @@ export const LogEntryExampleMessage: React.FunctionComponent<Props> = ({
columnValue={{ columnValue={{
columnId: datasetColumnId, columnId: datasetColumnId,
field: 'event.dataset', field: 'event.dataset',
value: humanFriendlyDataset, value: [humanFriendlyDataset],
highlights: [], highlights: [],
}} }}
highlights={noHighlights} highlights={noHighlights}

6
x-pack/plugins/infra/server/lib/adapters/framework/adapter_types.ts
View file

@ -15,6 +15,7 @@ import { PluginSetupContract as FeaturesPluginSetup } from '../../../../../../pl
import { SpacesPluginSetup } from '../../../../../../plugins/spaces/server'; import { SpacesPluginSetup } from '../../../../../../plugins/spaces/server';
import { PluginSetupContract as AlertingPluginContract } from '../../../../../alerts/server'; import { PluginSetupContract as AlertingPluginContract } from '../../../../../alerts/server';
import { MlPluginSetup } from '../../../../../ml/server'; import { MlPluginSetup } from '../../../../../ml/server';
import { JsonArray, JsonValue } from '../../../../common/typed_json';
export interface InfraServerPluginDeps { export interface InfraServerPluginDeps {
home: HomeServerPluginSetup; home: HomeServerPluginSetup;
@ -111,7 +112,10 @@ export type SearchHit = SearchResponse<object>['hits']['hits'][0];
export interface SortedSearchHit extends SearchHit { export interface SortedSearchHit extends SearchHit {
sort: any[]; sort: any[];
_source: { _source: {
[field: string]: any; [field: string]: JsonValue;
};
fields: {
[field: string]: JsonArray;
}; };
} }

41
x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts
View file

@ -4,18 +4,17 @@
* you may not use this file except in compliance with the Elastic License. * you may not use this file except in compliance with the Elastic License.
*/ */
/* eslint-disable @typescript-eslint/no-empty-interface */
import { timeMilliseconds } from 'd3-time'; import { timeMilliseconds } from 'd3-time';
import { fold, map } from 'fp-ts/lib/Either';
import { constant, identity } from 'fp-ts/lib/function';
import { pipe } from 'fp-ts/lib/pipeable';
import * as runtimeTypes from 'io-ts'; import * as runtimeTypes from 'io-ts';
import { compact, first } from 'lodash'; import { compact, first } from 'lodash';
import { pipe } from 'fp-ts/lib/pipeable';
import { map, fold } from 'fp-ts/lib/Either';
import { identity, constant } from 'fp-ts/lib/function';
import { RequestHandlerContext } from 'src/core/server'; import { RequestHandlerContext } from 'src/core/server';
import { JsonValue } from '../../../../common/typed_json'; import { JsonArray } from '../../../../common/typed_json';
import { import {
LogEntriesAdapter, LogEntriesAdapter,
LogItemHit,
LogEntriesParams, LogEntriesParams,
LogEntryDocument, LogEntryDocument,
LogEntryQuery, LogEntryQuery,
@ -28,13 +27,6 @@ import { KibanaFramework } from '../framework/kibana_framework_adapter';
const TIMESTAMP_FORMAT = 'epoch_millis'; const TIMESTAMP_FORMAT = 'epoch_millis';
interface LogItemHit {
_index: string;
_id: string;
fields: { [key: string]: [value: unknown] };
sort: [number, number];
}
export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter {
constructor(private readonly framework: KibanaFramework) {} constructor(private readonly framework: KibanaFramework) {}
@ -231,13 +223,14 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter {
function mapHitsToLogEntryDocuments(hits: SortedSearchHit[], fields: string[]): LogEntryDocument[] { function mapHitsToLogEntryDocuments(hits: SortedSearchHit[], fields: string[]): LogEntryDocument[] {
return hits.map((hit) => { return hits.map((hit) => {
const logFields = fields.reduce<{ [fieldName: string]: JsonValue }>( const logFields = fields.reduce<{ [fieldName: string]: JsonArray }>(
(flattenedFields, field) => { (flattenedFields, field) =>
if (field in hit.fields) { field in hit.fields
flattenedFields[field] = hit.fields[field][0]; ? {
} ...flattenedFields,
return flattenedFields; [field]: hit.fields[field],
}, }
: flattenedFields,
{} {}
); );
@ -338,8 +331,9 @@ const LogSummaryDateRangeBucketRuntimeType = runtimeTypes.intersection([
}), }),
]); ]);
export interface LogSummaryDateRangeBucket export type LogSummaryDateRangeBucket = runtimeTypes.TypeOf<
extends runtimeTypes.TypeOf<typeof LogSummaryDateRangeBucketRuntimeType> {} typeof LogSummaryDateRangeBucketRuntimeType
>;
const LogSummaryResponseRuntimeType = runtimeTypes.type({ const LogSummaryResponseRuntimeType = runtimeTypes.type({
aggregations: runtimeTypes.type({ aggregations: runtimeTypes.type({
@ -349,5 +343,4 @@ const LogSummaryResponseRuntimeType = runtimeTypes.type({
}), }),
}); });
export interface LogSummaryResponse export type LogSummaryResponse = runtimeTypes.TypeOf<typeof LogSummaryResponseRuntimeType>;
extends runtimeTypes.TypeOf<typeof LogSummaryResponseRuntimeType> {}

467
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.test.ts
View file

@ -13,251 +13,290 @@ describe('Filebeat Rules', () => {
describe('in ECS format', () => { describe('in ECS format', () => {
test('Apache2 Access', () => { test('Apache2 Access', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2016-12-26T16:22:13.000Z', '@timestamp': ['2016-12-26T16:22:13.000Z'],
'ecs.version': '1.0.0-beta2', 'ecs.version': ['1.0.0-beta2'],
'event.dataset': 'apache.access', 'event.dataset': ['apache.access'],
'event.module': 'apache', 'event.module': ['apache'],
'fileset.name': 'access', 'fileset.name': ['access'],
'http.request.method': 'GET', 'http.request.method': ['GET'],
'http.request.referrer': '-', 'http.request.referrer': ['-'],
'http.response.body.bytes': 499, 'http.response.body.bytes': [499],
'http.response.status_code': 404, 'http.response.status_code': [404],
'http.version': '1.1', 'http.version': ['1.1'],
'input.type': 'log', 'input.type': ['log'],
'log.offset': 73, 'log.offset': [73],
'service.type': 'apache', 'service.type': ['apache'],
'source.address': '192.168.33.1', 'source.address': ['192.168.33.1'],
'source.ip': '192.168.33.1', 'source.ip': ['192.168.33.1'],
'url.original': '/hello', 'url.original': ['/hello'],
'user.name': '-', 'user.name': ['-'],
'user_agent.device': 'Other', 'user_agent.device': ['Other'],
'user_agent.major': '50', 'user_agent.major': ['50'],
'user_agent.minor': '0', 'user_agent.minor': ['0'],
'user_agent.name': 'Firefox', 'user_agent.name': ['Firefox'],
'user_agent.original': 'user_agent.original': [
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0',
'user_agent.os.full_name': 'Mac OS X 10.12', ],
'user_agent.os.major': '10', 'user_agent.os.full_name': ['Mac OS X 10.12'],
'user_agent.os.minor': '12', 'user_agent.os.major': ['10'],
'user_agent.os.name': 'Mac OS X', 'user_agent.os.minor': ['12'],
'user_agent.os.name': ['Mac OS X'],
}; };
const highlights = { const highlights = {
'http.request.method': ['GET'], 'http.request.method': ['GET'],
}; };
expect(format(flattenedDocument, highlights)).toMatchInlineSnapshot(` expect(format(flattenedDocument, highlights)).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[", "constant": "[",
}, },
Object { Object {
"field": "event.module", "field": "event.module",
"highlights": Array [], "highlights": Array [],
"value": "apache", "value": Array [
}, "apache",
Object { ],
"constant": "][access] ", },
}, Object {
Object { "constant": "][access] ",
"field": "source.ip", },
"highlights": Array [], Object {
"value": "192.168.33.1", "field": "source.ip",
}, "highlights": Array [],
Object { "value": Array [
"constant": " ", "192.168.33.1",
}, ],
Object { },
"field": "user.name", Object {
"highlights": Array [], "constant": " ",
"value": "-", },
}, Object {
Object { "field": "user.name",
"constant": " \\"", "highlights": Array [],
}, "value": Array [
Object { "-",
"field": "http.request.method", ],
"highlights": Array [ },
"GET", Object {
], "constant": " \\"",
"value": "GET", },
}, Object {
Object { "field": "http.request.method",
"constant": " ", "highlights": Array [
}, "GET",
Object { ],
"field": "url.original", "value": Array [
"highlights": Array [], "GET",
"value": "/hello", ],
}, },
Object { Object {
"constant": " HTTP/", "constant": " ",
}, },
Object { Object {
"field": "http.version", "field": "url.original",
"highlights": Array [], "highlights": Array [],
"value": "1.1", "value": Array [
}, "/hello",
Object { ],
"constant": "\\" ", },
}, Object {
Object { "constant": " HTTP/",
"field": "http.response.status_code", },
"highlights": Array [], Object {
"value": "404", "field": "http.version",
}, "highlights": Array [],
Object { "value": Array [
"constant": " ", "1.1",
}, ],
Object { },
"field": "http.response.body.bytes", Object {
"highlights": Array [], "constant": "\\" ",
"value": "499", },
}, Object {
] "field": "http.response.status_code",
`); "highlights": Array [],
"value": Array [
404,
],
},
Object {
"constant": " ",
},
Object {
"field": "http.response.body.bytes",
"highlights": Array [],
"value": Array [
499,
],
},
]
`);
}); });
test('Apache2 Error', () => { test('Apache2 Error', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2016-12-26T16:22:08.000Z', '@timestamp': ['2016-12-26T16:22:08.000Z'],
'ecs.version': '1.0.0-beta2', 'ecs.version': ['1.0.0-beta2'],
'event.dataset': 'apache.error', 'event.dataset': ['apache.error'],
'event.module': 'apache', 'event.module': ['apache'],
'fileset.name': 'error', 'fileset.name': ['error'],
'input.type': 'log', 'input.type': ['log'],
'log.level': 'error', 'log.level': ['error'],
'log.offset': 0, 'log.offset': [0],
message: 'File does not exist: /var/www/favicon.ico', message: ['File does not exist: /var/www/favicon.ico'],
'service.type': 'apache', 'service.type': ['apache'],
'source.address': '192.168.33.1', 'source.address': ['192.168.33.1'],
'source.ip': '192.168.33.1', 'source.ip': ['192.168.33.1'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[apache][", "constant": "[apache][",
}, },
Object { Object {
"field": "log.level", "field": "log.level",
"highlights": Array [], "highlights": Array [],
"value": "error", "value": Array [
}, "error",
Object { ],
"constant": "] ", },
}, Object {
Object { "constant": "] ",
"field": "message", },
"highlights": Array [], Object {
"value": "File does not exist: /var/www/favicon.ico", "field": "message",
}, "highlights": Array [],
] "value": Array [
`); "File does not exist: /var/www/favicon.ico",
],
},
]
`);
}); });
}); });
describe('in pre-ECS format', () => { describe('in pre-ECS format', () => {
test('Apache2 Access', () => { test('Apache2 Access', () => {
const flattenedDocument = { const flattenedDocument = {
'apache2.access': true, 'apache2.access.remote_ip': ['192.168.1.42'],
'apache2.access.remote_ip': '192.168.1.42', 'apache2.access.user_name': ['admin'],
'apache2.access.user_name': 'admin', 'apache2.access.method': ['GET'],
'apache2.access.method': 'GET', 'apache2.access.url': ['/faqs'],
'apache2.access.url': '/faqs', 'apache2.access.http_version': ['1.1'],
'apache2.access.http_version': '1.1', 'apache2.access.response_code': ['200'],
'apache2.access.response_code': '200', 'apache2.access.body_sent.bytes': [1024],
'apache2.access.body_sent.bytes': 1024,
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[apache][access] ", "constant": "[apache][access] ",
}, },
Object { Object {
"field": "apache2.access.remote_ip", "field": "apache2.access.remote_ip",
"highlights": Array [], "highlights": Array [],
"value": "192.168.1.42", "value": Array [
}, "192.168.1.42",
Object { ],
"constant": " ", },
}, Object {
Object { "constant": " ",
"field": "apache2.access.user_name", },
"highlights": Array [], Object {
"value": "admin", "field": "apache2.access.user_name",
}, "highlights": Array [],
Object { "value": Array [
"constant": " \\"", "admin",
}, ],
Object { },
"field": "apache2.access.method", Object {
"highlights": Array [], "constant": " \\"",
"value": "GET", },
}, Object {
Object { "field": "apache2.access.method",
"constant": " ", "highlights": Array [],
}, "value": Array [
Object { "GET",
"field": "apache2.access.url", ],
"highlights": Array [], },
"value": "/faqs", Object {
}, "constant": " ",
Object { },
"constant": " HTTP/", Object {
}, "field": "apache2.access.url",
Object { "highlights": Array [],
"field": "apache2.access.http_version", "value": Array [
"highlights": Array [], "/faqs",
"value": "1.1", ],
}, },
Object { Object {
"constant": "\\" ", "constant": " HTTP/",
}, },
Object { Object {
"field": "apache2.access.response_code", "field": "apache2.access.http_version",
"highlights": Array [], "highlights": Array [],
"value": "200", "value": Array [
}, "1.1",
Object { ],
"constant": " ", },
}, Object {
Object { "constant": "\\" ",
"field": "apache2.access.body_sent.bytes", },
"highlights": Array [], Object {
"value": "1024", "field": "apache2.access.response_code",
}, "highlights": Array [],
] "value": Array [
`); "200",
],
},
Object {
"constant": " ",
},
Object {
"field": "apache2.access.body_sent.bytes",
"highlights": Array [],
"value": Array [
1024,
],
},
]
`);
}); });
test('Apache2 Error', () => { test('Apache2 Error', () => {
const flattenedDocument = { const flattenedDocument = {
'apache2.error.message': 'apache2.error.message': [
'AH00489: Apache/2.4.18 (Ubuntu) configured -- resuming normal operations', 'AH00489: Apache/2.4.18 (Ubuntu) configured -- resuming normal operations',
'apache2.error.level': 'notice', ],
'apache2.error.level': ['notice'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[apache][", "constant": "[apache][",
}, },
Object { Object {
"field": "apache2.error.level", "field": "apache2.error.level",
"highlights": Array [], "highlights": Array [],
"value": "notice", "value": Array [
}, "notice",
Object { ],
"constant": "] ", },
}, Object {
Object { "constant": "] ",
"field": "apache2.error.message", },
"highlights": Array [], Object {
"value": "AH00489: Apache/2.4.18 (Ubuntu) configured -- resuming normal operations", "field": "apache2.error.message",
}, "highlights": Array [],
] "value": Array [
`); "AH00489: Apache/2.4.18 (Ubuntu) configured -- resuming normal operations",
],
},
]
`);
}); });
}); });
}); });

2
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.ts
View file

@ -8,7 +8,7 @@ export const filebeatApache2Rules = [
{ {
// pre-ECS // pre-ECS
when: { when: {
exists: ['apache2.access'], existsPrefix: ['apache2.access'],
}, },
format: [ format: [
{ {

808
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.test.ts
View file

@ -13,345 +13,605 @@ describe('Filebeat Rules', () => {
describe('in ECS format', () => { describe('in ECS format', () => {
test('auditd log with outcome', () => { test('auditd log with outcome', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2016-12-07T02:17:21.515Z', '@timestamp': ['2016-12-07T02:17:21.515Z'],
'auditd.log': { 'auditd.log.addr': ['96.241.146.97'],
addr: '96.241.146.97', 'auditd.log.cipher': ['chacha20-poly1305@openssh.com'],
cipher: 'chacha20-poly1305@openssh.com', 'auditd.log.direction': ['from-server'],
direction: 'from-server', 'auditd.log.ksize': ['512'],
ksize: '512', 'auditd.log.laddr': ['10.142.0.2'],
laddr: '10.142.0.2', 'auditd.log.lport': ['22'],
lport: '22', 'auditd.log.pfs': ['curve25519-sha256@libssh.org'],
pfs: 'curve25519-sha256@libssh.org', 'auditd.log.rport': ['63927'],
rport: '63927', 'auditd.log.sequence': [406],
sequence: 406, 'auditd.log.ses': ['4294967295'],
ses: '4294967295', 'auditd.log.spid': ['1299'],
spid: '1299', 'auditd.log.subj': ['system_u:system_r:sshd_t:s0-s0:c0.c1023'],
subj: 'system_u:system_r:sshd_t:s0-s0:c0.c1023', 'ecs.version': ['1.0.0-beta2'],
}, 'event.action': ['crypto_session'],
'ecs.version': '1.0.0-beta2', 'event.dataset': ['auditd.log'],
'event.action': 'crypto_session', 'event.module': ['auditd'],
'event.dataset': 'auditd.log', 'event.outcome': ['success'],
'event.module': 'auditd', 'fileset.name': ['log'],
'event.outcome': 'success', 'input.type': ['log'],
'fileset.name': 'log', 'log.offset': [783],
'input.type': 'log', message: ['op=start'],
'log.offset': 783, 'process.executable': ['/usr/sbin/sshd'],
message: 'op=start', 'process.pid': [1298],
process: { executable: '/usr/sbin/sshd', pid: 1298 }, 'service.type': ['auditd'],
'service.type': 'auditd', 'user.audit.id': ['4294967295'],
user: { 'audit.id': '4294967295', id: '0', 'saved.id': '74' }, 'user.id': ['0'],
'user.saved.id': ['74'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[AuditD][", "constant": "[AuditD][",
}, },
Object { Object {
"field": "event.action", "field": "event.action",
"highlights": Array [], "highlights": Array [],
"value": "crypto_session", "value": Array [
}, "crypto_session",
Object { ],
"constant": "]", },
}, Object {
Object { "constant": "]",
"constant": " ", },
}, Object {
Object { "constant": " ",
"field": "event.outcome", },
"highlights": Array [], Object {
"value": "success", "field": "event.outcome",
}, "highlights": Array [],
Object { "value": Array [
"constant": " ", "success",
}, ],
Object { },
"constant": "user", Object {
}, "constant": " ",
Object { },
"constant": "=", Object {
}, "constant": "user",
Object { },
"field": "user", Object {
"highlights": Array [], "constant": "=",
"value": "{\\"audit.id\\":\\"4294967295\\",\\"id\\":\\"0\\",\\"saved.id\\":\\"74\\"}", },
}, Object {
Object { "field": "user.audit.id",
"constant": " ", "highlights": Array [],
}, "value": Array [
Object { "4294967295",
"constant": "process", ],
}, },
Object { Object {
"constant": "=", "field": "user.id",
}, "highlights": Array [],
Object { "value": Array [
"field": "process", "0",
"highlights": Array [], ],
"value": "{\\"executable\\":\\"/usr/sbin/sshd\\",\\"pid\\":1298}", },
}, Object {
Object { "field": "user.saved.id",
"constant": " ", "highlights": Array [],
}, "value": Array [
Object { "74",
"field": "auditd.log", ],
"highlights": Array [], },
"value": "{\\"addr\\":\\"96.241.146.97\\",\\"cipher\\":\\"chacha20-poly1305@openssh.com\\",\\"direction\\":\\"from-server\\",\\"ksize\\":\\"512\\",\\"laddr\\":\\"10.142.0.2\\",\\"lport\\":\\"22\\",\\"pfs\\":\\"curve25519-sha256@libssh.org\\",\\"rport\\":\\"63927\\",\\"sequence\\":406,\\"ses\\":\\"4294967295\\",\\"spid\\":\\"1299\\",\\"subj\\":\\"system_u:system_r:sshd_t:s0-s0:c0.c1023\\"}", Object {
}, "constant": " ",
Object { },
"constant": " ", Object {
}, "constant": "process",
Object { },
"field": "message", Object {
"highlights": Array [], "constant": "=",
"value": "op=start", },
}, Object {
] "field": "process.executable",
`); "highlights": Array [],
"value": Array [
"/usr/sbin/sshd",
],
},
Object {
"field": "process.pid",
"highlights": Array [],
"value": Array [
1298,
],
},
Object {
"constant": " ",
},
Object {
"field": "auditd.log.addr",
"highlights": Array [],
"value": Array [
"96.241.146.97",
],
},
Object {
"field": "auditd.log.cipher",
"highlights": Array [],
"value": Array [
"chacha20-poly1305@openssh.com",
],
},
Object {
"field": "auditd.log.direction",
"highlights": Array [],
"value": Array [
"from-server",
],
},
Object {
"field": "auditd.log.ksize",
"highlights": Array [],
"value": Array [
"512",
],
},
Object {
"field": "auditd.log.laddr",
"highlights": Array [],
"value": Array [
"10.142.0.2",
],
},
Object {
"field": "auditd.log.lport",
"highlights": Array [],
"value": Array [
"22",
],
},
Object {
"field": "auditd.log.pfs",
"highlights": Array [],
"value": Array [
"curve25519-sha256@libssh.org",
],
},
Object {
"field": "auditd.log.rport",
"highlights": Array [],
"value": Array [
"63927",
],
},
Object {
"field": "auditd.log.sequence",
"highlights": Array [],
"value": Array [
406,
],
},
Object {
"field": "auditd.log.ses",
"highlights": Array [],
"value": Array [
"4294967295",
],
},
Object {
"field": "auditd.log.spid",
"highlights": Array [],
"value": Array [
"1299",
],
},
Object {
"field": "auditd.log.subj",
"highlights": Array [],
"value": Array [
"system_u:system_r:sshd_t:s0-s0:c0.c1023",
],
},
Object {
"constant": " ",
},
Object {
"field": "message",
"highlights": Array [],
"value": Array [
"op=start",
],
},
]
`);
}); });
test('auditd log without outcome', () => { test('auditd log without outcome', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2017-01-31T20:17:14.891Z', '@timestamp': ['2017-01-31T20:17:14.891Z'],
'auditd.log': { 'auditd.log.a0': ['9'],
a0: '9', 'auditd.log.a1': ['7f564b2672a0'],
a1: '7f564b2672a0', 'auditd.log.a2': ['b8'],
a2: 'b8', 'auditd.log.a3': ['0'],
a3: '0', 'auditd.log.exit': ['184'],
exit: '184', 'auditd.log.items': ['0'],
items: '0', 'auditd.log.sequence': [18877199],
sequence: 18877199, 'auditd.log.ses': ['4294967295'],
ses: '4294967295', 'auditd.log.success': ['yes'],
success: 'yes', 'auditd.log.syscall': ['44'],
syscall: '44', 'auditd.log.tty': ['(none)'],
tty: '(none)', 'ecs.version': ['1.0.0-beta2'],
}, 'event.action': ['syscall'],
'ecs.version': '1.0.0-beta2', 'event.dataset': ['auditd.log'],
'event.action': 'syscall', 'event.module': ['auditd'],
'event.dataset': 'auditd.log', 'fileset.name': ['log'],
'event.module': 'auditd', 'host.architecture': ['x86_64'],
'fileset.name': 'log', 'input.type': ['log'],
'host.architecture': 'x86_64', 'log.offset': [174],
'input.type': 'log', 'process.executable': ['/usr/libexec/strongswan/charon (deleted)'],
'log.offset': 174, 'process.name': ['charon'],
process: { 'process.pid': [1281],
executable: '/usr/libexec/strongswan/charon (deleted)', 'process.ppid': [1240],
name: 'charon', 'service.type': ['auditd'],
pid: 1281, 'user.audit.id': ['4294967295'],
ppid: 1240, 'user.effective.group.id': ['0'],
}, 'user.effective.id': ['0'],
'service.type': 'auditd', 'user.filesystem.group.id': ['0'],
user: { 'user.filesystem.id': ['0'],
'audit.id': '4294967295', 'user.group.id': ['0'],
'effective.group.id': '0', 'user.id': ['0'],
'effective.id': '0', 'user.saved.group.id': ['0'],
'filesystem.group.id': '0', 'user.saved.id': ['0'],
'filesystem.id': '0',
'group.id': '0',
id: '0',
'saved.group.id': '0',
'saved.id': '0',
},
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[AuditD][", "constant": "[AuditD][",
}, },
Object { Object {
"field": "event.action", "field": "event.action",
"highlights": Array [], "highlights": Array [],
"value": "syscall", "value": Array [
}, "syscall",
Object { ],
"constant": "]", },
}, Object {
Object { "constant": "]",
"constant": " ", },
}, Object {
Object { "constant": " ",
"constant": "user", },
}, Object {
Object { "constant": "user",
"constant": "=", },
}, Object {
Object { "constant": "=",
"field": "user", },
"highlights": Array [], Object {
"value": "{\\"audit.id\\":\\"4294967295\\",\\"effective.group.id\\":\\"0\\",\\"effective.id\\":\\"0\\",\\"filesystem.group.id\\":\\"0\\",\\"filesystem.id\\":\\"0\\",\\"group.id\\":\\"0\\",\\"id\\":\\"0\\",\\"saved.group.id\\":\\"0\\",\\"saved.id\\":\\"0\\"}", "field": "user.audit.id",
}, "highlights": Array [],
Object { "value": Array [
"constant": " ", "4294967295",
}, ],
Object { },
"constant": "process", Object {
}, "field": "user.effective.group.id",
Object { "highlights": Array [],
"constant": "=", "value": Array [
}, "0",
Object { ],
"field": "process", },
"highlights": Array [], Object {
"value": "{\\"executable\\":\\"/usr/libexec/strongswan/charon (deleted)\\",\\"name\\":\\"charon\\",\\"pid\\":1281,\\"ppid\\":1240}", "field": "user.effective.id",
}, "highlights": Array [],
Object { "value": Array [
"constant": " ", "0",
}, ],
Object { },
"field": "auditd.log", Object {
"highlights": Array [], "field": "user.filesystem.group.id",
"value": "{\\"a0\\":\\"9\\",\\"a1\\":\\"7f564b2672a0\\",\\"a2\\":\\"b8\\",\\"a3\\":\\"0\\",\\"exit\\":\\"184\\",\\"items\\":\\"0\\",\\"sequence\\":18877199,\\"ses\\":\\"4294967295\\",\\"success\\":\\"yes\\",\\"syscall\\":\\"44\\",\\"tty\\":\\"(none)\\"}", "highlights": Array [],
}, "value": Array [
Object { "0",
"constant": " ", ],
}, },
Object { Object {
"field": "message", "field": "user.filesystem.id",
"highlights": Array [], "highlights": Array [],
"value": "undefined", "value": Array [
}, "0",
] ],
`); },
Object {
"field": "user.group.id",
"highlights": Array [],
"value": Array [
"0",
],
},
Object {
"field": "user.id",
"highlights": Array [],
"value": Array [
"0",
],
},
Object {
"field": "user.saved.group.id",
"highlights": Array [],
"value": Array [
"0",
],
},
Object {
"field": "user.saved.id",
"highlights": Array [],
"value": Array [
"0",
],
},
Object {
"constant": " ",
},
Object {
"constant": "process",
},
Object {
"constant": "=",
},
Object {
"field": "process.executable",
"highlights": Array [],
"value": Array [
"/usr/libexec/strongswan/charon (deleted)",
],
},
Object {
"field": "process.name",
"highlights": Array [],
"value": Array [
"charon",
],
},
Object {
"field": "process.pid",
"highlights": Array [],
"value": Array [
1281,
],
},
Object {
"field": "process.ppid",
"highlights": Array [],
"value": Array [
1240,
],
},
Object {
"constant": " ",
},
Object {
"field": "auditd.log.a0",
"highlights": Array [],
"value": Array [
"9",
],
},
Object {
"field": "auditd.log.a1",
"highlights": Array [],
"value": Array [
"7f564b2672a0",
],
},
Object {
"field": "auditd.log.a2",
"highlights": Array [],
"value": Array [
"b8",
],
},
Object {
"field": "auditd.log.a3",
"highlights": Array [],
"value": Array [
"0",
],
},
Object {
"field": "auditd.log.exit",
"highlights": Array [],
"value": Array [
"184",
],
},
Object {
"field": "auditd.log.items",
"highlights": Array [],
"value": Array [
"0",
],
},
Object {
"field": "auditd.log.sequence",
"highlights": Array [],
"value": Array [
18877199,
],
},
Object {
"field": "auditd.log.ses",
"highlights": Array [],
"value": Array [
"4294967295",
],
},
Object {
"field": "auditd.log.success",
"highlights": Array [],
"value": Array [
"yes",
],
},
Object {
"field": "auditd.log.syscall",
"highlights": Array [],
"value": Array [
"44",
],
},
Object {
"field": "auditd.log.tty",
"highlights": Array [],
"value": Array [
"(none)",
],
},
Object {
"constant": " ",
},
Object {
"field": "message",
"highlights": Array [],
"value": Array [],
},
]
`);
}); });
}); });
describe('in pre-ECS format', () => { describe('in pre-ECS format', () => {
test('auditd IPSEC rule', () => { test('auditd IPSEC rule', () => {
const event = { const event = {
'@timestamp': '2017-01-31T20:17:14.891Z', '@timestamp': ['2017-01-31T20:17:14.891Z'],
'auditd.log.auid': '4294967295', 'auditd.log.auid': ['4294967295'],
'auditd.log.dst': '192.168.0.0', 'auditd.log.dst': ['192.168.0.0'],
'auditd.log.dst_prefixlen': '16', 'auditd.log.dst_prefixlen': ['16'],
'auditd.log.op': 'SPD-delete', 'auditd.log.op': ['SPD-delete'],
'auditd.log.record_type': 'MAC_IPSEC_EVENT', 'auditd.log.record_type': ['MAC_IPSEC_EVENT'],
'auditd.log.res': '1', 'auditd.log.res': ['1'],
'auditd.log.sequence': 18877201, 'auditd.log.sequence': [18877201],
'auditd.log.ses': '4294967295', 'auditd.log.ses': ['4294967295'],
'auditd.log.src': '192.168.2.0', 'auditd.log.src': ['192.168.2.0'],
'auditd.log.src_prefixlen': '24', 'auditd.log.src_prefixlen': ['24'],
'ecs.version': '1.0.0-beta2', 'ecs.version': ['1.0.0-beta2'],
'event.dataset': 'auditd.log', 'event.dataset': ['auditd.log'],
'event.module': 'auditd', 'event.module': ['auditd'],
'fileset.name': 'log', 'fileset.name': ['log'],
'input.type': 'log', 'input.type': ['log'],
'log.offset': 0, 'log.offset': [0],
}; };
const message = format(event, {}); const message = format(event, {});
expect(message).toEqual([ expect(message).toEqual([
{ constant: '[AuditD][' }, { constant: '[AuditD][' },
{ field: 'auditd.log.record_type', highlights: [], value: 'MAC_IPSEC_EVENT' }, { field: 'auditd.log.record_type', highlights: [], value: ['MAC_IPSEC_EVENT'] },
{ constant: '] src:' }, { constant: '] src:' },
{ field: 'auditd.log.src', highlights: [], value: '192.168.2.0' }, { field: 'auditd.log.src', highlights: [], value: ['192.168.2.0'] },
{ constant: ' dst:' }, { constant: ' dst:' },
{ field: 'auditd.log.dst', highlights: [], value: '192.168.0.0' }, { field: 'auditd.log.dst', highlights: [], value: ['192.168.0.0'] },
{ constant: ' op:' }, { constant: ' op:' },
{ field: 'auditd.log.op', highlights: [], value: 'SPD-delete' }, { field: 'auditd.log.op', highlights: [], value: ['SPD-delete'] },
]); ]);
}); });
test('AuditD SYSCALL rule', () => { test('AuditD SYSCALL rule', () => {
const event = { const event = {
'@timestamp': '2017-01-31T20:17:14.891Z', '@timestamp': ['2017-01-31T20:17:14.891Z'],
'auditd.log.a0': '9', 'auditd.log.a0': ['9'],
'auditd.log.a1': '7f564b2672a0', 'auditd.log.a1': ['7f564b2672a0'],
'auditd.log.a2': 'b8', 'auditd.log.a2': ['b8'],
'auditd.log.a3': '0', 'auditd.log.a3': ['0'],
'auditd.log.arch': 'x86_64', 'auditd.log.arch': ['x86_64'],
'auditd.log.auid': '4294967295', 'auditd.log.auid': ['4294967295'],
'auditd.log.comm': 'charon', 'auditd.log.comm': ['charon'],
'auditd.log.egid': '0', 'auditd.log.egid': ['0'],
'auditd.log.euid': '0', 'auditd.log.euid': ['0'],
'auditd.log.exe': '/usr/libexec/strongswan/charon (deleted)', 'auditd.log.exe': ['/usr/libexec/strongswan/charon (deleted)'],
'auditd.log.exit': '184', 'auditd.log.exit': ['184'],
'auditd.log.fsgid': '0', 'auditd.log.fsgid': ['0'],
'auditd.log.fsuid': '0', 'auditd.log.fsuid': ['0'],
'auditd.log.gid': '0', 'auditd.log.gid': ['0'],
'auditd.log.items': '0', 'auditd.log.items': ['0'],
'auditd.log.pid': '1281', 'auditd.log.pid': ['1281'],
'auditd.log.ppid': '1240', 'auditd.log.ppid': ['1240'],
'auditd.log.record_type': 'SYSCALL', 'auditd.log.record_type': ['SYSCALL'],
'auditd.log.sequence': 18877199, 'auditd.log.sequence': [18877199],
'auditd.log.ses': '4294967295', 'auditd.log.ses': ['4294967295'],
'auditd.log.sgid': '0', 'auditd.log.sgid': ['0'],
'auditd.log.success': 'yes', 'auditd.log.success': ['yes'],
'auditd.log.suid': '0', 'auditd.log.suid': ['0'],
'auditd.log.syscall': '44', 'auditd.log.syscall': ['44'],
'auditd.log.tty': '(none)', 'auditd.log.tty': ['(none)'],
'auditd.log.uid': '0', 'auditd.log.uid': ['0'],
'ecs.version': '1.0.0-beta2', 'ecs.version': ['1.0.0-beta2'],
'event.dataset': 'auditd.log', 'event.dataset': ['auditd.log'],
'event.module': 'auditd', 'event.module': ['auditd'],
'fileset.name': 'log', 'fileset.name': ['log'],
'input.type': 'log', 'input.type': ['log'],
'log.offset': 174, 'log.offset': [174],
}; };
const message = format(event, {}); const message = format(event, {});
expect(message).toEqual([ expect(message).toEqual([
{ constant: '[AuditD][' }, { constant: '[AuditD][' },
{ field: 'auditd.log.record_type', highlights: [], value: 'SYSCALL' }, { field: 'auditd.log.record_type', highlights: [], value: ['SYSCALL'] },
{ constant: '] exe:' }, { constant: '] exe:' },
{ {
field: 'auditd.log.exe', field: 'auditd.log.exe',
highlights: [], highlights: [],
value: '/usr/libexec/strongswan/charon (deleted)', value: ['/usr/libexec/strongswan/charon (deleted)'],
}, },
{ constant: ' gid:' }, { constant: ' gid:' },
{ field: 'auditd.log.gid', highlights: [], value: '0' }, { field: 'auditd.log.gid', highlights: [], value: ['0'] },
{ constant: ' uid:' }, { constant: ' uid:' },
{ field: 'auditd.log.uid', highlights: [], value: '0' }, { field: 'auditd.log.uid', highlights: [], value: ['0'] },
{ constant: ' tty:' }, { constant: ' tty:' },
{ field: 'auditd.log.tty', highlights: [], value: '(none)' }, { field: 'auditd.log.tty', highlights: [], value: ['(none)'] },
{ constant: ' pid:' }, { constant: ' pid:' },
{ field: 'auditd.log.pid', highlights: [], value: '1281' }, { field: 'auditd.log.pid', highlights: [], value: ['1281'] },
{ constant: ' ppid:' }, { constant: ' ppid:' },
{ field: 'auditd.log.ppid', highlights: [], value: '1240' }, { field: 'auditd.log.ppid', highlights: [], value: ['1240'] },
]); ]);
}); });
test('AuditD events with msg rule', () => { test('AuditD events with msg rule', () => {
const event = { const event = {
'@timestamp': '2017-01-31T20:17:14.891Z', '@timestamp': ['2017-01-31T20:17:14.891Z'],
'auditd.log.auid': '4294967295', 'auditd.log.auid': ['4294967295'],
'auditd.log.record_type': 'EXAMPLE', 'auditd.log.record_type': ['EXAMPLE'],
'auditd.log.msg': 'some kind of message', 'auditd.log.msg': ['some kind of message'],
'ecs.version': '1.0.0-beta2', 'ecs.version': ['1.0.0-beta2'],
'event.dataset': 'auditd.log', 'event.dataset': ['auditd.log'],
'event.module': 'auditd', 'event.module': ['auditd'],
'fileset.name': 'log', 'fileset.name': ['log'],
'input.type': 'log', 'input.type': ['log'],
'log.offset': 174, 'log.offset': [174],
}; };
const message = format(event, {}); const message = format(event, {});
expect(message).toEqual([ expect(message).toEqual([
{ constant: '[AuditD][' }, { constant: '[AuditD][' },
{ field: 'auditd.log.record_type', highlights: [], value: 'EXAMPLE' }, { field: 'auditd.log.record_type', highlights: [], value: ['EXAMPLE'] },
{ constant: '] ' }, { constant: '] ' },
{ {
field: 'auditd.log.msg', field: 'auditd.log.msg',
highlights: [], highlights: [],
value: 'some kind of message', value: ['some kind of message'],
}, },
]); ]);
}); });
test('AuditD catchall rule', () => { test('AuditD catchall rule', () => {
const event = { const event = {
'@timestamp': '2017-01-31T20:17:14.891Z', '@timestamp': ['2017-01-31T20:17:14.891Z'],
'auditd.log.auid': '4294967295', 'auditd.log.auid': ['4294967295'],
'auditd.log.record_type': 'EXAMPLE', 'auditd.log.record_type': ['EXAMPLE'],
'ecs.version': '1.0.0-beta2', 'ecs.version': ['1.0.0-beta2'],
'event.dataset': 'auditd.log', 'event.dataset': ['auditd.log'],
'event.module': 'auditd', 'event.module': ['auditd'],
'fileset.name': 'log', 'fileset.name': ['log'],
'input.type': 'log', 'input.type': ['log'],
'log.offset': 174, 'log.offset': [174],
}; };
const message = format(event, {}); const message = format(event, {});
expect(message).toEqual([ expect(message).toEqual([
{ constant: '[AuditD][' }, { constant: '[AuditD][' },
{ field: 'auditd.log.record_type', highlights: [], value: 'EXAMPLE' }, { field: 'auditd.log.record_type', highlights: [], value: ['EXAMPLE'] },
{ constant: '] Event without message.' }, { constant: '] Event without message.' },
]); ]);
}); });

56
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.ts
View file

@ -4,24 +4,28 @@
* you may not use this file except in compliance with the Elastic License. * you may not use this file except in compliance with the Elastic License.
*/ */
import { labelField } from './helpers'; import { LogMessageFormattingRule } from '../rule_types';
import { labelFieldsPrefix } from './helpers';
const commonActionField = [{ constant: '[AuditD][' }, { field: 'event.action' }, { constant: ']' }]; const commonActionField = [{ constant: '[AuditD][' }, { field: 'event.action' }, { constant: ']' }];
const commonOutcomeField = [{ constant: ' ' }, { field: 'event.outcome' }]; const commonOutcomeField = [{ constant: ' ' }, { field: 'event.outcome' }];
export const filebeatAuditdRules = [ export const filebeatAuditdRules: LogMessageFormattingRule[] = [
{ {
// ECS format with outcome // ECS format with outcome
when: { when: {
exists: ['ecs.version', 'event.action', 'event.outcome', 'auditd.log'], all: [
{ exists: ['ecs.version', 'event.action', 'event.outcome'] },
{ existsPrefix: ['auditd.log'] },
],
}, },
format: [ format: [
...commonActionField, ...commonActionField,
...commonOutcomeField, ...commonOutcomeField,
...labelField('user', 'user'), ...labelFieldsPrefix('user', 'user'),
...labelField('process', 'process'), ...labelFieldsPrefix('process', 'process'),
{ constant: ' ' }, { constant: ' ' },
{ field: 'auditd.log' }, { fieldsPrefix: 'auditd.log' },
{ constant: ' ' }, { constant: ' ' },
{ field: 'message' }, { field: 'message' },
], ],
@ -29,14 +33,14 @@ export const filebeatAuditdRules = [
{ {
// ECS format without outcome // ECS format without outcome
when: { when: {
exists: ['ecs.version', 'event.action', 'auditd.log'], all: [{ exists: ['ecs.version', 'event.action'] }, { existsPrefix: ['auditd.log'] }],
}, },
format: [ format: [
...commonActionField, ...commonActionField,
...labelField('user', 'user'), ...labelFieldsPrefix('user', 'user'),
...labelField('process', 'process'), ...labelFieldsPrefix('process', 'process'),
{ constant: ' ' }, { constant: ' ' },
{ field: 'auditd.log' }, { fieldsPrefix: 'auditd.log' },
{ constant: ' ' }, { constant: ' ' },
{ field: 'message' }, { field: 'message' },
], ],
@ -44,10 +48,10 @@ export const filebeatAuditdRules = [
{ {
// pre-ECS IPSEC_EVENT Rule // pre-ECS IPSEC_EVENT Rule
when: { when: {
exists: ['auditd.log.record_type', 'auditd.log.src', 'auditd.log.dst', 'auditd.log.op'], all: [
values: { { exists: ['auditd.log.record_type', 'auditd.log.src', 'auditd.log.dst', 'auditd.log.op'] },
'auditd.log.record_type': 'MAC_IPSEC_EVENT', { values: { 'auditd.log.record_type': 'MAC_IPSEC_EVENT' } },
}, ],
}, },
format: [ format: [
{ constant: '[AuditD][' }, { constant: '[AuditD][' },
@ -63,18 +67,20 @@ export const filebeatAuditdRules = [
{ {
// pre-ECS SYSCALL Rule // pre-ECS SYSCALL Rule
when: { when: {
exists: [ all: [
'auditd.log.record_type', {
'auditd.log.exe', exists: [
'auditd.log.gid', 'auditd.log.record_type',
'auditd.log.uid', 'auditd.log.exe',
'auditd.log.tty', 'auditd.log.gid',
'auditd.log.pid', 'auditd.log.uid',
'auditd.log.ppid', 'auditd.log.tty',
'auditd.log.pid',
'auditd.log.ppid',
],
},
{ values: { 'auditd.log.record_type': 'SYSCALL' } },
], ],
values: {
'auditd.log.record_type': 'SYSCALL',
},
}, },
format: [ format: [
{ constant: '[AuditD][' }, { constant: '[AuditD][' },

1596
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.test.ts
View file

File diff suppressed because it is too large Load diff

240
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.test.ts
View file

@ -13,135 +13,155 @@ describe('Filebeat Rules', () => {
describe('in pre-ECS format', () => { describe('in pre-ECS format', () => {
test('icinga debug log', () => { test('icinga debug log', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2017-04-04T11:43:09.000Z', '@timestamp': ['2017-04-04T11:43:09.000Z'],
'event.dataset': 'icinga.debug', 'event.dataset': ['icinga.debug'],
'fileset.module': 'icinga', 'fileset.module': ['icinga'],
'fileset.name': 'debug', 'fileset.name': ['debug'],
'icinga.debug.facility': 'GraphiteWriter', 'icinga.debug.facility': ['GraphiteWriter'],
'icinga.debug.message': 'icinga.debug.message': [
"Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.",
'icinga.debug.severity': 'debug', ],
'input.type': 'log', 'icinga.debug.severity': ['debug'],
offset: 0, 'input.type': ['log'],
'prospector.type': 'log', offset: [0],
'prospector.type': ['log'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[Icinga][", "constant": "[Icinga][",
}, },
Object { Object {
"field": "icinga.debug.facility", "field": "icinga.debug.facility",
"highlights": Array [], "highlights": Array [],
"value": "GraphiteWriter", "value": Array [
}, "GraphiteWriter",
Object { ],
"constant": "][", },
}, Object {
Object { "constant": "][",
"field": "icinga.debug.severity", },
"highlights": Array [], Object {
"value": "debug", "field": "icinga.debug.severity",
}, "highlights": Array [],
Object { "value": Array [
"constant": "] ", "debug",
}, ],
Object { },
"field": "icinga.debug.message", Object {
"highlights": Array [], "constant": "] ",
"value": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", },
}, Object {
] "field": "icinga.debug.message",
`); "highlights": Array [],
"value": Array [
"Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.",
],
},
]
`);
}); });
test('icinga main log', () => { test('icinga main log', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2017-04-04T09:16:34.000Z', '@timestamp': ['2017-04-04T09:16:34.000Z'],
'event.dataset': 'icinga.main', 'event.dataset': ['icinga.main'],
'fileset.module': 'icinga', 'fileset.module': ['icinga'],
'fileset.name': 'main', 'fileset.name': ['main'],
'icinga.main.facility': 'Notification', 'icinga.main.facility': ['Notification'],
'icinga.main.message': 'icinga.main.message': [
"Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'",
'icinga.main.severity': 'information', ],
'input.type': 'log', 'icinga.main.severity': ['information'],
offset: 0, 'input.type': ['log'],
'prospector.type': 'log', offset: [0],
'prospector.type': ['log'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[Icinga][", "constant": "[Icinga][",
}, },
Object { Object {
"field": "icinga.main.facility", "field": "icinga.main.facility",
"highlights": Array [], "highlights": Array [],
"value": "Notification", "value": Array [
}, "Notification",
Object { ],
"constant": "][", },
}, Object {
Object { "constant": "][",
"field": "icinga.main.severity", },
"highlights": Array [], Object {
"value": "information", "field": "icinga.main.severity",
}, "highlights": Array [],
Object { "value": Array [
"constant": "] ", "information",
}, ],
Object { },
"field": "icinga.main.message", Object {
"highlights": Array [], "constant": "] ",
"value": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", },
}, Object {
] "field": "icinga.main.message",
`); "highlights": Array [],
"value": Array [
"Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'",
],
},
]
`);
}); });
test('icinga startup log', () => { test('icinga startup log', () => {
const flattenedDocument = { const flattenedDocument = {
'event.dataset': 'icinga.startup', 'event.dataset': ['icinga.startup'],
'fileset.module': 'icinga', 'fileset.module': ['icinga'],
'fileset.name': 'startup', 'fileset.name': ['startup'],
'icinga.startup.facility': 'cli', 'icinga.startup.facility': ['cli'],
'icinga.startup.message': 'Icinga application loader (version: r2.6.3-1)', 'icinga.startup.message': ['Icinga application loader (version: r2.6.3-1)'],
'icinga.startup.severity': 'information', 'icinga.startup.severity': ['information'],
'input.type': 'log', 'input.type': ['log'],
offset: 0, offset: [0],
'prospector.type': 'log', 'prospector.type': ['log'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[Icinga][", "constant": "[Icinga][",
}, },
Object { Object {
"field": "icinga.startup.facility", "field": "icinga.startup.facility",
"highlights": Array [], "highlights": Array [],
"value": "cli", "value": Array [
}, "cli",
Object { ],
"constant": "][", },
}, Object {
Object { "constant": "][",
"field": "icinga.startup.severity", },
"highlights": Array [], Object {
"value": "information", "field": "icinga.startup.severity",
}, "highlights": Array [],
Object { "value": Array [
"constant": "] ", "information",
}, ],
Object { },
"field": "icinga.startup.message", Object {
"highlights": Array [], "constant": "] ",
"value": "Icinga application loader (version: r2.6.3-1)", },
}, Object {
] "field": "icinga.startup.message",
`); "highlights": Array [],
"value": Array [
"Icinga application loader (version: r2.6.3-1)",
],
},
]
`);
}); });
}); });
}); });

1074
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts
View file

File diff suppressed because it is too large Load diff

84
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_kafka.test.ts
View file

@ -13,48 +13,54 @@ describe('Filebeat Rules', () => {
describe('in ECS format', () => { describe('in ECS format', () => {
test('kafka log', () => { test('kafka log', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2017-08-04T10:48:21.063Z', '@timestamp': ['2017-08-04T10:48:21.063Z'],
'ecs.version': '1.0.0-beta2', 'ecs.version': ['1.0.0-beta2'],
'event.dataset': 'kafka.log', 'event.dataset': ['kafka.log'],
'event.module': 'kafka', 'event.module': ['kafka'],
'fileset.name': 'log', 'fileset.name': ['log'],
'input.type': 'log', 'input.type': ['log'],
'kafka.log.class': 'kafka.controller.KafkaController', 'kafka.log.class': ['kafka.controller.KafkaController'],
'kafka.log.component': 'Controller 0', 'kafka.log.component': ['Controller 0'],
'log.level': 'INFO', 'log.level': ['INFO'],
'log.offset': 131, 'log.offset': [131],
message: '0 successfully elected as the controller', message: ['0 successfully elected as the controller'],
'service.type': 'kafka', 'service.type': ['kafka'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[", "constant": "[",
}, },
Object { Object {
"field": "event.dataset", "field": "event.dataset",
"highlights": Array [], "highlights": Array [],
"value": "kafka.log", "value": Array [
}, "kafka.log",
Object { ],
"constant": "][", },
}, Object {
Object { "constant": "][",
"field": "log.level", },
"highlights": Array [], Object {
"value": "INFO", "field": "log.level",
}, "highlights": Array [],
Object { "value": Array [
"constant": "] ", "INFO",
}, ],
Object { },
"field": "message", Object {
"highlights": Array [], "constant": "] ",
"value": "0 successfully elected as the controller", },
}, Object {
] "field": "message",
`); "highlights": Array [],
"value": Array [
"0 successfully elected as the controller",
],
},
]
`);
}); });
}); });
}); });

376
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts
View file

@ -13,194 +13,256 @@ describe('Filebeat Rules', () => {
describe('in ECS format', () => { describe('in ECS format', () => {
test('logstash log', () => { test('logstash log', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2017-10-23T14:20:12.046Z', '@timestamp': ['2017-10-23T14:20:12.046Z'],
'ecs.version': '1.0.0-beta2', 'ecs.version': ['1.0.0-beta2'],
'event.dataset': 'logstash.log', 'event.dataset': ['logstash.log'],
'event.module': 'logstash', 'event.module': ['logstash'],
'fileset.name': 'log', 'fileset.name': ['log'],
'input.type': 'log', 'input.type': ['log'],
'log.level': 'INFO', 'log.level': ['INFO'],
'log.offset': 0, 'log.offset': [0],
'logstash.log.module': 'logstash.modules.scaffold', 'logstash.log.module': ['logstash.modules.scaffold'],
message: message: [
'Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}', 'Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}',
'service.type': 'logstash', ],
'service.type': ['logstash'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[", "constant": "[",
}, },
Object { Object {
"field": "event.dataset", "field": "event.dataset",
"highlights": Array [], "highlights": Array [],
"value": "logstash.log", "value": Array [
}, "logstash.log",
Object { ],
"constant": "][", },
}, Object {
Object { "constant": "][",
"field": "log.level", },
"highlights": Array [], Object {
"value": "INFO", "field": "log.level",
}, "highlights": Array [],
Object { "value": Array [
"constant": "] ", "INFO",
}, ],
Object { },
"field": "message", Object {
"highlights": Array [], "constant": "] ",
"value": "Initializing module {:module_name=>\\"fb_apache\\", :directory=>\\"/usr/share/logstash/modules/fb_apache/configuration\\"}", },
}, Object {
] "field": "message",
`); "highlights": Array [],
"value": Array [
"Initializing module {:module_name=>\\"fb_apache\\", :directory=>\\"/usr/share/logstash/modules/fb_apache/configuration\\"}",
],
},
]
`);
}); });
test('logstash slowlog', () => { test('logstash slowlog', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2017-10-30T09:57:58.243Z', '@timestamp': ['2017-10-30T09:57:58.243Z'],
'ecs.version': '1.0.0-beta2', 'ecs.version': ['1.0.0-beta2'],
'event.dataset': 'logstash.slowlog', 'event.dataset': ['logstash.slowlog'],
'event.duration': 3027675106, 'event.duration': [3027675106],
'event.module': 'logstash', 'event.module': ['logstash'],
'fileset.name': 'slowlog', 'fileset.name': ['slowlog'],
'input.type': 'log', 'input.type': ['log'],
'log.level': 'WARN', 'log.level': ['WARN'],
'log.offset': 0, 'log.offset': [0],
'logstash.slowlog': { 'logstash.slowlog.event': [
event: '"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"',
'"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"', ],
module: 'slowlog.logstash.filters.sleep', 'logstash.slowlog.module': ['slowlog.logstash.filters.sleep'],
plugin_name: 'sleep', 'logstash.slowlog.plugin_name': ['sleep'],
plugin_params: 'logstash.slowlog.plugin_params': [
'{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}', '{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}',
plugin_type: 'filters', ],
took_in_millis: 3027, 'logstash.slowlog.plugin_type': ['filters'],
}, 'logstash.slowlog.took_in_millis': [3027],
'service.type': 'logstash', 'service.type': ['logstash'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[Logstash][", "constant": "[Logstash][",
}, },
Object { Object {
"field": "log.level", "field": "log.level",
"highlights": Array [], "highlights": Array [],
"value": "WARN", "value": Array [
}, "WARN",
Object { ],
"constant": "] ", },
}, Object {
Object { "constant": "] ",
"field": "logstash.slowlog", },
"highlights": Array [], Object {
"value": "{\\"event\\":\\"\\\\\\"{\\\\\\\\\\\\\\"@version\\\\\\\\\\\\\\":\\\\\\\\\\\\\\"1\\\\\\\\\\\\\\",\\\\\\\\\\\\\\"@timestamp\\\\\\\\\\\\\\":\\\\\\\\\\\\\\"2017-10-30T13:57:55.130Z\\\\\\\\\\\\\\",\\\\\\\\\\\\\\"host\\\\\\\\\\\\\\":\\\\\\\\\\\\\\"sashimi\\\\\\\\\\\\\\",\\\\\\\\\\\\\\"sequence\\\\\\\\\\\\\\":0,\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\":\\\\\\\\\\\\\\"Hello world!\\\\\\\\\\\\\\"}\\\\\\"\\",\\"module\\":\\"slowlog.logstash.filters.sleep\\",\\"plugin_name\\":\\"sleep\\",\\"plugin_params\\":\\"{\\\\\\"time\\\\\\"=>3, \\\\\\"id\\\\\\"=>\\\\\\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\\\\\\"}\\",\\"plugin_type\\":\\"filters\\",\\"took_in_millis\\":3027}", "field": "logstash.slowlog.event",
}, "highlights": Array [],
] "value": Array [
`); "\\"{\\\\\\"@version\\\\\\":\\\\\\"1\\\\\\",\\\\\\"@timestamp\\\\\\":\\\\\\"2017-10-30T13:57:55.130Z\\\\\\",\\\\\\"host\\\\\\":\\\\\\"sashimi\\\\\\",\\\\\\"sequence\\\\\\":0,\\\\\\"message\\\\\\":\\\\\\"Hello world!\\\\\\"}\\"",
],
},
Object {
"field": "logstash.slowlog.module",
"highlights": Array [],
"value": Array [
"slowlog.logstash.filters.sleep",
],
},
Object {
"field": "logstash.slowlog.plugin_name",
"highlights": Array [],
"value": Array [
"sleep",
],
},
Object {
"field": "logstash.slowlog.plugin_params",
"highlights": Array [],
"value": Array [
"{\\"time\\"=>3, \\"id\\"=>\\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\\"}",
],
},
Object {
"field": "logstash.slowlog.plugin_type",
"highlights": Array [],
"value": Array [
"filters",
],
},
Object {
"field": "logstash.slowlog.took_in_millis",
"highlights": Array [],
"value": Array [
3027,
],
},
]
`);
}); });
}); });
describe('in pre-ECS format', () => { describe('in pre-ECS format', () => {
test('logstash log', () => { test('logstash log', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2017-10-23T14:20:12.046Z', '@timestamp': ['2017-10-23T14:20:12.046Z'],
'event.dataset': 'logstash.log', 'event.dataset': ['logstash.log'],
'fileset.module': 'logstash', 'fileset.module': ['logstash'],
'fileset.name': 'log', 'fileset.name': ['log'],
'input.type': 'log', 'input.type': ['log'],
'logstash.log.level': 'INFO', 'logstash.log.level': ['INFO'],
'logstash.log.message': 'logstash.log.message': [
'Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}', 'Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}',
'logstash.log.module': 'logstash.modules.scaffold', ],
offset: 0, 'logstash.log.module': ['logstash.modules.scaffold'],
'prospector.type': 'log', offset: [0],
'prospector.type': ['log'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[Logstash][", "constant": "[Logstash][",
}, },
Object { Object {
"field": "logstash.log.level", "field": "logstash.log.level",
"highlights": Array [], "highlights": Array [],
"value": "INFO", "value": Array [
}, "INFO",
Object { ],
"constant": "] ", },
}, Object {
Object { "constant": "] ",
"field": "logstash.log.module", },
"highlights": Array [], Object {
"value": "logstash.modules.scaffold", "field": "logstash.log.module",
}, "highlights": Array [],
Object { "value": Array [
"constant": " - ", "logstash.modules.scaffold",
}, ],
Object { },
"field": "logstash.log.message", Object {
"highlights": Array [], "constant": " - ",
"value": "Initializing module {:module_name=>\\"fb_apache\\", :directory=>\\"/usr/share/logstash/modules/fb_apache/configuration\\"}", },
}, Object {
] "field": "logstash.log.message",
`); "highlights": Array [],
"value": Array [
"Initializing module {:module_name=>\\"fb_apache\\", :directory=>\\"/usr/share/logstash/modules/fb_apache/configuration\\"}",
],
},
]
`);
}); });
test('logstash slowlog', () => { test('logstash slowlog', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2017-10-30T09:57:58.243Z', '@timestamp': ['2017-10-30T09:57:58.243Z'],
'event.dataset': 'logstash.slowlog', 'event.dataset': ['logstash.slowlog'],
'fileset.module': 'logstash', 'fileset.module': ['logstash'],
'fileset.name': 'slowlog', 'fileset.name': ['slowlog'],
'input.type': 'log', 'input.type': ['log'],
'logstash.slowlog.event': 'logstash.slowlog.event': [
'"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"', '"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"',
'logstash.slowlog.level': 'WARN', ],
'logstash.slowlog.message': 'logstash.slowlog.level': ['WARN'],
'logstash.slowlog.message': [
'event processing time {:plugin_params=>{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"}', 'event processing time {:plugin_params=>{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"}',
'logstash.slowlog.module': 'slowlog.logstash.filters.sleep', ],
'logstash.slowlog.plugin_name': 'sleep', 'logstash.slowlog.module': ['slowlog.logstash.filters.sleep'],
'logstash.slowlog.plugin_params': 'logstash.slowlog.plugin_name': ['sleep'],
'logstash.slowlog.plugin_params': [
'{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}', '{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}',
'logstash.slowlog.plugin_type': 'filters', ],
'logstash.slowlog.took_in_millis': 3027, 'logstash.slowlog.plugin_type': ['filters'],
'logstash.slowlog.took_in_nanos': 3027675106, 'logstash.slowlog.took_in_millis': [3027],
offset: 0, 'logstash.slowlog.took_in_nanos': [3027675106],
'prospector.type': 'log', offset: [0],
'prospector.type': ['log'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[Logstash][", "constant": "[Logstash][",
}, },
Object { Object {
"field": "logstash.slowlog.level", "field": "logstash.slowlog.level",
"highlights": Array [], "highlights": Array [],
"value": "WARN", "value": Array [
}, "WARN",
Object { ],
"constant": "] ", },
}, Object {
Object { "constant": "] ",
"field": "logstash.slowlog.module", },
"highlights": Array [], Object {
"value": "slowlog.logstash.filters.sleep", "field": "logstash.slowlog.module",
}, "highlights": Array [],
Object { "value": Array [
"constant": " - ", "slowlog.logstash.filters.sleep",
}, ],
Object { },
"field": "logstash.slowlog.message", Object {
"highlights": Array [], "constant": " - ",
"value": "event processing time {:plugin_params=>{\\"time\\"=>3, \\"id\\"=>\\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\\"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>\\"{\\\\\\"@version\\\\\\":\\\\\\"1\\\\\\",\\\\\\"@timestamp\\\\\\":\\\\\\"2017-10-30T13:57:55.130Z\\\\\\",\\\\\\"host\\\\\\":\\\\\\"sashimi\\\\\\",\\\\\\"sequence\\\\\\":0,\\\\\\"message\\\\\\":\\\\\\"Hello world!\\\\\\"}\\"}", },
}, Object {
] "field": "logstash.slowlog.message",
`); "highlights": Array [],
"value": Array [
"event processing time {:plugin_params=>{\\"time\\"=>3, \\"id\\"=>\\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\\"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>\\"{\\\\\\"@version\\\\\\":\\\\\\"1\\\\\\",\\\\\\"@timestamp\\\\\\":\\\\\\"2017-10-30T13:57:55.130Z\\\\\\",\\\\\\"host\\\\\\":\\\\\\"sashimi\\\\\\",\\\\\\"sequence\\\\\\":0,\\\\\\"message\\\\\\":\\\\\\"Hello world!\\\\\\"}\\"}",
],
},
]
`);
}); });
}); });
}); });

4
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.ts
View file

@ -34,7 +34,7 @@ export const filebeatLogstashRules = [
{ {
// ECS // ECS
when: { when: {
exists: ['ecs.version', 'logstash.slowlog'], all: [{ exists: ['ecs.version'] }, { existsPrefix: ['logstash.slowlog'] }],
}, },
format: [ format: [
{ {
@ -47,7 +47,7 @@ export const filebeatLogstashRules = [
constant: '] ', constant: '] ',
}, },
{ {
field: 'logstash.slowlog', fieldsPrefix: 'logstash.slowlog',
}, },
], ],
}, },

65
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.test.ts
View file

@ -13,40 +13,45 @@ describe('Filebeat Rules', () => {
describe('in pre-ECS format', () => { describe('in pre-ECS format', () => {
test('mongodb log', () => { test('mongodb log', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2018-02-05T12:44:56.677Z', '@timestamp': ['2018-02-05T12:44:56.677Z'],
'event.dataset': 'mongodb.log', 'event.dataset': ['mongodb.log'],
'fileset.module': 'mongodb', 'fileset.module': ['mongodb'],
'fileset.name': 'log', 'fileset.name': ['log'],
'input.type': 'log', 'input.type': ['log'],
'mongodb.log.component': 'STORAGE', 'mongodb.log.component': ['STORAGE'],
'mongodb.log.context': 'initandlisten', 'mongodb.log.context': ['initandlisten'],
'mongodb.log.message': 'mongodb.log.message': [
'wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),', 'wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),',
'mongodb.log.severity': 'I', ],
offset: 281, 'mongodb.log.severity': ['I'],
'prospector.type': 'log', offset: [281],
'prospector.type': ['log'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[MongoDB][", "constant": "[MongoDB][",
}, },
Object { Object {
"field": "mongodb.log.component", "field": "mongodb.log.component",
"highlights": Array [], "highlights": Array [],
"value": "STORAGE", "value": Array [
}, "STORAGE",
Object { ],
"constant": "] ", },
}, Object {
Object { "constant": "] ",
"field": "mongodb.log.message", },
"highlights": Array [], Object {
"value": "wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),", "field": "mongodb.log.message",
}, "highlights": Array [],
] "value": Array [
`); "wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),",
],
},
]
`);
}); });
}); });
}); });

267
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.test.ts
View file

@ -13,139 +13,158 @@ describe('Filebeat Rules', () => {
describe('in ECS format', () => { describe('in ECS format', () => {
test('mysql error log', () => { test('mysql error log', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2016-12-09T12:08:33.335Z', '@timestamp': ['2016-12-09T12:08:33.335Z'],
'ecs.version': '1.0.0-beta2', 'ecs.version': ['1.0.0-beta2'],
'event.dataset': 'mysql.error', 'event.dataset': ['mysql.error'],
'event.module': 'mysql', 'event.module': ['mysql'],
'fileset.name': 'error', 'fileset.name': ['error'],
'input.type': 'log', 'input.type': ['log'],
'log.level': 'Warning', 'log.level': ['Warning'],
'log.offset': 92, 'log.offset': [92],
message: message: [
'TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).', 'TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).',
'mysql.thread_id': 0, ],
'service.type': 'mysql', 'mysql.thread_id': [0],
'service.type': ['mysql'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[", "constant": "[",
}, },
Object { Object {
"field": "event.dataset", "field": "event.dataset",
"highlights": Array [], "highlights": Array [],
"value": "mysql.error", "value": Array [
}, "mysql.error",
Object { ],
"constant": "][", },
}, Object {
Object { "constant": "][",
"field": "log.level", },
"highlights": Array [], Object {
"value": "Warning", "field": "log.level",
}, "highlights": Array [],
Object { "value": Array [
"constant": "] ", "Warning",
}, ],
Object { },
"field": "message", Object {
"highlights": Array [], "constant": "] ",
"value": "TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).", },
}, Object {
] "field": "message",
`); "highlights": Array [],
"value": Array [
"TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).",
],
},
]
`);
}); });
test('mysql slowlog', () => { test('mysql slowlog', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2018-08-07T08:27:47.000Z', '@timestamp': ['2018-08-07T08:27:47.000Z'],
'ecs.version': '1.0.0-beta2', 'ecs.version': ['1.0.0-beta2'],
'event.dataset': 'mysql.slowlog', 'event.dataset': ['mysql.slowlog'],
'event.duration': 4071491000, 'event.duration': [4071491000],
'event.module': 'mysql', 'event.module': ['mysql'],
'fileset.name': 'slowlog', 'fileset.name': ['slowlog'],
'input.type': 'log', 'input.type': ['log'],
'log.flags': ['multiline'], 'log.flags': ['multiline'],
'log.offset': 526, 'log.offset': [526],
'mysql.slowlog.current_user': 'appuser', 'mysql.slowlog.current_user': ['appuser'],
'mysql.slowlog.lock_time.sec': 0.000212, 'mysql.slowlog.lock_time.sec': [0.000212],
'mysql.slowlog.query': 'mysql.slowlog.query': [
'SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR ";") as mca_guid\n FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca\n WHERE cus.cus_guid = mcu.cus_guid\n AND cus.pro_code = \'CYB\'\n AND cus.cus_offline = 0\n AND mca.cus_guid = cus.cus_guid\n AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())\n AND mcu.mcu_crawlelements IS NOT NULL\n GROUP BY mcu.mcu_guid\n ORDER BY mcu.mcu_order ASC\n LIMIT 1000;', 'SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR ";") as mca_guid\n FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca\n WHERE cus.cus_guid = mcu.cus_guid\n AND cus.pro_code = \'CYB\'\n AND cus.cus_offline = 0\n AND mca.cus_guid = cus.cus_guid\n AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())\n AND mcu.mcu_crawlelements IS NOT NULL\n GROUP BY mcu.mcu_guid\n ORDER BY mcu.mcu_order ASC\n LIMIT 1000;',
'mysql.slowlog.rows_examined': 1489615, ],
'mysql.slowlog.rows_sent': 1000, 'mysql.slowlog.rows_examined': [1489615],
'mysql.thread_id': 10997316, 'mysql.slowlog.rows_sent': [1000],
'service.type': 'mysql', 'mysql.thread_id': [10997316],
'source.domain': 'apphost', 'service.type': ['mysql'],
'source.ip': '1.1.1.1', 'source.domain': ['apphost'],
'user.name': 'appuser', 'source.ip': ['1.1.1.1'],
'user.name': ['appuser'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[MySQL][slowlog] ", "constant": "[MySQL][slowlog] ",
}, },
Object { Object {
"field": "user.name", "field": "user.name",
"highlights": Array [], "highlights": Array [],
"value": "appuser", "value": Array [
}, "appuser",
Object { ],
"constant": "@", },
}, Object {
Object { "constant": "@",
"field": "source.domain", },
"highlights": Array [], Object {
"value": "apphost", "field": "source.domain",
}, "highlights": Array [],
Object { "value": Array [
"constant": " [", "apphost",
}, ],
Object { },
"field": "source.ip", Object {
"highlights": Array [], "constant": " [",
"value": "1.1.1.1", },
}, Object {
Object { "field": "source.ip",
"constant": "] ", "highlights": Array [],
}, "value": Array [
Object { "1.1.1.1",
"constant": " - ", ],
}, },
Object { Object {
"field": "event.duration", "constant": "] ",
"highlights": Array [], },
"value": "4071491000", Object {
}, "constant": " - ",
Object { },
"constant": " ns - ", Object {
}, "field": "event.duration",
Object { "highlights": Array [],
"field": "mysql.slowlog.query", "value": Array [
"highlights": Array [], 4071491000,
"value": "SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR \\";\\") as mca_guid ],
FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca },
WHERE cus.cus_guid = mcu.cus_guid Object {
AND cus.pro_code = 'CYB' "constant": " ns - ",
AND cus.cus_offline = 0 },
AND mca.cus_guid = cus.cus_guid Object {
AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE()) "field": "mysql.slowlog.query",
AND mcu.mcu_crawlelements IS NOT NULL "highlights": Array [],
GROUP BY mcu.mcu_guid "value": Array [
ORDER BY mcu.mcu_order ASC "SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR \\";\\") as mca_guid
LIMIT 1000;", FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca
}, WHERE cus.cus_guid = mcu.cus_guid
] AND cus.pro_code = 'CYB'
`); AND cus.cus_offline = 0
AND mca.cus_guid = cus.cus_guid
AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())
AND mcu.mcu_crawlelements IS NOT NULL
GROUP BY mcu.mcu_guid
ORDER BY mcu.mcu_order ASC
LIMIT 1000;",
],
},
]
`);
}); });
}); });
describe('in pre-ECS format', () => { describe('in pre-ECS format', () => {
test('mysql error log', () => { test('mysql error log', () => {
const errorDoc = { const errorDoc = {
'mysql.error.message': 'mysql.error.message': [
"Access denied for user 'petclinicdd'@'47.153.152.234' (using password: YES)", "Access denied for user 'petclinicdd'@'47.153.152.234' (using password: YES)",
],
}; };
const message = format(errorDoc, {}); const message = format(errorDoc, {});
expect(message).toEqual([ expect(message).toEqual([
@ -155,18 +174,18 @@ Array [
{ {
field: 'mysql.error.message', field: 'mysql.error.message',
highlights: [], highlights: [],
value: "Access denied for user 'petclinicdd'@'47.153.152.234' (using password: YES)", value: ["Access denied for user 'petclinicdd'@'47.153.152.234' (using password: YES)"],
}, },
]); ]);
}); });
test('mysql slow log', () => { test('mysql slow log', () => {
const errorDoc = { const errorDoc = {
'mysql.slowlog.query': 'select * from hosts', 'mysql.slowlog.query': ['select * from hosts'],
'mysql.slowlog.query_time.sec': 5, 'mysql.slowlog.query_time.sec': [5],
'mysql.slowlog.user': 'admin', 'mysql.slowlog.user': ['admin'],
'mysql.slowlog.ip': '192.168.1.42', 'mysql.slowlog.ip': ['192.168.1.42'],
'mysql.slowlog.host': 'webserver-01', 'mysql.slowlog.host': ['webserver-01'],
}; };
const message = format(errorDoc, {}); const message = format(errorDoc, {});
expect(message).toEqual([ expect(message).toEqual([
@ -176,7 +195,7 @@ Array [
{ {
field: 'mysql.slowlog.user', field: 'mysql.slowlog.user',
highlights: [], highlights: [],
value: 'admin', value: ['admin'],
}, },
{ {
constant: '@', constant: '@',
@ -184,7 +203,7 @@ Array [
{ {
field: 'mysql.slowlog.host', field: 'mysql.slowlog.host',
highlights: [], highlights: [],
value: 'webserver-01', value: ['webserver-01'],
}, },
{ {
constant: ' [', constant: ' [',
@ -192,7 +211,7 @@ Array [
{ {
field: 'mysql.slowlog.ip', field: 'mysql.slowlog.ip',
highlights: [], highlights: [],
value: '192.168.1.42', value: ['192.168.1.42'],
}, },
{ {
constant: '] ', constant: '] ',
@ -203,7 +222,7 @@ Array [
{ {
field: 'mysql.slowlog.query_time.sec', field: 'mysql.slowlog.query_time.sec',
highlights: [], highlights: [],
value: '5', value: [5],
}, },
{ {
constant: ' s - ', constant: ' s - ',
@ -211,7 +230,7 @@ Array [
{ {
field: 'mysql.slowlog.query', field: 'mysql.slowlog.query',
highlights: [], highlights: [],
value: 'select * from hosts', value: ['select * from hosts'],
}, },
]); ]);
}); });

475
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.test.ts
View file

@ -13,252 +13,293 @@ describe('Filebeat Rules', () => {
describe('in ECS format', () => { describe('in ECS format', () => {
test('Nginx Access', () => { test('Nginx Access', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2017-05-29T19:02:48.000Z', '@timestamp': ['2017-05-29T19:02:48.000Z'],
'ecs.version': '1.0.0-beta2', 'ecs.version': ['1.0.0-beta2'],
'event.dataset': 'nginx.access', 'event.dataset': ['nginx.access'],
'event.module': 'nginx', 'event.module': ['nginx'],
'fileset.name': 'access', 'fileset.name': ['access'],
'http.request.method': 'GET', 'http.request.method': ['GET'],
'http.request.referrer': '-', 'http.request.referrer': ['-'],
'http.response.body.bytes': 612, 'http.response.body.bytes': [612],
'http.response.status_code': 404, 'http.response.status_code': [404],
'http.version': '1.1', 'http.version': ['1.1'],
'input.type': 'log', 'input.type': ['log'],
'log.offset': 183, 'log.offset': [183],
'service.type': 'nginx', 'service.type': ['nginx'],
'source.ip': '172.17.0.1', 'source.ip': ['172.17.0.1'],
'url.original': '/stringpatch', 'url.original': ['/stringpatch'],
'user.name': '-', 'user.name': ['-'],
'user_agent.device': 'Other', 'user_agent.device': ['Other'],
'user_agent.major': '15', 'user_agent.major': ['15'],
'user_agent.minor': '0', 'user_agent.minor': ['0'],
'user_agent.name': 'Firefox Alpha', 'user_agent.name': ['Firefox Alpha'],
'user_agent.original': 'user_agent.original': [
'Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2', 'Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2',
'user_agent.os.full_name': 'Windows 7', ],
'user_agent.os.name': 'Windows 7', 'user_agent.os.full_name': ['Windows 7'],
'user_agent.patch': 'a2', 'user_agent.os.name': ['Windows 7'],
'user_agent.patch': ['a2'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[", "constant": "[",
}, },
Object { Object {
"field": "event.module", "field": "event.module",
"highlights": Array [], "highlights": Array [],
"value": "nginx", "value": Array [
}, "nginx",
Object { ],
"constant": "][access] ", },
}, Object {
Object { "constant": "][access] ",
"field": "source.ip", },
"highlights": Array [], Object {
"value": "172.17.0.1", "field": "source.ip",
}, "highlights": Array [],
Object { "value": Array [
"constant": " ", "172.17.0.1",
}, ],
Object { },
"field": "user.name", Object {
"highlights": Array [], "constant": " ",
"value": "-", },
}, Object {
Object { "field": "user.name",
"constant": " \\"", "highlights": Array [],
}, "value": Array [
Object { "-",
"field": "http.request.method", ],
"highlights": Array [], },
"value": "GET", Object {
}, "constant": " \\"",
Object { },
"constant": " ", Object {
}, "field": "http.request.method",
Object { "highlights": Array [],
"field": "url.original", "value": Array [
"highlights": Array [], "GET",
"value": "/stringpatch", ],
}, },
Object { Object {
"constant": " HTTP/", "constant": " ",
}, },
Object { Object {
"field": "http.version", "field": "url.original",
"highlights": Array [], "highlights": Array [],
"value": "1.1", "value": Array [
}, "/stringpatch",
Object { ],
"constant": "\\" ", },
}, Object {
Object { "constant": " HTTP/",
"field": "http.response.status_code", },
"highlights": Array [], Object {
"value": "404", "field": "http.version",
}, "highlights": Array [],
Object { "value": Array [
"constant": " ", "1.1",
}, ],
Object { },
"field": "http.response.body.bytes", Object {
"highlights": Array [], "constant": "\\" ",
"value": "612", },
}, Object {
] "field": "http.response.status_code",
`); "highlights": Array [],
"value": Array [
404,
],
},
Object {
"constant": " ",
},
Object {
"field": "http.response.body.bytes",
"highlights": Array [],
"value": Array [
612,
],
},
]
`);
}); });
test('Nginx Error', () => { test('Nginx Error', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2016-10-25T14:49:34.000Z', '@timestamp': ['2016-10-25T14:49:34.000Z'],
'ecs.version': '1.0.0-beta2', 'ecs.version': ['1.0.0-beta2'],
'event.dataset': 'nginx.error', 'event.dataset': ['nginx.error'],
'event.module': 'nginx', 'event.module': ['nginx'],
'fileset.name': 'error', 'fileset.name': ['error'],
'input.type': 'log', 'input.type': ['log'],
'log.level': 'error', 'log.level': ['error'],
'log.offset': 0, 'log.offset': [0],
message: message: [
'open() "/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: "GET /favicon.ico HTTP/1.1", host: "localhost:8080", referrer: "http://localhost:8080/"', 'open() "/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: "GET /favicon.ico HTTP/1.1", host: "localhost:8080", referrer: "http://localhost:8080/"',
'nginx.error.connection_id': 1, ],
'process.pid': 54053, 'nginx.error.connection_id': [1],
'process.thread.id': 0, 'process.pid': [54053],
'service.type': 'nginx', 'process.thread.id': [0],
'service.type': ['nginx'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[nginx]", "constant": "[nginx]",
}, },
Object { Object {
"constant": "[", "constant": "[",
}, },
Object { Object {
"field": "log.level", "field": "log.level",
"highlights": Array [], "highlights": Array [],
"value": "error", "value": Array [
}, "error",
Object { ],
"constant": "] ", },
}, Object {
Object { "constant": "] ",
"field": "message", },
"highlights": Array [], Object {
"value": "open() \\"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \\"GET /favicon.ico HTTP/1.1\\", host: \\"localhost:8080\\", referrer: \\"http://localhost:8080/\\"", "field": "message",
}, "highlights": Array [],
] "value": Array [
`); "open() \\"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \\"GET /favicon.ico HTTP/1.1\\", host: \\"localhost:8080\\", referrer: \\"http://localhost:8080/\\"",
],
},
]
`);
}); });
}); });
describe('in pre-ECS format', () => { describe('in pre-ECS format', () => {
test('Nginx Access', () => { test('Nginx Access', () => {
const flattenedDocument = { const flattenedDocument = {
'nginx.access': true, 'nginx.access': [true],
'nginx.access.remote_ip': '192.168.1.42', 'nginx.access.remote_ip': ['192.168.1.42'],
'nginx.access.user_name': 'admin', 'nginx.access.user_name': ['admin'],
'nginx.access.method': 'GET', 'nginx.access.method': ['GET'],
'nginx.access.url': '/faq', 'nginx.access.url': ['/faq'],
'nginx.access.http_version': '1.1', 'nginx.access.http_version': ['1.1'],
'nginx.access.body_sent.bytes': 1024, 'nginx.access.body_sent.bytes': [1024],
'nginx.access.response_code': 200, 'nginx.access.response_code': [200],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[nginx][access] ", "constant": "[nginx][access] ",
}, },
Object { Object {
"field": "nginx.access.remote_ip", "field": "nginx.access.remote_ip",
"highlights": Array [], "highlights": Array [],
"value": "192.168.1.42", "value": Array [
}, "192.168.1.42",
Object { ],
"constant": " ", },
}, Object {
Object { "constant": " ",
"field": "nginx.access.user_name", },
"highlights": Array [], Object {
"value": "admin", "field": "nginx.access.user_name",
}, "highlights": Array [],
Object { "value": Array [
"constant": " \\"", "admin",
}, ],
Object { },
"field": "nginx.access.method", Object {
"highlights": Array [], "constant": " \\"",
"value": "GET", },
}, Object {
Object { "field": "nginx.access.method",
"constant": " ", "highlights": Array [],
}, "value": Array [
Object { "GET",
"field": "nginx.access.url", ],
"highlights": Array [], },
"value": "/faq", Object {
}, "constant": " ",
Object { },
"constant": " HTTP/", Object {
}, "field": "nginx.access.url",
Object { "highlights": Array [],
"field": "nginx.access.http_version", "value": Array [
"highlights": Array [], "/faq",
"value": "1.1", ],
}, },
Object { Object {
"constant": "\\" ", "constant": " HTTP/",
}, },
Object { Object {
"field": "nginx.access.response_code", "field": "nginx.access.http_version",
"highlights": Array [], "highlights": Array [],
"value": "200", "value": Array [
}, "1.1",
Object { ],
"constant": " ", },
}, Object {
Object { "constant": "\\" ",
"field": "nginx.access.body_sent.bytes", },
"highlights": Array [], Object {
"value": "1024", "field": "nginx.access.response_code",
}, "highlights": Array [],
] "value": Array [
`); 200,
],
},
Object {
"constant": " ",
},
Object {
"field": "nginx.access.body_sent.bytes",
"highlights": Array [],
"value": Array [
1024,
],
},
]
`);
}); });
test('Nginx Error', () => { test('Nginx Error', () => {
const flattenedDocument = { const flattenedDocument = {
'nginx.error.message': 'nginx.error.message': [
'connect() failed (111: Connection refused) while connecting to upstream, client: 127.0.0.1, server: localhost, request: "GET /php-status?json= HTTP/1.1", upstream: "fastcgi://[::1]:9000", host: "localhost"', 'connect() failed (111: Connection refused) while connecting to upstream, client: 127.0.0.1, server: localhost, request: "GET /php-status?json= HTTP/1.1", upstream: "fastcgi://[::1]:9000", host: "localhost"',
'nginx.error.level': 'error', ],
'nginx.error.level': ['error'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[nginx]", "constant": "[nginx]",
}, },
Object { Object {
"constant": "[", "constant": "[",
}, },
Object { Object {
"field": "nginx.error.level", "field": "nginx.error.level",
"highlights": Array [], "highlights": Array [],
"value": "error", "value": Array [
}, "error",
Object { ],
"constant": "] ", },
}, Object {
Object { "constant": "] ",
"field": "nginx.error.message", },
"highlights": Array [], Object {
"value": "connect() failed (111: Connection refused) while connecting to upstream, client: 127.0.0.1, server: localhost, request: \\"GET /php-status?json= HTTP/1.1\\", upstream: \\"fastcgi://[::1]:9000\\", host: \\"localhost\\"", "field": "nginx.error.message",
}, "highlights": Array [],
] "value": Array [
`); "connect() failed (111: Connection refused) while connecting to upstream, client: 127.0.0.1, server: localhost, request: \\"GET /php-status?json= HTTP/1.1\\", upstream: \\"fastcgi://[::1]:9000\\", host: \\"localhost\\"",
],
},
]
`);
}); });
}); });
}); });

186
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.test.ts
View file

@ -13,65 +13,139 @@ describe('Filebeat Rules', () => {
describe('in pre-ECS format', () => { describe('in pre-ECS format', () => {
test('osquery result log', () => { test('osquery result log', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2017-12-28T14:40:08.000Z', '@timestamp': ['2017-12-28T14:40:08.000Z'],
'event.dataset': 'osquery.result', 'event.dataset': ['osquery.result'],
'fileset.module': 'osquery', 'fileset.module': ['osquery'],
'fileset.name': 'result', 'fileset.name': ['result'],
'input.type': 'log', 'input.type': ['log'],
offset: 0, offset: [0],
'osquery.result.action': 'removed', 'osquery.result.action': ['removed'],
'osquery.result.calendar_time': 'Thu Dec 28 14:40:08 2017 UTC', 'osquery.result.calendar_time': ['Thu Dec 28 14:40:08 2017 UTC'],
'osquery.result.columns': { 'osquery.result.columns.blocks': ['122061322'],
blocks: '122061322', 'osquery.result.columns.blocks_available': ['75966945'],
blocks_available: '75966945', 'osquery.result.columns.blocks_free': ['121274885'],
blocks_free: '121274885', 'osquery.result.columns.blocks_size': ['4096'],
blocks_size: '4096', 'osquery.result.columns.device': ['/dev/disk1s4'],
device: '/dev/disk1s4', 'osquery.result.columns.device_alias': ['/dev/disk1s4'],
device_alias: '/dev/disk1s4', 'osquery.result.columns.flags': ['345018372'],
flags: '345018372', 'osquery.result.columns.inodes': ['9223372036854775807'],
inodes: '9223372036854775807', 'osquery.result.columns.inodes_free': ['9223372036854775804'],
inodes_free: '9223372036854775804', 'osquery.result.columns.path': ['/private/var/vm'],
path: '/private/var/vm', 'osquery.result.columns.type': ['apfs'],
type: 'apfs', 'osquery.result.counter': ['1'],
}, 'osquery.result.decorations.host_uuid': ['4AB2906D-5516-5794-AF54-86D1D7F533F3'],
'osquery.result.counter': '1', 'osquery.result.decorations.username': ['tsg'],
'osquery.result.decorations.host_uuid': '4AB2906D-5516-5794-AF54-86D1D7F533F3', 'osquery.result.epoch': ['0'],
'osquery.result.decorations.username': 'tsg', 'osquery.result.host_identifier': ['192-168-0-4.rdsnet.ro'],
'osquery.result.epoch': '0', 'osquery.result.name': ['pack_it-compliance_mounts'],
'osquery.result.host_identifier': '192-168-0-4.rdsnet.ro', 'osquery.result.unix_time': ['1514472008'],
'osquery.result.name': 'pack_it-compliance_mounts', 'prospector.type': ['log'],
'osquery.result.unix_time': '1514472008',
'prospector.type': 'log',
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[Osquery][", "constant": "[Osquery][",
}, },
Object { Object {
"field": "osquery.result.action", "field": "osquery.result.action",
"highlights": Array [], "highlights": Array [],
"value": "removed", "value": Array [
}, "removed",
Object { ],
"constant": "] ", },
}, Object {
Object { "constant": "] ",
"field": "osquery.result.host_identifier", },
"highlights": Array [], Object {
"value": "192-168-0-4.rdsnet.ro", "field": "osquery.result.host_identifier",
}, "highlights": Array [],
Object { "value": Array [
"constant": " ", "192-168-0-4.rdsnet.ro",
}, ],
Object { },
"field": "osquery.result.columns", Object {
"highlights": Array [], "constant": " ",
"value": "{\\"blocks\\":\\"122061322\\",\\"blocks_available\\":\\"75966945\\",\\"blocks_free\\":\\"121274885\\",\\"blocks_size\\":\\"4096\\",\\"device\\":\\"/dev/disk1s4\\",\\"device_alias\\":\\"/dev/disk1s4\\",\\"flags\\":\\"345018372\\",\\"inodes\\":\\"9223372036854775807\\",\\"inodes_free\\":\\"9223372036854775804\\",\\"path\\":\\"/private/var/vm\\",\\"type\\":\\"apfs\\"}", },
}, Object {
] "field": "osquery.result.columns.blocks",
`); "highlights": Array [],
"value": Array [
"122061322",
],
},
Object {
"field": "osquery.result.columns.blocks_available",
"highlights": Array [],
"value": Array [
"75966945",
],
},
Object {
"field": "osquery.result.columns.blocks_free",
"highlights": Array [],
"value": Array [
"121274885",
],
},
Object {
"field": "osquery.result.columns.blocks_size",
"highlights": Array [],
"value": Array [
"4096",
],
},
Object {
"field": "osquery.result.columns.device",
"highlights": Array [],
"value": Array [
"/dev/disk1s4",
],
},
Object {
"field": "osquery.result.columns.device_alias",
"highlights": Array [],
"value": Array [
"/dev/disk1s4",
],
},
Object {
"field": "osquery.result.columns.flags",
"highlights": Array [],
"value": Array [
"345018372",
],
},
Object {
"field": "osquery.result.columns.inodes",
"highlights": Array [],
"value": Array [
"9223372036854775807",
],
},
Object {
"field": "osquery.result.columns.inodes_free",
"highlights": Array [],
"value": Array [
"9223372036854775804",
],
},
Object {
"field": "osquery.result.columns.path",
"highlights": Array [],
"value": Array [
"/private/var/vm",
],
},
Object {
"field": "osquery.result.columns.type",
"highlights": Array [],
"value": Array [
"apfs",
],
},
]
`);
}); });
}); });
}); });

2
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.ts
View file

@ -27,7 +27,7 @@ export const filebeatOsqueryRules = [
constant: ' ', constant: ' ',
}, },
{ {
field: 'osquery.result.columns', fieldsPrefix: 'osquery.result.columns',
}, },
], ],
}, },

221
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.test.ts
View file

@ -13,112 +13,129 @@ describe('Filebeat Rules', () => {
describe('in pre-ECS format', () => { describe('in pre-ECS format', () => {
test('traefik access log', () => { test('traefik access log', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2017-10-02T20:22:08.000Z', '@timestamp': ['2017-10-02T20:22:08.000Z'],
'event.dataset': 'traefik.access', 'event.dataset': ['traefik.access'],
'fileset.module': 'traefik', 'fileset.module': ['traefik'],
'fileset.name': 'access', 'fileset.name': ['access'],
'input.type': 'log', 'input.type': ['log'],
offset: 280, offset: [280],
'prospector.type': 'log', 'prospector.type': ['log'],
'traefik.access.backend_url': 'http://172.19.0.3:5601', 'traefik.access.backend_url': ['http://172.19.0.3:5601'],
'traefik.access.body_sent.bytes': 0, 'traefik.access.body_sent.bytes': [0],
'traefik.access.duration': 3, 'traefik.access.duration': [3],
'traefik.access.frontend_name': 'Host-host1', 'traefik.access.frontend_name': ['Host-host1'],
'traefik.access.geoip.city_name': 'Berlin', 'traefik.access.geoip.city_name': ['Berlin'],
'traefik.access.geoip.continent_name': 'Europe', 'traefik.access.geoip.continent_name': ['Europe'],
'traefik.access.geoip.country_iso_code': 'DE', 'traefik.access.geoip.country_iso_code': ['DE'],
'traefik.access.geoip.location.lat': 52.4908, 'traefik.access.geoip.location.lat': [52.4908],
'traefik.access.geoip.location.lon': 13.3275, 'traefik.access.geoip.location.lon': [13.3275],
'traefik.access.geoip.region_iso_code': 'DE-BE', 'traefik.access.geoip.region_iso_code': ['DE-BE'],
'traefik.access.geoip.region_name': 'Land Berlin', 'traefik.access.geoip.region_name': ['Land Berlin'],
'traefik.access.http_version': '1.1', 'traefik.access.http_version': ['1.1'],
'traefik.access.method': 'GET', 'traefik.access.method': ['GET'],
'traefik.access.referrer': 'http://example.com/login', 'traefik.access.referrer': ['http://example.com/login'],
'traefik.access.remote_ip': '85.181.35.98', 'traefik.access.remote_ip': ['85.181.35.98'],
'traefik.access.request_count': 271, 'traefik.access.request_count': [271],
'traefik.access.response_code': '304', 'traefik.access.response_code': ['304'],
'traefik.access.url': '/ui/favicons/favicon.ico', 'traefik.access.url': ['/ui/favicons/favicon.ico'],
'traefik.access.user_agent.device': 'Other', 'traefik.access.user_agent.device': ['Other'],
'traefik.access.user_agent.major': '61', 'traefik.access.user_agent.major': ['61'],
'traefik.access.user_agent.minor': '0', 'traefik.access.user_agent.minor': ['0'],
'traefik.access.user_agent.name': 'Chrome', 'traefik.access.user_agent.name': ['Chrome'],
'traefik.access.user_agent.original': 'traefik.access.user_agent.original': [
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
'traefik.access.user_agent.os': 'Linux', ],
'traefik.access.user_agent.os_name': 'Linux', 'traefik.access.user_agent.os': ['Linux'],
'traefik.access.user_agent.patch': '3163', 'traefik.access.user_agent.os_name': ['Linux'],
'traefik.access.user_identifier': '-', 'traefik.access.user_agent.patch': ['3163'],
'traefik.access.user_name': '-', 'traefik.access.user_identifier': ['-'],
'traefik.access.user_name': ['-'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[traefik][access] ", "constant": "[traefik][access] ",
}, },
Object { Object {
"field": "traefik.access.remote_ip", "field": "traefik.access.remote_ip",
"highlights": Array [], "highlights": Array [],
"value": "85.181.35.98", "value": Array [
}, "85.181.35.98",
Object { ],
"constant": " ", },
}, Object {
Object { "constant": " ",
"field": "traefik.access.frontend_name", },
"highlights": Array [], Object {
"value": "Host-host1", "field": "traefik.access.frontend_name",
}, "highlights": Array [],
Object { "value": Array [
"constant": " -> ", "Host-host1",
}, ],
Object { },
"field": "traefik.access.backend_url", Object {
"highlights": Array [], "constant": " -> ",
"value": "http://172.19.0.3:5601", },
}, Object {
Object { "field": "traefik.access.backend_url",
"constant": " \\"", "highlights": Array [],
}, "value": Array [
Object { "http://172.19.0.3:5601",
"field": "traefik.access.method", ],
"highlights": Array [], },
"value": "GET", Object {
}, "constant": " \\"",
Object { },
"constant": " ", Object {
}, "field": "traefik.access.method",
Object { "highlights": Array [],
"field": "traefik.access.url", "value": Array [
"highlights": Array [], "GET",
"value": "/ui/favicons/favicon.ico", ],
}, },
Object { Object {
"constant": " HTTP/", "constant": " ",
}, },
Object { Object {
"field": "traefik.access.http_version", "field": "traefik.access.url",
"highlights": Array [], "highlights": Array [],
"value": "1.1", "value": Array [
}, "/ui/favicons/favicon.ico",
Object { ],
"constant": "\\" ", },
}, Object {
Object { "constant": " HTTP/",
"field": "traefik.access.response_code", },
"highlights": Array [], Object {
"value": "304", "field": "traefik.access.http_version",
}, "highlights": Array [],
Object { "value": Array [
"constant": " ", "1.1",
}, ],
Object { },
"field": "traefik.access.body_sent.bytes", Object {
"highlights": Array [], "constant": "\\" ",
"value": "0", },
}, Object {
] "field": "traefik.access.response_code",
`); "highlights": Array [],
"value": Array [
"304",
],
},
Object {
"constant": " ",
},
Object {
"field": "traefik.access.body_sent.bytes",
"highlights": Array [],
"value": Array [
0,
],
},
]
`);
}); });
}); });
}); });

234
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.test.ts
View file

@ -15,154 +15,174 @@ describe('Generic Rules', () => {
describe('configurable message rules', () => { describe('configurable message rules', () => {
test('includes the event.dataset and log.level if present', () => { test('includes the event.dataset and log.level if present', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2016-12-26T16:22:13.000Z', '@timestamp': ['2016-12-26T16:22:13.000Z'],
'event.dataset': 'generic.test', 'event.dataset': ['generic.test'],
'log.level': 'TEST_LEVEL', 'log.level': ['TEST_LEVEL'],
first_generic_message: 'TEST_MESSAGE', first_generic_message: ['TEST_MESSAGE'],
}; };
const highlights = { const highlights = {
first_generic_message: ['TEST'], first_generic_message: ['TEST'],
}; };
expect(format(flattenedDocument, highlights)).toMatchInlineSnapshot(` expect(format(flattenedDocument, highlights)).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[", "constant": "[",
}, },
Object { Object {
"field": "event.dataset", "field": "event.dataset",
"highlights": Array [], "highlights": Array [],
"value": "generic.test", "value": Array [
}, "generic.test",
Object { ],
"constant": "][", },
}, Object {
Object { "constant": "][",
"field": "log.level", },
"highlights": Array [], Object {
"value": "TEST_LEVEL", "field": "log.level",
}, "highlights": Array [],
Object { "value": Array [
"constant": "] ", "TEST_LEVEL",
}, ],
Object { },
"field": "first_generic_message", Object {
"highlights": Array [ "constant": "] ",
"TEST", },
], Object {
"value": "TEST_MESSAGE", "field": "first_generic_message",
}, "highlights": Array [
] "TEST",
`); ],
"value": Array [
"TEST_MESSAGE",
],
},
]
`);
}); });
test('includes the log.level if present', () => { test('includes the log.level if present', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2016-12-26T16:22:13.000Z', '@timestamp': ['2016-12-26T16:22:13.000Z'],
'log.level': 'TEST_LEVEL', 'log.level': ['TEST_LEVEL'],
first_generic_message: 'TEST_MESSAGE', first_generic_message: ['TEST_MESSAGE'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[", "constant": "[",
}, },
Object { Object {
"field": "log.level", "field": "log.level",
"highlights": Array [], "highlights": Array [],
"value": "TEST_LEVEL", "value": Array [
}, "TEST_LEVEL",
Object { ],
"constant": "] ", },
}, Object {
Object { "constant": "] ",
"field": "first_generic_message", },
"highlights": Array [], Object {
"value": "TEST_MESSAGE", "field": "first_generic_message",
}, "highlights": Array [],
] "value": Array [
`); "TEST_MESSAGE",
],
},
]
`);
}); });
test('includes the message', () => { test('includes the message', () => {
const firstFlattenedDocument = { const firstFlattenedDocument = {
'@timestamp': '2016-12-26T16:22:13.000Z', '@timestamp': ['2016-12-26T16:22:13.000Z'],
first_generic_message: 'FIRST_TEST_MESSAGE', first_generic_message: ['FIRST_TEST_MESSAGE'],
}; };
expect(format(firstFlattenedDocument, {})).toMatchInlineSnapshot(` expect(format(firstFlattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"field": "first_generic_message", "field": "first_generic_message",
"highlights": Array [], "highlights": Array [],
"value": "FIRST_TEST_MESSAGE", "value": Array [
}, "FIRST_TEST_MESSAGE",
] ],
`); },
]
`);
const secondFlattenedDocument = { const secondFlattenedDocument = {
'@timestamp': '2016-12-26T16:22:13.000Z', '@timestamp': ['2016-12-26T16:22:13.000Z'],
second_generic_message: 'SECOND_TEST_MESSAGE', second_generic_message: ['SECOND_TEST_MESSAGE'],
}; };
expect(format(secondFlattenedDocument, {})).toMatchInlineSnapshot(` expect(format(secondFlattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"field": "second_generic_message", "field": "second_generic_message",
"highlights": Array [], "highlights": Array [],
"value": "SECOND_TEST_MESSAGE", "value": Array [
}, "SECOND_TEST_MESSAGE",
] ],
`); },
]
`);
}); });
}); });
describe('log.original fallback', () => { describe('log.original fallback', () => {
test('includes the event.dataset if present', () => { test('includes the event.dataset if present', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2016-12-26T16:22:13.000Z', '@timestamp': ['2016-12-26T16:22:13.000Z'],
'event.dataset': 'generic.test', 'event.dataset': ['generic.test'],
'log.original': 'TEST_MESSAGE', 'log.original': ['TEST_MESSAGE'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"constant": "[", "constant": "[",
}, },
Object { Object {
"field": "event.dataset", "field": "event.dataset",
"highlights": Array [], "highlights": Array [],
"value": "generic.test", "value": Array [
}, "generic.test",
Object { ],
"constant": "] ", },
}, Object {
Object { "constant": "] ",
"field": "log.original", },
"highlights": Array [], Object {
"value": "TEST_MESSAGE", "field": "log.original",
}, "highlights": Array [],
] "value": Array [
`); "TEST_MESSAGE",
],
},
]
`);
}); });
test('includes the original message', () => { test('includes the original message', () => {
const flattenedDocument = { const flattenedDocument = {
'@timestamp': '2016-12-26T16:22:13.000Z', '@timestamp': ['2016-12-26T16:22:13.000Z'],
'log.original': 'TEST_MESSAGE', 'log.original': ['TEST_MESSAGE'],
}; };
expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` expect(format(flattenedDocument, {})).toMatchInlineSnapshot(`
Array [ Array [
Object { Object {
"field": "log.original", "field": "log.original",
"highlights": Array [], "highlights": Array [],
"value": "TEST_MESSAGE", "value": Array [
}, "TEST_MESSAGE",
] ],
`); },
]
`);
}); });
}); });
}); });

7
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/helpers.ts
View file

@ -10,3 +10,10 @@ export const labelField = (label: string, field: string) => [
{ constant: '=' }, { constant: '=' },
{ field }, { field },
]; ];
export const labelFieldsPrefix = (label: string, fieldsPrefix: string) => [
{ constant: ' ' },
{ constant: label },
{ constant: '=' },
{ fieldsPrefix },
];

46
x-pack/plugins/infra/server/lib/domains/log_entries_domain/convert_document_source_to_log_item_fields.test.ts
View file

@ -4,66 +4,62 @@
* you may not use this file except in compliance with the Elastic License. * you may not use this file except in compliance with the Elastic License.
*/ */
import { convertDocumentSourceToLogItemFields } from './convert_document_source_to_log_item_fields'; import { convertESFieldsToLogItemFields } from './convert_document_source_to_log_item_fields';
describe('convertDocumentSourceToLogItemFields', () => { describe('convertESFieldsToLogItemFields', () => {
test('should convert document', () => { test('Converts the fields collection to LogItemFields', () => {
const doc = { const esFields = {
agent: { 'agent.hostname': ['demo-stack-client-01'],
hostname: 'demo-stack-client-01', 'agent.id': ['7adef8b6-2ab7-45cd-a0d5-b3baad735f1b'],
id: '7adef8b6-2ab7-45cd-a0d5-b3baad735f1b', 'agent.type': ['filebeat'],
type: 'filebeat', 'agent.ephemeral_id': ['a0c8164b-3564-4e32-b0bf-f4db5a7ae566'],
ephemeral_id: 'a0c8164b-3564-4e32-b0bf-f4db5a7ae566', 'agent.version': ['7.0.0'],
version: '7.0.0',
},
tags: ['prod', 'web'], tags: ['prod', 'web'],
metadata: [ metadata: [
{ key: 'env', value: 'prod' }, { key: 'env', value: 'prod' },
{ key: 'stack', value: 'web' }, { key: 'stack', value: 'web' },
], ],
host: { 'host.hostname': ['packer-virtualbox-iso-1546820004'],
hostname: 'packer-virtualbox-iso-1546820004', 'host.name': ['demo-stack-client-01'],
name: 'demo-stack-client-01',
},
}; };
const fields = convertDocumentSourceToLogItemFields(doc); const fields = convertESFieldsToLogItemFields(esFields);
expect(fields).toEqual([ expect(fields).toEqual([
{ {
field: 'agent.hostname', field: 'agent.hostname',
value: 'demo-stack-client-01', value: ['demo-stack-client-01'],
}, },
{ {
field: 'agent.id', field: 'agent.id',
value: '7adef8b6-2ab7-45cd-a0d5-b3baad735f1b', value: ['7adef8b6-2ab7-45cd-a0d5-b3baad735f1b'],
}, },
{ {
field: 'agent.type', field: 'agent.type',
value: 'filebeat', value: ['filebeat'],
}, },
{ {
field: 'agent.ephemeral_id', field: 'agent.ephemeral_id',
value: 'a0c8164b-3564-4e32-b0bf-f4db5a7ae566', value: ['a0c8164b-3564-4e32-b0bf-f4db5a7ae566'],
}, },
{ {
field: 'agent.version', field: 'agent.version',
value: '7.0.0', value: ['7.0.0'],
}, },
{ {
field: 'tags', field: 'tags',
value: '["prod","web"]', value: ['prod', 'web'],
}, },
{ {
field: 'metadata', field: 'metadata',
value: '[{"key":"env","value":"prod"},{"key":"stack","value":"web"}]', value: ['{"key":"env","value":"prod"}', '{"key":"stack","value":"web"}'],
}, },
{ {
field: 'host.hostname', field: 'host.hostname',
value: 'packer-virtualbox-iso-1546820004', value: ['packer-virtualbox-iso-1546820004'],
}, },
{ {
field: 'host.name', field: 'host.name',
value: 'demo-stack-client-01', value: ['demo-stack-client-01'],
}, },
]); ]);
}); });

46
x-pack/plugins/infra/server/lib/domains/log_entries_domain/convert_document_source_to_log_item_fields.ts
View file

@ -5,39 +5,21 @@
*/ */
import stringify from 'json-stable-stringify'; import stringify from 'json-stable-stringify';
import { isArray, isPlainObject } from 'lodash';
import { JsonObject } from '../../../../common/typed_json';
import { LogEntriesItemField } from '../../../../common/http_api'; import { LogEntriesItemField } from '../../../../common/http_api';
import { JsonArray } from '../../../../common/typed_json';
const isJsonObject = (subject: any): subject is JsonObject => { const serializeValue = (value: JsonArray): string[] => {
return isPlainObject(subject); return value.map((v) => {
}; if (typeof v === 'object' && v != null) {
return stringify(v);
const serializeValue = (value: any): string => { } else {
if (isArray(value) || isPlainObject(value)) { return `${v}`;
return stringify(value);
}
return `${value}`;
};
export const convertESFieldsToLogItemFields = (fields: {
[field: string]: [value: unknown];
}): LogEntriesItemField[] => {
return Object.keys(fields).map((field) => ({ field, value: serializeValue(fields[field][0]) }));
};
export const convertDocumentSourceToLogItemFields = (
source: JsonObject,
path: string[] = [],
fields: LogEntriesItemField[] = []
): LogEntriesItemField[] => {
return Object.keys(source).reduce((acc, key) => {
const value = source[key];
const nextPath = [...path, key];
if (isJsonObject(value)) {
return convertDocumentSourceToLogItemFields(value, nextPath, acc);
} }
const field = { field: nextPath.join('.'), value: serializeValue(value) }; });
return [...acc, field]; };
}, fields);
export const convertESFieldsToLogItemFields = (fields: {
[field: string]: JsonArray;
}): LogEntriesItemField[] => {
return Object.keys(fields).map((field) => ({ field, value: serializeValue(fields[field]) }));
}; };

20
x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts
View file

@ -7,7 +7,7 @@
import { sortBy } from 'lodash'; import { sortBy } from 'lodash';
import { RequestHandlerContext } from 'src/core/server'; import { RequestHandlerContext } from 'src/core/server';
import { JsonObject } from '../../../../common/typed_json'; import { JsonArray, JsonObject } from '../../../../common/typed_json';
import { import {
LogEntriesSummaryBucket, LogEntriesSummaryBucket,
LogEntriesSummaryHighlightsBucket, LogEntriesSummaryHighlightsBucket,
@ -163,8 +163,8 @@ export class InfraLogEntriesDomain {
return { return {
columnId: column.fieldColumn.id, columnId: column.fieldColumn.id,
field: column.fieldColumn.field, field: column.fieldColumn.field,
value: doc.fields[column.fieldColumn.field], value: doc.fields[column.fieldColumn.field] ?? [],
highlights: doc.highlights[column.fieldColumn.field] || [], highlights: doc.highlights[column.fieldColumn.field] ?? [],
}; };
} }
} }
@ -252,8 +252,8 @@ export class InfraLogEntriesDomain {
): Promise<LogEntriesItem> { ): Promise<LogEntriesItem> {
const document = await this.adapter.getLogItem(requestContext, id, sourceConfiguration); const document = await this.adapter.getLogItem(requestContext, id, sourceConfiguration);
const defaultFields = [ const defaultFields = [
{ field: '_index', value: document._index }, { field: '_index', value: [document._index] },
{ field: '_id', value: document._id }, { field: '_id', value: [document._id] },
]; ];
return { return {
@ -310,10 +310,10 @@ export class InfraLogEntriesDomain {
} }
} }
interface LogItemHit { export interface LogItemHit {
_index: string; _index: string;
_id: string; _id: string;
fields: { [field: string]: [value: unknown] }; fields: { [field: string]: [value: JsonArray] };
sort: [number, number]; sort: [number, number];
} }
@ -400,9 +400,9 @@ const createHighlightQueryDsl = (phrase: string, fields: string[]) => ({
const getContextFromDoc = (doc: LogEntryDocument): LogEntry['context'] => { const getContextFromDoc = (doc: LogEntryDocument): LogEntry['context'] => {
// Get all context fields, then test for the presence and type of the ones that go together // Get all context fields, then test for the presence and type of the ones that go together
const containerId = doc.fields['container.id']; const containerId = doc.fields['container.id']?.[0];
const hostName = doc.fields['host.name']; const hostName = doc.fields['host.name']?.[0];
const logFilePath = doc.fields['log.file.path']; const logFilePath = doc.fields['log.file.path']?.[0];
if (typeof containerId === 'string') { if (typeof containerId === 'string') {
return { 'container.id': containerId }; return { 'container.id': containerId };

109
x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts
View file

@ -4,11 +4,11 @@
* you may not use this file except in compliance with the Elastic License. * you may not use this file except in compliance with the Elastic License.
*/ */
import stringify from 'json-stable-stringify'; import { LogMessagePart } from '../../../../common/http_api/log_entries';
import { JsonArray, JsonValue } from '../../../../common/typed_json';
import { InfraLogMessageSegment } from '../../../graphql/types';
import { import {
LogMessageFormattingCondition, LogMessageFormattingCondition,
LogMessageFormattingFieldValueConditionValue,
LogMessageFormattingInstruction, LogMessageFormattingInstruction,
LogMessageFormattingRule, LogMessageFormattingRule,
} from './rule_types'; } from './rule_types';
@ -30,7 +30,7 @@ export function compileFormattingRules(
) )
) )
), ),
format(fields, highlights): InfraLogMessageSegment[] { format(fields, highlights): LogMessagePart[] {
for (const compiledRule of compiledRules) { for (const compiledRule of compiledRules) {
if (compiledRule.fulfillsCondition(fields)) { if (compiledRule.fulfillsCondition(fields)) {
return compiledRule.format(fields, highlights); return compiledRule.format(fields, highlights);
@ -59,16 +59,34 @@ const compileRule = (rule: LogMessageFormattingRule): CompiledLogMessageFormatti
const compileCondition = ( const compileCondition = (
condition: LogMessageFormattingCondition condition: LogMessageFormattingCondition
): CompiledLogMessageFormattingCondition => ): CompiledLogMessageFormattingCondition =>
[compileExistsCondition, compileFieldValueCondition].reduce( [
(compiledCondition, compile) => compile(condition) || compiledCondition, compileAllCondition,
catchAllCondition compileExistsCondition,
); compileExistsPrefixCondition,
compileFieldValueCondition,
].reduce((compiledCondition, compile) => compile(condition) || compiledCondition, falseCondition);
const catchAllCondition: CompiledLogMessageFormattingCondition = { const falseCondition: CompiledLogMessageFormattingCondition = {
conditionFields: [] as string[], conditionFields: [] as string[],
fulfillsCondition: () => false, fulfillsCondition: () => false,
}; };
const compileAllCondition = (
condition: LogMessageFormattingCondition
): CompiledLogMessageFormattingCondition | null => {
if (!('all' in condition)) {
return null;
}
const compiledConditions = condition.all.map(compileCondition);
return {
conditionFields: compiledConditions.flatMap(({ conditionFields }) => conditionFields),
fulfillsCondition: (fields: Fields) =>
compiledConditions.every(({ fulfillsCondition }) => fulfillsCondition(fields)),
};
};
const compileExistsCondition = (condition: LogMessageFormattingCondition) => const compileExistsCondition = (condition: LogMessageFormattingCondition) =>
'exists' in condition 'exists' in condition
? { ? {
@ -78,13 +96,24 @@ const compileExistsCondition = (condition: LogMessageFormattingCondition) =>
} }
: null; : null;
const compileExistsPrefixCondition = (condition: LogMessageFormattingCondition) =>
'existsPrefix' in condition
? {
conditionFields: condition.existsPrefix.map((prefix) => `${prefix}.*`),
fulfillsCondition: (fields: Fields) =>
condition.existsPrefix.every((fieldNamePrefix) =>
Object.keys(fields).some((field) => field.startsWith(`${fieldNamePrefix}.`))
),
}
: null;
const compileFieldValueCondition = (condition: LogMessageFormattingCondition) => const compileFieldValueCondition = (condition: LogMessageFormattingCondition) =>
'values' in condition 'values' in condition
? { ? {
conditionFields: Object.keys(condition.values), conditionFields: Object.keys(condition.values),
fulfillsCondition: (fields: Fields) => fulfillsCondition: (fields: Fields) =>
Object.entries(condition.values).every( Object.entries(condition.values).every(([fieldName, expectedValue]) =>
([fieldName, expectedValue]) => fields[fieldName] === expectedValue equalsOrContains(fields[fieldName] ?? [], expectedValue)
), ),
} }
: null; : null;
@ -116,7 +145,11 @@ const compileFormattingInstructions = (
const compileFormattingInstruction = ( const compileFormattingInstruction = (
formattingInstruction: LogMessageFormattingInstruction formattingInstruction: LogMessageFormattingInstruction
): CompiledLogMessageFormattingInstruction => ): CompiledLogMessageFormattingInstruction =>
[compileFieldReferenceFormattingInstruction, compileConstantFormattingInstruction].reduce( [
compileFieldReferenceFormattingInstruction,
compileFieldsPrefixReferenceFormattingInstruction,
compileConstantFormattingInstruction,
].reduce(
(compiledFormattingInstruction, compile) => (compiledFormattingInstruction, compile) =>
compile(formattingInstruction) || compiledFormattingInstruction, compile(formattingInstruction) || compiledFormattingInstruction,
catchAllFormattingInstruction catchAllFormattingInstruction
@ -138,19 +171,44 @@ const compileFieldReferenceFormattingInstruction = (
? { ? {
formattingFields: [formattingInstruction.field], formattingFields: [formattingInstruction.field],
format: (fields, highlights) => { format: (fields, highlights) => {
const value = fields[formattingInstruction.field]; const value = fields[formattingInstruction.field] ?? [];
const highlightedValues = highlights[formattingInstruction.field]; const highlightedValues = highlights[formattingInstruction.field] ?? [];
return [ return [
{ {
field: formattingInstruction.field, field: formattingInstruction.field,
value: typeof value === 'object' ? stringify(value) : `${value}`, value,
highlights: highlightedValues || [], highlights: highlightedValues,
}, },
]; ];
}, },
} }
: null; : null;
const compileFieldsPrefixReferenceFormattingInstruction = (
formattingInstruction: LogMessageFormattingInstruction
): CompiledLogMessageFormattingInstruction | null =>
'fieldsPrefix' in formattingInstruction
? {
formattingFields: [`${formattingInstruction.fieldsPrefix}.*`],
format: (fields, highlights) => {
const matchingFields = Object.keys(fields).filter((field) =>
field.startsWith(`${formattingInstruction.fieldsPrefix}.`)
);
return matchingFields.flatMap((field) => {
const value = fields[field] ?? [];
const highlightedValues = highlights[field] ?? [];
return [
{
field,
value,
highlights: highlightedValues,
},
];
});
},
}
: null;
const compileConstantFormattingInstruction = ( const compileConstantFormattingInstruction = (
formattingInstruction: LogMessageFormattingInstruction formattingInstruction: LogMessageFormattingInstruction
): CompiledLogMessageFormattingInstruction | null => ): CompiledLogMessageFormattingInstruction | null =>
@ -165,8 +223,21 @@ const compileConstantFormattingInstruction = (
} }
: null; : null;
const equalsOrContains = (
operand: JsonValue,
value: LogMessageFormattingFieldValueConditionValue
): boolean => {
if (Array.isArray(operand)) {
return operand.includes(value);
} else if (typeof operand === 'object' && operand !== null) {
return Object.values(operand).includes(value);
} else {
return operand === value;
}
};
export interface Fields { export interface Fields {
[fieldName: string]: string | number | object | boolean | null; [fieldName: string]: JsonArray;
} }
export interface Highlights { export interface Highlights {
@ -176,7 +247,7 @@ export interface Highlights {
export interface CompiledLogMessageFormattingRule { export interface CompiledLogMessageFormattingRule {
requiredFields: string[]; requiredFields: string[];
fulfillsCondition(fields: Fields): boolean; fulfillsCondition(fields: Fields): boolean;
format(fields: Fields, highlights: Highlights): InfraLogMessageSegment[]; format(fields: Fields, highlights: Highlights): LogMessagePart[];
} }
export interface CompiledLogMessageFormattingCondition { export interface CompiledLogMessageFormattingCondition {
@ -186,5 +257,5 @@ export interface CompiledLogMessageFormattingCondition {
export interface CompiledLogMessageFormattingInstruction { export interface CompiledLogMessageFormattingInstruction {
formattingFields: string[]; formattingFields: string[];
format(fields: Fields, highlights: Highlights): InfraLogMessageSegment[]; format(fields: Fields, highlights: Highlights): LogMessagePart[];
} }

21
x-pack/plugins/infra/server/lib/domains/log_entries_domain/rule_types.ts
View file

@ -4,33 +4,52 @@
* you may not use this file except in compliance with the Elastic License. * you may not use this file except in compliance with the Elastic License.
*/ */
import { JsonValue } from '../../../../common/typed_json';
export interface LogMessageFormattingRule { export interface LogMessageFormattingRule {
when: LogMessageFormattingCondition; when: LogMessageFormattingCondition;
format: LogMessageFormattingInstruction[]; format: LogMessageFormattingInstruction[];
} }
export type LogMessageFormattingCondition = export type LogMessageFormattingCondition =
| LogMessageFormattingAllCondition
| LogMessageFormattingExistsCondition | LogMessageFormattingExistsCondition
| LogMessageFormattingExistsPrefixCondition
| LogMessageFormattingFieldValueCondition; | LogMessageFormattingFieldValueCondition;
export interface LogMessageFormattingAllCondition {
all: LogMessageFormattingCondition[];
}
export interface LogMessageFormattingExistsCondition { export interface LogMessageFormattingExistsCondition {
exists: string[]; exists: string[];
} }
export interface LogMessageFormattingExistsPrefixCondition {
existsPrefix: string[];
}
export interface LogMessageFormattingFieldValueCondition { export interface LogMessageFormattingFieldValueCondition {
values: { values: {
[fieldName: string]: string | number | boolean | null; [fieldName: string]: LogMessageFormattingFieldValueConditionValue;
}; };
} }
export type LogMessageFormattingFieldValueConditionValue = JsonValue;
export type LogMessageFormattingInstruction = export type LogMessageFormattingInstruction =
| LogMessageFormattingFieldReference | LogMessageFormattingFieldReference
| LogMessageFormattingFieldsPrefixReference
| LogMessageFormattingConstant; | LogMessageFormattingConstant;
export interface LogMessageFormattingFieldReference { export interface LogMessageFormattingFieldReference {
field: string; field: string;
} }
export interface LogMessageFormattingFieldsPrefixReference {
fieldsPrefix: string;
}
export interface LogMessageFormattingConstant { export interface LogMessageFormattingConstant {
constant: string; constant: string;
} }

52
x-pack/test/api_integration/apis/metrics_ui/log_item.ts
View file

@ -44,107 +44,107 @@ export default function ({ getService }: FtrProviderContext) {
expect(logItem.fields).to.eql([ expect(logItem.fields).to.eql([
{ {
field: '@timestamp', field: '@timestamp',
value: '2018-10-17T19:42:22.000Z', value: ['2018-10-17T19:42:22.000Z'],
}, },
{ {
field: '_id', field: '_id',
value: 'yT2Mg2YBh-opCxJv8Vqj', value: ['yT2Mg2YBh-opCxJv8Vqj'],
}, },
{ {
field: '_index', field: '_index',
value: 'filebeat-7.0.0-alpha1-2018.10.17', value: ['filebeat-7.0.0-alpha1-2018.10.17'],
}, },
{ {
field: 'apache2.access.body_sent.bytes', field: 'apache2.access.body_sent.bytes',
value: '1336', value: ['1336'],
}, },
{ {
field: 'apache2.access.http_version', field: 'apache2.access.http_version',
value: '1.1', value: ['1.1'],
}, },
{ {
field: 'apache2.access.method', field: 'apache2.access.method',
value: 'GET', value: ['GET'],
}, },
{ {
field: 'apache2.access.referrer', field: 'apache2.access.referrer',
value: '-', value: ['-'],
}, },
{ {
field: 'apache2.access.remote_ip', field: 'apache2.access.remote_ip',
value: '10.128.0.11', value: ['10.128.0.11'],
}, },
{ {
field: 'apache2.access.response_code', field: 'apache2.access.response_code',
value: '200', value: ['200'],
}, },
{ {
field: 'apache2.access.url', field: 'apache2.access.url',
value: '/a-fresh-start-will-put-you-on-your-way', value: ['/a-fresh-start-will-put-you-on-your-way'],
}, },
{ {
field: 'apache2.access.user_agent.device', field: 'apache2.access.user_agent.device',
value: 'Other', value: ['Other'],
}, },
{ {
field: 'apache2.access.user_agent.name', field: 'apache2.access.user_agent.name',
value: 'Other', value: ['Other'],
}, },
{ {
field: 'apache2.access.user_agent.os', field: 'apache2.access.user_agent.os',
value: 'Other', value: ['Other'],
}, },
{ {
field: 'apache2.access.user_agent.os_name', field: 'apache2.access.user_agent.os_name',
value: 'Other', value: ['Other'],
}, },
{ {
field: 'apache2.access.user_name', field: 'apache2.access.user_name',
value: '-', value: ['-'],
}, },
{ {
field: 'beat.hostname', field: 'beat.hostname',
value: 'demo-stack-apache-01', value: ['demo-stack-apache-01'],
}, },
{ {
field: 'beat.name', field: 'beat.name',
value: 'demo-stack-apache-01', value: ['demo-stack-apache-01'],
}, },
{ {
field: 'beat.version', field: 'beat.version',
value: '7.0.0-alpha1', value: ['7.0.0-alpha1'],
}, },
{ {
field: 'fileset.module', field: 'fileset.module',
value: 'apache2', value: ['apache2'],
}, },
{ {
field: 'fileset.name', field: 'fileset.name',
value: 'access', value: ['access'],
}, },
{ {
field: 'host.name', field: 'host.name',
value: 'demo-stack-apache-01', value: ['demo-stack-apache-01'],
}, },
{ {
field: 'input.type', field: 'input.type',
value: 'log', value: ['log'],
}, },
{ {
field: 'offset', field: 'offset',
value: '5497614', value: ['5497614'],
}, },
{ {
field: 'prospector.type', field: 'prospector.type',
value: 'log', value: ['log'],
}, },
{ {
field: 'read_timestamp', field: 'read_timestamp',
value: '2018-10-17T19:42:23.160Z', value: ['2018-10-17T19:42:23.160Z'],
}, },
{ {
field: 'source', field: 'source',
value: '/var/log/apache2/access.log', value: ['/var/log/apache2/access.log'],
}, },
]); ]);
}); });