Optimizes the index queries to not block the NodeJS event loop (#75716)
## Summary Before this PR you can see event loop block times of: ```ts formatIndexFields: 7986.884ms ``` After this PR you will see event loop block times of: ```ts formatIndexFields: 85.012ms ``` within the file: ```ts x-pack/plugins/security_solution/server/lib/index_fields/elasticsearch_adapter.ts ``` For the GraphQL query of `SourceQuery`/`IndexFields` This also fixes the issue of `unknown` being returned to the front end by removing code that is no longer functioning as it was intended. Ensure during testing of this PR that blank/default and non exist indexes within `securitySolution:defaultIndex` still work as expected. Before, notice the `unknown` instead of the `filebeat-*`: <img width="733" alt="Screen Shot 2020-08-20 at 4 55 52 PM" src="https://user-images.githubusercontent.com/1151048/90949129-f5047900-e402-11ea-9278-b4c7bf5cd16d.png"> After: <img width="830" alt="Screen Shot 2020-08-20 at 4 56 03 PM" src="https://user-images.githubusercontent.com/1151048/90949133-02b9fe80-e403-11ea-8504-f5bbe043048a.png"> An explanation of how to see the block times for before and after --- For perf testing you first add timed testing to the file: ```ts x-pack/plugins/security_solution/server/lib/index_fields/elasticsearch_adapter.ts ``` Before this PR, around lines 42: ```ts console.time('formatIndexFields'); // <--- start timer const fields = formatIndexFields( responsesIndexFields, Object.keys(indexesAliasIndices) as IndexAlias[] ); console.timeEnd('formatIndexFields'); // <--- outputs the end timer return fields; ``` After this PR, around lines 42: ```ts console.time('formatIndexFields'); // <--- start timer const fields = await formatIndexFields(responsesIndexFields, indices); console.timeEnd('formatIndexFields'); // <--- outputs the end timer return fields; ``` And then reload the security solutions application web page here: ``` http://localhost:5601/app/security/timelines/default ``` Be sure to load it _twice_ for testing as NodeJS will sometimes report better numbers the second time as it does optimizations after the first time it encounters some code paths. You will begin to see numbers similar to this before this PR: ```ts formatIndexFields: 2553.279ms ``` This indicates that it is blocking the event loop for ~2.5 seconds befofe this fix. If you add additional indexes to your `securitySolution:defaultIndex` indexes that have additional fields then this amount will increase exponentially. For developers using our test servers I created two other indexes called delme-1 and delme-2 with additional mappings you can add like below ```ts apm-*-transaction*, auditbeat-*, endgame-*, filebeat-*, logs-*, packetbeat-*, winlogbeat-*, delme-1, delme-2 ``` <img width="980" alt="Screen Shot 2020-08-21 at 8 21 50 PM" src="https://user-images.githubusercontent.com/1151048/90949142-211ffa00-e403-11ea-8ab2-f66de977dce3.png"> Then you are going to see times approaching 8 seconds of blocking the event loop like so: ```ts formatIndexFields: 7986.884ms ``` After this fix on the first pass unoptimized it will report ```ts formatIndexFields: 373.082ms ``` Then after it optimizes the code paths on a second page load it will report ```ts formatIndexFields: 84.304ms ``` ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
This commit is contained in:
Frank Hassanabad
GitHub
parent
5f89e0003b
commit
ba9a607384
|
|
@ -6,16 +6,21 @@
|
|||
|
||||
import { sortBy } from 'lodash/fp';
|
||||
|
||||
import { formatIndexFields } from './elasticsearch_adapter';
|
||||
import {
|
||||
formatIndexFields,
|
||||
formatFirstFields,
|
||||
formatSecondFields,
|
||||
createFieldItem,
|
||||
} from './elasticsearch_adapter';
|
||||
import { mockAuditbeatIndexField, mockFilebeatIndexField, mockPacketbeatIndexField } from './mock';
|
||||
|
||||
describe('Index Fields', () => {
|
||||
describe('formatIndexFields', () => {
|
||||
test('Test Basic functionality', async () => {
|
||||
test('Basic functionality', async () => {
|
||||
expect(
|
||||
sortBy(
|
||||
'name',
|
||||
formatIndexFields(
|
||||
await formatIndexFields(
|
||||
[mockAuditbeatIndexField, mockFilebeatIndexField, mockPacketbeatIndexField],
|
||||
['auditbeat', 'filebeat', 'packetbeat']
|
||||
)
|
||||
|
|
@ -130,4 +135,557 @@ describe('Index Fields', () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatFirstFields', () => {
|
||||
test('Basic functionality', async () => {
|
||||
const fields = await formatFirstFields(
|
||||
[mockAuditbeatIndexField, mockFilebeatIndexField, mockPacketbeatIndexField],
|
||||
['auditbeat', 'filebeat', 'packetbeat']
|
||||
);
|
||||
expect(fields).toEqual([
|
||||
{
|
||||
description: 'Each document has an _id that uniquely identifies it',
|
||||
example: 'Y-6TfmcB0WOhS6qyMv3s',
|
||||
footnote: '',
|
||||
group: 1,
|
||||
level: 'core',
|
||||
name: '_id',
|
||||
required: true,
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: false,
|
||||
readFromDocValues: true,
|
||||
category: '_id',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'An index is like a ‘database’ in a relational database. It has a mapping which defines multiple types. An index is a logical namespace which maps to one or more primary shards and can have zero or more replica shards.',
|
||||
example: 'auditbeat-8.0.0-2019.02.19-000001',
|
||||
footnote: '',
|
||||
group: 1,
|
||||
level: 'core',
|
||||
name: '_index',
|
||||
required: true,
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
readFromDocValues: true,
|
||||
category: '_index',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.',
|
||||
example: '2016-05-23T08:05:34.853Z',
|
||||
name: '@timestamp',
|
||||
type: 'date',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'base',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.',
|
||||
example: '8a4f500f',
|
||||
name: 'agent.ephemeral_id',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.',
|
||||
example: 'foo',
|
||||
name: 'agent.name',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.',
|
||||
example: 'filebeat',
|
||||
name: 'agent.type',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description: 'Version of the agent.',
|
||||
example: '6.0.0-rc2',
|
||||
name: 'agent.version',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description: 'Each document has an _id that uniquely identifies it',
|
||||
example: 'Y-6TfmcB0WOhS6qyMv3s',
|
||||
footnote: '',
|
||||
group: 1,
|
||||
level: 'core',
|
||||
name: '_id',
|
||||
required: true,
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: false,
|
||||
readFromDocValues: true,
|
||||
category: '_id',
|
||||
indexes: ['filebeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'An index is like a ‘database’ in a relational database. It has a mapping which defines multiple types. An index is a logical namespace which maps to one or more primary shards and can have zero or more replica shards.',
|
||||
example: 'auditbeat-8.0.0-2019.02.19-000001',
|
||||
footnote: '',
|
||||
group: 1,
|
||||
level: 'core',
|
||||
name: '_index',
|
||||
required: true,
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
readFromDocValues: true,
|
||||
category: '_index',
|
||||
indexes: ['filebeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.',
|
||||
example: '2016-05-23T08:05:34.853Z',
|
||||
name: '@timestamp',
|
||||
type: 'date',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'base',
|
||||
indexes: ['filebeat'],
|
||||
},
|
||||
{
|
||||
name: 'agent.hostname',
|
||||
searchable: true,
|
||||
type: 'string',
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['filebeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.',
|
||||
example: 'foo',
|
||||
name: 'agent.name',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['filebeat'],
|
||||
},
|
||||
{
|
||||
description: 'Version of the agent.',
|
||||
example: '6.0.0-rc2',
|
||||
name: 'agent.version',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['filebeat'],
|
||||
},
|
||||
{
|
||||
description: 'Each document has an _id that uniquely identifies it',
|
||||
example: 'Y-6TfmcB0WOhS6qyMv3s',
|
||||
footnote: '',
|
||||
group: 1,
|
||||
level: 'core',
|
||||
name: '_id',
|
||||
required: true,
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: false,
|
||||
readFromDocValues: true,
|
||||
category: '_id',
|
||||
indexes: ['packetbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'An index is like a ‘database’ in a relational database. It has a mapping which defines multiple types. An index is a logical namespace which maps to one or more primary shards and can have zero or more replica shards.',
|
||||
example: 'auditbeat-8.0.0-2019.02.19-000001',
|
||||
footnote: '',
|
||||
group: 1,
|
||||
level: 'core',
|
||||
name: '_index',
|
||||
required: true,
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
readFromDocValues: true,
|
||||
category: '_index',
|
||||
indexes: ['packetbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.',
|
||||
example: '2016-05-23T08:05:34.853Z',
|
||||
name: '@timestamp',
|
||||
type: 'date',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'base',
|
||||
indexes: ['packetbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.',
|
||||
example: '8a4f500d',
|
||||
name: 'agent.id',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['packetbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.',
|
||||
example: 'filebeat',
|
||||
name: 'agent.type',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['packetbeat'],
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatSecondFields', () => {
|
||||
test('Basic functionality', async () => {
|
||||
const fields = await formatSecondFields([
|
||||
{
|
||||
description: 'Each document has an _id that uniquely identifies it',
|
||||
example: 'Y-6TfmcB0WOhS6qyMv3s',
|
||||
name: '_id',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: false,
|
||||
category: '_id',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'An index is like a ‘database’ in a relational database. It has a mapping which defines multiple types. An index is a logical namespace which maps to one or more primary shards and can have zero or more replica shards.',
|
||||
example: 'auditbeat-8.0.0-2019.02.19-000001',
|
||||
name: '_index',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: '_index',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.',
|
||||
example: '2016-05-23T08:05:34.853Z',
|
||||
name: '@timestamp',
|
||||
type: 'date',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'base',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.',
|
||||
example: '8a4f500f',
|
||||
name: 'agent.ephemeral_id',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.',
|
||||
example: 'foo',
|
||||
name: 'agent.name',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.',
|
||||
example: 'filebeat',
|
||||
name: 'agent.type',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description: 'Version of the agent.',
|
||||
example: '6.0.0-rc2',
|
||||
name: 'agent.version',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description: 'Each document has an _id that uniquely identifies it',
|
||||
example: 'Y-6TfmcB0WOhS6qyMv3s',
|
||||
name: '_id',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: false,
|
||||
category: '_id',
|
||||
indexes: ['filebeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'An index is like a ‘database’ in a relational database. It has a mapping which defines multiple types. An index is a logical namespace which maps to one or more primary shards and can have zero or more replica shards.',
|
||||
example: 'auditbeat-8.0.0-2019.02.19-000001',
|
||||
name: '_index',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: '_index',
|
||||
indexes: ['filebeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.',
|
||||
example: '2016-05-23T08:05:34.853Z',
|
||||
name: '@timestamp',
|
||||
type: 'date',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'base',
|
||||
indexes: ['filebeat'],
|
||||
},
|
||||
{
|
||||
name: 'agent.hostname',
|
||||
searchable: true,
|
||||
type: 'string',
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['filebeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.',
|
||||
example: 'foo',
|
||||
name: 'agent.name',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['filebeat'],
|
||||
},
|
||||
{
|
||||
description: 'Version of the agent.',
|
||||
example: '6.0.0-rc2',
|
||||
name: 'agent.version',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['filebeat'],
|
||||
},
|
||||
{
|
||||
description: 'Each document has an _id that uniquely identifies it',
|
||||
example: 'Y-6TfmcB0WOhS6qyMv3s',
|
||||
name: '_id',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: false,
|
||||
category: '_id',
|
||||
indexes: ['packetbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'An index is like a ‘database’ in a relational database. It has a mapping which defines multiple types. An index is a logical namespace which maps to one or more primary shards and can have zero or more replica shards.',
|
||||
example: 'auditbeat-8.0.0-2019.02.19-000001',
|
||||
name: '_index',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: '_index',
|
||||
indexes: ['packetbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.',
|
||||
example: '2016-05-23T08:05:34.853Z',
|
||||
name: '@timestamp',
|
||||
type: 'date',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'base',
|
||||
indexes: ['packetbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.',
|
||||
example: '8a4f500d',
|
||||
name: 'agent.id',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['packetbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.',
|
||||
example: 'filebeat',
|
||||
name: 'agent.type',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['packetbeat'],
|
||||
},
|
||||
]);
|
||||
expect(fields).toEqual([
|
||||
{
|
||||
description: 'Each document has an _id that uniquely identifies it',
|
||||
example: 'Y-6TfmcB0WOhS6qyMv3s',
|
||||
name: '_id',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: false,
|
||||
category: '_id',
|
||||
indexes: ['auditbeat', 'filebeat', 'packetbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'An index is like a ‘database’ in a relational database. It has a mapping which defines multiple types. An index is a logical namespace which maps to one or more primary shards and can have zero or more replica shards.',
|
||||
example: 'auditbeat-8.0.0-2019.02.19-000001',
|
||||
name: '_index',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: '_index',
|
||||
indexes: ['auditbeat', 'filebeat', 'packetbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.',
|
||||
example: '2016-05-23T08:05:34.853Z',
|
||||
name: '@timestamp',
|
||||
type: 'date',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'base',
|
||||
indexes: ['auditbeat', 'filebeat', 'packetbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.',
|
||||
example: '8a4f500f',
|
||||
name: 'agent.ephemeral_id',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['auditbeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.',
|
||||
example: 'foo',
|
||||
name: 'agent.name',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['auditbeat', 'filebeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.',
|
||||
example: 'filebeat',
|
||||
name: 'agent.type',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['auditbeat', 'packetbeat'],
|
||||
},
|
||||
{
|
||||
description: 'Version of the agent.',
|
||||
example: '6.0.0-rc2',
|
||||
name: 'agent.version',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['auditbeat', 'filebeat'],
|
||||
},
|
||||
{
|
||||
name: 'agent.hostname',
|
||||
searchable: true,
|
||||
type: 'string',
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['filebeat'],
|
||||
},
|
||||
{
|
||||
description:
|
||||
'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.',
|
||||
example: '8a4f500d',
|
||||
name: 'agent.id',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: true,
|
||||
category: 'agent',
|
||||
indexes: ['packetbeat'],
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createFieldItem', () => {
|
||||
test('Basic functionality', () => {
|
||||
const item = createFieldItem(
|
||||
['auditbeat'],
|
||||
{
|
||||
name: '_id',
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: false,
|
||||
},
|
||||
0
|
||||
);
|
||||
expect(item).toEqual({
|
||||
description: 'Each document has an _id that uniquely identifies it',
|
||||
example: 'Y-6TfmcB0WOhS6qyMv3s',
|
||||
footnote: '',
|
||||
group: 1,
|
||||
level: 'core',
|
||||
name: '_id',
|
||||
required: true,
|
||||
type: 'string',
|
||||
searchable: true,
|
||||
aggregatable: false,
|
||||
category: '_id',
|
||||
indexes: ['auditbeat'],
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,45 +4,25 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { isEmpty, get } from 'lodash/fp';
|
||||
import { isEmpty } from 'lodash/fp';
|
||||
|
||||
import { IndexField } from '../../graphql/types';
|
||||
import {
|
||||
baseCategoryFields,
|
||||
getDocumentation,
|
||||
getIndexAlias,
|
||||
hasDocumentation,
|
||||
IndexAlias,
|
||||
} from '../../utils/beat_schema';
|
||||
import { baseCategoryFields, getDocumentation, hasDocumentation } from '../../utils/beat_schema';
|
||||
import { FrameworkAdapter, FrameworkRequest } from '../framework';
|
||||
import { FieldsAdapter, IndexFieldDescriptor } from './types';
|
||||
|
||||
export class ElasticsearchIndexFieldAdapter implements FieldsAdapter {
|
||||
constructor(private readonly framework: FrameworkAdapter) {}
|
||||
|
||||
public async getIndexFields(request: FrameworkRequest, indices: string[]): Promise<IndexField[]> {
|
||||
const indexPatternsService = this.framework.getIndexPatternsService(request);
|
||||
const indexesAliasIndices = indices.reduce<Record<string, string[]>>((accumulator, indice) => {
|
||||
const key = getIndexAlias(indices, indice);
|
||||
|
||||
if (get(key, accumulator)) {
|
||||
accumulator[key] = [...accumulator[key], indice];
|
||||
} else {
|
||||
accumulator[key] = [indice];
|
||||
}
|
||||
return accumulator;
|
||||
}, {});
|
||||
const responsesIndexFields: IndexFieldDescriptor[][] = await Promise.all(
|
||||
Object.values(indexesAliasIndices).map((indicesByGroup) =>
|
||||
indexPatternsService.getFieldsForWildcard({
|
||||
pattern: indicesByGroup,
|
||||
})
|
||||
)
|
||||
);
|
||||
return formatIndexFields(
|
||||
responsesIndexFields,
|
||||
Object.keys(indexesAliasIndices) as IndexAlias[]
|
||||
const responsesIndexFields = await Promise.all(
|
||||
indices.map((index) => {
|
||||
return indexPatternsService.getFieldsForWildcard({
|
||||
pattern: index,
|
||||
});
|
||||
})
|
||||
);
|
||||
return formatIndexFields(responsesIndexFields, indices);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -63,51 +43,128 @@ const missingFields = [
|
|||
},
|
||||
];
|
||||
|
||||
export const formatIndexFields = (
|
||||
/**
|
||||
* Creates a single field item.
|
||||
*
|
||||
* This is a mutatious HOT CODE PATH function that will have array sizes up to 4.7 megs
|
||||
* in size at a time calling this function repeatedly. This function should be as optimized as possible
|
||||
* and should avoid any and all creation of new arrays, iterating over the arrays or performing
|
||||
* any n^2 operations.
|
||||
* @param indexesAlias The index alias
|
||||
* @param index The index its self
|
||||
* @param indexesAliasIdx The index within the alias
|
||||
*/
|
||||
export const createFieldItem = (
|
||||
indexesAlias: string[],
|
||||
index: IndexFieldDescriptor,
|
||||
indexesAliasIdx: number
|
||||
): IndexField => {
|
||||
const alias = indexesAlias[indexesAliasIdx];
|
||||
const splitName = index.name.split('.');
|
||||
const category = baseCategoryFields.includes(splitName[0]) ? 'base' : splitName[0];
|
||||
return {
|
||||
...(hasDocumentation(alias, index.name) ? getDocumentation(alias, index.name) : {}),
|
||||
...index,
|
||||
category,
|
||||
indexes: [alias],
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* This is a mutatious HOT CODE PATH function that will have array sizes up to 4.7 megs
|
||||
* in size at a time when being called. This function should be as optimized as possible
|
||||
* and should avoid any and all creation of new arrays, iterating over the arrays or performing
|
||||
* any n^2 operations. The `.push`, and `forEach` operations are expected within this function
|
||||
* to speed up performance.
|
||||
*
|
||||
* This intentionally waits for the next tick on the event loop to process as the large 4.7 megs
|
||||
* has already consumed a lot of the event loop processing up to this function and we want to give
|
||||
* I/O opportunity to occur by scheduling this on the next loop.
|
||||
* @param responsesIndexFields The response index fields to loop over
|
||||
* @param indexesAlias The index aliases such as filebeat-*
|
||||
*/
|
||||
export const formatFirstFields = async (
|
||||
responsesIndexFields: IndexFieldDescriptor[][],
|
||||
indexesAlias: IndexAlias[]
|
||||
): IndexField[] =>
|
||||
responsesIndexFields
|
||||
.reduce(
|
||||
(accumulator: IndexField[], indexFields: IndexFieldDescriptor[], indexesAliasIdx: number) => [
|
||||
...accumulator,
|
||||
...[...missingFields, ...indexFields].reduce(
|
||||
(itemAccumulator: IndexField[], index: IndexFieldDescriptor) => {
|
||||
const alias: IndexAlias = indexesAlias[indexesAliasIdx];
|
||||
const splitName = index.name.split('.');
|
||||
const category = baseCategoryFields.includes(splitName[0]) ? 'base' : splitName[0];
|
||||
return [
|
||||
...itemAccumulator,
|
||||
{
|
||||
...(hasDocumentation(alias, index.name) ? getDocumentation(alias, index.name) : {}),
|
||||
...index,
|
||||
category,
|
||||
indexes: [alias],
|
||||
} as IndexField,
|
||||
];
|
||||
indexesAlias: string[]
|
||||
): Promise<IndexField[]> => {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(() => {
|
||||
resolve(
|
||||
responsesIndexFields.reduce(
|
||||
(
|
||||
accumulator: IndexField[],
|
||||
indexFields: IndexFieldDescriptor[],
|
||||
indexesAliasIdx: number
|
||||
) => {
|
||||
missingFields.forEach((index) => {
|
||||
const item = createFieldItem(indexesAlias, index, indexesAliasIdx);
|
||||
accumulator.push(item);
|
||||
});
|
||||
indexFields.forEach((index) => {
|
||||
const item = createFieldItem(indexesAlias, index, indexesAliasIdx);
|
||||
accumulator.push(item);
|
||||
});
|
||||
return accumulator;
|
||||
},
|
||||
[]
|
||||
),
|
||||
],
|
||||
[]
|
||||
)
|
||||
.reduce((accumulator: IndexField[], indexfield: IndexField) => {
|
||||
const alreadyExistingIndexField = accumulator.findIndex(
|
||||
(acc) => acc.name === indexfield.name
|
||||
)
|
||||
);
|
||||
if (alreadyExistingIndexField > -1) {
|
||||
const existingIndexField = accumulator[alreadyExistingIndexField];
|
||||
return [
|
||||
...accumulator.slice(0, alreadyExistingIndexField),
|
||||
{
|
||||
...existingIndexField,
|
||||
description: isEmpty(existingIndexField.description)
|
||||
? indexfield.description
|
||||
: existingIndexField.description,
|
||||
indexes: Array.from(new Set([...existingIndexField.indexes, ...indexfield.indexes])),
|
||||
},
|
||||
...accumulator.slice(alreadyExistingIndexField + 1),
|
||||
];
|
||||
}
|
||||
return [...accumulator, indexfield];
|
||||
}, []);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* This is a mutatious HOT CODE PATH function that will have array sizes up to 4.7 megs
|
||||
* in size at a time when being called. This function should be as optimized as possible
|
||||
* and should avoid any and all creation of new arrays, iterating over the arrays or performing
|
||||
* any n^2 operations. The `.push`, and `forEach` operations are expected within this function
|
||||
* to speed up performance. The "indexFieldNameHash" side effect hash avoids additional expensive n^2
|
||||
* look ups.
|
||||
*
|
||||
* This intentionally waits for the next tick on the event loop to process as the large 4.7 megs
|
||||
* has already consumed a lot of the event loop processing up to this function and we want to give
|
||||
* I/O opportunity to occur by scheduling this on the next loop.
|
||||
* @param fields The index fields to create the secondary fields for
|
||||
*/
|
||||
export const formatSecondFields = async (fields: IndexField[]): Promise<IndexField[]> => {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(() => {
|
||||
const indexFieldNameHash: Record<string, number> = {};
|
||||
const reduced = fields.reduce((accumulator: IndexField[], indexfield: IndexField) => {
|
||||
const alreadyExistingIndexField = indexFieldNameHash[indexfield.name];
|
||||
if (alreadyExistingIndexField != null) {
|
||||
const existingIndexField = accumulator[alreadyExistingIndexField];
|
||||
if (isEmpty(accumulator[alreadyExistingIndexField].description)) {
|
||||
accumulator[alreadyExistingIndexField].description = indexfield.description;
|
||||
}
|
||||
accumulator[alreadyExistingIndexField].indexes = Array.from(
|
||||
new Set([...existingIndexField.indexes, ...indexfield.indexes])
|
||||
);
|
||||
return accumulator;
|
||||
}
|
||||
accumulator.push(indexfield);
|
||||
indexFieldNameHash[indexfield.name] = accumulator.length - 1;
|
||||
return accumulator;
|
||||
}, []);
|
||||
resolve(reduced);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Formats the index fields into a format the UI wants.
|
||||
*
|
||||
* NOTE: This will have array sizes up to 4.7 megs in size at a time when being called.
|
||||
* This function should be as optimized as possible and should avoid any and all creation
|
||||
* of new arrays, iterating over the arrays or performing any n^2 operations.
|
||||
* @param responsesIndexFields The response index fields to format
|
||||
* @param indexesAlias The index alias
|
||||
*/
|
||||
export const formatIndexFields = async (
|
||||
responsesIndexFields: IndexFieldDescriptor[][],
|
||||
indexesAlias: string[]
|
||||
): Promise<IndexField[]> => {
|
||||
const fields = await formatFirstFields(responsesIndexFields, indexesAlias);
|
||||
const secondFields = await formatSecondFields(fields);
|
||||
return secondFields;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
import { cloneDeep, isArray } from 'lodash/fp';
|
||||
|
||||
import { convertSchemaToAssociativeArray, getIndexSchemaDoc, getIndexAlias } from '.';
|
||||
import { convertSchemaToAssociativeArray, getIndexSchemaDoc } from '.';
|
||||
import { auditbeatSchema, filebeatSchema, packetbeatSchema } from './8.0.0';
|
||||
import { Schema } from './type';
|
||||
|
||||
|
|
@ -394,24 +394,4 @@ describe('Schema Beat', () => {
|
|||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getIndexAlias', () => {
|
||||
test('getIndexAlias handles values with leading wildcard', () => {
|
||||
const leadingWildcardIndex = '*-auditbeat-*';
|
||||
const result = getIndexAlias([leadingWildcardIndex], leadingWildcardIndex);
|
||||
expect(result).toBe(leadingWildcardIndex);
|
||||
});
|
||||
|
||||
test('getIndexAlias no match returns "unknown" string', () => {
|
||||
const index = 'auditbeat-*';
|
||||
const result = getIndexAlias([index], 'hello');
|
||||
expect(result).toBe('unknown');
|
||||
});
|
||||
|
||||
test('empty index should not cause an error to return although it will cause an invalid regular expression to occur', () => {
|
||||
const index = '';
|
||||
const result = getIndexAlias([index], 'hello');
|
||||
expect(result).toBe('unknown');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -76,21 +76,6 @@ const convertFieldsToAssociativeArray = (
|
|||
}, {})
|
||||
: {};
|
||||
|
||||
export const getIndexAlias = (defaultIndex: string[], indexName: string): string => {
|
||||
try {
|
||||
const found = defaultIndex.find((index) => `\\${indexName}`.match(`\\${index}`) != null);
|
||||
if (found != null) {
|
||||
return found;
|
||||
} else {
|
||||
return 'unknown';
|
||||
}
|
||||
} catch (error) {
|
||||
// if we encounter an error because the index contains invalid regular expressions then we should return an unknown
|
||||
// rather than blow up with a toaster error upstream
|
||||
return 'unknown';
|
||||
}
|
||||
};
|
||||
|
||||
export const getIndexSchemaDoc = memoize((index: string) => {
|
||||
if (index.match('auditbeat') != null) {
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
export type IndexAlias = 'auditbeat' | 'filebeat' | 'packetbeat' | 'ecs' | 'winlogbeat' | 'unknown';
|
||||
|
||||
/*
|
||||
* BEAT Interface
|
||||
*
|
||||
|
|
|
|||
Loading…
Reference in a new issue