maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fixing exceptions export format (#114920)

Browse source 
### Summary

Fixing exceptions export format and adding integration tests for it.
This commit is contained in:
Yara Tercero 2021-10-13 20:32:43 -07:00 committed by GitHub
parent 86f0733e56
commit 69a6cf329c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 294 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/kbn-securitysolution-list-api/src/api/index.ts
View file

@ -558,7 +558,7 @@ export const exportExceptionList = async ({
signal,
}: ExportExceptionListProps): Promise<Blob> =>
http.fetch<Blob>(`${EXCEPTION_LIST_URL}/_export`, {
method: 'GET',
method: 'POST',
query: { id, list_id: listId, namespace_type: namespaceType },
signal,
});

1
packages/kbn-securitysolution-utils/src/index.ts
View file

@ -7,3 +7,4 @@
*/
export * from './add_remove_id_to_item';
export * from './transform_data_to_ndjson';

88
packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.test.ts Normal file
View file

@ -0,0 +1,88 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { transformDataToNdjson } from './';
export const ANCHOR_DATE = '2020-02-20T03:57:54.037Z';
const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE) => ({
author: [],
id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
created_at: new Date(anchorDate).toISOString(),
updated_at: new Date(anchorDate).toISOString(),
created_by: 'elastic',
description: 'some description',
enabled: true,
false_positives: ['false positive 1', 'false positive 2'],
from: 'now-6m',
immutable: false,
name: 'Query with a rule id',
query: 'user.name: root or user.name: admin',
references: ['test 1', 'test 2'],
severity: 'high',
severity_mapping: [],
updated_by: 'elastic_kibana',
tags: ['some fake tag 1', 'some fake tag 2'],
to: 'now',
type: 'query',
threat: [],
version: 1,
status: 'succeeded',
status_date: '2020-02-22T16:47:50.047Z',
last_success_at: '2020-02-22T16:47:50.047Z',
last_success_message: 'succeeded',
output_index: '.siem-signals-default',
max_signals: 100,
risk_score: 55,
risk_score_mapping: [],
language: 'kuery',
rule_id: 'query-rule-id',
interval: '5m',
exceptions_list: [],
});
describe('transformDataToNdjson', () => {
test('if rules are empty it returns an empty string', () => {
const ruleNdjson = transformDataToNdjson([]);
expect(ruleNdjson).toEqual('');
});
test('single rule will transform with new line ending character for ndjson', () => {
const rule = getRulesSchemaMock();
const ruleNdjson = transformDataToNdjson([rule]);
expect(ruleNdjson.endsWith('\n')).toBe(true);
});
test('multiple rules will transform with two new line ending characters for ndjson', () => {
const result1 = getRulesSchemaMock();
const result2 = getRulesSchemaMock();
result2.id = 'some other id';
result2.rule_id = 'some other id';
result2.name = 'Some other rule';
const ruleNdjson = transformDataToNdjson([result1, result2]);
// this is how we count characters in JavaScript :-)
const count = ruleNdjson.split('\n').length - 1;
expect(count).toBe(2);
});
test('you can parse two rules back out without errors', () => {
const result1 = getRulesSchemaMock();
const result2 = getRulesSchemaMock();
result2.id = 'some other id';
result2.rule_id = 'some other id';
result2.name = 'Some other rule';
const ruleNdjson = transformDataToNdjson([result1, result2]);
const ruleStrings = ruleNdjson.split('\n');
const reParsed1 = JSON.parse(ruleStrings[0]);
const reParsed2 = JSON.parse(ruleStrings[1]);
expect(reParsed1).toEqual(result1);
expect(reParsed2).toEqual(result2);
});
});

16
packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.ts Normal file
View file

@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export const transformDataToNdjson = (data: unknown[]): string => {
if (data.length !== 0) {
const dataString = data.map((item) => JSON.stringify(item)).join('\n');
return `${dataString}\n`;
} else {
return '';
}
};

2
x-pack/plugins/lists/public/exceptions/api.test.ts
View file

@ -759,7 +759,7 @@ describe('Exceptions Lists API', () => {
});
expect(httpMock.fetch).toHaveBeenCalledWith('/api/exception_lists/_export', {
method: 'GET',
method: 'POST',
query: {
id: 'some-id',
list_id: 'list-id',

46
x-pack/plugins/lists/server/routes/export_exception_list_route.ts
View file

@ -6,6 +6,7 @@
*/
import { transformError } from '@kbn/securitysolution-es-utils';
import { transformDataToNdjson } from '@kbn/securitysolution-utils';
import { exportExceptionListQuerySchema } from '@kbn/securitysolution-io-ts-list-types';
import { EXCEPTION_LIST_URL } from '@kbn/securitysolution-list-constants';
@ -14,7 +15,7 @@ import type { ListsPluginRouter } from '../types';
import { buildRouteValidation, buildSiemResponse, getExceptionListClient } from './utils';
export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
router.get(
router.post(
{
options: {
tags: ['access:lists-read'],
@ -26,6 +27,7 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
},
async (context, request, response) => {
const siemResponse = buildSiemResponse(response);
try {
const { id, list_id: listId, namespace_type: namespaceType } = request.query;
const exceptionLists = getExceptionListClient(context);
@ -37,11 +39,10 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
if (exceptionList == null) {
return siemResponse.error({
body: `list_id: ${listId} does not exist`,
body: `exception list with list_id: ${listId} does not exist`,
statusCode: 400,
});
} else {
const { exportData: exportList } = getExport([exceptionList]);
const listItems = await exceptionLists.findExceptionListItem({
filter: undefined,
listId,
@ -51,19 +52,15 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
sortField: 'exception-list.created_at',
sortOrder: 'desc',
});
const exceptionItems = listItems?.data ?? [];
const { exportData: exportListItems, exportDetails } = getExport(listItems?.data ?? []);
const responseBody = [
exportList,
exportListItems,
{ exception_list_items_details: exportDetails },
];
const { exportData } = getExport([exceptionList, ...exceptionItems]);
const { exportDetails } = getExportDetails(exceptionItems);
// TODO: Allow the API to override the name of the file to export
const fileName = exceptionList.list_id;
return response.ok({
body: transformDataToNdjson(responseBody),
body: `${exportData}${exportDetails}`,
headers: {
'Content-Disposition': `attachment; filename="${fileName}"`,
'Content-Type': 'application/ndjson',
@ -81,24 +78,23 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
);
};
const transformDataToNdjson = (data: unknown[]): string => {
if (data.length !== 0) {
const dataString = data.map((dataItem) => JSON.stringify(dataItem)).join('\n');
return `${dataString}\n`;
} else {
return '';
}
};
export const getExport = (
data: unknown[]
): {
exportData: string;
exportDetails: string;
} => {
const ndjson = transformDataToNdjson(data);
const exportDetails = JSON.stringify({
exported_count: data.length,
});
return { exportData: ndjson, exportDetails: `${exportDetails}\n` };
return { exportData: ndjson };
};
export const getExportDetails = (
items: unknown[]
): {
exportDetails: string;
} => {
const exportDetails = JSON.stringify({
exported_list_items_count: items.length,
});
return { exportDetails: `${exportDetails}\n` };
};

2
x-pack/plugins/security_solution/cypress/downloads/test_exception_list.ndjson Normal file
View file

@ -0,0 +1,2 @@
{"_version":"WzQyNjA0LDFd","created_at":"2021-10-14T01:30:22.034Z","created_by":"elastic","description":"Test exception list description","id":"4c65a230-2c8e-11ec-be1c-2bbdec602f88","immutable":false,"list_id":"test_exception_list","name":"Test exception list","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"b04983b4-1617-441c-bb6c-c729281fa2e9","type":"detection","updated_at":"2021-10-14T01:30:22.036Z","updated_by":"elastic","version":1}
{"exported_list_items_count":0}

2
x-pack/plugins/security_solution/cypress/integration/exceptions/exceptions_table.spec.ts
View file

@ -81,7 +81,7 @@ describe('Exceptions Table', () => {
cy.wait('@export').then(({ response }) =>
cy
.wrap(response?.body!)
.wrap(response?.body)
.should('eql', expectedExportedExceptionList(this.exceptionListResponse))
);
});

3
x-pack/plugins/security_solution/cypress/objects/exception.ts
View file

@ -41,6 +41,5 @@ export const expectedExportedExceptionList = (
exceptionListResponse: Cypress.Response<ExceptionListItemSchema>
): string => {
const jsonrule = exceptionListResponse.body;
return `"{\\"_version\\":\\"${jsonrule._version}\\",\\"created_at\\":\\"${jsonrule.created_at}\\",\\"created_by\\":\\"elastic\\",\\"description\\":\\"${jsonrule.description}\\",\\"id\\":\\"${jsonrule.id}\\",\\"immutable\\":false,\\"list_id\\":\\"test_exception_list\\",\\"name\\":\\"Test exception list\\",\\"namespace_type\\":\\"single\\",\\"os_types\\":[],\\"tags\\":[],\\"tie_breaker_id\\":\\"${jsonrule.tie_breaker_id}\\",\\"type\\":\\"detection\\",\\"updated_at\\":\\"${jsonrule.updated_at}\\",\\"updated_by\\":\\"elastic\\",\\"version\\":1}\\n"\n""\n{"exception_list_items_details":"{\\"exported_count\\":0}\\n"}\n`;
return `{"_version":"${jsonrule._version}","created_at":"${jsonrule.created_at}","created_by":"elastic","description":"${jsonrule.description}","id":"${jsonrule.id}","immutable":false,"list_id":"test_exception_list","name":"Test exception list","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"${jsonrule.tie_breaker_id}","type":"detection","updated_at":"${jsonrule.updated_at}","updated_by":"elastic","version":1}\n{"exported_list_items_count":0}\n`;
};

3
x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.ts
View file

@ -5,11 +5,12 @@
* 2.0.
*/
import { transformDataToNdjson } from '@kbn/securitysolution-utils';
import { RulesClient } from '../../../../../alerting/server';
import { getNonPackagedRules } from './get_existing_prepackaged_rules';
import { getExportDetailsNdjson } from './get_export_details_ndjson';
import { transformAlertsToRules } from '../routes/rules/utils';
import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson';
export const getExportAll = async (
rulesClient: RulesClient,

2
x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_by_object_ids.ts
View file

@ -6,13 +6,13 @@
*/
import { chunk } from 'lodash';
import { transformDataToNdjson } from '@kbn/securitysolution-utils';
import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema';
import { RulesClient } from '../../../../../alerting/server';
import { getExportDetailsNdjson } from './get_export_details_ndjson';
import { isAlertType } from '../rules/types';
import { transformAlertToRule } from '../routes/rules/utils';
import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson';
import { INTERNAL_RULE_ID_KEY } from '../../../../common/constants';
import { findRules } from './find_rules';

3
x-pack/plugins/security_solution/server/lib/telemetry/sender.ts
View file

@ -8,10 +8,11 @@
import { cloneDeep } from 'lodash';
import axios from 'axios';
import { URL } from 'url';
import { transformDataToNdjson } from '@kbn/securitysolution-utils';
import { Logger } from 'src/core/server';
import { TelemetryPluginStart, TelemetryPluginSetup } from 'src/plugins/telemetry/server';
import { UsageCounter } from 'src/plugins/usage_collection/server';
import { transformDataToNdjson } from '../../utils/read_stream/create_stream_from_ndjson';
import {
TaskManagerSetupContract,
TaskManagerStartContract,

3
x-pack/plugins/security_solution/server/lib/timeline/routes/timelines/export_timelines/helpers.ts
View file

@ -6,6 +6,7 @@
*/
import { omit } from 'lodash/fp';
import { transformDataToNdjson } from '@kbn/securitysolution-utils';
import {
ExportedTimelines,
@ -15,8 +16,6 @@ import {
import { NoteSavedObject } from '../../../../../../common/types/timeline/note';
import { PinnedEventSavedObject } from '../../../../../../common/types/timeline/pinned_event';
import { transformDataToNdjson } from '../../../../../utils/read_stream/create_stream_from_ndjson';
import { FrameworkRequest } from '../../../../framework';
import * as noteLib from '../../../saved_object/notes';
import * as pinnedEventLib from '../../../saved_object/pinned_events';

71
x-pack/plugins/security_solution/server/utils/read_stream/create_stream_from_ndjson.test.ts
View file

@ -1,71 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { transformDataToNdjson } from './create_stream_from_ndjson';
import { ImportRulesSchemaDecoded } from '../../../common/detection_engine/schemas/request/import_rules_schema';
import { getRulesSchemaMock } from '../../../common/detection_engine/schemas/response/rules_schema.mocks';
export const getOutputSample = (): Partial<ImportRulesSchemaDecoded> => ({
rule_id: 'rule-1',
output_index: '.siem-signals',
risk_score: 50,
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'low',
interval: '5m',
type: 'query',
});
export const getSampleAsNdjson = (sample: Partial<ImportRulesSchemaDecoded>): string => {
return `${JSON.stringify(sample)}\n`;
};
describe('create_rules_stream_from_ndjson', () => {
describe('transformDataToNdjson', () => {
test('if rules are empty it returns an empty string', () => {
const ruleNdjson = transformDataToNdjson([]);
expect(ruleNdjson).toEqual('');
});
test('single rule will transform with new line ending character for ndjson', () => {
const rule = getRulesSchemaMock();
const ruleNdjson = transformDataToNdjson([rule]);
expect(ruleNdjson.endsWith('\n')).toBe(true);
});
test('multiple rules will transform with two new line ending characters for ndjson', () => {
const result1 = getRulesSchemaMock();
const result2 = getRulesSchemaMock();
result2.id = 'some other id';
result2.rule_id = 'some other id';
result2.name = 'Some other rule';
const ruleNdjson = transformDataToNdjson([result1, result2]);
// this is how we count characters in JavaScript :-)
const count = ruleNdjson.split('\n').length - 1;
expect(count).toBe(2);
});
test('you can parse two rules back out without errors', () => {
const result1 = getRulesSchemaMock();
const result2 = getRulesSchemaMock();
result2.id = 'some other id';
result2.rule_id = 'some other id';
result2.name = 'Some other rule';
const ruleNdjson = transformDataToNdjson([result1, result2]);
const ruleStrings = ruleNdjson.split('\n');
const reParsed1 = JSON.parse(ruleStrings[0]);
const reParsed2 = JSON.parse(ruleStrings[1]);
expect(reParsed1).toEqual(result1);
expect(reParsed2).toEqual(result2);
});
});
});

9
x-pack/plugins/security_solution/server/utils/read_stream/create_stream_from_ndjson.ts
View file

@ -48,12 +48,3 @@ export const createLimitStream = (limit: number): Transform => {
},
});
};
export const transformDataToNdjson = (data: unknown[]): string => {
if (data.length !== 0) {
const dataString = data.map((rule) => JSON.stringify(rule)).join('\n');
return `${dataString}\n`;
} else {
return '';
}
};

155
x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts Normal file
View file

@ -0,0 +1,155 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import type { CreateExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import { EXCEPTION_LIST_URL, EXCEPTION_LIST_ITEM_URL } from '@kbn/securitysolution-list-constants';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import {
removeExceptionListServerGeneratedProperties,
removeExceptionListItemServerGeneratedProperties,
binaryToString,
deleteAllExceptions,
} from '../../utils';
import { getCreateExceptionListMinimalSchemaMock } from '../../../../plugins/lists/common/schemas/request/create_exception_list_schema.mock';
import { getCreateExceptionListItemMinimalSchemaMock } from '../../../../plugins/lists/common/schemas/request/create_exception_list_item_schema.mock';
// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext): void => {
const supertest = getService('supertest');
const es = getService('es');
describe('export_exception_list_route', () => {
describe('exporting exception lists', () => {
afterEach(async () => {
await deleteAllExceptions(es);
});
it('should set the response content types to be expected', async () => {
// create an exception list
const { body } = await supertest
.post(EXCEPTION_LIST_URL)
.set('kbn-xsrf', 'true')
.send(getCreateExceptionListMinimalSchemaMock())
.expect(200);
// create an exception list item
await supertest
.post(EXCEPTION_LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(getCreateExceptionListItemMinimalSchemaMock())
.expect(200);
await supertest
.post(
`${EXCEPTION_LIST_URL}/_export?id=${body.id}&list_id=${body.list_id}&namespace_type=single`
)
.set('kbn-xsrf', 'true')
.expect('Content-Disposition', `attachment; filename="${body.list_id}"`)
.expect(200);
});
it('should return 404 if given ids that do not exist', async () => {
// create an exception list
await supertest
.post(EXCEPTION_LIST_URL)
.set('kbn-xsrf', 'true')
.send(getCreateExceptionListMinimalSchemaMock())
.expect(200);
// create an exception list item
await supertest
.post(EXCEPTION_LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(getCreateExceptionListItemMinimalSchemaMock())
.expect(200);
const { body: exportBody } = await supertest
.post(
`${EXCEPTION_LIST_URL}/_export?id=not_exist&list_id=not_exist&namespace_type=single`
)
.set('kbn-xsrf', 'true')
.expect(400);
expect(exportBody).to.eql({
message: 'exception list with list_id: not_exist does not exist',
status_code: 400,
});
});
it('should export a single list with a list id', async () => {
const { body } = await supertest
.post(EXCEPTION_LIST_URL)
.set('kbn-xsrf', 'true')
.send(getCreateExceptionListMinimalSchemaMock())
.expect(200);
const { body: itemBody } = await supertest
.post(EXCEPTION_LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(getCreateExceptionListItemMinimalSchemaMock())
.expect(200);
const { body: exportResult } = await supertest
.post(
`${EXCEPTION_LIST_URL}/_export?id=${body.id}&list_id=${body.list_id}&namespace_type=single`
)
.set('kbn-xsrf', 'true')
.expect(200)
.parse(binaryToString);
const exportedItemsToArray = exportResult.toString().split('\n');
const list = JSON.parse(exportedItemsToArray[0]);
const item = JSON.parse(exportedItemsToArray[1]);
expect(removeExceptionListServerGeneratedProperties(list)).to.eql(
removeExceptionListServerGeneratedProperties(body)
);
expect(removeExceptionListItemServerGeneratedProperties(item)).to.eql(
removeExceptionListItemServerGeneratedProperties(itemBody)
);
});
it('should export two list items with a list id', async () => {
const { body } = await supertest
.post(EXCEPTION_LIST_URL)
.set('kbn-xsrf', 'true')
.send(getCreateExceptionListMinimalSchemaMock())
.expect(200);
await supertest
.post(EXCEPTION_LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(getCreateExceptionListItemMinimalSchemaMock())
.expect(200);
const secondExceptionListItem: CreateExceptionListItemSchema = {
...getCreateExceptionListItemMinimalSchemaMock(),
item_id: 'some-list-item-id-2',
};
await supertest
.post(EXCEPTION_LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(secondExceptionListItem)
.expect(200);
const { body: exportResult } = await supertest
.post(
`${EXCEPTION_LIST_URL}/_export?id=${body.id}&list_id=${body.list_id}&namespace_type=single`
)
.set('kbn-xsrf', 'true')
.expect(200)
.parse(binaryToString);
const bodyString = exportResult.toString();
expect(bodyString.includes('some-list-item-id-2')).to.be(true);
expect(bodyString.includes('some-list-item-id')).to.be(true);
});
});
});
};

1
x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts
View file

@ -24,6 +24,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => {
loadTestFile(require.resolve('./find_list_items'));
loadTestFile(require.resolve('./import_list_items'));
loadTestFile(require.resolve('./export_list_items'));
loadTestFile(require.resolve('./export_exception_list'));
loadTestFile(require.resolve('./create_exception_lists'));
loadTestFile(require.resolve('./create_exception_list_items'));
loadTestFile(require.resolve('./read_exception_lists'));