maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Reporting] Fix ability to export CSV on searched data with frozen indices (#109976)

Browse source 
* use include frozen setting in csv export

* add api integration test

* add fixes

* Update x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts

* test polish

* update per feedback
This commit is contained in:
Tim Sullivan 2021-08-25 14:42:39 -07:00 committed by GitHub
parent 36eaf7fd58
commit 06c6168d34
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 150 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
x-pack/plugins/reporting/common/constants.ts
View file

@ -45,6 +45,7 @@ export const KBN_SCREENSHOT_HEADER_BLOCK_LIST = [
export const KBN_SCREENSHOT_HEADER_BLOCK_LIST_STARTS_WITH_PATTERN = ['proxy-'];
export const UI_SETTINGS_SEARCH_INCLUDE_FROZEN = 'search:includeFrozen';
export const UI_SETTINGS_CUSTOM_PDF_LOGO = 'xpackReporting:customPdfLogo';
export const UI_SETTINGS_CSV_SEPARATOR = 'csv:separator';
export const UI_SETTINGS_CSV_QUOTE_VALUES = 'csv:quoteValues';

10
x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts
View file

@ -198,7 +198,7 @@ it('calculates the bytes of the content', async () => {
Rx.of({
rawResponse: {
hits: {
hits: range(0, HITS_TOTAL).map((hit, i) => ({
hits: range(0, HITS_TOTAL).map(() => ({
fields: {
message: ['this is a great message'],
},
@ -248,7 +248,7 @@ it('warns if max size was reached', async () => {
Rx.of({
rawResponse: {
hits: {
hits: range(0, HITS_TOTAL).map((hit, i) => ({
hits: range(0, HITS_TOTAL).map(() => ({
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
@ -289,7 +289,7 @@ it('uses the scrollId to page all the data', async () => {
rawResponse: {
_scroll_id: 'awesome-scroll-hero',
hits: {
hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({
hits: range(0, HITS_TOTAL / 10).map(() => ({
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
@ -304,7 +304,7 @@ it('uses the scrollId to page all the data', async () => {
mockEsClient.asCurrentUser.scroll = jest.fn().mockResolvedValue({
body: {
hits: {
hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({
hits: range(0, HITS_TOTAL / 10).map(() => ({
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
@ -337,7 +337,7 @@ it('uses the scrollId to page all the data', async () => {
expect(mockDataClient.search).toHaveBeenCalledTimes(1);
expect(mockDataClient.search).toBeCalledWith(
{ params: { scroll: '30s', size: 500 } },
{ params: { ignore_throttled: true, scroll: '30s', size: 500 } },
{ strategy: 'es' }
);

7
x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts
View file

@ -83,8 +83,9 @@ export class CsvGenerator {
private async scan(
index: IndexPattern,
searchSource: ISearchSource,
scrollSettings: CsvExportSettings['scroll']
settings: CsvExportSettings
) {
const { scroll: scrollSettings, includeFrozen } = settings;
const searchBody = searchSource.getSearchRequestBody();
this.logger.debug(`executing search request`);
const searchParams = {
@ -93,8 +94,10 @@ export class CsvGenerator {
index: index.title,
scroll: scrollSettings.duration,
size: scrollSettings.size,
ignore_throttled: !includeFrozen,
},
};
const results = (
await this.clients.data.search(searchParams, { strategy: ES_SEARCH_STRATEGY }).toPromise()
).rawResponse as estypes.SearchResponse<unknown>;
@ -326,7 +329,7 @@ export class CsvGenerator {
let results: estypes.SearchResponse<unknown> | undefined;
if (scrollId == null) {
// open a scroll cursor in Elasticsearch
results = await this.scan(index, searchSource, scrollSettings);
results = await this.scan(index, searchSource, settings);
scrollId = results?._scroll_id;
if (results.hits?.total != null) {
totalRecords = results.hits.total as number;

4
x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts
View file

@ -9,6 +9,7 @@ import {
UI_SETTINGS_DATEFORMAT_TZ,
UI_SETTINGS_CSV_QUOTE_VALUES,
UI_SETTINGS_CSV_SEPARATOR,
UI_SETTINGS_SEARCH_INCLUDE_FROZEN,
} from '../../../../common/constants';
import { IUiSettingsClient } from 'kibana/server';
import { savedObjectsClientMock, uiSettingsServiceMock } from 'src/core/server/mocks';
@ -36,6 +37,8 @@ describe('getExportSettings', () => {
return ',';
case UI_SETTINGS_DATEFORMAT_TZ:
return 'Browser';
case UI_SETTINGS_SEARCH_INCLUDE_FROZEN:
return false;
}
return 'helo world';
@ -49,6 +52,7 @@ describe('getExportSettings', () => {
"checkForFormulas": undefined,
"escapeFormulaValues": undefined,
"escapeValue": [Function],
"includeFrozen": false,
"maxSizeBytes": undefined,
"scroll": Object {
"duration": undefined,

12
x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts
View file

@ -12,9 +12,10 @@ import { createEscapeValue } from '../../../../../../../src/plugins/data/common'
import { ReportingConfig } from '../../../';
import {
CSV_BOM_CHARS,
UI_SETTINGS_DATEFORMAT_TZ,
UI_SETTINGS_CSV_QUOTE_VALUES,
UI_SETTINGS_CSV_SEPARATOR,
UI_SETTINGS_DATEFORMAT_TZ,
UI_SETTINGS_SEARCH_INCLUDE_FROZEN,
} from '../../../../common/constants';
import { LevelLogger } from '../../../lib';
@ -30,6 +31,7 @@ export interface CsvExportSettings {
checkForFormulas: boolean;
escapeFormulaValues: boolean;
escapeValue: (value: string) => string;
includeFrozen: boolean;
}
export const getExportSettings = async (
@ -38,9 +40,7 @@ export const getExportSettings = async (
timezone: string | undefined,
logger: LevelLogger
): Promise<CsvExportSettings> => {
// Timezone
let setTimezone: string;
// timezone in job params?
if (timezone) {
setTimezone = timezone;
} else {
@ -59,8 +59,9 @@ export const getExportSettings = async (
}
}
// Separator, QuoteValues
const [separator, quoteValues] = await Promise.all([
// Advanced Settings that affect search export + CSV
const [includeFrozen, separator, quoteValues] = await Promise.all([
client.get(UI_SETTINGS_SEARCH_INCLUDE_FROZEN),
client.get(UI_SETTINGS_CSV_SEPARATOR),
client.get(UI_SETTINGS_CSV_QUOTE_VALUES),
]);
@ -76,6 +77,7 @@ export const getExportSettings = async (
duration: config.get('csv', 'scroll', 'duration'),
},
bom,
includeFrozen,
separator,
maxSizeBytes: config.get('csv', 'maxSizeBytes'),
checkForFormulas: config.get('csv', 'checkForFormulas'),

1
x-pack/test/reporting_api_integration/reporting_and_security/index.ts
View file

@ -29,5 +29,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./spaces'));
loadTestFile(require.resolve('./usage'));
loadTestFile(require.resolve('./ilm_migration_apis'));
loadTestFile(require.resolve('./search_frozen_indices'));
});
}

127
x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts Normal file
View file

@ -0,0 +1,127 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../ftr_provider_context';
// eslint-disable-next-line import/no-default-export
export default function ({ getService }: FtrProviderContext) {
const kibanaServer = getService('kibanaServer');
const supertestSvc = getService('supertest');
const esSupertest = getService('esSupertest');
const indexPatternId = 'cool-test-index-pattern';
async function callExportAPI() {
const job = {
browserTimezone: 'UTC',
columns: ['@timestamp', 'ip', 'utilization'],
searchSource: {
fields: [{ field: '*', include_unmapped: 'true' }],
filter: [
{
meta: { field: '@timestamp', index: indexPatternId, params: {} },
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: '2020-08-24T00:00:00.000Z',
lte: '2022-08-24T21:40:48.346Z',
},
},
},
],
index: indexPatternId,
parent: { filter: [], index: indexPatternId, query: { language: 'kuery', query: '' } },
sort: [{ '@timestamp': 'desc' }],
trackTotalHits: true,
},
title: 'Test search',
};
return await supertestSvc
.post(`/api/reporting/v1/generate/immediate/csv_searchsource`)
.set('kbn-xsrf', 'xxx')
.send(job);
}
describe('Frozen indices search', () => {
const reset = async () => {
await kibanaServer.uiSettings.replace({ 'search:includeFrozen': false });
try {
await esSupertest.delete('/test1,test2,test3');
await kibanaServer.savedObjects.delete({ type: 'index-pattern', id: indexPatternId });
} catch (err) {
// ignore 404 error
}
};
before(reset);
after(reset);
it('Search includes frozen indices based on Advanced Setting', async () => {
await kibanaServer.uiSettings.update({ 'csv:quoteValues': true });
// setup: add multiple indices of test data
await Promise.all([
esSupertest
.post('/test1/_doc')
.send({ '@timestamp': '2021-08-24T21:36:40Z', ip: '43.98.8.183', utilization: 18725 }),
esSupertest
.post('/test2/_doc')
.send({ '@timestamp': '2021-08-21T09:36:40Z', ip: '63.91.103.79', utilization: 8480 }),
esSupertest
.post('/test3/_doc')
.send({ '@timestamp': '2021-08-17T21:36:40Z', ip: '139.108.162.171', utilization: 3078 }),
]);
await esSupertest.post('/test*/_refresh');
// setup: create index pattern
const indexPatternCreateResponse = await kibanaServer.savedObjects.create({
type: 'index-pattern',
id: indexPatternId,
overwrite: true,
attributes: { title: 'test*', timeFieldName: '@timestamp' },
});
expect(indexPatternCreateResponse.id).to.be(indexPatternId);
// 1. check the initial data with a CSV export
const initialSearch = await callExportAPI();
expectSnapshot(initialSearch.text).toMatchInline(`
"\\"@timestamp\\",ip,utilization
\\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
\\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
\\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\"
"
`);
// 2. freeze an index in the pattern
await esSupertest.post('/test3/_freeze').expect(200);
await esSupertest.post('/test*/_refresh').expect(200);
// 3. recheck the search results
const afterFreezeSearch = await callExportAPI();
expectSnapshot(afterFreezeSearch.text).toMatchInline(`
"\\"@timestamp\\",ip,utilization
\\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
\\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
"
`);
// 4. update setting to allow searching frozen data
await kibanaServer.uiSettings.update({ 'search:includeFrozen': true });
// 5. recheck the search results
const afterAllowSearch = await callExportAPI();
expectSnapshot(afterAllowSearch.text).toMatchInline(`
"\\"@timestamp\\",ip,utilization
\\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
\\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
\\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\"
"
`);
});
});
}