From 4a148e60265c858b91a286135d1cf8ec3b2d5604 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Wed, 6 Feb 2019 09:42:17 -0700 Subject: [PATCH] Fix date formatting on server for CSV export (#29977) * Fix date formatting on server for CSV export * remove stray console.log * allow async to act in parallel * Log a warning when "Browser" is the timezone --- .../common/field_formats/types/date_server.js | 82 +++++++++++++++++++ .../kibana/server/field_formats/register.js | 4 +- .../export_types/csv/server/execute_job.js | 64 ++++++++++----- .../csv/server/lib/format_csv_values.js | 31 ++++--- .../csv/server/lib/generate_csv.js | 4 +- .../csv/server/lib/hit_iterator.js | 6 +- 6 files changed, 148 insertions(+), 43 deletions(-) create mode 100644 src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js diff --git a/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js b/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js new file mode 100644 index 000000000000..c39d13a07111 --- /dev/null +++ b/src/legacy/core_plugins/kibana/common/field_formats/types/date_server.js @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { memoize } from 'lodash'; +import moment from 'moment-timezone'; + +export function createDateOnServerFormat(FieldFormat) { + return class DateFormat extends FieldFormat { + constructor(params, getConfig) { + super(params); + + this.getConfig = getConfig; + this._memoizedConverter = memoize(val => { + if (val == null) { + return '-'; + } + + /* On the server, importing moment returns a new instance. Unlike on + * the client side, it doesn't have the dateFormat:tz configuration + * baked in. + * We need to set the timezone manually here. The date is taken in as + * UTC and converted into the desired timezone. */ + let date; + if (this._timeZone === 'Browser') { + // Assume a warning has been logged this can be unpredictable. It + // would be too verbose to log anything here. + date = moment.utc(val); + } else { + date = moment.utc(val).tz(this._timeZone); + } + + if (date.isValid()) { + return date.format(this._memoizedPattern); + } else { + return val; + } + }); + } + + getParamDefaults() { + return { + pattern: this.getConfig('dateFormat'), + timezone: this.getConfig('dateFormat:tz'), + }; + } + + _convert(val) { + // don't give away our ref to converter so we can hot-swap when config changes + const pattern = this.param('pattern'); + const timezone = this.param('timezone'); + + const timezoneChanged = this._timeZone !== timezone; + const datePatternChanged = this._memoizedPattern !== pattern; + if (timezoneChanged || datePatternChanged) { + this._timeZone = timezone; + this._memoizedPattern = pattern; + } + + return this._memoizedConverter(val); + } + + static id = 'date'; + static title = 'Date'; + static fieldType = 'date'; + }; +} diff --git a/src/legacy/core_plugins/kibana/server/field_formats/register.js b/src/legacy/core_plugins/kibana/server/field_formats/register.js index f1cf735028f5..acefdbd9bf6d 100644 --- a/src/legacy/core_plugins/kibana/server/field_formats/register.js +++ b/src/legacy/core_plugins/kibana/server/field_formats/register.js @@ -19,7 +19,7 @@ import { createUrlFormat } from '../../common/field_formats/types/url'; import { createBytesFormat } from '../../common/field_formats/types/bytes'; -import { createDateFormat } from '../../common/field_formats/types/date'; +import { createDateOnServerFormat } from '../../common/field_formats/types/date_server'; import { createDurationFormat } from '../../common/field_formats/types/duration'; import { createIpFormat } from '../../common/field_formats/types/ip'; import { createNumberFormat } from '../../common/field_formats/types/number'; @@ -34,7 +34,7 @@ import { createStaticLookupFormat } from '../../common/field_formats/types/stati export function registerFieldFormats(server) { server.registerFieldFormat(createUrlFormat); server.registerFieldFormat(createBytesFormat); - server.registerFieldFormat(createDateFormat); + server.registerFieldFormat(createDateOnServerFormat); server.registerFieldFormat(createDurationFormat); server.registerFieldFormat(createIpFormat); server.registerFieldFormat(createNumberFormat); diff --git a/x-pack/plugins/reporting/export_types/csv/server/execute_job.js b/x-pack/plugins/reporting/export_types/csv/server/execute_job.js index 4ef1658d88f8..d0351311f3eb 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/execute_job.js +++ b/x-pack/plugins/reporting/export_types/csv/server/execute_job.js @@ -15,7 +15,10 @@ function executeJobFn(server) { const { callWithRequest } = server.plugins.elasticsearch.getCluster('data'); const crypto = cryptoFactory(server); const config = server.config(); - const logger = createTaggedLogger(server, ['reporting', 'csv', 'debug']); + const logger = { + debug: createTaggedLogger(server, ['reporting', 'csv', 'debug']), + warn: createTaggedLogger(server, ['reporting', 'csv', 'warning']), + }; const generateCsv = createGenerateCsv(logger); const serverBasePath = config.get('server.basePath'); @@ -27,17 +30,23 @@ function executeJobFn(server) { metaFields, conflictedTypesFields, headers: serializedEncryptedHeaders, - basePath + basePath, } = job; let decryptedHeaders; try { decryptedHeaders = await crypto.decrypt(serializedEncryptedHeaders); } catch (e) { - throw new Error(i18n.translate('xpack.reporting.exportTypes.csv.executeJob.failedToDecryptReportJobDataErrorMessage', { - defaultMessage: 'Failed to decrypt report job data. Please ensure that {encryptionKey} is set and re-generate this report.', - values: { encryptionKey: 'xpack.reporting.encryptionKey' } - })); + throw new Error( + i18n.translate( + 'xpack.reporting.exportTypes.csv.executeJob.failedToDecryptReportJobDataErrorMessage', + { + defaultMessage: + 'Failed to decrypt report job data. Please ensure that {encryptionKey} is set and re-generate this report.', + values: { encryptionKey: 'xpack.reporting.encryptionKey' }, + } + ) + ); } const fakeRequest = { @@ -53,32 +62,47 @@ function executeJobFn(server) { }; const savedObjects = server.savedObjects; const savedObjectsClient = savedObjects.getScopedSavedObjectsClient(fakeRequest); - const uiSettings = server.uiSettingsServiceFactory({ - savedObjectsClient + const uiConfig = server.uiSettingsServiceFactory({ + savedObjectsClient, }); - const fieldFormats = await server.fieldFormatServiceFactory(uiSettings); - const formatsMap = fieldFormatMapFactory(indexPatternSavedObject, fieldFormats); + const [formatsMap, uiSettings] = await Promise.all([ + (async () => { + const fieldFormats = await server.fieldFormatServiceFactory(uiConfig); + return fieldFormatMapFactory(indexPatternSavedObject, fieldFormats); + })(), + (async () => { + const [separator, quoteValues, timezone] = await Promise.all([ + uiConfig.get('csv:separator'), + uiConfig.get('csv:quoteValues'), + uiConfig.get('dateFormat:tz'), + ]); - const separator = await uiSettings.get('csv:separator'); - const quoteValues = await uiSettings.get('csv:quoteValues'); - const maxSizeBytes = config.get('xpack.reporting.csv.maxSizeBytes'); - const scroll = config.get('xpack.reporting.csv.scroll'); + if (timezone === 'Browser') { + logger.warn(`Kibana Advanced Setting "dateFormat:tz" is set to "Browser". Dates will be formatted as UTC to avoid ambiguity.`); + } + + return { + separator, + quoteValues, + timezone, + }; + })(), + ]); const { content, maxSizeReached, size } = await generateCsv({ searchRequest, fields, - formatsMap, metaFields, conflictedTypesFields, callEndpoint, cancellationToken, + formatsMap, settings: { - separator, - quoteValues, - maxSizeBytes, - scroll - } + ...uiSettings, + maxSizeBytes: config.get('xpack.reporting.csv.maxSizeBytes'), + scroll: config.get('xpack.reporting.csv.scroll'), + }, }); return { diff --git a/x-pack/plugins/reporting/export_types/csv/server/lib/format_csv_values.js b/x-pack/plugins/reporting/export_types/csv/server/lib/format_csv_values.js index 4092480d2550..b54659da617d 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/lib/format_csv_values.js +++ b/x-pack/plugins/reporting/export_types/csv/server/lib/format_csv_values.js @@ -8,24 +8,23 @@ import { isObject, isNull, isUndefined } from 'lodash'; export function createFormatCsvValues(escapeValue, separator, fields, formatsMap) { return function formatCsvValues(values) { - return fields.map((field) => { - let value = values[field]; + return fields + .map(field => { + const value = values[field]; + if (isNull(value) || isUndefined(value)) { + return ''; + } - if (isNull(value) || isUndefined(value)) { - return ''; - } + let formattedValue = value; + if (formatsMap.has(field)) { + const formatter = formatsMap.get(field); + formattedValue = formatter.convert(value); + } - if (formatsMap.has(field)) { - const formatter = formatsMap.get(field); - value = formatter.convert(value); - } - - if (isObject(value)) { - return JSON.stringify(value); - } - - return value.toString(); - }) + return formattedValue; + }) + .map(value => (isObject(value) ? JSON.stringify(value) : value)) + .map(value => value.toString()) .map(escapeValue) .join(separator); }; diff --git a/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js b/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js index 2d0133238bf8..56a5ce3d80bb 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js +++ b/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js @@ -49,7 +49,7 @@ export function createGenerateCsv(logger) { } if (!builder.tryAppend(formatCsvValues(flattenHit(hit)) + '\n')) { - logger('max Size Reached'); + logger.warn('max Size Reached'); maxSizeReached = true; cancellationToken.cancel(); break; @@ -59,7 +59,7 @@ export function createGenerateCsv(logger) { await iterator.return(); } const size = builder.getSizeInBytes(); - logger(`finished generating, total size in bytes: ${size}`); + logger.debug(`finished generating, total size in bytes: ${size}`); return { content: builder.getString(), diff --git a/x-pack/plugins/reporting/export_types/csv/server/lib/hit_iterator.js b/x-pack/plugins/reporting/export_types/csv/server/lib/hit_iterator.js index 8c71b7244489..5ad618256872 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/lib/hit_iterator.js +++ b/x-pack/plugins/reporting/export_types/csv/server/lib/hit_iterator.js @@ -30,7 +30,7 @@ async function parseResponse(request) { export function createHitIterator(logger) { return async function* hitIterator(scrollSettings, callEndpoint, searchRequest, cancellationToken) { - logger('executing search request'); + logger.debug('executing search request'); function search(index, body) { return parseResponse(callEndpoint('search', { index, @@ -41,7 +41,7 @@ export function createHitIterator(logger) { } function scroll(scrollId) { - logger('executing scroll request'); + logger.debug('executing scroll request'); return parseResponse(callEndpoint('scroll', { scrollId, scroll: scrollSettings.duration @@ -49,7 +49,7 @@ export function createHitIterator(logger) { } function clearScroll(scrollId) { - logger('executing clearScroll request'); + logger.debug('executing clearScroll request'); return callEndpoint('clearScroll', { scrollId: [ scrollId ] });