maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Lens] Time shift metrics (#98781)

Browse source
This commit is contained in:
Joe Reuter 2021-06-02 15:58:47 +02:00 committed by GitHub
parent dfd6ec9243
commit 8cb3dbc4ab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
116 changed files with 3697 additions and 221 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md Normal file
View file

@ -0,0 +1,15 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [AggConfig](./kibana-plugin-plugins-data-public.aggconfig.md) &gt; [getTimeShift](./kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md)
## AggConfig.getTimeShift() method
<b>Signature:</b>
```typescript
getTimeShift(): undefined | moment.Duration;
```
<b>Returns:</b>
`undefined | moment.Duration`

15
docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md Normal file
View file

@ -0,0 +1,15 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [AggConfig](./kibana-plugin-plugins-data-public.aggconfig.md) &gt; [hasTimeShift](./kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md)
## AggConfig.hasTimeShift() method
<b>Signature:</b>
```typescript
hasTimeShift(): boolean;
```
<b>Returns:</b>
`boolean`

2
docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.md
View file

@ -46,8 +46,10 @@ export declare class AggConfig
| [getRequestAggs()](./kibana-plugin-plugins-data-public.aggconfig.getrequestaggs.md) | | |
| [getResponseAggs()](./kibana-plugin-plugins-data-public.aggconfig.getresponseaggs.md) | | |
| [getTimeRange()](./kibana-plugin-plugins-data-public.aggconfig.gettimerange.md) | | |
| [getTimeShift()](./kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md) | | |
| [getValue(bucket)](./kibana-plugin-plugins-data-public.aggconfig.getvalue.md) | | |
| [getValueBucketPath()](./kibana-plugin-plugins-data-public.aggconfig.getvaluebucketpath.md) | | Returns the bucket path containing the main value the agg will produce (e.g. for sum of bytes it will point to the sum, for median it will point to the 50 percentile in the percentile multi value bucket) |
| [hasTimeShift()](./kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md) | | |
| [isFilterable()](./kibana-plugin-plugins-data-public.aggconfig.isfilterable.md) | | |
| [makeLabel(percentageMode)](./kibana-plugin-plugins-data-public.aggconfig.makelabel.md) | | |
| [nextId(list)](./kibana-plugin-plugins-data-public.aggconfig.nextid.md) | <code>static</code> | Calculate the next id based on the ids in this list {<!-- -->array<!-- -->} list - a list of objects with id properties |

11
docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.forcenow.md Normal file
View file

@ -0,0 +1,11 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) &gt; [forceNow](./kibana-plugin-plugins-data-public.aggconfigs.forcenow.md)
## AggConfigs.forceNow property
<b>Signature:</b>
```typescript
forceNow?: Date;
```

72
docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md Normal file
View file

@ -0,0 +1,72 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) &gt; [getSearchSourceTimeFilter](./kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md)
## AggConfigs.getSearchSourceTimeFilter() method
<b>Signature:</b>
```typescript
getSearchSourceTimeFilter(forceNow?: Date): RangeFilter[] | {
meta: {
index: string | undefined;
params: {};
alias: string;
disabled: boolean;
negate: boolean;
};
query: {
bool: {
should: {
bool: {
filter: {
range: {
[x: string]: {
gte: string;
lte: string;
};
};
}[];
};
}[];
minimum_should_match: number;
};
};
}[];
```
## Parameters
| Parameter | Type | Description |
| --- | --- | --- |
| forceNow | <code>Date</code> | |
<b>Returns:</b>
`RangeFilter[] | {
meta: {
index: string | undefined;
params: {};
alias: string;
disabled: boolean;
negate: boolean;
};
query: {
bool: {
should: {
bool: {
filter: {
range: {
[x: string]: {
gte: string;
lte: string;
};
};
}[];
};
}[];
minimum_should_match: number;
};
};
}[]`

15
docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md Normal file
View file

@ -0,0 +1,15 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) &gt; [getTimeShiftInterval](./kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md)
## AggConfigs.getTimeShiftInterval() method
<b>Signature:</b>
```typescript
getTimeShiftInterval(): moment.Duration | undefined;
```
<b>Returns:</b>
`moment.Duration | undefined`

15
docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md Normal file
View file

@ -0,0 +1,15 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) &gt; [getTimeShifts](./kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md)
## AggConfigs.getTimeShifts() method
<b>Signature:</b>
```typescript
getTimeShifts(): Record<string, moment.Duration>;
```
<b>Returns:</b>
`Record<string, moment.Duration>`

15
docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md Normal file
View file

@ -0,0 +1,15 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) &gt; [hasTimeShifts](./kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md)
## AggConfigs.hasTimeShifts() method
<b>Signature:</b>
```typescript
hasTimeShifts(): boolean;
```
<b>Returns:</b>
`boolean`

7
docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md
View file

@ -22,6 +22,7 @@ export declare class AggConfigs
| --- | --- | --- | --- |
| [aggs](./kibana-plugin-plugins-data-public.aggconfigs.aggs.md) | | <code>IAggConfig[]</code> | |
| [createAggConfig](./kibana-plugin-plugins-data-public.aggconfigs.createaggconfig.md) | | <code>&lt;T extends AggConfig = AggConfig&gt;(params: CreateAggConfigParams, { addToAggConfigs }?: {</code><br/><code> addToAggConfigs?: boolean &#124; undefined;</code><br/><code> }) =&gt; T</code> | |
| [forceNow](./kibana-plugin-plugins-data-public.aggconfigs.forcenow.md) | | <code>Date</code> | |
| [hierarchical](./kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md) | | <code>boolean</code> | |
| [indexPattern](./kibana-plugin-plugins-data-public.aggconfigs.indexpattern.md) | | <code>IndexPattern</code> | |
| [timeFields](./kibana-plugin-plugins-data-public.aggconfigs.timefields.md) | | <code>string[]</code> | |
@ -43,8 +44,14 @@ export declare class AggConfigs
| [getRequestAggs()](./kibana-plugin-plugins-data-public.aggconfigs.getrequestaggs.md) | | |
| [getResponseAggById(id)](./kibana-plugin-plugins-data-public.aggconfigs.getresponseaggbyid.md) | | Find a response agg by it's id. This may be an agg in the aggConfigs, or one created specifically for a response value |
| [getResponseAggs()](./kibana-plugin-plugins-data-public.aggconfigs.getresponseaggs.md) | | Gets the AggConfigs (and possibly ResponseAggConfigs) that represent the values that will be produced when all aggs are run.<!-- -->With multi-value metric aggs it is possible for a single agg request to result in multiple agg values, which is why the length of a vis' responseValuesAggs may be different than the vis' aggs {<!-- -->array\[AggConfig\]<!-- -->} |
| [getSearchSourceTimeFilter(forceNow)](./kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md) | | |
| [getTimeShiftInterval()](./kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md) | | |
| [getTimeShifts()](./kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md) | | |
| [hasTimeShifts()](./kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md) | | |
| [jsonDataEquals(aggConfigs)](./kibana-plugin-plugins-data-public.aggconfigs.jsondataequals.md) | | Data-by-data comparison of this Aggregation Ignores the non-array indexes |
| [onSearchRequestStart(searchSource, options)](./kibana-plugin-plugins-data-public.aggconfigs.onsearchrequeststart.md) | | |
| [postFlightTransform(response)](./kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md) | | |
| [setForceNow(now)](./kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md) | | |
| [setTimeFields(timeFields)](./kibana-plugin-plugins-data-public.aggconfigs.settimefields.md) | | |
| [setTimeRange(timeRange)](./kibana-plugin-plugins-data-public.aggconfigs.settimerange.md) | | |
| [toDsl()](./kibana-plugin-plugins-data-public.aggconfigs.todsl.md) | | |

22
docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md Normal file
View file

@ -0,0 +1,22 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) &gt; [postFlightTransform](./kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md)
## AggConfigs.postFlightTransform() method
<b>Signature:</b>
```typescript
postFlightTransform(response: IEsSearchResponse<any>): IEsSearchResponse<any>;
```
## Parameters
| Parameter | Type | Description |
| --- | --- | --- |
| response | <code>IEsSearchResponse&lt;any&gt;</code> | |
<b>Returns:</b>
`IEsSearchResponse<any>`

22
docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md Normal file
View file

@ -0,0 +1,22 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) &gt; [setForceNow](./kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md)
## AggConfigs.setForceNow() method
<b>Signature:</b>
```typescript
setForceNow(now: Date | undefined): void;
```
## Parameters
| Parameter | Type | Description |
| --- | --- | --- |
| now | <code>Date &#124; undefined</code> | |
<b>Returns:</b>
`void`

27
src/plugins/data/common/search/aggs/agg_config.ts
View file

@ -6,6 +6,7 @@
* Side Public License, v 1.
*/
import moment from 'moment';
import _ from 'lodash';
import { i18n } from '@kbn/i18n';
import { Assign, Ensure } from '@kbn/utility-types';
@ -20,6 +21,7 @@ import {
import { IAggType } from './agg_type';
import { writeParams } from './agg_params';
import { IAggConfigs } from './agg_configs';
import { parseTimeShift } from './utils';
type State = string | number | boolean | null | undefined | SerializableState;
@ -172,6 +174,31 @@ export class AggConfig {
return _.get(this.params, key);
}
hasTimeShift(): boolean {
return Boolean(this.getParam('timeShift'));
}
getTimeShift(): undefined | moment.Duration {
const rawTimeShift = this.getParam('timeShift');
if (!rawTimeShift) return undefined;
const parsedTimeShift = parseTimeShift(rawTimeShift);
if (parsedTimeShift === 'invalid') {
throw new Error(`could not parse time shift ${rawTimeShift}`);
}
if (parsedTimeShift === 'previous') {
const timeShiftInterval = this.aggConfigs.getTimeShiftInterval();
if (timeShiftInterval) {
return timeShiftInterval;
} else if (!this.aggConfigs.timeRange) {
return;
}
return moment.duration(
moment(this.aggConfigs.timeRange.to).diff(this.aggConfigs.timeRange.from)
);
}
return parsedTimeShift;
}
write(aggs?: IAggConfigs) {
return writeParams<AggConfig>(this.type.params, this, aggs);
}

346
src/plugins/data/common/search/aggs/agg_configs.test.ts
View file

@ -14,6 +14,7 @@ import { mockAggTypesRegistry } from './test_helpers';
import type { IndexPatternField } from '../../index_patterns';
import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern';
import { stubIndexPattern, stubIndexPatternWithFields } from '../../../common/stubs';
import { IEsSearchResponse } from '..';
describe('AggConfigs', () => {
let indexPattern: IndexPattern;
@ -332,6 +333,109 @@ describe('AggConfigs', () => {
});
});
it('inserts a time split filters agg if there are multiple time shifts', () => {
const configStates = [
{
enabled: true,
type: 'terms',
schema: 'segment',
params: { field: 'clientip', size: 10 },
},
{ enabled: true, type: 'avg', schema: 'metric', params: { field: 'bytes' } },
{
enabled: true,
type: 'sum',
schema: 'metric',
params: { field: 'bytes', timeShift: '1d' },
},
];
indexPattern.fields.push({
name: 'timestamp',
type: 'date',
esTypes: ['date'],
aggregatable: true,
filterable: true,
searchable: true,
} as IndexPatternField);
const ac = new AggConfigs(indexPattern, configStates, { typesRegistry });
ac.timeFields = ['timestamp'];
ac.timeRange = {
from: '2021-05-05T00:00:00.000Z',
to: '2021-05-10T00:00:00.000Z',
};
const dsl = ac.toDsl();
const terms = ac.byName('terms')[0];
const avg = ac.byName('avg')[0];
const sum = ac.byName('sum')[0];
expect(dsl[terms.id].aggs.time_offset_split.filters.filters).toMatchInlineSnapshot(`
Object {
"0": Object {
"range": Object {
"timestamp": Object {
"gte": "2021-05-05T00:00:00.000Z",
"lte": "2021-05-10T00:00:00.000Z",
},
},
},
"86400000": Object {
"range": Object {
"timestamp": Object {
"gte": "2021-05-04T00:00:00.000Z",
"lte": "2021-05-09T00:00:00.000Z",
},
},
},
}
`);
expect(dsl[terms.id].aggs.time_offset_split.aggs).toHaveProperty(avg.id);
expect(dsl[terms.id].aggs.time_offset_split.aggs).toHaveProperty(sum.id);
});
it('does not insert a time split if there is a single time shift', () => {
const configStates = [
{
enabled: true,
type: 'terms',
schema: 'segment',
params: { field: 'clientip', size: 10 },
},
{
enabled: true,
type: 'avg',
schema: 'metric',
params: {
field: 'bytes',
timeShift: '1d',
},
},
{
enabled: true,
type: 'sum',
schema: 'metric',
params: { field: 'bytes', timeShift: '1d' },
},
];
const ac = new AggConfigs(indexPattern, configStates, { typesRegistry });
ac.timeFields = ['timestamp'];
ac.timeRange = {
from: '2021-05-05T00:00:00.000Z',
to: '2021-05-10T00:00:00.000Z',
};
const dsl = ac.toDsl();
const terms = ac.byName('terms')[0];
const avg = ac.byName('avg')[0];
const sum = ac.byName('sum')[0];
expect(dsl[terms.id].aggs).not.toHaveProperty('time_offset_split');
expect(dsl[terms.id].aggs).toHaveProperty(avg.id);
expect(dsl[terms.id].aggs).toHaveProperty(sum.id);
});
it('writes multiple metric aggregations at every level if the vis is hierarchical', () => {
const configStates = [
{ enabled: true, type: 'terms', schema: 'segment', params: { field: 'bytes', orderBy: 1 } },
@ -426,4 +530,246 @@ describe('AggConfigs', () => {
);
});
});
describe('#postFlightTransform', () => {
it('merges together splitted responses for multiple shifts', () => {
indexPattern = stubIndexPattern as IndexPattern;
indexPattern.fields.getByName = (name) => (({ name } as unknown) as IndexPatternField);
const configStates = [
{
enabled: true,
type: 'terms',
schema: 'segment',
params: { field: 'clientip', size: 10 },
},
{
enabled: true,
type: 'date_histogram',
schema: 'segment',
params: { field: '@timestamp', interval: '1d' },
},
{
enabled: true,
type: 'avg',
schema: 'metric',
params: {
field: 'bytes',
timeShift: '1d',
},
},
{
enabled: true,
type: 'sum',
schema: 'metric',
params: { field: 'bytes' },
},
];
const ac = new AggConfigs(indexPattern, configStates, { typesRegistry });
ac.timeFields = ['@timestamp'];
ac.timeRange = {
from: '2021-05-05T00:00:00.000Z',
to: '2021-05-10T00:00:00.000Z',
};
// 1 terms bucket (A), with 2 date buckets (7th and 8th of May)
// the bucket keys of the shifted time range will be shifted forward
const response = {
rawResponse: {
aggregations: {
'1': {
buckets: [
{
key: 'A',
time_offset_split: {
buckets: {
'0': {
2: {
buckets: [
{
// 2021-05-07
key: 1620345600000,
3: {
value: 1.1,
},
4: {
value: 2.2,
},
},
{
// 2021-05-08
key: 1620432000000,
doc_count: 26,
3: {
value: 3.3,
},
4: {
value: 4.4,
},
},
],
},
},
'86400000': {
2: {
buckets: [
{
// 2021-05-07
key: 1620345600000,
doc_count: 13,
3: {
value: 5.5,
},
4: {
value: 6.6,
},
},
{
// 2021-05-08
key: 1620432000000,
3: {
value: 7.7,
},
4: {
value: 8.8,
},
},
],
},
},
},
},
},
],
},
},
},
};
const mergedResponse = ac.postFlightTransform(
(response as unknown) as IEsSearchResponse<any>
);
expect(mergedResponse.rawResponse).toEqual({
aggregations: {
'1': {
buckets: [
{
'2': {
buckets: [
{
'4': {
value: 2.2,
},
// 2021-05-07
key: 1620345600000,
},
{
'3': {
value: 5.5,
},
'4': {
value: 4.4,
},
doc_count: 26,
doc_count_86400000: 13,
// 2021-05-08
key: 1620432000000,
},
{
'3': {
value: 7.7,
},
// 2021-05-09
key: 1620518400000,
},
],
},
key: 'A',
},
],
},
},
});
});
it('shifts date histogram keys and renames doc_count properties for single shift', () => {
indexPattern = stubIndexPattern as IndexPattern;
indexPattern.fields.getByName = (name) => (({ name } as unknown) as IndexPatternField);
const configStates = [
{
enabled: true,
type: 'date_histogram',
schema: 'segment',
params: { field: '@timestamp', interval: '1d' },
},
{
enabled: true,
type: 'avg',
schema: 'metric',
params: {
field: 'bytes',
timeShift: '1d',
},
},
];
const ac = new AggConfigs(indexPattern, configStates, { typesRegistry });
ac.timeFields = ['@timestamp'];
ac.timeRange = {
from: '2021-05-05T00:00:00.000Z',
to: '2021-05-10T00:00:00.000Z',
};
const response = {
rawResponse: {
aggregations: {
'1': {
buckets: [
{
// 2021-05-07
key: 1620345600000,
doc_count: 26,
2: {
value: 1.1,
},
},
{
// 2021-05-08
key: 1620432000000,
doc_count: 27,
2: {
value: 2.2,
},
},
],
},
},
},
};
const mergedResponse = ac.postFlightTransform(
(response as unknown) as IEsSearchResponse<any>
);
expect(mergedResponse.rawResponse).toEqual({
aggregations: {
'1': {
buckets: [
{
'2': {
value: 1.1,
},
doc_count_86400000: 26,
// 2021-05-08
key: 1620432000000,
},
{
'2': {
value: 2.2,
},
doc_count_86400000: 27,
// 2021-05-09
key: 1620518400000,
},
],
},
},
});
});
});
});

223
src/plugins/data/common/search/aggs/agg_configs.ts
View file

@ -6,17 +6,26 @@
* Side Public License, v 1.
*/
import _ from 'lodash';
import moment from 'moment';
import _, { cloneDeep } from 'lodash';
import { i18n } from '@kbn/i18n';
import { Assign } from '@kbn/utility-types';
import { Aggregate, Bucket } from '@elastic/elasticsearch/api/types';
import { ISearchOptions, ISearchSource } from 'src/plugins/data/public';
import {
IEsSearchResponse,
ISearchOptions,
ISearchSource,
RangeFilter,
} from 'src/plugins/data/public';
import { AggConfig, AggConfigSerialized, IAggConfig } from './agg_config';
import { IAggType } from './agg_type';
import { AggTypesRegistryStart } from './agg_types_registry';
import { AggGroupNames } from './agg_groups';
import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern';
import { TimeRange } from '../../../common';
import { TimeRange, getTime, isRangeFilter } from '../../../common';
import { IBucketAggConfig } from './buckets';
import { insertTimeShiftSplit, mergeTimeShifts } from './utils/time_splits';
function removeParentAggs(obj: any) {
for (const prop in obj) {
@ -48,6 +57,8 @@ export interface AggConfigsOptions {
export type CreateAggConfigParams = Assign<AggConfigSerialized, { type: string | IAggType }>;
export type GenericBucket = Bucket & { [property: string]: Aggregate };
/**
* @name AggConfigs
*
@ -66,6 +77,7 @@ export class AggConfigs {
public indexPattern: IndexPattern;
public timeRange?: TimeRange;
public timeFields?: string[];
public forceNow?: Date;
public hierarchical?: boolean = false;
private readonly typesRegistry: AggTypesRegistryStart;
@ -92,6 +104,10 @@ export class AggConfigs {
this.timeFields = timeFields;
}
setForceNow(now: Date | undefined) {
this.forceNow = now;
}
setTimeRange(timeRange: TimeRange) {
this.timeRange = timeRange;
@ -183,7 +199,13 @@ export class AggConfigs {
let dslLvlCursor: Record<string, any>;
let nestedMetrics: Array<{ config: AggConfig; dsl: Record<string, any> }> | [];
const timeShifts = this.getTimeShifts();
const hasMultipleTimeShifts = Object.keys(timeShifts).length > 1;
if (this.hierarchical) {
if (hasMultipleTimeShifts) {
throw new Error('Multiple time shifts not supported for hierarchical metrics');
}
// collect all metrics, and filter out the ones that we won't be copying
nestedMetrics = this.aggs
.filter(function (agg) {
@ -196,52 +218,67 @@ export class AggConfigs {
};
});
}
this.getRequestAggs()
.filter((config: AggConfig) => !config.type.hasNoDsl)
.forEach((config: AggConfig, i: number, list) => {
if (!dslLvlCursor) {
// start at the top level
dslLvlCursor = dslTopLvl;
} else {
const prevConfig: AggConfig = list[i - 1];
const prevDsl = dslLvlCursor[prevConfig.id];
const requestAggs = this.getRequestAggs();
const aggsWithDsl = requestAggs.filter((agg) => !agg.type.hasNoDsl).length;
const timeSplitIndex = this.getAll().findIndex(
(config) => 'splitForTimeShift' in config.type && config.type.splitForTimeShift(config, this)
);
// advance the cursor and nest under the previous agg, or
// put it on the same level if the previous agg doesn't accept
// sub aggs
dslLvlCursor = prevDsl?.aggs || dslLvlCursor;
}
requestAggs.forEach((config: AggConfig, i: number, list) => {
if (!dslLvlCursor) {
// start at the top level
dslLvlCursor = dslTopLvl;
} else {
const prevConfig: AggConfig = list[i - 1];
const prevDsl = dslLvlCursor[prevConfig.id];
const dsl = config.type.hasNoDslParams
? config.toDsl(this)
: (dslLvlCursor[config.id] = config.toDsl(this));
let subAggs: any;
// advance the cursor and nest under the previous agg, or
// put it on the same level if the previous agg doesn't accept
// sub aggs
dslLvlCursor = prevDsl?.aggs || dslLvlCursor;
}
parseParentAggs(dslLvlCursor, dsl);
if (hasMultipleTimeShifts) {
dslLvlCursor = insertTimeShiftSplit(this, config, timeShifts, dslLvlCursor);
}
if (config.type.type === AggGroupNames.Buckets && i < list.length - 1) {
// buckets that are not the last item in the list accept sub-aggs
subAggs = dsl.aggs || (dsl.aggs = {});
}
if (config.type.hasNoDsl) {
return;
}
if (subAggs) {
_.each(subAggs, (agg) => {
parseParentAggs(subAggs, agg);
});
}
if (subAggs && nestedMetrics) {
nestedMetrics.forEach((agg: any) => {
subAggs[agg.config.id] = agg.dsl;
// if a nested metric agg has parent aggs, we have to add them to every level of the tree
// to make sure "bucket_path" references in the nested metric agg itself are still working
if (agg.dsl.parentAggs) {
Object.entries(agg.dsl.parentAggs).forEach(([parentAggId, parentAgg]) => {
subAggs[parentAggId] = parentAgg;
});
}
});
}
});
const dsl = config.type.hasNoDslParams
? config.toDsl(this)
: (dslLvlCursor[config.id] = config.toDsl(this));
let subAggs: any;
parseParentAggs(dslLvlCursor, dsl);
if (
config.type.type === AggGroupNames.Buckets &&
(i < aggsWithDsl - 1 || timeSplitIndex > i)
) {
// buckets that are not the last item in the list of dsl producing aggs or have a time split coming up accept sub-aggs
subAggs = dsl.aggs || (dsl.aggs = {});
}
if (subAggs) {
_.each(subAggs, (agg) => {
parseParentAggs(subAggs, agg);
});
}
if (subAggs && nestedMetrics) {
nestedMetrics.forEach((agg: any) => {
subAggs[agg.config.id] = agg.dsl;
// if a nested metric agg has parent aggs, we have to add them to every level of the tree
// to make sure "bucket_path" references in the nested metric agg itself are still working
if (agg.dsl.parentAggs) {
Object.entries(agg.dsl.parentAggs).forEach(([parentAggId, parentAgg]) => {
subAggs[parentAggId] = parentAgg;
});
}
});
}
});
removeParentAggs(dslTopLvl);
return dslTopLvl;
@ -289,6 +326,104 @@ export class AggConfigs {
);
}
getTimeShifts(): Record<string, moment.Duration> {
const timeShifts: Record<string, moment.Duration> = {};
this.getAll()
.filter((agg) => agg.schema === 'metric')
.map((agg) => agg.getTimeShift())
.forEach((timeShift) => {
if (timeShift) {
timeShifts[String(timeShift.asMilliseconds())] = timeShift;
} else {
timeShifts[0] = moment.duration(0);
}
});
return timeShifts;
}
getTimeShiftInterval(): moment.Duration | undefined {
const splitAgg = (this.getAll().filter(
(agg) => agg.type.type === AggGroupNames.Buckets
) as IBucketAggConfig[]).find((agg) => agg.type.splitForTimeShift(agg, this));
return splitAgg?.type.getTimeShiftInterval(splitAgg);
}
hasTimeShifts(): boolean {
return this.getAll().some((agg) => agg.hasTimeShift());
}
getSearchSourceTimeFilter(forceNow?: Date) {
if (!this.timeFields || !this.timeRange) {
return [];
}
const timeRange = this.timeRange;
const timeFields = this.timeFields;
const timeShifts = this.getTimeShifts();
if (!this.hasTimeShifts()) {
return this.timeFields
.map((fieldName) => getTime(this.indexPattern, timeRange, { fieldName, forceNow }))
.filter(isRangeFilter);
}
return [
{
meta: {
index: this.indexPattern?.id,
params: {},
alias: '',
disabled: false,
negate: false,
},
query: {
bool: {
should: Object.entries(timeShifts).map(([, shift]) => {
return {
bool: {
filter: timeFields
.map(
(fieldName) =>
[
getTime(this.indexPattern, timeRange, { fieldName, forceNow }),
fieldName,
] as [RangeFilter | undefined, string]
)
.filter(([filter]) => isRangeFilter(filter))
.map(([filter, field]) => ({
range: {
[field]: {
gte: moment(filter?.range[field].gte).subtract(shift).toISOString(),
lte: moment(filter?.range[field].lte).subtract(shift).toISOString(),
},
},
})),
},
};
}),
minimum_should_match: 1,
},
},
},
];
}
postFlightTransform(response: IEsSearchResponse<any>) {
if (!this.hasTimeShifts()) {
return response;
}
const transformedRawResponse = cloneDeep(response.rawResponse);
if (!transformedRawResponse.aggregations) {
transformedRawResponse.aggregations = {
doc_count: response.rawResponse.hits?.total as Aggregate,
};
}
const aggCursor = transformedRawResponse.aggregations!;
mergeTimeShifts(this, aggCursor);
return {
...response,
rawResponse: transformedRawResponse,
};
}
getRequestAggById(id: string) {
return this.aggs.find((agg: AggConfig) => agg.id === id);
}

4
src/plugins/data/common/search/aggs/agg_type.ts
View file

@ -215,6 +215,10 @@ export class AggType<
return agg.id;
};
splitForTimeShift(agg: TAggConfig, aggs: IAggConfigs) {
return false;
}
/**
* Generic AggType Constructor
*

2
src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts
View file

@ -166,7 +166,7 @@ export const buildOtherBucketAgg = (
key: string
) => {
// make sure there are actually results for the buckets
if (aggregations[aggId].buckets.length < 1) {
if (aggregations[aggId]?.buckets.length < 1) {
noAggBucketResults = true;
return;
}

43
src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts
View file

@ -6,8 +6,9 @@
* Side Public License, v 1.
*/
import moment from 'moment';
import { IAggConfig } from '../agg_config';
import { KBN_FIELD_TYPES } from '../../../../common';
import { GenericBucket, IAggConfigs, KBN_FIELD_TYPES } from '../../../../common';
import { AggType, AggTypeConfig } from '../agg_type';
import { AggParamType } from '../param_types/agg';
@ -26,6 +27,14 @@ const bucketType = 'buckets';
interface BucketAggTypeConfig<TBucketAggConfig extends IAggConfig>
extends AggTypeConfig<TBucketAggConfig, BucketAggParam<TBucketAggConfig>> {
getKey?: (bucket: any, key: any, agg: IAggConfig) => any;
getShiftedKey?: (
agg: TBucketAggConfig,
key: string | number,
timeShift: moment.Duration
) => string | number;
orderBuckets?(agg: TBucketAggConfig, a: GenericBucket, b: GenericBucket): number;
splitForTimeShift?(agg: TBucketAggConfig, aggs: IAggConfigs): boolean;
getTimeShiftInterval?(agg: TBucketAggConfig): undefined | moment.Duration;
}
export class BucketAggType<TBucketAggConfig extends IAggConfig = IBucketAggConfig> extends AggType<
@ -35,6 +44,22 @@ export class BucketAggType<TBucketAggConfig extends IAggConfig = IBucketAggConfi
getKey: (bucket: any, key: any, agg: TBucketAggConfig) => any;
type = bucketType;
getShiftedKey(
agg: TBucketAggConfig,
key: string | number,
timeShift: moment.Duration
): string | number {
return key;
}
getTimeShiftInterval(agg: TBucketAggConfig): undefined | moment.Duration {
return undefined;
}
orderBuckets(agg: TBucketAggConfig, a: GenericBucket, b: GenericBucket): number {
return Number(a.key) - Number(b.key);
}
constructor(config: BucketAggTypeConfig<TBucketAggConfig>) {
super(config);
@ -43,6 +68,22 @@ export class BucketAggType<TBucketAggConfig extends IAggConfig = IBucketAggConfi
((bucket, key) => {
return key || bucket.key;
});
if (config.getShiftedKey) {
this.getShiftedKey = config.getShiftedKey;
}
if (config.orderBuckets) {
this.orderBuckets = config.orderBuckets;
}
if (config.getTimeShiftInterval) {
this.getTimeShiftInterval = config.getTimeShiftInterval;
}
if (config.splitForTimeShift) {
this.splitForTimeShift = config.splitForTimeShift;
}
}
}

11
src/plugins/data/common/search/aggs/buckets/date_histogram.ts
View file

@ -135,6 +135,17 @@ export const getDateHistogramBucketAgg = ({
},
};
},
getShiftedKey(agg, key, timeShift) {
return moment(key).add(timeShift).valueOf();
},
splitForTimeShift(agg, aggs) {
return aggs.hasTimeShifts() && Boolean(aggs.timeFields?.includes(agg.fieldName()));
},
getTimeShiftInterval(agg) {
const { useNormalizedEsInterval } = agg.params;
const interval = agg.buckets.getInterval(useNormalizedEsInterval);
return interval;
},
params: [
{
name: 'field',

49
src/plugins/data/common/search/aggs/buckets/terms.ts
View file

@ -9,6 +9,7 @@
import { noop } from 'lodash';
import { i18n } from '@kbn/i18n';
import moment from 'moment';
import { BucketAggType, IBucketAggConfig } from './bucket_agg_type';
import { BUCKET_TYPES } from './bucket_agg_types';
import { createFilterTerms } from './create_filter/terms';
@ -179,6 +180,54 @@ export const getTermsBucketAgg = () =>
return;
}
if (
aggs?.hasTimeShifts() &&
Object.keys(aggs?.getTimeShifts()).length > 1 &&
aggs.timeRange
) {
const shift = orderAgg.getTimeShift();
orderAgg = aggs.createAggConfig(
{
type: 'filtered_metric',
id: orderAgg.id,
params: {
customBucket: aggs
.createAggConfig(
{
type: 'filter',
id: 'shift',
params: {
filter: {
language: 'lucene',
query: {
range: {
[aggs.timeFields![0]]: {
gte: moment(aggs.timeRange.from)
.subtract(shift || 0)
.toISOString(),
lte: moment(aggs.timeRange.to)
.subtract(shift || 0)
.toISOString(),
},
},
},
},
},
},
{
addToAggConfigs: false,
}
)
.serialize(),
customMetric: orderAgg.serialize(),
},
enabled: false,
},
{
addToAggConfigs: false,
}
);
}
if (orderAgg.type.name === 'count') {
order._count = dir;
return;

1
src/plugins/data/common/search/aggs/metrics/avg_fn.test.ts
View file

@ -27,6 +27,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"field": "machine.os.keyword",
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "avg",

7
src/plugins/data/common/search/aggs/metrics/avg_fn.ts
View file

@ -62,6 +62,13 @@ export const aggAvg = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

3
src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.test.ts
View file

@ -29,6 +29,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"customMetric": undefined,
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "avg_bucket",
@ -42,11 +43,13 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"customMetric": undefined,
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "avg_bucket",
},
"json": undefined,
"timeShift": undefined,
}
`);
});

7
src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts
View file

@ -77,6 +77,13 @@ export const aggBucketAvg = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

3
src/plugins/data/common/search/aggs/metrics/bucket_max_fn.test.ts
View file

@ -29,6 +29,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"customMetric": undefined,
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "max_bucket",
@ -42,11 +43,13 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"customMetric": undefined,
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "max_bucket",
},
"json": undefined,
"timeShift": undefined,
}
`);
});

7
src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts
View file

@ -77,6 +77,13 @@ export const aggBucketMax = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

3
src/plugins/data/common/search/aggs/metrics/bucket_min_fn.test.ts
View file

@ -29,6 +29,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"customMetric": undefined,
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "min_bucket",
@ -42,11 +43,13 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"customMetric": undefined,
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "min_bucket",
},
"json": undefined,
"timeShift": undefined,
}
`);
});

7
src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts
View file

@ -77,6 +77,13 @@ export const aggBucketMin = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

3
src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.test.ts
View file

@ -29,6 +29,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"customMetric": undefined,
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "sum_bucket",
@ -42,11 +43,13 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"customMetric": undefined,
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "sum_bucket",
},
"json": undefined,
"timeShift": undefined,
}
`);
});

7
src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts
View file

@ -77,6 +77,13 @@ export const aggBucketSum = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

1
src/plugins/data/common/search/aggs/metrics/cardinality_fn.test.ts
View file

@ -27,6 +27,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"field": "machine.os.keyword",
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "cardinality",

7
src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts
View file

@ -67,6 +67,13 @@ export const aggCardinality = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

7
src/plugins/data/common/search/aggs/metrics/count.ts
View file

@ -31,7 +31,12 @@ export const getCountMetricAgg = () =>
};
},
getValue(agg, bucket) {
return bucket.doc_count;
const timeShift = agg.getTimeShift();
if (!timeShift) {
return bucket.doc_count;
} else {
return bucket[`doc_count_${timeShift.asMilliseconds()}`];
}
},
isScalable() {
return true;

1
src/plugins/data/common/search/aggs/metrics/count_fn.test.ts
View file

@ -23,6 +23,7 @@ describe('agg_expression_functions', () => {
"id": undefined,
"params": Object {
"customLabel": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "count",

7
src/plugins/data/common/search/aggs/metrics/count_fn.ts
View file

@ -54,6 +54,13 @@ export const aggCount = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

4
src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.test.ts
View file

@ -29,6 +29,7 @@ describe('agg_expression_functions', () => {
"customMetric": undefined,
"json": undefined,
"metricAgg": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "cumulative_sum",
@ -54,6 +55,7 @@ describe('agg_expression_functions', () => {
"customMetric": undefined,
"json": undefined,
"metricAgg": "sum",
"timeShift": undefined,
},
"schema": undefined,
"type": "cumulative_sum",
@ -81,12 +83,14 @@ describe('agg_expression_functions', () => {
"customMetric": undefined,
"json": undefined,
"metricAgg": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "cumulative_sum",
},
"json": undefined,
"metricAgg": undefined,
"timeShift": undefined,
}
`);
});

7
src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts
View file

@ -81,6 +81,13 @@ export const aggCumulativeSum = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

4
src/plugins/data/common/search/aggs/metrics/derivative_fn.test.ts
View file

@ -29,6 +29,7 @@ describe('agg_expression_functions', () => {
"customMetric": undefined,
"json": undefined,
"metricAgg": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "derivative",
@ -54,6 +55,7 @@ describe('agg_expression_functions', () => {
"customMetric": undefined,
"json": undefined,
"metricAgg": "sum",
"timeShift": undefined,
},
"schema": undefined,
"type": "derivative",
@ -81,12 +83,14 @@ describe('agg_expression_functions', () => {
"customMetric": undefined,
"json": undefined,
"metricAgg": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "derivative",
},
"json": undefined,
"metricAgg": undefined,
"timeShift": undefined,
}
`);
});

7
src/plugins/data/common/search/aggs/metrics/derivative_fn.ts
View file

@ -81,6 +81,13 @@ export const aggDerivative = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

2
src/plugins/data/common/search/aggs/metrics/filtered_metric.ts
View file

@ -42,7 +42,7 @@ export const getFilteredMetricAgg = () => {
getValue(agg, bucket) {
const customMetric = agg.getParam('customMetric');
const customBucket = agg.getParam('customBucket');
return customMetric.getValue(bucket[customBucket.id]);
return bucket && bucket[customBucket.id] && customMetric.getValue(bucket[customBucket.id]);
},
getValueBucketPath(agg) {
const customBucket = agg.getParam('customBucket');

3
src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.test.ts
View file

@ -28,6 +28,7 @@ describe('agg_expression_functions', () => {
"customBucket": undefined,
"customLabel": undefined,
"customMetric": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "filtered_metric",
@ -40,10 +41,12 @@ describe('agg_expression_functions', () => {
"customBucket": undefined,
"customLabel": undefined,
"customMetric": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "filtered_metric",
},
"timeShift": undefined,
}
`);
});

7
src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts
View file

@ -72,6 +72,13 @@ export const aggFilteredMetric = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

1
src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.test.ts
View file

@ -27,6 +27,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"field": "machine.os.keyword",
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "geo_bounds",

7
src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts
View file

@ -67,6 +67,13 @@ export const aggGeoBounds = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

1
src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.test.ts
View file

@ -27,6 +27,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"field": "machine.os.keyword",
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "geo_centroid",

7
src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts
View file

@ -67,6 +67,13 @@ export const aggGeoCentroid = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

1
src/plugins/data/common/search/aggs/metrics/max_fn.test.ts
View file

@ -27,6 +27,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"field": "machine.os.keyword",
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "max",

7
src/plugins/data/common/search/aggs/metrics/max_fn.ts
View file

@ -62,6 +62,13 @@ export const aggMax = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

2
src/plugins/data/common/search/aggs/metrics/median.ts
View file

@ -46,7 +46,7 @@ export const getMedianMetricAgg = () => {
{ name: 'percents', default: [50], shouldShow: () => false, serialize: () => undefined },
],
getValue(agg, bucket) {
return bucket[agg.id].values['50.0'];
return bucket[agg.id]?.values['50.0'];
},
});
};

1
src/plugins/data/common/search/aggs/metrics/median_fn.test.ts
View file

@ -27,6 +27,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"field": "machine.os.keyword",
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "median",

7
src/plugins/data/common/search/aggs/metrics/median_fn.ts
View file

@ -67,6 +67,13 @@ export const aggMedian = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

19
src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts
View file

@ -11,7 +11,8 @@ import { AggType, AggTypeConfig } from '../agg_type';
import { AggParamType } from '../param_types/agg';
import { AggConfig } from '../agg_config';
import { METRIC_TYPES } from './metric_agg_types';
import { FieldTypes } from '../param_types';
import { BaseParamType, FieldTypes } from '../param_types';
import { AggGroupNames } from '../agg_groups';
export interface IMetricAggConfig extends AggConfig {
type: InstanceType<typeof MetricAggType>;
@ -47,6 +48,14 @@ export class MetricAggType<TMetricAggConfig extends AggConfig = IMetricAggConfig
constructor(config: MetricAggTypeConfig<TMetricAggConfig>) {
super(config);
this.params.push(
new BaseParamType({
name: 'timeShift',
type: 'string',
write: () => {},
}) as MetricAggParam<TMetricAggConfig>
);
this.getValue =
config.getValue ||
((agg, bucket) => {
@ -69,6 +78,14 @@ export class MetricAggType<TMetricAggConfig extends AggConfig = IMetricAggConfig
});
this.isScalable = config.isScalable || (() => false);
// split at this point if there are time shifts and this is the first metric
this.splitForTimeShift = (agg, aggs) =>
aggs.hasTimeShifts() &&
aggs.byType(AggGroupNames.Metrics)[0] === agg &&
!aggs
.byType(AggGroupNames.Buckets)
.some((bucketAgg) => bucketAgg.type.splitForTimeShift(bucketAgg, aggs));
}
}

1
src/plugins/data/common/search/aggs/metrics/min_fn.test.ts
View file

@ -27,6 +27,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"field": "machine.os.keyword",
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "min",

7
src/plugins/data/common/search/aggs/metrics/min_fn.ts
View file

@ -62,6 +62,13 @@ export const aggMin = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

4
src/plugins/data/common/search/aggs/metrics/moving_avg_fn.test.ts
View file

@ -30,6 +30,7 @@ describe('agg_expression_functions', () => {
"json": undefined,
"metricAgg": undefined,
"script": undefined,
"timeShift": undefined,
"window": undefined,
},
"schema": undefined,
@ -59,6 +60,7 @@ describe('agg_expression_functions', () => {
"json": undefined,
"metricAgg": "sum",
"script": "test",
"timeShift": undefined,
"window": 10,
},
"schema": undefined,
@ -88,6 +90,7 @@ describe('agg_expression_functions', () => {
"json": undefined,
"metricAgg": undefined,
"script": undefined,
"timeShift": undefined,
"window": undefined,
},
"schema": undefined,
@ -96,6 +99,7 @@ describe('agg_expression_functions', () => {
"json": undefined,
"metricAgg": undefined,
"script": undefined,
"timeShift": undefined,
"window": undefined,
}
`);

7
src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts
View file

@ -94,6 +94,13 @@ export const aggMovingAvg = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

2
src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.test.ts
View file

@ -27,6 +27,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"field": "machine.os.keyword",
"json": undefined,
"timeShift": undefined,
"values": undefined,
},
"schema": undefined,
@ -51,6 +52,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"field": "machine.os.keyword",
"json": undefined,
"timeShift": undefined,
"values": Array [
1,
2,

7
src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts
View file

@ -74,6 +74,13 @@ export const aggPercentileRanks = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

2
src/plugins/data/common/search/aggs/metrics/percentiles_fn.test.ts
View file

@ -28,6 +28,7 @@ describe('agg_expression_functions', () => {
"field": "machine.os.keyword",
"json": undefined,
"percents": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "percentiles",
@ -56,6 +57,7 @@ describe('agg_expression_functions', () => {
2,
3,
],
"timeShift": undefined,
},
"schema": undefined,
"type": "percentiles",

7
src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts
View file

@ -74,6 +74,13 @@ export const aggPercentiles = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

4
src/plugins/data/common/search/aggs/metrics/serial_diff_fn.test.ts
View file

@ -29,6 +29,7 @@ describe('agg_expression_functions', () => {
"customMetric": undefined,
"json": undefined,
"metricAgg": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "serial_diff",
@ -54,6 +55,7 @@ describe('agg_expression_functions', () => {
"customMetric": undefined,
"json": undefined,
"metricAgg": "sum",
"timeShift": undefined,
},
"schema": undefined,
"type": "serial_diff",
@ -81,12 +83,14 @@ describe('agg_expression_functions', () => {
"customMetric": undefined,
"json": undefined,
"metricAgg": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "serial_diff",
},
"json": undefined,
"metricAgg": undefined,
"timeShift": undefined,
}
`);
});

7
src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts
View file

@ -81,6 +81,13 @@ export const aggSerialDiff = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

7
src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts
View file

@ -74,6 +74,13 @@ export const aggSinglePercentile = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

1
src/plugins/data/common/search/aggs/metrics/std_deviation_fn.test.ts
View file

@ -27,6 +27,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"field": "machine.os.keyword",
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "std_dev",

7
src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts
View file

@ -67,6 +67,13 @@ export const aggStdDeviation = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

1
src/plugins/data/common/search/aggs/metrics/sum_fn.test.ts
View file

@ -27,6 +27,7 @@ describe('agg_expression_functions', () => {
"customLabel": undefined,
"field": "machine.os.keyword",
"json": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "sum",

7
src/plugins/data/common/search/aggs/metrics/sum_fn.ts
View file

@ -62,6 +62,13 @@ export const aggSum = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

2
src/plugins/data/common/search/aggs/metrics/top_hit_fn.test.ts
View file

@ -32,6 +32,7 @@ describe('agg_expression_functions', () => {
"size": undefined,
"sortField": undefined,
"sortOrder": undefined,
"timeShift": undefined,
},
"schema": undefined,
"type": "top_hits",
@ -64,6 +65,7 @@ describe('agg_expression_functions', () => {
"size": 6,
"sortField": "_score",
"sortOrder": "asc",
"timeShift": undefined,
},
"schema": "whatever",
"type": "top_hits",

7
src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts
View file

@ -94,6 +94,13 @@ export const aggTopHit = (): FunctionDefinition => ({
defaultMessage: 'Represents a custom label for this aggregation',
}),
},
timeShift: {
types: ['string'],
help: i18n.translate('data.search.aggs.metrics.timeShift.help', {
defaultMessage:
'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.',
}),
},
},
fn: (input, args) => {
const { id, enabled, schema, ...rest } = args;

1
src/plugins/data/common/search/aggs/types.ts
View file

@ -132,6 +132,7 @@ export type AggsStart = Assign<AggsCommonStart, { types: AggTypesRegistryStart }
export interface BaseAggParams {
json?: string;
customLabel?: string;
timeShift?: string;
}
/** @internal */

1
src/plugins/data/common/search/aggs/utils/index.ts
View file

@ -15,3 +15,4 @@ export * from './ip_address';
export * from './prop_filter';
export * from './to_angular_json';
export * from './infer_time_zone';
export * from './parse_time_shift';

29
src/plugins/data/common/search/aggs/utils/parse_time_shift.ts Normal file
View file

@ -0,0 +1,29 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import moment from 'moment';
const allowedUnits = ['s', 'm', 'h', 'd', 'w', 'M', 'y'] as const;
type AllowedUnit = typeof allowedUnits[number];
/**
* This method parses a string into a time shift duration.
* If parsing fails, 'invalid' is returned.
* Allowed values are the string 'previous' and an integer followed by the units s,m,h,d,w,M,y
* */
export const parseTimeShift = (val: string): moment.Duration | 'previous' | 'invalid' => {
const trimmedVal = val.trim();
if (trimmedVal === 'previous') {
return 'previous';
}
const [, amount, unit] = trimmedVal.match(/^(\d+)(\w)$/) || [];
const parsedAmount = Number(amount);
if (Number.isNaN(parsedAmount) || !allowedUnits.includes(unit as AllowedUnit)) {
return 'invalid';
}
return moment.duration(Number(amount), unit as AllowedUnit);
};

447
src/plugins/data/common/search/aggs/utils/time_splits.ts Normal file
View file

@ -0,0 +1,447 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import moment from 'moment';
import _, { isArray } from 'lodash';
import {
Aggregate,
FiltersAggregate,
FiltersBucketItem,
MultiBucketAggregate,
} from '@elastic/elasticsearch/api/types';
import { AggGroupNames } from '../agg_groups';
import { GenericBucket, AggConfigs, getTime, AggConfig } from '../../../../common';
import { IBucketAggConfig } from '../buckets';
/**
* This function will transform an ES response containg a time split (using a filters aggregation before the metrics or date histogram aggregation),
* merging together all branches for the different time ranges into a single response structure which can be tabified into a single table.
*
* If there is just a single time shift, there are no separate branches per time range - in this case only the date histogram keys are shifted by the
* configured amount of time.
*
* To do this, the following steps are taken:
* * Traverse the response tree, tracking the current agg config
* * Once the node which would contain the time split object is found, merge all separate time range buckets into a single layer of buckets of the parent agg
* * Recursively repeat this process for all nested sub-buckets
*
* Example input:
* ```
* "aggregations" : {
"product" : {
"buckets" : [
{
"key" : "Product A",
"doc_count" : 512,
"first_year" : {
"doc_count" : 418,
"overall_revenue" : {
"value" : 2163634.0
}
},
"time_offset_split" : {
"buckets" : {
"-1y" : {
"doc_count" : 420,
"year" : {
"buckets" : [
{
"key_as_string" : "2018",
"doc_count" : 81,
"revenue" : {
"value" : 505124.0
}
},
{
"key_as_string" : "2019",
"doc_count" : 65,
"revenue" : {
"value" : 363058.0
}
}
]
}
},
"regular" : {
"doc_count" : 418,
"year" : {
"buckets" : [
{
"key_as_string" : "2019",
"doc_count" : 65,
"revenue" : {
"value" : 363058.0
}
},
{
"key_as_string" : "2020",
"doc_count" : 84,
"revenue" : {
"value" : 392924.0
}
}
]
}
}
}
}
},
{
"key" : "Product B",
"doc_count" : 248,
"first_year" : {
"doc_count" : 215,
"overall_revenue" : {
"value" : 1315547.0
}
},
"time_offset_split" : {
"buckets" : {
"-1y" : {
"doc_count" : 211,
"year" : {
"buckets" : [
{
"key_as_string" : "2018",
"key" : 1618963200000,
"doc_count" : 28,
"revenue" : {
"value" : 156543.0
}
},
// ...
* ```
*
* Example output:
* ```
* "aggregations" : {
"product" : {
"buckets" : [
{
"key" : "Product A",
"doc_count" : 512,
"first_year" : {
"doc_count" : 418,
"overall_revenue" : {
"value" : 2163634.0
}
},
"year" : {
"buckets" : [
{
"key_as_string" : "2019",
"doc_count" : 81,
"revenue_regular" : {
"value" : 505124.0
},
"revenue_-1y" : {
"value" : 302736.0
}
},
{
"key_as_string" : "2020",
"doc_count" : 78,
"revenue_regular" : {
"value" : 392924.0
},
"revenue_-1y" : {
"value" : 363058.0
},
}
// ...
* ```
*
*
* @param aggConfigs The agg configs instance
* @param aggCursor The root aggregations object from the response which will be mutated in place
*/
export function mergeTimeShifts(aggConfigs: AggConfigs, aggCursor: Record<string, Aggregate>) {
const timeShifts = aggConfigs.getTimeShifts();
const hasMultipleTimeShifts = Object.keys(timeShifts).length > 1;
const requestAggs = aggConfigs.getRequestAggs();
const bucketAggs = aggConfigs.aggs.filter(
(agg) => agg.type.type === AggGroupNames.Buckets
) as IBucketAggConfig[];
const mergeAggLevel = (
target: GenericBucket,
source: GenericBucket,
shift: moment.Duration,
aggIndex: number
) => {
Object.entries(source).forEach(([key, val]) => {
// copy over doc count into special key
if (typeof val === 'number' && key === 'doc_count') {
if (shift.asMilliseconds() === 0) {
target.doc_count = val;
} else {
target[`doc_count_${shift.asMilliseconds()}`] = val;
}
} else if (typeof val !== 'object') {
// other meta keys not of interest
return;
} else {
// a sub-agg
const agg = requestAggs.find((requestAgg) => key.indexOf(requestAgg.id) === 0);
if (agg && agg.type.type === AggGroupNames.Metrics) {
const timeShift = agg.getTimeShift();
if (
(timeShift && timeShift.asMilliseconds() === shift.asMilliseconds()) ||
(shift.asMilliseconds() === 0 && !timeShift)
) {
// this is a metric from the current time shift, copy it over
target[key] = source[key];
}
} else if (agg && agg === bucketAggs[aggIndex]) {
const bucketAgg = agg as IBucketAggConfig;
// expected next bucket sub agg
const subAggregate = val as Aggregate;
const buckets = ('buckets' in subAggregate ? subAggregate.buckets : undefined) as
| GenericBucket[]
| Record<string, GenericBucket>
| undefined;
if (!target[key]) {
// sub aggregate only exists in shifted branch, not in base branch - create dummy aggregate
// which will be filled with shifted data
target[key] = {
buckets: isArray(buckets) ? [] : {},
};
}
const baseSubAggregate = target[key] as Aggregate;
// only supported bucket formats in agg configs are array of buckets and record of buckets for filters
const baseBuckets = ('buckets' in baseSubAggregate
? baseSubAggregate.buckets
: undefined) as GenericBucket[] | Record<string, GenericBucket> | undefined;
// merge
if (isArray(buckets) && isArray(baseBuckets)) {
const baseBucketMap: Record<string, GenericBucket> = {};
baseBuckets.forEach((bucket) => {
baseBucketMap[String(bucket.key)] = bucket;
});
buckets.forEach((bucket) => {
const bucketKey = bucketAgg.type.getShiftedKey(bucketAgg, bucket.key, shift);
// if a bucket is missing in the map, create an empty one
if (!baseBucketMap[bucketKey]) {
baseBucketMap[String(bucketKey)] = {
key: bucketKey,
} as GenericBucket;
}
mergeAggLevel(baseBucketMap[bucketKey], bucket, shift, aggIndex + 1);
});
(baseSubAggregate as MultiBucketAggregate).buckets = Object.values(
baseBucketMap
).sort((a, b) => bucketAgg.type.orderBuckets(bucketAgg, a, b));
} else if (baseBuckets && buckets && !isArray(baseBuckets)) {
Object.entries(buckets).forEach(([bucketKey, bucket]) => {
// if a bucket is missing in the base response, create an empty one
if (!baseBuckets[bucketKey]) {
baseBuckets[bucketKey] = {} as GenericBucket;
}
mergeAggLevel(baseBuckets[bucketKey], bucket, shift, aggIndex + 1);
});
}
}
}
});
};
const transformTimeShift = (cursor: Record<string, Aggregate>, aggIndex: number): undefined => {
const shouldSplit = aggConfigs.aggs[aggIndex].type.splitForTimeShift(
aggConfigs.aggs[aggIndex],
aggConfigs
);
if (shouldSplit) {
// multiple time shifts caused a filters agg in the tree we have to merge
if (hasMultipleTimeShifts && cursor.time_offset_split) {
const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate).buckets as Record<
string,
FiltersBucketItem
>;
const subTree = {};
Object.entries(timeShifts).forEach(([key, shift]) => {
mergeAggLevel(
subTree as GenericBucket,
timeShiftedBuckets[key] as GenericBucket,
shift,
aggIndex
);
});
delete cursor.time_offset_split;
Object.assign(cursor, subTree);
} else {
// otherwise we have to "merge" a single level to shift all keys
const [[, shift]] = Object.entries(timeShifts);
const subTree = {};
mergeAggLevel(subTree, cursor, shift, aggIndex);
Object.assign(cursor, subTree);
}
return;
}
// recurse deeper into the response object
Object.keys(cursor).forEach((subAggId) => {
const subAgg = cursor[subAggId];
if (typeof subAgg !== 'object' || !('buckets' in subAgg)) {
return;
}
if (isArray(subAgg.buckets)) {
subAgg.buckets.forEach((bucket) => transformTimeShift(bucket, aggIndex + 1));
} else {
Object.values(subAgg.buckets).forEach((bucket) => transformTimeShift(bucket, aggIndex + 1));
}
});
};
transformTimeShift(aggCursor, 0);
}
/**
* Inserts a filters aggregation into the aggregation tree which splits buckets to fetch data for all time ranges
* configured in metric aggregations.
*
* The current agg config can implement `splitForTimeShift` to force insertion of the time split filters aggregation
* before the dsl of the agg config (date histogram and metrics aggregations do this)
*
* Example aggregation tree without time split:
* ```
* "aggs": {
"product": {
"terms": {
"field": "product",
"size": 3,
"order": { "overall_revenue": "desc" }
},
"aggs": {
"overall_revenue": {
"sum": {
"field": "sales"
}
},
"year": {
"date_histogram": {
"field": "timestamp",
"interval": "year"
},
"aggs": {
"revenue": {
"sum": {
"field": "sales"
}
}
}
// ...
* ```
*
* Same aggregation tree with inserted time split:
* ```
* "aggs": {
"product": {
"terms": {
"field": "product",
"size": 3,
"order": { "first_year>overall_revenue": "desc" }
},
"aggs": {
"first_year": {
"filter": {
"range": {
"timestamp": {
"gte": "2019",
"lte": "2020"
}
}
},
"aggs": {
"overall_revenue": {
"sum": {
"field": "sales"
}
}
}
},
"time_offset_split": {
"filters": {
"filters": {
"regular": {
"range": {
"timestamp": {
"gte": "2019",
"lte": "2020"
}
}
},
"-1y": {
"range": {
"timestamp": {
"gte": "2018",
"lte": "2019"
}
}
}
}
},
"aggs": {
"year": {
"date_histogram": {
"field": "timestamp",
"interval": "year"
},
"aggs": {
"revenue": {
"sum": {
"field": "sales"
}
}
}
}
}
}
}
* ```
*/
export function insertTimeShiftSplit(
aggConfigs: AggConfigs,
config: AggConfig,
timeShifts: Record<string, moment.Duration>,
dslLvlCursor: Record<string, any>
) {
if ('splitForTimeShift' in config.type && !config.type.splitForTimeShift(config, aggConfigs)) {
return dslLvlCursor;
}
if (!aggConfigs.timeFields || aggConfigs.timeFields.length < 1) {
throw new Error('Time shift can only be used with configured time field');
}
if (!aggConfigs.timeRange) {
throw new Error('Time shift can only be used with configured time range');
}
const timeRange = aggConfigs.timeRange;
const filters: Record<string, unknown> = {};
const timeField = aggConfigs.timeFields[0];
Object.entries(timeShifts).forEach(([key, shift]) => {
const timeFilter = getTime(aggConfigs.indexPattern, timeRange, {
fieldName: timeField,
forceNow: aggConfigs.forceNow,
});
if (timeFilter) {
filters[key] = {
range: {
[timeField]: {
gte: moment(timeFilter.range[timeField].gte).subtract(shift).toISOString(),
lte: moment(timeFilter.range[timeField].lte).subtract(shift).toISOString(),
},
},
};
}
});
dslLvlCursor.time_offset_split = {
filters: {
filters,
},
aggs: {},
};
return dslLvlCursor.time_offset_split.aggs;
}

1
src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts
View file

@ -42,6 +42,7 @@ describe('esaggs expression function - public', () => {
toDsl: jest.fn().mockReturnValue({ aggs: {} }),
onSearchRequestStart: jest.fn(),
setTimeFields: jest.fn(),
setForceNow: jest.fn(),
} as unknown) as jest.Mocked<IAggConfigs>,
filters: undefined,
indexPattern: ({ id: 'logstash-*' } as unknown) as jest.Mocked<IndexPattern>,

29
src/plugins/data/common/search/expressions/esaggs/request_handler.ts
View file

@ -9,15 +9,7 @@
import { i18n } from '@kbn/i18n';
import { Adapters } from 'src/plugins/inspector/common';
import {
calculateBounds,
Filter,
getTime,
IndexPattern,
isRangeFilter,
Query,
TimeRange,
} from '../../../../common';
import { calculateBounds, Filter, IndexPattern, Query, TimeRange } from '../../../../common';
import { IAggConfigs } from '../../aggs';
import { ISearchStartSearchSource } from '../../search_source';
@ -70,8 +62,15 @@ export const handleRequest = async ({
const timeFilterSearchSource = searchSource.createChild({ callParentStartHandlers: true });
const requestSearchSource = timeFilterSearchSource.createChild({ callParentStartHandlers: true });
// If timeFields have been specified, use the specified ones, otherwise use primary time field of index
// pattern if it's available.
const defaultTimeField = indexPattern?.getTimeField?.();
const defaultTimeFields = defaultTimeField ? [defaultTimeField.name] : [];
const allTimeFields = timeFields && timeFields.length > 0 ? timeFields : defaultTimeFields;
aggs.setTimeRange(timeRange as TimeRange);
aggs.setTimeFields(timeFields);
aggs.setForceNow(forceNow);
aggs.setTimeFields(allTimeFields);
// For now we need to mirror the history of the passed search source, since
// the request inspector wouldn't work otherwise.
@ -90,19 +89,11 @@ export const handleRequest = async ({
return aggs.onSearchRequestStart(paramSearchSource, options);
});
// If timeFields have been specified, use the specified ones, otherwise use primary time field of index
// pattern if it's available.
const defaultTimeField = indexPattern?.getTimeField?.();
const defaultTimeFields = defaultTimeField ? [defaultTimeField.name] : [];
const allTimeFields = timeFields && timeFields.length > 0 ? timeFields : defaultTimeFields;
// If a timeRange has been specified and we had at least one timeField available, create range
// filters for that those time fields
if (timeRange && allTimeFields.length > 0) {
timeFilterSearchSource.setField('filter', () => {
return allTimeFields
.map((fieldName) => getTime(indexPattern, timeRange, { fieldName, forceNow }))
.filter(isRangeFilter);
return aggs.getSearchSourceTimeFilter(forceNow);
});
}

33
src/plugins/data/common/search/search_source/search_source.ts
View file

@ -75,7 +75,13 @@ import { estypes } from '@elastic/elasticsearch';
import { normalizeSortRequest } from './normalize_sort_request';
import { fieldWildcardFilter } from '../../../../kibana_utils/common';
import { IIndexPattern, IndexPattern, IndexPatternField } from '../../index_patterns';
import { AggConfigs, ES_SEARCH_STRATEGY, ISearchGeneric, ISearchOptions } from '../..';
import {
AggConfigs,
ES_SEARCH_STRATEGY,
IEsSearchResponse,
ISearchGeneric,
ISearchOptions,
} from '../..';
import type {
ISearchSource,
SearchFieldValue,
@ -414,6 +420,15 @@ export class SearchSource {
}
}
private postFlightTransform(response: IEsSearchResponse<any>) {
const aggs = this.getField('aggs');
if (aggs instanceof AggConfigs) {
return aggs.postFlightTransform(response);
} else {
return response;
}
}
private async fetchOthers(response: estypes.SearchResponse<any>, options: ISearchOptions) {
const aggs = this.getField('aggs');
if (aggs instanceof AggConfigs) {
@ -451,24 +466,26 @@ export class SearchSource {
if (isErrorResponse(response)) {
obs.error(response);
} else if (isPartialResponse(response)) {
obs.next(response);
obs.next(this.postFlightTransform(response));
} else {
if (!this.hasPostFlightRequests()) {
obs.next(response);
obs.next(this.postFlightTransform(response));
obs.complete();
} else {
// Treat the complete response as partial, then run the postFlightRequests.
obs.next({
...response,
...this.postFlightTransform(response),
isPartial: true,
isRunning: true,
});
const sub = from(this.fetchOthers(response.rawResponse, options)).subscribe({
next: (responseWithOther) => {
obs.next({
...response,
rawResponse: responseWithOther,
});
obs.next(
this.postFlightTransform({
...response,
rawResponse: responseWithOther!,
})
);
},
error: (e) => {
obs.error(e);

2
src/plugins/data/common/search/tabify/tabify.ts
View file

@ -139,7 +139,7 @@ export function tabifyAggResponse(
const write = new TabbedAggResponseWriter(aggConfigs, respOpts || {});
const topLevelBucket: AggResponseBucket = {
...esResponse.aggregations,
doc_count: esResponse.hits?.total,
doc_count: esResponse.aggregations?.doc_count || esResponse.hits?.total,
};
collectBucket(aggConfigs, write, topLevelBucket, '', 1);

47
src/plugins/data/public/public.api.md
View file

@ -7,11 +7,13 @@
import { $Values } from '@kbn/utility-types';
import { Action } from 'history';
import { Adapters as Adapters_2 } from 'src/plugins/inspector/common';
import { Aggregate } from '@elastic/elasticsearch/api/types';
import { ApiResponse } from '@elastic/elasticsearch/lib/Transport';
import { ApplicationStart } from 'kibana/public';
import { Assign } from '@kbn/utility-types';
import { BfetchPublicSetup } from 'src/plugins/bfetch/public';
import Boom from '@hapi/boom';
import { Bucket } from '@elastic/elasticsearch/api/types';
import { ConfigDeprecationProvider } from '@kbn/config';
import { CoreSetup } from 'src/core/public';
import { CoreSetup as CoreSetup_2 } from 'kibana/public';
@ -46,6 +48,7 @@ import { Href } from 'history';
import { HttpSetup } from 'kibana/public';
import { IAggConfigs as IAggConfigs_2 } from 'src/plugins/data/public';
import { IconType } from '@elastic/eui';
import { IEsSearchResponse as IEsSearchResponse_2 } from 'src/plugins/data/public';
import { IncomingHttpHeaders } from 'http';
import { InjectedIntl } from '@kbn/i18n/react';
import { ISearchOptions as ISearchOptions_2 } from 'src/plugins/data/public';
@ -74,6 +77,7 @@ import * as PropTypes from 'prop-types';
import { PublicContract } from '@kbn/utility-types';
import { PublicMethodsOf } from '@kbn/utility-types';
import { PublicUiSettingsParams } from 'src/core/server/types';
import { RangeFilter as RangeFilter_2 } from 'src/plugins/data/public';
import React from 'react';
import * as React_3 from 'react';
import { RecursiveReadonly } from '@kbn/utility-types';
@ -152,9 +156,13 @@ export class AggConfig {
// (undocumented)
getTimeRange(): import("../../../public").TimeRange | undefined;
// (undocumented)
getTimeShift(): undefined | moment.Duration;
// (undocumented)
getValue(bucket: any): any;
getValueBucketPath(): string;
// (undocumented)
hasTimeShift(): boolean;
// (undocumented)
id: string;
// (undocumented)
isFilterable(): boolean;
@ -245,6 +253,8 @@ export class AggConfigs {
addToAggConfigs?: boolean | undefined;
}) => T;
// (undocumented)
forceNow?: Date;
// (undocumented)
getAll(): AggConfig[];
// (undocumented)
getRequestAggById(id: string): AggConfig | undefined;
@ -253,6 +263,39 @@ export class AggConfigs {
getResponseAggById(id: string): AggConfig | undefined;
getResponseAggs(): AggConfig[];
// (undocumented)
getSearchSourceTimeFilter(forceNow?: Date): RangeFilter_2[] | {
meta: {
index: string | undefined;
params: {};
alias: string;
disabled: boolean;
negate: boolean;
};
query: {
bool: {
should: {
bool: {
filter: {
range: {
[x: string]: {
gte: string;
lte: string;
};
};
}[];
};
}[];
minimum_should_match: number;
};
};
}[];
// (undocumented)
getTimeShiftInterval(): moment.Duration | undefined;
// (undocumented)
getTimeShifts(): Record<string, moment.Duration>;
// (undocumented)
hasTimeShifts(): boolean;
// (undocumented)
hierarchical?: boolean;
// (undocumented)
indexPattern: IndexPattern;
@ -260,6 +303,10 @@ export class AggConfigs {
// (undocumented)
onSearchRequestStart(searchSource: ISearchSource_2, options?: ISearchOptions_2): Promise<[unknown, unknown, unknown, unknown, unknown, unknown, unknown, unknown, unknown, unknown]>;
// (undocumented)
postFlightTransform(response: IEsSearchResponse_2<any>): IEsSearchResponse_2<any>;
// (undocumented)
setForceNow(now: Date | undefined): void;
// (undocumented)
setTimeFields(timeFields: string[] | undefined): void;
// (undocumented)
setTimeRange(timeRange: TimeRange): void;

4
src/plugins/data/server/server.api.md
View file

@ -6,8 +6,10 @@
import { $Values } from '@kbn/utility-types';
import { Adapters } from 'src/plugins/inspector/common';
import { Aggregate } from '@elastic/elasticsearch/api/types';
import { Assign } from '@kbn/utility-types';
import { BfetchServerSetup } from 'src/plugins/bfetch/server';
import { Bucket } from '@elastic/elasticsearch/api/types';
import { ConfigDeprecationProvider } from '@kbn/config';
import { CoreSetup } from 'src/core/server';
import { CoreSetup as CoreSetup_2 } from 'kibana/server';
@ -32,6 +34,7 @@ import { ExpressionsServerSetup } from 'src/plugins/expressions/server';
import { ExpressionValueBoxed } from 'src/plugins/expressions/common';
import { FormatFactory as FormatFactory_2 } from 'src/plugins/data/common/field_formats/utils';
import { IAggConfigs as IAggConfigs_2 } from 'src/plugins/data/public';
import { IEsSearchResponse as IEsSearchResponse_2 } from 'src/plugins/data/public';
import { IScopedClusterClient } from 'src/core/server';
import { ISearchOptions as ISearchOptions_2 } from 'src/plugins/data/public';
import { ISearchSource } from 'src/plugins/data/public';
@ -52,6 +55,7 @@ import { Plugin as Plugin_2 } from 'src/core/server';
import { Plugin as Plugin_3 } from 'kibana/server';
import { PluginInitializerContext as PluginInitializerContext_2 } from 'src/core/server';
import { PublicMethodsOf } from '@kbn/utility-types';
import { RangeFilter } from 'src/plugins/data/public';
import { RecursiveReadonly } from '@kbn/utility-types';
import { RequestAdapter } from 'src/plugins/inspector/common';
import { RequestHandlerContext } from 'src/core/server';

3
src/plugins/vis_default_editor/public/components/agg_params_helper.ts
View file

@ -72,6 +72,9 @@ function getAggParamsToRender({
if (hideCustomLabel && param.name === 'customLabel') {
return;
}
if (param.name === 'timeShift') {
return;
}
// if field param exists, compute allowed fields
if (param.type === 'field') {
let availableFields: IndexPatternField[] = (param as IFieldParamType).getAvailableFields(agg);

371
test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts Normal file
View file

@ -0,0 +1,371 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import expect from '@kbn/expect';
import { Datatable } from 'src/plugins/expressions';
import { ExpectExpression, expectExpressionProvider } from './helpers';
import { FtrProviderContext } from '../../../functional/ftr_provider_context';
function getCell(esaggsResult: any, row: number, column: number): unknown | undefined {
const columnId = esaggsResult?.columns[column]?.id;
if (!columnId) {
return;
}
return esaggsResult?.rows[row]?.[columnId];
}
function checkShift(rows: Datatable['rows'], columns: Datatable['columns'], metricIndex = 1) {
rows.shift();
rows.pop();
rows.forEach((_, index) => {
if (index < rows.length - 1) {
expect(getCell({ rows, columns }, index, metricIndex + 1)).to.be(
getCell({ rows, columns }, index + 1, metricIndex)
);
}
});
}
export default function ({
getService,
updateBaselines,
}: FtrProviderContext & { updateBaselines: boolean }) {
let expectExpression: ExpectExpression;
describe('esaggs timeshift tests', () => {
before(() => {
expectExpression = expectExpressionProvider({ getService, updateBaselines });
});
const timeRange = {
from: '2015-09-21T00:00:00Z',
to: '2015-09-22T00:00:00Z',
};
it('shifts single metric', async () => {
const expression = `
kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
| esaggs index={indexPatternLoad id='logstash-*'}
aggs={aggCount id="1" enabled=true schema="metric" timeShift="1d"}
`;
const result = await expectExpression('esaggs_shift_single_metric', expression).getResponse();
expect(getCell(result, 0, 0)).to.be(4763);
});
it('shifts multiple metrics', async () => {
const expression = `
kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
| esaggs index={indexPatternLoad id='logstash-*'}
aggs={aggCount id="1" enabled=true schema="metric" timeShift="12h"}
aggs={aggCount id="2" enabled=true schema="metric" timeShift="1d"}
aggs={aggCount id="3" enabled=true schema="metric"}
`;
const result = await expectExpression('esaggs_shift_multi_metric', expression).getResponse();
expect(getCell(result, 0, 0)).to.be(4629);
expect(getCell(result, 0, 1)).to.be(4763);
expect(getCell(result, 0, 2)).to.be(4618);
});
it('shifts single percentile', async () => {
const expression = `
kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
| esaggs index={indexPatternLoad id='logstash-*'}
aggs={aggSinglePercentile id="1" enabled=true schema="metric" field="bytes" percentile=95}
aggs={aggSinglePercentile id="2" enabled=true schema="metric" field="bytes" percentile=95 timeShift="1d"}
`;
const result = await expectExpression(
'esaggs_shift_single_percentile',
expression
).getResponse();
// percentile is not stable
expect(getCell(result, 0, 0)).to.be.within(10000, 20000);
expect(getCell(result, 0, 1)).to.be.within(10000, 20000);
});
it('shifts multiple percentiles', async () => {
const expression = `
kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
| esaggs index={indexPatternLoad id='logstash-*'}
aggs={aggPercentiles id="1" enabled=true schema="metric" field="bytes" percents=5 percents=95}
aggs={aggPercentiles id="2" enabled=true schema="metric" field="bytes" percents=5 percents=95 timeShift="1d"}
`;
const result = await expectExpression(
'esaggs_shift_multi_percentile',
expression
).getResponse();
// percentile is not stable
expect(getCell(result, 0, 0)).to.be.within(100, 1000);
expect(getCell(result, 0, 1)).to.be.within(10000, 20000);
expect(getCell(result, 0, 2)).to.be.within(100, 1000);
expect(getCell(result, 0, 3)).to.be.within(10000, 20000);
});
it('shifts date histogram', async () => {
const expression = `
kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
| esaggs index={indexPatternLoad id='logstash-*'}
aggs={aggDateHistogram id="1" enabled=true schema="bucket" field="@timestamp" interval="1h"}
aggs={aggAvg id="2" field="bytes" enabled=true schema="metric" timeShift="1h"}
aggs={aggAvg id="3" field="bytes" enabled=true schema="metric"}
`;
const result: Datatable = await expectExpression(
'esaggs_shift_date_histogram',
expression
).getResponse();
expect(result.rows.length).to.be(25);
checkShift(result.rows, result.columns);
});
it('shifts filtered metrics', async () => {
const expression = `
kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
| esaggs index={indexPatternLoad id='logstash-*'}
aggs={aggDateHistogram id="1" enabled=true schema="bucket" field="@timestamp" interval="1h"}
aggs={aggFilteredMetric
id="2"
customBucket={aggFilter
id="2-filter"
enabled=true
schema="bucket"
filter='{"language":"kuery","query":"geo.src:US"}'
}
customMetric={aggAvg id="3"
field="bytes"
enabled=true
schema="metric"
}
enabled=true
schema="metric"
timeShift="1h"
}
aggs={aggFilteredMetric
id="4"
customBucket={aggFilter
id="4-filter"
enabled=true
schema="bucket"
filter='{"language":"kuery","query":"geo.src:US"}'
}
customMetric={aggAvg id="5"
field="bytes"
enabled=true
schema="metric"
}
enabled=true
schema="metric"
}
`;
const result: Datatable = await expectExpression(
'esaggs_shift_filtered_metrics',
expression
).getResponse();
expect(result.rows.length).to.be(25);
checkShift(result.rows, result.columns);
});
it('shifts terms', async () => {
const expression = `
kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
| esaggs index={indexPatternLoad id='logstash-*'}
aggs={aggTerms id="1" field="geo.src" size="3" enabled=true schema="bucket" orderAgg={aggCount id="order" enabled=true schema="metric"} otherBucket=true}
aggs={aggAvg id="2" field="bytes" enabled=true schema="metric" timeShift="1d"}
aggs={aggAvg id="3" field="bytes" enabled=true schema="metric"}
`;
const result: Datatable = await expectExpression(
'esaggs_shift_terms',
expression
).getResponse();
expect(result.rows).to.eql([
{
'col-0-1': 'CN',
'col-1-2': 40,
'col-2-3': 5806.404352806415,
},
{
'col-0-1': 'IN',
'col-1-2': 7901,
'col-2-3': 5838.315923566879,
},
{
'col-0-1': 'US',
'col-1-2': 7440,
'col-2-3': 5614.142857142857,
},
{
'col-0-1': '__other__',
'col-1-2': 5766.575645756458,
'col-2-3': 5742.1265576323985,
},
]);
});
it('shifts filters', async () => {
const expression = `
kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
| esaggs index={indexPatternLoad id='logstash-*'}
aggs={aggFilters id="1" filters='[{"input":{"query":"geo.src:\\"US\\" ","language":"kuery"},"label":""},{"input":{"query":"geo.src: \\"CN\\"","language":"kuery"},"label":""}]'}
aggs={aggFilters id="2" filters='[{"input":{"query":"geo.dest:\\"US\\" ","language":"kuery"},"label":""},{"input":{"query":"geo.dest: \\"CN\\"","language":"kuery"},"label":""}]'}
aggs={aggAvg id="3" field="bytes" enabled=true schema="metric" timeShift="2h"}
aggs={aggAvg id="4" field="bytes" enabled=true schema="metric"}
`;
const result: Datatable = await expectExpression(
'esaggs_shift_filters',
expression
).getResponse();
expect(result.rows).to.eql([
{
'col-0-1': 'geo.src:"US" ',
'col-1-2': 'geo.dest:"US" ',
'col-2-3': 5956.9,
'col-3-4': 5956.9,
},
{
'col-0-1': 'geo.src:"US" ',
'col-1-2': 'geo.dest: "CN"',
'col-2-3': 5127.854838709677,
'col-3-4': 5085.746031746032,
},
{
'col-0-1': 'geo.src: "CN"',
'col-1-2': 'geo.dest:"US" ',
'col-2-3': 5648.25,
'col-3-4': 5643.793650793651,
},
{
'col-0-1': 'geo.src: "CN"',
'col-1-2': 'geo.dest: "CN"',
'col-2-3': 5842.858823529412,
'col-3-4': 5842.858823529412,
},
]);
});
it('shifts histogram', async () => {
const expression = `
kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
| esaggs index={indexPatternLoad id='logstash-*'}
aggs={aggHistogram id="1" field="bytes" interval=5000 enabled=true schema="bucket"}
aggs={aggCount id="2" enabled=true schema="metric"}
aggs={aggCount id="3" enabled=true schema="metric" timeShift="6h"}
`;
const result: Datatable = await expectExpression(
'esaggs_shift_histogram',
expression
).getResponse();
expect(result.rows).to.eql([
{
'col-0-1': 0,
'col-1-2': 2020,
'col-2-3': 2036,
},
{
'col-0-1': 5000,
'col-1-2': 2360,
'col-2-3': 2358,
},
{
'col-0-1': 10000,
'col-1-2': 126,
'col-2-3': 127,
},
{
'col-0-1': 15000,
'col-1-2': 112,
'col-2-3': 108,
},
]);
});
it('shifts sibling pipeline aggs', async () => {
const expression = `
kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
| esaggs index={indexPatternLoad id='logstash-*'}
aggs={aggBucketSum id="1" enabled=true schema="metric" customBucket={aggTerms id="2" enabled="true" schema="bucket" field="geo.src" size="3"} customMetric={aggCount id="4" enabled="true" schema="metric"}}
aggs={aggBucketSum id="5" enabled=true schema="metric" timeShift="1d" customBucket={aggTerms id="6" enabled="true" schema="bucket" field="geo.src" size="3"} customMetric={aggCount id="7" enabled="true" schema="metric"}}
`;
const result: Datatable = await expectExpression(
'esaggs_shift_sibling_pipeline_aggs',
expression
).getResponse();
expect(getCell(result, 0, 0)).to.be(2050);
expect(getCell(result, 0, 1)).to.be(2053);
});
it('shifts parent pipeline aggs', async () => {
const expression = `
kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
| esaggs index={indexPatternLoad id='logstash-*'}
aggs={aggDateHistogram id="1" enabled=true schema="bucket" field="@timestamp" interval="3h" min_doc_count=0}
aggs={aggMovingAvg id="2" enabled=true schema="metric" metricAgg="custom" window=5 script="MovingFunctions.unweightedAvg(values)" timeShift="3h" customMetric={aggCount id="2-metric" enabled="true" schema="metric"}}
`;
const result: Datatable = await expectExpression(
'esaggs_shift_parent_pipeline_aggs',
expression
).getResponse();
expect(result.rows).to.eql([
{
'col-0-1': 1442791800000,
'col-1-2': null,
},
{
'col-0-1': 1442802600000,
'col-1-2': 30,
},
{
'col-0-1': 1442813400000,
'col-1-2': 30.5,
},
{
'col-0-1': 1442824200000,
'col-1-2': 69.66666666666667,
},
{
'col-0-1': 1442835000000,
'col-1-2': 198.5,
},
{
'col-0-1': 1442845800000,
'col-1-2': 415.6,
},
{
'col-0-1': 1442856600000,
'col-1-2': 702.2,
},
{
'col-0-1': 1442867400000,
'col-1-2': 859.8,
},
{
'col-0-1': 1442878200000,
'col-1-2': 878.4,
},
]);
});
it('metrics at all levels should work for single shift', async () => {
const expression = `
kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
| esaggs index={indexPatternLoad id='logstash-*'} metricsAtAllLevels=true
aggs={aggCount id="1" enabled=true schema="metric" timeShift="1d"}
`;
const result = await expectExpression('esaggs_shift_single_metric', expression).getResponse();
expect(getCell(result, 0, 0)).to.be(4763);
});
it('metrics at all levels should fail for multiple shifts', async () => {
const expression = `
kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
| esaggs index={indexPatternLoad id='logstash-*'} metricsAtAllLevels=true
aggs={aggCount id="1" enabled=true schema="metric" timeShift="1d"}
aggs={aggCount id="2" enabled=true schema="metric"}
`;
const result = await expectExpression('esaggs_shift_single_metric', expression).getResponse();
expect(result.type).to.be('error');
});
});
}

1
test/interpreter_functional/test_suites/run_pipeline/index.ts
View file

@ -36,5 +36,6 @@ export default function ({ getService, getPageObjects, loadTestFile }: FtrProvid
loadTestFile(require.resolve('./tag_cloud'));
loadTestFile(require.resolve('./metric'));
loadTestFile(require.resolve('./esaggs'));
loadTestFile(require.resolve('./esaggs_timeshift'));
});
}

75
x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx
View file

@ -18,6 +18,8 @@ import {
EuiButtonEmpty,
EuiLink,
EuiPageContentBody,
EuiButton,
EuiSpacer,
} from '@elastic/eui';
import { CoreStart, ApplicationStart } from 'kibana/public';
import {
@ -80,7 +82,11 @@ export interface WorkspacePanelProps {
}
interface WorkspaceState {
expressionBuildError?: Array<{ shortMessage: string; longMessage: string }>;
expressionBuildError?: Array<{
shortMessage: string;
longMessage: string;
fixAction?: { label: string; newState: (framePublicAPI: FramePublicAPI) => Promise<unknown> };
}>;
expandError: boolean;
}
@ -335,6 +341,7 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({
localState={{ ...localState, configurationValidationError, missingRefsErrors }}
ExpressionRendererComponent={ExpressionRendererComponent}
application={core.application}
activeDatasourceId={activeDatasourceId}
/>
);
};
@ -398,6 +405,7 @@ export const VisualizationWrapper = ({
ExpressionRendererComponent,
dispatch,
application,
activeDatasourceId,
}: {
expression: string | null | undefined;
framePublicAPI: FramePublicAPI;
@ -406,11 +414,16 @@ export const VisualizationWrapper = ({
dispatch: (action: Action) => void;
setLocalState: (dispatch: (prevState: WorkspaceState) => WorkspaceState) => void;
localState: WorkspaceState & {
configurationValidationError?: Array<{ shortMessage: string; longMessage: string }>;
configurationValidationError?: Array<{
shortMessage: string;
longMessage: string;
fixAction?: { label: string; newState: (framePublicAPI: FramePublicAPI) => Promise<unknown> };
}>;
missingRefsErrors?: Array<{ shortMessage: string; longMessage: string }>;
};
ExpressionRendererComponent: ReactExpressionRendererType;
application: ApplicationStart;
activeDatasourceId: string | null;
}) => {
const context: ExecutionContextSearch = useMemo(
() => ({
@ -440,6 +453,41 @@ export const VisualizationWrapper = ({
[dispatchLens]
);
function renderFixAction(
validationError:
| {
shortMessage: string;
longMessage: string;
fixAction?:
| { label: string; newState: (framePublicAPI: FramePublicAPI) => Promise<unknown> }
| undefined;
}
| undefined
) {
return (
validationError &&
validationError.fixAction &&
activeDatasourceId && (
<>
<EuiButton
data-test-subj="errorFixAction"
onClick={async () => {
const newState = await validationError.fixAction?.newState(framePublicAPI);
dispatch({
type: 'UPDATE_DATASOURCE_STATE',
datasourceId: activeDatasourceId,
updater: newState,
});
}}
>
{validationError.fixAction.label}
</EuiButton>
<EuiSpacer />
</>
)
);
}
if (localState.configurationValidationError?.length) {
let showExtraErrors = null;
let showExtraErrorsAction = null;
@ -448,14 +496,17 @@ export const VisualizationWrapper = ({
if (localState.expandError) {
showExtraErrors = localState.configurationValidationError
.slice(1)
.map(({ longMessage }) => (
<p
key={longMessage}
className="eui-textBreakWord"
data-test-subj="configuration-failure-error"
>
{longMessage}
</p>
.map((validationError) => (
<>
<p
key={validationError.longMessage}
className="eui-textBreakWord"
data-test-subj="configuration-failure-error"
>
{validationError.longMessage}
</p>
{renderFixAction(validationError)}
</>
));
} else {
showExtraErrorsAction = (
@ -487,6 +538,7 @@ export const VisualizationWrapper = ({
<p className="eui-textBreakWord" data-test-subj="configuration-failure-error">
{localState.configurationValidationError[0].longMessage}
</p>
{renderFixAction(localState.configurationValidationError?.[0])}
{showExtraErrors}
</>
@ -546,6 +598,7 @@ export const VisualizationWrapper = ({
}
if (localState.expressionBuildError?.length) {
const firstError = localState.expressionBuildError[0];
return (
<EuiFlexGroup>
<EuiFlexItem>
@ -559,7 +612,7 @@ export const VisualizationWrapper = ({
/>
</p>
<p>{localState.expressionBuildError[0].longMessage}</p>
<p>{firstError.longMessage}</p>
</>
}
iconColor="danger"

17
x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx
View file

@ -60,9 +60,20 @@ export function WorkspacePanelWrapper({
},
[dispatch, activeVisualization]
);
const warningMessages =
activeVisualization?.getWarningMessages &&
activeVisualization.getWarningMessages(visualizationState, framePublicAPI);
const warningMessages: React.ReactNode[] = [];
if (activeVisualization?.getWarningMessages) {
warningMessages.push(
...(activeVisualization.getWarningMessages(visualizationState, framePublicAPI) || [])
);
}
Object.entries(datasourceStates).forEach(([datasourceId, datasourceState]) => {
const datasource = datasourceMap[datasourceId];
if (!datasourceState.isLoading && datasource.getWarningMessages) {
warningMessages.push(
...(datasource.getWarningMessages(datasourceState.state, framePublicAPI) || [])
);
}
});
return (
<>
<div>

11
x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/advanced_options.tsx
View file

@ -20,9 +20,7 @@ export function AdvancedOptions(props: {
}) {
const [popoverOpen, setPopoverOpen] = useState(false);
const popoverOptions = props.options.filter((option) => option.showInPopover);
const inlineOptions = props.options
.filter((option) => option.inlineElement)
.map((option) => React.cloneElement(option.inlineElement!, { key: option.dataTestSubj }));
const inlineOptions = props.options.filter((option) => option.inlineElement);
return (
<>
@ -74,7 +72,12 @@ export function AdvancedOptions(props: {
{inlineOptions.length > 0 && (
<>
<EuiSpacer size="s" />
{inlineOptions}
{inlineOptions.map((option, index) => (
<>
{React.cloneElement(option.inlineElement!, { key: option.dataTestSubj })}
{index !== inlineOptions.length - 1 && <EuiSpacer size="s" />}
</>
))}
</>
)}
</>

34
x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx
View file

@ -43,6 +43,7 @@ import { ReferenceEditor } from './reference_editor';
import { setTimeScaling, TimeScaling } from './time_scaling';
import { defaultFilter, Filtering, setFilter } from './filtering';
import { AdvancedOptions } from './advanced_options';
import { setTimeShift, TimeShift } from './time_shift';
import { useDebouncedValue } from '../../shared_components';
const operationPanels = getOperationDisplay();
@ -142,6 +143,7 @@ export function DimensionEditor(props: DimensionEditorProps) {
}, [fieldByOperation, operationWithoutField]);
const [filterByOpenInitially, setFilterByOpenInitally] = useState(false);
const [timeShiftedFocused, setTimeShiftFocused] = useState(false);
// Operations are compatible if they match inputs. They are always compatible in
// the empty state. Field-based operations are not compatible with field-less operations.
@ -506,6 +508,38 @@ export function DimensionEditor(props: DimensionEditorProps) {
/>
) : null,
},
{
title: i18n.translate('xpack.lens.indexPattern.timeShift.label', {
defaultMessage: 'Time shift',
}),
dataTestSubj: 'indexPattern-time-shift-enable',
onClick: () => {
setTimeShiftFocused(true);
setStateWrapper(setTimeShift(columnId, state.layers[layerId], ''));
},
showInPopover: Boolean(
operationDefinitionMap[selectedColumn.operationType].shiftable &&
selectedColumn.timeShift === undefined &&
(currentIndexPattern.timeFieldName ||
Object.values(state.layers[layerId].columns).some(
(col) => col.operationType === 'date_histogram'
))
),
inlineElement:
operationDefinitionMap[selectedColumn.operationType].shiftable &&
selectedColumn.timeShift !== undefined ? (
<TimeShift
indexPattern={currentIndexPattern}
selectedColumn={selectedColumn}
columnId={columnId}
layer={state.layers[layerId]}
updateLayer={setStateWrapper}
isFocused={timeShiftedFocused}
activeData={props.activeData}
layerId={layerId}
/>
) : null,
},
]}
/>
)}

193
x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx
View file

@ -33,6 +33,9 @@ import { OperationMetadata } from '../../types';
import { DateHistogramIndexPatternColumn } from '../operations/definitions/date_histogram';
import { getFieldByNameFactory } from '../pure_helpers';
import { Filtering } from './filtering';
import { TimeShift } from './time_shift';
import { DimensionEditor } from './dimension_editor';
import { AdvancedOptions } from './advanced_options';
jest.mock('../loader');
jest.mock('../query_input', () => ({
@ -1319,6 +1322,196 @@ describe('IndexPatternDimensionEditorPanel', () => {
});
});
describe('time shift', () => {
function getProps(colOverrides: Partial<IndexPatternColumn>) {
return {
...defaultProps,
state: getStateWithColumns({
datecolumn: {
dataType: 'date',
isBucketed: true,
label: '',
customLabel: true,
operationType: 'date_histogram',
sourceField: 'ts',
params: {
interval: '1d',
},
},
col2: {
dataType: 'number',
isBucketed: false,
label: 'Count of records',
operationType: 'count',
sourceField: 'Records',
...colOverrides,
} as IndexPatternColumn,
}),
columnId: 'col2',
};
}
it('should not show custom options if time shift is not available', () => {
const props = {
...defaultProps,
state: getStateWithColumns({
col2: {
dataType: 'number',
isBucketed: false,
label: 'Count of records',
operationType: 'count',
sourceField: 'Records',
} as IndexPatternColumn,
}),
columnId: 'col2',
};
wrapper = shallow(
<IndexPatternDimensionEditorComponent
{...props}
state={{
...props.state,
indexPatterns: {
'1': {
...props.state.indexPatterns['1'],
timeFieldName: undefined,
},
},
}}
/>
);
expect(
wrapper
.find(DimensionEditor)
.dive()
.find(AdvancedOptions)
.dive()
.find('[data-test-subj="indexPattern-time-shift-enable"]')
).toHaveLength(0);
});
it('should show custom options if time shift is available', () => {
wrapper = shallow(<IndexPatternDimensionEditorComponent {...getProps({})} />);
expect(
wrapper
.find(DimensionEditor)
.dive()
.find(AdvancedOptions)
.dive()
.find('[data-test-subj="indexPattern-time-shift-enable"]')
).toHaveLength(1);
});
it('should show current time shift if set', () => {
wrapper = mount(<IndexPatternDimensionEditorComponent {...getProps({ timeShift: '1d' })} />);
expect(wrapper.find(TimeShift).find(EuiComboBox).prop('selectedOptions')[0].value).toEqual(
'1d'
);
});
it('should allow to set time shift initially', () => {
const props = getProps({});
wrapper = shallow(<IndexPatternDimensionEditorComponent {...props} />);
wrapper
.find(DimensionEditor)
.dive()
.find(AdvancedOptions)
.dive()
.find('[data-test-subj="indexPattern-time-shift-enable"]')
.prop('onClick')!({} as MouseEvent);
expect((props.setState as jest.Mock).mock.calls[0][0](props.state)).toEqual({
...props.state,
layers: {
first: {
...props.state.layers.first,
columns: {
...props.state.layers.first.columns,
col2: expect.objectContaining({
timeShift: '',
}),
},
},
},
});
});
it('should carry over time shift to other operation if possible', () => {
const props = getProps({
timeShift: '1d',
sourceField: 'bytes',
operationType: 'sum',
label: 'Sum of bytes per hour',
});
wrapper = mount(<IndexPatternDimensionEditorComponent {...props} />);
wrapper
.find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]')
.simulate('click');
expect((props.setState as jest.Mock).mock.calls[0][0](props.state)).toEqual({
...props.state,
layers: {
first: {
...props.state.layers.first,
columns: {
...props.state.layers.first.columns,
col2: expect.objectContaining({
timeShift: '1d',
}),
},
},
},
});
});
it('should allow to change time shift', () => {
const props = getProps({
timeShift: '1d',
});
wrapper = mount(<IndexPatternDimensionEditorComponent {...props} />);
wrapper.find(TimeShift).find(EuiComboBox).prop('onCreateOption')!('1h', []);
expect((props.setState as jest.Mock).mock.calls[0][0](props.state)).toEqual({
...props.state,
layers: {
first: {
...props.state.layers.first,
columns: {
...props.state.layers.first.columns,
col2: expect.objectContaining({
timeShift: '1h',
}),
},
},
},
});
});
it('should allow to time shift', () => {
const props = getProps({
timeShift: '1h',
});
wrapper = mount(<IndexPatternDimensionEditorComponent {...props} />);
wrapper
.find('[data-test-subj="indexPattern-time-shift-remove"]')
.find(EuiButtonIcon)
.prop('onClick')!(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
{} as any
);
expect((props.setState as jest.Mock).mock.calls[0][0](props.state)).toEqual({
...props.state,
layers: {
first: {
...props.state.layers.first,
columns: {
...props.state.layers.first.columns,
col2: expect.objectContaining({
timeShift: undefined,
}),
},
},
},
});
});
});
describe('filtering', () => {
function getProps(colOverrides: Partial<IndexPatternColumn>) {
return {

8
x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_scaling.tsx
View file

@ -27,7 +27,13 @@ export function setTimeScaling(
const currentColumn = layer.columns[columnId];
const label = currentColumn.customLabel
? currentColumn.label
: adjustTimeScaleLabelSuffix(currentColumn.label, currentColumn.timeScale, timeScale);
: adjustTimeScaleLabelSuffix(
currentColumn.label,
currentColumn.timeScale,
timeScale,
currentColumn.timeShift,
currentColumn.timeShift
);
return {
...layer,
columns: {

394
x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx Normal file
View file

@ -0,0 +1,394 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EuiButtonIcon } from '@elastic/eui';
import { EuiFormRow, EuiFlexItem, EuiFlexGroup } from '@elastic/eui';
import { EuiComboBox } from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import { uniq } from 'lodash';
import { i18n } from '@kbn/i18n';
import React, { useEffect, useRef, useState } from 'react';
import { Query } from 'src/plugins/data/public';
import { search } from '../../../../../../src/plugins/data/public';
import { parseTimeShift } from '../../../../../../src/plugins/data/common';
import {
adjustTimeScaleLabelSuffix,
IndexPatternColumn,
operationDefinitionMap,
} from '../operations';
import { IndexPattern, IndexPatternLayer, IndexPatternPrivateState } from '../types';
import { IndexPatternDimensionEditorProps } from './dimension_panel';
import { FramePublicAPI } from '../../types';
// to do: get the language from uiSettings
export const defaultFilter: Query = {
query: '',
language: 'kuery',
};
export function setTimeShift(
columnId: string,
layer: IndexPatternLayer,
timeShift: string | undefined
) {
const trimmedTimeShift = timeShift?.trim();
const currentColumn = layer.columns[columnId];
const label = currentColumn.customLabel
? currentColumn.label
: adjustTimeScaleLabelSuffix(
currentColumn.label,
currentColumn.timeScale,
currentColumn.timeScale,
currentColumn.timeShift,
trimmedTimeShift
);
return {
...layer,
columns: {
...layer.columns,
[columnId]: {
...layer.columns[columnId],
label,
timeShift: trimmedTimeShift,
},
},
};
}
const timeShiftOptions = [
{
label: i18n.translate('xpack.lens.indexPattern.timeShift.hour', {
defaultMessage: '1 hour (1h)',
}),
value: '1h',
},
{
label: i18n.translate('xpack.lens.indexPattern.timeShift.3hours', {
defaultMessage: '3 hours (3h)',
}),
value: '3h',
},
{
label: i18n.translate('xpack.lens.indexPattern.timeShift.6hours', {
defaultMessage: '6 hours (6h)',
}),
value: '6h',
},
{
label: i18n.translate('xpack.lens.indexPattern.timeShift.12hours', {
defaultMessage: '12 hours (12h)',
}),
value: '12h',
},
{
label: i18n.translate('xpack.lens.indexPattern.timeShift.day', {
defaultMessage: '1 day (1d)',
}),
value: '1d',
},
{
label: i18n.translate('xpack.lens.indexPattern.timeShift.week', {
defaultMessage: '1 week (1w)',
}),
value: '1w',
},
{
label: i18n.translate('xpack.lens.indexPattern.timeShift.month', {
defaultMessage: '1 month (1M)',
}),
value: '1M',
},
{
label: i18n.translate('xpack.lens.indexPattern.timeShift.3months', {
defaultMessage: '3 months (3M)',
}),
value: '3M',
},
{
label: i18n.translate('xpack.lens.indexPattern.timeShift.6months', {
defaultMessage: '6 months (6M)',
}),
value: '6M',
},
{
label: i18n.translate('xpack.lens.indexPattern.timeShift.year', {
defaultMessage: '1 year (1y)',
}),
value: '1y',
},
{
label: i18n.translate('xpack.lens.indexPattern.timeShift.previous', {
defaultMessage: 'Previous',
}),
value: 'previous',
},
];
export function TimeShift({
selectedColumn,
columnId,
layer,
updateLayer,
indexPattern,
isFocused,
activeData,
layerId,
}: {
selectedColumn: IndexPatternColumn;
indexPattern: IndexPattern;
columnId: string;
layer: IndexPatternLayer;
updateLayer: (newLayer: IndexPatternLayer) => void;
isFocused: boolean;
activeData: IndexPatternDimensionEditorProps['activeData'];
layerId: string;
}) {
const focusSetRef = useRef(false);
const [localValue, setLocalValue] = useState(selectedColumn.timeShift);
useEffect(() => {
setLocalValue(selectedColumn.timeShift);
}, [selectedColumn.timeShift]);
const selectedOperation = operationDefinitionMap[selectedColumn.operationType];
if (!selectedOperation.shiftable || selectedColumn.timeShift === undefined) {
return null;
}
let dateHistogramInterval: null | moment.Duration = null;
const dateHistogramColumn = layer.columnOrder.find(
(colId) => layer.columns[colId].operationType === 'date_histogram'
);
if (!dateHistogramColumn && !indexPattern.timeFieldName) {
return null;
}
if (dateHistogramColumn && activeData && activeData[layerId] && activeData[layerId]) {
const column = activeData[layerId].columns.find((col) => col.id === dateHistogramColumn);
if (column) {
dateHistogramInterval = search.aggs.parseInterval(
search.aggs.getDateHistogramMetaDataByDatatableColumn(column)?.interval || ''
);
}
}
function isValueTooSmall(parsedValue: ReturnType<typeof parseTimeShift>) {
return (
dateHistogramInterval &&
parsedValue &&
typeof parsedValue === 'object' &&
parsedValue.asMilliseconds() < dateHistogramInterval.asMilliseconds()
);
}
function isValueNotMultiple(parsedValue: ReturnType<typeof parseTimeShift>) {
return (
dateHistogramInterval &&
parsedValue &&
typeof parsedValue === 'object' &&
!Number.isInteger(parsedValue.asMilliseconds() / dateHistogramInterval.asMilliseconds())
);
}
const parsedLocalValue = localValue && parseTimeShift(localValue);
const isLocalValueInvalid = Boolean(parsedLocalValue === 'invalid');
const localValueTooSmall = parsedLocalValue && isValueTooSmall(parsedLocalValue);
const localValueNotMultiple = parsedLocalValue && isValueNotMultiple(parsedLocalValue);
function getSelectedOption() {
if (!localValue) return [];
const goodPick = timeShiftOptions.filter(({ value }) => value === localValue);
if (goodPick.length > 0) return goodPick;
return [
{
value: localValue,
label: localValue,
},
];
}
return (
<div
ref={(r) => {
if (r && isFocused) {
const timeShiftInput = r.querySelector('[data-test-subj="comboBoxSearchInput"]');
if (!focusSetRef.current && timeShiftInput instanceof HTMLInputElement) {
focusSetRef.current = true;
timeShiftInput.focus();
}
}
}}
>
<EuiFormRow
display="columnCompressed"
fullWidth
label={i18n.translate('xpack.lens.indexPattern.timeShift.label', {
defaultMessage: 'Time shift',
})}
helpText={i18n.translate('xpack.lens.indexPattern.timeShift.help', {
defaultMessage: 'Enter the time shift number and unit',
})}
error={
(localValueTooSmall &&
i18n.translate('xpack.lens.indexPattern.timeShift.tooSmallHelp', {
defaultMessage:
'Time shift should to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram',
})) ||
(localValueNotMultiple &&
i18n.translate('xpack.lens.indexPattern.timeShift.noMultipleHelp', {
defaultMessage:
'Time shift should be a multiple of the date histogram interval. Either adjust time shift or date histogram interval',
}))
}
isInvalid={Boolean(isLocalValueInvalid || localValueTooSmall || localValueNotMultiple)}
>
<EuiFlexGroup gutterSize="s" alignItems="center">
<EuiFlexItem>
<EuiComboBox
fullWidth
compressed
isClearable={false}
data-test-subj="indexPattern-dimension-time-shift"
placeholder={i18n.translate('xpack.lens.indexPattern.timeShiftPlaceholder', {
defaultMessage: 'Time shift (e.g. 1d)',
})}
options={timeShiftOptions.filter(({ value }) => {
const parsedValue = parseTimeShift(value);
return (
parsedValue && !isValueTooSmall(parsedValue) && !isValueNotMultiple(parsedValue)
);
})}
selectedOptions={getSelectedOption()}
singleSelection={{ asPlainText: true }}
isInvalid={isLocalValueInvalid}
onCreateOption={(val) => {
const parsedVal = parseTimeShift(val);
if (parsedVal !== 'invalid') {
updateLayer(setTimeShift(columnId, layer, val));
} else {
setLocalValue(val);
}
}}
onChange={(choices) => {
if (choices.length === 0) {
updateLayer(setTimeShift(columnId, layer, ''));
setLocalValue('');
return;
}
const choice = choices[0].value as string;
const parsedVal = parseTimeShift(choice);
if (parsedVal !== 'invalid') {
updateLayer(setTimeShift(columnId, layer, choice));
} else {
setLocalValue(choice);
}
}}
/>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiButtonIcon
data-test-subj="indexPattern-time-shift-remove"
color="danger"
aria-label={i18n.translate('xpack.lens.timeShift.removeLabel', {
defaultMessage: 'Remove time shift',
})}
onClick={() => {
updateLayer(setTimeShift(columnId, layer, undefined));
}}
iconType="cross"
/>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFormRow>
</div>
);
}
export function getTimeShiftWarningMessages(
state: IndexPatternPrivateState,
{ activeData }: FramePublicAPI
) {
if (!state) return;
const warningMessages: React.ReactNode[] = [];
Object.entries(state.layers).forEach(([layerId, layer]) => {
let dateHistogramInterval: null | string = null;
const dateHistogramColumn = layer.columnOrder.find(
(colId) => layer.columns[colId].operationType === 'date_histogram'
);
if (!dateHistogramColumn) {
return;
}
if (dateHistogramColumn && activeData && activeData[layerId]) {
const column = activeData[layerId].columns.find((col) => col.id === dateHistogramColumn);
if (column) {
dateHistogramInterval =
search.aggs.getDateHistogramMetaDataByDatatableColumn(column)?.interval || null;
}
}
if (dateHistogramInterval === null) {
return;
}
const shiftInterval = search.aggs.parseInterval(dateHistogramInterval)!.asMilliseconds();
let timeShifts: number[] = [];
const timeShiftMap: Record<number, string[]> = {};
Object.entries(layer.columns).forEach(([columnId, column]) => {
if (column.isBucketed) return;
let duration: number = 0;
if (column.timeShift) {
const parsedTimeShift = parseTimeShift(column.timeShift);
if (parsedTimeShift === 'previous' || parsedTimeShift === 'invalid') {
return;
}
duration = parsedTimeShift.asMilliseconds();
}
timeShifts.push(duration);
if (!timeShiftMap[duration]) {
timeShiftMap[duration] = [];
}
timeShiftMap[duration].push(columnId);
});
timeShifts = uniq(timeShifts);
if (timeShifts.length < 2) {
return;
}
timeShifts.forEach((timeShift) => {
if (timeShift === 0) return;
if (timeShift < shiftInterval) {
timeShiftMap[timeShift].forEach((columnId) => {
warningMessages.push(
<FormattedMessage
key={`small-${columnId}`}
id="xpack.lens.indexPattern.timeShiftSmallWarning"
defaultMessage="{label} uses a time shift of {columnTimeShift} which is smaller than the date histogram interval of {interval}. To prevent mismatched data, use a multiple of {interval} as time shift."
values={{
label: <strong>{layer.columns[columnId].label}</strong>,
interval: <strong>{dateHistogramInterval}</strong>,
columnTimeShift: <strong>{layer.columns[columnId].timeShift}</strong>,
}}
/>
);
});
} else if (!Number.isInteger(timeShift / shiftInterval)) {
timeShiftMap[timeShift].forEach((columnId) => {
warningMessages.push(
<FormattedMessage
key={`multiple-${columnId}`}
id="xpack.lens.indexPattern.timeShiftMultipleWarning"
defaultMessage="{label} uses a time shift of {columnTimeShift} which is not a multiple of the date histogram interval of {interval}. To prevent mismatched data, use a multiple of {interval} as time shift."
values={{
label: <strong>{layer.columns[columnId].label}</strong>,
interval: dateHistogramInterval,
columnTimeShift: layer.columns[columnId].timeShift!,
}}
/>
);
});
}
});
});
return warningMessages;
}

169
x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts
View file

@ -7,7 +7,7 @@
import { IStorageWrapper } from 'src/plugins/kibana_utils/public';
import { getIndexPatternDatasource, IndexPatternColumn } from './indexpattern';
import { DatasourcePublicAPI, Operation, Datasource } from '../types';
import { DatasourcePublicAPI, Operation, Datasource, FramePublicAPI } from '../types';
import { coreMock } from 'src/core/public/mocks';
import { IndexPatternPersistedState, IndexPatternPrivateState } from './types';
import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks';
@ -18,6 +18,7 @@ import { operationDefinitionMap, getErrorMessages } from './operations';
import { createMockedFullReference } from './operations/mocks';
import { indexPatternFieldEditorPluginMock } from 'src/plugins/index_pattern_field_editor/public/mocks';
import { uiActionsPluginMock } from '../../../../../src/plugins/ui_actions/public/mocks';
import React from 'react';
jest.mock('./loader');
jest.mock('../id_generator');
@ -500,6 +501,43 @@ describe('IndexPattern Data Source', () => {
expect(ast.chain[0].arguments.timeFields).toEqual(['timestamp', 'another_datefield']);
});
it('should pass time shift parameter to metric agg functions', async () => {
const queryBaseState: IndexPatternBaseState = {
currentIndexPatternId: '1',
layers: {
first: {
indexPatternId: '1',
columnOrder: ['col2', 'col1'],
columns: {
col1: {
label: 'Count of records',
dataType: 'number',
isBucketed: false,
sourceField: 'Records',
operationType: 'count',
timeShift: '1d',
},
col2: {
label: 'Date',
dataType: 'date',
isBucketed: true,
operationType: 'date_histogram',
sourceField: 'timestamp',
params: {
interval: 'auto',
},
},
},
},
},
};
const state = enrichBaseState(queryBaseState);
const ast = indexPatternDatasource.toExpression(state, 'first') as Ast;
expect((ast.chain[0].arguments.aggs[1] as Ast).chain[0].arguments.timeShift).toEqual(['1d']);
});
it('should wrap filtered metrics in filtered metric aggregation', async () => {
const queryBaseState: IndexPatternBaseState = {
currentIndexPatternId: '1',
@ -1267,6 +1305,135 @@ describe('IndexPattern Data Source', () => {
});
});
describe('#getWarningMessages', () => {
it('should return mismatched time shifts', () => {
const state: IndexPatternPrivateState = {
indexPatternRefs: [],
existingFields: {},
isFirstExistenceFetch: false,
indexPatterns: expectedIndexPatterns,
layers: {
first: {
indexPatternId: '1',
columnOrder: ['col1', 'col2', 'col3', 'col4', 'col5', 'col6'],
columns: {
col1: {
operationType: 'date_histogram',
params: {
interval: '12h',
},
label: '',
dataType: 'date',
isBucketed: true,
sourceField: 'timestamp',
},
col2: {
operationType: 'count',
label: '',
dataType: 'number',
isBucketed: false,
sourceField: 'records',
},
col3: {
operationType: 'count',
timeShift: '1h',
label: '',
dataType: 'number',
isBucketed: false,
sourceField: 'records',
},
col4: {
operationType: 'count',
timeShift: '13h',
label: '',
dataType: 'number',
isBucketed: false,
sourceField: 'records',
},
col5: {
operationType: 'count',
timeShift: '1w',
label: '',
dataType: 'number',
isBucketed: false,
sourceField: 'records',
},
col6: {
operationType: 'count',
timeShift: 'previous',
label: '',
dataType: 'number',
isBucketed: false,
sourceField: 'records',
},
},
},
},
currentIndexPatternId: '1',
};
const warnings = indexPatternDatasource.getWarningMessages!(state, ({
activeData: {
first: {
type: 'datatable',
rows: [],
columns: [
{
id: 'col1',
name: 'col1',
meta: {
type: 'date',
source: 'esaggs',
sourceParams: {
type: 'date_histogram',
params: {
used_interval: '12h',
},
},
},
},
],
},
},
} as unknown) as FramePublicAPI);
expect(warnings!.length).toBe(2);
expect((warnings![0] as React.ReactElement).props.id).toEqual(
'xpack.lens.indexPattern.timeShiftSmallWarning'
);
expect((warnings![1] as React.ReactElement).props.id).toEqual(
'xpack.lens.indexPattern.timeShiftMultipleWarning'
);
});
it('should prepend each error with its layer number on multi-layer chart', () => {
(getErrorMessages as jest.Mock).mockClear();
(getErrorMessages as jest.Mock).mockReturnValueOnce(['error 1', 'error 2']);
const state: IndexPatternPrivateState = {
indexPatternRefs: [],
existingFields: {},
isFirstExistenceFetch: false,
indexPatterns: expectedIndexPatterns,
layers: {
first: {
indexPatternId: '1',
columnOrder: [],
columns: {},
},
second: {
indexPatternId: '1',
columnOrder: [],
columns: {},
},
},
currentIndexPatternId: '1',
};
expect(indexPatternDatasource.getErrorMessages(state)).toEqual([
{ longMessage: 'Layer 1 error: error 1', shortMessage: '' },
{ longMessage: 'Layer 1 error: error 2', shortMessage: '' },
]);
expect(getErrorMessages).toHaveBeenCalledTimes(2);
});
});
describe('#updateStateOnCloseDimension', () => {
it('should not update when there are no incomplete columns', () => {
expect(

23
x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx
View file

@ -55,6 +55,7 @@ import { deleteColumn, isReferenced } from './operations';
import { UiActionsStart } from '../../../../../src/plugins/ui_actions/public';
import { GeoFieldWorkspacePanel } from '../editor_frame_service/editor_frame/workspace_panel/geo_field_workspace_panel';
import { DraggingIdentifier } from '../drag_drop';
import { getTimeShiftWarningMessages } from './dimension_panel/time_shift';
export { OperationType, IndexPatternColumn, deleteColumn } from './operations';
@ -407,13 +408,20 @@ export function getIndexPatternDatasource({
}
// Forward the indexpattern as well, as it is required by some operationType checks
const layerErrors = Object.values(state.layers).map((layer) =>
(getErrorMessages(layer, state.indexPatterns[layer.indexPatternId]) ?? []).map(
(message) => ({
shortMessage: '', // Not displayed currently
longMessage: message,
})
)
const layerErrors = Object.entries(state.layers).map(([layerId, layer]) =>
(
getErrorMessages(
layer,
state.indexPatterns[layer.indexPatternId],
state,
layerId,
core
) ?? []
).map((message) => ({
shortMessage: '', // Not displayed currently
longMessage: typeof message === 'string' ? message : message.message,
fixAction: typeof message === 'object' ? message.fixAction : undefined,
}))
);
// Single layer case, no need to explain more
@ -449,6 +457,7 @@ export function getIndexPatternDatasource({
});
return messages.length ? messages : undefined;
},
getWarningMessages: getTimeShiftWarningMessages,
checkIntegrity: (state) => {
const ids = Object.values(state.layers || {}).map(({ indexPatternId }) => indexPatternId);
return ids.filter((id) => !state.indexPatterns[id]);

8
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx
View file

@ -70,7 +70,8 @@ export const counterRateOperation: OperationDefinition<
ref && 'sourceField' in ref
? indexPattern.getFieldByName(ref.sourceField)?.displayName
: undefined,
column.timeScale
column.timeScale,
column.timeShift
);
},
toExpression: (layer, columnId) => {
@ -84,7 +85,8 @@ export const counterRateOperation: OperationDefinition<
metric && 'sourceField' in metric
? indexPattern.getFieldByName(metric.sourceField)?.displayName
: undefined,
timeScale
timeScale,
previousColumn?.timeShift
),
dataType: 'number',
operationType: 'counter_rate',
@ -92,6 +94,7 @@ export const counterRateOperation: OperationDefinition<
scale: 'ratio',
references: referenceIds,
timeScale,
timeShift: previousColumn?.timeShift,
filter: getFilter(previousColumn, columnParams),
params: getFormatFromPreviousColumn(previousColumn),
};
@ -118,4 +121,5 @@ export const counterRateOperation: OperationDefinition<
},
timeScalingMode: 'mandatory',
filterable: true,
shiftable: true,
};

15
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx
View file

@ -13,11 +13,12 @@ import {
getErrorsForDateReference,
dateBasedOperationToExpression,
hasDateField,
buildLabelFunction,
} from './utils';
import { OperationDefinition } from '..';
import { getFormatFromPreviousColumn, getFilter } from '../helpers';
const ofName = (name?: string) => {
const ofName = buildLabelFunction((name?: string) => {
return i18n.translate('xpack.lens.indexPattern.cumulativeSumOf', {
defaultMessage: 'Cumulative sum of {name}',
values: {
@ -28,7 +29,7 @@ const ofName = (name?: string) => {
}),
},
});
};
});
export type CumulativeSumIndexPatternColumn = FormattedIndexPatternColumn &
ReferenceBasedIndexPatternColumn & {
@ -67,7 +68,9 @@ export const cumulativeSumOperation: OperationDefinition<
return ofName(
ref && 'sourceField' in ref
? indexPattern.getFieldByName(ref.sourceField)?.displayName
: undefined
: undefined,
undefined,
column.timeShift
);
},
toExpression: (layer, columnId) => {
@ -79,12 +82,15 @@ export const cumulativeSumOperation: OperationDefinition<
label: ofName(
ref && 'sourceField' in ref
? indexPattern.getFieldByName(ref.sourceField)?.displayName
: undefined
: undefined,
undefined,
previousColumn?.timeShift
),
dataType: 'number',
operationType: 'cumulative_sum',
isBucketed: false,
scale: 'ratio',
timeShift: previousColumn?.timeShift,
filter: getFilter(previousColumn, columnParams),
references: referenceIds,
params: getFormatFromPreviousColumn(previousColumn),
@ -111,4 +117,5 @@ export const cumulativeSumOperation: OperationDefinition<
)?.join(', ');
},
filterable: true,
shiftable: true,
};

6
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx
View file

@ -66,7 +66,7 @@ export const derivativeOperation: OperationDefinition<
}
},
getDefaultLabel: (column, indexPattern, columns) => {
return ofName(columns[column.references[0]]?.label, column.timeScale);
return ofName(columns[column.references[0]]?.label, column.timeScale, column.timeShift);
},
toExpression: (layer, columnId) => {
return dateBasedOperationToExpression(layer, columnId, 'derivative');
@ -74,7 +74,7 @@ export const derivativeOperation: OperationDefinition<
buildColumn: ({ referenceIds, previousColumn, layer }, columnParams) => {
const ref = layer.columns[referenceIds[0]];
return {
label: ofName(ref?.label, previousColumn?.timeScale),
label: ofName(ref?.label, previousColumn?.timeScale, previousColumn?.timeShift),
dataType: 'number',
operationType: OPERATION_NAME,
isBucketed: false,
@ -82,6 +82,7 @@ export const derivativeOperation: OperationDefinition<
references: referenceIds,
timeScale: previousColumn?.timeScale,
filter: getFilter(previousColumn, columnParams),
timeShift: previousColumn?.timeShift,
params: getFormatFromPreviousColumn(previousColumn),
};
},
@ -108,4 +109,5 @@ export const derivativeOperation: OperationDefinition<
},
timeScalingMode: 'optional',
filterable: true,
shiftable: true,
};

6
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx
View file

@ -76,7 +76,7 @@ export const movingAverageOperation: OperationDefinition<
}
},
getDefaultLabel: (column, indexPattern, columns) => {
return ofName(columns[column.references[0]]?.label, column.timeScale);
return ofName(columns[column.references[0]]?.label, column.timeScale, column.timeShift);
},
toExpression: (layer, columnId) => {
return dateBasedOperationToExpression(layer, columnId, 'moving_average', {
@ -90,7 +90,7 @@ export const movingAverageOperation: OperationDefinition<
const metric = layer.columns[referenceIds[0]];
const { window = WINDOW_DEFAULT_VALUE } = columnParams;
return {
label: ofName(metric?.label, previousColumn?.timeScale),
label: ofName(metric?.label, previousColumn?.timeScale, previousColumn?.timeShift),
dataType: 'number',
operationType: 'moving_average',
isBucketed: false,
@ -98,6 +98,7 @@ export const movingAverageOperation: OperationDefinition<
references: referenceIds,
timeScale: previousColumn?.timeScale,
filter: getFilter(previousColumn, columnParams),
timeShift: previousColumn?.timeShift,
params: {
window,
...getFormatFromPreviousColumn(previousColumn),
@ -129,6 +130,7 @@ export const movingAverageOperation: OperationDefinition<
},
timeScalingMode: 'optional',
filterable: true,
shiftable: true,
};
function MovingAverageParamEditor({

5
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts
View file

@ -16,10 +16,11 @@ import { operationDefinitionMap } from '..';
export const buildLabelFunction = (ofName: (name?: string) => string) => (
name?: string,
timeScale?: TimeScaleUnit
timeScale?: TimeScaleUnit,
timeShift?: string
) => {
const rawLabel = ofName(name);
return adjustTimeScaleLabelSuffix(rawLabel, undefined, timeScale);
return adjustTimeScaleLabelSuffix(rawLabel, undefined, timeScale, undefined, timeShift);
};
/**

37
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx
View file

@ -17,6 +17,7 @@ import {
getSafeName,
getFilter,
} from './helpers';
import { adjustTimeScaleLabelSuffix } from '../time_scale_utils';
const supportedTypes = new Set([
'string',
@ -33,13 +34,19 @@ const SCALE = 'ratio';
const OPERATION_TYPE = 'unique_count';
const IS_BUCKETED = false;
function ofName(name: string) {
return i18n.translate('xpack.lens.indexPattern.cardinalityOf', {
defaultMessage: 'Unique count of {name}',
values: {
name,
},
});
function ofName(name: string, timeShift: string | undefined) {
return adjustTimeScaleLabelSuffix(
i18n.translate('xpack.lens.indexPattern.cardinalityOf', {
defaultMessage: 'Unique count of {name}',
values: {
name,
},
}),
undefined,
undefined,
undefined,
timeShift
);
}
export interface CardinalityIndexPatternColumn
@ -76,21 +83,19 @@ export const cardinalityOperation: OperationDefinition<CardinalityIndexPatternCo
);
},
filterable: true,
operationParams: [
{ name: 'kql', type: 'string', required: false },
{ name: 'lucene', type: 'string', required: false },
],
getDefaultLabel: (column, indexPattern) => ofName(getSafeName(column.sourceField, indexPattern)),
shiftable: true,
getDefaultLabel: (column, indexPattern) =>
ofName(getSafeName(column.sourceField, indexPattern), column.timeShift),
buildColumn({ field, previousColumn }, columnParams) {
return {
label: ofName(field.displayName),
label: ofName(field.displayName, previousColumn?.timeShift),
dataType: 'number',
operationType: OPERATION_TYPE,
scale: SCALE,
sourceField: field.name,
isBucketed: IS_BUCKETED,
filter: getFilter(previousColumn, columnParams),
timeShift: previousColumn?.timeShift,
params: getFormatFromPreviousColumn(previousColumn),
};
},
@ -100,12 +105,14 @@ export const cardinalityOperation: OperationDefinition<CardinalityIndexPatternCo
enabled: true,
schema: 'metric',
field: column.sourceField,
// time shift is added to wrapping aggFilteredMetric if filter is set
timeShift: column.filter ? undefined : column.timeShift,
}).toAst();
},
onFieldChange: (oldColumn, field) => {
return {
...oldColumn,
label: ofName(field.displayName),
label: ofName(field.displayName, oldColumn.timeShift),
sourceField: field.name,
};
},

1
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts
View file

@ -16,6 +16,7 @@ export interface BaseIndexPatternColumn extends Operation {
customLabel?: boolean;
timeScale?: TimeScaleUnit;
filter?: Query;
timeShift?: string;
}
// Formatting can optionally be added to any column

29
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx
View file

@ -38,7 +38,13 @@ export const countOperation: OperationDefinition<CountIndexPatternColumn, 'field
onFieldChange: (oldColumn, field) => {
return {
...oldColumn,
label: adjustTimeScaleLabelSuffix(field.displayName, undefined, oldColumn.timeScale),
label: adjustTimeScaleLabelSuffix(
field.displayName,
undefined,
oldColumn.timeScale,
undefined,
oldColumn.timeShift
),
sourceField: field.name,
};
},
@ -51,10 +57,23 @@ export const countOperation: OperationDefinition<CountIndexPatternColumn, 'field
};
}
},
getDefaultLabel: (column) => adjustTimeScaleLabelSuffix(countLabel, undefined, column.timeScale),
getDefaultLabel: (column) =>
adjustTimeScaleLabelSuffix(
countLabel,
undefined,
column.timeScale,
undefined,
column.timeShift
),
buildColumn({ field, previousColumn }, columnParams) {
return {
label: adjustTimeScaleLabelSuffix(countLabel, undefined, previousColumn?.timeScale),
label: adjustTimeScaleLabelSuffix(
countLabel,
undefined,
previousColumn?.timeScale,
undefined,
previousColumn?.timeShift
),
dataType: 'number',
operationType: 'count',
isBucketed: false,
@ -62,6 +81,7 @@ export const countOperation: OperationDefinition<CountIndexPatternColumn, 'field
sourceField: field.name,
timeScale: previousColumn?.timeScale,
filter: getFilter(previousColumn, columnParams),
timeShift: previousColumn?.timeShift,
params:
previousColumn?.dataType === 'number' &&
previousColumn.params &&
@ -82,6 +102,8 @@ export const countOperation: OperationDefinition<CountIndexPatternColumn, 'field
id: columnId,
enabled: true,
schema: 'metric',
// time shift is added to wrapping aggFilteredMetric if filter is set
timeShift: column.filter ? undefined : column.timeShift,
}).toAst();
},
isTransferable: () => {
@ -89,4 +111,5 @@ export const countOperation: OperationDefinition<CountIndexPatternColumn, 'field
},
timeScalingMode: 'optional',
filterable: true,
shiftable: true,
};

53
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx
View file

@ -23,7 +23,7 @@ import {
EuiTextColor,
} from '@elastic/eui';
import { updateColumnParam } from '../layer_helpers';
import { OperationDefinition } from './index';
import { OperationDefinition, ParamEditorProps } from './index';
import { FieldBasedIndexPatternColumn } from './column_types';
import {
AggFunctionsMapping,
@ -35,6 +35,7 @@ import {
import { buildExpressionFunction } from '../../../../../../../src/plugins/expressions/public';
import { getInvalidFieldMessage, getSafeName } from './helpers';
import { HelpPopover, HelpPopoverButton } from '../../help_popover';
import { IndexPatternLayer } from '../../types';
const { isValidInterval } = search.aggs;
const autoInterval = 'auto';
@ -48,6 +49,28 @@ export interface DateHistogramIndexPatternColumn extends FieldBasedIndexPatternC
};
}
function getMultipleDateHistogramsErrorMessage(layer: IndexPatternLayer, columnId: string) {
const usesTimeShift = Object.values(layer.columns).some(
(col) => col.timeShift && col.timeShift !== ''
);
if (!usesTimeShift) {
return undefined;
}
const dateHistograms = layer.columnOrder.filter(
(colId) => layer.columns[colId].operationType === 'date_histogram'
);
if (dateHistograms.length < 2) {
return undefined;
}
return i18n.translate('xpack.lens.indexPattern.multipleDateHistogramsError', {
defaultMessage:
'"{dimensionLabel}" is not the only date histogram. When using time shifts, make sure to only use one date histogram.',
values: {
dimensionLabel: layer.columns[columnId].label,
},
});
}
export const dateHistogramOperation: OperationDefinition<
DateHistogramIndexPatternColumn,
'field'
@ -60,7 +83,13 @@ export const dateHistogramOperation: OperationDefinition<
priority: 5, // Highest priority level used
operationParams: [{ name: 'interval', type: 'string', required: false }],
getErrorMessage: (layer, columnId, indexPattern) =>
getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern),
[
...(getInvalidFieldMessage(
layer.columns[columnId] as FieldBasedIndexPatternColumn,
indexPattern
) || []),
getMultipleDateHistogramsErrorMessage(layer, columnId) || '',
].filter(Boolean),
getHelpMessage: (props) => <AutoDateHistogramPopover {...props} />,
getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type }) => {
if (
@ -150,7 +179,15 @@ export const dateHistogramOperation: OperationDefinition<
extended_bounds: JSON.stringify({}),
}).toAst();
},
paramEditor: ({ layer, columnId, currentColumn, updateLayer, dateRange, data, indexPattern }) => {
paramEditor: function ParamEditor({
layer,
columnId,
currentColumn,
updateLayer,
dateRange,
data,
indexPattern,
}: ParamEditorProps<DateHistogramIndexPatternColumn>) {
const field = currentColumn && indexPattern.getFieldByName(currentColumn.sourceField);
const intervalIsRestricted =
field!.aggregationRestrictions && field!.aggregationRestrictions.date_histogram;
@ -225,10 +262,11 @@ export const dateHistogramOperation: OperationDefinition<
disabled={calendarOnlyIntervals.has(interval.unit)}
isInvalid={!isValid}
onChange={(e) => {
setInterval({
const newInterval = {
...interval,
value: e.target.value,
});
};
setInterval(newInterval);
}}
/>
</EuiFlexItem>
@ -238,10 +276,11 @@ export const dateHistogramOperation: OperationDefinition<
data-test-subj="lensDateHistogramUnit"
value={interval.unit}
onChange={(e) => {
setInterval({
const newInterval = {
...interval,
unit: e.target.value,
});
};
setInterval(newInterval);
}}
isInvalid={!isValid}
options={[

43
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from 'kibana/public';
import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup, CoreStart } from 'kibana/public';
import { IStorageWrapper } from 'src/plugins/kibana_utils/public';
import { termsOperation, TermsIndexPatternColumn } from './terms';
import { filtersOperation, FiltersIndexPatternColumn } from './filters';
@ -42,13 +42,14 @@ import {
FormulaIndexPatternColumn,
} from './formula';
import { lastValueOperation, LastValueIndexPatternColumn } from './last_value';
import { OperationMetadata } from '../../../types';
import { FramePublicAPI, OperationMetadata } from '../../../types';
import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types';
import { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../types';
import { DateRange } from '../../../../common';
import { ExpressionAstFunction } from '../../../../../../../src/plugins/expressions/public';
import { DataPublicPluginStart } from '../../../../../../../src/plugins/data/public';
import { RangeIndexPatternColumn, rangeOperation } from './ranges';
import { IndexPatternDimensionEditorProps } from '../../dimension_panel';
/**
* A union type of all available column types. If a column is of an unknown type somewhere
@ -160,6 +161,7 @@ export interface ParamEditorProps<C> {
http: HttpSetup;
dateRange: DateRange;
data: DataPublicPluginStart;
activeData?: IndexPatternDimensionEditorProps['activeData'];
operationDefinitionMap: Record<string, GenericOperationDefinition>;
}
@ -240,7 +242,22 @@ interface BaseOperationDefinitionProps<C extends BaseIndexPatternColumn> {
columnId: string,
indexPattern: IndexPattern,
operationDefinitionMap?: Record<string, GenericOperationDefinition>
) => string[] | undefined;
) =>
| Array<
| string
| {
message: string;
fixAction?: {
label: string;
newState: (
core: CoreStart,
frame: FramePublicAPI,
layerId: string
) => Promise<IndexPatternLayer>;
};
}
>
| undefined;
/*
* Flag whether this operation can be scaled by time unit if a date histogram is available.
@ -255,6 +272,7 @@ interface BaseOperationDefinitionProps<C extends BaseIndexPatternColumn> {
* autocomplete.
*/
filterable?: boolean;
shiftable?: boolean;
getHelpMessage?: (props: HelpProps<C>) => React.ReactNode;
/*
@ -366,12 +384,27 @@ interface FieldBasedOperationDefinition<C extends BaseIndexPatternColumn> {
* - Requires a date histogram operation somewhere before it in order
* - Missing references
*/
getErrorMessage: (
getErrorMessage?: (
layer: IndexPatternLayer,
columnId: string,
indexPattern: IndexPattern,
operationDefinitionMap?: Record<string, GenericOperationDefinition>
) => string[] | undefined;
) =>
| Array<
| string
| {
message: string;
fixAction?: {
label: string;
newState: (
core: CoreStart,
frame: FramePublicAPI,
layerId: string
) => Promise<IndexPatternLayer>;
};
}
>
| undefined;
}
export interface RequiredReference {

Some files were not shown because too many files have changed in this diff Show more