Remove MathJS Feature (#15652)

* Reverting a258f1af4

* Revert "[TSVB] Add support for Math Aggregation to tables (#14553)"

This reverts commit d2537d8039.

* Revert "Math Aggregation to support Sibling Aggs for TSVB (#13681)"

This reverts commit 9c9fb17fef.

* removing old partial bucket cliping from table vis

* Adding support for percentiles for bucket_script args

* removing old partial bucket cliping for table vis

* Fixing support for drop last bucket

* Adding unsupported agg message
This commit is contained in:
Chris Cowan 2018-01-09 12:25:33 -07:00
parent 59726c0b75
commit f423729a88
23 changed files with 58 additions and 333 deletions

View file

@ -151,7 +151,6 @@
"lodash": "3.10.1",
"lru-cache": "4.1.1",
"markdown-it": "8.3.2",
"mathjs": "3.16.2",
"minimatch": "2.0.10",
"mkdirp": "0.5.1",
"moment": "2.13.0",

View file

@ -16,7 +16,7 @@ describe('aggLookup', () => {
it('returns options for all aggs', () => {
const options = createOptions();
expect(options).to.have.length(29);
expect(options).to.have.length(28);
options.forEach((option) => {
expect(option).to.have.property('label');
expect(option).to.have.property('value');
@ -32,13 +32,13 @@ describe('aggLookup', () => {
it('returns options for pipeline', () => {
const options = createOptions('pipeline');
expect(options).to.have.length(15);
expect(options).to.have.length(14);
expect(options.every(opt => !isBasicAgg({ type: opt.value }))).to.equal(true);
});
it('returns options for all if given unknown key', () => {
const options = createOptions('foo');
expect(options).to.have.length(29);
expect(options).to.have.length(28);
});
});

View file

@ -15,7 +15,7 @@ describe('calculateLabel(metric, metrics)', () => {
});
it('returns "Calcuation" for a bucket script metric', () => {
expect(calculateLabel({ type: 'calculation' })).to.equal('Bucket Script');
expect(calculateLabel({ type: 'calculation' })).to.equal('Calculation');
});
it('returns formated label for series_agg', () => {

View file

@ -1,7 +1,7 @@
import _ from 'lodash';
const lookup = {
'count': 'Count',
'calculation': 'Bucket Script',
'calculation': 'Calculation',
'std_deviation': 'Std. Deviation',
'variance': 'Variance',
'sum_of_squares': 'Sum of Sq.',
@ -24,7 +24,6 @@ const lookup = {
'sum_of_squares_bucket': 'Overall Sum of Sq.',
'std_deviation_bucket': 'Overall Std. Deviation',
'series_agg': 'Series Agg',
'math': 'Math',
'serial_diff': 'Serial Difference',
'filter_ratio': 'Filter Ratio',
'positive_only': 'Positive Only',
@ -44,7 +43,6 @@ const pipeline = [
'sum_of_squares_bucket',
'std_deviation_bucket',
'series_agg',
'math',
'serial_diff',
'positive_only'
];

View file

@ -19,8 +19,7 @@ export default function calculateLabel(metric, metrics) {
if (metric.alias) return metric.alias;
if (metric.type === 'count') return 'Count';
if (metric.type === 'calculation') return 'Bucket Script';
if (metric.type === 'math') return 'Math';
if (metric.type === 'calculation') return 'Calculation';
if (metric.type === 'series_agg') return `Series Agg (${metric.function})`;
if (metric.type === 'filter_ratio') return 'Filter Ratio';
if (metric.type === 'static') return `Static Value of ${metric.value}`;

View file

@ -1,14 +1,14 @@
import PropTypes from 'prop-types';
import React from 'react';
import StdAgg from './std_agg';
import aggToComponent from '../lib/agg_to_component';
import { sortable } from 'react-anything-sortable';
import { UnsupportedAgg } from './unsupported_agg';
function Agg(props) {
const { model } = props;
let Component = aggToComponent[model.type];
if (!Component) {
Component = StdAgg;
Component = UnsupportedAgg;
}
const style = {
cursor: 'default',

View file

@ -20,12 +20,13 @@ const metricAggs = [
];
const pipelineAggs = [
{ label: 'Bucket Script', value: 'calculation' },
{ label: 'Calculation', value: 'calculation' },
{ label: 'Cumulative Sum', value: 'cumulative_sum' },
{ label: 'Derivative', value: 'derivative' },
{ label: 'Moving Average', value: 'moving_average' },
{ label: 'Positive Only', value: 'positive_only' },
{ label: 'Serial Difference', value: 'serial_diff' },
{ label: 'Series Agg', value: 'series_agg' }
];
const siblingAggs = [
@ -38,11 +39,6 @@ const siblingAggs = [
{ label: 'Overall Variance', value: 'variance_bucket' }
];
const specialAggs = [
{ label: 'Series Agg', value: 'series_agg' },
{ label: 'Math', value: 'math' }
];
class AggSelectOption extends Component {
constructor(props) {
@ -148,9 +144,7 @@ function AggSelect(props) {
{ label: 'Parent Pipeline Aggregations', value: null, pipeline: true, heading: true, disabled: true },
...pipelineAggs.filter(filterByPanelType(panelType)).map(agg => ({ ...agg, disabled: !enablePipelines })),
{ label: 'Sibling Pipeline Aggregations', value: null, pipeline: true, heading: true, disabled: true },
...siblingAggs.map(agg => ({ ...agg, disabled: !enablePipelines })),
{ label: 'Special Aggregations', value: null, pipeline: true, heading: true, disabled: true },
...specialAggs.map(agg => ({ ...agg, disabled: !enablePipelines }))
...siblingAggs.map(agg => ({ ...agg, disabled: !enablePipelines }))
];
}

View file

@ -1,98 +0,0 @@
import React, { Component } from 'react';
import PropTypes from 'prop-types';
import _ from 'lodash';
import uuid from 'uuid';
import AggRow from './agg_row';
import AggSelect from './agg_select';
import createChangeHandler from '../lib/create_change_handler';
import createSelectHandler from '../lib/create_select_handler';
import createTextHandler from '../lib/create_text_handler';
import Vars from './vars';
class MathAgg extends Component {
componentWillMount() {
if (!this.props.model.variables) {
this.props.onChange(_.assign({}, this.props.model, {
variables: [{ id: uuid.v1() }]
}));
}
}
render() {
const { siblings } = this.props;
const defaults = { script: '' };
const model = { ...defaults, ...this.props.model };
const handleChange = createChangeHandler(this.props.onChange, model);
const handleSelectChange = createSelectHandler(handleChange);
const handleTextChange = createTextHandler(handleChange);
return (
<AggRow
disableDelete={this.props.disableDelete}
model={this.props.model}
onAdd={this.props.onAdd}
onDelete={this.props.onDelete}
siblings={this.props.siblings}
>
<div className="vis_editor__row_item">
<div>
<div className="vis_editor__label">Aggregation</div>
<AggSelect
siblings={this.props.siblings}
value={model.type}
onChange={handleSelectChange('type')}
/>
<div className="vis_editor__variables">
<div className="vis_editor__label">Variables</div>
<Vars
metrics={siblings}
onChange={handleChange}
name="variables"
model={model}
includeSiblings={true}
/>
</div>
<div className="vis_editor__row_item">
<label className="vis_editor__label" htmlFor="mathExpressionInput">Expression</label>
<textarea
id="mathExpressionInput"
aria-describedby="mathExpressionDescription"
className="vis_editor__input-grows-100"
onChange={handleTextChange('script')}
>
{model.script}
</textarea>
<div className="vis_editor__note" id="mathExpressionDescription">
This field uses basic math expresions (see <a href="http://mathjs.org/docs/expressions/syntax.html" target="_blank">MathJS</a>) - Variables
are keys on the <code>params</code> object, i.e. <code>params.&lt;name&gt;</code> To access all the data use
<code>params._all.&lt;name&gt;.values</code> for an array of the values and <code>params._all.&lt;name&gt;.timestamps</code>
for an array of the timestamps. <code>params._timestamp</code> is available for the current bucket&apos;s timestamp,
<code>params._index</code> is available for the current bucket&apos;s index, and <code>params._interval</code>s available
for the interval in milliseconds.
</div>
</div>
</div>
</div>
</AggRow>
);
}
}
MathAgg.propTypes = {
disableDelete: PropTypes.bool,
fields: PropTypes.object,
model: PropTypes.object,
onAdd: PropTypes.func,
onChange: PropTypes.func,
onDelete: PropTypes.func,
panel: PropTypes.object,
series: PropTypes.object,
siblings: PropTypes.array,
};
export default MathAgg;

View file

@ -20,14 +20,10 @@ function createTypeFilter(restrict, exclude) {
// This filters out sibling aggs, percentiles, and special aggs (like Series Agg)
export function filterRows(includeSiblings) {
return row => {
if (includeSiblings) return !/^series/.test(row.type) && !/^percentile/.test(row.type) && row.type !== 'math';
return !/_bucket$/.test(row.type)
&& !/^series/.test(row.type)
&& !/^percentile/.test(row.type)
&& row.type !== 'math';
};
export function filterRows(row) {
return !/_bucket$/.test(row.type)
&& !/^series/.test(row.type)
&& !/^percentile/.test(row.type);
}
function MetricSelect(props) {
@ -36,8 +32,7 @@ function MetricSelect(props) {
metric,
onChange,
value,
exclude,
includeSiblings
exclude
} = props;
const metrics = props.metrics
@ -63,7 +58,7 @@ function MetricSelect(props) {
const options = siblings
.filter(filterRows(includeSiblings))
.filter(filterRows)
.map(row => {
const label = calculateLabel(row, metrics);
return { value: row.id, label };
@ -85,7 +80,6 @@ MetricSelect.defaultProps = {
exclude: [],
metric: {},
restrict: 'none',
includeSiblings: false
};
MetricSelect.propTypes = {
@ -94,8 +88,7 @@ MetricSelect.propTypes = {
metric: PropTypes.object,
onChange: PropTypes.func,
restrict: PropTypes.string,
value: PropTypes.string,
includeSiblings: PropTypes.bool
value: PropTypes.string
};
export default MetricSelect;

View file

@ -0,0 +1,18 @@
import AggRow from './agg_row';
import React from 'react';
export function UnsupportedAgg(props) {
return (
<AggRow
disableDelete={props.disableDelete}
model={props.model}
onAdd={props.onAdd}
onDelete={props.onDelete}
siblings={props.siblings}
>
<div className="vis_editor__row_item">
<p>The <code>{props.model.type}</code> aggregation is no longer supported.</p>
</div>
</AggRow>
);
}

View file

@ -42,7 +42,6 @@ class CalculationVars extends Component {
metrics={this.props.metrics}
metric={this.props.model}
value={row.field}
includeSiblings={this.props.includeSiblings}
/>
</div>
<div className="vis_editor__calc_vars-control">
@ -70,16 +69,14 @@ class CalculationVars extends Component {
}
CalculationVars.defaultProps = {
name: 'variables',
includeSiblings: false
name: 'variables'
};
CalculationVars.propTypes = {
metrics: PropTypes.array,
model: PropTypes.object,
name: PropTypes.string,
onChange: PropTypes.func,
includeSiblings: PropTypes.bool
onChange: PropTypes.func
};
export default CalculationVars;

View file

@ -12,7 +12,6 @@ import { PositiveOnlyAgg } from '../aggs/positive_only';
import { FilterRatioAgg } from '../aggs/filter_ratio';
import { PercentileRankAgg } from '../aggs/percentile_rank';
import { Static } from '../aggs/static';
import MathAgg from '../aggs/math';
export default {
count: StdAgg,
avg: StdAgg,
@ -41,8 +40,7 @@ export default {
serial_diff: SerialDiffAgg,
filter_ratio: FilterRatioAgg,
positive_only: PositiveOnlyAgg,
static: Static,
math: MathAgg
static: Static
};

View file

@ -58,11 +58,6 @@
margin: 0 10px 0 0;
}
}
.vis_editor__note {
.vis_editor__label;
font-style: italic;
}
.vis_editor__input {
padding: 8px 10px;
border-radius: @borderRadius;
@ -337,7 +332,6 @@
margin-bottom: 2px;
padding: 10px;
align-items: center;
.vis_editor__note,
.vis_editor__label {
margin-bottom: 5px;
font-size: 12px;

View file

@ -8,8 +8,7 @@ describe('getSplits(resp, panel, series)', () => {
aggregations: {
SERIES: {
timeseries: { buckets: [] },
SIBAGG: { value: 1 },
meta: { bucketSize: 10 }
SIBAGG: { value: 1 }
}
}
};
@ -27,7 +26,6 @@ describe('getSplits(resp, panel, series)', () => {
{
id: 'SERIES',
label: 'Overall Average of Average of cpu',
meta: { bucketSize: 10 },
color: '#FF0000',
timeseries: { buckets: [] },
SIBAGG: { value: 1 }
@ -50,8 +48,7 @@ describe('getSplits(resp, panel, series)', () => {
timeseries: { buckets: [] },
SIBAGG: { value: 2 }
}
],
meta: { bucketSize: 10 }
]
}
}
};
@ -72,7 +69,6 @@ describe('getSplits(resp, panel, series)', () => {
id: 'SERIES:example-01',
key: 'example-01',
label: 'example-01',
meta: { bucketSize: 10 },
color: '#FF0000',
timeseries: { buckets: [] },
SIBAGG: { value: 1 }
@ -81,7 +77,6 @@ describe('getSplits(resp, panel, series)', () => {
id: 'SERIES:example-02',
key: 'example-02',
label: 'example-02',
meta: { bucketSize: 10 },
color: '#FF0000',
timeseries: { buckets: [] },
SIBAGG: { value: 2 }
@ -104,8 +99,7 @@ describe('getSplits(resp, panel, series)', () => {
timeseries: { buckets: [] },
SIBAGG: { value: 2 }
}
],
meta: { bucketSize: 10 }
]
}
}
};
@ -126,7 +120,6 @@ describe('getSplits(resp, panel, series)', () => {
id: 'SERIES:example-01',
key: 'example-01',
label: 'example-01',
meta: { bucketSize: 10 },
color: '#FF0000',
timeseries: { buckets: [] },
SIBAGG: { value: 1 }
@ -135,7 +128,6 @@ describe('getSplits(resp, panel, series)', () => {
id: 'SERIES:example-02',
key: 'example-02',
label: 'example-02',
meta: { bucketSize: 10 },
color: '#930000',
timeseries: { buckets: [] },
SIBAGG: { value: 2 }
@ -154,8 +146,7 @@ describe('getSplits(resp, panel, series)', () => {
'filter-2': {
timeseries: { buckets: [] },
}
},
meta: { bucketSize: 10 }
}
}
}
};
@ -177,7 +168,6 @@ describe('getSplits(resp, panel, series)', () => {
id: 'SERIES:filter-1',
key: 'filter-1',
label: '200s',
meta: { bucketSize: 10 },
color: '#F00',
timeseries: { buckets: [] },
},
@ -185,7 +175,6 @@ describe('getSplits(resp, panel, series)', () => {
id: 'SERIES:filter-2',
key: 'filter-2',
label: '300s',
meta: { bucketSize: 10 },
color: '#0F0',
timeseries: { buckets: [] },
}

View file

@ -5,7 +5,6 @@ import getLastMetric from './get_last_metric';
import getSplitColors from './get_split_colors';
import { formatKey } from './format_key';
export default function getSplits(resp, panel, series) {
const meta = _.get(resp, `aggregations.${series.id}.meta`);
const color = new Color(series.color);
const metric = getLastMetric(series);
if (_.has(resp, `aggregations.${series.id}.buckets`)) {
@ -17,7 +16,6 @@ export default function getSplits(resp, panel, series) {
bucket.id = `${series.id}:${bucket.key}`;
bucket.label = formatKey(bucket.key, series);
bucket.color = panel.type === 'top_n' ? color.hex() : colors.shift();
bucket.meta = meta;
return bucket;
});
}
@ -29,7 +27,6 @@ export default function getSplits(resp, panel, series) {
bucket.key = filter.id;
bucket.color = filter.color;
bucket.label = filter.label || filter.filter || '*';
bucket.meta = meta;
return bucket;
});
}
@ -49,8 +46,7 @@ export default function getSplits(resp, panel, series) {
id: series.id,
label: series.label || calculateLabel(metric, series.metrics),
color: color.hex(),
...mergeObj,
meta
...mergeObj
}
];
}

View file

@ -50,11 +50,6 @@ describe('dateHistogram(req, panel, series)', () => {
}
}
}
},
meta: {
bucketSize: 10,
intervalString: '10s',
timeField: '@timestamp'
}
}
}
@ -81,11 +76,6 @@ describe('dateHistogram(req, panel, series)', () => {
}
}
}
},
meta: {
bucketSize: 10,
intervalString: '10s',
timeField: '@timestamp'
}
}
}
@ -115,11 +105,6 @@ describe('dateHistogram(req, panel, series)', () => {
}
}
}
},
meta: {
bucketSize: 20,
intervalString: '20s',
timeField: 'timestamp'
}
}
}

View file

@ -5,7 +5,7 @@ import { set } from 'lodash';
export default function dateHistogram(req, panel, series) {
return next => doc => {
const { timeField, interval } = getIntervalAndTimefield(panel, series);
const { bucketSize, intervalString } = getBucketSize(req, interval);
const { intervalString } = getBucketSize(req, interval);
const { from, to } = offsetTime(req, series.offset_time);
const { timezone } = req.payload.timerange;
@ -19,11 +19,6 @@ export default function dateHistogram(req, panel, series) {
max: to.valueOf()
}
});
set(doc, `aggs.${series.id}.meta`, {
timeField,
intervalString,
bucketSize
});
return next(doc);
};
}

View file

@ -1,4 +1,4 @@
import { set } from 'lodash';
import _ from 'lodash';
import getBucketSize from '../../helpers/get_bucket_size';
import getIntervalAndTimefield from '../../get_interval_and_timefield';
import getTimerange from '../../helpers/get_timerange';
@ -7,11 +7,11 @@ import { calculateAggRoot } from './calculate_agg_root';
export default function dateHistogram(req, panel) {
return next => doc => {
const { timeField, interval } = getIntervalAndTimefield(panel);
const { bucketSize, intervalString } = getBucketSize(req, interval);
const { intervalString } = getBucketSize(req, interval);
const { from, to } = getTimerange(req);
panel.series.forEach(column => {
const aggRoot = calculateAggRoot(doc, column);
set(doc, `${aggRoot}.timeseries.date_histogram`, {
_.set(doc, `${aggRoot}.timeseries.date_histogram`, {
field: timeField,
interval: intervalString,
min_doc_count: 0,
@ -20,11 +20,6 @@ export default function dateHistogram(req, panel) {
max: to.valueOf()
}
});
set(doc, aggRoot.replace(/\.aggs$/, '.meta'), {
timeField,
intervalString,
bucketSize
});
});
return next(doc);
};

View file

@ -6,7 +6,6 @@ import stdMetric from './std_metric';
import stdSibling from './std_sibling';
import timeShift from './time_shift';
import { dropLastBucket } from './drop_last_bucket';
import { mathAgg } from './math';
export default [
percentile,
@ -14,7 +13,6 @@ export default [
stdDeviationSibling,
stdMetric,
stdSibling,
mathAgg,
seriesAgg,
timeShift,
dropLastBucket

View file

@ -1,110 +0,0 @@
const percentileValueMatch = /\[([0-9\.]+)\]$/;
import { startsWith, flatten, values, first, last } from 'lodash';
import getDefaultDecoration from '../../helpers/get_default_decoration';
import getSiblingAggValue from '../../helpers/get_sibling_agg_value';
import getSplits from '../../helpers/get_splits';
import mapBucket from '../../helpers/map_bucket';
import mathjs from 'mathjs';
const limitedEval = mathjs.eval;
mathjs.import({
'import': function () { throw new Error('Function import is not allowed in your expression.'); },
'createUnit': function () { throw new Error('Function createUnit is not allowed in your expression.'); },
'eval': function () { throw new Error('Function eval is not allowed in your expression.'); },
'parse': function () { throw new Error('Function parse is not allowed in your expression.'); },
'simplify': function () { throw new Error('Function simplify is not allowed in your expression.'); },
'derivative': function () { throw new Error('Function derivative is not allowed in your expression.'); }
}, { override: true });
export function mathAgg(resp, panel, series) {
return next => results => {
const mathMetric = last(series.metrics);
if (mathMetric.type !== 'math') return next(results);
// Filter the results down to only the ones that match the series.id. Sometimes
// there will be data from other series mixed in.
results = results.filter(s => {
if (s.id.split(/:/)[0] === series.id) {
return false;
}
return true;
});
const decoration = getDefaultDecoration(series);
const splits = getSplits(resp, panel, series);
const mathSeries = splits.map((split) => {
if (mathMetric.variables.length) {
// Gather the data for the splits. The data will either be a sibling agg or
// a standard metric/pipeline agg
const splitData = mathMetric.variables.reduce((acc, v) => {
const metric = series.metrics.find(m => startsWith(v.field, m.id));
if (!metric) return acc;
if (/_bucket$/.test(metric.type)) {
acc[v.name] = split.timeseries.buckets.map(bucket => {
return [bucket.key, getSiblingAggValue(split, metric)];
});
} else {
const percentileMatch = v.field.match(percentileValueMatch);
const m = percentileMatch ? { ...metric, percent: percentileMatch[1] } : { ...metric };
acc[v.name] = split.timeseries.buckets.map(mapBucket(m));
}
return acc;
}, {});
// Create an params._all so the users can access the entire series of data
// in the Math.js equation
const all = Object.keys(splitData).reduce((acc, key) => {
acc[key] = {
values: splitData[key].map(x => x[1]),
timestamps: splitData[key].map(x => x[0])
};
return acc;
}, {});
// Get the first var and check that it shows up in the split data otherwise
// we need to return an empty array for the data since we can't opperate
// without the first varaible
const firstVar = first(mathMetric.variables);
if (!splitData[firstVar.name]) {
return {
id: split.id,
label: split.label,
color: split.color,
data: [],
...decoration
};
}
// Use the first var to collect all the timestamps
const timestamps = splitData[firstVar.name].map(r => first(r));
// Map the timestamps to actual data
const data = timestamps.map((ts, index) => {
const params = mathMetric.variables.reduce((acc, v) => {
acc[v.name] = last(splitData[v.name].find(row => row[0] === ts));
return acc;
}, {});
// If some of the values are null, return the timestamp and null, this is
// a safety check for the user
const someNull = values(params).some(v => v == null);
if (someNull) return [ts, null];
// calculate the result based on the user's script and return the value
const result = limitedEval(mathMetric.script, {
params: {
...params,
_index: index,
_timestamp: ts,
_all: all,
_interval: split.meta.bucketSize * 1000
}
});
// if the result is an object (usually when the user is working with maps and functions) flatten the results and return the last value.
if (typeof result === 'object') return [ts, last(flatten(result.valueOf()))];
return [ts, result];
});
return {
id: split.id,
label: split.label,
color: split.color,
data,
...decoration
};
}
});
return next(results.concat(mathSeries));
};
}

View file

@ -2,14 +2,12 @@
import stdMetric from './std_metric';
import stdSibling from './std_sibling';
import seriesAgg from './series_agg';
import { math } from './math';
import { dropLastBucketFn } from './drop_last_bucket';
export default [
// percentile,
stdMetric,
stdSibling,
math,
seriesAgg,
dropLastBucketFn
];

View file

@ -1,8 +0,0 @@
import { mathAgg } from '../series/math';
export function math(bucket, panel, series) {
return next => results => {
const mathFn = mathAgg({ aggregations: bucket }, panel, series);
return mathFn(next)(results);
};
}

View file

@ -99,21 +99,16 @@ describe('buildRequestBody(req)', () => {
filter: {
match_all: {}
},
meta: {
timeField: '@timestamp',
bucketSize: 10,
intervalString: '10s'
},
aggs: {
timeseries: {
date_histogram: {
field: '@timestamp',
interval: '10s',
min_doc_count: 0,
time_zone: 'UTC',
extended_bounds: {
min: 1485463055881,
max: 1485463955881
'aggs': {
'timeseries': {
'date_histogram': {
'field': '@timestamp',
'interval': '10s',
'min_doc_count': 0,
'time_zone': 'UTC',
'extended_bounds': {
'min': 1485463055881,
'max': 1485463955881
}
},
aggs: {