maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Lens] Create managedReference type for formulas (#99729)

Browse source 
* [Lens] Create managedReference type for formulas

* Fix test failures

* Fix i18n types

* Delete managedReference when replacing

* Tests for formula

* Refactoring from code review

Co-authored-by: Joe Reuter <johannes.reuter@elastic.co>
Co-authored-by: Marco Liberati <marco.liberati@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Wylie Conlon 2021-05-14 17:07:21 -04:00 committed by GitHub
parent b95586f0f4
commit 47f4bfc782
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
45 changed files with 3634 additions and 381 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

198
x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx
View file

@ -29,12 +29,7 @@ import { ReactWrapper } from 'enzyme';
import { DragDrop, ChildDragDropProvider } from '../../../drag_drop';
import { fromExpression } from '@kbn/interpreter/common';
import { coreMock } from 'src/core/public/mocks';
import {
DataPublicPluginStart,
esFilters,
IFieldType,
IIndexPattern,
} from '../../../../../../../src/plugins/data/public';
import { esFilters, IFieldType, IIndexPattern } from '../../../../../../../src/plugins/data/public';
import { UiActionsStart } from '../../../../../../../src/plugins/ui_actions/public';
import { uiActionsPluginMock } from '../../../../../../../src/plugins/ui_actions/public/mocks';
import { TriggerContract } from '../../../../../../../src/plugins/ui_actions/public/triggers';
@ -55,6 +50,25 @@ function createCoreStartWithPermissions(newCapabilities = defaultPermissions) {
return core;
}
function getDefaultProps() {
return {
activeDatasourceId: 'mock',
datasourceStates: {},
datasourceMap: {},
framePublicAPI: createMockFramePublicAPI(),
activeVisualizationId: 'vis',
visualizationState: {},
dispatch: () => {},
ExpressionRenderer: createExpressionRendererMock(),
core: createCoreStartWithPermissions(),
plugins: {
uiActions: uiActionsPluginMock.createStartContract(),
data: dataPluginMock.createStartContract(),
},
getSuggestionForField: () => undefined,
};
}
describe('workspace_panel', () => {
let mockVisualization: jest.Mocked<Visualization>;
let mockVisualization2: jest.Mocked<Visualization>;
@ -62,21 +76,18 @@ describe('workspace_panel', () => {
let expressionRendererMock: jest.Mock<React.ReactElement, [ReactExpressionRendererProps]>;
let uiActionsMock: jest.Mocked<UiActionsStart>;
let dataMock: jest.Mocked<DataPublicPluginStart>;
let trigger: jest.Mocked<TriggerContract>;
let instance: ReactWrapper<WorkspacePanelProps>;
beforeEach(() => {
// These are used in specific tests to assert function calls
trigger = ({ exec: jest.fn() } as unknown) as jest.Mocked<TriggerContract>;
uiActionsMock = uiActionsPluginMock.createStartContract();
dataMock = dataPluginMock.createStartContract();
uiActionsMock.getTrigger.mockReturnValue(trigger);
mockVisualization = createMockVisualization();
mockVisualization2 = createMockVisualization();
mockDatasource = createMockDatasource('a');
expressionRendererMock = createExpressionRendererMock();
});
@ -87,23 +98,14 @@ describe('workspace_panel', () => {
it('should render an explanatory text if no visualization is active', () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
datasourceStates={{}}
datasourceMap={{}}
framePublicAPI={createMockFramePublicAPI()}
{...getDefaultProps()}
activeVisualizationId={null}
visualizationMap={{
vis: mockVisualization,
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
expect(instance.find('[data-test-subj="empty-workspace"]')).toHaveLength(2);
expect(instance.find(expressionRendererMock)).toHaveLength(0);
});
@ -111,20 +113,10 @@ describe('workspace_panel', () => {
it('should render an explanatory text if the visualization does not produce an expression', () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
datasourceStates={{}}
datasourceMap={{}}
framePublicAPI={createMockFramePublicAPI()}
activeVisualizationId="vis"
{...getDefaultProps()}
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => null },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
@ -135,20 +127,10 @@ describe('workspace_panel', () => {
it('should render an explanatory text if the datasource does not produce an expression', () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
datasourceStates={{}}
datasourceMap={{}}
framePublicAPI={createMockFramePublicAPI()}
activeVisualizationId="vis"
{...getDefaultProps()}
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
@ -166,7 +148,7 @@ describe('workspace_panel', () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
state: {},
@ -177,16 +159,10 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
@ -204,10 +180,11 @@ describe('workspace_panel', () => {
};
mockDatasource.toExpression.mockReturnValue('datasource');
mockDatasource.getLayers.mockReturnValue(['first']);
const props = getDefaultProps();
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...props}
datasourceStates={{
mock: {
state: {},
@ -218,16 +195,11 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
plugins={{ ...props.plugins, uiActions: uiActionsMock }}
/>
);
@ -251,7 +223,7 @@ describe('workspace_panel', () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
state: {},
@ -262,16 +234,11 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={dispatch}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
@ -298,7 +265,7 @@ describe('workspace_panel', () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
state: {},
@ -314,16 +281,10 @@ describe('workspace_panel', () => {
mock2: mockDatasource2,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
@ -382,7 +343,7 @@ describe('workspace_panel', () => {
await act(async () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
state: {},
@ -393,16 +354,10 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
});
@ -439,7 +394,7 @@ describe('workspace_panel', () => {
await act(async () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
state: {},
@ -450,16 +405,10 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
});
@ -494,7 +443,7 @@ describe('workspace_panel', () => {
};
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
// define a layer with an indexpattern not available
@ -506,16 +455,9 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
@ -532,7 +474,7 @@ describe('workspace_panel', () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
// define a layer with an indexpattern not available
@ -548,16 +490,11 @@ describe('workspace_panel', () => {
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
// Use cannot navigate to the management page
core={createCoreStartWithPermissions({
navLinks: { management: false },
management: { kibana: { indexPatterns: true } },
})}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
@ -575,7 +512,7 @@ describe('workspace_panel', () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
// define a layer with an indexpattern not available
@ -587,20 +524,14 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
// user can go to management, but indexPatterns management is not accessible
core={createCoreStartWithPermissions({
navLinks: { management: true },
management: { kibana: { indexPatterns: false } },
})}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
@ -621,7 +552,7 @@ describe('workspace_panel', () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
state: {},
@ -632,16 +563,9 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
@ -663,7 +587,7 @@ describe('workspace_panel', () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
state: {},
@ -674,16 +598,9 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: mockVisualization,
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
@ -707,7 +624,7 @@ describe('workspace_panel', () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
state: {},
@ -718,16 +635,9 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: mockVisualization,
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
@ -748,7 +658,7 @@ describe('workspace_panel', () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
state: {},
@ -759,16 +669,9 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
@ -787,7 +690,7 @@ describe('workspace_panel', () => {
await act(async () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
state: {},
@ -798,16 +701,10 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
});
@ -832,7 +729,7 @@ describe('workspace_panel', () => {
await act(async () => {
instance = mount(
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
state: {},
@ -843,16 +740,10 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={framePublicAPI}
activeVisualizationId="vis"
visualizationMap={{
vis: { ...mockVisualization, toExpression: () => 'vis' },
}}
visualizationState={{}}
dispatch={() => {}}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={() => undefined}
/>
);
});
@ -900,7 +791,7 @@ describe('workspace_panel', () => {
dropTargetsByOrder={undefined}
>
<WorkspacePanel
activeDatasourceId={'mock'}
{...getDefaultProps()}
datasourceStates={{
mock: {
state: {},
@ -911,16 +802,11 @@ describe('workspace_panel', () => {
mock: mockDatasource,
}}
framePublicAPI={frame}
activeVisualizationId={'vis'}
visualizationMap={{
vis: mockVisualization,
vis2: mockVisualization2,
}}
visualizationState={{}}
dispatch={mockDispatch}
ExpressionRenderer={expressionRendererMock}
core={createCoreStartWithPermissions()}
plugins={{ uiActions: uiActionsMock, data: dataMock }}
getSuggestionForField={mockGetSuggestionForField}
/>
</ChildDragDropProvider>

8
x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx
View file

@ -151,6 +151,7 @@ export function DimensionEditor(props: DimensionEditorProps) {
const possibleOperations = useMemo(() => {
return Object.values(operationDefinitionMap)
.filter(({ hidden }) => !hidden)
.sort((op1, op2) => {
return op1.displayName.localeCompare(op2.displayName);
})
@ -242,6 +243,7 @@ export function DimensionEditor(props: DimensionEditorProps) {
onClick() {
if (
operationDefinitionMap[operationType].input === 'none' ||
operationDefinitionMap[operationType].input === 'managedReference' ||
operationDefinitionMap[operationType].input === 'fullReference'
) {
// Clear invalid state because we are reseting to a valid column
@ -319,7 +321,8 @@ export function DimensionEditor(props: DimensionEditorProps) {
// Need to workout early on the error to decide whether to show this or an help text
const fieldErrorMessage =
(selectedOperationDefinition?.input !== 'fullReference' ||
((selectedOperationDefinition?.input !== 'fullReference' &&
selectedOperationDefinition?.input !== 'managedReference') ||
(incompleteOperation && operationDefinitionMap[incompleteOperation].input === 'field')) &&
getErrorMessage(
selectedColumn,
@ -447,6 +450,7 @@ export function DimensionEditor(props: DimensionEditorProps) {
currentColumn={state.layers[layerId].columns[columnId]}
dateRange={dateRange}
indexPattern={currentIndexPattern}
operationDefinitionMap={operationDefinitionMap}
{...services}
/>
</>
@ -586,7 +590,7 @@ export function DimensionEditor(props: DimensionEditorProps) {
function getErrorMessage(
selectedColumn: IndexPatternColumn | undefined,
incompleteOperation: boolean,
input: 'none' | 'field' | 'fullReference' | undefined,
input: 'none' | 'field' | 'fullReference' | 'managedReference' | undefined,
fieldInvalid: boolean
) {
if (selectedColumn && incompleteOperation) {

97
x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx
View file

@ -25,14 +25,13 @@ import {
import { mountWithIntl as mount, shallowWithIntl as shallow } from '@kbn/test/jest';
import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup, CoreSetup } from 'kibana/public';
import { IStorageWrapper } from 'src/plugins/kibana_utils/public';
import { generateId } from '../../id_generator';
import { IndexPatternPrivateState } from '../types';
import { IndexPatternColumn, replaceColumn } from '../operations';
import { documentField } from '../document_field';
import { OperationMetadata } from '../../types';
import { DateHistogramIndexPatternColumn } from '../operations/definitions/date_histogram';
import { getFieldByNameFactory } from '../pure_helpers';
import { DimensionEditor } from './dimension_editor';
import { AdvancedOptions } from './advanced_options';
import { Filtering } from './filtering';
jest.mock('../loader');
@ -48,6 +47,7 @@ jest.mock('lodash', () => {
debounce: (fn: unknown) => fn,
};
});
jest.mock('../../id_generator');
const fields = [
{
@ -388,6 +388,15 @@ describe('IndexPatternDimensionEditorPanel', () => {
);
});
it('should not display hidden operation types', () => {
wrapper = mount(<IndexPatternDimensionEditorComponent {...defaultProps} />);
const items: EuiListGroupItemProps[] = wrapper.find(EuiListGroup).prop('listItems') || [];
expect(items.find(({ id }) => id === 'math')).toBeUndefined();
expect(items.find(({ id }) => id === 'formula')).toBeUndefined();
});
it('should indicate that reference-based operations are not compatible when they are incomplete', () => {
wrapper = mount(
<IndexPatternDimensionEditorComponent
@ -1072,7 +1081,7 @@ describe('IndexPatternDimensionEditorPanel', () => {
}
it('should not show custom options if time scaling is not available', () => {
wrapper = shallow(
wrapper = mount(
<IndexPatternDimensionEditorComponent
{...getProps({
operationType: 'average',
@ -1080,25 +1089,23 @@ describe('IndexPatternDimensionEditorPanel', () => {
})}
/>
);
wrapper
.find('[data-test-subj="indexPattern-advanced-popover"]')
.hostNodes()
.simulate('click');
expect(
wrapper
.find(DimensionEditor)
.dive()
.find(AdvancedOptions)
.dive()
.find('[data-test-subj="indexPattern-time-scaling-enable"]')
wrapper.find('[data-test-subj="indexPattern-time-scaling-enable"]').hostNodes()
).toHaveLength(0);
});
it('should show custom options if time scaling is available', () => {
wrapper = shallow(<IndexPatternDimensionEditorComponent {...getProps({})} />);
wrapper = mount(<IndexPatternDimensionEditorComponent {...getProps({})} />);
wrapper
.find('[data-test-subj="indexPattern-advanced-popover"]')
.hostNodes()
.simulate('click');
expect(
wrapper
.find(DimensionEditor)
.dive()
.find(AdvancedOptions)
.dive()
.find('[data-test-subj="indexPattern-time-scaling-enable"]')
wrapper.find('[data-test-subj="indexPattern-time-scaling-enable"]').hostNodes()
).toHaveLength(1);
});
@ -1114,14 +1121,15 @@ describe('IndexPatternDimensionEditorPanel', () => {
it('should allow to set time scaling initially', () => {
const props = getProps({});
wrapper = shallow(<IndexPatternDimensionEditorComponent {...props} />);
wrapper = mount(<IndexPatternDimensionEditorComponent {...props} />);
wrapper
.find('[data-test-subj="indexPattern-advanced-popover"]')
.hostNodes()
.simulate('click');
wrapper
.find(DimensionEditor)
.dive()
.find(AdvancedOptions)
.dive()
.find('[data-test-subj="indexPattern-time-scaling-enable"]')
.prop('onClick')!({} as MouseEvent);
.hostNodes()
.simulate('click');
expect(props.setState).toHaveBeenCalledWith(
{
...props.state,
@ -1205,6 +1213,10 @@ describe('IndexPatternDimensionEditorPanel', () => {
it('should allow to change time scaling', () => {
const props = getProps({ timeScale: 's', label: 'Count of records per second' });
wrapper = mount(<IndexPatternDimensionEditorComponent {...props} />);
wrapper
.find('[data-test-subj="indexPattern-advanced-popover"]')
.hostNodes()
.simulate('click');
wrapper
.find('[data-test-subj="indexPattern-time-scaling-unit"]')
.find(EuiSelect)
@ -1321,33 +1333,32 @@ describe('IndexPatternDimensionEditorPanel', () => {
}
it('should not show custom options if time scaling is not available', () => {
wrapper = shallow(
wrapper = mount(
<IndexPatternDimensionEditorComponent
{...getProps({
operationType: 'terms',
sourceField: 'bytes',
params: {
orderDirection: 'asc',
orderBy: { type: 'alphabetical' },
size: 5,
},
})}
/>
);
expect(
wrapper
.find(DimensionEditor)
.dive()
.find(AdvancedOptions)
.dive()
.find('[data-test-subj="indexPattern-filter-by-enable"]')
wrapper.find('[data-test-subj="indexPattern-advanced-popover"]').hostNodes()
).toHaveLength(0);
});
it('should show custom options if filtering is available', () => {
wrapper = shallow(<IndexPatternDimensionEditorComponent {...getProps({})} />);
wrapper = mount(<IndexPatternDimensionEditorComponent {...getProps({})} />);
wrapper
.find('[data-test-subj="indexPattern-advanced-popover"]')
.hostNodes()
.simulate('click');
expect(
wrapper
.find(DimensionEditor)
.dive()
.find(AdvancedOptions)
.dive()
.find('[data-test-subj="indexPattern-filter-by-enable"]')
wrapper.find('[data-test-subj="indexPattern-filter-by-enable"]').hostNodes()
).toHaveLength(1);
});
@ -1364,14 +1375,15 @@ describe('IndexPatternDimensionEditorPanel', () => {
it('should allow to set filter initially', () => {
const props = getProps({});
wrapper = shallow(<IndexPatternDimensionEditorComponent {...props} />);
wrapper = mount(<IndexPatternDimensionEditorComponent {...props} />);
wrapper
.find('[data-test-subj="indexPattern-advanced-popover"]')
.hostNodes()
.simulate('click');
wrapper
.find(DimensionEditor)
.dive()
.find(AdvancedOptions)
.dive()
.find('[data-test-subj="indexPattern-filter-by-enable"]')
.prop('onClick')!({} as MouseEvent);
.hostNodes()
.simulate('click');
expect(props.setState).toHaveBeenCalledWith(
{
...props.state,
@ -1934,6 +1946,7 @@ describe('IndexPatternDimensionEditorPanel', () => {
});
it('should hide the top level field selector when switching from non-reference to reference', () => {
(generateId as jest.Mock).mockReturnValue(`second`);
wrapper = mount(<IndexPatternDimensionEditorComponent {...defaultProps} />);
expect(wrapper.find('ReferenceEditor')).toHaveLength(0);

6
x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts
View file

@ -904,7 +904,7 @@ describe('IndexPatternDimensionEditorPanel', () => {
layers: {
first: {
...testState.layers.first,
columnOrder: ['ref1', 'col1', 'ref1Copy', 'col1Copy'],
columnOrder: ['col1', 'ref1', 'ref1Copy', 'col1Copy'],
columns: {
ref1: testState.layers.first.columns.ref1,
col1: testState.layers.first.columns.col1,
@ -974,7 +974,7 @@ describe('IndexPatternDimensionEditorPanel', () => {
layers: {
first: {
...testState.layers.first,
columnOrder: ['ref1', 'ref2', 'col1', 'ref1Copy', 'ref2Copy', 'col1Copy'],
columnOrder: ['col1', 'ref1', 'ref2', 'ref1Copy', 'col1Copy', 'ref2Copy'],
columns: {
ref1: testState.layers.first.columns.ref1,
ref2: testState.layers.first.columns.ref2,
@ -1061,8 +1061,8 @@ describe('IndexPatternDimensionEditorPanel', () => {
'col1',
'innerRef1Copy',
'ref1Copy',
'ref2Copy',
'col1Copy',
'ref2Copy',
],
columns: {
innerRef1: testState.layers.first.columns.innerRef1,

2
x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/on_drop_handler.ts
View file

@ -114,7 +114,7 @@ function onMoveCompatible(
const modifiedLayer = copyColumn({
layer,
columnId,
targetId: columnId,
sourceColumnId: droppedItem.columnId,
sourceColumn,
shouldDeleteSource,

25
x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.test.tsx
View file

@ -304,6 +304,31 @@ describe('reference editor', () => {
);
});
it('should not display hidden sub-function types', () => {
// This may happen for saved objects after changing the type of a field
wrapper = mount(
<ReferenceEditor
{...getDefaultArgs()}
validation={{
input: ['field', 'fullReference', 'managedReference'],
validateMetadata: (meta: OperationMetadata) => true,
}}
/>
);
const subFunctionSelect = wrapper
.find('[data-test-subj="indexPattern-reference-function"]')
.first();
expect(subFunctionSelect.prop('isInvalid')).toEqual(true);
expect(subFunctionSelect.prop('selectedOptions')).not.toEqual(
expect.arrayContaining([expect.objectContaining({ value: 'math' })])
);
expect(subFunctionSelect.prop('selectedOptions')).not.toEqual(
expect.arrayContaining([expect.objectContaining({ value: 'formula' })])
);
});
it('should hide the function selector when using a field-only selection style', () => {
wrapper = mount(
<ReferenceEditor

8
x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx
View file

@ -41,7 +41,7 @@ const operationPanels = getOperationDisplay();
export interface ReferenceEditorProps {
layer: IndexPatternLayer;
selectionStyle: 'full' | 'field';
selectionStyle: 'full' | 'field' | 'hidden';
validation: RequiredReference;
columnId: string;
updateLayer: (newLayer: IndexPatternLayer) => void;
@ -92,6 +92,7 @@ export function ReferenceEditor(props: ReferenceEditorProps) {
const operationByField: Partial<Record<string, Set<OperationType>>> = {};
const fieldByOperation: Partial<Record<OperationType, Set<string>>> = {};
Object.values(operationDefinitionMap)
.filter(({ hidden }) => !hidden)
.sort((op1, op2) => {
return op1.displayName.localeCompare(op2.displayName);
})
@ -197,6 +198,10 @@ export function ReferenceEditor(props: ReferenceEditorProps) {
return;
}
if (selectionStyle === 'hidden') {
return null;
}
const selectedOption = incompleteOperation
? [functionOptions.find(({ value }) => value === incompleteOperation)!]
: column
@ -340,6 +345,7 @@ export function ReferenceEditor(props: ReferenceEditorProps) {
columnId={columnId}
indexPattern={currentIndexPattern}
dateRange={dateRange}
operationDefinitionMap={operationDefinitionMap}
{...services}
/>
</>

113
x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts
View file

@ -15,7 +15,7 @@ import { Ast } from '@kbn/interpreter/common';
import { chartPluginMock } from '../../../../../src/plugins/charts/public/mocks';
import { getFieldByNameFactory } from './pure_helpers';
import { operationDefinitionMap, getErrorMessages } from './operations';
import { createMockedReferenceOperation } from './operations/mocks';
import { createMockedFullReference } from './operations/mocks';
import { indexPatternFieldEditorPluginMock } from 'src/plugins/index_pattern_field_editor/public/mocks';
import { uiActionsPluginMock } from '../../../../../src/plugins/ui_actions/public/mocks';
@ -289,6 +289,30 @@ describe('IndexPattern Data Source', () => {
expect(indexPatternDatasource.toExpression(state, 'first')).toEqual(null);
});
it('should generate an empty expression when there is a formula without aggs', async () => {
const queryBaseState: IndexPatternBaseState = {
currentIndexPatternId: '1',
layers: {
first: {
indexPatternId: '1',
columnOrder: ['col1'],
columns: {
col1: {
label: 'Formula',
dataType: 'number',
isBucketed: false,
operationType: 'formula',
references: [],
params: {},
},
},
},
},
};
const state = enrichBaseState(queryBaseState);
expect(indexPatternDatasource.toExpression(state, 'first')).toEqual(null);
});
it('should generate an expression for an aggregated query', async () => {
const queryBaseState: IndexPatternBaseState = {
currentIndexPatternId: '1',
@ -817,7 +841,7 @@ describe('IndexPattern Data Source', () => {
describe('references', () => {
beforeEach(() => {
// @ts-expect-error we are inserting an invalid type
operationDefinitionMap.testReference = createMockedReferenceOperation();
operationDefinitionMap.testReference = createMockedFullReference();
// @ts-expect-error we are inserting an invalid type
operationDefinitionMap.testReference.toExpression.mockReturnValue(['mock']);
@ -900,6 +924,91 @@ describe('IndexPattern Data Source', () => {
}),
});
});
it('should topologically sort references', () => {
// This is a real example of count() + count()
const queryBaseState: IndexPatternBaseState = {
currentIndexPatternId: '1',
layers: {
first: {
indexPatternId: '1',
columnOrder: ['date', 'count', 'formula', 'countX0', 'math'],
columns: {
count: {
label: 'count',
dataType: 'number',
operationType: 'count',
isBucketed: false,
scale: 'ratio',
sourceField: 'Records',
customLabel: true,
},
date: {
label: 'timestamp',
dataType: 'date',
operationType: 'date_histogram',
sourceField: 'timestamp',
isBucketed: true,
scale: 'interval',
params: {
interval: 'auto',
},
},
formula: {
label: 'Formula',
dataType: 'number',
operationType: 'formula',
isBucketed: false,
scale: 'ratio',
params: {
formula: 'count() + count()',
isFormulaBroken: false,
},
references: ['math'],
},
countX0: {
label: 'countX0',
dataType: 'number',
operationType: 'count',
isBucketed: false,
scale: 'ratio',
sourceField: 'Records',
customLabel: true,
},
math: {
label: 'math',
dataType: 'number',
operationType: 'math',
isBucketed: false,
scale: 'ratio',
params: {
tinymathAst: {
type: 'function',
name: 'add',
// @ts-expect-error String args are not valid tinymath, but signals something unique to Lens
args: ['countX0', 'count'],
location: {
min: 0,
max: 17,
},
text: 'count() + count()',
},
},
references: ['countX0', 'count'],
customLabel: true,
},
},
},
},
};
const state = enrichBaseState(queryBaseState);
const ast = indexPatternDatasource.toExpression(state, 'first') as Ast;
const chainLength = ast.chain.length;
expect(ast.chain[chainLength - 2].arguments.name).toEqual(['math']);
expect(ast.chain[chainLength - 1].arguments.id).toEqual(['formula']);
});
});
});

4
x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts
View file

@ -12,6 +12,7 @@ const actualMocks = jest.requireActual('../mocks');
jest.spyOn(actualOperations.operationDefinitionMap.date_histogram, 'paramEditor');
jest.spyOn(actualOperations.operationDefinitionMap.terms, 'onOtherColumnChanged');
jest.spyOn(actualHelpers, 'copyColumn');
jest.spyOn(actualHelpers, 'insertOrReplaceColumn');
jest.spyOn(actualHelpers, 'insertNewColumn');
jest.spyOn(actualHelpers, 'replaceColumn');
@ -30,6 +31,7 @@ export const {
} = actualOperations;
export const {
copyColumn,
insertOrReplaceColumn,
insertNewColumn,
replaceColumn,
@ -50,4 +52,4 @@ export const {
export const { adjustTimeScaleLabelSuffix, DEFAULT_TIME_SCALE } = actualTimeScaleUtils;
export const { createMockedReferenceOperation } = actualMocks;
export const { createMockedFullReference } = actualMocks;

8
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx
View file

@ -17,7 +17,7 @@ import {
} from './utils';
import { DEFAULT_TIME_SCALE } from '../../time_scale_utils';
import { OperationDefinition } from '..';
import { getFormatFromPreviousColumn } from '../helpers';
import { getFormatFromPreviousColumn, getFilter } from '../helpers';
const ofName = buildLabelFunction((name?: string) => {
return i18n.translate('xpack.lens.indexPattern.CounterRateOf', {
@ -50,7 +50,7 @@ export const counterRateOperation: OperationDefinition<
selectionStyle: 'field',
requiredReferences: [
{
input: ['field'],
input: ['field', 'managedReference'],
specificOperations: ['max'],
validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed,
},
@ -76,7 +76,7 @@ export const counterRateOperation: OperationDefinition<
toExpression: (layer, columnId) => {
return dateBasedOperationToExpression(layer, columnId, 'lens_counter_rate');
},
buildColumn: ({ referenceIds, previousColumn, layer, indexPattern }) => {
buildColumn: ({ referenceIds, previousColumn, layer, indexPattern }, columnParams) => {
const metric = layer.columns[referenceIds[0]];
const timeScale = previousColumn?.timeScale || DEFAULT_TIME_SCALE;
return {
@ -92,7 +92,7 @@ export const counterRateOperation: OperationDefinition<
scale: 'ratio',
references: referenceIds,
timeScale,
filter: previousColumn?.filter,
filter: getFilter(previousColumn, columnParams),
params: getFormatFromPreviousColumn(previousColumn),
};
},

8
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx
View file

@ -15,7 +15,7 @@ import {
hasDateField,
} from './utils';
import { OperationDefinition } from '..';
import { getFormatFromPreviousColumn } from '../helpers';
import { getFormatFromPreviousColumn, getFilter } from '../helpers';
const ofName = (name?: string) => {
return i18n.translate('xpack.lens.indexPattern.cumulativeSumOf', {
@ -48,7 +48,7 @@ export const cumulativeSumOperation: OperationDefinition<
selectionStyle: 'field',
requiredReferences: [
{
input: ['field'],
input: ['field', 'managedReference'],
specificOperations: ['count', 'sum'],
validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed,
},
@ -73,7 +73,7 @@ export const cumulativeSumOperation: OperationDefinition<
toExpression: (layer, columnId) => {
return dateBasedOperationToExpression(layer, columnId, 'cumulative_sum');
},
buildColumn: ({ referenceIds, previousColumn, layer, indexPattern }) => {
buildColumn: ({ referenceIds, previousColumn, layer, indexPattern }, columnParams) => {
const ref = layer.columns[referenceIds[0]];
return {
label: ofName(
@ -85,7 +85,7 @@ export const cumulativeSumOperation: OperationDefinition<
operationType: 'cumulative_sum',
isBucketed: false,
scale: 'ratio',
filter: previousColumn?.filter,
filter: getFilter(previousColumn, columnParams),
references: referenceIds,
params: getFormatFromPreviousColumn(previousColumn),
};

8
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx
View file

@ -17,7 +17,7 @@ import {
} from './utils';
import { adjustTimeScaleOnOtherColumnChange } from '../../time_scale_utils';
import { OperationDefinition } from '..';
import { getFormatFromPreviousColumn } from '../helpers';
import { getFormatFromPreviousColumn, getFilter } from '../helpers';
const OPERATION_NAME = 'differences';
@ -52,7 +52,7 @@ export const derivativeOperation: OperationDefinition<
selectionStyle: 'full',
requiredReferences: [
{
input: ['field'],
input: ['field', 'managedReference'],
validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed,
},
],
@ -71,7 +71,7 @@ export const derivativeOperation: OperationDefinition<
toExpression: (layer, columnId) => {
return dateBasedOperationToExpression(layer, columnId, 'derivative');
},
buildColumn: ({ referenceIds, previousColumn, layer }) => {
buildColumn: ({ referenceIds, previousColumn, layer }, columnParams) => {
const ref = layer.columns[referenceIds[0]];
return {
label: ofName(ref?.label, previousColumn?.timeScale),
@ -81,7 +81,7 @@ export const derivativeOperation: OperationDefinition<
scale: 'ratio',
references: referenceIds,
timeScale: previousColumn?.timeScale,
filter: previousColumn?.filter,
filter: getFilter(previousColumn, columnParams),
params: getFormatFromPreviousColumn(previousColumn),
};
},

22
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx
View file

@ -19,7 +19,12 @@ import {
hasDateField,
} from './utils';
import { updateColumnParam } from '../../layer_helpers';
import { getFormatFromPreviousColumn, isValidNumber, useDebounceWithOptions } from '../helpers';
import {
getFormatFromPreviousColumn,
isValidNumber,
useDebounceWithOptions,
getFilter,
} from '../helpers';
import { adjustTimeScaleOnOtherColumnChange } from '../../time_scale_utils';
import { HelpPopover, HelpPopoverButton } from '../../../help_popover';
import type { OperationDefinition, ParamEditorProps } from '..';
@ -37,6 +42,8 @@ const ofName = buildLabelFunction((name?: string) => {
});
});
const WINDOW_DEFAULT_VALUE = 5;
export type MovingAverageIndexPatternColumn = FormattedIndexPatternColumn &
ReferenceBasedIndexPatternColumn & {
operationType: 'moving_average';
@ -58,10 +65,11 @@ export const movingAverageOperation: OperationDefinition<
selectionStyle: 'full',
requiredReferences: [
{
input: ['field'],
input: ['field', 'managedReference'],
validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed,
},
],
operationParams: [{ name: 'window', type: 'number', required: true }],
getPossibleOperation: (indexPattern) => {
if (hasDateField(indexPattern)) {
return {
@ -79,8 +87,12 @@ export const movingAverageOperation: OperationDefinition<
window: [(layer.columns[columnId] as MovingAverageIndexPatternColumn).params.window],
});
},
buildColumn: ({ referenceIds, previousColumn, layer }) => {
buildColumn: (
{ referenceIds, previousColumn, layer },
columnParams = { window: WINDOW_DEFAULT_VALUE }
) => {
const metric = layer.columns[referenceIds[0]];
const { window = WINDOW_DEFAULT_VALUE } = columnParams;
return {
label: ofName(metric?.label, previousColumn?.timeScale),
dataType: 'number',
@ -89,9 +101,9 @@ export const movingAverageOperation: OperationDefinition<
scale: 'ratio',
references: referenceIds,
timeScale: previousColumn?.timeScale,
filter: previousColumn?.filter,
filter: getFilter(previousColumn, columnParams),
params: {
window: 5,
window,
...getFormatFromPreviousColumn(previousColumn),
},
};

4
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.test.ts
View file

@ -7,7 +7,7 @@
import { checkReferences } from './utils';
import { operationDefinitionMap } from '..';
import { createMockedReferenceOperation } from '../../mocks';
import { createMockedFullReference } from '../../mocks';
// Mock prevents issue with circular loading
jest.mock('..');
@ -15,7 +15,7 @@ jest.mock('..');
describe('utils', () => {
beforeEach(() => {
// @ts-expect-error test-only operation type
operationDefinitionMap.testReference = createMockedReferenceOperation();
operationDefinitionMap.testReference = createMockedFullReference();
});
describe('checkReferences', () => {

16
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx
View file

@ -11,7 +11,12 @@ import { buildExpressionFunction } from '../../../../../../../src/plugins/expres
import { OperationDefinition } from './index';
import { FormattedIndexPatternColumn, FieldBasedIndexPatternColumn } from './column_types';
import { getFormatFromPreviousColumn, getInvalidFieldMessage, getSafeName } from './helpers';
import {
getFormatFromPreviousColumn,
getInvalidFieldMessage,
getSafeName,
getFilter,
} from './helpers';
const supportedTypes = new Set([
'string',
@ -71,8 +76,13 @@ export const cardinalityOperation: OperationDefinition<CardinalityIndexPatternCo
);
},
filterable: true,
operationParams: [
{ name: 'kql', type: 'string', required: false },
{ name: 'lucene', type: 'string', required: false },
],
getDefaultLabel: (column, indexPattern) => ofName(getSafeName(column.sourceField, indexPattern)),
buildColumn({ field, previousColumn }) {
buildColumn({ field, previousColumn }, columnParams) {
return {
label: ofName(field.displayName),
dataType: 'number',
@ -80,7 +90,7 @@ export const cardinalityOperation: OperationDefinition<CardinalityIndexPatternCo
scale: SCALE,
sourceField: field.name,
isBucketed: IS_BUCKETED,
filter: previousColumn?.filter,
filter: getFilter(previousColumn, columnParams),
params: getFormatFromPreviousColumn(previousColumn),
};
},

6
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx
View file

@ -11,7 +11,7 @@ import { buildExpressionFunction } from '../../../../../../../src/plugins/expres
import { OperationDefinition } from './index';
import { FormattedIndexPatternColumn, FieldBasedIndexPatternColumn } from './column_types';
import { IndexPatternField } from '../../types';
import { getInvalidFieldMessage } from './helpers';
import { getInvalidFieldMessage, getFilter } from './helpers';
import {
adjustTimeScaleLabelSuffix,
adjustTimeScaleOnOtherColumnChange,
@ -52,7 +52,7 @@ export const countOperation: OperationDefinition<CountIndexPatternColumn, 'field
}
},
getDefaultLabel: (column) => adjustTimeScaleLabelSuffix(countLabel, undefined, column.timeScale),
buildColumn({ field, previousColumn }) {
buildColumn({ field, previousColumn }, columnParams) {
return {
label: adjustTimeScaleLabelSuffix(countLabel, undefined, previousColumn?.timeScale),
dataType: 'number',
@ -61,7 +61,7 @@ export const countOperation: OperationDefinition<CountIndexPatternColumn, 'field
scale: 'ratio',
sourceField: field.name,
timeScale: previousColumn?.timeScale,
filter: previousColumn?.filter,
filter: getFilter(previousColumn, columnParams),
params:
previousColumn?.dataType === 'number' &&
previousColumn.params &&

1
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx
View file

@ -97,6 +97,7 @@ const defaultOptions = {
data: dataStart,
http: {} as HttpSetup,
indexPattern: indexPattern1,
operationDefinitionMap: {},
};
describe('date_histogram', () => {

5
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx
View file

@ -58,6 +58,7 @@ export const dateHistogramOperation: OperationDefinition<
}),
input: 'field',
priority: 5, // Highest priority level used
operationParams: [{ name: 'interval', type: 'string', required: false }],
getErrorMessage: (layer, columnId, indexPattern) =>
getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern),
getHelpMessage: (props) => <AutoDateHistogramPopover {...props} />,
@ -75,8 +76,8 @@ export const dateHistogramOperation: OperationDefinition<
}
},
getDefaultLabel: (column, indexPattern) => getSafeName(column.sourceField, indexPattern),
buildColumn({ field }) {
let interval = autoInterval;
buildColumn({ field }, columnParams) {
let interval = columnParams?.interval ?? autoInterval;
let timeZone: string | undefined;
if (field.aggregationRestrictions && field.aggregationRestrictions.date_histogram) {
interval = restrictedInterval(field.aggregationRestrictions) as string;

1
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx
View file

@ -27,6 +27,7 @@ const defaultProps = {
data: dataPluginMock.createStartContract(),
http: {} as HttpSetup,
indexPattern: createMockedIndexPattern(),
operationDefinitionMap: {},
};
// mocking random id generator function

987
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx Normal file
View file

@ -0,0 +1,987 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { createMockedIndexPattern } from '../../../mocks';
import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from '../index';
import { FormulaIndexPatternColumn } from './formula';
import { regenerateLayerFromAst } from './parse';
import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../../types';
import { tinymathFunctions } from './util';
jest.mock('../../layer_helpers', () => {
return {
getColumnOrder: ({ columns }: { columns: Record<string, IndexPatternColumn> }) =>
Object.keys(columns),
};
});
const operationDefinitionMap: Record<string, GenericOperationDefinition> = {
average: ({
input: 'field',
buildColumn: ({ field }: { field: IndexPatternField }) => ({
label: 'avg',
dataType: 'number',
operationType: 'average',
sourceField: field.name,
isBucketed: false,
scale: 'ratio',
timeScale: false,
}),
} as unknown) as GenericOperationDefinition,
terms: { input: 'field' } as GenericOperationDefinition,
sum: { input: 'field' } as GenericOperationDefinition,
last_value: { input: 'field' } as GenericOperationDefinition,
max: { input: 'field' } as GenericOperationDefinition,
count: ({
input: 'field',
filterable: true,
buildColumn: ({ field }: { field: IndexPatternField }) => ({
label: 'avg',
dataType: 'number',
operationType: 'count',
sourceField: field.name,
isBucketed: false,
scale: 'ratio',
timeScale: false,
}),
} as unknown) as GenericOperationDefinition,
derivative: { input: 'fullReference' } as GenericOperationDefinition,
moving_average: ({
input: 'fullReference',
operationParams: [{ name: 'window', type: 'number', required: true }],
buildColumn: ({ references }: { references: string[] }) => ({
label: 'moving_average',
dataType: 'number',
operationType: 'moving_average',
isBucketed: false,
scale: 'ratio',
timeScale: false,
params: { window: 5 },
references,
}),
getErrorMessage: () => ['mock error'],
} as unknown) as GenericOperationDefinition,
cumulative_sum: { input: 'fullReference' } as GenericOperationDefinition,
};
describe('formula', () => {
let layer: IndexPatternLayer;
beforeEach(() => {
layer = {
indexPatternId: '1',
columnOrder: ['col1'],
columns: {
col1: {
label: 'Top value of category',
dataType: 'string',
isBucketed: true,
operationType: 'terms',
params: {
orderBy: { type: 'alphabetical' },
size: 3,
orderDirection: 'asc',
},
sourceField: 'category',
},
},
};
});
describe('buildColumn', () => {
let indexPattern: IndexPattern;
beforeEach(() => {
layer = {
indexPatternId: '1',
columnOrder: ['col1'],
columns: {
col1: {
label: 'Average',
dataType: 'number',
operationType: 'average',
isBucketed: false,
scale: 'ratio',
sourceField: 'bytes',
},
},
};
indexPattern = createMockedIndexPattern();
});
it('should start with an empty formula if no previous column is detected', () => {
expect(
formulaOperation.buildColumn({
layer: {
indexPatternId: '1',
columnOrder: [],
columns: {},
},
indexPattern,
})
).toEqual({
label: 'Formula',
dataType: 'number',
operationType: 'formula',
isBucketed: false,
scale: 'ratio',
params: {},
references: [],
});
});
it('should move into Formula previous operation', () => {
expect(
formulaOperation.buildColumn({
previousColumn: layer.columns.col1,
layer,
indexPattern,
})
).toEqual({
label: 'Formula',
dataType: 'number',
operationType: 'formula',
isBucketed: false,
scale: 'ratio',
params: { isFormulaBroken: false, formula: 'average(bytes)' },
references: [],
});
});
it('it should move over explicit format param if set', () => {
expect(
formulaOperation.buildColumn({
previousColumn: {
...layer.columns.col1,
params: {
format: {
id: 'number',
params: {
decimals: 2,
},
},
},
} as IndexPatternColumn,
layer,
indexPattern,
})
).toEqual({
label: 'Formula',
dataType: 'number',
operationType: 'formula',
isBucketed: false,
scale: 'ratio',
params: {
isFormulaBroken: false,
formula: 'average(bytes)',
format: {
id: 'number',
params: {
decimals: 2,
},
},
},
references: [],
});
});
it('it should move over kql arguments if set', () => {
expect(
formulaOperation.buildColumn({
previousColumn: {
...layer.columns.col1,
filter: {
language: 'kuery',
// Need to test with multiple replaces due to string replace
query: `category.keyword: "Men's Clothing" or category.keyword: "Men's Shoes"`,
},
} as IndexPatternColumn,
layer,
indexPattern,
})
).toEqual({
label: 'Formula',
dataType: 'number',
operationType: 'formula',
isBucketed: false,
scale: 'ratio',
params: {
isFormulaBroken: false,
formula: `average(bytes, kql='category.keyword: "Men\\'s Clothing" or category.keyword: "Men\\'s Shoes"')`,
},
references: [],
});
});
it('it should move over lucene arguments without', () => {
expect(
formulaOperation.buildColumn({
previousColumn: {
...layer.columns.col1,
operationType: 'count',
sourceField: 'Records',
filter: {
language: 'lucene',
query: `*`,
},
} as IndexPatternColumn,
layer,
indexPattern,
})
).toEqual({
label: 'Formula',
dataType: 'number',
operationType: 'formula',
isBucketed: false,
scale: 'ratio',
params: {
isFormulaBroken: false,
formula: `count(lucene='*')`,
},
references: [],
});
});
it('should move over previous operation parameter if set - only numeric', () => {
expect(
formulaOperation.buildColumn(
{
previousColumn: {
label: 'Moving Average',
dataType: 'number',
operationType: 'moving_average',
isBucketed: false,
scale: 'ratio',
references: ['col2'],
timeScale: 'd',
params: { window: 3 },
},
layer: {
indexPatternId: '1',
columnOrder: [],
columns: {
col1: {
label: 'Moving Average',
dataType: 'number',
operationType: 'moving_average',
isBucketed: false,
scale: 'ratio',
references: ['col2'],
timeScale: 'd',
params: { window: 3 },
},
col2: {
dataType: 'number',
isBucketed: false,
label: 'col1X0',
operationType: 'average',
scale: 'ratio',
sourceField: 'bytes',
timeScale: 'd',
},
},
},
indexPattern,
},
{},
operationDefinitionMap
)
).toEqual({
label: 'Formula',
dataType: 'number',
operationType: 'formula',
isBucketed: false,
scale: 'ratio',
params: {
isFormulaBroken: false,
formula: 'moving_average(average(bytes), window=3)',
},
references: [],
});
});
it('should not move previous column configuration if not numeric', () => {
expect(
formulaOperation.buildColumn(
{
previousColumn: {
label: 'Top value of category',
dataType: 'string',
isBucketed: true,
operationType: 'terms',
params: {
orderBy: { type: 'alphabetical' },
size: 3,
orderDirection: 'asc',
},
sourceField: 'category',
},
layer: {
indexPatternId: '1',
columnOrder: [],
columns: {
col1: {
label: 'Top value of category',
dataType: 'string',
isBucketed: true,
operationType: 'terms',
params: {
orderBy: { type: 'alphabetical' },
size: 3,
orderDirection: 'asc',
},
sourceField: 'category',
},
},
},
indexPattern,
},
{},
operationDefinitionMap
)
).toEqual({
label: 'Formula',
dataType: 'number',
operationType: 'formula',
isBucketed: false,
scale: 'ratio',
params: {},
references: [],
});
});
});
describe('regenerateLayerFromAst()', () => {
let indexPattern: IndexPattern;
let currentColumn: FormulaIndexPatternColumn;
function testIsBrokenFormula(formula: string) {
expect(
regenerateLayerFromAst(
formula,
layer,
'col1',
currentColumn,
indexPattern,
operationDefinitionMap
).newLayer
).toEqual({
...layer,
columns: {
...layer.columns,
col1: {
...currentColumn,
params: {
...currentColumn.params,
formula,
isFormulaBroken: true,
},
},
},
});
}
beforeEach(() => {
indexPattern = createMockedIndexPattern();
currentColumn = {
label: 'Formula',
dataType: 'number',
operationType: 'formula',
isBucketed: false,
scale: 'ratio',
params: { formula: '', isFormulaBroken: false },
references: [],
};
});
it('should mutate the layer with new columns for valid formula expressions', () => {
expect(
regenerateLayerFromAst(
'average(bytes)',
layer,
'col1',
currentColumn,
indexPattern,
operationDefinitionMap
).newLayer
).toEqual({
...layer,
columnOrder: ['col1X0', 'col1X1', 'col1'],
columns: {
...layer.columns,
col1: {
...currentColumn,
references: ['col1X1'],
params: {
...currentColumn.params,
formula: 'average(bytes)',
isFormulaBroken: false,
},
},
col1X0: {
customLabel: true,
dataType: 'number',
isBucketed: false,
label: 'col1X0',
operationType: 'average',
scale: 'ratio',
sourceField: 'bytes',
timeScale: false,
},
col1X1: {
customLabel: true,
dataType: 'number',
isBucketed: false,
label: 'col1X1',
operationType: 'math',
params: {
tinymathAst: 'col1X0',
},
references: ['col1X0'],
scale: 'ratio',
},
},
});
});
it('returns no change but error if the formula cannot be parsed', () => {
const formulas = [
'+',
'average((',
'average((bytes)',
'average(bytes) +',
'average(""',
'moving_average(average(bytes), window=)',
'average(bytes) + moving_average(average(bytes), window=)',
];
for (const formula of formulas) {
testIsBrokenFormula(formula);
}
});
it('returns no change but error if field is used with no Lens wrapping operation', () => {
testIsBrokenFormula('bytes');
});
it('returns no change but error if at least one field in the formula is missing', () => {
const formulas = [
'noField',
'average(noField)',
'noField + 1',
'derivative(average(noField))',
'average(bytes) + derivative(average(noField))',
];
for (const formula of formulas) {
testIsBrokenFormula(formula);
}
});
it('returns no change but error if at least one operation in the formula is missing', () => {
const formulas = [
'noFn()',
'noFn(bytes)',
'average(bytes) + noFn()',
'derivative(noFn())',
'noFn() + noFnTwo()',
'noFn(noFnTwo())',
'noFn() + noFnTwo() + 5',
'average(bytes) + derivative(noFn())',
'derivative(average(bytes) + noFn())',
];
for (const formula of formulas) {
testIsBrokenFormula(formula);
}
});
it('returns no change but error if one operation has the wrong first argument', () => {
const formulas = [
'average(7)',
'average()',
'average(average(bytes))',
'average(1 + 2)',
'average(bytes + 5)',
'average(bytes + bytes)',
'derivative(7)',
'derivative(bytes + 7)',
'derivative(bytes + bytes)',
'derivative(bytes + average(bytes))',
'derivative(bytes + 7 + average(bytes))',
];
for (const formula of formulas) {
testIsBrokenFormula(formula);
}
});
it('returns no change but error if an argument is passed to count operation', () => {
const formulas = ['count(7)', 'count("bytes")', 'count(bytes)'];
for (const formula of formulas) {
testIsBrokenFormula(formula);
}
});
it('returns no change but error if a required parameter is not passed to the operation in formula', () => {
const formula = 'moving_average(average(bytes))';
testIsBrokenFormula(formula);
});
it('returns no change but error if a required parameter passed with the wrong type in formula', () => {
const formula = 'moving_average(average(bytes), window="m")';
testIsBrokenFormula(formula);
});
it('returns error if a required parameter is passed multiple time', () => {
const formula = 'moving_average(average(bytes), window=7, window=3)';
testIsBrokenFormula(formula);
});
it('returns error if a math operation has less arguments than required', () => {
const formula = 'pow(5)';
testIsBrokenFormula(formula);
});
it('returns error if a math operation has the wrong argument type', () => {
const formula = 'pow(bytes)';
testIsBrokenFormula(formula);
});
it('returns the locations of each function', () => {
expect(
regenerateLayerFromAst(
'moving_average(average(bytes), window=7) + count()',
layer,
'col1',
currentColumn,
indexPattern,
operationDefinitionMap
).locations
).toEqual({
col1X0: { min: 15, max: 29 },
col1X2: { min: 0, max: 41 },
col1X3: { min: 43, max: 50 },
});
});
});
describe('getErrorMessage', () => {
let indexPattern: IndexPattern;
function getNewLayerWithFormula(formula: string, isBroken = true): IndexPatternLayer {
return {
columns: {
col1: {
label: 'Formula',
dataType: 'number',
operationType: 'formula',
isBucketed: false,
scale: 'ratio',
params: { formula, isFormulaBroken: isBroken },
references: [],
},
},
columnOrder: [],
indexPatternId: '',
};
}
beforeEach(() => {
indexPattern = createMockedIndexPattern();
});
it('returns undefined if count is passed without arguments', () => {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula('count()'),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(undefined);
});
it('returns undefined if count is passed with only a named argument', () => {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula(`count(kql='*')`, false),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(undefined);
});
it('returns a syntax error if the kql argument does not parse', () => {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula(`count(kql='invalid: "')`, false),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual([
`Expected "(", "{", value, whitespace but """ found.
invalid: "
---------^`,
]);
});
it('returns undefined if a field operation is passed with the correct first argument', () => {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula('average(bytes)'),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(undefined);
// note that field names can be wrapped in quotes as well
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula('average("bytes")'),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(undefined);
});
it('returns undefined if a fullReference operation is passed with the correct first argument', () => {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula('derivative(average(bytes))'),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(undefined);
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula('derivative(average("bytes"))'),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(undefined);
});
it('returns undefined if a fullReference operation is passed with the arguments', () => {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula('moving_average(average(bytes), window=7)'),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(undefined);
});
it('returns an error if field is used with no Lens wrapping operation', () => {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula('bytes'),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual([`The field bytes cannot be used without operation`]);
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula('bytes + bytes'),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual([`The operation add does not accept any field as argument`]);
});
it('returns an error if parsing a syntax invalid formula', () => {
const formulas = [
'+',
'average((',
'average((bytes)',
'average(bytes) +',
'average(""',
'moving_average(average(bytes), window=)',
];
for (const formula of formulas) {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula(formula),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual([`The Formula ${formula} cannot be parsed`]);
}
});
it('returns an error if the field is missing', () => {
const formulas = [
'noField',
'average(noField)',
'noField + 1',
'derivative(average(noField))',
];
for (const formula of formulas) {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula(formula),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(['Field noField not found']);
}
});
it('returns an error with plural form correctly handled', () => {
const formulas = ['noField + noField2', 'noField + 1 + noField2'];
for (const formula of formulas) {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula(formula),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(['Fields noField, noField2 not found']);
}
});
it('returns an error if an operation is unknown', () => {
const formulas = ['noFn()', 'noFn(bytes)', 'average(bytes) + noFn()', 'derivative(noFn())'];
for (const formula of formulas) {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula(formula),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(['Operation noFn not found']);
}
const multipleFnFormulas = ['noFn() + noFnTwo()', 'noFn(noFnTwo())'];
for (const formula of multipleFnFormulas) {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula(formula),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(['Operations noFn, noFnTwo not found']);
}
});
it('returns an error if field operation in formula have the wrong first argument', () => {
const formulas = [
'average(7)',
'average()',
'average(average(bytes))',
'average(1 + 2)',
'average(bytes + 5)',
'average(bytes + bytes)',
'derivative(7)',
];
for (const formula of formulas) {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula(formula),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(
// some formulas may contain more errors
expect.arrayContaining([
expect.stringMatching(
`The first argument for ${formula.substring(0, formula.indexOf('('))}`
),
])
);
}
});
it('returns an error if an argument is passed to count() operation', () => {
const formulas = ['count(7)', 'count("bytes")', 'count(bytes)'];
for (const formula of formulas) {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula(formula),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(['The operation count does not accept any field as argument']);
}
});
it('returns an error if an operation with required parameters does not receive them', () => {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula('moving_average(average(bytes))'),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual([
'The operation moving_average in the Formula is missing the following parameters: window',
]);
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula('moving_average(average(bytes), myparam=7)'),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual([
'The operation moving_average in the Formula is missing the following parameters: window',
]);
});
it('returns an error if a parameter is passed to an operation with no parameters', () => {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula('average(bytes, myparam=7)'),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(['The operation average does not accept any parameter']);
});
it('returns an error if the parameter passed to an operation is of the wrong type', () => {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula('moving_average(average(bytes), window="m")'),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual([
'The parameters for the operation moving_average in the Formula are of the wrong type: window',
]);
});
it('returns no error for the demo formula example', () => {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula(`
moving_average(
cumulative_sum(
7 * clamp(sum(bytes), 0, last_value(memory) + max(memory))
), window=10
)
`),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(undefined);
});
it('returns no error if a math operation is passed to fullReference operations', () => {
const formulas = [
'derivative(7+1)',
'derivative(7+average(bytes))',
'moving_average(7+average(bytes), window=7)',
];
for (const formula of formulas) {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula(formula),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(undefined);
}
});
it('returns errors if math operations are used with no arguments', () => {
const formulas = [
'derivative(7+1)',
'derivative(7+average(bytes))',
'moving_average(7+average(bytes), window=7)',
];
for (const formula of formulas) {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula(formula),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual(undefined);
}
});
// there are 4 types of errors for math functions:
// * no argument passed
// * too many arguments passed
// * field passed
// * missing argument
const errors = [
(operation: string) =>
`The first argument for ${operation} should be a operation name. Found ()`,
(operation: string) => `The operation ${operation} has too many arguments`,
(operation: string) => `The operation ${operation} does not accept any field as argument`,
(operation: string) => {
const required = tinymathFunctions[operation].positionalArguments.filter(
({ optional }) => !optional
);
return `The operation ${operation} in the Formula is missing ${
required.length - 1
} arguments: ${required
.slice(1)
.map(({ name }) => name)
.join(', ')}`;
},
];
// we'll try to map all of these here in this test
for (const fn of Object.keys(tinymathFunctions)) {
it(`returns an error for the math functions available: ${fn}`, () => {
const nArgs = tinymathFunctions[fn].positionalArguments;
// start with the first 3 types
const formulas = [
`${fn}()`,
`${fn}(1, 2, 3, 4, 5)`,
// to simplify a bit, add the required number of args by the function filled with the field name
`${fn}(${Array(nArgs.length).fill('bytes').join(', ')})`,
];
// add the fourth check only for those functions with more than 1 arg required
const enableFourthCheck = nArgs.filter(({ optional }) => !optional).length > 1;
if (enableFourthCheck) {
formulas.push(`${fn}(1)`);
}
formulas.forEach((formula, i) => {
expect(
formulaOperation.getErrorMessage!(
getNewLayerWithFormula(formula),
'col1',
indexPattern,
operationDefinitionMap
)
).toEqual([errors[i](fn)]);
});
});
}
});
});

155
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx Normal file
View file

@ -0,0 +1,155 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { i18n } from '@kbn/i18n';
import { OperationDefinition } from '../index';
import { ReferenceBasedIndexPatternColumn } from '../column_types';
import { IndexPattern } from '../../../types';
import { runASTValidation, tryToParse } from './validation';
import { regenerateLayerFromAst } from './parse';
import { generateFormula } from './generate';
const defaultLabel = i18n.translate('xpack.lens.indexPattern.formulaLabel', {
defaultMessage: 'Formula',
});
export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternColumn {
operationType: 'formula';
params: {
formula?: string;
isFormulaBroken?: boolean;
// last value on numeric fields can be formatted
format?: {
id: string;
params?: {
decimals: number;
};
};
};
}
export const formulaOperation: OperationDefinition<
FormulaIndexPatternColumn,
'managedReference'
> = {
type: 'formula',
displayName: defaultLabel,
getDefaultLabel: (column, indexPattern) => defaultLabel,
input: 'managedReference',
hidden: true,
getDisabledStatus(indexPattern: IndexPattern) {
return undefined;
},
getErrorMessage(layer, columnId, indexPattern, operationDefinitionMap) {
const column = layer.columns[columnId] as FormulaIndexPatternColumn;
if (!column.params.formula || !operationDefinitionMap) {
return;
}
const { root, error } = tryToParse(column.params.formula);
if (error || !root) {
return [error!.message];
}
const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap);
return errors.length ? errors.map(({ message }) => message) : undefined;
},
getPossibleOperation() {
return {
dataType: 'number',
isBucketed: false,
scale: 'ratio',
};
},
toExpression: (layer, columnId) => {
const currentColumn = layer.columns[columnId] as FormulaIndexPatternColumn;
const params = currentColumn.params;
// TODO: improve this logic
const useDisplayLabel = currentColumn.label !== defaultLabel;
const label = !params?.isFormulaBroken
? useDisplayLabel
? currentColumn.label
: params?.formula
: '';
return [
{
type: 'function',
function: 'mapColumn',
arguments: {
id: [columnId],
name: [label || ''],
exp: [
{
type: 'expression',
chain: [
{
type: 'function',
function: 'math',
arguments: {
expression: [
currentColumn.references.length ? `"${currentColumn.references[0]}"` : ``,
],
},
},
],
},
],
},
},
];
},
buildColumn({ previousColumn, layer, indexPattern }, _, operationDefinitionMap) {
let previousFormula = '';
if (previousColumn) {
previousFormula = generateFormula(
previousColumn,
layer,
previousFormula,
operationDefinitionMap
);
}
// carry over the format settings from previous operation for seamless transfer
// NOTE: this works only for non-default formatters set in Lens
let prevFormat = {};
if (previousColumn?.params && 'format' in previousColumn.params) {
prevFormat = { format: previousColumn.params.format };
}
return {
label: 'Formula',
dataType: 'number',
operationType: 'formula',
isBucketed: false,
scale: 'ratio',
params: previousFormula
? { formula: previousFormula, isFormulaBroken: false, ...prevFormat }
: { ...prevFormat },
references: [],
};
},
isTransferable: () => {
return true;
},
createCopy(layer, sourceId, targetId, indexPattern, operationDefinitionMap) {
const currentColumn = layer.columns[sourceId] as FormulaIndexPatternColumn;
const tempLayer = {
...layer,
columns: {
...layer.columns,
[targetId]: { ...currentColumn },
},
};
const { newLayer } = regenerateLayerFromAst(
currentColumn.params.formula ?? '',
tempLayer,
targetId,
currentColumn,
indexPattern,
operationDefinitionMap
);
return newLayer;
},
};

90
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts Normal file
View file

@ -0,0 +1,90 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { isObject } from 'lodash';
import { GenericOperationDefinition, IndexPatternColumn } from '../index';
import { ReferenceBasedIndexPatternColumn } from '../column_types';
import { IndexPatternLayer } from '../../../types';
// Just handle two levels for now
type OperationParams = Record<string, string | number | Record<string, string | number>>;
export function getSafeFieldName(fieldName: string | undefined) {
// clean up the "Records" field for now
if (!fieldName || fieldName === 'Records') {
return '';
}
return fieldName;
}
export function generateFormula(
previousColumn: ReferenceBasedIndexPatternColumn | IndexPatternColumn,
layer: IndexPatternLayer,
previousFormula: string,
operationDefinitionMap: Record<string, GenericOperationDefinition> | undefined
) {
if ('references' in previousColumn) {
const metric = layer.columns[previousColumn.references[0]];
if (metric && 'sourceField' in metric && metric.dataType === 'number') {
const fieldName = getSafeFieldName(metric.sourceField);
// TODO need to check the input type from the definition
previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName})`;
}
} else {
if (previousColumn && 'sourceField' in previousColumn && previousColumn.dataType === 'number') {
previousFormula += `${previousColumn.operationType}(${getSafeFieldName(
previousColumn?.sourceField
)}`;
}
}
const formulaNamedArgs = extractParamsForFormula(previousColumn, operationDefinitionMap);
if (formulaNamedArgs.length) {
previousFormula +=
', ' + formulaNamedArgs.map(({ name, value }) => `${name}=${value}`).join(', ');
}
if (previousColumn.filter) {
if (previousColumn.operationType !== 'count') {
previousFormula += ', ';
}
previousFormula +=
(previousColumn.filter.language === 'kuery' ? 'kql=' : 'lucene=') +
`'${previousColumn.filter.query.replace(/'/g, `\\'`)}'`; // replace all
}
if (previousFormula) {
// close the formula at the end
previousFormula += ')';
}
return previousFormula;
}
function extractParamsForFormula(
column: IndexPatternColumn | ReferenceBasedIndexPatternColumn,
operationDefinitionMap: Record<string, GenericOperationDefinition> | undefined
) {
if (!operationDefinitionMap) {
return [];
}
const def = operationDefinitionMap[column.operationType];
if ('operationParams' in def && column.params) {
return (def.operationParams || []).flatMap(({ name, required }) => {
const value = (column.params as OperationParams)![name];
if (isObject(value)) {
return Object.keys(value).map((subName) => ({
name: `${name}-${subName}`,
value: value[subName] as string | number,
required,
}));
}
return {
name,
value,
required,
};
});
}
return [];
}

10
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts Normal file
View file

@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { formulaOperation, FormulaIndexPatternColumn } from './formula';
export { regenerateLayerFromAst } from './parse';
export { mathOperation, MathIndexPatternColumn } from './math';

111
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx Normal file
View file

@ -0,0 +1,111 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { TinymathAST } from '@kbn/tinymath';
import { OperationDefinition } from '../index';
import { ReferenceBasedIndexPatternColumn } from '../column_types';
import { IndexPattern } from '../../../types';
export interface MathIndexPatternColumn extends ReferenceBasedIndexPatternColumn {
operationType: 'math';
params: {
tinymathAst: TinymathAST | string;
// last value on numeric fields can be formatted
format?: {
id: string;
params?: {
decimals: number;
};
};
};
}
export const mathOperation: OperationDefinition<MathIndexPatternColumn, 'managedReference'> = {
type: 'math',
displayName: 'Math',
hidden: true,
getDefaultLabel: (column, indexPattern) => 'Math',
input: 'managedReference',
getDisabledStatus(indexPattern: IndexPattern) {
return undefined;
},
getPossibleOperation() {
return {
dataType: 'number',
isBucketed: false,
scale: 'ratio',
};
},
toExpression: (layer, columnId) => {
const column = layer.columns[columnId] as MathIndexPatternColumn;
return [
{
type: 'function',
function: 'mapColumn',
arguments: {
id: [columnId],
name: [columnId],
exp: [
{
type: 'expression',
chain: [
{
type: 'function',
function: 'math',
arguments: {
expression: [astToString(column.params.tinymathAst)],
onError: ['null'],
},
},
],
},
],
},
},
];
},
buildColumn() {
return {
label: 'Math',
dataType: 'number',
operationType: 'math',
isBucketed: false,
scale: 'ratio',
params: {
tinymathAst: '',
},
references: [],
};
},
isTransferable: (column, newIndexPattern) => {
// TODO has to check all children
return true;
},
createCopy: (layer) => {
return { ...layer };
},
};
function astToString(ast: TinymathAST | string): string | number {
if (typeof ast === 'number') {
return ast;
}
if (typeof ast === 'string') {
// Double quotes around uuids like 1234-5678X2 to avoid ambiguity
return `"${ast}"`;
}
if (ast.type === 'variable') {
return ast.value;
}
if (ast.type === 'namedArgument') {
if (ast.name === 'kql' || ast.name === 'lucene') {
return `${ast.name}='${ast.value}'`;
}
return `${ast.name}=${ast.value}`;
}
return `${ast.name}(${ast.args.map(astToString).join(',')})`;
}

210
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts Normal file
View file

@ -0,0 +1,210 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { isObject } from 'lodash';
import type { TinymathAST, TinymathVariable, TinymathLocation } from '@kbn/tinymath';
import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from '../index';
import { IndexPattern, IndexPatternLayer } from '../../../types';
import { mathOperation } from './math';
import { documentField } from '../../../document_field';
import { runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation';
import { findVariables, getOperationParams, groupArgsByType } from './util';
import { FormulaIndexPatternColumn } from './formula';
import { getColumnOrder } from '../../layer_helpers';
function getManagedId(mainId: string, index: number) {
return `${mainId}X${index}`;
}
function parseAndExtract(
text: string,
layer: IndexPatternLayer,
columnId: string,
indexPattern: IndexPattern,
operationDefinitionMap: Record<string, GenericOperationDefinition>
) {
const { root, error } = tryToParse(text);
if (error || !root) {
return { extracted: [], isValid: false };
}
// before extracting the data run the validation task and throw if invalid
const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap);
if (errors.length) {
return { extracted: [], isValid: false };
}
/*
{ name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] }
*/
const extracted = extractColumns(columnId, operationDefinitionMap, root, layer, indexPattern);
return { extracted, isValid: true };
}
function extractColumns(
idPrefix: string,
operations: Record<string, GenericOperationDefinition>,
ast: TinymathAST,
layer: IndexPatternLayer,
indexPattern: IndexPattern
): Array<{ column: IndexPatternColumn; location?: TinymathLocation }> {
const columns: Array<{ column: IndexPatternColumn; location?: TinymathLocation }> = [];
function parseNode(node: TinymathAST) {
if (typeof node === 'number' || node.type !== 'function') {
// leaf node
return node;
}
const nodeOperation = operations[node.name];
if (!nodeOperation) {
// it's a regular math node
const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array<
number | TinymathVariable
>;
return {
...node,
args: consumedArgs,
};
}
// split the args into types for better TS experience
const { namedArguments, variables, functions } = groupArgsByType(node.args);
// operation node
if (nodeOperation.input === 'field') {
const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v));
// a validation task passed before executing this and checked already there's a field
const field = shouldHaveFieldArgument(node)
? indexPattern.getFieldByName(fieldName.value)!
: documentField;
const mappedParams = getOperationParams(nodeOperation, namedArguments || []);
const newCol = (nodeOperation as OperationDefinition<
IndexPatternColumn,
'field'
>).buildColumn(
{
layer,
indexPattern,
field,
},
mappedParams
);
const newColId = getManagedId(idPrefix, columns.length);
newCol.customLabel = true;
newCol.label = newColId;
columns.push({ column: newCol, location: node.location });
// replace by new column id
return newColId;
}
if (nodeOperation.input === 'fullReference') {
const [referencedOp] = functions;
const consumedParam = parseNode(referencedOp);
const subNodeVariables = consumedParam ? findVariables(consumedParam) : [];
const mathColumn = mathOperation.buildColumn({
layer,
indexPattern,
});
mathColumn.references = subNodeVariables.map(({ value }) => value);
mathColumn.params.tinymathAst = consumedParam!;
columns.push({ column: mathColumn });
mathColumn.customLabel = true;
mathColumn.label = getManagedId(idPrefix, columns.length - 1);
const mappedParams = getOperationParams(nodeOperation, namedArguments || []);
const newCol = (nodeOperation as OperationDefinition<
IndexPatternColumn,
'fullReference'
>).buildColumn(
{
layer,
indexPattern,
referenceIds: [getManagedId(idPrefix, columns.length - 1)],
},
mappedParams
);
const newColId = getManagedId(idPrefix, columns.length);
newCol.customLabel = true;
newCol.label = newColId;
columns.push({ column: newCol, location: node.location });
// replace by new column id
return newColId;
}
}
const root = parseNode(ast);
if (root === undefined) {
return [];
}
const variables = findVariables(root);
const mathColumn = mathOperation.buildColumn({
layer,
indexPattern,
});
mathColumn.references = variables.map(({ value }) => value);
mathColumn.params.tinymathAst = root!;
const newColId = getManagedId(idPrefix, columns.length);
mathColumn.customLabel = true;
mathColumn.label = newColId;
columns.push({ column: mathColumn });
return columns;
}
export function regenerateLayerFromAst(
text: string,
layer: IndexPatternLayer,
columnId: string,
currentColumn: FormulaIndexPatternColumn,
indexPattern: IndexPattern,
operationDefinitionMap: Record<string, GenericOperationDefinition>
) {
const { extracted, isValid } = parseAndExtract(
text,
layer,
columnId,
indexPattern,
operationDefinitionMap
);
const columns = { ...layer.columns };
const locations: Record<string, TinymathLocation> = {};
Object.keys(columns).forEach((k) => {
if (k.startsWith(columnId)) {
delete columns[k];
}
});
extracted.forEach(({ column, location }, index) => {
columns[getManagedId(columnId, index)] = column;
if (location) locations[getManagedId(columnId, index)] = location;
});
columns[columnId] = {
...currentColumn,
params: {
...currentColumn.params,
formula: text,
isFormulaBroken: !isValid,
},
references: !isValid ? [] : [getManagedId(columnId, extracted.length - 1)],
};
return {
newLayer: {
...layer,
columns,
columnOrder: getColumnOrder({
...layer,
columns,
}),
},
locations,
};
}

25
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/types.ts Normal file
View file

@ -0,0 +1,25 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
TinymathAST,
TinymathFunction,
TinymathNamedArgument,
TinymathVariable,
} from 'packages/kbn-tinymath';
export type GroupedNodes = {
[Key in TinymathNamedArgument['type']]: TinymathNamedArgument[];
} &
{
[Key in TinymathVariable['type']]: Array<TinymathVariable | string | number>;
} &
{
[Key in TinymathFunction['type']]: TinymathFunction[];
};
export type TinymathNodeTypes = Exclude<TinymathAST, number>;

317
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts Normal file
View file

@ -0,0 +1,317 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { groupBy, isObject } from 'lodash';
import { i18n } from '@kbn/i18n';
import type {
TinymathAST,
TinymathFunction,
TinymathNamedArgument,
TinymathVariable,
} from 'packages/kbn-tinymath';
import type { OperationDefinition, IndexPatternColumn } from '../index';
import type { GroupedNodes } from './types';
export function groupArgsByType(args: TinymathAST[]) {
const { namedArgument, variable, function: functions } = groupBy<TinymathAST>(
args,
(arg: TinymathAST) => {
return isObject(arg) ? arg.type : 'variable';
}
) as GroupedNodes;
// better naming
return {
namedArguments: namedArgument || [],
variables: variable || [],
functions: functions || [],
};
}
export function getValueOrName(node: TinymathAST) {
if (!isObject(node)) {
return node;
}
if (node.type !== 'function') {
return node.value;
}
return node.name;
}
export function getOperationParams(
operation:
| OperationDefinition<IndexPatternColumn, 'field'>
| OperationDefinition<IndexPatternColumn, 'fullReference'>,
params: TinymathNamedArgument[] = []
): Record<string, string | number> {
const formalArgs: Record<string, string> = (operation.operationParams || []).reduce(
(memo: Record<string, string>, { name, type }) => {
memo[name] = type;
return memo;
},
{}
);
return params.reduce<Record<string, string | number>>((args, { name, value }) => {
if (formalArgs[name]) {
args[name] = value;
}
if (operation.filterable && (name === 'kql' || name === 'lucene')) {
args[name] = value;
}
return args;
}, {});
}
// Todo: i18n everything here
export const tinymathFunctions: Record<
string,
{
positionalArguments: Array<{
name: string;
optional?: boolean;
}>;
// help: React.ReactElement;
// Help is in Markdown format
help: string;
}
> = {
add: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.left', { defaultMessage: 'left' }) },
{ name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) },
],
help: `
Also works with + symbol
Example: ${'`count() + sum(bytes)`'}
Example: ${'`add(count(), 5)`'}
`,
},
subtract: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.left', { defaultMessage: 'left' }) },
{ name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) },
],
help: `
Also works with ${'`-`'} symbol
Example: ${'`subtract(sum(bytes), avg(bytes))`'}
`,
},
multiply: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.left', { defaultMessage: 'left' }) },
{ name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) },
],
help: `
Also works with ${'`*`'} symbol
Example: ${'`multiply(sum(bytes), 2)`'}
`,
},
divide: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.left', { defaultMessage: 'left' }) },
{ name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) },
],
help: `
Also works with ${'`/`'} symbol
Example: ${'`ceil(sum(bytes))`'}
`,
},
abs: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
],
help: `
Absolute value
Example: ${'`abs(sum(bytes))`'}
`,
},
cbrt: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
],
help: `
Cube root of value
Example: ${'`cbrt(sum(bytes))`'}
`,
},
ceil: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
],
help: `
Ceiling of value, rounds up
Example: ${'`ceil(sum(bytes))`'}
`,
},
clamp: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
{ name: i18n.translate('xpack.lens.formula.min', { defaultMessage: 'min' }) },
{ name: i18n.translate('xpack.lens.formula.max', { defaultMessage: 'max' }) },
],
help: `
Limits the value from a minimum to maximum
Example: ${'`ceil(sum(bytes))`'}
`,
},
cube: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
],
help: `
Limits the value from a minimum to maximum
Example: ${'`ceil(sum(bytes))`'}
`,
},
exp: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
],
help: `
Raises <em>e</em> to the nth power.
Example: ${'`exp(sum(bytes))`'}
`,
},
fix: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
],
help: `
For positive values, takes the floor. For negative values, takes the ceiling.
Example: ${'`fix(sum(bytes))`'}
`,
},
floor: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
],
help: `
Round down to nearest integer value
Example: ${'`floor(sum(bytes))`'}
`,
},
log: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
{
name: i18n.translate('xpack.lens.formula.base', { defaultMessage: 'base' }),
optional: true,
},
],
help: `
Logarithm with optional base. The natural base <em>e</em> is used as default.
Example: ${'`log(sum(bytes))`'}
Example: ${'`log(sum(bytes), 2)`'}
`,
},
// TODO: check if this is valid for Tinymath
// log10: {
// positionalArguments: [
// { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
// ],
// help: `
// Base 10 logarithm.
// Example: ${'`log10(sum(bytes))`'}
// `,
// },
mod: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
{
name: i18n.translate('xpack.lens.formula.base', { defaultMessage: 'base' }),
optional: true,
},
],
help: `
Remainder after dividing the function by a number
Example: ${'`mod(sum(bytes), 2)`'}
`,
},
pow: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
{
name: i18n.translate('xpack.lens.formula.base', { defaultMessage: 'base' }),
},
],
help: `
Raises the value to a certain power. The second argument is required
Example: ${'`pow(sum(bytes), 3)`'}
`,
},
round: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
{
name: i18n.translate('xpack.lens.formula.decimals', { defaultMessage: 'decimals' }),
optional: true,
},
],
help: `
Rounds to a specific number of decimal places, default of 0
Example: ${'`round(sum(bytes))`'}
Example: ${'`round(sum(bytes), 2)`'}
`,
},
sqrt: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
],
help: `
Square root of a positive value only
Example: ${'`sqrt(sum(bytes))`'}
`,
},
square: {
positionalArguments: [
{ name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) },
],
help: `
Raise the value to the 2nd power
Example: ${'`square(sum(bytes))`'}
`,
},
};
export function isMathNode(node: TinymathAST) {
return isObject(node) && node.type === 'function' && tinymathFunctions[node.name];
}
export function findMathNodes(root: TinymathAST | string): TinymathFunction[] {
function flattenMathNodes(node: TinymathAST | string): TinymathFunction[] {
if (!isObject(node) || node.type !== 'function' || !isMathNode(node)) {
return [];
}
return [node, ...node.args.flatMap(flattenMathNodes)].filter(Boolean);
}
return flattenMathNodes(root);
}
// traverse a tree and find all string leaves
export function findVariables(node: TinymathAST | string): TinymathVariable[] {
if (typeof node === 'string') {
return [
{
type: 'variable',
value: node,
text: node,
location: { min: 0, max: 0 },
},
];
}
if (node == null) {
return [];
}
if (typeof node === 'number' || node.type === 'namedArgument') {
return [];
}
if (node.type === 'variable') {
// leaf node
return [node];
}
return node.args.flatMap(findVariables);
}

687
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts Normal file
View file

@ -0,0 +1,687 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { isObject } from 'lodash';
import { i18n } from '@kbn/i18n';
import { parse, TinymathLocation } from '@kbn/tinymath';
import type { TinymathAST, TinymathFunction, TinymathNamedArgument } from '@kbn/tinymath';
import { esKuery, esQuery } from '../../../../../../../../src/plugins/data/public';
import {
findMathNodes,
findVariables,
getOperationParams,
getValueOrName,
groupArgsByType,
isMathNode,
tinymathFunctions,
} from './util';
import type { OperationDefinition, IndexPatternColumn, GenericOperationDefinition } from '../index';
import type { IndexPattern, IndexPatternLayer } from '../../../types';
import type { TinymathNodeTypes } from './types';
interface ValidationErrors {
missingField: { message: string; type: { variablesLength: number; variablesList: string } };
missingOperation: {
message: string;
type: { operationLength: number; operationsList: string };
};
missingParameter: {
message: string;
type: { operation: string; params: string };
};
wrongTypeParameter: {
message: string;
type: { operation: string; params: string };
};
wrongFirstArgument: {
message: string;
type: { operation: string; type: string; argument: string | number };
};
cannotAcceptParameter: { message: string; type: { operation: string } };
shouldNotHaveField: { message: string; type: { operation: string } };
tooManyArguments: { message: string; type: { operation: string } };
fieldWithNoOperation: {
message: string;
type: { field: string };
};
failedParsing: { message: string; type: { expression: string } };
duplicateArgument: {
message: string;
type: { operation: string; params: string };
};
missingMathArgument: {
message: string;
type: { operation: string; count: number; params: string };
};
}
type ErrorTypes = keyof ValidationErrors;
type ErrorValues<K extends ErrorTypes> = ValidationErrors[K]['type'];
export interface ErrorWrapper {
message: string;
locations: TinymathLocation[];
severity?: 'error' | 'warning';
}
export function isParsingError(message: string) {
return message.includes('Failed to parse expression');
}
function findFunctionNodes(root: TinymathAST | string): TinymathFunction[] {
function flattenFunctionNodes(node: TinymathAST | string): TinymathFunction[] {
if (!isObject(node) || node.type !== 'function') {
return [];
}
return [node, ...node.args.flatMap(flattenFunctionNodes)].filter(Boolean);
}
return flattenFunctionNodes(root);
}
export function hasInvalidOperations(
node: TinymathAST | string,
operations: Record<string, GenericOperationDefinition>
): { names: string[]; locations: TinymathLocation[] } {
const nodes = findFunctionNodes(node).filter((v) => !isMathNode(v) && !operations[v.name]);
return {
// avoid duplicates
names: Array.from(new Set(nodes.map(({ name }) => name))),
locations: nodes.map(({ location }) => location),
};
}
export const getQueryValidationError = (
query: string,
language: 'kql' | 'lucene',
indexPattern: IndexPattern
): string | undefined => {
try {
if (language === 'kql') {
esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query), indexPattern);
} else {
esQuery.luceneStringToDsl(query);
}
return;
} catch (e) {
return e.message;
}
};
function getMessageFromId<K extends ErrorTypes>({
messageId,
values: { ...values },
locations,
}: {
messageId: K;
values: ErrorValues<K>;
locations: TinymathLocation[];
}): ErrorWrapper {
let message: string;
// Use a less strict type instead of doing a typecast on each message type
const out = (values as unknown) as Record<string, string>;
switch (messageId) {
case 'wrongFirstArgument':
message = i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', {
defaultMessage:
'The first argument for {operation} should be a {type} name. Found {argument}',
values: { operation: out.operation, type: out.type, argument: out.argument },
});
break;
case 'shouldNotHaveField':
message = i18n.translate('xpack.lens.indexPattern.formulaFieldNotRequired', {
defaultMessage: 'The operation {operation} does not accept any field as argument',
values: { operation: out.operation },
});
break;
case 'cannotAcceptParameter':
message = i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', {
defaultMessage: 'The operation {operation} does not accept any parameter',
values: { operation: out.operation },
});
break;
case 'missingParameter':
message = i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', {
defaultMessage:
'The operation {operation} in the Formula is missing the following parameters: {params}',
values: { operation: out.operation, params: out.params },
});
break;
case 'wrongTypeParameter':
message = i18n.translate('xpack.lens.indexPattern.formulaExpressionWrongType', {
defaultMessage:
'The parameters for the operation {operation} in the Formula are of the wrong type: {params}',
values: { operation: out.operation, params: out.params },
});
break;
case 'duplicateArgument':
message = i18n.translate('xpack.lens.indexPattern.formulaOperationDuplicateParams', {
defaultMessage:
'The parameters for the operation {operation} have been declared multiple times: {params}',
values: { operation: out.operation, params: out.params },
});
break;
case 'missingField':
message = i18n.translate('xpack.lens.indexPattern.formulaFieldNotFound', {
defaultMessage:
'{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found',
values: { variablesLength: out.variablesLength, variablesList: out.variablesList },
});
break;
case 'missingOperation':
message = i18n.translate('xpack.lens.indexPattern.operationsNotFound', {
defaultMessage:
'{operationLength, plural, one {Operation} other {Operations}} {operationsList} not found',
values: { operationLength: out.operationLength, operationsList: out.operationsList },
});
break;
case 'fieldWithNoOperation':
message = i18n.translate('xpack.lens.indexPattern.fieldNoOperation', {
defaultMessage: 'The field {field} cannot be used without operation',
values: { field: out.field },
});
break;
case 'failedParsing':
message = i18n.translate('xpack.lens.indexPattern.formulaExpressionParseError', {
defaultMessage: 'The Formula {expression} cannot be parsed',
values: { expression: out.expression },
});
break;
case 'tooManyArguments':
message = i18n.translate('xpack.lens.indexPattern.formulaWithTooManyArguments', {
defaultMessage: 'The operation {operation} has too many arguments',
values: { operation: out.operation },
});
break;
case 'missingMathArgument':
message = i18n.translate('xpack.lens.indexPattern.formulaMathMissingArgument', {
defaultMessage:
'The operation {operation} in the Formula is missing {count} arguments: {params}',
values: { operation: out.operation, count: out.count, params: out.params },
});
break;
// case 'mathRequiresFunction':
// message = i18n.translate('xpack.lens.indexPattern.formulaMathRequiresFunctionLabel', {
// defaultMessage; 'The function {name} requires an Elasticsearch function',
// values: { ...values },
// });
// break;
default:
message = 'no Error found';
break;
}
return { message, locations };
}
export function tryToParse(
formula: string
): { root: TinymathAST; error: null } | { root: null; error: ErrorWrapper } {
let root;
try {
root = parse(formula);
} catch (e) {
return {
root: null,
error: getMessageFromId({
messageId: 'failedParsing',
values: {
expression: formula,
},
locations: [],
}),
};
}
return { root, error: null };
}
export function runASTValidation(
ast: TinymathAST,
layer: IndexPatternLayer,
indexPattern: IndexPattern,
operations: Record<string, GenericOperationDefinition>
) {
return [
...checkMissingVariableOrFunctions(ast, layer, indexPattern, operations),
...runFullASTValidation(ast, layer, indexPattern, operations),
];
}
function checkVariableEdgeCases(ast: TinymathAST, missingVariables: Set<string>) {
const invalidVariableErrors = [];
if (isObject(ast) && ast.type === 'variable' && !missingVariables.has(ast.value)) {
invalidVariableErrors.push(
getMessageFromId({
messageId: 'fieldWithNoOperation',
values: {
field: ast.value,
},
locations: [ast.location],
})
);
}
return invalidVariableErrors;
}
function checkMissingVariableOrFunctions(
ast: TinymathAST,
layer: IndexPatternLayer,
indexPattern: IndexPattern,
operations: Record<string, GenericOperationDefinition>
): ErrorWrapper[] {
const missingErrors: ErrorWrapper[] = [];
const missingOperations = hasInvalidOperations(ast, operations);
if (missingOperations.names.length) {
missingErrors.push(
getMessageFromId({
messageId: 'missingOperation',
values: {
operationLength: missingOperations.names.length,
operationsList: missingOperations.names.join(', '),
},
locations: missingOperations.locations,
})
);
}
const missingVariables = findVariables(ast).filter(
// filter empty string as well?
({ value }) => !indexPattern.getFieldByName(value) && !layer.columns[value]
);
// need to check the arguments here: check only strings for now
if (missingVariables.length) {
missingErrors.push(
getMessageFromId({
messageId: 'missingField',
values: {
variablesLength: missingVariables.length,
variablesList: missingVariables.map(({ value }) => value).join(', '),
},
locations: missingVariables.map(({ location }) => location),
})
);
}
const invalidVariableErrors = checkVariableEdgeCases(
ast,
new Set(missingVariables.map(({ value }) => value))
);
return [...missingErrors, ...invalidVariableErrors];
}
function getQueryValidationErrors(
namedArguments: TinymathNamedArgument[] | undefined,
indexPattern: IndexPattern
): ErrorWrapper[] {
const errors: ErrorWrapper[] = [];
(namedArguments ?? []).forEach((arg) => {
if (arg.name === 'kql' || arg.name === 'lucene') {
const message = getQueryValidationError(arg.value, arg.name, indexPattern);
if (message) {
errors.push({
message,
locations: [arg.location],
});
}
}
});
return errors;
}
function validateNameArguments(
node: TinymathFunction,
nodeOperation:
| OperationDefinition<IndexPatternColumn, 'field'>
| OperationDefinition<IndexPatternColumn, 'fullReference'>,
namedArguments: TinymathNamedArgument[] | undefined,
indexPattern: IndexPattern
) {
const errors = [];
const missingParams = getMissingParams(nodeOperation, namedArguments);
if (missingParams.length) {
errors.push(
getMessageFromId({
messageId: 'missingParameter',
values: {
operation: node.name,
params: missingParams.map(({ name }) => name).join(', '),
},
locations: [node.location],
})
);
}
const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments);
if (wrongTypeParams.length) {
errors.push(
getMessageFromId({
messageId: 'wrongTypeParameter',
values: {
operation: node.name,
params: wrongTypeParams.map(({ name }) => name).join(', '),
},
locations: [node.location],
})
);
}
const duplicateParams = getDuplicateParams(namedArguments);
if (duplicateParams.length) {
errors.push(
getMessageFromId({
messageId: 'duplicateArgument',
values: {
operation: node.name,
params: duplicateParams.join(', '),
},
locations: [node.location],
})
);
}
const queryValidationErrors = getQueryValidationErrors(namedArguments, indexPattern);
if (queryValidationErrors.length) {
errors.push(...queryValidationErrors);
}
return errors;
}
function runFullASTValidation(
ast: TinymathAST,
layer: IndexPatternLayer,
indexPattern: IndexPattern,
operations: Record<string, GenericOperationDefinition>
): ErrorWrapper[] {
const missingVariables = findVariables(ast).filter(
// filter empty string as well?
({ value }) => !indexPattern.getFieldByName(value) && !layer.columns[value]
);
const missingVariablesSet = new Set(missingVariables.map(({ value }) => value));
function validateNode(node: TinymathAST): ErrorWrapper[] {
if (!isObject(node) || node.type !== 'function') {
return [];
}
const nodeOperation = operations[node.name];
const errors: ErrorWrapper[] = [];
const { namedArguments, functions, variables } = groupArgsByType(node.args);
const [firstArg] = node?.args || [];
if (!nodeOperation) {
errors.push(...validateMathNodes(node, missingVariablesSet));
// carry on with the validation for all the functions within the math operation
if (functions?.length) {
return errors.concat(functions.flatMap((fn) => validateNode(fn)));
}
} else {
if (nodeOperation.input === 'field') {
if (shouldHaveFieldArgument(node)) {
if (!isFirstArgumentValidType(firstArg, 'variable')) {
if (isMathNode(firstArg)) {
errors.push(
getMessageFromId({
messageId: 'wrongFirstArgument',
values: {
operation: node.name,
type: 'field',
argument: `math operation`,
},
locations: [node.location],
})
);
} else {
errors.push(
getMessageFromId({
messageId: 'wrongFirstArgument',
values: {
operation: node.name,
type: 'field',
argument: getValueOrName(firstArg),
},
locations: [node.location],
})
);
}
}
} else {
// Named arguments only
if (functions?.length || variables?.length) {
errors.push(
getMessageFromId({
messageId: 'shouldNotHaveField',
values: {
operation: node.name,
},
locations: [node.location],
})
);
}
}
if (!canHaveParams(nodeOperation) && namedArguments.length) {
errors.push(
getMessageFromId({
messageId: 'cannotAcceptParameter',
values: {
operation: node.name,
},
locations: [node.location],
})
);
} else {
const argumentsErrors = validateNameArguments(
node,
nodeOperation,
namedArguments,
indexPattern
);
if (argumentsErrors.length) {
errors.push(...argumentsErrors);
}
}
return errors;
}
if (nodeOperation.input === 'fullReference') {
// What about fn(7 + 1)? We may want to allow that
// In general this should be handled down the Esaggs route rather than here
if (
!isFirstArgumentValidType(firstArg, 'function') ||
(isMathNode(firstArg) && validateMathNodes(firstArg, missingVariablesSet).length)
) {
errors.push(
getMessageFromId({
messageId: 'wrongFirstArgument',
values: {
operation: node.name,
type: 'operation',
argument: getValueOrName(firstArg),
},
locations: [node.location],
})
);
}
if (!canHaveParams(nodeOperation) && namedArguments.length) {
errors.push(
getMessageFromId({
messageId: 'cannotAcceptParameter',
values: {
operation: node.name,
},
locations: [node.location],
})
);
} else {
const argumentsErrors = validateNameArguments(
node,
nodeOperation,
namedArguments,
indexPattern
);
if (argumentsErrors.length) {
errors.push(...argumentsErrors);
}
}
}
return errors.concat(validateNode(functions[0]));
}
return errors;
}
return validateNode(ast);
}
export function canHaveParams(
operation:
| OperationDefinition<IndexPatternColumn, 'field'>
| OperationDefinition<IndexPatternColumn, 'fullReference'>
) {
return Boolean((operation.operationParams || []).length) || operation.filterable;
}
export function getInvalidParams(
operation:
| OperationDefinition<IndexPatternColumn, 'field'>
| OperationDefinition<IndexPatternColumn, 'fullReference'>,
params: TinymathNamedArgument[] = []
) {
return validateParams(operation, params).filter(
({ isMissing, isCorrectType, isRequired }) => (isMissing && isRequired) || !isCorrectType
);
}
export function getMissingParams(
operation:
| OperationDefinition<IndexPatternColumn, 'field'>
| OperationDefinition<IndexPatternColumn, 'fullReference'>,
params: TinymathNamedArgument[] = []
) {
return validateParams(operation, params).filter(
({ isMissing, isRequired }) => isMissing && isRequired
);
}
export function getWrongTypeParams(
operation:
| OperationDefinition<IndexPatternColumn, 'field'>
| OperationDefinition<IndexPatternColumn, 'fullReference'>,
params: TinymathNamedArgument[] = []
) {
return validateParams(operation, params).filter(
({ isCorrectType, isMissing }) => !isCorrectType && !isMissing
);
}
function getDuplicateParams(params: TinymathNamedArgument[] = []) {
const uniqueArgs = Object.create(null);
for (const { name } of params) {
const counter = uniqueArgs[name] || 0;
uniqueArgs[name] = counter + 1;
}
const uniqueNames = Object.keys(uniqueArgs);
if (params.length > uniqueNames.length) {
return uniqueNames.filter((name) => uniqueArgs[name] > 1);
}
return [];
}
export function validateParams(
operation:
| OperationDefinition<IndexPatternColumn, 'field'>
| OperationDefinition<IndexPatternColumn, 'fullReference'>,
params: TinymathNamedArgument[] = []
) {
const paramsObj = getOperationParams(operation, params);
const formalArgs = [...(operation.operationParams ?? [])];
if (operation.filterable) {
formalArgs.push(
{ name: 'kql', type: 'string', required: false },
{ name: 'lucene', type: 'string', required: false }
);
}
return formalArgs.map(({ name, type, required }) => ({
name,
isMissing: !(name in paramsObj),
isCorrectType: typeof paramsObj[name] === type,
isRequired: required,
}));
}
export function shouldHaveFieldArgument(node: TinymathFunction) {
return !['count'].includes(node.name);
}
export function isFirstArgumentValidType(arg: TinymathAST, type: TinymathNodeTypes['type']) {
return isObject(arg) && arg.type === type;
}
export function validateMathNodes(root: TinymathAST, missingVariableSet: Set<string>) {
const mathNodes = findMathNodes(root);
const errors: ErrorWrapper[] = [];
mathNodes.forEach((node: TinymathFunction) => {
const { positionalArguments } = tinymathFunctions[node.name];
if (!node.args.length) {
// we can stop here
return errors.push(
getMessageFromId({
messageId: 'wrongFirstArgument',
values: {
operation: node.name,
type: 'operation',
argument: `()`,
},
locations: [node.location],
})
);
}
if (node.args.length > positionalArguments.length) {
errors.push(
getMessageFromId({
messageId: 'tooManyArguments',
values: {
operation: node.name,
},
locations: [node.location],
})
);
}
// no need to iterate all the arguments, one field is anough to trigger the error
const hasFieldAsArgument = positionalArguments.some((requirements, index) => {
const arg = node.args[index];
if (arg != null && typeof arg !== 'number') {
return arg.type === 'variable' && !missingVariableSet.has(arg.value);
}
});
if (hasFieldAsArgument) {
errors.push(
getMessageFromId({
messageId: 'shouldNotHaveField',
values: {
operation: node.name,
},
locations: [node.location],
})
);
}
const mandatoryArguments = positionalArguments.filter(({ optional }) => !optional);
// if there is only 1 mandatory arg, this is already handled by the wrongFirstArgument check
if (mandatoryArguments.length > 1 && node.args.length < mandatoryArguments.length) {
const missingArgs = positionalArguments.filter(
({ name, optional }, i) => !optional && node.args[i] == null
);
errors.push(
getMessageFromId({
messageId: 'missingMathArgument',
values: {
operation: node.name,
count: mandatoryArguments.length - node.args.length,
params: missingArgs.map(({ name }) => name).join(', '),
},
locations: [node.location],
})
);
}
});
return errors;
}

2
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.test.ts
View file

@ -37,7 +37,7 @@ describe('helpers', () => {
createMockedIndexPattern()
);
expect(messages).toHaveLength(1);
expect(messages![0]).toEqual('Field timestamp was not found');
expect(messages![0]).toEqual('Field timestamp is of the wrong type');
});
it('returns no message if all fields are matching', () => {

52
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx
View file

@ -54,14 +54,37 @@ export function getInvalidFieldMessage(
operationDefinition.getPossibleOperationForField(field) !== undefined
)
);
return isInvalid
? [
i18n.translate('xpack.lens.indexPattern.fieldNotFound', {
defaultMessage: 'Field {invalidField} was not found',
values: { invalidField: sourceField },
const isWrongType = Boolean(
sourceField &&
operationDefinition &&
field &&
!operationDefinition.isTransferable(
column as IndexPatternColumn,
indexPattern,
operationDefinitionMap
)
);
if (isInvalid) {
if (isWrongType) {
return [
i18n.translate('xpack.lens.indexPattern.fieldWrongType', {
defaultMessage: 'Field {invalidField} is of the wrong type',
values: {
invalidField: sourceField,
},
}),
]
: undefined;
];
}
return [
i18n.translate('xpack.lens.indexPattern.fieldNotFound', {
defaultMessage: 'Field {invalidField} was not found',
values: { invalidField: sourceField },
}),
];
}
return undefined;
}
export function getSafeName(name: string, indexPattern: IndexPattern): string {
@ -100,3 +123,18 @@ export function getFormatFromPreviousColumn(previousColumn: IndexPatternColumn |
? { format: previousColumn.params.format }
: undefined;
}
export function getFilter(
previousColumn: IndexPatternColumn | undefined,
columnParams: { kql?: string | undefined; lucene?: string | undefined } | undefined
) {
let filter = previousColumn?.filter;
if (columnParams) {
if ('kql' in columnParams) {
filter = { query: columnParams.kql ?? '', language: 'kuery' };
} else if ('lucene' in columnParams) {
filter = { query: columnParams.lucene ?? '', language: 'lucene' };
}
}
return filter;
}

115
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts
View file

@ -35,6 +35,12 @@ import {
MovingAverageIndexPatternColumn,
} from './calculations';
import { countOperation, CountIndexPatternColumn } from './count';
import {
mathOperation,
MathIndexPatternColumn,
formulaOperation,
FormulaIndexPatternColumn,
} from './formula';
import { lastValueOperation, LastValueIndexPatternColumn } from './last_value';
import { OperationMetadata } from '../../../types';
import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types';
@ -66,7 +72,9 @@ export type IndexPatternColumn =
| CumulativeSumIndexPatternColumn
| CounterRateIndexPatternColumn
| DerivativeIndexPatternColumn
| MovingAverageIndexPatternColumn;
| MovingAverageIndexPatternColumn
| MathIndexPatternColumn
| FormulaIndexPatternColumn;
export type FieldBasedIndexPatternColumn = Extract<IndexPatternColumn, { sourceField: string }>;
@ -115,6 +123,8 @@ const internalOperationDefinitions = [
counterRateOperation,
derivativeOperation,
movingAverageOperation,
mathOperation,
formulaOperation,
];
export { termsOperation } from './terms';
@ -131,6 +141,7 @@ export {
derivativeOperation,
movingAverageOperation,
} from './calculations';
export { formulaOperation } from './formula/formula';
/**
* Properties passed to the operation-specific part of the popover editor
@ -147,6 +158,7 @@ export interface ParamEditorProps<C> {
http: HttpSetup;
dateRange: DateRange;
data: DataPublicPluginStart;
operationDefinitionMap: Record<string, GenericOperationDefinition>;
}
export interface HelpProps<C> {
@ -198,7 +210,11 @@ interface BaseOperationDefinitionProps<C extends BaseIndexPatternColumn> {
* If this function returns false, the column is removed when switching index pattern
* for a layer
*/
isTransferable: (column: C, newIndexPattern: IndexPattern) => boolean;
isTransferable: (
column: C,
newIndexPattern: IndexPattern,
operationDefinitionMap: Record<string, GenericOperationDefinition>
) => boolean;
/**
* Transfering a column to another index pattern. This can be used to
* adjust operation specific settings such as reacting to aggregation restrictions
@ -220,7 +236,8 @@ interface BaseOperationDefinitionProps<C extends BaseIndexPatternColumn> {
getErrorMessage?: (
layer: IndexPatternLayer,
columnId: string,
indexPattern: IndexPattern
indexPattern: IndexPattern,
operationDefinitionMap?: Record<string, GenericOperationDefinition>
) => string[] | undefined;
/*
@ -230,9 +247,18 @@ interface BaseOperationDefinitionProps<C extends BaseIndexPatternColumn> {
* If set to optional, time scaling won't be enabled by default and can be removed.
*/
timeScalingMode?: TimeScalingMode;
/**
* Filterable operations can have a KQL or Lucene query added at the dimension level.
* This flag is used by the formula to assign the kql= and lucene= named arguments and set up
* autocomplete.
*/
filterable?: boolean;
getHelpMessage?: (props: HelpProps<C>) => React.ReactNode;
/*
* Operations can be used as middleware for other operations, hence not shown in the panel UI
*/
hidden?: boolean;
}
interface BaseBuildColumnArgs {
@ -240,15 +266,28 @@ interface BaseBuildColumnArgs {
indexPattern: IndexPattern;
}
interface OperationParam {
name: string;
type: string;
required?: boolean;
}
interface FieldlessOperationDefinition<C extends BaseIndexPatternColumn> {
input: 'none';
/**
* The specification of the arguments used by the operations used for both validation,
* and use from external managed operations
*/
operationParams?: OperationParam[];
/**
* Builds the column object for the given parameters. Should include default p
*/
buildColumn: (
arg: BaseBuildColumnArgs & {
previousColumn?: IndexPatternColumn;
}
},
columnParams?: (IndexPatternColumn & C)['params']
) => C;
/**
* Returns the meta data of the operation if applied. Undefined
@ -270,6 +309,12 @@ interface FieldlessOperationDefinition<C extends BaseIndexPatternColumn> {
interface FieldBasedOperationDefinition<C extends BaseIndexPatternColumn> {
input: 'field';
/**
* The specification of the arguments used by the operations used for both validation,
* and use from external managed operations
*/
operationParams?: OperationParam[];
/**
* Returns the meta data of the operation if applied to the given field. Undefined
* if the field is not applicable to the operation.
@ -282,7 +327,8 @@ interface FieldBasedOperationDefinition<C extends BaseIndexPatternColumn> {
arg: BaseBuildColumnArgs & {
field: IndexPatternField;
previousColumn?: IndexPatternColumn;
}
},
columnParams?: (IndexPatternColumn & C)['params'] & { kql?: string; lucene?: string }
) => C;
/**
* This method will be called if the user changes the field of an operation.
@ -320,7 +366,8 @@ interface FieldBasedOperationDefinition<C extends BaseIndexPatternColumn> {
getErrorMessage: (
layer: IndexPatternLayer,
columnId: string,
indexPattern: IndexPattern
indexPattern: IndexPattern,
operationDefinitionMap?: Record<string, GenericOperationDefinition>
) => string[] | undefined;
}
@ -333,6 +380,7 @@ export interface RequiredReference {
// operation types. The main use case is Cumulative Sum, where we need to only take the
// sum of Count or sum of Sum.
specificOperations?: OperationType[];
multi?: boolean;
}
// Full reference uses one or more reference operations which are visible to the user
@ -345,12 +393,19 @@ interface FullReferenceOperationDefinition<C extends BaseIndexPatternColumn> {
*/
requiredReferences: RequiredReference[];
/**
* The specification of the arguments used by the operations used for both validation,
* and use from external managed operations
*/
operationParams?: OperationParam[];
/**
* The type of UI that is shown in the editor for this function:
* - full: List of sub-functions and fields
* - field: List of fields, selects first operation per field
* - hidden: Do not allow to use operation directly
*/
selectionStyle: 'full' | 'field';
selectionStyle: 'full' | 'field' | 'hidden';
/**
* Builds the column object for the given parameters. Should include default p
@ -359,6 +414,10 @@ interface FullReferenceOperationDefinition<C extends BaseIndexPatternColumn> {
arg: BaseBuildColumnArgs & {
referenceIds: string[];
previousColumn?: IndexPatternColumn;
},
columnParams?: (ReferenceBasedIndexPatternColumn & C)['params'] & {
kql?: string;
lucene?: string;
}
) => ReferenceBasedIndexPatternColumn & C;
/**
@ -376,10 +435,49 @@ interface FullReferenceOperationDefinition<C extends BaseIndexPatternColumn> {
) => ExpressionAstFunction[];
}
interface ManagedReferenceOperationDefinition<C extends BaseIndexPatternColumn> {
input: 'managedReference';
/**
* Builds the column object for the given parameters. Should include default p
*/
buildColumn: (
arg: BaseBuildColumnArgs & {
previousColumn?: IndexPatternColumn | ReferenceBasedIndexPatternColumn;
},
columnParams?: (ReferenceBasedIndexPatternColumn & C)['params'],
operationDefinitionMap?: Record<string, GenericOperationDefinition>
) => ReferenceBasedIndexPatternColumn & C;
/**
* Returns the meta data of the operation if applied. Undefined
* if the operation can't be added with these fields.
*/
getPossibleOperation: () => OperationMetadata | undefined;
/**
* A chain of expression functions which will transform the table
*/
toExpression: (
layer: IndexPatternLayer,
columnId: string,
indexPattern: IndexPattern
) => ExpressionAstFunction[];
/**
* Managed references control the IDs of their inner columns, so we need to be able to copy from the
* root level
*/
createCopy: (
layer: IndexPatternLayer,
sourceColumnId: string,
targetColumnId: string,
indexPattern: IndexPattern,
operationDefinitionMap: Record<string, GenericOperationDefinition>
) => IndexPatternLayer;
}
interface OperationDefinitionMap<C extends BaseIndexPatternColumn> {
field: FieldBasedOperationDefinition<C>;
none: FieldlessOperationDefinition<C>;
fullReference: FullReferenceOperationDefinition<C>;
managedReference: ManagedReferenceOperationDefinition<C>;
}
/**
@ -405,7 +503,8 @@ export type OperationType = typeof internalOperationDefinitions[number]['type'];
export type GenericOperationDefinition =
| OperationDefinition<IndexPatternColumn, 'field'>
| OperationDefinition<IndexPatternColumn, 'none'>
| OperationDefinition<IndexPatternColumn, 'fullReference'>;
| OperationDefinition<IndexPatternColumn, 'fullReference'>
| OperationDefinition<IndexPatternColumn, 'managedReference'>;
/**
* List of all available operation definitions

1
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx
View file

@ -29,6 +29,7 @@ const defaultProps = {
...createMockedIndexPattern(),
hasRestrictions: false,
} as IndexPattern,
operationDefinitionMap: {},
};
describe('last_value', () => {

11
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx
View file

@ -15,7 +15,12 @@ import { FieldBasedIndexPatternColumn } from './column_types';
import { IndexPatternField, IndexPattern } from '../../types';
import { updateColumnParam } from '../layer_helpers';
import { DataType } from '../../../types';
import { getFormatFromPreviousColumn, getInvalidFieldMessage, getSafeName } from './helpers';
import {
getFormatFromPreviousColumn,
getInvalidFieldMessage,
getSafeName,
getFilter,
} from './helpers';
function ofName(name: string) {
return i18n.translate('xpack.lens.indexPattern.lastValueOf', {
@ -141,7 +146,7 @@ export const lastValueOperation: OperationDefinition<LastValueIndexPatternColumn
}
return errorMessages.length ? errorMessages : undefined;
},
buildColumn({ field, previousColumn, indexPattern }) {
buildColumn({ field, previousColumn, indexPattern }, columnParams) {
const sortField = isTimeFieldNameDateField(indexPattern)
? indexPattern.timeFieldName
: indexPattern.fields.find((f) => f.type === 'date')?.name;
@ -161,7 +166,7 @@ export const lastValueOperation: OperationDefinition<LastValueIndexPatternColumn
isBucketed: false,
scale: field.type === 'string' ? 'ordinal' : 'ratio',
sourceField: field.name,
filter: previousColumn?.filter,
filter: getFilter(previousColumn, columnParams),
params: {
sortField,
...getFormatFromPreviousColumn(previousColumn),

16
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx
View file

@ -8,7 +8,12 @@
import { i18n } from '@kbn/i18n';
import { buildExpressionFunction } from '../../../../../../../src/plugins/expressions/public';
import { OperationDefinition } from './index';
import { getFormatFromPreviousColumn, getInvalidFieldMessage, getSafeName } from './helpers';
import {
getFormatFromPreviousColumn,
getInvalidFieldMessage,
getSafeName,
getFilter,
} from './helpers';
import {
FormattedIndexPatternColumn,
FieldBasedIndexPatternColumn,
@ -89,8 +94,8 @@ function buildMetricOperation<T extends MetricColumn<string>>({
: (layer.columns[thisColumnId] as T),
getDefaultLabel: (column, indexPattern, columns) =>
labelLookup(getSafeName(column.sourceField, indexPattern), column),
buildColumn: ({ field, previousColumn }) =>
({
buildColumn: ({ field, previousColumn }, columnParams) => {
return {
label: labelLookup(field.displayName, previousColumn),
dataType: 'number',
operationType: type,
@ -98,9 +103,10 @@ function buildMetricOperation<T extends MetricColumn<string>>({
isBucketed: false,
scale: 'ratio',
timeScale: optionalTimeScaling ? previousColumn?.timeScale : undefined,
filter: previousColumn?.filter,
filter: getFilter(previousColumn, columnParams),
params: getFormatFromPreviousColumn(previousColumn),
} as T),
} as T;
},
onFieldChange: (oldColumn, field) => {
return {
...oldColumn,

39
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx
View file

@ -31,6 +31,7 @@ const defaultProps = {
...createMockedIndexPattern(),
hasRestrictions: false,
} as IndexPattern,
operationDefinitionMap: {},
};
describe('percentile', () => {
@ -178,6 +179,41 @@ describe('percentile', () => {
expect(percentileColumn.params.percentile).toEqual(95);
expect(percentileColumn.label).toEqual('95th percentile of test');
});
it('should create a percentile from formula', () => {
const indexPattern = createMockedIndexPattern();
const bytesField = indexPattern.fields.find(({ name }) => name === 'bytes')!;
bytesField.displayName = 'test';
const percentileColumn = percentileOperation.buildColumn(
{
indexPattern,
field: bytesField,
layer: { columns: {}, columnOrder: [], indexPatternId: '' },
},
{ percentile: 75 }
);
expect(percentileColumn.dataType).toEqual('number');
expect(percentileColumn.params.percentile).toEqual(75);
expect(percentileColumn.label).toEqual('75th percentile of test');
});
it('should create a percentile from formula with filter', () => {
const indexPattern = createMockedIndexPattern();
const bytesField = indexPattern.fields.find(({ name }) => name === 'bytes')!;
bytesField.displayName = 'test';
const percentileColumn = percentileOperation.buildColumn(
{
indexPattern,
field: bytesField,
layer: { columns: {}, columnOrder: [], indexPatternId: '' },
},
{ percentile: 75, kql: 'bytes > 100' }
);
expect(percentileColumn.dataType).toEqual('number');
expect(percentileColumn.params.percentile).toEqual(75);
expect(percentileColumn.filter).toEqual({ language: 'kuery', query: 'bytes > 100' });
expect(percentileColumn.label).toEqual('75th percentile of test');
});
});
describe('isTransferable', () => {
@ -202,7 +238,8 @@ describe('percentile', () => {
percentile: 95,
},
},
indexPattern
indexPattern,
{}
)
).toBeTruthy();
});

9
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx
View file

@ -17,6 +17,7 @@ import {
getSafeName,
isValidNumber,
useDebounceWithOptions,
getFilter,
} from './helpers';
import { FieldBasedIndexPatternColumn } from './column_types';
@ -51,6 +52,7 @@ export const percentileOperation: OperationDefinition<PercentileIndexPatternColu
defaultMessage: 'Percentile',
}),
input: 'field',
operationParams: [{ name: 'percentile', type: 'number', required: false }],
filterable: true,
getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType }) => {
if (supportedFieldTypes.includes(fieldType) && aggregatable && !aggregationRestrictions) {
@ -73,13 +75,14 @@ export const percentileOperation: OperationDefinition<PercentileIndexPatternColu
},
getDefaultLabel: (column, indexPattern, columns) =>
ofName(getSafeName(column.sourceField, indexPattern), column.params.percentile),
buildColumn: ({ field, previousColumn, indexPattern }) => {
buildColumn: ({ field, previousColumn, indexPattern }, columnParams) => {
const existingPercentileParam =
previousColumn?.operationType === 'percentile' &&
previousColumn.params &&
'percentile' in previousColumn.params &&
previousColumn.params.percentile;
const newPercentileParam = existingPercentileParam || DEFAULT_PERCENTILE_VALUE;
const newPercentileParam =
columnParams?.percentile ?? (existingPercentileParam || DEFAULT_PERCENTILE_VALUE);
return {
label: ofName(getSafeName(field.name, indexPattern), newPercentileParam),
dataType: 'number',
@ -87,7 +90,7 @@ export const percentileOperation: OperationDefinition<PercentileIndexPatternColu
sourceField: field.name,
isBucketed: false,
scale: 'ratio',
filter: previousColumn?.filter,
filter: getFilter(previousColumn, columnParams),
params: {
percentile: newPercentileParam,
...getFormatFromPreviousColumn(previousColumn),

1
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx
View file

@ -90,6 +90,7 @@ const defaultOptions = {
{ name: sourceField, type: 'number', displayName: sourceField },
]),
},
operationDefinitionMap: {},
};
describe('ranges', () => {

1
x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx
View file

@ -28,6 +28,7 @@ const defaultProps = {
data: dataPluginMock.createStartContract(),
http: {} as HttpSetup,
indexPattern: createMockedIndexPattern(),
operationDefinitionMap: {},
};
describe('terms', () => {

294
x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts
View file

@ -7,6 +7,7 @@
import type { OperationMetadata } from '../../types';
import {
copyColumn,
insertNewColumn,
replaceColumn,
updateColumnParam,
@ -23,7 +24,7 @@ import type { IndexPattern, IndexPatternLayer } from '../types';
import { documentField } from '../document_field';
import { getFieldByNameFactory } from '../pure_helpers';
import { generateId } from '../../id_generator';
import { createMockedReferenceOperation } from './mocks';
import { createMockedFullReference } from './mocks';
jest.mock('../operations');
jest.mock('../../id_generator');
@ -89,13 +90,126 @@ describe('state_helpers', () => {
(generateId as jest.Mock).mockImplementation(() => `id${++count}`);
// @ts-expect-error we are inserting an invalid type
operationDefinitionMap.testReference = createMockedReferenceOperation();
operationDefinitionMap.testReference = createMockedFullReference();
});
afterEach(() => {
delete operationDefinitionMap.testReference;
});
describe('copyColumn', () => {
it('should recursively modify a formula and update the math ast', () => {
const source = {
dataType: 'number' as const,
isBucketed: false,
label: 'Formula',
operationType: 'formula' as const,
params: {
formula: 'moving_average(sum(bytes), window=5)',
isFormulaBroken: false,
},
references: ['formulaX3'],
};
const math = {
customLabel: true,
dataType: 'number' as const,
isBucketed: false,
label: 'math',
operationType: 'math' as const,
params: { tinymathAst: 'formulaX2' },
references: ['formulaX2'],
};
const sum = {
customLabel: true,
dataType: 'number' as const,
isBucketed: false,
label: 'formulaX0',
operationType: 'sum' as const,
scale: 'ratio' as const,
sourceField: 'bytes',
};
const movingAvg = {
customLabel: true,
dataType: 'number' as const,
isBucketed: false,
label: 'formulaX2',
operationType: 'moving_average' as const,
params: { window: 5 },
references: ['formulaX1'],
};
expect(
copyColumn({
layer: {
indexPatternId: '',
columnOrder: [],
columns: {
source,
formulaX0: sum,
formulaX1: math,
formulaX2: movingAvg,
formulaX3: {
...math,
label: 'formulaX3',
references: ['formulaX2'],
params: { tinymathAst: 'formulaX2' },
},
},
},
targetId: 'copy',
sourceColumn: source,
shouldDeleteSource: false,
indexPattern,
sourceColumnId: 'source',
})
).toEqual({
indexPatternId: '',
columnOrder: [
'source',
'formulaX0',
'formulaX1',
'formulaX2',
'formulaX3',
'copyX0',
'copyX1',
'copyX2',
'copyX3',
'copy',
],
columns: {
source,
formulaX0: sum,
formulaX1: math,
formulaX2: movingAvg,
formulaX3: {
...math,
label: 'formulaX3',
references: ['formulaX2'],
params: { tinymathAst: 'formulaX2' },
},
copy: expect.objectContaining({ ...source, references: ['copyX3'] }),
copyX0: expect.objectContaining({ ...sum, label: 'copyX0' }),
copyX1: expect.objectContaining({
...math,
label: 'copyX1',
references: ['copyX0'],
params: { tinymathAst: 'copyX0' },
}),
copyX2: expect.objectContaining({
...movingAvg,
label: 'copyX2',
references: ['copyX1'],
}),
copyX3: expect.objectContaining({
...math,
label: 'copyX3',
references: ['copyX2'],
params: { tinymathAst: 'copyX2' },
}),
},
});
});
});
describe('insertNewColumn', () => {
it('should throw for invalid operations', () => {
expect(() => {
@ -195,7 +309,7 @@ describe('state_helpers', () => {
).toEqual(expect.objectContaining({ columnOrder: ['col1', 'col2'] }));
});
it('should insert a metric after buckets, but before references', () => {
it('should insert a metric after references', () => {
const layer: IndexPatternLayer = {
indexPatternId: '1',
columnOrder: ['col1'],
@ -231,7 +345,7 @@ describe('state_helpers', () => {
field: documentField,
visualizationGroups: [],
})
).toEqual(expect.objectContaining({ columnOrder: ['col1', 'col2', 'col3'] }));
).toEqual(expect.objectContaining({ columnOrder: ['col1', 'col3', 'col2'] }));
});
it('should insert new buckets at the end of previous buckets', () => {
@ -1074,7 +1188,7 @@ describe('state_helpers', () => {
referenceIds: ['id1'],
})
);
expect(result.columnOrder).toEqual(['id1', 'col1']);
expect(result.columnOrder).toEqual(['col1', 'id1']);
expect(result.columns).toEqual(
expect.objectContaining({
id1: expectedColumn,
@ -1196,7 +1310,7 @@ describe('state_helpers', () => {
op: 'testReference',
});
expect(result.columnOrder).toEqual(['id1', 'col1']);
expect(result.columnOrder).toEqual(['col1', 'id1']);
expect(result.columns).toEqual({
id1: expect.objectContaining({
operationType: 'average',
@ -1426,6 +1540,83 @@ describe('state_helpers', () => {
);
});
it('should transition from managedReference to fullReference by deleting the managedReference', () => {
const math = {
customLabel: true,
dataType: 'number' as const,
isBucketed: false,
label: 'math',
operationType: 'math' as const,
};
const layer: IndexPatternLayer = {
indexPatternId: '',
columnOrder: [],
columns: {
source: {
dataType: 'number' as const,
isBucketed: false,
label: 'Formula',
operationType: 'formula' as const,
params: {
formula: 'moving_average(sum(bytes), window=5)',
isFormulaBroken: false,
},
references: ['formulaX3'],
},
formulaX0: {
customLabel: true,
dataType: 'number' as const,
isBucketed: false,
label: 'formulaX0',
operationType: 'sum' as const,
scale: 'ratio' as const,
sourceField: 'bytes',
},
formulaX1: {
...math,
label: 'formulaX1',
references: ['formulaX0'],
params: { tinymathAst: 'formulaX0' },
},
formulaX2: {
customLabel: true,
dataType: 'number' as const,
isBucketed: false,
label: 'formulaX2',
operationType: 'moving_average' as const,
params: { window: 5 },
references: ['formulaX1'],
},
formulaX3: {
...math,
label: 'formulaX3',
references: ['formulaX2'],
params: { tinymathAst: 'formulaX2' },
},
},
};
expect(
replaceColumn({
layer,
indexPattern,
columnId: 'source',
// @ts-expect-error not statically available
op: 'secondTest',
})
).toEqual(
expect.objectContaining({
columnOrder: ['source'],
columns: {
source: expect.objectContaining({
operationType: 'secondTest',
references: ['id1'],
}),
},
})
);
});
it('should transition by using the field from the previous reference if nothing else works (case new5)', () => {
const layer: IndexPatternLayer = {
indexPatternId: '1',
@ -1459,7 +1650,7 @@ describe('state_helpers', () => {
})
).toEqual(
expect.objectContaining({
columnOrder: ['id1', 'output'],
columnOrder: ['output', 'id1'],
columns: {
id1: expect.objectContaining({
sourceField: 'timestamp',
@ -2051,58 +2242,78 @@ describe('state_helpers', () => {
).toEqual(['col1', 'col3', 'col2']);
});
it('should correctly sort references to other references', () => {
it('does not topologically sort formulas, but keeps the relative order', () => {
expect(
getColumnOrder({
columnOrder: [],
indexPatternId: '',
columnOrder: [],
columns: {
bucket: {
label: 'Top values of category',
dataType: 'string',
count: {
label: 'count',
dataType: 'number',
operationType: 'count',
isBucketed: false,
scale: 'ratio',
sourceField: 'Records',
customLabel: true,
},
date: {
label: 'timestamp',
dataType: 'date',
operationType: 'date_histogram',
sourceField: 'timestamp',
isBucketed: true,
// Private
operationType: 'terms',
sourceField: 'category',
scale: 'interval',
params: {
size: 5,
orderBy: {
type: 'alphabetical',
},
orderDirection: 'asc',
interval: 'auto',
},
},
metric: {
label: 'Average of bytes',
formula: {
label: 'Formula',
dataType: 'number',
operationType: 'formula',
isBucketed: false,
// Private
operationType: 'average',
sourceField: 'bytes',
scale: 'ratio',
params: {
formula: 'count() + count()',
isFormulaBroken: false,
},
references: ['math'],
},
ref2: {
label: 'Ref2',
countX0: {
label: 'countX0',
dataType: 'number',
operationType: 'count',
isBucketed: false,
// @ts-expect-error only for testing
operationType: 'testReference',
references: ['ref1'],
scale: 'ratio',
sourceField: 'Records',
customLabel: true,
},
ref1: {
label: 'Ref',
math: {
label: 'math',
dataType: 'number',
operationType: 'math',
isBucketed: false,
// @ts-expect-error only for testing
operationType: 'testReference',
references: ['bucket'],
scale: 'ratio',
params: {
tinymathAst: {
type: 'function',
name: 'add',
// @ts-expect-error String args are not valid tinymath, but signals something unique to Lens
args: ['countX0', 'count'],
location: {
min: 0,
max: 17,
},
text: 'count() + count()',
},
},
references: ['countX0', 'count'],
customLabel: true,
},
},
})
).toEqual(['bucket', 'metric', 'ref1', 'ref2']);
).toEqual(['date', 'count', 'formula', 'countX0', 'math']);
});
});
@ -2459,7 +2670,8 @@ describe('state_helpers', () => {
},
},
'col1',
indexPattern
indexPattern,
operationDefinitionMap
);
});
});

204
x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts
View file

@ -6,6 +6,7 @@
*/
import _, { partition } from 'lodash';
import { getSortScoreByPriority } from './operations';
import type { OperationMetadata, VisualizationDimensionGroupConfig } from '../../types';
import {
operationDefinitionMap,
@ -15,9 +16,9 @@ import {
RequiredReference,
} from './definitions';
import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../types';
import { getSortScoreByPriority } from './operations';
import { generateId } from '../../id_generator';
import { ReferenceBasedIndexPatternColumn } from './definitions/column_types';
import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './definitions/formula';
interface ColumnChange {
op: OperationType;
@ -32,7 +33,7 @@ interface ColumnChange {
interface ColumnCopy {
layer: IndexPatternLayer;
columnId: string;
targetId: string;
sourceColumn: IndexPatternColumn;
sourceColumnId: string;
indexPattern: IndexPattern;
@ -41,16 +42,19 @@ interface ColumnCopy {
export function copyColumn({
layer,
columnId,
targetId,
sourceColumn,
shouldDeleteSource,
indexPattern,
sourceColumnId,
}: ColumnCopy): IndexPatternLayer {
let modifiedLayer = {
...layer,
columns: copyReferencesRecursively(layer.columns, sourceColumn, columnId),
};
let modifiedLayer = copyReferencesRecursively(
layer,
sourceColumn,
sourceColumnId,
targetId,
indexPattern
);
if (shouldDeleteSource) {
modifiedLayer = deleteColumn({
@ -64,16 +68,25 @@ export function copyColumn({
}
function copyReferencesRecursively(
columns: Record<string, IndexPatternColumn>,
layer: IndexPatternLayer,
sourceColumn: IndexPatternColumn,
columnId: string
) {
sourceId: string,
targetId: string,
indexPattern: IndexPattern
): IndexPatternLayer {
let columns = { ...layer.columns };
if ('references' in sourceColumn) {
if (columns[columnId]) {
return columns;
if (columns[targetId]) {
return layer;
}
const def = operationDefinitionMap[sourceColumn.operationType];
if ('createCopy' in def) {
// Allow managed references to recursively insert new columns
return def.createCopy(layer, sourceId, targetId, indexPattern, operationDefinitionMap);
}
sourceColumn?.references.forEach((ref, index) => {
// TODO: Add an option to assign IDs without generating the new one
const newId = generateId();
const refColumn = { ...columns[ref] };
@ -82,10 +95,10 @@ function copyReferencesRecursively(
// and visible columns shouldn't be copied
const refColumnWithInnerRefs =
'references' in refColumn
? copyReferencesRecursively(columns, refColumn, newId) // if a column has references, copy them too
? copyReferencesRecursively(layer, refColumn, sourceId, newId, indexPattern).columns // if a column has references, copy them too
: { [newId]: refColumn };
const newColumn = columns[columnId];
const newColumn = columns[targetId];
let references = [newId];
if (newColumn && 'references' in newColumn) {
references = newColumn.references;
@ -95,7 +108,7 @@ function copyReferencesRecursively(
columns = {
...columns,
...refColumnWithInnerRefs,
[columnId]: {
[targetId]: {
...sourceColumn,
references,
},
@ -104,10 +117,11 @@ function copyReferencesRecursively(
} else {
columns = {
...columns,
[columnId]: sourceColumn,
[targetId]: sourceColumn,
};
}
return columns;
return { ...layer, columns, columnOrder: getColumnOrder({ ...layer, columns }) };
}
export function insertOrReplaceColumn(args: ColumnChange): IndexPatternLayer {
@ -141,12 +155,12 @@ export function insertNewColumn({
const baseOptions = { indexPattern, previousColumn: layer.columns[columnId] };
if (operationDefinition.input === 'none') {
if (operationDefinition.input === 'none' || operationDefinition.input === 'managedReference') {
if (field) {
throw new Error(`Can't create operation ${op} with the provided field ${field.name}`);
}
const possibleOperation = operationDefinition.getPossibleOperation();
const isBucketed = Boolean(possibleOperation.isBucketed);
const isBucketed = Boolean(possibleOperation?.isBucketed);
const addOperationFn = isBucketed ? addBucket : addMetric;
return updateDefaultLabels(
addOperationFn(
@ -333,6 +347,19 @@ export function replaceColumn({
tempLayer = resetIncomplete(tempLayer, columnId);
if (previousDefinition.input === 'managedReference') {
// Every transition away from a managedReference resets it, we don't have a way to keep the state
tempLayer = deleteColumn({ layer: tempLayer, columnId, indexPattern });
return insertNewColumn({
layer: tempLayer,
columnId,
indexPattern,
op,
field,
visualizationGroups,
});
}
if (operationDefinition.input === 'fullReference') {
return applyReferenceTransition({
layer: tempLayer,
@ -395,6 +422,54 @@ export function replaceColumn({
}
}
// TODO: Refactor all this to be more generic and know less about Formula
// if managed it has to look at the full picture to have a seamless transition
if (operationDefinition.input === 'managedReference') {
const newColumn = copyCustomLabel(
operationDefinition.buildColumn(
{ ...baseOptions, layer: tempLayer },
previousColumn.params,
operationDefinitionMap
),
previousColumn
) as FormulaIndexPatternColumn;
// now remove the previous references
if (previousDefinition.input === 'fullReference') {
(previousColumn as ReferenceBasedIndexPatternColumn).references.forEach((id: string) => {
tempLayer = deleteColumn({ layer: tempLayer, columnId: id, indexPattern });
});
}
const basicLayer = { ...tempLayer, columns: { ...tempLayer.columns, [columnId]: newColumn } };
// rebuild the references again for the specific AST generated
let newLayer;
try {
newLayer = newColumn.params.formula
? regenerateLayerFromAst(
newColumn.params.formula,
basicLayer,
columnId,
newColumn,
indexPattern,
operationDefinitionMap
).newLayer
: basicLayer;
} catch (e) {
newLayer = basicLayer;
}
return updateDefaultLabels(
{
...tempLayer,
columnOrder: getColumnOrder(newLayer),
columns: adjustColumnReferencesForChangedColumn(newLayer, columnId),
},
indexPattern
);
}
// This logic comes after the transitions because they need to look at previous columns
if (previousDefinition.input === 'fullReference') {
(previousColumn as ReferenceBasedIndexPatternColumn).references.forEach((id: string) => {
@ -976,8 +1051,12 @@ export function deleteColumn({
);
}
// Derives column order from column object, respects existing columnOrder
// when possible, but also allows new columns to be added to the order
// Column order mostly affects the visual order in the UI. It is derived
// from the columns objects, respecting any existing columnOrder relationships,
// but allowing new columns to be inserted
//
// This does NOT topologically sort references, as this would cause the order in the UI
// to change. Reference order is determined before creating the pipeline in to_expression
export function getColumnOrder(layer: IndexPatternLayer): string[] {
const entries = Object.entries(layer.columns);
entries.sort(([idA], [idB]) => {
@ -992,16 +1071,6 @@ export function getColumnOrder(layer: IndexPatternLayer): string[] {
}
});
// If a reference has another reference as input, put it last in sort order
entries.sort(([idA, a], [idB, b]) => {
if ('references' in a && a.references.includes(idB)) {
return 1;
}
if ('references' in b && b.references.includes(idA)) {
return -1;
}
return 0;
});
const [aggregations, metrics] = _.partition(entries, ([, col]) => col.isBucketed);
return aggregations.map(([id]) => id).concat(metrics.map(([id]) => id));
@ -1019,8 +1088,22 @@ export function getExistingColumnGroups(layer: IndexPatternLayer): [string[], st
/**
* Returns true if the given column can be applied to the given index pattern
*/
export function isColumnTransferable(column: IndexPatternColumn, newIndexPattern: IndexPattern) {
return operationDefinitionMap[column.operationType].isTransferable(column, newIndexPattern);
export function isColumnTransferable(
column: IndexPatternColumn,
newIndexPattern: IndexPattern,
layer: IndexPatternLayer
): boolean {
return (
operationDefinitionMap[column.operationType].isTransferable(
column,
newIndexPattern,
operationDefinitionMap
) &&
(!('references' in column) ||
column.references.every((columnId) =>
isColumnTransferable(layer.columns[columnId], newIndexPattern, layer)
))
);
}
export function updateLayerIndexPattern(
@ -1028,15 +1111,7 @@ export function updateLayerIndexPattern(
newIndexPattern: IndexPattern
): IndexPatternLayer {
const keptColumns: IndexPatternLayer['columns'] = _.pickBy(layer.columns, (column) => {
if ('references' in column) {
return (
isColumnTransferable(column, newIndexPattern) &&
column.references.every((columnId) =>
isColumnTransferable(layer.columns[columnId], newIndexPattern)
)
);
}
return isColumnTransferable(column, newIndexPattern);
return isColumnTransferable(column, newIndexPattern, layer);
});
const newColumns: IndexPatternLayer['columns'] = _.mapValues(keptColumns, (column) => {
const operationDefinition = operationDefinitionMap[column.operationType];
@ -1069,7 +1144,7 @@ export function getErrorMessages(
.flatMap(([columnId, column]) => {
const def = operationDefinitionMap[column.operationType];
if (def.getErrorMessage) {
return def.getErrorMessage(layer, columnId, indexPattern);
return def.getErrorMessage(layer, columnId, indexPattern, operationDefinitionMap);
}
})
// remove the undefined values
@ -1147,6 +1222,23 @@ export function resetIncomplete(layer: IndexPatternLayer, columnId: string): Ind
return { ...layer, incompleteColumns };
}
// managedReferences have a relaxed policy about operation allowed, so let them pass
function maybeValidateOperations({
column,
validation,
}: {
column: IndexPatternColumn;
validation: RequiredReference;
}) {
if (!validation.specificOperations) {
return true;
}
if (operationDefinitionMap[column.operationType].input === 'managedReference') {
return true;
}
return validation.specificOperations.includes(column.operationType);
}
export function isColumnValidAsReference({
column,
validation,
@ -1159,7 +1251,29 @@ export function isColumnValidAsReference({
const operationDefinition = operationDefinitionMap[operationType];
return (
validation.input.includes(operationDefinition.input) &&
(!validation.specificOperations || validation.specificOperations.includes(operationType)) &&
maybeValidateOperations({
column,
validation,
}) &&
validation.validateMetadata(column)
);
}
export function getManagedColumnsFrom(
columnId: string,
columns: Record<string, IndexPatternColumn>
): Array<[string, IndexPatternColumn]> {
const allNodes: Record<string, string[]> = {};
Object.entries(columns).forEach(([id, col]) => {
allNodes[id] = 'references' in col ? [...col.references] : [];
});
const queue: string[] = allNodes[columnId];
const store: Array<[string, IndexPatternColumn]> = [];
while (queue.length > 0) {
const nextId = queue.shift()!;
store.push([nextId, columns[nextId]]);
queue.push(...allNodes[nextId]);
}
return store.filter(([, column]) => column);
}

2
x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts
View file

@ -8,7 +8,7 @@
import type { OperationMetadata } from '../../types';
import type { OperationType } from './definitions';
export const createMockedReferenceOperation = () => {
export const createMockedFullReference = () => {
return {
input: 'fullReference',
displayName: 'Reference test',

8
x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts
View file

@ -354,6 +354,14 @@ describe('getOperationTypesForField', () => {
"operationType": "last_value",
"type": "field",
},
Object {
"operationType": "math",
"type": "managedReference",
},
Object {
"operationType": "formula",
"type": "managedReference",
},
],
},
Object {

71
x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts
View file

@ -106,6 +106,10 @@ type OperationFieldTuple =
| {
type: 'fullReference';
operationType: OperationType;
}
| {
type: 'managedReference';
operationType: OperationType;
};
/**
@ -138,7 +142,11 @@ type OperationFieldTuple =
* ]
* ```
*/
export function getAvailableOperationsByMetadata(indexPattern: IndexPattern) {
export function getAvailableOperationsByMetadata(
indexPattern: IndexPattern,
// For consistency in testing
customOperationDefinitionMap?: Record<string, GenericOperationDefinition>
) {
const operationByMetadata: Record<
string,
{ operationMetaData: OperationMetadata; operations: OperationFieldTuple[] }
@ -161,36 +169,49 @@ export function getAvailableOperationsByMetadata(indexPattern: IndexPattern) {
}
};
operationDefinitions.sort(getSortScoreByPriority).forEach((operationDefinition) => {
if (operationDefinition.input === 'field') {
indexPattern.fields.forEach((field) => {
(customOperationDefinitionMap
? Object.values(customOperationDefinitionMap)
: operationDefinitions
)
.sort(getSortScoreByPriority)
.forEach((operationDefinition) => {
if (operationDefinition.input === 'field') {
indexPattern.fields.forEach((field) => {
addToMap(
{
type: 'field',
operationType: operationDefinition.type,
field: field.name,
},
operationDefinition.getPossibleOperationForField(field)
);
});
} else if (operationDefinition.input === 'none') {
addToMap(
{
type: 'field',
type: 'none',
operationType: operationDefinition.type,
field: field.name,
},
operationDefinition.getPossibleOperationForField(field)
);
});
} else if (operationDefinition.input === 'none') {
addToMap(
{
type: 'none',
operationType: operationDefinition.type,
},
operationDefinition.getPossibleOperation()
);
} else if (operationDefinition.input === 'fullReference') {
const validOperation = operationDefinition.getPossibleOperation(indexPattern);
if (validOperation) {
addToMap(
{ type: 'fullReference', operationType: operationDefinition.type },
validOperation
operationDefinition.getPossibleOperation()
);
} else if (operationDefinition.input === 'fullReference') {
const validOperation = operationDefinition.getPossibleOperation(indexPattern);
if (validOperation) {
addToMap(
{ type: 'fullReference', operationType: operationDefinition.type },
validOperation
);
}
} else if (operationDefinition.input === 'managedReference') {
const validOperation = operationDefinition.getPossibleOperation();
if (validOperation) {
addToMap(
{ type: 'managedReference', operationType: operationDefinition.type },
validOperation
);
}
}
}
});
});
return Object.values(operationByMetadata);
}

43
x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts
View file

@ -60,22 +60,26 @@ function getExpressionForLayer(
const [referenceEntries, esAggEntries] = partition(
columnEntries,
([, col]) => operationDefinitionMap[col.operationType]?.input === 'fullReference'
([, col]) =>
operationDefinitionMap[col.operationType]?.input === 'fullReference' ||
operationDefinitionMap[col.operationType]?.input === 'managedReference'
);
if (referenceEntries.length || esAggEntries.length) {
const aggs: ExpressionAstExpressionBuilder[] = [];
const expressions: ExpressionAstFunction[] = [];
referenceEntries.forEach(([colId, col]) => {
sortedReferences(referenceEntries).forEach((colId) => {
const col = columns[colId];
const def = operationDefinitionMap[col.operationType];
if (def.input === 'fullReference') {
if (def.input === 'fullReference' || def.input === 'managedReference') {
expressions.push(...def.toExpression(layer, colId, indexPattern));
}
});
esAggEntries.forEach(([colId, col]) => {
const def = operationDefinitionMap[col.operationType];
if (def.input !== 'fullReference') {
if (def.input !== 'fullReference' && def.input !== 'managedReference') {
const wrapInFilter = Boolean(def.filterable && col.filter);
let aggAst = def.toEsAggsFn(
col,
@ -112,6 +116,10 @@ function getExpressionForLayer(
}
});
if (esAggEntries.length === 0) {
// Return early if there are no aggs, for example if the user has an empty formula
return null;
}
const idMap = esAggEntries.reduce((currentIdMap, [colId, column], index) => {
const esAggsId = `col-${index}-${colId}`;
return {
@ -245,6 +253,33 @@ function getExpressionForLayer(
return null;
}
// Topologically sorts references so that we can execute them in sequence
function sortedReferences(columns: Array<readonly [string, IndexPatternColumn]>) {
const allNodes: Record<string, string[]> = {};
columns.forEach(([id, col]) => {
allNodes[id] = 'references' in col ? col.references : [];
});
// remove real metric references
columns.forEach(([id]) => {
allNodes[id] = allNodes[id].filter((refId) => !!allNodes[refId]);
});
const ordered: string[] = [];
while (ordered.length < columns.length) {
Object.keys(allNodes).forEach((id) => {
if (allNodes[id].length === 0) {
ordered.push(id);
delete allNodes[id];
Object.keys(allNodes).forEach((k) => {
allNodes[k] = allNodes[k].filter((i) => i !== id);
});
}
});
}
return ordered;
}
export function toExpression(
state: IndexPatternPrivateState,
layerId: string,

14
x-pack/plugins/lens/public/indexpattern_datasource/utils.ts
View file

@ -62,7 +62,12 @@ export function isColumnInvalid(
Boolean(getReferencesErrors(layer, column, indexPattern).filter(Boolean).length);
return (
!!operationDefinition.getErrorMessage?.(layer, columnId, indexPattern) || referencesHaveErrors
!!operationDefinition.getErrorMessage?.(
layer,
columnId,
indexPattern,
operationDefinitionMap
) || referencesHaveErrors
);
}
@ -74,7 +79,12 @@ function getReferencesErrors(
return column.references?.map((referenceId: string) => {
const referencedOperation = layer.columns[referenceId]?.operationType;
const referencedDefinition = operationDefinitionMap[referencedOperation];
return referencedDefinition?.getErrorMessage?.(layer, referenceId, indexPattern);
return referencedDefinition?.getErrorMessage?.(
layer,
referenceId,
indexPattern,
operationDefinitionMap
);
});
}