From f5fdacc68197bb82339b311117bba5e2202c35a8 Mon Sep 17 00:00:00 2001 From: Thomas Neirynck Date: Mon, 12 Jul 2021 15:28:03 +0200 Subject: [PATCH] [Maps] Auto generate legends and styles from mvt data (#94811) (#105203) --- api_docs/maps.json | 8 +- x-pack/plugins/maps/common/constants.ts | 5 +- .../layer_descriptor_types.ts | 20 ++ .../style_property_descriptor_types.ts | 14 +- .../maps/common/get_centroid_features.test.ts | 4 +- .../maps/common/get_centroid_features.ts | 8 +- .../maps/common/get_geometry_counts.ts | 45 +++++ .../maps/common/pluck_category_field_meta.ts | 46 +++++ .../maps/common/pluck_range_field_meta.ts | 34 ++++ .../public/actions/data_request_actions.ts | 13 +- .../maps/public/actions/layer_actions.ts | 21 ++ .../classes/fields/agg/count_agg_field.ts | 2 +- .../blended_vector_layer.ts | 8 +- .../maps/public/classes/layers/layer.tsx | 10 +- .../tiled_vector_layer.test.tsx.snap | 5 +- .../tiled_vector_layer.test.tsx | 2 +- .../tiled_vector_layer/tiled_vector_layer.tsx | 111 ++++++++++- .../classes/layers/vector_layer/index.ts | 7 +- .../layers/vector_layer/vector_layer.tsx | 40 ++-- .../categorical_data_mapping_popover.tsx | 2 - .../ordinal_data_mapping_popover.tsx | 2 - .../dynamic_color_property.test.tsx.snap | 23 +-- .../dynamic_color_property.test.tsx | 21 +- .../properties/dynamic_style_property.tsx | 183 +++++++++++------- .../classes/styles/vector/vector_style.tsx | 132 +++++++++++-- .../classes/util/mb_filter_expressions.ts | 4 +- .../connected_components/mb_map/index.ts | 9 +- .../connected_components/mb_map/mb_map.tsx | 35 +++- .../mb_map/tile_status_tracker.test.ts | 6 +- .../mb_map/tile_status_tracker.ts | 16 +- x-pack/plugins/maps/server/mvt/get_tile.ts | 172 ++++++++++++++-- x-pack/plugins/maps/server/mvt/mvt_routes.ts | 2 +- .../apis/maps/get_grid_tile.js | 52 ++++- .../api_integration/apis/maps/get_tile.js | 45 ++++- .../functional/apps/maps/mapbox_styles.js | 13 +- 35 files changed, 876 insertions(+), 244 deletions(-) create mode 100644 x-pack/plugins/maps/common/get_geometry_counts.ts create mode 100644 x-pack/plugins/maps/common/pluck_category_field_meta.ts create mode 100644 x-pack/plugins/maps/common/pluck_range_field_meta.ts diff --git a/api_docs/maps.json b/api_docs/maps.json index de1a1f9f6f55..4f9d4996b038 100644 --- a/api_docs/maps.json +++ b/api_docs/maps.json @@ -3085,13 +3085,13 @@ }, { "parentPluginId": "maps", - "id": "def-common.KBN_TOO_MANY_FEATURES_PROPERTY", + "id": "def-common.KBN_METADATA_FEATURE", "type": "string", "tags": [], - "label": "KBN_TOO_MANY_FEATURES_PROPERTY", + "label": "KBN_METADATA_FEATURE", "description": [], "signature": [ - "\"__kbn_too_many_features__\"" + "\"__kbn_metadata_feature__\"" ], "source": { "path": "x-pack/plugins/maps/common/constants.ts", @@ -3582,4 +3582,4 @@ } ] } -} \ No newline at end of file +} diff --git a/x-pack/plugins/maps/common/constants.ts b/x-pack/plugins/maps/common/constants.ts index e1ec3e269b33..e2de2c412e82 100644 --- a/x-pack/plugins/maps/common/constants.ts +++ b/x-pack/plugins/maps/common/constants.ts @@ -50,7 +50,10 @@ export const MVT_GETGRIDTILE_API_PATH = 'mvt/getGridTile'; export const MVT_SOURCE_LAYER_NAME = 'source_layer'; // Identifies vector tile "too many features" feature. // "too many features" feature is a box showing area that contains too many features for single ES search response -export const KBN_TOO_MANY_FEATURES_PROPERTY = '__kbn_too_many_features__'; +export const KBN_METADATA_FEATURE = '__kbn_metadata_feature__'; +export const KBN_FEATURE_COUNT = '__kbn_feature_count__'; +export const KBN_IS_TILE_COMPLETE = '__kbn_is_tile_complete__'; +export const KBN_VECTOR_SHAPE_TYPE_COUNTS = '__kbn_vector_shape_type_counts__'; export const KBN_TOO_MANY_FEATURES_IMAGE_ID = '__kbn_too_many_features_image_id__'; // Identifies centroid feature. // Centroids are a single point for representing lines, multiLines, polygons, and multiPolygons diff --git a/x-pack/plugins/maps/common/descriptor_types/layer_descriptor_types.ts b/x-pack/plugins/maps/common/descriptor_types/layer_descriptor_types.ts index 80baebc9afa5..740da8493c53 100644 --- a/x-pack/plugins/maps/common/descriptor_types/layer_descriptor_types.ts +++ b/x-pack/plugins/maps/common/descriptor_types/layer_descriptor_types.ts @@ -8,13 +8,22 @@ /* eslint-disable @typescript-eslint/consistent-type-definitions */ import { Query } from 'src/plugins/data/public'; +import { Feature } from 'geojson'; import { + FieldMeta, HeatmapStyleDescriptor, StyleDescriptor, VectorStyleDescriptor, } from './style_property_descriptor_types'; import { DataRequestDescriptor } from './data_request_descriptor_types'; import { AbstractSourceDescriptor, TermJoinSourceDescriptor } from './source_descriptor_types'; +import { VectorShapeTypeCounts } from '../get_geometry_counts'; +import { + KBN_FEATURE_COUNT, + KBN_IS_TILE_COMPLETE, + KBN_METADATA_FEATURE, + KBN_VECTOR_SHAPE_TYPE_COUNTS, +} from '../constants'; export type Attribution = { label: string; @@ -26,6 +35,16 @@ export type JoinDescriptor = { right: TermJoinSourceDescriptor; }; +export type TileMetaFeature = Feature & { + properties: { + [KBN_METADATA_FEATURE]: true; + [KBN_IS_TILE_COMPLETE]: boolean; + [KBN_FEATURE_COUNT]: number; + [KBN_VECTOR_SHAPE_TYPE_COUNTS]: VectorShapeTypeCounts; + fieldMeta?: FieldMeta; + }; +}; + export type LayerDescriptor = { __dataRequests?: DataRequestDescriptor[]; __isInErrorState?: boolean; @@ -33,6 +52,7 @@ export type LayerDescriptor = { __errorMessage?: string; __trackedLayerDescriptor?: LayerDescriptor; __areTilesLoaded?: boolean; + __metaFromTiles?: TileMetaFeature[]; alpha?: number; attribution?: Attribution; id: string; diff --git a/x-pack/plugins/maps/common/descriptor_types/style_property_descriptor_types.ts b/x-pack/plugins/maps/common/descriptor_types/style_property_descriptor_types.ts index fb44c9443c79..9c14dd088665 100644 --- a/x-pack/plugins/maps/common/descriptor_types/style_property_descriptor_types.ts +++ b/x-pack/plugins/maps/common/descriptor_types/style_property_descriptor_types.ts @@ -225,14 +225,16 @@ export type GeometryTypes = { isPolygonsOnly: boolean; }; +export type FieldMeta = { + [key: string]: { + range?: RangeFieldMeta; + categories?: CategoryFieldMeta; + }; +}; + export type StyleMetaDescriptor = { geometryTypes?: GeometryTypes; - fieldMeta: { - [key: string]: { - range?: RangeFieldMeta; - categories?: CategoryFieldMeta; - }; - }; + fieldMeta: FieldMeta; }; export type VectorStyleDescriptor = StyleDescriptor & { diff --git a/x-pack/plugins/maps/common/get_centroid_features.test.ts b/x-pack/plugins/maps/common/get_centroid_features.test.ts index 57879989644f..0fac9dc3a355 100644 --- a/x-pack/plugins/maps/common/get_centroid_features.test.ts +++ b/x-pack/plugins/maps/common/get_centroid_features.test.ts @@ -44,7 +44,7 @@ test('should not create centroid feature for point and multipoint', () => { expect(centroidFeatures.length).toBe(0); }); -test('should not create centroid for too many features polygon', () => { +test('should not create centroid for the metadata polygon', () => { const polygonFeature: Feature = { type: 'Feature', geometry: { @@ -60,7 +60,7 @@ test('should not create centroid for too many features polygon', () => { ], }, properties: { - __kbn_too_many_features__: true, + __kbn_metadata_feature__: true, prop0: 'value0', prop1: 0.0, }, diff --git a/x-pack/plugins/maps/common/get_centroid_features.ts b/x-pack/plugins/maps/common/get_centroid_features.ts index ce82b7d875fb..edf8167f9c15 100644 --- a/x-pack/plugins/maps/common/get_centroid_features.ts +++ b/x-pack/plugins/maps/common/get_centroid_features.ts @@ -21,11 +21,7 @@ import turfArea from '@turf/area'; import turfCenterOfMass from '@turf/center-of-mass'; import turfLength from '@turf/length'; import { lineString, polygon } from '@turf/helpers'; -import { - GEO_JSON_TYPE, - KBN_IS_CENTROID_FEATURE, - KBN_TOO_MANY_FEATURES_PROPERTY, -} from './constants'; +import { GEO_JSON_TYPE, KBN_IS_CENTROID_FEATURE, KBN_METADATA_FEATURE } from './constants'; export function getCentroidFeatures(featureCollection: FeatureCollection): Feature[] { const centroids = []; @@ -33,7 +29,7 @@ export function getCentroidFeatures(featureCollection: FeatureCollection): Featu const feature = featureCollection.features[i]; // do not add centroid for kibana added features - if (feature.properties?.[KBN_TOO_MANY_FEATURES_PROPERTY]) { + if (feature.properties?.[KBN_METADATA_FEATURE]) { continue; } diff --git a/x-pack/plugins/maps/common/get_geometry_counts.ts b/x-pack/plugins/maps/common/get_geometry_counts.ts new file mode 100644 index 000000000000..2a3368560c76 --- /dev/null +++ b/x-pack/plugins/maps/common/get_geometry_counts.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Feature } from 'geojson'; +import { GEO_JSON_TYPE, VECTOR_SHAPE_TYPE } from './constants'; + +export interface VectorShapeTypeCounts { + [VECTOR_SHAPE_TYPE.POINT]: number; + [VECTOR_SHAPE_TYPE.LINE]: number; + [VECTOR_SHAPE_TYPE.POLYGON]: number; +} + +export function countVectorShapeTypes(features: Feature[]): VectorShapeTypeCounts { + const vectorShapeTypeCounts: VectorShapeTypeCounts = { + [VECTOR_SHAPE_TYPE.POINT]: 0, + [VECTOR_SHAPE_TYPE.LINE]: 0, + [VECTOR_SHAPE_TYPE.POLYGON]: 0, + }; + + for (let i = 0; i < features.length; i++) { + const feature: Feature = features[i]; + if ( + feature.geometry.type === GEO_JSON_TYPE.POINT || + feature.geometry.type === GEO_JSON_TYPE.MULTI_POINT + ) { + vectorShapeTypeCounts[VECTOR_SHAPE_TYPE.POINT] += 1; + } else if ( + feature.geometry.type === GEO_JSON_TYPE.LINE_STRING || + feature.geometry.type === GEO_JSON_TYPE.MULTI_LINE_STRING + ) { + vectorShapeTypeCounts[VECTOR_SHAPE_TYPE.LINE] += 1; + } else if ( + feature.geometry.type === GEO_JSON_TYPE.POLYGON || + feature.geometry.type === GEO_JSON_TYPE.MULTI_POLYGON + ) { + vectorShapeTypeCounts[VECTOR_SHAPE_TYPE.POLYGON] += 1; + } + } + + return vectorShapeTypeCounts; +} diff --git a/x-pack/plugins/maps/common/pluck_category_field_meta.ts b/x-pack/plugins/maps/common/pluck_category_field_meta.ts new file mode 100644 index 000000000000..c71316f864a8 --- /dev/null +++ b/x-pack/plugins/maps/common/pluck_category_field_meta.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Feature } from 'geojson'; +import { CategoryFieldMeta } from './descriptor_types'; + +export function pluckCategoryFieldMeta( + features: Feature[], + name: string, + size: number +): CategoryFieldMeta | null { + const counts = new Map(); + for (let i = 0; i < features.length; i++) { + const feature = features[i]; + const term = feature.properties ? feature.properties[name] : undefined; + // properties object may be sparse, so need to check if the field is effectively present + if (typeof term !== undefined) { + if (counts.has(term)) { + counts.set(term, counts.get(term) + 1); + } else { + counts.set(term, 1); + } + } + } + + return trimCategories(counts, size); +} + +export function trimCategories(counts: Map, size: number): CategoryFieldMeta { + const ordered = []; + for (const [key, value] of counts) { + ordered.push({ key, count: value }); + } + + ordered.sort((a, b) => { + return b.count - a.count; + }); + const truncated = ordered.slice(0, size); + return { + categories: truncated, + } as CategoryFieldMeta; +} diff --git a/x-pack/plugins/maps/common/pluck_range_field_meta.ts b/x-pack/plugins/maps/common/pluck_range_field_meta.ts new file mode 100644 index 000000000000..b0bf03896892 --- /dev/null +++ b/x-pack/plugins/maps/common/pluck_range_field_meta.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Feature } from 'geojson'; +import { RangeFieldMeta } from './descriptor_types'; + +export function pluckRangeFieldMeta( + features: Feature[], + name: string, + parseValue: (rawValue: unknown) => number +): RangeFieldMeta | null { + let min = Infinity; + let max = -Infinity; + for (let i = 0; i < features.length; i++) { + const feature = features[i]; + const newValue = feature.properties ? parseValue(feature.properties[name]) : NaN; + if (!isNaN(newValue)) { + min = Math.min(min, newValue); + max = Math.max(max, newValue); + } + } + + return min === Infinity || max === -Infinity + ? null + : ({ + min, + max, + delta: max - min, + } as RangeFieldMeta); +} diff --git a/x-pack/plugins/maps/public/actions/data_request_actions.ts b/x-pack/plugins/maps/public/actions/data_request_actions.ts index 1b65650fc3fd..47350474eef0 100644 --- a/x-pack/plugins/maps/public/actions/data_request_actions.ts +++ b/x-pack/plugins/maps/public/actions/data_request_actions.ts @@ -16,7 +16,6 @@ import { FeatureCollection } from 'geojson'; import { MapStoreState } from '../reducers/store'; import { KBN_IS_CENTROID_FEATURE, - LAYER_STYLE_TYPE, LAYER_TYPE, SOURCE_DATA_REQUEST_ID, } from '../../common/constants'; @@ -49,7 +48,6 @@ import { IVectorLayer } from '../classes/layers/vector_layer'; import { DataMeta, MapExtent, MapFilters } from '../../common/descriptor_types'; import { DataRequestAbortError } from '../classes/util/data_request'; import { scaleBounds, turfBboxToBounds } from '../../common/elasticsearch_util'; -import { IVectorStyle } from '../classes/styles/vector/vector_style'; const FIT_TO_BOUNDS_SCALE_FACTOR = 0.1; @@ -95,14 +93,12 @@ export function updateStyleMeta(layerId: string | null) { if (!layer) { return; } - const sourceDataRequest = layer.getSourceDataRequest(); - const style = layer.getCurrentStyle(); - if (!style || !sourceDataRequest || style.getType() !== LAYER_STYLE_TYPE.VECTOR) { + + const styleMeta = await layer.getStyleMetaDescriptorFromLocalFeatures(); + if (!styleMeta) { return; } - const styleMeta = await (style as IVectorStyle).pluckStyleMetaFromSourceDataRequest( - sourceDataRequest - ); + dispatch({ type: SET_LAYER_STYLE_META, layerId, @@ -249,6 +245,7 @@ function endDataLoad( dispatch(unregisterCancelCallback(requestToken)); const dataRequest = getDataRequestDescriptor(getState(), layerId, dataId); if (dataRequest && dataRequest.dataRequestToken !== requestToken) { + // todo - investigate - this may arise with failing style meta request and should not throw in that case throw new DataRequestAbortError(); } diff --git a/x-pack/plugins/maps/public/actions/layer_actions.ts b/x-pack/plugins/maps/public/actions/layer_actions.ts index 0072fcceb0d2..eef325ca67cc 100644 --- a/x-pack/plugins/maps/public/actions/layer_actions.ts +++ b/x-pack/plugins/maps/public/actions/layer_actions.ts @@ -47,6 +47,7 @@ import { JoinDescriptor, LayerDescriptor, StyleDescriptor, + TileMetaFeature, } from '../../common/descriptor_types'; import { ILayer } from '../classes/layers/layer'; import { IVectorLayer } from '../classes/layers/vector_layer'; @@ -591,3 +592,23 @@ export function setAreTilesLoaded(layerId: string, areTilesLoaded: boolean) { newValue: areTilesLoaded, }; } + +export function updateMetaFromTiles(layerId: string, mbMetaFeatures: TileMetaFeature[]) { + return async ( + dispatch: ThunkDispatch, + getState: () => MapStoreState + ) => { + const layer = getLayerById(layerId, getState()); + if (!layer) { + return; + } + + dispatch({ + type: UPDATE_LAYER_PROP, + id: layerId, + propName: '__metaFromTiles', + newValue: mbMetaFeatures, + }); + await dispatch(updateStyleMeta(layerId)); + }; +} diff --git a/x-pack/plugins/maps/public/classes/fields/agg/count_agg_field.ts b/x-pack/plugins/maps/public/classes/fields/agg/count_agg_field.ts index 56b966ae1585..7f5cd577e928 100644 --- a/x-pack/plugins/maps/public/classes/fields/agg/count_agg_field.ts +++ b/x-pack/plugins/maps/public/classes/fields/agg/count_agg_field.ts @@ -94,7 +94,7 @@ export class CountAggField implements IESAggField { } supportsAutoDomain(): boolean { - return this._canReadFromGeoJson ? true : this.supportsFieldMeta(); + return true; } canReadFromGeoJson(): boolean { diff --git a/x-pack/plugins/maps/public/classes/layers/blended_vector_layer/blended_vector_layer.ts b/x-pack/plugins/maps/public/classes/layers/blended_vector_layer/blended_vector_layer.ts index 9bfa74825c33..5db22ff5354a 100644 --- a/x-pack/plugins/maps/public/classes/layers/blended_vector_layer/blended_vector_layer.ts +++ b/x-pack/plugins/maps/public/classes/layers/blended_vector_layer/blended_vector_layer.ts @@ -184,9 +184,9 @@ export class BlendedVectorLayer extends VectorLayer implements IVectorLayer { private readonly _isClustered: boolean; private readonly _clusterSource: ESGeoGridSource; - private readonly _clusterStyle: IVectorStyle; + private readonly _clusterStyle: VectorStyle; private readonly _documentSource: ESSearchSource; - private readonly _documentStyle: IVectorStyle; + private readonly _documentStyle: VectorStyle; constructor(options: BlendedVectorLayerArguments) { super({ @@ -195,7 +195,7 @@ export class BlendedVectorLayer extends VectorLayer implements IVectorLayer { }); this._documentSource = this._source as ESSearchSource; // VectorLayer constructor sets _source as document source - this._documentStyle = this._style as IVectorStyle; // VectorLayer constructor sets _style as document source + this._documentStyle = this._style; // VectorLayer constructor sets _style as document source this._clusterSource = getClusterSource(this._documentSource, this._documentStyle); const clusterStyleDescriptor = getClusterStyleDescriptor( @@ -279,7 +279,7 @@ export class BlendedVectorLayer extends VectorLayer implements IVectorLayer { return this._documentSource; } - getCurrentStyle(): IVectorStyle { + getCurrentStyle(): VectorStyle { return this._isClustered ? this._clusterStyle : this._documentStyle; } diff --git a/x-pack/plugins/maps/public/classes/layers/layer.tsx b/x-pack/plugins/maps/public/classes/layers/layer.tsx index ef41c157a2b1..5244882d41e0 100644 --- a/x-pack/plugins/maps/public/classes/layers/layer.tsx +++ b/x-pack/plugins/maps/public/classes/layers/layer.tsx @@ -10,7 +10,7 @@ import type { Map as MbMap } from '@kbn/mapbox-gl'; import { Query } from 'src/plugins/data/public'; import _ from 'lodash'; -import React, { ReactElement, ReactNode } from 'react'; +import React, { ReactElement } from 'react'; import { EuiIcon } from '@elastic/eui'; import uuid from 'uuid/v4'; import { FeatureCollection } from 'geojson'; @@ -37,6 +37,7 @@ import { MapExtent, StyleDescriptor, Timeslice, + StyleMetaDescriptor, } from '../../../common/descriptor_types'; import { ImmutableSourceProperty, ISource, SourceEditorArgs } from '../sources/source'; import { DataRequestContext } from '../../actions'; @@ -104,10 +105,11 @@ export interface ILayer { getCustomIconAndTooltipContent(): CustomIconAndTooltipContent; getDescriptor(): LayerDescriptor; getGeoFieldNames(): string[]; + getStyleMetaDescriptorFromLocalFeatures(): Promise; } export type CustomIconAndTooltipContent = { - icon: ReactNode; + icon: ReactElement; tooltipContent?: string | null; areResultsTrimmed?: boolean; }; @@ -521,4 +523,8 @@ export class AbstractLayer implements ILayer { const source = this.getSource(); return source.isESSource() ? [(source as IESSource).getGeoFieldName()] : []; } + + async getStyleMetaDescriptorFromLocalFeatures(): Promise { + return null; + } } diff --git a/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/__snapshots__/tiled_vector_layer.test.tsx.snap b/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/__snapshots__/tiled_vector_layer.test.tsx.snap index 3a542ce9bca1..d3b96936a85a 100644 --- a/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/__snapshots__/tiled_vector_layer.test.tsx.snap +++ b/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/__snapshots__/tiled_vector_layer.test.tsx.snap @@ -1,8 +1,9 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`icon should use vector icon 1`] = ` +exports[`icon should use no data icon 1`] = ` `; diff --git a/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.test.tsx b/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.test.tsx index e71d32669a56..e1f134cdf2a8 100644 --- a/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.test.tsx +++ b/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.test.tsx @@ -96,7 +96,7 @@ describe('visiblity', () => { }); describe('icon', () => { - it('should use vector icon', async () => { + it('should use no data icon', async () => { const layer: TiledVectorLayer = createLayer({}, {}); const iconAndTooltipContent = layer.getCustomIconAndTooltipContent(); diff --git a/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.tsx b/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.tsx index 2ad6a5ef73c6..880a47d0981c 100644 --- a/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.tsx +++ b/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.tsx @@ -5,29 +5,41 @@ * 2.0. */ -import React from 'react'; import type { Map as MbMap, GeoJSONSource as MbGeoJSONSource, VectorSource as MbVectorSource, } from '@kbn/mapbox-gl'; -import { EuiIcon } from '@elastic/eui'; import { Feature } from 'geojson'; import uuid from 'uuid/v4'; import { parse as parseUrl } from 'url'; +import { i18n } from '@kbn/i18n'; import { IVectorStyle, VectorStyle } from '../../styles/vector/vector_style'; -import { SOURCE_DATA_REQUEST_ID, LAYER_TYPE } from '../../../../common/constants'; -import { VectorLayer, VectorLayerArguments } from '../vector_layer'; +import { + KBN_FEATURE_COUNT, + KBN_IS_TILE_COMPLETE, + KBN_METADATA_FEATURE, + LAYER_TYPE, + SOURCE_DATA_REQUEST_ID, +} from '../../../../common/constants'; +import { + VectorLayer, + VectorLayerArguments, + NO_RESULTS_ICON_AND_TOOLTIPCONTENT, +} from '../vector_layer'; import { ITiledSingleLayerVectorSource } from '../../sources/tiled_single_layer_vector_source'; import { DataRequestContext } from '../../../actions'; import { Timeslice, + StyleMetaDescriptor, VectorLayerDescriptor, VectorSourceRequestMeta, + TileMetaFeature, } from '../../../../common/descriptor_types'; import { MVTSingleLayerVectorSourceConfig } from '../../sources/mvt_single_layer_vector_source/types'; import { canSkipSourceUpdate } from '../../util/can_skip_fetch'; import { isRefreshOnlyQuery } from '../../util/is_refresh_only_query'; +import { CustomIconAndTooltipContent } from '../layer'; export class TiledVectorLayer extends VectorLayer { static type = LAYER_TYPE.TILED_VECTOR; @@ -54,9 +66,45 @@ export class TiledVectorLayer extends VectorLayer { this._source = source as ITiledSingleLayerVectorSource; } - getCustomIconAndTooltipContent() { + _getMetaFromTiles(): TileMetaFeature[] { + return this._descriptor.__metaFromTiles || []; + } + + getCustomIconAndTooltipContent(): CustomIconAndTooltipContent { + const tileMetas = this._getMetaFromTiles(); + if (!tileMetas.length) { + return NO_RESULTS_ICON_AND_TOOLTIPCONTENT; + } + + const totalFeaturesCount: number = tileMetas.reduce((acc: number, tileMeta: Feature) => { + const count = tileMeta && tileMeta.properties ? tileMeta.properties[KBN_FEATURE_COUNT] : 0; + return count + acc; + }, 0); + + if (totalFeaturesCount === 0) { + return NO_RESULTS_ICON_AND_TOOLTIPCONTENT; + } + + const isIncomplete: boolean = tileMetas.some((tileMeta: Feature) => { + return !tileMeta?.properties?.[KBN_IS_TILE_COMPLETE]; + }); + return { - icon: , + icon: this.getCurrentStyle().getIcon(), + tooltipContent: isIncomplete + ? i18n.translate('xpack.maps.tiles.resultsTrimmedMsg', { + defaultMessage: `Results limited to {count} documents.`, + values: { + count: totalFeaturesCount, + }, + }) + : i18n.translate('xpack.maps.tiles.resultsCompleteMsg', { + defaultMessage: `Found {count} documents.`, + values: { + count: totalFeaturesCount, + }, + }), + areResultsTrimmed: isIncomplete, }; } @@ -188,6 +236,47 @@ export class TiledVectorLayer extends VectorLayer { this._setMbCentroidProperties(mbMap, sourceMeta.layerName); } + queryTileMetaFeatures(mbMap: MbMap): TileMetaFeature[] | null { + // @ts-ignore + const mbSource = mbMap.getSource(this._getMbSourceId()); + if (!mbSource) { + return null; + } + + const sourceDataRequest = this.getSourceDataRequest(); + if (!sourceDataRequest) { + return null; + } + const sourceMeta: MVTSingleLayerVectorSourceConfig = sourceDataRequest.getData() as MVTSingleLayerVectorSourceConfig; + if (sourceMeta.layerName === '') { + return null; + } + + // querySourceFeatures can return duplicated features when features cross tile boundaries. + // Tile meta will never have duplicated features since by there nature, tile meta is a feature contained within a single tile + const mbFeatures = mbMap.querySourceFeatures(this._getMbSourceId(), { + sourceLayer: sourceMeta.layerName, + filter: ['==', ['get', KBN_METADATA_FEATURE], true], + }); + + const metaFeatures: TileMetaFeature[] = mbFeatures.map((mbFeature: Feature) => { + const parsedProperties: Record = {}; + for (const key in mbFeature.properties) { + if (mbFeature.properties.hasOwnProperty(key)) { + parsedProperties[key] = JSON.parse(mbFeature.properties[key]); // mvt properties cannot be nested geojson + } + } + return { + type: 'Feature', + id: mbFeature.id, + geometry: mbFeature.geometry, + properties: parsedProperties, + } as TileMetaFeature; + }); + + return metaFeatures as TileMetaFeature[]; + } + _requiresPrevSourceCleanup(mbMap: MbMap): boolean { const mbSource = mbMap.getSource(this._getMbSourceId()) as MbVectorSource | MbGeoJSONSource; if (!mbSource) { @@ -252,4 +341,14 @@ export class TiledVectorLayer extends VectorLayer { getFeatureById(id: string | number): Feature | null { return null; } + + async getStyleMetaDescriptorFromLocalFeatures(): Promise { + const style = this.getCurrentStyle(); + if (!style) { + return null; + } + + const metaFromTiles = this._getMetaFromTiles(); + return await style.pluckStyleMetaFromTileMeta(metaFromTiles); + } } diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/index.ts b/x-pack/plugins/maps/public/classes/layers/vector_layer/index.ts index b6777f8a5e45..3c8449c5aa4a 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_layer/index.ts +++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/index.ts @@ -6,4 +6,9 @@ */ export { addGeoJsonMbSource, getVectorSourceBounds, syncVectorSource } from './utils'; -export { IVectorLayer, VectorLayer, VectorLayerArguments } from './vector_layer'; +export { + IVectorLayer, + VectorLayer, + VectorLayerArguments, + NO_RESULTS_ICON_AND_TOOLTIPCONTENT, +} from './vector_layer'; diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx index f8d141574e5f..959064b3daab 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx +++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx @@ -23,12 +23,13 @@ import { SOURCE_FORMATTERS_DATA_REQUEST_ID, FEATURE_VISIBLE_PROPERTY_NAME, EMPTY_FEATURE_COLLECTION, - KBN_TOO_MANY_FEATURES_PROPERTY, + KBN_METADATA_FEATURE, LAYER_TYPE, FIELD_ORIGIN, KBN_TOO_MANY_FEATURES_IMAGE_ID, FieldFormatter, SUPPORTS_FEATURE_EDITING_REQUEST_ID, + KBN_IS_TILE_COMPLETE, } from '../../../../common/constants'; import { JoinTooltipProperty } from '../../tooltips/join_tooltip_property'; import { DataRequestAbortError } from '../../util/data_request'; @@ -49,6 +50,7 @@ import { DynamicStylePropertyOptions, MapFilters, MapQuery, + StyleMetaDescriptor, Timeslice, VectorJoinSourceRequestMeta, VectorLayerDescriptor, @@ -102,9 +104,18 @@ export interface IVectorLayer extends ILayer { deleteFeature(featureId: string): Promise; } +const noResultsIcon = ; +export const NO_RESULTS_ICON_AND_TOOLTIPCONTENT = { + icon: noResultsIcon, + tooltipContent: i18n.translate('xpack.maps.vectorLayer.noResultsFoundTooltip', { + defaultMessage: `No results found.`, + }), +}; + export class VectorLayer extends AbstractLayer implements IVectorLayer { static type = LAYER_TYPE.VECTOR; - protected readonly _style: IVectorStyle; + + protected readonly _style: VectorStyle; private readonly _joins: InnerJoin[]; static createDescriptor( @@ -157,7 +168,7 @@ export class VectorLayer extends AbstractLayer implements IVectorLayer { return this._style; } - getCurrentStyle(): IVectorStyle { + getCurrentStyle(): VectorStyle { return this._style; } @@ -211,14 +222,8 @@ export class VectorLayer extends AbstractLayer implements IVectorLayer { getCustomIconAndTooltipContent(): CustomIconAndTooltipContent { const featureCollection = this._getSourceFeatureCollection(); - const noResultsIcon = ; if (!featureCollection || featureCollection.features.length === 0) { - return { - icon: noResultsIcon, - tooltipContent: i18n.translate('xpack.maps.vectorLayer.noResultsFoundTooltip', { - defaultMessage: `No results found.`, - }), - }; + return NO_RESULTS_ICON_AND_TOOLTIPCONTENT; } if ( @@ -956,9 +961,9 @@ export class VectorLayer extends AbstractLayer implements IVectorLayer { } mbMap.addLayer(mbLayer); mbMap.setFilter(tooManyFeaturesLayerId, [ - '==', - ['get', KBN_TOO_MANY_FEATURES_PROPERTY], - true, + 'all', + ['==', ['get', KBN_METADATA_FEATURE], true], + ['==', ['get', KBN_IS_TILE_COMPLETE], false], ]); mbMap.setPaintProperty( tooManyFeaturesLayerId, @@ -1168,4 +1173,13 @@ export class VectorLayer extends AbstractLayer implements IVectorLayer { const layerSource = this.getSource(); await layerSource.deleteFeature(featureId); } + + async getStyleMetaDescriptorFromLocalFeatures(): Promise { + const sourceDataRequest = this.getSourceDataRequest(); + const style = this.getCurrentStyle(); + if (!style || !sourceDataRequest) { + return null; + } + return await style.pluckStyleMetaFromSourceDataRequest(sourceDataRequest); + } } diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/data_mapping/categorical_data_mapping_popover.tsx b/x-pack/plugins/maps/public/classes/styles/vector/components/data_mapping/categorical_data_mapping_popover.tsx index e03dda06a3ef..3fb36364bee4 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/data_mapping/categorical_data_mapping_popover.tsx +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/data_mapping/categorical_data_mapping_popover.tsx @@ -15,7 +15,6 @@ import { FieldMetaOptions } from '../../../../../../common/descriptor_types'; interface Props { fieldMetaOptions: FieldMetaOptions; onChange: (updatedOptions: DynamicOptions) => void; - switchDisabled: boolean; } export function CategoricalDataMappingPopover(props: Props) { @@ -40,7 +39,6 @@ export function CategoricalDataMappingPopover(props: Props{' '} { fieldMetaOptions: FieldMetaOptions; styleName: VECTOR_STYLES; onChange: (updatedOptions: DynamicOptions) => void; - switchDisabled: boolean; dataMappingFunction: DATA_MAPPING_FUNCTION; supportedDataMappingFunctions: DATA_MAPPING_FUNCTION[]; } @@ -169,7 +168,6 @@ export function OrdinalDataMappingPopover(props: Props{' '} -`; - -exports[`renderDataMappingPopover Should enable toggle when field is backed by geojson-source 1`] = ` - `; diff --git a/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_color_property.test.tsx b/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_color_property.test.tsx index b7e0133881ee..dfe358586cb9 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_color_property.test.tsx +++ b/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_color_property.test.tsx @@ -642,7 +642,7 @@ test('Should read out ordinal type correctly', async () => { }); describe('renderDataMappingPopover', () => { - test('Should enable toggle when field is backed by geojson-source', () => { + test('Should render OrdinalDataMappingPopover', () => { const colorStyle = makeProperty( { color: 'Blues', @@ -656,23 +656,4 @@ describe('renderDataMappingPopover', () => { const legendRow = colorStyle.renderDataMappingPopover(() => {}); expect(legendRow).toMatchSnapshot(); }); - - test('Should disable toggle when field is not backed by geojson source', () => { - const nonGeoJsonField = Object.create(mockField); - nonGeoJsonField.canReadFromGeoJson = () => { - return false; - }; - const colorStyle = makeProperty( - { - color: 'Blues', - type: undefined, - fieldMetaOptions, - }, - undefined, - nonGeoJsonField - ); - - const legendRow = colorStyle.renderDataMappingPopover(() => {}); - expect(legendRow).toMatchSnapshot(); - }); }); diff --git a/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_style_property.tsx b/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_style_property.tsx index 0841bb7546d9..7f0e6027e7af 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_style_property.tsx +++ b/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_style_property.tsx @@ -27,17 +27,24 @@ import { OrdinalDataMappingPopover, } from '../components/data_mapping'; import { + Category, CategoryFieldMeta, FieldMetaOptions, PercentilesFieldMeta, RangeFieldMeta, StyleMetaData, + TileMetaFeature, } from '../../../../../common/descriptor_types'; import { IField } from '../../../fields/field'; import { IVectorLayer } from '../../../layers/vector_layer'; import { InnerJoin } from '../../../joins/inner_join'; import { IVectorStyle } from '../vector_style'; import { getComputedFieldName } from '../style_util'; +import { pluckRangeFieldMeta } from '../../../../../common/pluck_range_field_meta'; +import { + pluckCategoryFieldMeta, + trimCategories, +} from '../../../../../common/pluck_category_field_meta'; export interface IDynamicStyleProperty extends IStyleProperty { getFieldMetaOptions(): FieldMetaOptions; @@ -56,6 +63,10 @@ export interface IDynamicStyleProperty extends IStyleProperty { getFieldMetaRequest(): Promise; pluckOrdinalStyleMetaFromFeatures(features: Feature[]): RangeFieldMeta | null; pluckCategoricalStyleMetaFromFeatures(features: Feature[]): CategoryFieldMeta | null; + pluckOrdinalStyleMetaFromTileMetaFeatures(features: TileMetaFeature[]): RangeFieldMeta | null; + pluckCategoricalStyleMetaFromTileMetaFeatures( + features: TileMetaFeature[] + ): CategoryFieldMeta | null; getValueSuggestions(query: string): Promise; enrichGeoJsonAndMbFeatureState( featureCollection: FeatureCollection, @@ -103,29 +114,37 @@ export class DynamicStyleProperty return join ? join.getSourceMetaDataRequestId() : null; } - getRangeFieldMeta() { + _getRangeFieldMetaFromLocalFeatures() { const style = this._layer.getStyle() as IVectorStyle; const styleMeta = style.getStyleMeta(); const fieldName = this.getFieldName(); - const rangeFieldMetaFromLocalFeatures = styleMeta.getRangeFieldMetaDescriptor(fieldName); + return styleMeta.getRangeFieldMetaDescriptor(fieldName); + } - if (!this.isFieldMetaEnabled()) { - return rangeFieldMetaFromLocalFeatures; - } - - const dataRequestId = this._getStyleMetaDataRequestId(fieldName); + _getRangeFieldMetaFromStyleMetaRequest(): RangeFieldMeta | null { + const dataRequestId = this._getStyleMetaDataRequestId(this.getFieldName()); if (!dataRequestId) { - return rangeFieldMetaFromLocalFeatures; + return null; } const styleMetaDataRequest = this._layer.getDataRequest(dataRequestId); if (!styleMetaDataRequest || !styleMetaDataRequest.hasData()) { - return rangeFieldMetaFromLocalFeatures; + return null; } const data = styleMetaDataRequest.getData() as StyleMetaData; const rangeFieldMeta = this._pluckOrdinalStyleMetaFromFieldMetaData(data); - return rangeFieldMeta ? rangeFieldMeta : rangeFieldMetaFromLocalFeatures; + return rangeFieldMeta ? rangeFieldMeta : null; + } + + getRangeFieldMeta(): RangeFieldMeta | null { + const rangeFieldMetaFromLocalFeatures = this._getRangeFieldMetaFromLocalFeatures(); + if (!this.isFieldMetaEnabled()) { + return rangeFieldMetaFromLocalFeatures; + } + + const rangeFieldMetaFromServer = this._getRangeFieldMetaFromStyleMetaRequest(); + return rangeFieldMetaFromServer ? rangeFieldMetaFromServer : rangeFieldMetaFromLocalFeatures; } getPercentilesFieldMeta() { @@ -150,29 +169,39 @@ export class DynamicStyleProperty return percentilesValuesToFieldMeta(percentiles); } - getCategoryFieldMeta() { + _getCategoryFieldMetaFromLocalFeatures() { const style = this._layer.getStyle() as IVectorStyle; const styleMeta = style.getStyleMeta(); const fieldName = this.getFieldName(); - const categoryFieldMetaFromLocalFeatures = styleMeta.getCategoryFieldMetaDescriptor(fieldName); + return styleMeta.getCategoryFieldMetaDescriptor(fieldName); + } + + _getCategoryFieldMetaFromStyleMetaRequest() { + const dataRequestId = this._getStyleMetaDataRequestId(this.getFieldName()); + if (!dataRequestId) { + return null; + } + + const styleMetaDataRequest = this._layer.getDataRequest(dataRequestId); + if (!styleMetaDataRequest || !styleMetaDataRequest.hasData()) { + return null; + } + + const data = styleMetaDataRequest.getData() as StyleMetaData; + return this._pluckCategoricalStyleMetaFromFieldMetaData(data); + } + + getCategoryFieldMeta(): CategoryFieldMeta | null { + const categoryFieldMetaFromLocalFeatures = this._getCategoryFieldMetaFromLocalFeatures(); if (!this.isFieldMetaEnabled()) { return categoryFieldMetaFromLocalFeatures; } - const dataRequestId = this._getStyleMetaDataRequestId(fieldName); - if (!dataRequestId) { - return categoryFieldMetaFromLocalFeatures; - } - - const styleMetaDataRequest = this._layer.getDataRequest(dataRequestId); - if (!styleMetaDataRequest || !styleMetaDataRequest.hasData()) { - return categoryFieldMetaFromLocalFeatures; - } - - const data = styleMetaDataRequest.getData() as StyleMetaData; - const rangeFieldMeta = this._pluckCategoricalStyleMetaFromFieldMetaData(data); - return rangeFieldMeta ? rangeFieldMeta : categoryFieldMetaFromLocalFeatures; + const categoricalFieldMetaFromServer = this._getCategoryFieldMetaFromStyleMetaRequest(); + return categoricalFieldMetaFromServer + ? categoricalFieldMetaFromServer + : categoryFieldMetaFromLocalFeatures; } getField() { @@ -277,7 +306,9 @@ export class DynamicStyleProperty : DATA_MAPPING_FUNCTION.INTERPOLATE; } - pluckOrdinalStyleMetaFromFeatures(features: Feature[]) { + pluckOrdinalStyleMetaFromTileMetaFeatures( + metaFeatures: TileMetaFeature[] + ): RangeFieldMeta | null { if (!this.isOrdinal()) { return null; } @@ -285,59 +316,74 @@ export class DynamicStyleProperty const name = this.getFieldName(); let min = Infinity; let max = -Infinity; - for (let i = 0; i < features.length; i++) { - const feature = features[i]; - const newValue = parseFloat(feature.properties ? feature.properties[name] : null); - if (!isNaN(newValue)) { - min = Math.min(min, newValue); - max = Math.max(max, newValue); + for (let i = 0; i < metaFeatures.length; i++) { + const fieldMeta = metaFeatures[i].properties.fieldMeta; + if (fieldMeta && fieldMeta[name] && fieldMeta[name].range) { + min = Math.min(fieldMeta[name].range?.min as number, min); + max = Math.max(fieldMeta[name].range?.max as number, max); } } - - return min === Infinity || max === -Infinity - ? null - : ({ - min, - max, - delta: max - min, - } as RangeFieldMeta); + return { + min, + max, + delta: max - min, + }; } - pluckCategoricalStyleMetaFromFeatures(features: Feature[]) { + pluckCategoricalStyleMetaFromTileMetaFeatures( + metaFeatures: TileMetaFeature[] + ): CategoryFieldMeta | null { const size = this.getNumberOfCategories(); if (!this.isCategorical() || size <= 0) { return null; } - const counts = new Map(); - for (let i = 0; i < features.length; i++) { - const feature = features[i]; - const term = feature.properties ? feature.properties[this.getFieldName()] : undefined; - // properties object may be sparse, so need to check if the field is effectively present - if (typeof term !== undefined) { - if (counts.has(term)) { - counts.set(term, counts.get(term) + 1); - } else { - counts.set(term, 1); + const name = this.getFieldName(); + + const counts = new Map(); + for (let i = 0; i < metaFeatures.length; i++) { + const fieldMeta = metaFeatures[i].properties.fieldMeta; + if (fieldMeta && fieldMeta[name] && fieldMeta[name].categories) { + const categoryFieldMeta: CategoryFieldMeta = fieldMeta[name] + .categories as CategoryFieldMeta; + for (let c = 0; c < categoryFieldMeta.categories.length; c++) { + const category: Category = categoryFieldMeta.categories[c]; + // properties object may be sparse, so need to check if the field is effectively present + if (typeof category.key !== undefined) { + if (counts.has(category.key)) { + counts.set(category.key, (counts.get(category.key) as number) + category.count); + } else { + counts.set(category.key, category.count); + } + } } } } - const ordered = []; - for (const [key, value] of counts) { - ordered.push({ key, count: value }); - } - - ordered.sort((a, b) => { - return b.count - a.count; - }); - const truncated = ordered.slice(0, size); - return { - categories: truncated, - } as CategoryFieldMeta; + return trimCategories(counts, size); } - _pluckOrdinalStyleMetaFromFieldMetaData(styleMetaData: StyleMetaData) { + pluckOrdinalStyleMetaFromFeatures(features: Feature[]): RangeFieldMeta | null { + if (!this.isOrdinal()) { + return null; + } + + const name = this.getFieldName(); + return pluckRangeFieldMeta(features, name, (rawValue: unknown) => { + return parseFloat(rawValue as string); + }); + } + + pluckCategoricalStyleMetaFromFeatures(features: Feature[]): CategoryFieldMeta | null { + const size = this.getNumberOfCategories(); + if (!this.isCategorical() || size <= 0) { + return null; + } + + return pluckCategoryFieldMeta(features, this.getFieldName(), size); + } + + _pluckOrdinalStyleMetaFromFieldMetaData(styleMetaData: StyleMetaData): RangeFieldMeta | null { if (!this.isOrdinal() || !this._field) { return null; } @@ -361,7 +407,9 @@ export class DynamicStyleProperty }; } - _pluckCategoricalStyleMetaFromFieldMetaData(styleMetaData: StyleMetaData) { + _pluckCategoricalStyleMetaFromFieldMetaData( + styleMetaData: StyleMetaData + ): CategoryFieldMeta | null { if (!this.isCategorical() || !this._field) { return null; } @@ -399,21 +447,16 @@ export class DynamicStyleProperty if (!this.supportsFieldMeta()) { return null; } - - const switchDisabled = !!this._field && !this._field.canReadFromGeoJson(); - return this.isCategorical() ? ( fieldMetaOptions={this.getFieldMetaOptions()} onChange={onChange} - switchDisabled={switchDisabled} /> ) : ( fieldMetaOptions={this.getFieldMetaOptions()} styleName={this.getStyleName()} onChange={onChange} - switchDisabled={switchDisabled} dataMappingFunction={this.getDataMappingFunction()} supportedDataMappingFunctions={this._getSupportedDataMappingFunctions()} /> diff --git a/x-pack/plugins/maps/public/classes/styles/vector/vector_style.tsx b/x-pack/plugins/maps/public/classes/styles/vector/vector_style.tsx index 7578695d7ac6..695ae95b1b43 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/vector_style.tsx +++ b/x-pack/plugins/maps/public/classes/styles/vector/vector_style.tsx @@ -16,6 +16,7 @@ import { FIELD_ORIGIN, GEO_JSON_TYPE, KBN_IS_CENTROID_FEATURE, + KBN_VECTOR_SHAPE_TYPE_COUNTS, LAYER_STYLE_TYPE, SOURCE_FORMATTERS_DATA_REQUEST_ID, STYLE_TYPE, @@ -63,6 +64,7 @@ import { StyleMetaDescriptor, StylePropertyField, StylePropertyOptions, + TileMetaFeature, VectorStyleDescriptor, VectorStylePropertiesDescriptor, } from '../../../../common/descriptor_types'; @@ -74,6 +76,7 @@ import { IVectorLayer } from '../../layers/vector_layer'; import { IVectorSource } from '../../sources/vector_source'; import { createStyleFieldsHelper, StyleFieldsHelper } from './style_fields_helper'; import { IESAggField } from '../../fields/agg'; +import { VectorShapeTypeCounts } from '../../../../common/get_geometry_counts'; const POINTS = [GEO_JSON_TYPE.POINT, GEO_JSON_TYPE.MULTI_POINT]; const LINES = [GEO_JSON_TYPE.LINE_STRING, GEO_JSON_TYPE.MULTI_LINE_STRING]; @@ -89,9 +92,9 @@ export interface IVectorStyle extends IStyle { previousFields: IField[], mapColors: string[] ): Promise<{ hasChanges: boolean; nextStyleDescriptor?: VectorStyleDescriptor }>; - pluckStyleMetaFromSourceDataRequest(sourceDataRequest: DataRequest): Promise; isTimeAware: () => boolean; getIcon: () => ReactElement; + getIconFromGeometryTypes: (isLinesOnly: boolean, isPointsOnly: boolean) => ReactElement; hasLegendDetails: () => Promise; renderLegendDetails: () => ReactElement; clearFeatureState: (featureCollection: FeatureCollection, mbMap: MbMap, sourceId: string) => void; @@ -488,9 +491,89 @@ export class VectorStyle implements IVectorStyle { ); } - async pluckStyleMetaFromSourceDataRequest(sourceDataRequest: DataRequest) { - const features = _.get(sourceDataRequest.getData(), 'features', []); + async pluckStyleMetaFromTileMeta(metaFeatures: TileMetaFeature[]): Promise { + const shapeTypeCountMeta: VectorShapeTypeCounts = metaFeatures.reduce( + (accumulator: VectorShapeTypeCounts, tileMeta: TileMetaFeature) => { + if ( + !tileMeta || + !tileMeta.properties || + !tileMeta.properties[KBN_VECTOR_SHAPE_TYPE_COUNTS] + ) { + return accumulator; + } + accumulator[VECTOR_SHAPE_TYPE.POINT] += + tileMeta.properties[KBN_VECTOR_SHAPE_TYPE_COUNTS][VECTOR_SHAPE_TYPE.POINT]; + accumulator[VECTOR_SHAPE_TYPE.LINE] += + tileMeta.properties[KBN_VECTOR_SHAPE_TYPE_COUNTS][VECTOR_SHAPE_TYPE.LINE]; + accumulator[VECTOR_SHAPE_TYPE.POLYGON] += + tileMeta.properties[KBN_VECTOR_SHAPE_TYPE_COUNTS][VECTOR_SHAPE_TYPE.POLYGON]; + + return accumulator; + }, + { + [VECTOR_SHAPE_TYPE.POLYGON]: 0, + [VECTOR_SHAPE_TYPE.LINE]: 0, + [VECTOR_SHAPE_TYPE.POINT]: 0, + } + ); + + const isLinesOnly = + shapeTypeCountMeta[VECTOR_SHAPE_TYPE.LINE] > 0 && + shapeTypeCountMeta[VECTOR_SHAPE_TYPE.POINT] === 0 && + shapeTypeCountMeta[VECTOR_SHAPE_TYPE.POLYGON] === 0; + const isPointsOnly = + shapeTypeCountMeta[VECTOR_SHAPE_TYPE.LINE] === 0 && + shapeTypeCountMeta[VECTOR_SHAPE_TYPE.POINT] > 0 && + shapeTypeCountMeta[VECTOR_SHAPE_TYPE.POLYGON] === 0; + const isPolygonsOnly = + shapeTypeCountMeta[VECTOR_SHAPE_TYPE.LINE] === 0 && + shapeTypeCountMeta[VECTOR_SHAPE_TYPE.POINT] === 0 && + shapeTypeCountMeta[VECTOR_SHAPE_TYPE.POLYGON] > 0; + + const styleMeta: StyleMetaDescriptor = { + geometryTypes: { + isPointsOnly, + isLinesOnly, + isPolygonsOnly, + }, + fieldMeta: {}, + }; + + const dynamicProperties = this.getDynamicPropertiesArray(); + if (dynamicProperties.length === 0 || !metaFeatures) { + // no additional meta data to pull from source data request. + return styleMeta; + } + + dynamicProperties.forEach((dynamicProperty) => { + const ordinalStyleMeta = dynamicProperty.pluckOrdinalStyleMetaFromTileMetaFeatures( + metaFeatures + ); + const categoricalStyleMeta = dynamicProperty.pluckCategoricalStyleMetaFromTileMetaFeatures( + metaFeatures + ); + + const name = dynamicProperty.getFieldName(); + if (!styleMeta.fieldMeta[name]) { + styleMeta.fieldMeta[name] = {}; + } + if (categoricalStyleMeta) { + styleMeta.fieldMeta[name].categories = categoricalStyleMeta; + } + + if (ordinalStyleMeta) { + styleMeta.fieldMeta[name].range = ordinalStyleMeta; + } + }); + + return styleMeta; + } + + async pluckStyleMetaFromSourceDataRequest( + sourceDataRequest: DataRequest + ): Promise { + const features = _.get(sourceDataRequest.getData(), 'features', []); const supportedFeatures = await this._source.getSupportedShapeTypes(); const hasFeatureType = { [VECTOR_SHAPE_TYPE.POINT]: false, @@ -548,21 +631,25 @@ export class VectorStyle implements IVectorStyle { return styleMeta; } - dynamicProperties.forEach((dynamicProperty) => { - const categoricalStyleMeta = dynamicProperty.pluckCategoricalStyleMetaFromFeatures(features); - const ordinalStyleMeta = dynamicProperty.pluckOrdinalStyleMetaFromFeatures(features); - const name = dynamicProperty.getFieldName(); - if (!styleMeta.fieldMeta[name]) { - styleMeta.fieldMeta[name] = {}; - } - if (categoricalStyleMeta) { - styleMeta.fieldMeta[name].categories = categoricalStyleMeta; - } + dynamicProperties.forEach( + (dynamicProperty: IDynamicStyleProperty) => { + const categoricalStyleMeta = dynamicProperty.pluckCategoricalStyleMetaFromFeatures( + features + ); + const ordinalStyleMeta = dynamicProperty.pluckOrdinalStyleMetaFromFeatures(features); + const name = dynamicProperty.getFieldName(); + if (!styleMeta.fieldMeta[name]) { + styleMeta.fieldMeta[name] = {}; + } + if (categoricalStyleMeta) { + styleMeta.fieldMeta[name].categories = categoricalStyleMeta; + } - if (ordinalStyleMeta) { - styleMeta.fieldMeta[name].range = ordinalStyleMeta; + if (ordinalStyleMeta) { + styleMeta.fieldMeta[name].range = ordinalStyleMeta; + } } - }); + ); return styleMeta; } @@ -653,8 +740,7 @@ export class VectorStyle implements IVectorStyle { : (this._iconStyleProperty as StaticIconProperty).getOptions().value; } - getIcon = () => { - const isLinesOnly = this._getIsLinesOnly(); + getIconFromGeometryTypes(isLinesOnly: boolean, isPointsOnly: boolean) { let strokeColor; if (isLinesOnly) { strokeColor = extractColorFromStyleProperty( @@ -676,14 +762,20 @@ export class VectorStyle implements IVectorStyle { return ( ); - }; + } + + getIcon() { + const isLinesOnly = this._getIsLinesOnly(); + const isPointsOnly = this._getIsPointsOnly(); + return this.getIconFromGeometryTypes(isLinesOnly, isPointsOnly); + } _getLegendDetailStyleProperties = () => { return this.getDynamicPropertiesArray().filter((styleProperty) => { diff --git a/x-pack/plugins/maps/public/classes/util/mb_filter_expressions.ts b/x-pack/plugins/maps/public/classes/util/mb_filter_expressions.ts index 9568ef5c35bb..eb5896392971 100644 --- a/x-pack/plugins/maps/public/classes/util/mb_filter_expressions.ts +++ b/x-pack/plugins/maps/public/classes/util/mb_filter_expressions.ts @@ -9,7 +9,7 @@ import { GEO_JSON_TYPE, FEATURE_VISIBLE_PROPERTY_NAME, KBN_IS_CENTROID_FEATURE, - KBN_TOO_MANY_FEATURES_PROPERTY, + KBN_METADATA_FEATURE, } from '../../../common/constants'; import { Timeslice } from '../../../common/descriptor_types'; @@ -19,7 +19,7 @@ export interface TimesliceMaskConfig { timeslice: Timeslice; } -export const EXCLUDE_TOO_MANY_FEATURES_BOX = ['!=', ['get', KBN_TOO_MANY_FEATURES_PROPERTY], true]; +export const EXCLUDE_TOO_MANY_FEATURES_BOX = ['!=', ['get', KBN_METADATA_FEATURE], true]; export const EXCLUDE_CENTROID_FEATURES = ['!=', ['get', KBN_IS_CENTROID_FEATURE], true]; function getFilterExpression( diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/index.ts b/x-pack/plugins/maps/public/connected_components/mb_map/index.ts index b9b4b184318f..3084d3b9c9f3 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/index.ts +++ b/x-pack/plugins/maps/public/connected_components/mb_map/index.ts @@ -8,7 +8,7 @@ import { AnyAction } from 'redux'; import { ThunkDispatch } from 'redux-thunk'; import { connect } from 'react-redux'; -import { MBMap } from './mb_map'; +import { MbMap } from './mb_map'; import { clearGoto, clearMouseCoordinates, @@ -19,6 +19,7 @@ import { setAreTilesLoaded, setMapInitError, setMouseCoordinates, + updateMetaFromTiles, } from '../../actions'; import { getGoto, @@ -33,6 +34,7 @@ import { getDrawMode, getIsFullScreen } from '../../selectors/ui_selectors'; import { getInspectorAdapters } from '../../reducers/non_serializable_instances'; import { MapStoreState } from '../../reducers/store'; import { DRAW_MODE } from '../../../common'; +import { TileMetaFeature } from '../../../common/descriptor_types'; function mapStateToProps(state: MapStoreState) { return { @@ -79,8 +81,11 @@ function mapDispatchToProps(dispatch: ThunkDispatch void; timeslice?: Timeslice; + updateMetaFromTiles: (layerId: string, features: TileMetaFeature[]) => void; featureModeActive: boolean; filterModeActive: boolean; } @@ -77,7 +86,7 @@ interface State { mbMap: MapboxMap | undefined; } -export class MBMap extends Component { +export class MbMap extends Component { private _checker?: ResizeChecker; private _isMounted: boolean = false; private _containerRef: HTMLDivElement | null = null; @@ -116,6 +125,16 @@ export class MBMap extends Component { this.props.onMapDestroyed(); } + // This keeps track of the latest update calls, per layerId + _queryForMeta = (layer: ILayer) => { + if (this.state.mbMap && layer.isVisible() && layer.getType() === LAYER_TYPE.TILED_VECTOR) { + const mbFeatures = (layer as TiledVectorLayer).queryTileMetaFeatures(this.state.mbMap); + if (mbFeatures !== null) { + this.props.updateMetaFromTiles(layer.getId(), mbFeatures); + } + } + }; + _debouncedSync = _.debounce(() => { if (this._isMounted && this.props.isMapReady && this.state.mbMap) { const hasLayerListChanged = this._prevLayerList !== this.props.layerList; // Comparing re-select memoized instance so no deep equals needed @@ -184,7 +203,10 @@ export class MBMap extends Component { this._tileStatusTracker = new TileStatusTracker({ mbMap, getCurrentLayerList: () => this.props.layerList, - setAreTilesLoaded: this.props.setAreTilesLoaded, + updateTileStatus: (layer: ILayer, areTilesLoaded: boolean) => { + this.props.setAreTilesLoaded(layer.getId(), areTilesLoaded); + this._queryForMeta(layer); + }, }); const tooManyFeaturesImageSrc = @@ -250,6 +272,7 @@ export class MBMap extends Component { this.props.extentChanged(this._getMapState()); }, 100) ); + // Attach event only if view control is visible, which shows lat/lon if (!this.props.settings.hideViewControl) { const throttledSetMouseCoordinates = _.throttle((e: MapMouseEvent) => { diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.test.ts b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.test.ts index 6b47fe3e6e65..c2995802339c 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.test.ts +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.test.ts @@ -84,8 +84,8 @@ describe('TileStatusTracker', () => { const loadedMap: Map = new Map(); new TileStatusTracker({ mbMap: (mockMbMap as unknown) as MbMap, - setAreTilesLoaded: (layerId, areTilesLoaded) => { - loadedMap.set(layerId, areTilesLoaded); + updateTileStatus: (layer, areTilesLoaded) => { + loadedMap.set(layer.getId(), areTilesLoaded); }, getCurrentLayerList: () => { return [ @@ -127,7 +127,7 @@ describe('TileStatusTracker', () => { const mockMbMap = new MockMbMap(); const tileStatusTracker = new TileStatusTracker({ mbMap: (mockMbMap as unknown) as MbMap, - setAreTilesLoaded: () => {}, + updateTileStatus: () => {}, getCurrentLayerList: () => { return []; }, diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.ts b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.ts index fc99cd3067d0..b25bbf63a1e4 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.ts +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.ts @@ -24,7 +24,7 @@ interface Tile { export class TileStatusTracker { private _tileCache: Tile[]; private readonly _mbMap: MapboxMap; - private readonly _setAreTilesLoaded: (layerId: string, areTilesLoaded: boolean) => void; + private readonly _updateTileStatus: (layer: ILayer, areTilesLoaded: boolean) => void; private readonly _getCurrentLayerList: () => ILayer[]; private readonly _onSourceDataLoading = (e: MapSourceDataEvent) => { if ( @@ -47,7 +47,7 @@ export class TileStatusTracker { mbSourceId: e.sourceId, mbTile: e.tile, }); - this._updateTileStatus(); + this._updateTileStatusForAllLayers(); } } }; @@ -76,15 +76,15 @@ export class TileStatusTracker { constructor({ mbMap, - setAreTilesLoaded, + updateTileStatus, getCurrentLayerList, }: { mbMap: MapboxMap; - setAreTilesLoaded: (layerId: string, areTilesLoaded: boolean) => void; + updateTileStatus: (layer: ILayer, areTilesLoaded: boolean) => void; getCurrentLayerList: () => ILayer[]; }) { this._tileCache = []; - this._setAreTilesLoaded = setAreTilesLoaded; + this._updateTileStatus = updateTileStatus; this._getCurrentLayerList = getCurrentLayerList; this._mbMap = mbMap; @@ -93,7 +93,7 @@ export class TileStatusTracker { this._mbMap.on('sourcedata', this._onSourceData); } - _updateTileStatus = _.debounce(() => { + _updateTileStatusForAllLayers = _.debounce(() => { this._tileCache = this._tileCache.filter((tile) => { return typeof tile.mbTile.aborted === 'boolean' ? !tile.mbTile.aborted : true; }); @@ -108,7 +108,7 @@ export class TileStatusTracker { break; } } - this._setAreTilesLoaded(layer.getId(), !atLeastOnePendingTile); + this._updateTileStatus(layer, !atLeastOnePendingTile); } }, 100); @@ -119,7 +119,7 @@ export class TileStatusTracker { if (trackedIndex >= 0) { this._tileCache.splice(trackedIndex, 1); - this._updateTileStatus(); + this._updateTileStatusForAllLayers(); } }; diff --git a/x-pack/plugins/maps/server/mvt/get_tile.ts b/x-pack/plugins/maps/server/mvt/get_tile.ts index 82d216298650..41a2c550198f 100644 --- a/x-pack/plugins/maps/server/mvt/get_tile.ts +++ b/x-pack/plugins/maps/server/mvt/get_tile.ts @@ -12,15 +12,21 @@ import vtpbf from 'vt-pbf'; import { Logger } from 'src/core/server'; import type { DataRequestHandlerContext } from 'src/plugins/data/server'; import { Feature, FeatureCollection, Polygon } from 'geojson'; +import { countVectorShapeTypes } from '../../common/get_geometry_counts'; import { + COUNT_PROP_NAME, ES_GEO_FIELD_TYPE, FEATURE_ID_PROPERTY_NAME, GEOTILE_GRID_AGG_NAME, - KBN_TOO_MANY_FEATURES_PROPERTY, + KBN_FEATURE_COUNT, + KBN_IS_TILE_COMPLETE, + KBN_METADATA_FEATURE, + KBN_VECTOR_SHAPE_TYPE_COUNTS, MAX_ZOOM, MVT_SOURCE_LAYER_NAME, RENDER_AS, SUPER_FINE_ZOOM_DELTA, + VECTOR_SHAPE_TYPE, } from '../../common/constants'; import { @@ -28,11 +34,18 @@ import { convertRegularRespToGeoJson, hitsToGeoJson, isTotalHitsGreaterThan, + formatEnvelopeAsPolygon, TotalHits, } from '../../common/elasticsearch_util'; import { flattenHit } from './util'; import { ESBounds, tileToESBbox } from '../../common/geo_tile_utils'; import { getCentroidFeatures } from '../../common/get_centroid_features'; +import { pluckRangeFieldMeta } from '../../common/pluck_range_field_meta'; +import { FieldMeta, TileMetaFeature } from '../../common/descriptor_types'; +import { pluckCategoryFieldMeta } from '../../common/pluck_category_field_meta'; + +// heuristic. largest color-palette has 30 colors. 1 color is used for 'other'. +const TERM_COUNT = 30 - 1; function isAbortError(error: Error) { return error.message === 'Request aborted' || error.message === 'Aborted'; @@ -48,7 +61,6 @@ export async function getGridTile({ z, requestBody = {}, requestType = RENDER_AS.POINT, - geoFieldType = ES_GEO_FIELD_TYPE.GEO_POINT, searchSessionId, abortSignal, }: { @@ -60,7 +72,7 @@ export async function getGridTile({ context: DataRequestHandlerContext; logger: Logger; requestBody: any; - requestType: RENDER_AS; + requestType: RENDER_AS.GRID | RENDER_AS.POINT; geoFieldType: ES_GEO_FIELD_TYPE; searchSessionId?: string; abortSignal: AbortSignal; @@ -91,6 +103,79 @@ export async function getGridTile({ ) .toPromise(); const features: Feature[] = convertRegularRespToGeoJson(response.rawResponse, requestType); + + if (features.length) { + const bounds = formatEnvelopeAsPolygon({ + maxLat: tileBounds.top_left.lat, + minLat: tileBounds.bottom_right.lat, + maxLon: tileBounds.bottom_right.lon, + minLon: tileBounds.top_left.lon, + }); + + const fieldNames = new Set(); + features.forEach((feature) => { + for (const key in feature.properties) { + if (feature.properties.hasOwnProperty(key) && key !== 'key' && key !== 'gridCentroid') { + fieldNames.add(key); + } + } + }); + + const fieldMeta: FieldMeta = {}; + fieldNames.forEach((fieldName: string) => { + const rangeMeta = pluckRangeFieldMeta(features, fieldName, (rawValue: unknown) => { + if (fieldName === COUNT_PROP_NAME) { + return parseFloat(rawValue as string); + } else if (typeof rawValue === 'number') { + return rawValue; + } else if (rawValue) { + return parseFloat((rawValue as { value: string }).value); + } else { + return NaN; + } + }); + + const categoryMeta = pluckCategoryFieldMeta(features, fieldName, TERM_COUNT); + + if (!fieldMeta[fieldName]) { + fieldMeta[fieldName] = {}; + } + + if (rangeMeta) { + fieldMeta[fieldName].range = rangeMeta; + } + + if (categoryMeta) { + fieldMeta[fieldName].categories = categoryMeta; + } + }); + + const metaDataFeature: TileMetaFeature = { + type: 'Feature', + properties: { + [KBN_METADATA_FEATURE]: true, + [KBN_FEATURE_COUNT]: features.length, + [KBN_IS_TILE_COMPLETE]: true, + [KBN_VECTOR_SHAPE_TYPE_COUNTS]: + requestType === RENDER_AS.GRID + ? { + [VECTOR_SHAPE_TYPE.POINT]: 0, + [VECTOR_SHAPE_TYPE.LINE]: 0, + [VECTOR_SHAPE_TYPE.POLYGON]: features.length, + } + : { + [VECTOR_SHAPE_TYPE.POINT]: features.length, + [VECTOR_SHAPE_TYPE.LINE]: 0, + [VECTOR_SHAPE_TYPE.POLYGON]: 0, + }, + fieldMeta, + }, + geometry: bounds, + }; + + features.push(metaDataFeature); + } + const featureCollection: FeatureCollection = { features, type: 'FeatureCollection', @@ -99,6 +184,8 @@ export async function getGridTile({ return createMvtTile(featureCollection, z, x, y); } catch (e) { if (!isAbortError(e)) { + // These are often circuit breaking exceptions + // Should return a tile with some error message logger.warn(`Cannot generate grid-tile for ${z}/${x}/${y}: ${e.message}`); } return null; @@ -188,19 +275,25 @@ export async function getTile({ ) .toPromise(); - features = [ - { - type: 'Feature', - properties: { - [KBN_TOO_MANY_FEATURES_PROPERTY]: true, + const metaDataFeature: TileMetaFeature = { + type: 'Feature', + properties: { + [KBN_METADATA_FEATURE]: true, + [KBN_IS_TILE_COMPLETE]: false, + [KBN_FEATURE_COUNT]: 0, + [KBN_VECTOR_SHAPE_TYPE_COUNTS]: { + [VECTOR_SHAPE_TYPE.POINT]: 0, + [VECTOR_SHAPE_TYPE.LINE]: 0, + [VECTOR_SHAPE_TYPE.POLYGON]: 0, }, - geometry: esBboxToGeoJsonPolygon( - // @ts-expect-error @elastic/elasticsearch no way to declare aggregations for search response - bboxResponse.rawResponse.aggregations.data_bounds.bounds, - tileToESBbox(x, y, z) - ), }, - ]; + geometry: esBboxToGeoJsonPolygon( + // @ts-expect-error @elastic/elasticsearch no way to declare aggregations for search response + bboxResponse.rawResponse.aggregations.data_bounds.bounds, + tileToESBbox(x, y, z) + ), + }; + features = [metaDataFeature]; } else { const documentsResponse = await context .search!.search( @@ -217,7 +310,6 @@ export async function getTile({ ) .toPromise(); - // Todo: pass in epochMillies-fields const featureCollection = hitsToGeoJson( // @ts-expect-error hitsToGeoJson should be refactored to accept estypes.SearchHit documentsResponse.rawResponse.hits.hits, @@ -238,6 +330,56 @@ export async function getTile({ props[FEATURE_ID_PROPERTY_NAME] = features[i].id; } } + + const counts = countVectorShapeTypes(features); + + const fieldNames = new Set(); + features.forEach((feature) => { + for (const key in feature.properties) { + if ( + feature.properties.hasOwnProperty(key) && + key !== '_index' && + key !== '_id' && + key !== FEATURE_ID_PROPERTY_NAME + ) { + fieldNames.add(key); + } + } + }); + + const fieldMeta: FieldMeta = {}; + fieldNames.forEach((fieldName: string) => { + const rangeMeta = pluckRangeFieldMeta(features, fieldName, (rawValue: unknown) => { + return typeof rawValue === 'number' ? rawValue : NaN; + }); + const categoryMeta = pluckCategoryFieldMeta(features, fieldName, TERM_COUNT); + + if (!fieldMeta[fieldName]) { + fieldMeta[fieldName] = {}; + } + + if (rangeMeta) { + fieldMeta[fieldName].range = rangeMeta; + } + + if (categoryMeta) { + fieldMeta[fieldName].categories = categoryMeta; + } + }); + + const metadataFeature: TileMetaFeature = { + type: 'Feature', + properties: { + [KBN_METADATA_FEATURE]: true, + [KBN_IS_TILE_COMPLETE]: true, + [KBN_VECTOR_SHAPE_TYPE_COUNTS]: counts, + [KBN_FEATURE_COUNT]: features.length, + fieldMeta, + }, + geometry: esBboxToGeoJsonPolygon(tileToESBbox(x, y, z), tileToESBbox(x, y, z)), + }; + + features.push(metadataFeature); } const featureCollection: FeatureCollection = { diff --git a/x-pack/plugins/maps/server/mvt/mvt_routes.ts b/x-pack/plugins/maps/server/mvt/mvt_routes.ts index b844d5400e2f..01a89aff1a66 100644 --- a/x-pack/plugins/maps/server/mvt/mvt_routes.ts +++ b/x-pack/plugins/maps/server/mvt/mvt_routes.ts @@ -123,7 +123,7 @@ export function initMVTRoutes({ z: parseInt((params as any).z, 10) as number, index: query.index as string, requestBody: requestBodyDSL as any, - requestType: query.requestType as RENDER_AS, + requestType: query.requestType as RENDER_AS.POINT | RENDER_AS.GRID, geoFieldType: query.geoFieldType as ES_GEO_FIELD_TYPE, searchSessionId: query.searchSessionId, abortSignal: abortController.signal, diff --git a/x-pack/test/api_integration/apis/maps/get_grid_tile.js b/x-pack/test/api_integration/apis/maps/get_grid_tile.js index c929a1af7b15..fdb8b2187bbb 100644 --- a/x-pack/test/api_integration/apis/maps/get_grid_tile.js +++ b/x-pack/test/api_integration/apis/maps/get_grid_tile.js @@ -33,13 +33,37 @@ export default function ({ getService }) { const jsonTile = new VectorTile(new Protobuf(resp.body)); const layer = jsonTile.layers[MVT_SOURCE_LAYER_NAME]; - expect(layer.length).to.be(1); + expect(layer.length).to.be(2); + + // Cluster feature const clusterFeature = layer.feature(0); expect(clusterFeature.type).to.be(1); expect(clusterFeature.extent).to.be(4096); expect(clusterFeature.id).to.be(undefined); expect(clusterFeature.properties).to.eql({ doc_count: 1, avg_of_bytes: 9252 }); expect(clusterFeature.loadGeometry()).to.eql([[{ x: 87, y: 667 }]]); + + // Metadata feature + const metadataFeature = layer.feature(1); + expect(metadataFeature.type).to.be(3); + expect(metadataFeature.extent).to.be(4096); + expect(metadataFeature.properties).to.eql({ + __kbn_metadata_feature__: true, + __kbn_feature_count__: 1, + __kbn_is_tile_complete__: true, + __kbn_vector_shape_type_counts__: '{"POINT":1,"LINE":0,"POLYGON":0}', + fieldMeta: + '{"doc_count":{"range":{"min":1,"max":1,"delta":0},"categories":{"categories":[{"key":1,"count":1}]}},"avg_of_bytes":{"range":{"min":9252,"max":9252,"delta":0},"categories":{"categories":[{"key":9252,"count":1}]}}}', + }); + expect(metadataFeature.loadGeometry()).to.eql([ + [ + { x: 0, y: 0 }, + { x: 4096, y: 0 }, + { x: 4096, y: 4096 }, + { x: 0, y: 4096 }, + { x: 0, y: 0 }, + ], + ]); }); it('should return vector tile containing grid features', async () => { @@ -58,7 +82,7 @@ export default function ({ getService }) { const jsonTile = new VectorTile(new Protobuf(resp.body)); const layer = jsonTile.layers[MVT_SOURCE_LAYER_NAME]; - expect(layer.length).to.be(2); + expect(layer.length).to.be(3); const gridFeature = layer.feature(0); expect(gridFeature.type).to.be(3); @@ -75,7 +99,29 @@ export default function ({ getService }) { ], ]); - const clusterFeature = layer.feature(1); + // Metadata feature + const metadataFeature = layer.feature(1); + expect(metadataFeature.type).to.be(3); + expect(metadataFeature.extent).to.be(4096); + expect(metadataFeature.properties).to.eql({ + __kbn_metadata_feature__: true, + __kbn_feature_count__: 1, + __kbn_is_tile_complete__: true, + __kbn_vector_shape_type_counts__: '{"POINT":0,"LINE":0,"POLYGON":1}', + fieldMeta: + '{"doc_count":{"range":{"min":1,"max":1,"delta":0},"categories":{"categories":[{"key":1,"count":1}]}},"avg_of_bytes":{"range":{"min":9252,"max":9252,"delta":0},"categories":{"categories":[{"key":9252,"count":1}]}}}', + }); + expect(metadataFeature.loadGeometry()).to.eql([ + [ + { x: 0, y: 0 }, + { x: 4096, y: 0 }, + { x: 4096, y: 4096 }, + { x: 0, y: 4096 }, + { x: 0, y: 0 }, + ], + ]); + + const clusterFeature = layer.feature(2); expect(clusterFeature.type).to.be(1); expect(clusterFeature.extent).to.be(4096); expect(clusterFeature.id).to.be(undefined); diff --git a/x-pack/test/api_integration/apis/maps/get_tile.js b/x-pack/test/api_integration/apis/maps/get_tile.js index f75d43cf0cee..03a16175931a 100644 --- a/x-pack/test/api_integration/apis/maps/get_tile.js +++ b/x-pack/test/api_integration/apis/maps/get_tile.js @@ -29,7 +29,9 @@ export default function ({ getService }) { const jsonTile = new VectorTile(new Protobuf(resp.body)); const layer = jsonTile.layers[MVT_SOURCE_LAYER_NAME]; - expect(layer.length).to.be(2); + expect(layer.length).to.be(3); // 2 docs + the metadata feature + + // 1st doc const feature = layer.feature(0); expect(feature.type).to.be(1); expect(feature.extent).to.be(4096); @@ -42,6 +44,29 @@ export default function ({ getService }) { ['machine.os.raw']: 'ios', }); expect(feature.loadGeometry()).to.eql([[{ x: 44, y: 2382 }]]); + + // Metadata feature + const metadataFeature = layer.feature(2); + expect(metadataFeature.type).to.be(3); + expect(metadataFeature.extent).to.be(4096); + expect(metadataFeature.id).to.be(undefined); + expect(metadataFeature.properties).to.eql({ + __kbn_feature_count__: 2, + __kbn_is_tile_complete__: true, + __kbn_metadata_feature__: true, + __kbn_vector_shape_type_counts__: '{"POINT":2,"LINE":0,"POLYGON":0}', + fieldMeta: + '{"machine.os.raw":{"categories":{"categories":[{"key":"ios","count":1},{"count":1}]}},"bytes":{"range":{"min":9252,"max":9583,"delta":331},"categories":{"categories":[{"key":9252,"count":1},{"key":9583,"count":1}]}}}', + }); + expect(metadataFeature.loadGeometry()).to.eql([ + [ + { x: 0, y: 4096 }, + { x: 0, y: 0 }, + { x: 4096, y: 0 }, + { x: 4096, y: 4096 }, + { x: 0, y: 4096 }, + ], + ]); }); it('should return vector tile containing bounds when count exceeds size', async () => { @@ -61,12 +86,18 @@ export default function ({ getService }) { const jsonTile = new VectorTile(new Protobuf(resp.body)); const layer = jsonTile.layers[MVT_SOURCE_LAYER_NAME]; expect(layer.length).to.be(1); - const feature = layer.feature(0); - expect(feature.type).to.be(3); - expect(feature.extent).to.be(4096); - expect(feature.id).to.be(undefined); - expect(feature.properties).to.eql({ __kbn_too_many_features__: true }); - expect(feature.loadGeometry()).to.eql([ + + const metadataFeature = layer.feature(0); + expect(metadataFeature.type).to.be(3); + expect(metadataFeature.extent).to.be(4096); + expect(metadataFeature.id).to.be(undefined); + expect(metadataFeature.properties).to.eql({ + __kbn_metadata_feature__: true, + __kbn_feature_count__: 0, + __kbn_is_tile_complete__: false, + __kbn_vector_shape_type_counts__: '{"POINT":0,"LINE":0,"POLYGON":0}', + }); + expect(metadataFeature.loadGeometry()).to.eql([ [ { x: 44, y: 2382 }, { x: 44, y: 1913 }, diff --git a/x-pack/test/functional/apps/maps/mapbox_styles.js b/x-pack/test/functional/apps/maps/mapbox_styles.js index 31cb366f826c..a0a858c30372 100644 --- a/x-pack/test/functional/apps/maps/mapbox_styles.js +++ b/x-pack/test/functional/apps/maps/mapbox_styles.js @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; +import { KBN_IS_TILE_COMPLETE, KBN_METADATA_FEATURE } from '../../../../plugins/maps/common'; export default function ({ getPageObjects, getService }) { const PageObjects = getPageObjects(['maps']); @@ -40,7 +41,7 @@ export default function ({ getPageObjects, getService }) { maxzoom: 24, filter: [ 'all', - ['!=', ['get', '__kbn_too_many_features__'], true], + ['!=', ['get', '__kbn_metadata_feature__'], true], ['!=', ['get', '__kbn_is_centroid_feature__'], true], ['any', ['==', ['geometry-type'], 'Point'], ['==', ['geometry-type'], 'MultiPoint']], ['==', ['get', '__kbn_isvisibleduetojoin__'], true], @@ -121,7 +122,7 @@ export default function ({ getPageObjects, getService }) { maxzoom: 24, filter: [ 'all', - ['!=', ['get', '__kbn_too_many_features__'], true], + ['!=', ['get', '__kbn_metadata_feature__'], true], ['!=', ['get', '__kbn_is_centroid_feature__'], true], ['any', ['==', ['geometry-type'], 'Polygon'], ['==', ['geometry-type'], 'MultiPolygon']], ['==', ['get', '__kbn_isvisibleduetojoin__'], true], @@ -198,7 +199,7 @@ export default function ({ getPageObjects, getService }) { maxzoom: 24, filter: [ 'all', - ['!=', ['get', '__kbn_too_many_features__'], true], + ['!=', ['get', '__kbn_metadata_feature__'], true], ['!=', ['get', '__kbn_is_centroid_feature__'], true], [ 'any', @@ -224,7 +225,11 @@ export default function ({ getPageObjects, getService }) { source: 'n1t6f', minzoom: 0, maxzoom: 24, - filter: ['==', ['get', '__kbn_too_many_features__'], true], + filter: [ + 'all', + ['==', ['get', KBN_METADATA_FEATURE], true], + ['==', ['get', KBN_IS_TILE_COMPLETE], false], + ], layout: { visibility: 'visible' }, paint: { 'fill-pattern': '__kbn_too_many_features_image_id__', 'fill-opacity': 0.75 }, });