Add esTypes property to index pattern field (#35251) (#35438)

Adds an array property to the Field object containing all of the types the field is mapped as across ES indices.
2019-04-23 13:36:30 -04:00 · 2019-04-23 13:36:30 -04:00 · 11a05bd8a4
commit 11a05bd8a4
parent 618d48b142
22 changed files with 114 additions and 7 deletions
--- a/packages/kbn-es-query/src/fixtures/index_pattern_response.json
+++ b/packages/kbn-es-query/src/fixtures/index_pattern_response.json
@ -5,6 +5,7 @@
    {
      "name": "bytes",
      "type": "number",
+      "esTypes": ["long"],
      "count": 10,
      "scripted": false,
      "searchable": true,
@ -14,6 +15,7 @@
    {
      "name": "ssl",
      "type": "boolean",
+      "esTypes": ["boolean"],
      "count": 20,
      "scripted": false,
      "searchable": true,
@ -23,6 +25,7 @@
    {
      "name": "@timestamp",
      "type": "date",
+      "esTypes": ["date"],
      "count": 30,
      "scripted": false,
      "searchable": true,
@ -32,6 +35,7 @@
    {
      "name": "time",
      "type": "date",
+      "esTypes": ["date"],
      "count": 30,
      "scripted": false,
      "searchable": true,
@ -41,6 +45,7 @@
    {
      "name": "@tags",
      "type": "string",
+      "esTypes": ["keyword"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -50,6 +55,7 @@
    {
      "name": "utc_time",
      "type": "date",
+      "esTypes": ["date"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -59,6 +65,7 @@
    {
      "name": "phpmemory",
      "type": "number",
+      "esTypes": ["integer"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -68,6 +75,7 @@
    {
      "name": "ip",
      "type": "ip",
+      "esTypes": ["ip"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -77,6 +85,7 @@
    {
      "name": "request_body",
      "type": "attachment",
+      "esTypes": ["attachment"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -86,6 +95,7 @@
    {
      "name": "point",
      "type": "geo_point",
+      "esTypes": ["geo_point"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -95,6 +105,7 @@
    {
      "name": "area",
      "type": "geo_shape",
+      "esTypes": ["geo_shape"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -104,6 +115,7 @@
    {
      "name": "hashed",
      "type": "murmur3",
+      "esTypes": ["murmur3"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -113,6 +125,7 @@
    {
      "name": "geo.coordinates",
      "type": "geo_point",
+      "esTypes": ["geo_point"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -122,6 +135,7 @@
    {
      "name": "extension",
      "type": "string",
+      "esTypes": ["keyword"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -131,6 +145,7 @@
    {
      "name": "machine.os",
      "type": "string",
+      "esTypes": ["text"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -140,6 +155,7 @@
    {
      "name": "machine.os.raw",
      "type": "string",
+      "esTypes": ["keyword"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -151,6 +167,7 @@
    {
      "name": "geo.src",
      "type": "string",
+      "esTypes": ["keyword"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -160,6 +177,7 @@
    {
      "name": "_id",
      "type": "string",
+      "esTypes": ["_id"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -169,6 +187,7 @@
    {
      "name": "_type",
      "type": "string",
+      "esTypes": ["_type"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -178,6 +197,7 @@
    {
      "name": "_source",
      "type": "_source",
+      "esTypes": ["_source"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -187,6 +207,7 @@
    {
      "name": "non-filterable",
      "type": "string",
+      "esTypes": ["text"],
      "count": 0,
      "scripted": false,
      "searchable": false,
@ -196,6 +217,7 @@
    {
      "name": "non-sortable",
      "type": "string",
+      "esTypes": ["text"],
      "count": 0,
      "scripted": false,
      "searchable": false,
@ -205,6 +227,7 @@
    {
      "name": "custom_user_field",
      "type": "conflict",
+      "esTypes": ["long", "text"],
      "count": 0,
      "scripted": false,
      "searchable": true,
--- a/src/fixtures/logstash_fields.js
+++ b/src/fixtures/logstash_fields.js
@ -78,6 +78,7 @@ function stubbedLogstashFields() {
    return {
      name,
      type,
+      esTypes: [esType],
      readFromDocValues: shouldReadFieldFromDocValues(aggregatable, esType),
      aggregatable,
      searchable,
--- a/src/legacy/core_plugins/metrics/public/components/splits/snapshots/terms.test.js.snap
+++ b/src/legacy/core_plugins/metrics/public/components/splits/snapshots/terms.test.js.snap
@ -45,6 +45,9 @@ exports[`src/legacy/core_plugins/metrics/public/components/splits/terms.test.js
              "kibana_sample_data_flights": Array [
                Object {
                  "aggregatable": true,
+                  "esTypes": Array [
+                    "keyword",
+                  ],
                  "name": "OriginCityName",
                  "readFromDocValues": true,
                  "searchable": true,
--- a/src/legacy/core_plugins/metrics/public/components/splits/terms.test.js
+++ b/src/legacy/core_plugins/metrics/public/components/splits/terms.test.js
@ -51,7 +51,8 @@ describe('src/legacy/core_plugins/metrics/public/components/splits/terms.test.js
            name: 'OriginCityName',
            readFromDocValues: true,
            searchable: true,
-            type: 'string'
+            type: 'string',
+            esTypes: ['keyword'],
          }
        ]
      },
--- a/src/legacy/server/index_patterns/service/index_patterns_service.d.ts
+++ b/src/legacy/server/index_patterns/service/index_patterns_service.d.ts
@ -29,6 +29,7 @@ export interface FieldDescriptor {
  readFromDocValues: boolean;
  searchable: boolean;
  type: string;
+  esTypes: string[];
  parent?: string;
  subType?: string;
 }
--- a/src/legacy/server/index_patterns/service/lib/field_capabilities/field_caps_response.js
+++ b/src/legacy/server/index_patterns/service/lib/field_capabilities/field_caps_response.js
@ -102,6 +102,7 @@ export function readFieldCapsResponse(fieldCapsResponse) {
      return {
        name: fieldName,
        type: 'conflict',
+        esTypes: types,
        searchable: isSearchable,
        aggregatable: isAggregatable,
        readFromDocValues: false,
@ -116,6 +117,7 @@ export function readFieldCapsResponse(fieldCapsResponse) {
    return {
      name: fieldName,
      type: castEsToKbnFieldTypeName(esType),
+      esTypes: types,
      searchable: isSearchable,
      aggregatable: isAggregatable,
      readFromDocValues: shouldReadFieldFromDocValues(isAggregatable, esType),
--- a/src/legacy/server/index_patterns/service/lib/field_capabilities/field_caps_response.test.js
+++ b/src/legacy/server/index_patterns/service/lib/field_capabilities/field_caps_response.test.js
@ -40,7 +40,7 @@ describe('index_patterns/field_capabilities/field_caps_response', () => {
        expect(fields).toHaveLength(22);
      });

-      it('includes only name, type, searchable, aggregatable, readFromDocValues, and maybe conflictDescriptions, parent, ' +
+      it('includes only name, type, esTypes, searchable, aggregatable, readFromDocValues, and maybe conflictDescriptions, parent, ' +
        'and subType of each field', () => {
        const responseClone = cloneDeep(esResponse);
        // try to trick it into including an extra field
@ -53,6 +53,7 @@ describe('index_patterns/field_capabilities/field_caps_response', () => {
          expect(Object.keys(fieldWithoutOptionalKeys)).toEqual([
            'name',
            'type',
+            'esTypes',
            'searchable',
            'aggregatable',
            'readFromDocValues'
@ -76,6 +77,14 @@ describe('index_patterns/field_capabilities/field_caps_response', () => {
        });
      });

+      it('should include the original ES types found for each field across indices', () => {
+        const fields = readFieldCapsResponse(esResponse);
+        fields.forEach((field) => {
+          const fixtureTypes = Object.keys(esResponse.fields[field.name]);
+          expect(field.esTypes).toEqual(fixtureTypes);
+        });
+      });
+
      it('returns fields with multiple types as conflicts', () => {
        const fields = readFieldCapsResponse(esResponse);
        const conflicts = fields.filter(f => f.type === 'conflict');
@ -83,6 +92,7 @@ describe('index_patterns/field_capabilities/field_caps_response', () => {
          {
            name: 'success',
            type: 'conflict',
+            esTypes: ['boolean', 'keyword'],
            searchable: true,
            aggregatable: true,
            readFromDocValues: false,
--- a/src/legacy/server/sample_data/data_sets/ecommerce/saved_objects.js
+++ b/src/legacy/server/sample_data/data_sets/ecommerce/saved_objects.js
--- a/src/legacy/server/sample_data/data_sets/flights/saved_objects.js
+++ b/src/legacy/server/sample_data/data_sets/flights/saved_objects.js
--- a/src/legacy/server/sample_data/data_sets/logs/saved_objects.js
+++ b/src/legacy/server/sample_data/data_sets/logs/saved_objects.js
--- a/src/legacy/ui/public/index_patterns/_field.d.ts
+++ b/src/legacy/ui/public/index_patterns/_field.d.ts
@ -20,6 +20,9 @@
 export interface Field {
  name: string;
  type: string;
+  // esTypes might be undefined on old index patterns that have not been refreshed since we added
+  // this prop. It is also undefined on scripted fields.
+  esTypes?: string[];
  aggregatable: boolean;
  filterable: boolean;
  searchable: boolean;
--- a/src/legacy/ui/public/index_patterns/_field.js
+++ b/src/legacy/ui/public/index_patterns/_field.js
@ -72,6 +72,7 @@ export function Field(indexPattern, spec) {

  obj.fact('name');
  obj.fact('type');
+  obj.fact('esTypes');
  obj.writ('count', spec.count || 0);

  // scripted objs
--- a/src/legacy/ui/public/index_patterns/fixtures/index.ts
+++ b/src/legacy/ui/public/index_patterns/fixtures/index.ts
@ -22,6 +22,7 @@ import { Field, IndexPattern } from '../index';
 export const mockFields: Field[] = [
  {
    name: 'machine.os',
+    esTypes: ['text'],
    type: 'string',
    aggregatable: false,
    searchable: false,
@ -30,6 +31,7 @@ export const mockFields: Field[] = [
  {
    name: 'machine.os.raw',
    type: 'string',
+    esTypes: ['keyword'],
    aggregatable: true,
    searchable: true,
    filterable: true,
@ -37,6 +39,7 @@ export const mockFields: Field[] = [
  {
    name: 'not.filterable',
    type: 'string',
+    esTypes: ['text'],
    aggregatable: true,
    searchable: false,
    filterable: false,
@ -44,6 +47,7 @@ export const mockFields: Field[] = [
  {
    name: 'bytes',
    type: 'number',
+    esTypes: ['long'],
    aggregatable: true,
    searchable: true,
    filterable: true,
@ -51,6 +55,7 @@ export const mockFields: Field[] = [
  {
    name: '@timestamp',
    type: 'date',
+    esTypes: ['date'],
    aggregatable: true,
    searchable: true,
    filterable: true,
@ -58,6 +63,7 @@ export const mockFields: Field[] = [
  {
    name: 'clientip',
    type: 'ip',
+    esTypes: ['ip'],
    aggregatable: true,
    searchable: true,
    filterable: true,
@ -65,6 +71,7 @@ export const mockFields: Field[] = [
  {
    name: 'bool.field',
    type: 'boolean',
+    esTypes: ['boolean'],
    aggregatable: true,
    searchable: true,
    filterable: true,
--- a/src/legacy/ui/public/query_bar/components/query_bar.test.tsx
+++ b/src/legacy/ui/public/query_bar/components/query_bar.test.tsx
@ -69,6 +69,7 @@ const mockIndexPattern = {
    {
      name: 'response',
      type: 'number',
+      esTypes: ['integer'],
      aggregatable: true,
      filterable: true,
      searchable: true,
--- a/src/legacy/ui/public/timefilter/get_time.test.ts
+++ b/src/legacy/ui/public/timefilter/get_time.test.ts
@ -34,7 +34,14 @@ describe('get_time', () => {
          title: 'test',
          timeFieldName: 'date',
          fields: [
-            { name: 'date', type: 'date', aggregatable: true, searchable: true, filterable: true },
+            {
+              name: 'date',
+              type: 'date',
+              esTypes: ['date'],
+              aggregatable: true,
+              searchable: true,
+              filterable: true,
+            },
          ],
        },
        { from: 'now-60y', to: 'now' }
--- a/src/legacy/ui/public/vis/editors/config/editor_config_providers.test.ts
+++ b/src/legacy/ui/public/vis/editors/config/editor_config_providers.test.ts
@ -29,6 +29,7 @@ describe('EditorConfigProvider', () => {
      {
        name: 'response',
        type: 'number',
+        esTypes: ['integer'],
        aggregatable: true,
        filterable: true,
        searchable: true,
--- a/test/api_integration/apis/index_patterns/fields_for_time_pattern_route/pattern.js
+++ b/test/api_integration/apis/index_patterns/fields_for_time_pattern_route/pattern.js
@ -41,6 +41,7 @@ export default function ({ getService }) {
              {
                name: '@timestamp',
                type: 'date',
+                esTypes: ['date'],
                aggregatable: true,
                searchable: true,
                readFromDocValues: true,
@ -48,6 +49,7 @@ export default function ({ getService }) {
              {
                name: 'Jan01',
                type: 'boolean',
+                esTypes: ['boolean'],
                aggregatable: true,
                searchable: true,
                readFromDocValues: true,
@ -55,6 +57,7 @@ export default function ({ getService }) {
              {
                name: 'Jan02',
                type: 'boolean',
+                esTypes: ['boolean'],
                aggregatable: true,
                searchable: true,
                readFromDocValues: true,
@ -78,6 +81,7 @@ export default function ({ getService }) {
              {
                name: '@timestamp',
                type: 'date',
+                esTypes: ['date'],
                aggregatable: true,
                searchable: true,
                readFromDocValues: true,
@ -85,6 +89,7 @@ export default function ({ getService }) {
              {
                name: 'Jan02',
                type: 'boolean',
+                esTypes: ['boolean'],
                aggregatable: true,
                searchable: true,
                readFromDocValues: true,
@ -109,6 +114,7 @@ export default function ({ getService }) {
              {
                name: '@timestamp',
                type: 'date',
+                esTypes: ['date'],
                aggregatable: true,
                searchable: true,
                readFromDocValues: true,
@ -116,6 +122,7 @@ export default function ({ getService }) {
              {
                name: 'Jan02',
                type: 'boolean',
+                esTypes: ['boolean'],
                aggregatable: true,
                searchable: true,
                readFromDocValues: true,
--- a/test/api_integration/apis/index_patterns/fields_for_wildcard_route/conflicts.js
+++ b/test/api_integration/apis/index_patterns/fields_for_wildcard_route/conflicts.js
@ -38,6 +38,7 @@ export default function ({ getService }) {
              {
                name: '@timestamp',
                type: 'date',
+                esTypes: ['date'],
                aggregatable: true,
                searchable: true,
                readFromDocValues: true,
@ -45,6 +46,7 @@ export default function ({ getService }) {
              {
                name: 'number_conflict',
                type: 'number',
+                esTypes: ['integer', 'float'],
                aggregatable: true,
                searchable: true,
                readFromDocValues: true,
@ -52,6 +54,7 @@ export default function ({ getService }) {
              {
                name: 'string_conflict',
                type: 'string',
+                esTypes: ['text', 'keyword'],
                aggregatable: true,
                searchable: true,
                readFromDocValues: false,
@ -59,6 +62,7 @@ export default function ({ getService }) {
              {
                name: 'success',
                type: 'conflict',
+                esTypes: ['boolean', 'keyword'],
                aggregatable: true,
                searchable: true,
                readFromDocValues: false,
--- a/test/api_integration/apis/index_patterns/fields_for_wildcard_route/response.js
+++ b/test/api_integration/apis/index_patterns/fields_for_wildcard_route/response.js
@ -41,6 +41,7 @@ export default function ({ getService }) {
          fields: [
            {
              type: 'boolean',
+              esTypes: ['boolean'],
              searchable: true,
              aggregatable: true,
              name: 'bar',
@ -48,6 +49,7 @@ export default function ({ getService }) {
            },
            {
              type: 'string',
+              esTypes: ['text'],
              searchable: true,
              aggregatable: false,
              name: 'baz',
@ -55,6 +57,7 @@ export default function ({ getService }) {
            },
            {
              type: 'string',
+              esTypes: ['keyword'],
              searchable: true,
              aggregatable: true,
              name: 'baz.keyword',
@ -64,6 +67,7 @@ export default function ({ getService }) {
            },
            {
              type: 'number',
+              esTypes: ['long'],
              searchable: true,
              aggregatable: true,
              name: 'foo',
@ -90,6 +94,7 @@ export default function ({ getService }) {
            {
              aggregatable: true,
              name: '_id',
+              esTypes: ['_id'],
              readFromDocValues: false,
              searchable: true,
              type: 'string',
@ -97,12 +102,14 @@ export default function ({ getService }) {
            {
              aggregatable: false,
              name: '_source',
+              esTypes: ['_source'],
              readFromDocValues: false,
              searchable: false,
              type: '_source',
            },
            {
              type: 'boolean',
+              esTypes: ['boolean'],
              searchable: true,
              aggregatable: true,
              name: 'bar',
@ -111,12 +118,14 @@ export default function ({ getService }) {
            {
              aggregatable: false,
              name: 'baz',
+              esTypes: ['text'],
              readFromDocValues: false,
              searchable: true,
              type: 'string',
            },
            {
              type: 'string',
+              esTypes: ['keyword'],
              searchable: true,
              aggregatable: true,
              name: 'baz.keyword',
@ -133,6 +142,7 @@ export default function ({ getService }) {
            },
            {
              type: 'number',
+              esTypes: ['long'],
              searchable: true,
              aggregatable: true,
              name: 'foo',
--- a/test/common/config.js
+++ b/test/common/config.js
@ -55,7 +55,7 @@ export default function () {
        `--elasticsearch.username=${servers.elasticsearch.username}`,
        `--elasticsearch.password=${servers.elasticsearch.password}`,
        `--kibana.disableWelcomeScreen=true`,
-        `--server.maxPayloadBytes=1648576`,
+        `--server.maxPayloadBytes=1679958`,
      ],
    },

--- a/x-pack/plugins/kuery_autocomplete/public/autocomplete_providers/fixtures/index_pattern_response.json
+++ b/x-pack/plugins/kuery_autocomplete/public/autocomplete_providers/fixtures/index_pattern_response.json
@ -5,6 +5,7 @@
    {
      "name": "bytes",
      "type": "number",
+      "esTypes": ["long"],
      "count": 10,
      "scripted": false,
      "searchable": true,
@ -14,6 +15,7 @@
    {
      "name": "ssl",
      "type": "boolean",
+      "esTypes": ["boolean"],
      "count": 20,
      "scripted": false,
      "searchable": true,
@ -23,6 +25,7 @@
    {
      "name": "@timestamp",
      "type": "date",
+      "esTypes": ["date"],
      "count": 30,
      "scripted": false,
      "searchable": true,
@ -32,6 +35,7 @@
    {
      "name": "time",
      "type": "date",
+      "esTypes": ["date"],
      "count": 30,
      "scripted": false,
      "searchable": true,
@ -41,6 +45,7 @@
    {
      "name": "@tags",
      "type": "string",
+      "esTypes": ["keyword"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -50,6 +55,7 @@
    {
      "name": "utc_time",
      "type": "date",
+      "esTypes": ["date"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -59,6 +65,7 @@
    {
      "name": "phpmemory",
      "type": "number",
+      "esTypes": ["integer"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -68,6 +75,7 @@
    {
      "name": "ip",
      "type": "ip",
+      "esTypes": ["ip"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -77,6 +85,7 @@
    {
      "name": "request_body",
      "type": "attachment",
+      "esTypes": ["attachment"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -86,6 +95,7 @@
    {
      "name": "point",
      "type": "geo_point",
+      "esTypes": ["geo_point"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -95,6 +105,7 @@
    {
      "name": "area",
      "type": "geo_shape",
+      "esTypes": ["geo_shape"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -104,6 +115,7 @@
    {
      "name": "hashed",
      "type": "murmur3",
+      "esTypes": ["murmur3"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -113,6 +125,7 @@
    {
      "name": "geo.coordinates",
      "type": "geo_point",
+      "esTypes": ["geo_point"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -122,6 +135,7 @@
    {
      "name": "extension",
      "type": "string",
+      "esTypes": ["keyword"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -131,6 +145,7 @@
    {
      "name": "machine.os",
      "type": "string",
+      "esTypes": ["text"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -140,6 +155,7 @@
    {
      "name": "machine.os.raw",
      "type": "string",
+      "esTypes": ["keyword"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -151,6 +167,7 @@
    {
      "name": "geo.src",
      "type": "string",
+      "esTypes": ["keyword"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -160,6 +177,7 @@
    {
      "name": "_id",
      "type": "string",
+      "esTypes": ["_id"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -169,6 +187,7 @@
    {
      "name": "_type",
      "type": "string",
+      "esTypes": ["_type"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -178,6 +197,7 @@
    {
      "name": "_source",
      "type": "_source",
+      "esTypes": ["_source"],
      "count": 0,
      "scripted": false,
      "searchable": true,
@ -187,6 +207,7 @@
    {
      "name": "non-filterable",
      "type": "string",
+      "esTypes": ["text"],
      "count": 0,
      "scripted": false,
      "searchable": false,
@ -196,6 +217,7 @@
    {
      "name": "non-sortable",
      "type": "string",
+      "esTypes": ["text"],
      "count": 0,
      "scripted": false,
      "searchable": false,
@ -205,6 +227,7 @@
    {
      "name": "custom_user_field",
      "type": "conflict",
+      "esTypes": ["long", "text"],
      "count": 0,
      "scripted": false,
      "searchable": true,
--- a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js
+++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js
@ -25,6 +25,7 @@ describe('Rollup Search Strategy', () => {
          {
            name: 'day_of_week.terms.value',
            type: 'object',
+            esTypes: ['object'],
            searchable: false,
            aggregatable: false,
          },
@ -146,6 +147,7 @@ describe('Rollup Search Strategy', () => {
          readFromDocValues: true,
          searchable: true,
          type: 'object',
+          esTypes: ['object'],
        },
      ]);
    });