[data.search.searchSource] Update SearchSource to use Fields API. (#82383)

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getfields.md

@@ -21,7 +21,8 @@ getFields(): {
         size?: number | undefined;
         source?: string | boolean | string[] | undefined;
         version?: boolean | undefined;
-        fields?: string | boolean | string[] | undefined;
+        fields?: SearchFieldValue[] | undefined;
+        fieldsFromSource?: string | boolean | string[] | undefined;
         index?: import("../..").IndexPattern | undefined;
         searchAfter?: import("./types").EsQuerySearchAfter | undefined;
         timeout?: string | undefined;
@@ -42,7 +43,8 @@ getFields(): {
         size?: number | undefined;
         source?: string | boolean | string[] | undefined;
         version?: boolean | undefined;
-        fields?: string | boolean | string[] | undefined;
+        fields?: SearchFieldValue[] | undefined;
+        fieldsFromSource?: string | boolean | string[] | undefined;
         index?: import("../..").IndexPattern | undefined;
         searchAfter?: import("./types").EsQuerySearchAfter | undefined;
         timeout?: string | undefined;

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.fields.md

@@ -4,8 +4,10 @@
 
 ## SearchSourceFields.fields property
 
+Retrieve fields via the search Fields API
+
 <b>Signature:</b>
 
 ```typescript
-fields?: NameList;
+fields?: SearchFieldValue[];
 ```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.fieldsfromsource.md

@@ -0,0 +1,18 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [SearchSourceFields](./kibana-plugin-plugins-data-public.searchsourcefields.md) &gt; [fieldsFromSource](./kibana-plugin-plugins-data-public.searchsourcefields.fieldsfromsource.md)
+
+## SearchSourceFields.fieldsFromSource property
+
+> Warning: This API is now obsolete.
+> 
+> It is recommended to use `fields` wherever possible.
+> 
+
+Retreive fields directly from \_source (legacy behavior)
+
+<b>Signature:</b>
+
+```typescript
+fieldsFromSource?: NameList;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md

@@ -17,7 +17,8 @@ export interface SearchSourceFields
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [aggs](./kibana-plugin-plugins-data-public.searchsourcefields.aggs.md) | <code>any</code> | [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) |
-|  [fields](./kibana-plugin-plugins-data-public.searchsourcefields.fields.md) | <code>NameList</code> |  |
+|  [fields](./kibana-plugin-plugins-data-public.searchsourcefields.fields.md) | <code>SearchFieldValue[]</code> | Retrieve fields via the search Fields API |
+|  [fieldsFromSource](./kibana-plugin-plugins-data-public.searchsourcefields.fieldsfromsource.md) | <code>NameList</code> | Retreive fields directly from \_source (legacy behavior) |
 |  [filter](./kibana-plugin-plugins-data-public.searchsourcefields.filter.md) | <code>Filter[] &#124; Filter &#124; (() =&gt; Filter[] &#124; Filter &#124; undefined)</code> | [Filter](./kibana-plugin-plugins-data-public.filter.md) |
 |  [from](./kibana-plugin-plugins-data-public.searchsourcefields.from.md) | <code>number</code> |  |
 |  [highlight](./kibana-plugin-plugins-data-public.searchsourcefields.highlight.md) | <code>any</code> |  |

examples/search_examples/public/components/app.tsx

@@ -23,7 +23,8 @@ import { FormattedMessage, I18nProvider } from '@kbn/i18n/react';
 import { BrowserRouter as Router } from 'react-router-dom';
 
 import {
-  EuiButton,
+  EuiButtonEmpty,
+  EuiCodeBlock,
   EuiPage,
   EuiPageBody,
   EuiPageContent,
@@ -32,6 +33,7 @@ import {
   EuiTitle,
   EuiText,
   EuiFlexGrid,
+  EuiFlexGroup,
   EuiFlexItem,
   EuiCheckbox,
   EuiSpacer,
@@ -68,6 +70,11 @@ interface SearchExamplesAppDeps {
   data: DataPublicPluginStart;
 }
 
+function getNumeric(fields?: IndexPatternField[]) {
+  if (!fields) return [];
+  return fields?.filter((f) => f.type === 'number' && f.aggregatable);
+}
+
 function formatFieldToComboBox(field?: IndexPatternField | null) {
   if (!field) return [];
   return formatFieldsToComboBox([field]);
@@ -95,8 +102,13 @@ export const SearchExamplesApp = ({
   const [getCool, setGetCool] = useState<boolean>(false);
   const [timeTook, setTimeTook] = useState<number | undefined>();
   const [indexPattern, setIndexPattern] = useState<IndexPattern | null>();
-  const [numericFields, setNumericFields] = useState<IndexPatternField[]>();
+  const [fields, setFields] = useState<IndexPatternField[]>();
-  const [selectedField, setSelectedField] = useState<IndexPatternField | null | undefined>();
+  const [selectedFields, setSelectedFields] = useState<IndexPatternField[]>([]);
+  const [selectedNumericField, setSelectedNumericField] = useState<
+    IndexPatternField | null | undefined
+  >();
+  const [request, setRequest] = useState<Record<string, any>>({});
+  const [response, setResponse] = useState<Record<string, any>>({});
 
   // Fetch the default index pattern using the `data.indexPatterns` service, as the component is mounted.
   useEffect(() => {
@@ -110,24 +122,23 @@ export const SearchExamplesApp = ({
 
   // Update the fields list every time the index pattern is modified.
   useEffect(() => {
-    const fields = indexPattern?.fields.filter(
+    setFields(indexPattern?.fields);
-      (field) => field.type === 'number' && field.aggregatable
-    );
-    setNumericFields(fields);
-    setSelectedField(fields?.length ? fields[0] : null);
   }, [indexPattern]);
+  useEffect(() => {
+    setSelectedNumericField(fields?.length ? getNumeric(fields)[0] : null);
+  }, [fields]);
 
   const doAsyncSearch = async (strategy?: string) => {
-    if (!indexPattern || !selectedField) return;
+    if (!indexPattern || !selectedNumericField) return;
 
     // Constuct the query portion of the search request
     const query = data.query.getEsQuery(indexPattern);
 
     // Constuct the aggregations portion of the search request by using the `data.search.aggs` service.
-    const aggs = [{ type: 'avg', params: { field: selectedField.name } }];
+    const aggs = [{ type: 'avg', params: { field: selectedNumericField!.name } }];
     const aggsDsl = data.search.aggs.createAggConfigs(indexPattern, aggs).toDsl();
 
-    const request = {
+    const req = {
       params: {
         index: indexPattern.title,
         body: {
@@ -140,23 +151,26 @@ export const SearchExamplesApp = ({
     };
 
     // Submit the search request using the `data.search` service.
+    setRequest(req.params.body);
     const searchSubscription$ = data.search
-      .search(request, {
+      .search(req, {
         strategy,
       })
       .subscribe({
-        next: (response) => {
+        next: (res) => {
-          if (isCompleteResponse(response)) {
+          if (isCompleteResponse(res)) {
-            setTimeTook(response.rawResponse.took);
+            setResponse(res.rawResponse);
-            const avgResult: number | undefined = response.rawResponse.aggregations
+            setTimeTook(res.rawResponse.took);
-              ? response.rawResponse.aggregations[1].value
+            const avgResult: number | undefined = res.rawResponse.aggregations
+              ? res.rawResponse.aggregations[1].value
               : undefined;
             const message = (
               <EuiText>
-                Searched {response.rawResponse.hits.total} documents. <br />
+                Searched {res.rawResponse.hits.total} documents. <br />
-                The average of {selectedField.name} is {avgResult ? Math.floor(avgResult) : 0}.
+                The average of {selectedNumericField!.name} is{' '}
+                {avgResult ? Math.floor(avgResult) : 0}.
                 <br />
-                Is it Cool? {String((response as IMyStrategyResponse).cool)}
+                Is it Cool? {String((res as IMyStrategyResponse).cool)}
               </EuiText>
             );
             notifications.toasts.addSuccess({
@@ -164,7 +178,7 @@ export const SearchExamplesApp = ({
               text: mountReactNode(message),
             });
             searchSubscription$.unsubscribe();
-          } else if (isErrorResponse(response)) {
+          } else if (isErrorResponse(res)) {
             // TODO: Make response error status clearer
             notifications.toasts.addWarning('An error has occurred');
             searchSubscription$.unsubscribe();
@@ -176,6 +190,50 @@ export const SearchExamplesApp = ({
       });
   };
 
+  const doSearchSourceSearch = async () => {
+    if (!indexPattern) return;
+
+    const query = data.query.queryString.getQuery();
+    const filters = data.query.filterManager.getFilters();
+    const timefilter = data.query.timefilter.timefilter.createFilter(indexPattern);
+    if (timefilter) {
+      filters.push(timefilter);
+    }
+
+    try {
+      const searchSource = await data.search.searchSource.create();
+
+      searchSource
+        .setField('index', indexPattern)
+        .setField('filter', filters)
+        .setField('query', query)
+        .setField('fields', selectedFields.length ? selectedFields.map((f) => f.name) : ['*']);
+
+      if (selectedNumericField) {
+        searchSource.setField('aggs', () => {
+          return data.search.aggs
+            .createAggConfigs(indexPattern, [
+              { type: 'avg', params: { field: selectedNumericField.name } },
+            ])
+            .toDsl();
+        });
+      }
+
+      setRequest(await searchSource.getSearchRequestBody());
+      const res = await searchSource.fetch();
+      setResponse(res);
+
+      const message = <EuiText>Searched {res.hits.total} documents.</EuiText>;
+      notifications.toasts.addSuccess({
+        title: 'Query result',
+        text: mountReactNode(message),
+      });
+    } catch (e) {
+      setResponse(e.body);
+      notifications.toasts.addWarning(`An error has occurred: ${e.message}`);
+    }
+  };
+
   const onClickHandler = () => {
     doAsyncSearch();
   };
@@ -185,22 +243,24 @@ export const SearchExamplesApp = ({
   };
 
   const onServerClickHandler = async () => {
-    if (!indexPattern || !selectedField) return;
+    if (!indexPattern || !selectedNumericField) return;
     try {
-      const response = await http.get(SERVER_SEARCH_ROUTE_PATH, {
+      const res = await http.get(SERVER_SEARCH_ROUTE_PATH, {
         query: {
           index: indexPattern.title,
-          field: selectedField.name,
+          field: selectedNumericField!.name,
         },
       });
 
-      notifications.toasts.addSuccess(`Server returned ${JSON.stringify(response)}`);
+      notifications.toasts.addSuccess(`Server returned ${JSON.stringify(res)}`);
     } catch (e) {
       notifications.toasts.addDanger('Failed to run search');
     }
   };
 
-  if (!indexPattern) return null;
+  const onSearchSourceClickHandler = () => {
+    doSearchSourceSearch();
+  };
 
   return (
     <Router basename={basename}>
@@ -212,7 +272,7 @@ export const SearchExamplesApp = ({
             useDefaultBehaviors={true}
             indexPatterns={indexPattern ? [indexPattern] : undefined}
           />
-          <EuiPage restrictWidth="1000px">
+          <EuiPage>
             <EuiPageBody>
               <EuiPageHeader>
                 <EuiTitle size="l">
@@ -227,106 +287,178 @@ export const SearchExamplesApp = ({
               </EuiPageHeader>
               <EuiPageContent>
                 <EuiPageContentBody>
-                  <EuiText>
+                  <EuiFlexGrid columns={3}>
-                    <EuiFlexGrid columns={1}>
+                    <EuiFlexItem style={{ width: '40%' }}>
-                      <EuiFlexItem>
+                      <EuiText>
-                        <EuiFormLabel>Index Pattern</EuiFormLabel>
+                        <EuiFlexGrid columns={2}>
-                        <IndexPatternSelect
+                          <EuiFlexItem>
-                          placeholder={i18n.translate(
+                            <EuiFormLabel>Index Pattern</EuiFormLabel>
-                            'backgroundSessionExample.selectIndexPatternPlaceholder',
+                            <IndexPatternSelect
-                            {
+                              placeholder={i18n.translate(
-                              defaultMessage: 'Select index pattern',
+                                'backgroundSessionExample.selectIndexPatternPlaceholder',
-                            }
+                                {
-                          )}
+                                  defaultMessage: 'Select index pattern',
-                          indexPatternId={indexPattern?.id || ''}
+                                }
-                          onChange={async (newIndexPatternId: any) => {
+                              )}
-                            const newIndexPattern = await data.indexPatterns.get(newIndexPatternId);
+                              indexPatternId={indexPattern?.id || ''}
-                            setIndexPattern(newIndexPattern);
+                              onChange={async (newIndexPatternId: any) => {
-                          }}
+                                const newIndexPattern = await data.indexPatterns.get(
-                          isClearable={false}
+                                  newIndexPatternId
+                                );
+                                setIndexPattern(newIndexPattern);
+                              }}
+                              isClearable={false}
+                            />
+                          </EuiFlexItem>
+                          <EuiFlexItem>
+                            <EuiFormLabel>Numeric Field to Aggregate</EuiFormLabel>
+                            <EuiComboBox
+                              options={formatFieldsToComboBox(getNumeric(fields))}
+                              selectedOptions={formatFieldToComboBox(selectedNumericField)}
+                              singleSelection={true}
+                              onChange={(option) => {
+                                const fld = indexPattern?.getFieldByName(option[0].label);
+                                setSelectedNumericField(fld || null);
+                              }}
+                              sortMatchesBy="startsWith"
+                            />
+                          </EuiFlexItem>
+                        </EuiFlexGrid>
+                        <EuiFlexGroup>
+                          <EuiFlexItem>
+                            <EuiFormLabel>
+                              Fields to query (leave blank to include all fields)
+                            </EuiFormLabel>
+                            <EuiComboBox
+                              options={formatFieldsToComboBox(fields)}
+                              selectedOptions={formatFieldsToComboBox(selectedFields)}
+                              singleSelection={false}
+                              onChange={(option) => {
+                                const flds = option
+                                  .map((opt) => indexPattern?.getFieldByName(opt?.label))
+                                  .filter((f) => f);
+                                setSelectedFields(flds.length ? (flds as IndexPatternField[]) : []);
+                              }}
+                              sortMatchesBy="startsWith"
+                            />
+                          </EuiFlexItem>
+                        </EuiFlexGroup>
+                      </EuiText>
+                      <EuiSpacer />
+                      <EuiTitle size="s">
+                        <h3>
+                          Searching Elasticsearch using <EuiCode>data.search</EuiCode>
+                        </h3>
+                      </EuiTitle>
+                      <EuiText>
+                        If you want to fetch data from Elasticsearch, you can use the different
+                        services provided by the <EuiCode>data</EuiCode> plugin. These help you get
+                        the index pattern and search bar configuration, format them into a DSL query
+                        and send it to Elasticsearch.
+                        <EuiSpacer />
+                        <EuiButtonEmpty size="xs" onClick={onClickHandler} iconType="play">
+                          <FormattedMessage
+                            id="searchExamples.buttonText"
+                            defaultMessage="Request from low-level client (data.search.search)"
+                          />
+                        </EuiButtonEmpty>
+                        <EuiButtonEmpty
+                          size="xs"
+                          onClick={onSearchSourceClickHandler}
+                          iconType="play"
+                        >
+                          <FormattedMessage
+                            id="searchExamples.searchSource.buttonText"
+                            defaultMessage="Request from high-level client (data.search.searchSource)"
+                          />
+                        </EuiButtonEmpty>
+                      </EuiText>
+                      <EuiSpacer />
+                      <EuiTitle size="s">
+                        <h3>Writing a custom search strategy</h3>
+                      </EuiTitle>
+                      <EuiText>
+                        If you want to do some pre or post processing on the server, you might want
+                        to create a custom search strategy. This example uses such a strategy,
+                        passing in custom input and receiving custom output back.
+                        <EuiSpacer />
+                        <EuiCheckbox
+                          id="GetCool"
+                          label={
+                            <FormattedMessage
+                              id="searchExamples.getCoolCheckbox"
+                              defaultMessage="Get cool parameter?"
+                            />
+                          }
+                          checked={getCool}
+                          onChange={(event) => setGetCool(event.target.checked)}
                         />
-                      </EuiFlexItem>
+                        <EuiButtonEmpty
-                      <EuiFlexItem>
+                          size="xs"
-                        <EuiFormLabel>Numeric Fields</EuiFormLabel>
+                          onClick={onMyStrategyClickHandler}
-                        <EuiComboBox
+                          iconType="play"
-                          options={formatFieldsToComboBox(numericFields)}
+                        >
-                          selectedOptions={formatFieldToComboBox(selectedField)}
+                          <FormattedMessage
-                          singleSelection={true}
+                            id="searchExamples.myStrategyButtonText"
-                          onChange={(option) => {
+                            defaultMessage="Request from low-level client via My Strategy"
-                            const field = indexPattern.getFieldByName(option[0].label);
+                          />
-                            setSelectedField(field || null);
+                        </EuiButtonEmpty>
-                          }}
+                      </EuiText>
-                          sortMatchesBy="startsWith"
+                      <EuiSpacer />
-                        />
+                      <EuiTitle size="s">
-                      </EuiFlexItem>
+                        <h3>Using search on the server</h3>
-                    </EuiFlexGrid>
+                      </EuiTitle>
-                  </EuiText>
+                      <EuiText>
-                  <EuiText>
+                        You can also run your search request from the server, without registering a
-                    <FormattedMessage
+                        search strategy. This request does not take the configuration of{' '}
-                      id="searchExamples.timestampText"
+                        <EuiCode>TopNavMenu</EuiCode> into account, but you could pass those down to
-                      defaultMessage="Last query took: {time} ms"
+                        the server as well.
-                      values={{ time: timeTook || 'Unknown' }}
+                        <EuiSpacer />
-                    />
+                        <EuiButtonEmpty size="xs" onClick={onServerClickHandler} iconType="play">
-                  </EuiText>
+                          <FormattedMessage
-                  <EuiSpacer />
+                            id="searchExamples.myServerButtonText"
-                  <EuiTitle size="s">
+                            defaultMessage="Request from low-level client on the server"
-                    <h3>
+                          />
-                      Searching Elasticsearch using <EuiCode>data.search</EuiCode>
+                        </EuiButtonEmpty>
-                    </h3>
+                      </EuiText>
-                  </EuiTitle>
+                    </EuiFlexItem>
-                  <EuiText>
+                    <EuiFlexItem style={{ width: '30%' }}>
-                    If you want to fetch data from Elasticsearch, you can use the different services
+                      <EuiTitle size="xs">
-                    provided by the <EuiCode>data</EuiCode> plugin. These help you get the index
+                        <h4>Request</h4>
-                    pattern and search bar configuration, format them into a DSL query and send it
+                      </EuiTitle>
-                    to Elasticsearch.
+                      <EuiText size="xs">Search body sent to ES</EuiText>
-                    <EuiSpacer />
+                      <EuiCodeBlock
-                    <EuiButton type="primary" size="s" onClick={onClickHandler}>
+                        language="json"
-                      <FormattedMessage id="searchExamples.buttonText" defaultMessage="Get data" />
+                        fontSize="s"
-                    </EuiButton>
+                        paddingSize="s"
-                  </EuiText>
+                        overflowHeight={450}
-                  <EuiSpacer />
+                        isCopyable
-                  <EuiTitle size="s">
+                      >
-                    <h3>Writing a custom search strategy</h3>
+                        {JSON.stringify(request, null, 2)}
-                  </EuiTitle>
+                      </EuiCodeBlock>
-                  <EuiText>
+                    </EuiFlexItem>
-                    If you want to do some pre or post processing on the server, you might want to
+                    <EuiFlexItem style={{ width: '30%' }}>
-                    create a custom search strategy. This example uses such a strategy, passing in
+                      <EuiTitle size="xs">
-                    custom input and receiving custom output back.
+                        <h4>Response</h4>
-                    <EuiSpacer />
+                      </EuiTitle>
-                    <EuiCheckbox
+                      <EuiText size="xs">
-                      id="GetCool"
-                      label={
                         <FormattedMessage
-                          id="searchExamples.getCoolCheckbox"
+                          id="searchExamples.timestampText"
-                          defaultMessage="Get cool parameter?"
+                          defaultMessage="Took: {time} ms"
+                          values={{ time: timeTook || 'Unknown' }}
                         />
-                      }
+                      </EuiText>
-                      checked={getCool}
+                      <EuiCodeBlock
-                      onChange={(event) => setGetCool(event.target.checked)}
+                        language="json"
-                    />
+                        fontSize="s"
-                    <EuiButton type="primary" size="s" onClick={onMyStrategyClickHandler}>
+                        paddingSize="s"
-                      <FormattedMessage
+                        overflowHeight={450}
-                        id="searchExamples.myStrategyButtonText"
+                        isCopyable
-                        defaultMessage="Get data via My Strategy"
+                      >
-                      />
+                        {JSON.stringify(response, null, 2)}
-                    </EuiButton>
+                      </EuiCodeBlock>
-                  </EuiText>
+                    </EuiFlexItem>
-                  <EuiSpacer />
+                  </EuiFlexGrid>
-                  <EuiTitle size="s">
-                    <h3>Using search on the server</h3>
-                  </EuiTitle>
-                  <EuiText>
-                    You can also run your search request from the server, without registering a
-                    search strategy. This request does not take the configuration of{' '}
-                    <EuiCode>TopNavMenu</EuiCode> into account, but you could pass those down to the
-                    server as well.
-                    <EuiButton type="primary" size="s" onClick={onServerClickHandler}>
-                      <FormattedMessage
-                        id="searchExamples.myServerButtonText"
-                        defaultMessage="Get data on the server"
-                      />
-                    </EuiButton>
-                  </EuiText>
                 </EuiPageContentBody>
               </EuiPageContent>
             </EuiPageBody>

src/plugins/data/common/search/search_source/filter_docvalue_fields.test.ts

@@ -1,30 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { filterDocvalueFields } from './filter_docvalue_fields';
-
-test('Should exclude docvalue_fields that are not contained in fields', () => {
-  const docvalueFields = [
-    'my_ip_field',
-    { field: 'my_keyword_field' },
-    { field: 'my_date_field', format: 'epoch_millis' },
-  ];
-  const out = filterDocvalueFields(docvalueFields, ['my_ip_field', 'my_keyword_field']);
-  expect(out).toEqual(['my_ip_field', { field: 'my_keyword_field' }]);
-});

src/plugins/data/common/search/search_source/filter_docvalue_fields.ts

@@ -1,33 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-interface DocvalueField {
-  field: string;
-  [key: string]: unknown;
-}
-
-export function filterDocvalueFields(
-  docvalueFields: Array<string | DocvalueField>,
-  fields: string[]
-) {
-  return docvalueFields.filter((docValue) => {
-    const docvalueFieldName = typeof docValue === 'string' ? docValue : docValue.field;
-    return fields.includes(docvalueFieldName);
-  });
-}

src/plugins/data/common/search/search_source/search_source.test.ts

@@ -29,7 +29,7 @@ jest.mock('./legacy', () => ({
 
 const getComputedFields = () => ({
   storedFields: [],
-  scriptFields: [],
+  scriptFields: {},
   docvalueFields: [],
 });
 
@@ -51,6 +51,7 @@ const indexPattern2 = ({
 describe('SearchSource', () => {
   let mockSearchMethod: any;
   let searchSourceDependencies: SearchSourceDependencies;
+  let searchSource: SearchSource;
 
   beforeEach(() => {
     mockSearchMethod = jest.fn().mockReturnValue(of({ rawResponse: '' }));
@@ -64,19 +65,12 @@ describe('SearchSource', () => {
         loadingCount$: new BehaviorSubject(0),
       },
     };
-  });
 
-  describe('#setField()', () => {
+    searchSource = new SearchSource({}, searchSourceDependencies);
-    test('sets the value for the property', () => {
-      const searchSource = new SearchSource({}, searchSourceDependencies);
-      searchSource.setField('aggs', 5);
-      expect(searchSource.getField('aggs')).toBe(5);
-    });
   });
 
   describe('#getField()', () => {
     test('gets the value for the property', () => {
-      const searchSource = new SearchSource({}, searchSourceDependencies);
       searchSource.setField('aggs', 5);
       expect(searchSource.getField('aggs')).toBe(5);
     });
@@ -84,52 +78,391 @@ describe('SearchSource', () => {
 
   describe('#removeField()', () => {
     test('remove property', () => {
-      const searchSource = new SearchSource({}, searchSourceDependencies);
+      searchSource = new SearchSource({}, searchSourceDependencies);
       searchSource.setField('aggs', 5);
       searchSource.removeField('aggs');
       expect(searchSource.getField('aggs')).toBeFalsy();
     });
   });
 
-  describe(`#setField('index')`, () => {
+  describe('#setField() / #flatten', () => {
-    describe('auto-sourceFiltering', () => {
+    test('sets the value for the property', () => {
-      describe('new index pattern assigned', () => {
+      searchSource.setField('aggs', 5);
-        test('generates a searchSource filter', async () => {
+      expect(searchSource.getField('aggs')).toBe(5);
-          const searchSource = new SearchSource({}, searchSourceDependencies);
+    });
-          expect(searchSource.getField('index')).toBe(undefined);
-          expect(searchSource.getField('source')).toBe(undefined);
-          searchSource.setField('index', indexPattern);
-          expect(searchSource.getField('index')).toBe(indexPattern);
-          const request = await searchSource.getSearchRequestBody();
-          expect(request._source).toBe(mockSource);
-        });
 
-        test('removes created searchSource filter on removal', async () => {
+    describe('computed fields handling', () => {
-          const searchSource = new SearchSource({}, searchSourceDependencies);
+      test('still provides computed fields when no fields are specified', async () => {
-          searchSource.setField('index', indexPattern);
+        searchSource.setField('index', ({
-          searchSource.setField('index', undefined);
+          ...indexPattern,
-          const request = await searchSource.getSearchRequestBody();
+          getComputedFields: () => ({
-          expect(request._source).toBe(undefined);
+            storedFields: ['hello'],
+            scriptFields: { world: {} },
+            docvalueFields: ['@timestamp'],
+          }),
+        } as unknown) as IndexPattern);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.stored_fields).toEqual(['hello']);
+        expect(request.script_fields).toEqual({ world: {} });
+        expect(request.fields).toEqual(['@timestamp']);
+      });
+
+      test('never includes docvalue_fields', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: {},
+            docvalueFields: ['@timestamp'],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['@timestamp']);
+        searchSource.setField('fieldsFromSource', ['foo']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request).not.toHaveProperty('docvalue_fields');
+      });
+
+      test('overrides computed docvalue fields with ones that are provided', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: {},
+            docvalueFields: ['hello'],
+          }),
+        } as unknown) as IndexPattern);
+        // @ts-expect-error TS won't like using this field name, but technically it's possible.
+        searchSource.setField('docvalue_fields', ['world']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request).toHaveProperty('docvalue_fields');
+        expect(request.docvalue_fields).toEqual(['world']);
+      });
+
+      test('allows explicitly provided docvalue fields to override fields API when fetching fieldsFromSource', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: {},
+            docvalueFields: [{ field: 'a', format: 'date_time' }],
+          }),
+        } as unknown) as IndexPattern);
+        // @ts-expect-error TS won't like using this field name, but technically it's possible.
+        searchSource.setField('docvalue_fields', [{ field: 'b', format: 'date_time' }]);
+        searchSource.setField('fields', ['c']);
+        searchSource.setField('fieldsFromSource', ['a', 'b', 'd']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request).toHaveProperty('docvalue_fields');
+        expect(request._source.includes).toEqual(['c', 'a', 'b', 'd']);
+        expect(request.docvalue_fields).toEqual([{ field: 'b', format: 'date_time' }]);
+        expect(request.fields).toEqual(['c', { field: 'a', format: 'date_time' }]);
+      });
+
+      test('allows you to override computed fields if you provide a format', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: {},
+            docvalueFields: [{ field: 'hello', format: 'date_time' }],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', [{ field: 'hello', format: 'strict_date_time' }]);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request).toHaveProperty('fields');
+        expect(request.fields).toEqual([{ field: 'hello', format: 'strict_date_time' }]);
+      });
+
+      test('injects a date format for computed docvalue fields if none is provided', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: {},
+            docvalueFields: [{ field: 'hello', format: 'date_time' }],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['hello']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request).toHaveProperty('fields');
+        expect(request.fields).toEqual([{ field: 'hello', format: 'date_time' }]);
+      });
+
+      test('injects a date format for computed docvalue fields while merging other properties', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: {},
+            docvalueFields: [{ field: 'hello', format: 'date_time', a: 'test', b: 'test' }],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', [{ field: 'hello', a: 'a', c: 'c' }]);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request).toHaveProperty('fields');
+        expect(request.fields).toEqual([
+          { field: 'hello', format: 'date_time', a: 'a', b: 'test', c: 'c' },
+        ]);
+      });
+
+      test('merges provided script fields with computed fields', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: { hello: {} },
+            docvalueFields: [],
+          }),
+        } as unknown) as IndexPattern);
+        // @ts-expect-error TS won't like using this field name, but technically it's possible.
+        searchSource.setField('script_fields', { world: {} });
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request).toHaveProperty('script_fields');
+        expect(request.script_fields).toEqual({
+          hello: {},
+          world: {},
         });
       });
 
-      describe('new index pattern assigned over another', () => {
+      test(`requests any fields that aren't script_fields from stored_fields`, async () => {
-        test('replaces searchSource filter with new', async () => {
+        searchSource.setField('index', ({
-          const searchSource = new SearchSource({}, searchSourceDependencies);
+          ...indexPattern,
-          searchSource.setField('index', indexPattern);
+          getComputedFields: () => ({
-          searchSource.setField('index', indexPattern2);
+            storedFields: [],
-          expect(searchSource.getField('index')).toBe(indexPattern2);
+            scriptFields: { hello: {} },
-          const request = await searchSource.getSearchRequestBody();
+            docvalueFields: [],
-          expect(request._source).toBe(mockSource2);
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['hello', 'a', { field: 'c' }]);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.script_fields).toEqual({ hello: {} });
+        expect(request.stored_fields).toEqual(['a', 'c']);
+      });
+
+      test('ignores objects without a `field` property when setting stored_fields', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: { hello: {} },
+            docvalueFields: [],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['hello', 'a', { foo: 'c' }]);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.script_fields).toEqual({ hello: {} });
+        expect(request.stored_fields).toEqual(['a']);
+      });
+
+      test(`requests any fields that aren't script_fields from stored_fields with fieldsFromSource`, async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: { hello: {} },
+            docvalueFields: [],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fieldsFromSource', ['hello', 'a']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.script_fields).toEqual({ hello: {} });
+        expect(request.stored_fields).toEqual(['a']);
+      });
+
+      test('defaults to * for stored fields when no fields are provided', async () => {
+        const requestA = await searchSource.getSearchRequestBody();
+        expect(requestA.stored_fields).toEqual(['*']);
+
+        searchSource.setField('fields', ['*']);
+        const requestB = await searchSource.getSearchRequestBody();
+        expect(requestB.stored_fields).toEqual(['*']);
+      });
+
+      test('defaults to * for stored fields when no fields are provided with fieldsFromSource', async () => {
+        searchSource.setField('fieldsFromSource', ['*']);
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.stored_fields).toEqual(['*']);
+      });
+    });
+
+    describe('source filters handling', () => {
+      test('excludes docvalue fields based on source filtering', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: {},
+            docvalueFields: ['@timestamp', 'exclude-me'],
+          }),
+        } as unknown) as IndexPattern);
+        // @ts-expect-error Typings for excludes filters need to be fixed.
+        searchSource.setField('source', { excludes: ['exclude-*'] });
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual(['@timestamp']);
+      });
+
+      test('defaults to source filters from index pattern', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: {},
+            docvalueFields: ['@timestamp', 'foo-bar', 'foo-baz'],
+          }),
+        } as unknown) as IndexPattern);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual(['@timestamp']);
+      });
+
+      test('filters script fields to only include specified fields', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: { hello: {}, world: {} },
+            docvalueFields: [],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['hello']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.script_fields).toEqual({ hello: {} });
+      });
+    });
+
+    describe('handling for when specific fields are provided', () => {
+      test('fieldsFromSource will request any fields outside of script_fields from _source & stored fields', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: { hello: {}, world: {} },
+            docvalueFields: ['@timestamp'],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fieldsFromSource', [
+          'hello',
+          'world',
+          '@timestamp',
+          'foo-a',
+          'bar-b',
+        ]);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request._source).toEqual({
+          includes: ['@timestamp', 'bar-b'],
+        });
+        expect(request.stored_fields).toEqual(['@timestamp', 'bar-b']);
+      });
+
+      test('filters request when a specific list of fields is provided', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: ['*'],
+            scriptFields: { hello: {}, world: {} },
+            docvalueFields: ['@timestamp', 'date'],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['hello', '@timestamp', 'foo-a', 'bar']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual(['hello', '@timestamp', 'bar']);
+        expect(request.script_fields).toEqual({ hello: {} });
+        expect(request.stored_fields).toEqual(['@timestamp', 'bar']);
+      });
+
+      test('filters request when a specific list of fields is provided with fieldsFromSource', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: ['*'],
+            scriptFields: { hello: {}, world: {} },
+            docvalueFields: ['@timestamp', 'date'],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fieldsFromSource', ['hello', '@timestamp', 'foo-a', 'bar']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request._source).toEqual({
+          includes: ['@timestamp', 'bar'],
+        });
+        expect(request.fields).toEqual(['@timestamp']);
+        expect(request.script_fields).toEqual({ hello: {} });
+        expect(request.stored_fields).toEqual(['@timestamp', 'bar']);
+      });
+
+      test('filters request when a specific list of fields is provided with fieldsFromSource or fields', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: ['*'],
+            scriptFields: { hello: {}, world: {} },
+            docvalueFields: ['@timestamp', 'date', 'time'],
+          }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['hello', '@timestamp', 'foo-a', 'bar']);
+        searchSource.setField('fieldsFromSource', ['foo-b', 'date', 'baz']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request._source).toEqual({
+          includes: ['@timestamp', 'bar', 'date', 'baz'],
+        });
+        expect(request.fields).toEqual(['hello', '@timestamp', 'bar', 'date']);
+        expect(request.script_fields).toEqual({ hello: {} });
+        expect(request.stored_fields).toEqual(['@timestamp', 'bar', 'date', 'baz']);
+      });
+    });
+
+    describe(`#setField('index')`, () => {
+      describe('auto-sourceFiltering', () => {
+        describe('new index pattern assigned', () => {
+          test('generates a searchSource filter', async () => {
+            expect(searchSource.getField('index')).toBe(undefined);
+            expect(searchSource.getField('source')).toBe(undefined);
+            searchSource.setField('index', indexPattern);
+            expect(searchSource.getField('index')).toBe(indexPattern);
+            const request = await searchSource.getSearchRequestBody();
+            expect(request._source).toBe(mockSource);
+          });
+
+          test('removes created searchSource filter on removal', async () => {
+            searchSource.setField('index', indexPattern);
+            searchSource.setField('index', undefined);
+            const request = await searchSource.getSearchRequestBody();
+            expect(request._source).toBe(undefined);
+          });
         });
 
-        test('removes created searchSource filter on removal', async () => {
+        describe('new index pattern assigned over another', () => {
-          const searchSource = new SearchSource({}, searchSourceDependencies);
+          test('replaces searchSource filter with new', async () => {
-          searchSource.setField('index', indexPattern);
+            searchSource.setField('index', indexPattern);
-          searchSource.setField('index', indexPattern2);
+            searchSource.setField('index', indexPattern2);
-          searchSource.setField('index', undefined);
+            expect(searchSource.getField('index')).toBe(indexPattern2);
-          const request = await searchSource.getSearchRequestBody();
+            const request = await searchSource.getSearchRequestBody();
-          expect(request._source).toBe(undefined);
+            expect(request._source).toBe(mockSource2);
+          });
+
+          test('removes created searchSource filter on removal', async () => {
+            searchSource.setField('index', indexPattern);
+            searchSource.setField('index', indexPattern2);
+            searchSource.setField('index', undefined);
+            const request = await searchSource.getSearchRequestBody();
+            expect(request._source).toBe(undefined);
+          });
         });
       });
     });
@@ -137,7 +470,7 @@ describe('SearchSource', () => {
 
   describe('#onRequestStart()', () => {
     test('should be called when starting a request', async () => {
-      const searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies);
+      searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies);
       const fn = jest.fn();
       searchSource.onRequestStart(fn);
       const options = {};
@@ -147,7 +480,7 @@ describe('SearchSource', () => {
 
     test('should not be called on parent searchSource', async () => {
       const parent = new SearchSource({}, searchSourceDependencies);
-      const searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies);
+      searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies);
 
       const fn = jest.fn();
       searchSource.onRequestStart(fn);
@@ -162,12 +495,12 @@ describe('SearchSource', () => {
 
     test('should be called on parent searchSource if callParentStartHandlers is true', async () => {
       const parent = new SearchSource({}, searchSourceDependencies);
-      const searchSource = new SearchSource(
+      searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies).setParent(
-        { index: indexPattern },
+        parent,
-        searchSourceDependencies
+        {
-      ).setParent(parent, {
+          callParentStartHandlers: true,
-        callParentStartHandlers: true,
+        }
-      });
+      );
 
       const fn = jest.fn();
       searchSource.onRequestStart(fn);
@@ -192,7 +525,7 @@ describe('SearchSource', () => {
     });
 
     test('should call msearch', async () => {
-      const searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies);
+      searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies);
       const options = {};
       await searchSource.fetch(options);
       expect(fetchSoon).toBeCalledTimes(1);
@@ -201,7 +534,7 @@ describe('SearchSource', () => {
 
   describe('#search service fetch()', () => {
     test('should call msearch', async () => {
-      const searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies);
+      searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies);
       const options = {};
 
       await searchSource.fetch(options);
@@ -212,7 +545,6 @@ describe('SearchSource', () => {
   describe('#serialize', () => {
     test('should reference index patterns', () => {
       const indexPattern123 = { id: '123' } as IndexPattern;
-      const searchSource = new SearchSource({}, searchSourceDependencies);
       searchSource.setField('index', indexPattern123);
       const { searchSourceJSON, references } = searchSource.serialize();
       expect(references[0].id).toEqual('123');
@@ -221,7 +553,6 @@ describe('SearchSource', () => {
     });
 
     test('should add other fields', () => {
-      const searchSource = new SearchSource({}, searchSourceDependencies);
       searchSource.setField('highlightAll', true);
       searchSource.setField('from', 123456);
       const { searchSourceJSON } = searchSource.serialize();
@@ -230,7 +561,6 @@ describe('SearchSource', () => {
     });
 
     test('should omit sort and size', () => {
-      const searchSource = new SearchSource({}, searchSourceDependencies);
       searchSource.setField('highlightAll', true);
       searchSource.setField('from', 123456);
       searchSource.setField('sort', { field: SortDirection.asc });
@@ -240,7 +570,6 @@ describe('SearchSource', () => {
     });
 
     test('should serialize filters', () => {
-      const searchSource = new SearchSource({}, searchSourceDependencies);
       const filter = [
         {
           query: 'query',
@@ -257,7 +586,6 @@ describe('SearchSource', () => {
     });
 
     test('should reference index patterns in filters separately from index field', () => {
-      const searchSource = new SearchSource({}, searchSourceDependencies);
       const indexPattern123 = { id: '123' } as IndexPattern;
       searchSource.setField('index', indexPattern123);
       const filter = [

src/plugins/data/common/search/search_source/search_source.ts

@@ -70,14 +70,18 @@
  */
 
 import { setWith } from '@elastic/safer-lodash-set';
-import { uniqueId, uniq, extend, pick, difference, omit, isObject, keys, isFunction } from 'lodash';
+import { uniqueId, keyBy, pick, difference, omit, isObject, isFunction } from 'lodash';
 import { map } from 'rxjs/operators';
 import { normalizeSortRequest } from './normalize_sort_request';
-import { filterDocvalueFields } from './filter_docvalue_fields';
 import { fieldWildcardFilter } from '../../../../kibana_utils/common';
 import { IIndexPattern } from '../../index_patterns';
 import { ISearchGeneric, ISearchOptions } from '../..';
-import type { ISearchSource, SearchSourceOptions, SearchSourceFields } from './types';
+import type {
+  ISearchSource,
+  SearchFieldValue,
+  SearchSourceOptions,
+  SearchSourceFields,
+} from './types';
 import { FetchHandlers, RequestFailure, getSearchParamsFromRequest, SearchRequest } from './fetch';
 
 import { getEsQueryConfig, buildEsQuery, Filter, UI_SETTINGS } from '../../../common';
@@ -404,7 +408,11 @@ export class SearchSource {
       case 'query':
         return addToRoot(key, (data[key] || []).concat(val));
       case 'fields':
-        const fields = uniq((data[key] || []).concat(val));
+        // uses new Fields API
+        return addToBody('fields', val);
+      case 'fieldsFromSource':
+        // preserves legacy behavior
+        const fields = [...new Set((data[key] || []).concat(val))];
         return addToRoot(key, fields);
       case 'index':
       case 'type':
@@ -451,49 +459,127 @@ export class SearchSource {
   }
 
   private flatten() {
+    const { getConfig } = this.dependencies;
     const searchRequest = this.mergeProps();
 
     searchRequest.body = searchRequest.body || {};
-    const { body, index, fields, query, filters, highlightAll } = searchRequest;
+    const { body, index, query, filters, highlightAll } = searchRequest;
     searchRequest.indexType = this.getIndexType(index);
 
-    const computedFields = index ? index.getComputedFields() : {};
+    // get some special field types from the index pattern
+    const { docvalueFields, scriptFields, storedFields } = index
+      ? index.getComputedFields()
+      : {
+          docvalueFields: [],
+          scriptFields: {},
+          storedFields: ['*'],
+        };
 
-    body.stored_fields = computedFields.storedFields;
+    const fieldListProvided = !!body.fields;
-    body.script_fields = body.script_fields || {};
+    const getFieldName = (fld: string | Record<string, any>): string =>
-    extend(body.script_fields, computedFields.scriptFields);
+      typeof fld === 'string' ? fld : fld.field;
 
-    const defaultDocValueFields = computedFields.docvalueFields
+    // set defaults
-      ? computedFields.docvalueFields
+    let fieldsFromSource = searchRequest.fieldsFromSource || [];
-      : [];
+    body.fields = body.fields || [];
-    body.docvalue_fields = body.docvalue_fields || defaultDocValueFields;
+    body.script_fields = {
+      ...body.script_fields,
+      ...scriptFields,
+    };
+    body.stored_fields = storedFields;
 
-    if (!body.hasOwnProperty('_source') && index) {
+    // apply source filters from index pattern if specified by the user
-      body._source = index.getSourceFiltering();
+    let filteredDocvalueFields = docvalueFields;
+    if (index) {
+      const sourceFilters = index.getSourceFiltering();
+      if (!body.hasOwnProperty('_source')) {
+        body._source = sourceFilters;
+      }
+      if (body._source.excludes) {
+        const filter = fieldWildcardFilter(
+          body._source.excludes,
+          getConfig(UI_SETTINGS.META_FIELDS)
+        );
+        // also apply filters to provided fields & default docvalueFields
+        body.fields = body.fields.filter((fld: SearchFieldValue) => filter(getFieldName(fld)));
+        fieldsFromSource = fieldsFromSource.filter((fld: SearchFieldValue) =>
+          filter(getFieldName(fld))
+        );
+        filteredDocvalueFields = filteredDocvalueFields.filter((fld: SearchFieldValue) =>
+          filter(getFieldName(fld))
+        );
+      }
     }
 
-    const { getConfig } = this.dependencies;
+    // specific fields were provided, so we need to exclude any others
-
+    if (fieldListProvided || fieldsFromSource.length) {
-    if (body._source) {
+      const bodyFieldNames = body.fields.map((field: string | Record<string, any>) =>
-      // exclude source fields for this index pattern specified by the user
+        getFieldName(field)
-      const filter = fieldWildcardFilter(body._source.excludes, getConfig(UI_SETTINGS.META_FIELDS));
-      body.docvalue_fields = body.docvalue_fields.filter((docvalueField: any) =>
-        filter(docvalueField.field)
       );
-    }
+      const uniqFieldNames = [...new Set([...bodyFieldNames, ...fieldsFromSource])];
 
-    // if we only want to search for certain fields
+      // filter down script_fields to only include items specified
-    if (fields) {
+      body.script_fields = pick(
-      // filter out the docvalue_fields, and script_fields to only include those that we are concerned with
+        body.script_fields,
-      body.docvalue_fields = filterDocvalueFields(body.docvalue_fields, fields);
+        Object.keys(body.script_fields).filter((f) => uniqFieldNames.includes(f))
-      body.script_fields = pick(body.script_fields, fields);
-
-      // request the remaining fields from both stored_fields and _source
-      const remainingFields = difference(fields, keys(body.script_fields));
-      body.stored_fields = remainingFields;
-      setWith(body, '_source.includes', remainingFields, (nsValue) =>
-        isObject(nsValue) ? {} : nsValue
       );
+
+      // request the remaining fields from stored_fields just in case, since the
+      // fields API does not handle stored fields
+      const remainingFields = difference(uniqFieldNames, Object.keys(body.script_fields)).filter(
+        Boolean
+      );
+
+      // only include unique values
+      body.stored_fields = [...new Set(remainingFields)];
+
+      if (fieldsFromSource.length) {
+        // include remaining fields in _source
+        setWith(body, '_source.includes', remainingFields, (nsValue) =>
+          isObject(nsValue) ? {} : nsValue
+        );
+
+        // if items that are in the docvalueFields are provided, we should
+        // make sure those are added to the fields API unless they are
+        // already set in docvalue_fields
+        body.fields = [
+          ...body.fields,
+          ...filteredDocvalueFields.filter((fld: SearchFieldValue) => {
+            return (
+              fieldsFromSource.includes(getFieldName(fld)) &&
+              !(body.docvalue_fields || [])
+                .map((d: string | Record<string, any>) => getFieldName(d))
+                .includes(getFieldName(fld))
+            );
+          }),
+        ];
+
+        // delete fields array if it is still set to the empty default
+        if (!fieldListProvided && body.fields.length === 0) delete body.fields;
+      } else {
+        // remove _source, since everything's coming from fields API, scripted, or stored fields
+        body._source = false;
+
+        // if items that are in the docvalueFields are provided, we should
+        // inject the format from the computed fields if one isn't given
+        const docvaluesIndex = keyBy(filteredDocvalueFields, 'field');
+        body.fields = body.fields.map((fld: SearchFieldValue) => {
+          const fieldName = getFieldName(fld);
+          if (Object.keys(docvaluesIndex).includes(fieldName)) {
+            // either provide the field object from computed docvalues,
+            // or merge the user-provided field with the one in docvalues
+            return typeof fld === 'string'
+              ? docvaluesIndex[fld]
+              : {
+                  ...docvaluesIndex[fieldName],
+                  ...fld,
+                };
+          }
+          return fld;
+        });
+      }
+    } else {
+      body.fields = filteredDocvalueFields;
     }
 
     const esQueryConfigs = getEsQueryConfig({ get: getConfig });

src/plugins/data/common/search/search_source/types.ts

@@ -59,6 +59,13 @@ export interface SortDirectionNumeric {
 
 export type EsQuerySortValue = Record<string, SortDirection | SortDirectionNumeric>;
 
+interface SearchField {
+  [key: string]: SearchFieldValue;
+}
+
+// @internal
+export type SearchFieldValue = string | SearchField;
+
 /**
  * search source fields
  */
@@ -86,7 +93,16 @@ export interface SearchSourceFields {
   size?: number;
   source?: NameList;
   version?: boolean;
-  fields?: NameList;
+  /**
+   * Retrieve fields via the search Fields API
+   */
+  fields?: SearchFieldValue[];
+  /**
+   * Retreive fields directly from _source (legacy behavior)
+   *
+   * @deprecated It is recommended to use `fields` wherever possible.
+   */
+  fieldsFromSource?: NameList;
   /**
    * {@link IndexPatternService}
    */

src/plugins/data/public/public.api.md

@@ -2171,7 +2171,8 @@ export class SearchSource {
         size?: number | undefined;
         source?: string | boolean | string[] | undefined;
         version?: boolean | undefined;
-        fields?: string | boolean | string[] | undefined;
+        fields?: SearchFieldValue[] | undefined;
+        fieldsFromSource?: string | boolean | string[] | undefined;
         index?: import("../..").IndexPattern | undefined;
         searchAfter?: import("./types").EsQuerySearchAfter | undefined;
         timeout?: string | undefined;
@@ -2205,8 +2206,9 @@ export class SearchSource {
 export interface SearchSourceFields {
     // (undocumented)
     aggs?: any;
-    // (undocumented)
+    fields?: SearchFieldValue[];
-    fields?: NameList;
+    // @deprecated
+    fieldsFromSource?: NameList;
     // (undocumented)
     filter?: Filter[] | Filter | (() => Filter[] | Filter | undefined);
     // (undocumented)
@@ -2406,6 +2408,7 @@ export const UI_SETTINGS: {
 // src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:64:5 - (ae-forgotten-export) The symbol "FormatFieldFn" needs to be exported by the entry point index.d.ts
 // src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:135:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts
 // src/plugins/data/common/search/aggs/types.ts:113:51 - (ae-forgotten-export) The symbol "AggTypesRegistryStart" needs to be exported by the entry point index.d.ts
+// src/plugins/data/common/search/search_source/search_source.ts:197:7 - (ae-forgotten-export) The symbol "SearchFieldValue" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/field_formats/field_formats_service.ts:67:3 - (ae-forgotten-export) The symbol "FormatFactory" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/index.ts:66:23 - (ae-forgotten-export) The symbol "FILTERS" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/index.ts:66:23 - (ae-forgotten-export) The symbol "getDisplayValueFromFilter" needs to be exported by the entry point index.d.ts

src/plugins/discover/public/application/helpers/get_sharing_data.test.ts

@@ -46,10 +46,8 @@ describe('getSharingData', () => {
         ],
         "searchRequest": Object {
           "body": Object {
-            "_source": Object {
+            "_source": Object {},
-              "includes": Array [],
+            "fields": undefined,
-            },
-            "docvalue_fields": Array [],
             "query": Object {
               "bool": Object {
                 "filter": Array [],
@@ -60,7 +58,7 @@ describe('getSharingData', () => {
             },
             "script_fields": Object {},
             "sort": Array [],
-            "stored_fields": Array [],
+            "stored_fields": undefined,
           },
           "index": "the-index-pattern-title",
         },

src/plugins/discover/public/application/helpers/get_sharing_data.ts

@@ -63,7 +63,7 @@ export async function getSharingData(
     index.timeFieldName || '',
     config.get(DOC_HIDE_TIME_COLUMN_SETTING)
   );
-  searchSource.setField('fields', searchFields);
+  searchSource.setField('fieldsFromSource', searchFields);
   searchSource.setField(
     'sort',
     getSortForSearchSource(state.sort as SortOrder[], index, config.get(SORT_DEFAULT_ORDER_SETTING))

src/plugins/index_pattern_management/public/components/edit_index_pattern/source_filters_table/components/add_filter/__snapshots__/add_filter.test.tsx.snap

@@ -8,7 +8,7 @@ exports[`AddFilter should ignore strings with just spaces 1`] = `
     <EuiFieldText
       fullWidth={true}
       onChange={[Function]}
-      placeholder="source filter, accepts wildcards (e.g., \`user*\` to filter fields starting with 'user')"
+      placeholder="field filter, accepts wildcards (e.g., \`user*\` to filter fields starting with 'user')"
       value=""
     />
   </EuiFlexItem>
@@ -35,7 +35,7 @@ exports[`AddFilter should render normally 1`] = `
     <EuiFieldText
       fullWidth={true}
       onChange={[Function]}
-      placeholder="source filter, accepts wildcards (e.g., \`user*\` to filter fields starting with 'user')"
+      placeholder="field filter, accepts wildcards (e.g., \`user*\` to filter fields starting with 'user')"
       value=""
     />
   </EuiFlexItem>

src/plugins/index_pattern_management/public/components/edit_index_pattern/source_filters_table/components/add_filter/add_filter.tsx

@@ -31,7 +31,7 @@ const sourcePlaceholder = i18n.translate(
   'indexPatternManagement.editIndexPattern.sourcePlaceholder',
   {
     defaultMessage:
-      "source filter, accepts wildcards (e.g., `user*` to filter fields starting with 'user')",
+      "field filter, accepts wildcards (e.g., `user*` to filter fields starting with 'user')",
   }
 );
 

src/plugins/index_pattern_management/public/components/edit_index_pattern/source_filters_table/components/confirmation_modal/__snapshots__/confirmation_modal.test.tsx.snap

@@ -23,7 +23,7 @@ exports[`Header should render normally 1`] = `
     onConfirm={[Function]}
     title={
       <FormattedMessage
-        defaultMessage="Delete source filter '{value}'?"
+        defaultMessage="Delete field filter '{value}'?"
         id="indexPatternManagement.editIndexPattern.source.deleteSourceFilterLabel"
         values={
           Object {

src/plugins/index_pattern_management/public/components/edit_index_pattern/source_filters_table/components/confirmation_modal/confirmation_modal.tsx

@@ -42,7 +42,7 @@ export const DeleteFilterConfirmationModal = ({
         title={
           <FormattedMessage
             id="indexPatternManagement.editIndexPattern.source.deleteSourceFilterLabel"
-            defaultMessage="Delete source filter '{value}'?"
+            defaultMessage="Delete field filter '{value}'?"
             values={{
               value: filterToDeleteValue,
             }}

src/plugins/index_pattern_management/public/components/edit_index_pattern/source_filters_table/components/header/__snapshots__/header.test.tsx.snap

@@ -7,7 +7,7 @@ exports[`Header should render normally 1`] = `
   >
     <h3>
       <FormattedMessage
-        defaultMessage="Source filters"
+        defaultMessage="Field filters"
         id="indexPatternManagement.editIndexPattern.sourceHeader"
         values={Object {}}
       />
@@ -16,7 +16,7 @@ exports[`Header should render normally 1`] = `
   <EuiText>
     <p>
       <FormattedMessage
-        defaultMessage="Source filters can be used to exclude one or more fields when fetching the document source. This happens when viewing a document in the Discover app, or with a table displaying results from a saved search in the Dashboard app. Each row is built using the source of a single document, and if you have documents with large or unimportant fields you may benefit from filtering those out at this lower level."
+        defaultMessage="Field filters can be used to exclude one or more fields when fetching a document. This happens when viewing a document in the Discover app, or with a table displaying results from a saved search in the Dashboard app. If you have documents with large or unimportant fields you may benefit from filtering those out at this lower level."
         id="indexPatternManagement.editIndexPattern.sourceLabel"
         values={Object {}}
       />

src/plugins/index_pattern_management/public/components/edit_index_pattern/source_filters_table/components/header/header.tsx

@@ -28,7 +28,7 @@ export const Header = () => (
       <h3>
         <FormattedMessage
           id="indexPatternManagement.editIndexPattern.sourceHeader"
-          defaultMessage="Source filters"
+          defaultMessage="Field filters"
         />
       </h3>
     </EuiTitle>
@@ -36,10 +36,9 @@ export const Header = () => (
       <p>
         <FormattedMessage
           id="indexPatternManagement.editIndexPattern.sourceLabel"
-          defaultMessage="Source filters can be used to exclude one or more fields when fetching the document source. This happens when
+          defaultMessage="Field filters can be used to exclude one or more fields when fetching a document. This happens when
-          viewing a document in the Discover app, or with a table displaying results from a saved search in the Dashboard app. Each row is
+          viewing a document in the Discover app, or with a table displaying results from a saved search in the Dashboard app.
-          built using the source of a single document, and if you have documents with large or unimportant fields you may benefit from
+          If you have documents with large or unimportant fields you may benefit from filtering those out at this lower level."
-          filtering those out at this lower level."
         />
       </p>
       <p>

src/plugins/index_pattern_management/public/components/edit_index_pattern/tabs/utils.ts

@@ -67,7 +67,7 @@ function getTitle(type: string, filteredCount: Dictionary<number>, totalCount: D
       break;
     case 'sourceFilters':
       title = i18n.translate('indexPatternManagement.editIndexPattern.tabs.sourceHeader', {
-        defaultMessage: 'Source filters',
+        defaultMessage: 'Field filters',
       });
       break;
   }

x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.test.ts

@@ -113,7 +113,7 @@ describe('ESSearchSource', () => {
         });
         const urlTemplateWithMeta = await esSearchSource.getUrlTemplateWithMeta(searchFilters);
         expect(urlTemplateWithMeta.urlTemplate).toBe(
-          `rootdir/api/maps/mvt/getTile;?x={x}&y={y}&z={z}&geometryFieldName=bar&index=foobar-title-*&requestBody=(foobar:ES_DSL_PLACEHOLDER,params:('0':('0':index,'1':(fields:(),title:'foobar-title-*')),'1':('0':size,'1':1000),'2':('0':filter,'1':!()),'3':('0':query),'4':('0':index,'1':(fields:(),title:'foobar-title-*')),'5':('0':query,'1':(language:KQL,query:'tooltipField: foobar',queryLastTriggeredAt:'2019-04-25T20:53:22.331Z')),'6':('0':fields,'1':!(tooltipField,styleField)),'7':('0':source,'1':!(tooltipField,styleField))))&geoFieldType=geo_shape`
+          `rootdir/api/maps/mvt/getTile;?x={x}&y={y}&z={z}&geometryFieldName=bar&index=foobar-title-*&requestBody=(foobar:ES_DSL_PLACEHOLDER,params:('0':('0':index,'1':(fields:(),title:'foobar-title-*')),'1':('0':size,'1':1000),'2':('0':filter,'1':!()),'3':('0':query),'4':('0':index,'1':(fields:(),title:'foobar-title-*')),'5':('0':query,'1':(language:KQL,query:'tooltipField: foobar',queryLastTriggeredAt:'2019-04-25T20:53:22.331Z')),'6':('0':fieldsFromSource,'1':!(tooltipField,styleField)),'7':('0':source,'1':!(tooltipField,styleField))))&geoFieldType=geo_shape`
         );
       });
     });

x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx

@@ -375,7 +375,7 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye
       maxResultWindow,
       initialSearchContext
     );
-    searchSource.setField('fields', searchFilters.fieldNames); // Setting "fields" filters out unused scripted fields
+    searchSource.setField('fieldsFromSource', searchFilters.fieldNames); // Setting "fields" filters out unused scripted fields
     if (sourceOnlyFields.length === 0) {
       searchSource.setField('source', false); // do not need anything from _source
     } else {
@@ -505,7 +505,7 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye
     };
 
     searchSource.setField('query', query);
-    searchSource.setField('fields', this._getTooltipPropertyNames());
+    searchSource.setField('fieldsFromSource', this._getTooltipPropertyNames());
 
     const resp = await searchSource.fetch();
 
@@ -708,7 +708,7 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye
       indexSettings.maxResultWindow,
       initialSearchContext
     );
-    searchSource.setField('fields', searchFilters.fieldNames); // Setting "fields" filters out unused scripted fields
+    searchSource.setField('fieldsFromSource', searchFilters.fieldNames); // Setting "fields" filters out unused scripted fields
     if (sourceOnlyFields.length === 0) {
       searchSource.setField('source', false); // do not need anything from _source
     } else {

x-pack/test/functional/apps/maps/mvt_super_fine.js

@@ -32,7 +32,7 @@ export default function ({ getPageObjects, getService }) {
 
       //Source should be correct
       expect(mapboxStyle.sources[MB_VECTOR_SOURCE_ID].tiles[0]).to.equal(
-        "/api/maps/mvt/getGridTile?x={x}&y={y}&z={z}&geometryFieldName=geo.coordinates&index=logstash-*&requestBody=(_source:(excludes:!()),aggs:(gridSplit:(aggs:(gridCentroid:(geo_centroid:(field:geo.coordinates)),max_of_bytes:(max:(field:bytes))),geotile_grid:(bounds:!n,field:geo.coordinates,precision:!n,shard_size:65535,size:65535))),docvalue_fields:!((field:'@timestamp',format:date_time),(field:'relatedContent.article:modified_time',format:date_time),(field:'relatedContent.article:published_time',format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((match_all:()),(range:('@timestamp':(format:strict_date_optional_time,gte:'2015-09-20T00:00:00.000Z',lte:'2015-09-20T01:00:00.000Z')))),must:!(),must_not:!(),should:!())),script_fields:(hour_of_day:(script:(lang:painless,source:'doc[!'@timestamp!'].value.getHour()'))),size:0,stored_fields:!('*'))&requestType=grid&geoFieldType=geo_point"
+        "/api/maps/mvt/getGridTile?x={x}&y={y}&z={z}&geometryFieldName=geo.coordinates&index=logstash-*&requestBody=(_source:(excludes:!()),aggs:(gridSplit:(aggs:(gridCentroid:(geo_centroid:(field:geo.coordinates)),max_of_bytes:(max:(field:bytes))),geotile_grid:(bounds:!n,field:geo.coordinates,precision:!n,shard_size:65535,size:65535))),fields:!((field:'@timestamp',format:date_time),(field:'relatedContent.article:modified_time',format:date_time),(field:'relatedContent.article:published_time',format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((match_all:()),(range:('@timestamp':(format:strict_date_optional_time,gte:'2015-09-20T00:00:00.000Z',lte:'2015-09-20T01:00:00.000Z')))),must:!(),must_not:!(),should:!())),script_fields:(hour_of_day:(script:(lang:painless,source:'doc[!'@timestamp!'].value.getHour()'))),size:0,stored_fields:!('*'))&requestType=grid&geoFieldType=geo_point"
       );
 
       //Should correctly load meta for style-rule (sigma is set to 1, opacity to 1)