Add _bulk_create endpoint to /api/saved_objects (#20498) (#20615)

* add _bulk_create rest endpoint

* document bulk_create api

* provide 409 status code when conflicts exist

* add overwrite and version to documenation

* clean up assert statements and 2 places where bulkCreate is getting used with new output

* properly stub bulkCreate return

* remove version of documenation example

docs/api/saved-objects.asciidoc

@@ -13,6 +13,7 @@ permanently breaks Kibana in a future version.
 * <<saved-objects-api-bulk-get>>
 * <<saved-objects-api-find>>
 * <<saved-objects-api-create>>
+* <<saved-objects-api-bulk-create>>
 * <<saved-objects-api-update>>
 * <<saved-objects-api-delete>>
 
@@ -20,5 +21,6 @@ include::saved-objects/get.asciidoc[]
 include::saved-objects/bulk_get.asciidoc[]
 include::saved-objects/find.asciidoc[]
 include::saved-objects/create.asciidoc[]
+include::saved-objects/bulk_create.asciidoc[]
 include::saved-objects/update.asciidoc[]
 include::saved-objects/delete.asciidoc[]

docs/api/saved-objects/bulk_create.asciidoc

@@ -0,0 +1,102 @@
+[[saved-objects-api-bulk-create]]
+=== Bulk Create Objects
+
+experimental[This functionality is *experimental* and may be changed or removed completely in a future release.]
+
+The bulk-create saved object API enables you to persist multiple Kibana saved
+objects.
+
+==== Request
+
+`POST /api/saved_objects/_bulk_create`
+
+
+==== Query Parameters
+
+`overwrite` (optional)::
+  (boolean) If true, will overwrite the document with the same ID.
+
+
+==== Request Body
+
+The request body must be a JSON array containing objects, each of which
+contains the following properties:
+
+`type` (required)::
+  (string) Valid options, include: `visualization`, `dashboard`, `search`, `index-pattern`, `config`, and `timelion-sheet`
+
+`id` (optional)::
+  (string) Enables specifying an ID to use, as opposed to one being randomly generated
+
+`attributes` (required)::
+  (object) The data to persist
+
+`version` (optional)::
+  (number) Enables specifying a version
+
+
+==== Response body
+
+The response body will have a top level `saved_objects` property that contains
+an array of objects, which represent the response for each of the requested
+objects. The order of the objects in the response is identical to the order of
+the objects in the request.
+
+For any saved object that could not be persisted, an error object will exist in its
+place.
+
+
+==== Examples
+
+The following example attempts to persist an index pattern with id
+`my-pattern` and a dashboard with id `my-dashboard`, but only the index pattern
+could be persisted because there was an id collision with `my-dashboard` as a saved object with that id alread exists.
+
+[source,js]
+--------------------------------------------------
+POST api/saved_objects/_bulk_create
+[
+  {
+    "type": "index-pattern",
+    "id": "my-pattern",
+    "attributes": {
+      "title": "my-pattern-*"
+    }
+  },
+  {
+    "type": "dashboard",
+    "id": "my-dashboard",
+    "attributes": {
+      "title": "Look at my dashboard"
+    }
+  }
+]
+--------------------------------------------------
+// KIBANA
+
+A successful call returns a response code of `200` and a response body
+containing a JSON structure similar to the following example:
+
+[source,js]
+--------------------------------------------------
+{
+  "saved_objects": [
+    {
+      "id": "my-pattern",
+      "type": "index-pattern",
+      "version": 1,
+      "attributes": {
+        "title": "my-pattern-*"
+      }
+    },
+    {
+      "id": "my-dashboard",
+      "type": "dashboard",
+      "error": {
+        "statusCode": 409,
+        "message": "version conflict, document already exists"
+      }
+    }
+  ]
+}
+--------------------------------------------------

src/core_plugins/kibana/server/lib/import/import_dashboards.js

@@ -29,6 +29,6 @@ export async function importDashboards(req) {
   const docs = payload.objects
     .filter(item => !exclude.includes(item.type));
 
-  const objects = await savedObjectsClient.bulkCreate(docs, { overwrite });
-  return { objects };
+  const results = await savedObjectsClient.bulkCreate(docs, { overwrite });
+  return { objects: results.saved_objects };
 }

src/core_plugins/kibana/server/lib/import/import_dashboards.test.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { importDashboards } from '../import_dashboards';
+import { importDashboards } from './import_dashboards';
 import sinon from 'sinon';
 import { expect } from 'chai';
 
@@ -26,7 +26,7 @@ describe('importDashboards(req)', () => {
   let req;
   let bulkCreateStub;
   beforeEach(() => {
-    bulkCreateStub = sinon.stub().returns(Promise.resolve());
+    bulkCreateStub = sinon.stub().returns(Promise.resolve({ saved_objects: [] }));
     req = {
       query: {},
       payload: {
@@ -45,7 +45,7 @@ describe('importDashboards(req)', () => {
 
   });
 
-  it('should call bulkCreate with each asset', () => {
+  test('should call bulkCreate with each asset', () => {
     return importDashboards(req).then(() => {
       expect(bulkCreateStub.calledOnce).to.equal(true);
       expect(bulkCreateStub.args[0][0]).to.eql([
@@ -55,7 +55,7 @@ describe('importDashboards(req)', () => {
     });
   });
 
-  it('should call bulkCreate with overwrite true if force is truthy', () => {
+  test('should call bulkCreate with overwrite true if force is truthy', () => {
     req.query = { force: 'true' };
     return importDashboards(req).then(() => {
       expect(bulkCreateStub.calledOnce).to.equal(true);
@@ -63,7 +63,7 @@ describe('importDashboards(req)', () => {
     });
   });
 
-  it('should exclude types based on exclude argument', () => {
+  test('should exclude types based on exclude argument', () => {
     req.query = { exclude: 'visualization' };
     return importDashboards(req).then(() => {
       expect(bulkCreateStub.calledOnce).to.equal(true);

src/server/sample_data/routes/install.js

@@ -112,7 +112,7 @@ export const createInstallRoute = () => ({
         }
 
         const createResults = await request.getSavedObjectsClient().bulkCreate(sampleDataset.savedObjects, { overwrite: true });
-        const errors = createResults.filter(savedObjectCreateResult => {
+        const errors = createResults.saved_objects.filter(savedObjectCreateResult => {
           return savedObjectCreateResult.hasOwnProperty('error');
         });
         if (errors.length > 0) {

src/server/saved_objects/routes/bulk_create.js

@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Joi from 'joi';
+
+export const createBulkCreateRoute = (prereqs) => ({
+  path: '/api/saved_objects/_bulk_create',
+  method: 'POST',
+  config: {
+    pre: [prereqs.getSavedObjectsClient],
+    validate: {
+      query: Joi.object().keys({
+        overwrite: Joi.boolean().default(false)
+      }).default(),
+      payload: Joi.array().items(Joi.object({
+        type: Joi.string().required(),
+        id: Joi.string(),
+        attributes: Joi.object().required(),
+        version: Joi.number(),
+      }).required())
+    },
+    handler(request, reply) {
+      const { overwrite } = request.query;
+      const { savedObjectsClient } = request.pre;
+
+      reply(savedObjectsClient.bulkCreate(request.payload, { overwrite }));
+    }
+  }
+});

src/server/saved_objects/routes/index.js

@@ -17,6 +17,7 @@
  * under the License.
  */
 
+export { createBulkCreateRoute } from './bulk_create';
 export { createBulkGetRoute } from './bulk_get';
 export { createCreateRoute } from './create';
 export { createDeleteRoute } from './delete';

src/server/saved_objects/saved_objects_mixin.js

@@ -20,6 +20,7 @@
 import { createSavedObjectsService } from './service';
 
 import {
+  createBulkCreateRoute,
   createBulkGetRoute,
   createCreateRoute,
   createDeleteRoute,
@@ -45,6 +46,7 @@ export function savedObjectsMixin(kbnServer, server) {
     },
   };
 
+  server.route(createBulkCreateRoute(prereqs));
   server.route(createBulkGetRoute(prereqs));
   server.route(createCreateRoute(prereqs));
   server.route(createDeleteRoute(prereqs));

src/server/saved_objects/service/lib/repository.js

@@ -101,7 +101,7 @@ export class SavedObjectsRepository {
    * @param {array} objects - [{ type, id, attributes }]
    * @param {object} [options={}]
    * @property {boolean} [options.overwrite=false] - overwrites existing documents
-   * @returns {promise} - [{ id, type, version, attributes, error: { message } }]
+   * @returns {promise} -  {saved_objects: [[{ id, type, version, attributes, error: { message } }]}
    */
   async bulkCreate(objects, options = {}) {
     const {
@@ -135,37 +135,46 @@ export class SavedObjectsRepository {
       ]), []),
     });
 
-    return items.map((response, i) => {
-      const {
-        error,
-        _id: responseId,
-        _version: version,
-      } = Object.values(response)[0];
+    return {
+      saved_objects: items.map((response, i) => {
+        const {
+          error,
+          _id: responseId,
+          _version: version,
+        } = Object.values(response)[0];
 
-      const {
-        id = responseId,
-        type,
-        attributes,
-      } = objects[i];
+        const {
+          id = responseId,
+          type,
+          attributes,
+        } = objects[i];
+
+        if (error) {
+          if (error.type === 'version_conflict_engine_exception') {
+            return {
+              id,
+              type,
+              error: { statusCode: 409, message: 'version conflict, document already exists' }
+            };
+          }
+          return {
+            id,
+            type,
+            error: {
+              message: error.reason || JSON.stringify(error)
+            }
+          };
+        }
 
-      if (error) {
         return {
           id,
           type,
-          error: {
-            message: error.reason || JSON.stringify(error)
-          }
+          updated_at: time,
+          version,
+          attributes
         };
-      }
-
-      return {
-        id,
-        type,
-        updated_at: time,
-        version,
-        attributes
-      };
-    });
+      })
+    };
   }
 
   /**

src/server/saved_objects/service/lib/repository.test.js

@@ -271,19 +271,21 @@ describe('SavedObjectsRepository', () => {
         { type: 'index-pattern', id: 'two', attributes: { title: 'Test Two' } }
       ]);
 
-      expect(response).toEqual([
-        {
-          id: 'one',
-          type: 'config',
-          error: { message: 'type[config] missing' }
-        }, {
-          id: 'two',
-          type: 'index-pattern',
-          version: 2,
-          ...mockTimestampFields,
-          attributes: { title: 'Test Two' },
-        }
-      ]);
+      expect(response).toEqual({
+        saved_objects: [
+          {
+            id: 'one',
+            type: 'config',
+            error: { message: 'type[config] missing' }
+          }, {
+            id: 'two',
+            type: 'index-pattern',
+            version: 2,
+            ...mockTimestampFields,
+            attributes: { title: 'Test Two' },
+          }
+        ]
+      });
     });
 
     it('formats Elasticsearch response', async () => {
@@ -309,21 +311,23 @@ describe('SavedObjectsRepository', () => {
         { type: 'index-pattern', id: 'two', attributes: { title: 'Test Two' } }
       ]);
 
-      expect(response).toEqual([
-        {
-          id: 'one',
-          type: 'config',
-          version: 2,
-          ...mockTimestampFields,
-          attributes: { title: 'Test One' },
-        }, {
-          id: 'two',
-          type: 'index-pattern',
-          version: 2,
-          ...mockTimestampFields,
-          attributes: { title: 'Test Two' },
-        }
-      ]);
+      expect(response).toEqual({
+        saved_objects: [
+          {
+            id: 'one',
+            type: 'config',
+            version: 2,
+            ...mockTimestampFields,
+            attributes: { title: 'Test One' },
+          }, {
+            id: 'two',
+            type: 'index-pattern',
+            version: 2,
+            ...mockTimestampFields,
+            attributes: { title: 'Test Two' },
+          }
+        ]
+      });
     });
   });
 

src/server/saved_objects/service/saved_objects_client.js

@@ -114,7 +114,7 @@ export class SavedObjectsClient {
    * @param {array} objects - [{ type, id, attributes }]
    * @param {object} [options={}]
    * @property {boolean} [options.overwrite=false] - overwrites existing documents
-   * @returns {promise} - [{ id, type, version, attributes, error: { message } }]
+   * @returns {promise} - { saved_objects: [{ id, type, version, attributes, error: { message } }]}
    */
   async bulkCreate(objects, options = {}) {
     return this._repository.bulkCreate(objects, options);

src/ui/public/saved_objects/saved_objects_client.js

@@ -51,15 +51,15 @@ export class SavedObjectsClient {
   }
 
   /**
-  * Persists an object
-  *
-  * @param {string} type
-  * @param {object} [attributes={}]
-  * @param {object} [options={}]
-  * @property {string} [options.id] - force id on creation, not recommended
-  * @property {boolean} [options.overwrite=false]
-  * @returns {promise} - SavedObject({ id, type, version, attributes })
-  */
+   * Persists an object
+   *
+   * @param {string} type
+   * @param {object} [attributes={}]
+   * @param {object} [options={}]
+   * @property {string} [options.id] - force id on creation, not recommended
+   * @property {boolean} [options.overwrite=false]
+   * @returns {promise} - SavedObject({ id, type, version, attributes })
+   */
   create(type, attributes = {}, options = {}) {
     if (!type || !attributes) {
       return this._PromiseCtor.reject(new Error('requires type and attributes'));
@@ -72,6 +72,23 @@ export class SavedObjectsClient {
       .then(resp => this.createSavedObject(resp));
   }
 
+  /**
+   * Creates multiple documents at once
+   *
+   * @param {array} objects - [{ type, id, attributes }]
+   * @param {object} [options={}]
+   * @property {boolean} [options.overwrite=false]
+   * @returns {promise} - { savedObjects: [{ id, type, version, attributes, error: { message } }]}
+   */
+  bulkCreate = (objects = [], options = {}) => {
+    const url = this._getUrl(['_bulk_create'], _.pick(options, ['overwrite']));
+
+    return this._request('POST', url, objects).then(resp => {
+      resp.saved_objects = resp.saved_objects.map(d => this.createSavedObject(d));
+      return keysToCamelCaseShallow(resp);
+    });
+  }
+
   /**
    * Deletes an object
    *

x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.js

@@ -366,7 +366,7 @@ export class DataRecognizer {
     if (filteredSavedObjects.length) {
       results = await this.savedObjectsClient.bulkCreate(filteredSavedObjects);
     }
-    return results;
+    return results.saved_objects;
   }
 
   // save the jobs.