diff --git a/docs/api/saved-objects/import.asciidoc b/docs/api/saved-objects/import.asciidoc index d1df7c380812..ed90625e9550 100644 --- a/docs/api/saved-objects/import.asciidoc +++ b/docs/api/saved-objects/import.asciidoc @@ -86,11 +86,58 @@ containing a JSON structure similar to the following example: { "id": "my-pattern", "type": "index-pattern", + "title": "my-pattern-*", "error": { - "statusCode": 409, - "message": "version conflict, document already exists", + "type": "conflict" }, }, ], } -------------------------------------------------- + +The following example imports a visualization and dashboard but the index pattern for the visualization reference doesn't exist. + +[source,js] +-------------------------------------------------- +POST api/saved_objects/_import +Content-Type: multipart/form-data; boundary=EXAMPLE +--EXAMPLE +Content-Disposition: form-data; name="file"; filename="export.ndjson" +Content-Type: application/ndjson + +{"type":"visualization","id":"my-vis","attributes":{"title":"my-vis"},"references":[{"name":"ref_0","type":"index-pattern","id":"my-pattern-*"}]} +{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"},"references":[{"name":"ref_0","type":"visualization","id":"my-vis"}]} +--EXAMPLE-- +-------------------------------------------------- +// KIBANA + +The call returns a response code of `200` and a response body +containing a JSON structure similar to the following example: + +[source,js] +-------------------------------------------------- + "success": false, + "successCount": 0, + "errors": [ + { + "id": "my-vis", + "type": "visualization", + "title": "my-vis", + "error": { + "type": "missing_references", + "references": [ + { + "type": "index-pattern", + "id": "my-pattern-*" + } + ], + "blocking": [ + { + "type": "dashboard", + "id": "my-dashboard" + } + ] + } + } + ] +-------------------------------------------------- diff --git a/docs/api/saved-objects/resolve_import_errors.asciidoc b/docs/api/saved-objects/resolve_import_errors.asciidoc index 545afa75aa42..6221229c4d55 100644 --- a/docs/api/saved-objects/resolve_import_errors.asciidoc +++ b/docs/api/saved-objects/resolve_import_errors.asciidoc @@ -3,7 +3,7 @@ experimental[This functionality is *experimental* and may be changed or removed completely in a future release.] -The resolve import errors API enables you to resolve errors given by the import API by either overwriting specific saved objects or changing references to a newly created object. +The resolve import errors API enables you to resolve errors given by the import API by either retrying certain saved objects, overwriting specific saved objects or changing references to different saved objects. Note: You cannot access this endpoint via the Console in Kibana. @@ -16,27 +16,20 @@ Note: You cannot access this endpoint via the Console in Kibana. The request body must be of type multipart/form-data. `file`:: - (ndjson) The same new line delimited JSON objects given to the import API. + The same file given to the import API. -`overwrites` (optional):: - (array) A list of `type` and `id` objects allowed to be overwritten on import. - -`replaceReferences` (optional):: - (array) A list of `type`, `from` and `to` used to change imported saved object references to. - -`skips` (optional):: - (array) A list of `type` and `id` objects to skip importing. +`retries`:: + (array) A list of `type`, `id`, `replaceReferences` and `overwrite` objects to retry importing. The property `replaceReferences` is a list of `type`, `from` and `to` used to change the object's references. ==== Response body The response body will have a top level `success` property that indicates -if the import was successful or not as well as a `successCount` indicating how many records are successfully resolved. -In the scenario the import wasn't successful a top level `errors` array will contain the objects that failed to import. +if resolving errors was successful or not as well as a `successCount` indicating how many records are successfully resolved. +In the scenario resolving errors wasn't successful, a top level `errors` array will contain the objects that failed to be resolved. ==== Examples -The following example resolves errors for an index pattern and dashboard but indicates to skip the index pattern. -This will cause the index pattern to not be in the system and the dashboard to overwrite the existing saved object. +The following example retries importing a dashboard. [source,js] -------------------------------------------------- @@ -46,14 +39,9 @@ Content-Type: multipart/form-data; boundary=EXAMPLE Content-Disposition: form-data; name="file"; filename="export.ndjson" Content-Type: application/ndjson -{"type":"index-pattern","id":"my-pattern","attributes":{"title":"my-pattern-*"}} {"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"}} --EXAMPLE -Content-Disposition: form-data; name="skips" - -[{"type":"index-pattern","id":"my-pattern"}] ---EXAMPLE -Content-Disposition: form-data; name="overwrites" +Content-Disposition: form-data; name="retries" [{"type":"dashboard","id":"my-dashboard"}] --EXAMPLE-- @@ -71,8 +59,7 @@ containing a JSON structure similar to the following example: } -------------------------------------------------- -The following example resolves errors for a visualization and dashboard but indicates -to replace the dashboard references to another visualization. +The following example resolves errors for a dashboard. This will cause the dashboard to overwrite the existing saved object. [source,js] -------------------------------------------------- @@ -82,12 +69,42 @@ Content-Type: multipart/form-data; boundary=EXAMPLE Content-Disposition: form-data; name="file"; filename="export.ndjson" Content-Type: application/ndjson -{"type":"visualization","id":"my-vis","attributes":{"title":"Look at my visualization"}} -{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"},"references":[{"name":"panel_0","type":"visualization","id":"my-vis"}]} +{"type":"index-pattern","id":"my-pattern","attributes":{"title":"my-pattern-*"}} +{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"}} --EXAMPLE -Content-Disposition: form-data; name="replaceReferences" +Content-Disposition: form-data; name="retries" -[{"type":"visualization","from":"my-vis","to":"my-vis-2"}] +[{"type":"dashboard","id":"my-dashboard","overwrite":true}] +--EXAMPLE-- +-------------------------------------------------- +// KIBANA + +A successful call returns a response code of `200` and a response body +containing a JSON structure similar to the following example: + +[source,js] +-------------------------------------------------- +{ + "success": true, + "successCount": 1 +} +-------------------------------------------------- + +The following example resolves errors for a visualization by replacing the index pattern to another. + +[source,js] +-------------------------------------------------- +POST api/saved_objects/_resolve_import_errors +Content-Type: multipart/form-data; boundary=EXAMPLE +--EXAMPLE +Content-Disposition: form-data; name="file"; filename="export.ndjson" +Content-Type: application/ndjson + +{"type":"visualization","id":"my-vis","attributes":{"title":"Look at my visualization"},"references":[{"name":"ref_0","type":"index-pattern","id":"missing"}]} +--EXAMPLE +Content-Disposition: form-data; name="retries" + +[{"type":"visualization","id":"my-vis","replaceReferences":[{"type":"index-pattern","from":"missing","to":"existing"}]}] --EXAMPLE-- -------------------------------------------------- // KIBANA diff --git a/src/legacy/server/saved_objects/import/create_objects_filter.test.ts b/src/legacy/server/saved_objects/import/create_objects_filter.test.ts index 7ccbc7dde835..327fb3a45f00 100644 --- a/src/legacy/server/saved_objects/import/create_objects_filter.test.ts +++ b/src/legacy/server/saved_objects/import/create_objects_filter.test.ts @@ -20,34 +20,45 @@ import { createObjectsFilter } from './create_objects_filter'; describe('createObjectsFilter()', () => { - test('filters should return false when contains empty parameters', () => { - const fn = createObjectsFilter([], [], []); + test('filter should return false when contains empty parameters', () => { + const fn = createObjectsFilter([]); expect(fn({ type: 'a', id: '1', attributes: {}, references: [] })).toEqual(false); }); - test('filters should exclude skips', () => { - const fn = createObjectsFilter( - [ - { - type: 'a', - id: '1', - }, - ], - [], - [ - { - type: 'b', - from: '1', - to: '2', - }, - ] - ); + test('filter should return true for objects that are being retried', () => { + const fn = createObjectsFilter([ + { + type: 'a', + id: '1', + overwrite: false, + replaceReferences: [], + }, + ]); expect( fn({ type: 'a', id: '1', attributes: {}, - references: [{ name: 'ref_0', type: 'b', id: '1' }], + references: [], + }) + ).toEqual(true); + }); + + test(`filter should return false for objects that aren't being retried`, () => { + const fn = createObjectsFilter([ + { + type: 'a', + id: '1', + overwrite: false, + replaceReferences: [], + }, + ]); + expect( + fn({ + type: 'b', + id: '1', + attributes: {}, + references: [], }) ).toEqual(false); expect( @@ -55,131 +66,8 @@ describe('createObjectsFilter()', () => { type: 'a', id: '2', attributes: {}, - references: [{ name: 'ref_0', type: 'b', id: '1' }], - }) - ).toEqual(true); - }); - - test('filter should include references to replace', () => { - const fn = createObjectsFilter( - [], - [], - [ - { - type: 'b', - from: '1', - to: '2', - }, - ] - ); - expect( - fn({ - type: 'a', - id: '1', - attributes: {}, - references: [ - { - name: 'ref_0', - type: 'b', - id: '1', - }, - ], - }) - ).toEqual(true); - expect( - fn({ - type: 'a', - id: '1', - attributes: {}, - references: [ - { - name: 'ref_0', - type: 'b', - id: '2', - }, - ], + references: [], }) ).toEqual(false); }); - - test('filter should include objects to overwrite', () => { - const fn = createObjectsFilter( - [], - [ - { - type: 'a', - id: '1', - }, - ], - [] - ); - expect(fn({ type: 'a', id: '1', attributes: {}, references: [] })).toEqual(true); - expect(fn({ type: 'a', id: '2', attributes: {}, references: [] })).toEqual(false); - }); - - test('filter should work with skips, overwrites and replaceReferences', () => { - const fn = createObjectsFilter( - [ - { - type: 'a', - id: '1', - }, - ], - [ - { - type: 'a', - id: '2', - }, - ], - [ - { - type: 'b', - from: '1', - to: '2', - }, - ] - ); - expect( - fn({ - type: 'a', - id: '1', - attributes: {}, - references: [ - { - name: 'ref_0', - type: 'b', - id: '1', - }, - ], - }) - ).toEqual(false); - expect( - fn({ - type: 'a', - id: '2', - attributes: {}, - references: [ - { - name: 'ref_0', - type: 'b', - id: '2', - }, - ], - }) - ).toEqual(true); - expect( - fn({ - type: 'a', - id: '3', - attributes: {}, - references: [ - { - name: 'ref_0', - type: 'b', - id: '1', - }, - ], - }) - ).toEqual(true); - }); }); diff --git a/src/legacy/server/saved_objects/import/create_objects_filter.ts b/src/legacy/server/saved_objects/import/create_objects_filter.ts index 71edc3b4eaaa..aacf8112f255 100644 --- a/src/legacy/server/saved_objects/import/create_objects_filter.ts +++ b/src/legacy/server/saved_objects/import/create_objects_filter.ts @@ -18,37 +18,11 @@ */ import { SavedObject } from '../service'; +import { Retry } from './types'; -export function createObjectsFilter( - skips: Array<{ - type: string; - id: string; - }>, - overwrites: Array<{ - type: string; - id: string; - }>, - replaceReferences: Array<{ - type: string; - from: string; - to: string; - }> -) { - const refReplacements = replaceReferences.map(ref => `${ref.type}:${ref.from}`); +export function createObjectsFilter(retries: Retry[]) { + const retryKeys = new Set(retries.map(retry => `${retry.type}:${retry.id}`)); return (obj: SavedObject) => { - if (skips.some(skipObj => skipObj.type === obj.type && skipObj.id === obj.id)) { - return false; - } - if ( - overwrites.some(overwriteObj => overwriteObj.type === obj.type && overwriteObj.id === obj.id) - ) { - return true; - } - for (const reference of obj.references || []) { - if (refReplacements.includes(`${reference.type}:${reference.id}`)) { - return true; - } - } - return false; + return retryKeys.has(`${obj.type}:${obj.id}`); }; } diff --git a/src/legacy/server/saved_objects/import/extract_errors.test.ts b/src/legacy/server/saved_objects/import/extract_errors.test.ts index 6f21c7b54647..ad2b1467923a 100644 --- a/src/legacy/server/saved_objects/import/extract_errors.test.ts +++ b/src/legacy/server/saved_objects/import/extract_errors.test.ts @@ -23,7 +23,7 @@ import { extractErrors } from './extract_errors'; describe('extractErrors()', () => { test('returns empty array when no errors exist', () => { const savedObjects: SavedObject[] = []; - const result = extractErrors(savedObjects); + const result = extractErrors(savedObjects, savedObjects); expect(result).toMatchInlineSnapshot(`Array []`); }); @@ -32,13 +32,17 @@ describe('extractErrors()', () => { { id: '1', type: 'dashboard', - attributes: {}, + attributes: { + title: 'My Dashboard 1', + }, references: [], }, { id: '2', type: 'dashboard', - attributes: {}, + attributes: { + title: 'My Dashboard 2', + }, references: [], error: { statusCode: 409, @@ -48,7 +52,9 @@ describe('extractErrors()', () => { { id: '3', type: 'dashboard', - attributes: {}, + attributes: { + title: 'My Dashboard 3', + }, references: [], error: { statusCode: 400, @@ -56,7 +62,7 @@ describe('extractErrors()', () => { }, }, ]; - const result = extractErrors(savedObjects); + const result = extractErrors(savedObjects, savedObjects); expect(result).toMatchInlineSnapshot(` Array [ Object { @@ -64,6 +70,7 @@ Array [ "type": "conflict", }, "id": "2", + "title": "My Dashboard 2", "type": "dashboard", }, Object { @@ -73,6 +80,7 @@ Array [ "type": "unknown", }, "id": "3", + "title": "My Dashboard 3", "type": "dashboard", }, ] diff --git a/src/legacy/server/saved_objects/import/extract_errors.ts b/src/legacy/server/saved_objects/import/extract_errors.ts index 5418198c058b..6ae9562a1f3a 100644 --- a/src/legacy/server/saved_objects/import/extract_errors.ts +++ b/src/legacy/server/saved_objects/import/extract_errors.ts @@ -20,14 +20,29 @@ import { SavedObject } from '../service'; import { ImportError } from './types'; -export function extractErrors(savedObjects: SavedObject[]) { +export function extractErrors( + savedObjectResults: SavedObject[], + savedObjectsToImport: SavedObject[] +) { const errors: ImportError[] = []; - for (const savedObject of savedObjects) { + const originalSavedObjectsMap = new Map(); + for (const savedObject of savedObjectsToImport) { + originalSavedObjectsMap.set(`${savedObject.type}:${savedObject.id}`, savedObject); + } + for (const savedObject of savedObjectResults) { if (savedObject.error) { + const originalSavedObject = originalSavedObjectsMap.get( + `${savedObject.type}:${savedObject.id}` + ); + const title = + originalSavedObject && + originalSavedObject.attributes && + originalSavedObject.attributes.title; if (savedObject.error.statusCode === 409) { errors.push({ id: savedObject.id, type: savedObject.type, + title, error: { type: 'conflict', }, @@ -37,6 +52,7 @@ export function extractErrors(savedObjects: SavedObject[]) { errors.push({ id: savedObject.id, type: savedObject.type, + title, error: { ...savedObject.error, type: 'unknown', diff --git a/src/legacy/server/saved_objects/import/import_saved_objects.test.ts b/src/legacy/server/saved_objects/import/import_saved_objects.test.ts index 2dccb2960167..95aff38f27e2 100644 --- a/src/legacy/server/saved_objects/import/import_saved_objects.test.ts +++ b/src/legacy/server/saved_objects/import/import_saved_objects.test.ts @@ -26,25 +26,33 @@ describe('importSavedObjects()', () => { { id: '1', type: 'index-pattern', - attributes: {}, + attributes: { + title: 'My Index Pattern', + }, references: [], }, { id: '2', type: 'search', - attributes: {}, + attributes: { + title: 'My Search', + }, references: [], }, { id: '3', type: 'visualization', - attributes: {}, + attributes: { + title: 'My Visualization', + }, references: [], }, { id: '4', type: 'dashboard', - attributes: {}, + attributes: { + title: 'My Dashboard', + }, references: [], }, ]; @@ -60,13 +68,27 @@ describe('importSavedObjects()', () => { }; beforeEach(() => { - savedObjectsClient.bulkCreate.mockReset(); - savedObjectsClient.bulkGet.mockReset(); - savedObjectsClient.create.mockReset(); - savedObjectsClient.delete.mockReset(); - savedObjectsClient.find.mockReset(); - savedObjectsClient.get.mockReset(); - savedObjectsClient.update.mockReset(); + jest.resetAllMocks(); + }); + + test('returns early when no objects exist', async () => { + const readStream = new Readable({ + read() { + this.push(null); + }, + }); + const result = await importSavedObjects({ + readStream, + objectLimit: 1, + overwrite: false, + savedObjectsClient, + }); + expect(result).toMatchInlineSnapshot(` +Object { + "success": true, + "successCount": 0, +} +`); }); test('calls bulkCreate without overwrite', async () => { @@ -98,25 +120,33 @@ Object { Array [ Array [ Object { - "attributes": Object {}, + "attributes": Object { + "title": "My Index Pattern", + }, "id": "1", "references": Array [], "type": "index-pattern", }, Object { - "attributes": Object {}, + "attributes": Object { + "title": "My Search", + }, "id": "2", "references": Array [], "type": "search", }, Object { - "attributes": Object {}, + "attributes": Object { + "title": "My Visualization", + }, "id": "3", "references": Array [], "type": "visualization", }, Object { - "attributes": Object {}, + "attributes": Object { + "title": "My Dashboard", + }, "id": "4", "references": Array [], "type": "dashboard", @@ -166,25 +196,33 @@ Object { Array [ Array [ Object { - "attributes": Object {}, + "attributes": Object { + "title": "My Index Pattern", + }, "id": "1", "references": Array [], "type": "index-pattern", }, Object { - "attributes": Object {}, + "attributes": Object { + "title": "My Search", + }, "id": "2", "references": Array [], "type": "search", }, Object { - "attributes": Object {}, + "attributes": Object { + "title": "My Visualization", + }, "id": "3", "references": Array [], "type": "visualization", }, Object { - "attributes": Object {}, + "attributes": Object { + "title": "My Dashboard", + }, "id": "4", "references": Array [], "type": "dashboard", @@ -205,7 +243,7 @@ Object { `); }); - test('extracts errors', async () => { + test('extracts errors for conflicts', async () => { const readStream = new Readable({ read() { savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n')); @@ -237,6 +275,7 @@ Object { "type": "conflict", }, "id": "1", + "title": "My Index Pattern", "type": "index-pattern", }, Object { @@ -244,6 +283,7 @@ Object { "type": "conflict", }, "id": "2", + "title": "My Search", "type": "search", }, Object { @@ -251,6 +291,7 @@ Object { "type": "conflict", }, "id": "3", + "title": "My Visualization", "type": "visualization", }, Object { @@ -258,12 +299,122 @@ Object { "type": "conflict", }, "id": "4", + "title": "My Dashboard", "type": "dashboard", }, ], "success": false, "successCount": 0, } +`); + }); + + test('validates references', async () => { + const readStream = new Readable({ + read() { + this.push( + JSON.stringify({ + id: '1', + type: 'search', + attributes: { + title: 'My Search', + }, + references: [ + { + name: 'ref_0', + type: 'index-pattern', + id: '2', + }, + ], + }) + '\n' + ); + this.push( + JSON.stringify({ + id: '3', + type: 'visualization', + attributes: { + title: 'My Visualization', + }, + references: [ + { + name: 'ref_0', + type: 'search', + id: '1', + }, + ], + }) + '\n' + ); + this.push(null); + }, + }); + savedObjectsClient.bulkGet.mockResolvedValueOnce({ + saved_objects: [ + { + type: 'index-pattern', + id: '2', + error: { + statusCode: 404, + message: 'Not found', + }, + }, + ], + }); + const result = await importSavedObjects({ + readStream, + objectLimit: 4, + overwrite: false, + savedObjectsClient, + }); + expect(result).toMatchInlineSnapshot(` +Object { + "errors": Array [ + Object { + "error": Object { + "blocking": Array [ + Object { + "id": "3", + "type": "visualization", + }, + ], + "references": Array [ + Object { + "id": "2", + "type": "index-pattern", + }, + ], + "type": "missing_references", + }, + "id": "1", + "title": "My Search", + "type": "search", + }, + ], + "success": false, + "successCount": 0, +} +`); + expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(` +[MockFunction] { + "calls": Array [ + Array [ + Array [ + Object { + "fields": Array [ + "id", + ], + "id": "2", + "type": "index-pattern", + }, + ], + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Promise {}, + }, + ], +} `); }); }); diff --git a/src/legacy/server/saved_objects/import/import_saved_objects.ts b/src/legacy/server/saved_objects/import/import_saved_objects.ts index 01324071eb31..7637a1dfb423 100644 --- a/src/legacy/server/saved_objects/import/import_saved_objects.ts +++ b/src/legacy/server/saved_objects/import/import_saved_objects.ts @@ -43,13 +43,14 @@ export async function importSavedObjects({ overwrite, savedObjectsClient, }: ImportSavedObjectsOptions): Promise { + // Get the objects to import const objectsFromStream = await collectSavedObjects(readStream, objectLimit); - + // Validate references const { filteredObjects, errors: validationErrors } = await validateReferences( objectsFromStream, savedObjectsClient ); - + // Exit early if no objects to import if (filteredObjects.length === 0) { return { success: validationErrors.length === 0, @@ -57,15 +58,18 @@ export async function importSavedObjects({ ...(validationErrors.length ? { errors: validationErrors } : {}), }; } - + // Create objects in bulk const bulkCreateResult = await savedObjectsClient.bulkCreate(filteredObjects, { overwrite, }); - const errors = [...validationErrors, ...extractErrors(bulkCreateResult.saved_objects)]; + const errors = [ + ...validationErrors, + ...extractErrors(bulkCreateResult.saved_objects, filteredObjects), + ]; return { success: errors.length === 0, - successCount: objectsFromStream.length - errors.length, + successCount: bulkCreateResult.saved_objects.filter(obj => !obj.error).length, ...(errors.length ? { errors } : {}), }; } diff --git a/src/legacy/server/saved_objects/import/resolve_import_errors.test.ts b/src/legacy/server/saved_objects/import/resolve_import_errors.test.ts index 2d88e8199825..4885630dfb72 100644 --- a/src/legacy/server/saved_objects/import/resolve_import_errors.test.ts +++ b/src/legacy/server/saved_objects/import/resolve_import_errors.test.ts @@ -26,25 +26,33 @@ describe('resolveImportErrors()', () => { { id: '1', type: 'index-pattern', - attributes: {}, + attributes: { + title: 'My Index Pattern', + }, references: [], }, { id: '2', type: 'search', - attributes: {}, + attributes: { + title: 'My Search', + }, references: [], }, { id: '3', type: 'visualization', - attributes: {}, + attributes: { + title: 'My Visualization', + }, references: [], }, { id: '4', type: 'dashboard', - attributes: {}, + attributes: { + title: 'My Dashboard', + }, references: [ { name: 'panel_0', @@ -66,13 +74,7 @@ describe('resolveImportErrors()', () => { }; beforeEach(() => { - savedObjectsClient.bulkCreate.mockReset(); - savedObjectsClient.bulkGet.mockReset(); - savedObjectsClient.create.mockReset(); - savedObjectsClient.delete.mockReset(); - savedObjectsClient.find.mockReset(); - savedObjectsClient.get.mockReset(); - savedObjectsClient.update.mockReset(); + jest.resetAllMocks(); }); test('works with empty parameters', async () => { @@ -83,15 +85,13 @@ describe('resolveImportErrors()', () => { }, }); savedObjectsClient.bulkCreate.mockResolvedValue({ - saved_objects: savedObjects, + saved_objects: [], }); const result = await resolveImportErrors({ readStream, objectLimit: 4, - skips: [], - overwrites: [], + retries: [], savedObjectsClient, - replaceReferences: [], }); expect(result).toMatchInlineSnapshot(` Object { @@ -102,66 +102,28 @@ Object { expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`[MockFunction]`); }); - test('works with skips', async () => { + test('works with retries', async () => { const readStream = new Readable({ read() { savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n')); this.push(null); }, }); - savedObjectsClient.bulkCreate.mockResolvedValue({ - saved_objects: savedObjects, + savedObjectsClient.bulkCreate.mockResolvedValueOnce({ + saved_objects: savedObjects.filter(obj => obj.type === 'visualization' && obj.id === '3'), }); const result = await resolveImportErrors({ readStream, objectLimit: 4, - skips: [ - { - type: 'dashboard', - id: '4', - }, - ], - overwrites: [], - savedObjectsClient, - replaceReferences: [ + retries: [ { type: 'visualization', - from: '3', - to: '30', - }, - ], - }); - expect(result).toMatchInlineSnapshot(` -Object { - "success": true, - "successCount": 0, -} -`); - expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`[MockFunction]`); - }); - - test('works with overwrites', async () => { - const readStream = new Readable({ - read() { - savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n')); - this.push(null); - }, - }); - savedObjectsClient.bulkCreate.mockResolvedValue({ - saved_objects: savedObjects, - }); - const result = await resolveImportErrors({ - readStream, - objectLimit: 4, - skips: [], - overwrites: [ - { - type: 'index-pattern', - id: '1', + id: '3', + replaceReferences: [], + overwrite: false, }, ], savedObjectsClient, - replaceReferences: [], }); expect(result).toMatchInlineSnapshot(` Object { @@ -175,7 +137,64 @@ Object { Array [ Array [ Object { - "attributes": Object {}, + "attributes": Object { + "title": "My Visualization", + }, + "id": "3", + "references": Array [], + "type": "visualization", + }, + ], + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Promise {}, + }, + ], +} +`); + }); + + test('works with overwrites', async () => { + const readStream = new Readable({ + read() { + savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n')); + this.push(null); + }, + }); + savedObjectsClient.bulkCreate.mockResolvedValue({ + saved_objects: savedObjects.filter(obj => obj.type === 'index-pattern' && obj.id === '1'), + }); + const result = await resolveImportErrors({ + readStream, + objectLimit: 4, + retries: [ + { + type: 'index-pattern', + id: '1', + overwrite: true, + replaceReferences: [], + }, + ], + savedObjectsClient, + }); + expect(result).toMatchInlineSnapshot(` +Object { + "success": true, + "successCount": 1, +} +`); + expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(` +[MockFunction] { + "calls": Array [ + Array [ + Array [ + Object { + "attributes": Object { + "title": "My Index Pattern", + }, "id": "1", "references": Array [], "type": "index-pattern", @@ -204,21 +223,26 @@ Object { }, }); savedObjectsClient.bulkCreate.mockResolvedValue({ - saved_objects: savedObjects, + saved_objects: savedObjects.filter(obj => obj.type === 'dashboard' && obj.id === '4'), }); const result = await resolveImportErrors({ readStream, objectLimit: 4, - skips: [], - overwrites: [], - savedObjectsClient, - replaceReferences: [ + retries: [ { - type: 'visualization', - from: '3', - to: '13', + type: 'dashboard', + id: '4', + overwrite: false, + replaceReferences: [ + { + type: 'visualization', + from: '3', + to: '13', + }, + ], }, ], + savedObjectsClient, }); expect(result).toMatchInlineSnapshot(` Object { @@ -232,7 +256,9 @@ Object { Array [ Array [ Object { - "attributes": Object {}, + "attributes": Object { + "title": "My Dashboard", + }, "id": "4", "references": Array [ Object { @@ -244,9 +270,198 @@ Object { "type": "dashboard", }, ], - Object { - "overwrite": true, + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Promise {}, + }, + ], +} +`); + }); + + test('extracts errors for conflicts', async () => { + const readStream = new Readable({ + read() { + savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n')); + this.push(null); }, + }); + savedObjectsClient.bulkCreate.mockResolvedValue({ + saved_objects: savedObjects.map(savedObject => ({ + type: savedObject.type, + id: savedObject.id, + error: { + statusCode: 409, + message: 'conflict', + }, + })), + }); + const result = await resolveImportErrors({ + readStream, + objectLimit: 4, + retries: savedObjects.map(obj => ({ + type: obj.type, + id: obj.id, + overwrite: false, + replaceReferences: [], + })), + savedObjectsClient, + }); + expect(result).toMatchInlineSnapshot(` +Object { + "errors": Array [ + Object { + "error": Object { + "type": "conflict", + }, + "id": "1", + "title": "My Index Pattern", + "type": "index-pattern", + }, + Object { + "error": Object { + "type": "conflict", + }, + "id": "2", + "title": "My Search", + "type": "search", + }, + Object { + "error": Object { + "type": "conflict", + }, + "id": "3", + "title": "My Visualization", + "type": "visualization", + }, + Object { + "error": Object { + "type": "conflict", + }, + "id": "4", + "title": "My Dashboard", + "type": "dashboard", + }, + ], + "success": false, + "successCount": 0, +} +`); + }); + + test('validates references', async () => { + const readStream = new Readable({ + read() { + this.push( + JSON.stringify({ + id: '1', + type: 'search', + attributes: { + title: 'My Search', + }, + references: [ + { + name: 'ref_0', + type: 'index-pattern', + id: '2', + }, + ], + }) + '\n' + ); + this.push( + JSON.stringify({ + id: '3', + type: 'visualization', + attributes: { + title: 'My Visualization', + }, + references: [ + { + name: 'ref_0', + type: 'search', + id: '1', + }, + ], + }) + '\n' + ); + this.push(null); + }, + }); + savedObjectsClient.bulkGet.mockResolvedValueOnce({ + saved_objects: [ + { + type: 'index-pattern', + id: '2', + error: { + statusCode: 404, + message: 'Not found', + }, + }, + ], + }); + const result = await resolveImportErrors({ + readStream, + objectLimit: 2, + retries: [ + { + type: 'search', + id: '1', + overwrite: false, + replaceReferences: [], + }, + { + type: 'visualization', + id: '3', + overwrite: false, + replaceReferences: [], + }, + ], + savedObjectsClient, + }); + expect(result).toMatchInlineSnapshot(` +Object { + "errors": Array [ + Object { + "error": Object { + "blocking": Array [ + Object { + "id": "3", + "type": "visualization", + }, + ], + "references": Array [ + Object { + "id": "2", + "type": "index-pattern", + }, + ], + "type": "missing_references", + }, + "id": "1", + "title": "My Search", + "type": "search", + }, + ], + "success": false, + "successCount": 0, +} +`); + expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(` +[MockFunction] { + "calls": Array [ + Array [ + Array [ + Object { + "fields": Array [ + "id", + ], + "id": "2", + "type": "index-pattern", + }, + ], ], ], "results": Array [ diff --git a/src/legacy/server/saved_objects/import/resolve_import_errors.ts b/src/legacy/server/saved_objects/import/resolve_import_errors.ts index 827d73655308..3b14694472bb 100644 --- a/src/legacy/server/saved_objects/import/resolve_import_errors.ts +++ b/src/legacy/server/saved_objects/import/resolve_import_errors.ts @@ -22,25 +22,15 @@ import { SavedObjectsClient } from '../service'; import { collectSavedObjects } from './collect_saved_objects'; import { createObjectsFilter } from './create_objects_filter'; import { extractErrors } from './extract_errors'; -import { ImportError } from './types'; +import { splitOverwrites } from './split_overwrites'; +import { ImportError, Retry } from './types'; +import { validateReferences } from './validate_references'; interface ResolveImportErrorsOptions { readStream: Readable; objectLimit: number; savedObjectsClient: SavedObjectsClient; - overwrites: Array<{ - type: string; - id: string; - }>; - replaceReferences: Array<{ - type: string; - from: string; - to: string; - }>; - skips: Array<{ - type: string; - id: string; - }>; + retries: Retry[]; } interface ImportResponse { @@ -52,38 +42,65 @@ interface ImportResponse { export async function resolveImportErrors({ readStream, objectLimit, - skips, - overwrites, + retries, savedObjectsClient, - replaceReferences, }: ResolveImportErrorsOptions): Promise { + let successCount = 0; let errors: ImportError[] = []; - const filter = createObjectsFilter(skips, overwrites, replaceReferences); + const filter = createObjectsFilter(retries); + + // Get the objects to resolve errors const objectsToResolve = await collectSavedObjects(readStream, objectLimit, filter); - // Replace references - const refReplacementsMap: Record = {}; - for (const { type, to, from } of replaceReferences) { - refReplacementsMap[`${type}:${from}`] = to; + // Create a map of references to replace for each object to avoid iterating through + // retries for every object to resolve + const retriesReferencesMap = new Map(); + for (const retry of retries) { + const map: { [key: string]: string } = {}; + for (const { type, from, to } of retry.replaceReferences) { + map[`${type}:${from}`] = to; + } + retriesReferencesMap.set(`${retry.type}:${retry.id}`, map); } + + // Replace references for (const savedObject of objectsToResolve) { + const refMap = retriesReferencesMap.get(`${savedObject.type}:${savedObject.id}`); + if (!refMap) { + continue; + } for (const reference of savedObject.references || []) { - if (refReplacementsMap[`${reference.type}:${reference.id}`]) { - reference.id = refReplacementsMap[`${reference.type}:${reference.id}`]; + if (refMap[`${reference.type}:${reference.id}`]) { + reference.id = refMap[`${reference.type}:${reference.id}`]; } } } - if (objectsToResolve.length) { - const bulkCreateResult = await savedObjectsClient.bulkCreate(objectsToResolve, { + // Validate references + const { filteredObjects, errors: validationErrors } = await validateReferences( + objectsToResolve, + savedObjectsClient + ); + errors = errors.concat(validationErrors); + + // Bulk create in two batches, overwrites and non-overwrites + const { objectsToOverwrite, objectsToNotOverwrite } = splitOverwrites(filteredObjects, retries); + if (objectsToOverwrite.length) { + const bulkCreateResult = await savedObjectsClient.bulkCreate(objectsToOverwrite, { overwrite: true, }); - errors = extractErrors(bulkCreateResult.saved_objects); + errors = errors.concat(extractErrors(bulkCreateResult.saved_objects, objectsToOverwrite)); + successCount += bulkCreateResult.saved_objects.filter(obj => !obj.error).length; + } + if (objectsToNotOverwrite.length) { + const bulkCreateResult = await savedObjectsClient.bulkCreate(objectsToNotOverwrite); + errors = errors.concat(extractErrors(bulkCreateResult.saved_objects, objectsToNotOverwrite)); + successCount += bulkCreateResult.saved_objects.filter(obj => !obj.error).length; } return { + successCount, success: errors.length === 0, - successCount: objectsToResolve.length - errors.length, ...(errors.length ? { errors } : {}), }; } diff --git a/src/legacy/server/saved_objects/import/split_overwrites.test.ts b/src/legacy/server/saved_objects/import/split_overwrites.test.ts new file mode 100644 index 000000000000..2fd9d70db56d --- /dev/null +++ b/src/legacy/server/saved_objects/import/split_overwrites.test.ts @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { splitOverwrites } from './split_overwrites'; + +describe('splitOverwrites()', () => { + test('should split array accordingly', () => { + const retries = [ + { + type: 'a', + id: '1', + overwrite: true, + replaceReferences: [], + }, + { + id: '2', + type: 'b', + overwrite: false, + replaceReferences: [], + }, + { + type: 'c', + id: '3', + overwrite: true, + replaceReferences: [], + }, + ]; + const savedObjects = [ + { + id: '1', + type: 'a', + attributes: {}, + references: [], + }, + { + id: '2', + type: 'b', + attributes: {}, + references: [], + }, + { + id: '3', + type: 'c', + attributes: {}, + references: [], + }, + ]; + const result = splitOverwrites(savedObjects, retries); + expect(result).toMatchInlineSnapshot(` +Object { + "objectsToNotOverwrite": Array [ + Object { + "attributes": Object {}, + "id": "2", + "references": Array [], + "type": "b", + }, + ], + "objectsToOverwrite": Array [ + Object { + "attributes": Object {}, + "id": "1", + "references": Array [], + "type": "a", + }, + Object { + "attributes": Object {}, + "id": "3", + "references": Array [], + "type": "c", + }, + ], +} +`); + }); +}); diff --git a/src/legacy/server/saved_objects/import/split_overwrites.ts b/src/legacy/server/saved_objects/import/split_overwrites.ts new file mode 100644 index 000000000000..5609308f755f --- /dev/null +++ b/src/legacy/server/saved_objects/import/split_overwrites.ts @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { SavedObject } from '../service'; +import { Retry } from './types'; + +export function splitOverwrites(savedObjects: SavedObject[], retries: Retry[]) { + const objectsToOverwrite: SavedObject[] = []; + const objectsToNotOverwrite: SavedObject[] = []; + const overwrites = retries + .filter(retry => retry.overwrite) + .map(retry => `${retry.type}:${retry.id}`); + + for (const savedObject of savedObjects) { + if (overwrites.includes(`${savedObject.type}:${savedObject.id}`)) { + objectsToOverwrite.push(savedObject); + } else { + objectsToNotOverwrite.push(savedObject); + } + } + + return { objectsToOverwrite, objectsToNotOverwrite }; +} diff --git a/src/legacy/server/saved_objects/import/types.ts b/src/legacy/server/saved_objects/import/types.ts index b3933966a32d..9deb9d7ac9b3 100644 --- a/src/legacy/server/saved_objects/import/types.ts +++ b/src/legacy/server/saved_objects/import/types.ts @@ -17,26 +17,42 @@ * under the License. */ -interface ConflictError { +export interface Retry { + type: string; + id: string; + overwrite: boolean; + replaceReferences: Array<{ + type: string; + from: string; + to: string; + }>; +} + +export interface ConflictError { type: 'conflict'; } -interface UnknownError { +export interface UnknownError { type: 'unknown'; message: string; statusCode: number; } -interface MissingReferencesError { +export interface MissingReferencesError { type: 'missing_references'; references: Array<{ type: string; id: string; }>; + blocking: Array<{ + type: string; + id: string; + }>; } export interface ImportError { id: string; type: string; + title?: string; error: ConflictError | MissingReferencesError | UnknownError; } diff --git a/src/legacy/server/saved_objects/import/validate_references.test.ts b/src/legacy/server/saved_objects/import/validate_references.test.ts index 159f40b4cf2b..e69ca99fb10f 100644 --- a/src/legacy/server/saved_objects/import/validate_references.test.ts +++ b/src/legacy/server/saved_objects/import/validate_references.test.ts @@ -299,7 +299,9 @@ Object { { id: '2', type: 'visualization', - attributes: {}, + attributes: { + title: 'My Visualization 2', + }, references: [ { name: 'ref_0', @@ -311,7 +313,9 @@ Object { { id: '4', type: 'visualization', - attributes: {}, + attributes: { + title: 'My Visualization 4', + }, references: [ { name: 'ref_0', @@ -342,6 +346,7 @@ Object { "errors": Array [ Object { "error": Object { + "blocking": Array [], "references": Array [ Object { "id": "3", @@ -351,10 +356,12 @@ Object { "type": "missing_references", }, "id": "2", + "title": "My Visualization 2", "type": "visualization", }, Object { "error": Object { + "blocking": Array [], "references": Array [ Object { "id": "5", @@ -372,6 +379,7 @@ Object { "type": "missing_references", }, "id": "4", + "title": "My Visualization 4", "type": "visualization", }, ], @@ -583,4 +591,36 @@ Object { `); expect(savedObjectsClient.bulkGet).toHaveBeenCalledTimes(0); }); + + test('throws when bulkGet fails', async () => { + savedObjectsClient.bulkGet.mockResolvedValue({ + saved_objects: [ + { + id: '1', + type: 'index-pattern', + error: { + statusCode: 400, + message: 'Error', + }, + }, + ], + }); + const savedObjects = [ + { + id: '2', + type: 'visualization', + attributes: {}, + references: [ + { + name: 'ref_0', + type: 'index-pattern', + id: '1', + }, + ], + }, + ]; + await expect( + validateReferences(savedObjects, savedObjectsClient) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"Bad Request"`); + }); }); diff --git a/src/legacy/server/saved_objects/import/validate_references.ts b/src/legacy/server/saved_objects/import/validate_references.ts index 3b5570949467..e44cacd99248 100644 --- a/src/legacy/server/saved_objects/import/validate_references.ts +++ b/src/legacy/server/saved_objects/import/validate_references.ts @@ -17,37 +17,61 @@ * under the License. */ +import Boom from 'boom'; import { SavedObject, SavedObjectsClient } from '../service'; import { ImportError } from './types'; -const ENFORCED_TYPES = ['index-pattern', 'search']; +const REF_TYPES_TO_VLIDATE = ['index-pattern', 'search']; + +function filterReferencesToValidate({ type }: { type: string }) { + return REF_TYPES_TO_VLIDATE.includes(type); +} export async function getNonExistingReferenceAsKeys( savedObjects: SavedObject[], savedObjectsClient: SavedObjectsClient ) { const collector = new Map(); + // Collect all references within objects for (const savedObject of savedObjects) { - for (const { type, id } of savedObject.references || []) { - if (!ENFORCED_TYPES.includes(type)) { - continue; - } + const filteredReferences = (savedObject.references || []).filter(filterReferencesToValidate); + for (const { type, id } of filteredReferences) { collector.set(`${type}:${id}`, { type, id }); } } + + // Remove objects that could be references for (const savedObject of savedObjects) { collector.delete(`${savedObject.type}:${savedObject.id}`); } - if (collector.size) { - const bulkGetOpts = Array.from(collector.values()).map(obj => ({ ...obj, fields: ['id'] })); - const bulkGetResponse = await savedObjectsClient.bulkGet(bulkGetOpts); - for (const savedObject of bulkGetResponse.saved_objects) { - if (savedObject.error) { - continue; - } - collector.delete(`${savedObject.type}:${savedObject.id}`); - } + if (collector.size === 0) { + return []; } + + // Fetch references to see if they exist + const bulkGetOpts = Array.from(collector.values()).map(obj => ({ ...obj, fields: ['id'] })); + const bulkGetResponse = await savedObjectsClient.bulkGet(bulkGetOpts); + + // Error handling + const erroredObjects = bulkGetResponse.saved_objects.filter( + obj => obj.error && obj.error.statusCode !== 404 + ); + if (erroredObjects.length) { + const err = Boom.badRequest(); + err.output.payload.attributes = { + objects: erroredObjects, + }; + throw err; + } + + // Cleanup collector + for (const savedObject of bulkGetResponse.saved_objects) { + if (savedObject.error) { + continue; + } + collector.delete(`${savedObject.type}:${savedObject.id}`); + } + return [...collector.keys()]; } @@ -55,39 +79,59 @@ export async function validateReferences( savedObjects: SavedObject[], savedObjectsClient: SavedObjectsClient ) { - const errors: ImportError[] = []; - + const errorMap: { [key: string]: ImportError } = {}; const nonExistingReferenceKeys = await getNonExistingReferenceAsKeys( savedObjects, savedObjectsClient ); // Filter out objects with missing references, add to error object - const filteredObjects = savedObjects.filter(savedObject => { + let filteredObjects = savedObjects.filter(savedObject => { const missingReferences = []; - for (const { type: refType, id: refId } of savedObject.references || []) { - if (!ENFORCED_TYPES.includes(refType)) { - continue; - } + const enforcedTypeReferences = (savedObject.references || []).filter( + filterReferencesToValidate + ); + for (const { type: refType, id: refId } of enforcedTypeReferences) { if (nonExistingReferenceKeys.includes(`${refType}:${refId}`)) { missingReferences.push({ type: refType, id: refId }); } } - if (missingReferences.length) { - errors.push({ - id: savedObject.id, - type: savedObject.type, - error: { - type: 'missing_references', - references: missingReferences, - }, - }); + if (missingReferences.length === 0) { + return true; } - return missingReferences.length === 0; + errorMap[`${savedObject.type}:${savedObject.id}`] = { + id: savedObject.id, + type: savedObject.type, + title: savedObject.attributes && savedObject.attributes.title, + error: { + type: 'missing_references', + references: missingReferences, + blocking: [], + }, + }; + return false; + }); + + // Filter out objects that reference objects within the import but are missing_references + // For example: visualization referencing a search that is missing an index pattern needs to be filtered out + filteredObjects = filteredObjects.filter(savedObject => { + let isBlocked = false; + for (const reference of savedObject.references || []) { + const referencedObjectError = errorMap[`${reference.type}:${reference.id}`]; + if (!referencedObjectError || referencedObjectError.error.type !== 'missing_references') { + continue; + } + referencedObjectError.error.blocking.push({ + type: savedObject.type, + id: savedObject.id, + }); + isBlocked = true; + } + return !isBlocked; }); return { - errors, + errors: Object.values(errorMap), filteredObjects, }; } diff --git a/src/legacy/server/saved_objects/routes/import.test.ts b/src/legacy/server/saved_objects/routes/import.test.ts index 1d87f4c7157c..a907de8d7c69 100644 --- a/src/legacy/server/saved_objects/routes/import.test.ts +++ b/src/legacy/server/saved_objects/routes/import.test.ts @@ -36,13 +36,7 @@ describe('POST /api/saved_objects/_import', () => { beforeEach(() => { server = createMockServer(); - savedObjectsClient.bulkCreate.mockReset(); - savedObjectsClient.bulkGet.mockReset(); - savedObjectsClient.create.mockReset(); - savedObjectsClient.delete.mockReset(); - savedObjectsClient.find.mockReset(); - savedObjectsClient.get.mockReset(); - savedObjectsClient.update.mockReset(); + jest.resetAllMocks(); const prereqs = { getSavedObjectsClient: { @@ -180,6 +174,7 @@ describe('POST /api/saved_objects/_import', () => { { id: 'my-pattern', type: 'index-pattern', + title: 'my-pattern-*', error: { type: 'conflict', }, @@ -187,4 +182,88 @@ describe('POST /api/saved_objects/_import', () => { ], }); }); + + test('imports a visualization with missing references', async () => { + // NOTE: changes to this scenario should be reflected in the docs + const request = { + method: 'POST', + url: '/api/saved_objects/_import', + payload: [ + '--EXAMPLE', + 'Content-Disposition: form-data; name="file"; filename="export.ndjson"', + 'Content-Type: application/ndjson', + '', + '{"type":"visualization","id":"my-vis","attributes":{"title":"my-vis"},"references":[{"name":"ref_0","type":"index-pattern","id":"my-pattern-*"}]}', + '{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"},"references":[{"name":"ref_0","type":"visualization","id":"my-vis"}]}', + '--EXAMPLE--', + ].join('\r\n'), + headers: { + 'content-Type': 'multipart/form-data; boundary=EXAMPLE', + }, + }; + savedObjectsClient.bulkGet.mockResolvedValueOnce({ + saved_objects: [ + { + id: 'my-pattern-*', + type: 'index-pattern', + error: { + statusCode: 404, + message: 'Not found', + }, + }, + ], + }); + const { payload, statusCode } = await server.inject(request); + const response = JSON.parse(payload); + expect(statusCode).toBe(200); + expect(response).toEqual({ + success: false, + successCount: 0, + errors: [ + { + id: 'my-vis', + type: 'visualization', + title: 'my-vis', + error: { + type: 'missing_references', + references: [ + { + type: 'index-pattern', + id: 'my-pattern-*', + }, + ], + blocking: [ + { + type: 'dashboard', + id: 'my-dashboard', + }, + ], + }, + }, + ], + }); + expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(` +[MockFunction] { + "calls": Array [ + Array [ + Array [ + Object { + "fields": Array [ + "id", + ], + "id": "my-pattern-*", + "type": "index-pattern", + }, + ], + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Promise {}, + }, + ], +} +`); + }); }); diff --git a/src/legacy/server/saved_objects/routes/resolve_import_errors.test.ts b/src/legacy/server/saved_objects/routes/resolve_import_errors.test.ts index f4bd08e5b3ff..0ab2965655b0 100644 --- a/src/legacy/server/saved_objects/routes/resolve_import_errors.test.ts +++ b/src/legacy/server/saved_objects/routes/resolve_import_errors.test.ts @@ -36,13 +36,7 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => { beforeEach(() => { server = createMockServer(); - savedObjectsClient.bulkCreate.mockReset(); - savedObjectsClient.bulkGet.mockReset(); - savedObjectsClient.create.mockReset(); - savedObjectsClient.delete.mockReset(); - savedObjectsClient.find.mockReset(); - savedObjectsClient.get.mockReset(); - savedObjectsClient.update.mockReset(); + jest.resetAllMocks(); const prereqs = { getSavedObjectsClient: { @@ -66,6 +60,10 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => { 'Content-Type: application/ndjson', '', '', + '--BOUNDARY', + 'Content-Disposition: form-data; name="retries"', + '', + '[]', '--BOUNDARY--', ].join('\r\n'), headers: { @@ -79,7 +77,68 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => { expect(savedObjectsClient.bulkCreate).toHaveBeenCalledTimes(0); }); - test('resolves conflicts for an index pattern and dashboard but skips the index pattern', async () => { + test('retries importin a dashboard', async () => { + // NOTE: changes to this scenario should be reflected in the docs + const request = { + method: 'POST', + url: '/api/saved_objects/_resolve_import_errors', + payload: [ + '--EXAMPLE', + 'Content-Disposition: form-data; name="file"; filename="export.ndjson"', + 'Content-Type: application/ndjson', + '', + '{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"}}', + '--EXAMPLE', + 'Content-Disposition: form-data; name="retries"', + '', + '[{"type":"dashboard","id":"my-dashboard"}]', + '--EXAMPLE--', + ].join('\r\n'), + headers: { + 'content-Type': 'multipart/form-data; boundary=EXAMPLE', + }, + }; + savedObjectsClient.bulkCreate.mockResolvedValueOnce({ + saved_objects: [ + { + type: 'dashboard', + id: 'my-dashboard', + attributes: { + title: 'Look at my dashboard', + }, + }, + ], + }); + const { payload, statusCode } = await server.inject(request); + const response = JSON.parse(payload); + expect(statusCode).toBe(200); + expect(response).toEqual({ success: true, successCount: 1 }); + expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(` +[MockFunction] { + "calls": Array [ + Array [ + Array [ + Object { + "attributes": Object { + "title": "Look at my dashboard", + }, + "id": "my-dashboard", + "type": "dashboard", + }, + ], + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Promise {}, + }, + ], +} +`); + }); + + test('resolves conflicts for dashboard', async () => { // NOTE: changes to this scenario should be reflected in the docs const request = { method: 'POST', @@ -92,13 +151,9 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => { '{"type":"index-pattern","id":"my-pattern","attributes":{"title":"my-pattern-*"}}', '{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"}}', '--EXAMPLE', - 'Content-Disposition: form-data; name="skips"', + 'Content-Disposition: form-data; name="retries"', '', - '[{"type":"index-pattern","id":"my-pattern"}]', - '--EXAMPLE', - 'Content-Disposition: form-data; name="overwrites"', - '', - '[{"type":"dashboard","id":"my-dashboard"}]', + '[{"type":"dashboard","id":"my-dashboard","overwrite":true}]', '--EXAMPLE--', ].join('\r\n'), headers: { @@ -158,12 +213,11 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => { 'Content-Disposition: form-data; name="file"; filename="export.ndjson"', 'Content-Type: application/ndjson', '', - '{"type":"visualization","id":"my-vis","attributes":{"title":"Look at my visualization"}}', - '{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"},"references":[{"name":"panel_0","type":"visualization","id":"my-vis"}]}', + '{"type":"visualization","id":"my-vis","attributes":{"title":"Look at my visualization"},"references":[{"name":"ref_0","type":"index-pattern","id":"missing"}]}', '--EXAMPLE', - 'Content-Disposition: form-data; name="replaceReferences"', + 'Content-Disposition: form-data; name="retries"', '', - '[{"type":"visualization","from":"my-vis","to":"my-vis-2"}]', + '[{"type":"visualization","id":"my-vis","replaceReferences":[{"type":"index-pattern","from":"missing","to":"existing"}]}]', '--EXAMPLE--', ].join('\r\n'), headers: { @@ -173,21 +227,31 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => { savedObjectsClient.bulkCreate.mockResolvedValueOnce({ saved_objects: [ { - type: 'dashboard', - id: 'my-dashboard', + type: 'visualization', + id: 'my-vis', attributes: { - title: 'Look at my dashboard', + title: 'Look at my visualization', }, references: [ { - name: 'panel_0', - type: 'visualization', - id: 'my-vis-2', + name: 'ref_0', + type: 'index-pattern', + id: 'existing', }, ], }, ], }); + savedObjectsClient.bulkGet.mockResolvedValueOnce({ + saved_objects: [ + { + id: 'existing', + type: 'index-pattern', + attributes: {}, + references: [], + }, + ], + }); const { payload, statusCode } = await server.inject(request); const response = JSON.parse(payload); expect(statusCode).toBe(200); @@ -199,22 +263,42 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => { Array [ Object { "attributes": Object { - "title": "Look at my dashboard", + "title": "Look at my visualization", }, - "id": "my-dashboard", + "id": "my-vis", "references": Array [ Object { - "id": "my-vis-2", - "name": "panel_0", - "type": "visualization", + "id": "existing", + "name": "ref_0", + "type": "index-pattern", }, ], - "type": "dashboard", + "type": "visualization", + }, + ], + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Promise {}, + }, + ], +} +`); + expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(` +[MockFunction] { + "calls": Array [ + Array [ + Array [ + Object { + "fields": Array [ + "id", + ], + "id": "existing", + "type": "index-pattern", }, ], - Object { - "overwrite": true, - }, ], ], "results": Array [ diff --git a/src/legacy/server/saved_objects/routes/resolve_import_errors.ts b/src/legacy/server/saved_objects/routes/resolve_import_errors.ts index 71c472fae6ea..1d61dc10978e 100644 --- a/src/legacy/server/saved_objects/routes/resolve_import_errors.ts +++ b/src/legacy/server/saved_objects/routes/resolve_import_errors.ts @@ -38,18 +38,15 @@ interface ImportRequest extends Hapi.Request { }; payload: { file: HapiReadableStream; - overwrites: Array<{ - type: string; - id: string; - }>; - replaceReferences: Array<{ - type: string; - from: string; - to: string; - }>; - skips: Array<{ + retries: Array<{ type: string; id: string; + overwrite: boolean; + replaceReferences: Array<{ + type: string; + from: string; + to: string; + }>; }>; }; } @@ -67,31 +64,24 @@ export const createResolveImportErrorsRoute = (prereqs: Prerequisites, server: H validate: { payload: Joi.object({ file: Joi.object().required(), - overwrites: Joi.array() + retries: Joi.array() .items( Joi.object({ type: Joi.string().required(), id: Joi.string().required(), + overwrite: Joi.boolean().default(false), + replaceReferences: Joi.array() + .items( + Joi.object({ + type: Joi.string().required(), + from: Joi.string().required(), + to: Joi.string().required(), + }) + ) + .default([]), }) ) - .default([]), - replaceReferences: Joi.array() - .items( - Joi.object({ - type: Joi.string().required(), - from: Joi.string().required(), - to: Joi.string().required(), - }) - ) - .default([]), - skips: Joi.array() - .items( - Joi.object({ - type: Joi.string().required(), - id: Joi.string().required(), - }) - ) - .default([]), + .required(), }).default(), }, }, @@ -99,16 +89,16 @@ export const createResolveImportErrorsRoute = (prereqs: Prerequisites, server: H const { savedObjectsClient } = request.pre; const { filename } = request.payload.file.hapi; const fileExtension = extname(filename).toLowerCase(); + if (fileExtension !== '.ndjson') { return Boom.badRequest(`Invalid file extension ${fileExtension}`); } + return await resolveImportErrors({ savedObjectsClient, readStream: request.payload.file, + retries: request.payload.retries, objectLimit: request.server.config().get('savedObjects.maxImportExportSize'), - skips: request.payload.skips, - overwrites: request.payload.overwrites, - replaceReferences: request.payload.replaceReferences, }); }, }); diff --git a/test/api_integration/apis/saved_objects/import.js b/test/api_integration/apis/saved_objects/import.js index 19fd051e5f44..98dc729bdc87 100644 --- a/test/api_integration/apis/saved_objects/import.js +++ b/test/api_integration/apis/saved_objects/import.js @@ -70,6 +70,7 @@ export default function ({ getService }) { { id: '91200a00-9efd-11e7-acb3-3dab96693fab', type: 'index-pattern', + title: 'logstash-*', error: { type: 'conflict', } @@ -77,6 +78,7 @@ export default function ({ getService }) { { id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', type: 'visualization', + title: 'Count of requests', error: { type: 'conflict', } @@ -84,6 +86,7 @@ export default function ({ getService }) { { id: 'be3733a0-9efe-11e7-acb3-3dab96693fab', type: 'dashboard', + title: 'Requests', error: { type: 'conflict', } @@ -161,6 +164,7 @@ export default function ({ getService }) { id: '1', error: { type: 'missing_references', + blocking: [], references: [ { type: 'index-pattern', diff --git a/test/api_integration/apis/saved_objects/resolve_import_errors.js b/test/api_integration/apis/saved_objects/resolve_import_errors.js index 84867960a8e3..665a36fe8816 100644 --- a/test/api_integration/apis/saved_objects/resolve_import_errors.js +++ b/test/api_integration/apis/saved_objects/resolve_import_errors.js @@ -32,6 +32,7 @@ export default function ({ getService }) { it('should return 200 and import nothing when empty parameters are passed in', async () => { await supertest .post('/api/saved_objects/_resolve_import_errors') + .field('retries', '[]') .attach('file', join(__dirname, '../../fixtures/import.ndjson')) .expect(200) .then((resp) => { @@ -45,18 +46,21 @@ export default function ({ getService }) { it('should return 200 and import everything when overwrite parameters contains all objects', async () => { await supertest .post('/api/saved_objects/_resolve_import_errors') - .field('overwrites', JSON.stringify([ + .field('retries', JSON.stringify([ { type: 'index-pattern', id: '91200a00-9efd-11e7-acb3-3dab96693fab', + overwrite: true, }, { type: 'visualization', id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', + overwrite: true, }, { type: 'dashboard', id: 'be3733a0-9efe-11e7-acb3-3dab96693fab', + overwrite: true, }, ])) .attach('file', join(__dirname, '../../fixtures/import.ndjson')) @@ -72,7 +76,7 @@ export default function ({ getService }) { it('should return 400 when no file passed in', async () => { await supertest .post('/api/saved_objects/_resolve_import_errors') - .field('skips', '[]') + .field('retries', '[]') .expect(400) .then((resp) => { expect(resp.body).to.eql({ @@ -84,7 +88,26 @@ export default function ({ getService }) { }); }); - it('should return 200 when replacing references', async () => { + it('should return 400 when resolving conflicts with a file containing more than 10,000 objects', async () => { + const fileChunks = []; + for (let i = 0; i < 10001; i++) { + fileChunks.push(`{"type":"visualization","id":"${i}","attributes":{},"references":[]}`); + } + await supertest + .post('/api/saved_objects/_resolve_import_errors') + .field('retries', '[]') + .attach('file', Buffer.from(fileChunks.join('\n'), 'utf8'), 'export.ndjson') + .expect(400) + .then((resp) => { + expect(resp.body).to.eql({ + statusCode: 400, + error: 'Bad Request', + message: 'Can\'t import more than 10000 objects', + }); + }); + }); + + it('should return 200 with errors when missing references', async () => { const objToInsert = { id: '1', type: 'visualization', @@ -94,58 +117,44 @@ export default function ({ getService }) { references: [ { name: 'ref_0', - type: 'search', - id: '1', + type: 'index-pattern', + id: '2', }, - ] + ], }; await supertest .post('/api/saved_objects/_resolve_import_errors') - .field('replaceReferences', JSON.stringify( + .field('retries', JSON.stringify( [ { - type: 'search', - from: '1', - to: '2', - } + type: 'visualization', + id: '1', + }, ] )) .attach('file', Buffer.from(JSON.stringify(objToInsert), 'utf8'), 'export.ndjson') .expect(200) .then((resp) => { expect(resp.body).to.eql({ - success: true, - successCount: 1, - }); - }); - await supertest - .get('/api/saved_objects/visualization/1') - .expect(200) - .then((resp) => { - expect(resp.body.references).to.eql([ - { - name: 'ref_0', - type: 'search', - id: '2', - }, - ]); - }); - }); - - it('should return 400 when resolving conflicts with a file containing more than 10,000 objects', async () => { - const fileChunks = []; - for (let i = 0; i < 10001; i++) { - fileChunks.push(`{"type":"visualization","id":"${i}","attributes":{},"references":[]}`); - } - await supertest - .post('/api/saved_objects/_resolve_import_errors') - .attach('file', Buffer.from(fileChunks.join('\n'), 'utf8'), 'export.ndjson') - .expect(400) - .then((resp) => { - expect(resp.body).to.eql({ - statusCode: 400, - error: 'Bad Request', - message: 'Can\'t import more than 10000 objects', + success: false, + successCount: 0, + errors: [ + { + id: '1', + type: 'visualization', + title: 'My favorite vis', + error: { + type: 'missing_references', + blocking: [], + references: [ + { + type: 'index-pattern', + id: '2', + }, + ], + }, + }, + ], }); }); }); @@ -159,22 +168,7 @@ export default function ({ getService }) { it('should return 200 when skipping all the records', async () => { await supertest .post('/api/saved_objects/_resolve_import_errors') - .field('skips', JSON.stringify( - [ - { - id: '91200a00-9efd-11e7-acb3-3dab96693fab', - type: 'index-pattern', - }, - { - id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', - type: 'visualization', - }, - { - id: 'be3733a0-9efe-11e7-acb3-3dab96693fab', - type: 'dashboard', - }, - ] - )) + .field('retries', '[]') .attach('file', join(__dirname, '../../fixtures/import.ndjson')) .expect(200) .then((resp) => { @@ -185,19 +179,22 @@ export default function ({ getService }) { it('should return 200 when manually overwriting each object', async () => { await supertest .post('/api/saved_objects/_resolve_import_errors') - .field('overwrites', JSON.stringify( + .field('retries', JSON.stringify( [ { id: '91200a00-9efd-11e7-acb3-3dab96693fab', type: 'index-pattern', + overwrite: true, }, { id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', type: 'visualization', + overwrite: true, }, { id: 'be3733a0-9efe-11e7-acb3-3dab96693fab', type: 'dashboard', + overwrite: true, }, ] )) @@ -211,19 +208,12 @@ export default function ({ getService }) { it('should return 200 with only one record when overwriting 1 and skipping 1', async () => { await supertest .post('/api/saved_objects/_resolve_import_errors') - .field('overwrites', JSON.stringify( + .field('retries', JSON.stringify( [ { id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', type: 'visualization', - }, - ] - )) - .field('skips', JSON.stringify( - [ - { - id: '91200a00-9efd-11e7-acb3-3dab96693fab', - type: 'index-pattern', + overwrite: true, }, ] )) @@ -233,6 +223,60 @@ export default function ({ getService }) { expect(resp.body).to.eql({ success: true, successCount: 1 }); }); }); + + it('should return 200 when replacing references', async () => { + const objToInsert = { + id: '1', + type: 'visualization', + attributes: { + title: 'My favorite vis', + }, + references: [ + { + name: 'ref_0', + type: 'index-pattern', + id: '2', + }, + ] + }; + await supertest + .post('/api/saved_objects/_resolve_import_errors') + .field('retries', JSON.stringify( + [ + { + type: 'visualization', + id: '1', + replaceReferences: [ + { + type: 'index-pattern', + from: '2', + to: '91200a00-9efd-11e7-acb3-3dab96693fab', + }, + ], + }, + ] + )) + .attach('file', Buffer.from(JSON.stringify(objToInsert), 'utf8'), 'export.ndjson') + .expect(200) + .then((resp) => { + expect(resp.body).to.eql({ + success: true, + successCount: 1, + }); + }); + await supertest + .get('/api/saved_objects/visualization/1') + .expect(200) + .then((resp) => { + expect(resp.body.references).to.eql([ + { + name: 'ref_0', + type: 'index-pattern', + id: '91200a00-9efd-11e7-acb3-3dab96693fab', + }, + ]); + }); + }); }); }); }); diff --git a/x-pack/test/saved_object_api_integration/common/suites/import.ts b/x-pack/test/saved_object_api_integration/common/suites/import.ts index 816b68d3eed3..5ccb42ac2982 100644 --- a/x-pack/test/saved_object_api_integration/common/suites/import.ts +++ b/x-pack/test/saved_object_api_integration/common/suites/import.ts @@ -61,6 +61,7 @@ export function importTestSuiteFactory(es: any, esArchiver: any, supertest: Supe { id: '1', type: 'wigwags', + title: 'Wigwags title', error: { message: `Unsupported saved object type: 'wigwags': Bad Request`, statusCode: 400, diff --git a/x-pack/test/saved_object_api_integration/common/suites/resolve_import_errors.ts b/x-pack/test/saved_object_api_integration/common/suites/resolve_import_errors.ts index 0a06ca064bee..7419de868be5 100644 --- a/x-pack/test/saved_object_api_integration/common/suites/resolve_import_errors.ts +++ b/x-pack/test/saved_object_api_integration/common/suites/resolve_import_errors.ts @@ -65,6 +65,7 @@ export function resolveImportErrorsTestSuiteFactory( { id: '1', type: 'wigwags', + title: 'Wigwags title', error: { message: `Unsupported saved object type: 'wigwags': Bad Request`, statusCode: 400, @@ -116,11 +117,12 @@ export function resolveImportErrorsTestSuiteFactory( .post(`${getUrlPrefix(spaceId)}/api/saved_objects/_resolve_import_errors`) .auth(user.username, user.password) .field( - 'overwrites', + 'retries', JSON.stringify([ { type: 'dashboard', id: `${getIdPrefix(spaceId)}a01b2f57-fcfd-4864-b735-09e28f0d815e`, + overwrite: true, }, ]) ) @@ -147,15 +149,17 @@ export function resolveImportErrorsTestSuiteFactory( .post(`${getUrlPrefix(spaceId)}/api/saved_objects/_resolve_import_errors`) .auth(user.username, user.password) .field( - 'overwrites', + 'retries', JSON.stringify([ { type: 'wigwags', id: '1', + overwrite: true, }, { type: 'dashboard', id: `${getIdPrefix(spaceId)}a01b2f57-fcfd-4864-b735-09e28f0d815e`, + overwrite: true, }, ]) )