Modify saved object import APIs to handle special use cases from the previous import process (#34161)
* Modify import APIs to handle special use cases from the previous import process * Cleanup * Add more examples to the docs * Make title come from data inside file * Fix some broken tests * Fix docs * Fix docs wording * Apply PR feedback pt1 * Apply PR feedback pt2
This commit is contained in:
parent
5c2267f2ca
commit
51e6a009ee
22 changed files with 1288 additions and 514 deletions
|
@ -86,11 +86,58 @@ containing a JSON structure similar to the following example:
|
|||
{
|
||||
"id": "my-pattern",
|
||||
"type": "index-pattern",
|
||||
"title": "my-pattern-*",
|
||||
"error": {
|
||||
"statusCode": 409,
|
||||
"message": "version conflict, document already exists",
|
||||
"type": "conflict"
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
--------------------------------------------------
|
||||
|
||||
The following example imports a visualization and dashboard but the index pattern for the visualization reference doesn't exist.
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
POST api/saved_objects/_import
|
||||
Content-Type: multipart/form-data; boundary=EXAMPLE
|
||||
--EXAMPLE
|
||||
Content-Disposition: form-data; name="file"; filename="export.ndjson"
|
||||
Content-Type: application/ndjson
|
||||
|
||||
{"type":"visualization","id":"my-vis","attributes":{"title":"my-vis"},"references":[{"name":"ref_0","type":"index-pattern","id":"my-pattern-*"}]}
|
||||
{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"},"references":[{"name":"ref_0","type":"visualization","id":"my-vis"}]}
|
||||
--EXAMPLE--
|
||||
--------------------------------------------------
|
||||
// KIBANA
|
||||
|
||||
The call returns a response code of `200` and a response body
|
||||
containing a JSON structure similar to the following example:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
"success": false,
|
||||
"successCount": 0,
|
||||
"errors": [
|
||||
{
|
||||
"id": "my-vis",
|
||||
"type": "visualization",
|
||||
"title": "my-vis",
|
||||
"error": {
|
||||
"type": "missing_references",
|
||||
"references": [
|
||||
{
|
||||
"type": "index-pattern",
|
||||
"id": "my-pattern-*"
|
||||
}
|
||||
],
|
||||
"blocking": [
|
||||
{
|
||||
"type": "dashboard",
|
||||
"id": "my-dashboard"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
--------------------------------------------------
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
experimental[This functionality is *experimental* and may be changed or removed completely in a future release.]
|
||||
|
||||
The resolve import errors API enables you to resolve errors given by the import API by either overwriting specific saved objects or changing references to a newly created object.
|
||||
The resolve import errors API enables you to resolve errors given by the import API by either retrying certain saved objects, overwriting specific saved objects or changing references to different saved objects.
|
||||
|
||||
Note: You cannot access this endpoint via the Console in Kibana.
|
||||
|
||||
|
@ -16,27 +16,20 @@ Note: You cannot access this endpoint via the Console in Kibana.
|
|||
The request body must be of type multipart/form-data.
|
||||
|
||||
`file`::
|
||||
(ndjson) The same new line delimited JSON objects given to the import API.
|
||||
The same file given to the import API.
|
||||
|
||||
`overwrites` (optional)::
|
||||
(array) A list of `type` and `id` objects allowed to be overwritten on import.
|
||||
|
||||
`replaceReferences` (optional)::
|
||||
(array) A list of `type`, `from` and `to` used to change imported saved object references to.
|
||||
|
||||
`skips` (optional)::
|
||||
(array) A list of `type` and `id` objects to skip importing.
|
||||
`retries`::
|
||||
(array) A list of `type`, `id`, `replaceReferences` and `overwrite` objects to retry importing. The property `replaceReferences` is a list of `type`, `from` and `to` used to change the object's references.
|
||||
|
||||
==== Response body
|
||||
|
||||
The response body will have a top level `success` property that indicates
|
||||
if the import was successful or not as well as a `successCount` indicating how many records are successfully resolved.
|
||||
In the scenario the import wasn't successful a top level `errors` array will contain the objects that failed to import.
|
||||
if resolving errors was successful or not as well as a `successCount` indicating how many records are successfully resolved.
|
||||
In the scenario resolving errors wasn't successful, a top level `errors` array will contain the objects that failed to be resolved.
|
||||
|
||||
==== Examples
|
||||
|
||||
The following example resolves errors for an index pattern and dashboard but indicates to skip the index pattern.
|
||||
This will cause the index pattern to not be in the system and the dashboard to overwrite the existing saved object.
|
||||
The following example retries importing a dashboard.
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -46,14 +39,9 @@ Content-Type: multipart/form-data; boundary=EXAMPLE
|
|||
Content-Disposition: form-data; name="file"; filename="export.ndjson"
|
||||
Content-Type: application/ndjson
|
||||
|
||||
{"type":"index-pattern","id":"my-pattern","attributes":{"title":"my-pattern-*"}}
|
||||
{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"}}
|
||||
--EXAMPLE
|
||||
Content-Disposition: form-data; name="skips"
|
||||
|
||||
[{"type":"index-pattern","id":"my-pattern"}]
|
||||
--EXAMPLE
|
||||
Content-Disposition: form-data; name="overwrites"
|
||||
Content-Disposition: form-data; name="retries"
|
||||
|
||||
[{"type":"dashboard","id":"my-dashboard"}]
|
||||
--EXAMPLE--
|
||||
|
@ -71,8 +59,7 @@ containing a JSON structure similar to the following example:
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
The following example resolves errors for a visualization and dashboard but indicates
|
||||
to replace the dashboard references to another visualization.
|
||||
The following example resolves errors for a dashboard. This will cause the dashboard to overwrite the existing saved object.
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -82,12 +69,42 @@ Content-Type: multipart/form-data; boundary=EXAMPLE
|
|||
Content-Disposition: form-data; name="file"; filename="export.ndjson"
|
||||
Content-Type: application/ndjson
|
||||
|
||||
{"type":"visualization","id":"my-vis","attributes":{"title":"Look at my visualization"}}
|
||||
{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"},"references":[{"name":"panel_0","type":"visualization","id":"my-vis"}]}
|
||||
{"type":"index-pattern","id":"my-pattern","attributes":{"title":"my-pattern-*"}}
|
||||
{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"}}
|
||||
--EXAMPLE
|
||||
Content-Disposition: form-data; name="replaceReferences"
|
||||
Content-Disposition: form-data; name="retries"
|
||||
|
||||
[{"type":"visualization","from":"my-vis","to":"my-vis-2"}]
|
||||
[{"type":"dashboard","id":"my-dashboard","overwrite":true}]
|
||||
--EXAMPLE--
|
||||
--------------------------------------------------
|
||||
// KIBANA
|
||||
|
||||
A successful call returns a response code of `200` and a response body
|
||||
containing a JSON structure similar to the following example:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
{
|
||||
"success": true,
|
||||
"successCount": 1
|
||||
}
|
||||
--------------------------------------------------
|
||||
|
||||
The following example resolves errors for a visualization by replacing the index pattern to another.
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
POST api/saved_objects/_resolve_import_errors
|
||||
Content-Type: multipart/form-data; boundary=EXAMPLE
|
||||
--EXAMPLE
|
||||
Content-Disposition: form-data; name="file"; filename="export.ndjson"
|
||||
Content-Type: application/ndjson
|
||||
|
||||
{"type":"visualization","id":"my-vis","attributes":{"title":"Look at my visualization"},"references":[{"name":"ref_0","type":"index-pattern","id":"missing"}]}
|
||||
--EXAMPLE
|
||||
Content-Disposition: form-data; name="retries"
|
||||
|
||||
[{"type":"visualization","id":"my-vis","replaceReferences":[{"type":"index-pattern","from":"missing","to":"existing"}]}]
|
||||
--EXAMPLE--
|
||||
--------------------------------------------------
|
||||
// KIBANA
|
||||
|
|
|
@ -20,34 +20,45 @@
|
|||
import { createObjectsFilter } from './create_objects_filter';
|
||||
|
||||
describe('createObjectsFilter()', () => {
|
||||
test('filters should return false when contains empty parameters', () => {
|
||||
const fn = createObjectsFilter([], [], []);
|
||||
test('filter should return false when contains empty parameters', () => {
|
||||
const fn = createObjectsFilter([]);
|
||||
expect(fn({ type: 'a', id: '1', attributes: {}, references: [] })).toEqual(false);
|
||||
});
|
||||
|
||||
test('filters should exclude skips', () => {
|
||||
const fn = createObjectsFilter(
|
||||
[
|
||||
test('filter should return true for objects that are being retried', () => {
|
||||
const fn = createObjectsFilter([
|
||||
{
|
||||
type: 'a',
|
||||
id: '1',
|
||||
overwrite: false,
|
||||
replaceReferences: [],
|
||||
},
|
||||
],
|
||||
[],
|
||||
[
|
||||
{
|
||||
type: 'b',
|
||||
from: '1',
|
||||
to: '2',
|
||||
},
|
||||
]
|
||||
);
|
||||
]);
|
||||
expect(
|
||||
fn({
|
||||
type: 'a',
|
||||
id: '1',
|
||||
attributes: {},
|
||||
references: [{ name: 'ref_0', type: 'b', id: '1' }],
|
||||
references: [],
|
||||
})
|
||||
).toEqual(true);
|
||||
});
|
||||
|
||||
test(`filter should return false for objects that aren't being retried`, () => {
|
||||
const fn = createObjectsFilter([
|
||||
{
|
||||
type: 'a',
|
||||
id: '1',
|
||||
overwrite: false,
|
||||
replaceReferences: [],
|
||||
},
|
||||
]);
|
||||
expect(
|
||||
fn({
|
||||
type: 'b',
|
||||
id: '1',
|
||||
attributes: {},
|
||||
references: [],
|
||||
})
|
||||
).toEqual(false);
|
||||
expect(
|
||||
|
@ -55,131 +66,8 @@ describe('createObjectsFilter()', () => {
|
|||
type: 'a',
|
||||
id: '2',
|
||||
attributes: {},
|
||||
references: [{ name: 'ref_0', type: 'b', id: '1' }],
|
||||
})
|
||||
).toEqual(true);
|
||||
});
|
||||
|
||||
test('filter should include references to replace', () => {
|
||||
const fn = createObjectsFilter(
|
||||
[],
|
||||
[],
|
||||
[
|
||||
{
|
||||
type: 'b',
|
||||
from: '1',
|
||||
to: '2',
|
||||
},
|
||||
]
|
||||
);
|
||||
expect(
|
||||
fn({
|
||||
type: 'a',
|
||||
id: '1',
|
||||
attributes: {},
|
||||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
type: 'b',
|
||||
id: '1',
|
||||
},
|
||||
],
|
||||
})
|
||||
).toEqual(true);
|
||||
expect(
|
||||
fn({
|
||||
type: 'a',
|
||||
id: '1',
|
||||
attributes: {},
|
||||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
type: 'b',
|
||||
id: '2',
|
||||
},
|
||||
],
|
||||
references: [],
|
||||
})
|
||||
).toEqual(false);
|
||||
});
|
||||
|
||||
test('filter should include objects to overwrite', () => {
|
||||
const fn = createObjectsFilter(
|
||||
[],
|
||||
[
|
||||
{
|
||||
type: 'a',
|
||||
id: '1',
|
||||
},
|
||||
],
|
||||
[]
|
||||
);
|
||||
expect(fn({ type: 'a', id: '1', attributes: {}, references: [] })).toEqual(true);
|
||||
expect(fn({ type: 'a', id: '2', attributes: {}, references: [] })).toEqual(false);
|
||||
});
|
||||
|
||||
test('filter should work with skips, overwrites and replaceReferences', () => {
|
||||
const fn = createObjectsFilter(
|
||||
[
|
||||
{
|
||||
type: 'a',
|
||||
id: '1',
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
type: 'a',
|
||||
id: '2',
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
type: 'b',
|
||||
from: '1',
|
||||
to: '2',
|
||||
},
|
||||
]
|
||||
);
|
||||
expect(
|
||||
fn({
|
||||
type: 'a',
|
||||
id: '1',
|
||||
attributes: {},
|
||||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
type: 'b',
|
||||
id: '1',
|
||||
},
|
||||
],
|
||||
})
|
||||
).toEqual(false);
|
||||
expect(
|
||||
fn({
|
||||
type: 'a',
|
||||
id: '2',
|
||||
attributes: {},
|
||||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
type: 'b',
|
||||
id: '2',
|
||||
},
|
||||
],
|
||||
})
|
||||
).toEqual(true);
|
||||
expect(
|
||||
fn({
|
||||
type: 'a',
|
||||
id: '3',
|
||||
attributes: {},
|
||||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
type: 'b',
|
||||
id: '1',
|
||||
},
|
||||
],
|
||||
})
|
||||
).toEqual(true);
|
||||
});
|
||||
});
|
||||
|
|
|
@ -18,37 +18,11 @@
|
|||
*/
|
||||
|
||||
import { SavedObject } from '../service';
|
||||
import { Retry } from './types';
|
||||
|
||||
export function createObjectsFilter(
|
||||
skips: Array<{
|
||||
type: string;
|
||||
id: string;
|
||||
}>,
|
||||
overwrites: Array<{
|
||||
type: string;
|
||||
id: string;
|
||||
}>,
|
||||
replaceReferences: Array<{
|
||||
type: string;
|
||||
from: string;
|
||||
to: string;
|
||||
}>
|
||||
) {
|
||||
const refReplacements = replaceReferences.map(ref => `${ref.type}:${ref.from}`);
|
||||
export function createObjectsFilter(retries: Retry[]) {
|
||||
const retryKeys = new Set<string>(retries.map(retry => `${retry.type}:${retry.id}`));
|
||||
return (obj: SavedObject) => {
|
||||
if (skips.some(skipObj => skipObj.type === obj.type && skipObj.id === obj.id)) {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
overwrites.some(overwriteObj => overwriteObj.type === obj.type && overwriteObj.id === obj.id)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
for (const reference of obj.references || []) {
|
||||
if (refReplacements.includes(`${reference.type}:${reference.id}`)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
return retryKeys.has(`${obj.type}:${obj.id}`);
|
||||
};
|
||||
}
|
||||
|
|
|
@ -23,7 +23,7 @@ import { extractErrors } from './extract_errors';
|
|||
describe('extractErrors()', () => {
|
||||
test('returns empty array when no errors exist', () => {
|
||||
const savedObjects: SavedObject[] = [];
|
||||
const result = extractErrors(savedObjects);
|
||||
const result = extractErrors(savedObjects, savedObjects);
|
||||
expect(result).toMatchInlineSnapshot(`Array []`);
|
||||
});
|
||||
|
||||
|
@ -32,13 +32,17 @@ describe('extractErrors()', () => {
|
|||
{
|
||||
id: '1',
|
||||
type: 'dashboard',
|
||||
attributes: {},
|
||||
attributes: {
|
||||
title: 'My Dashboard 1',
|
||||
},
|
||||
references: [],
|
||||
},
|
||||
{
|
||||
id: '2',
|
||||
type: 'dashboard',
|
||||
attributes: {},
|
||||
attributes: {
|
||||
title: 'My Dashboard 2',
|
||||
},
|
||||
references: [],
|
||||
error: {
|
||||
statusCode: 409,
|
||||
|
@ -48,7 +52,9 @@ describe('extractErrors()', () => {
|
|||
{
|
||||
id: '3',
|
||||
type: 'dashboard',
|
||||
attributes: {},
|
||||
attributes: {
|
||||
title: 'My Dashboard 3',
|
||||
},
|
||||
references: [],
|
||||
error: {
|
||||
statusCode: 400,
|
||||
|
@ -56,7 +62,7 @@ describe('extractErrors()', () => {
|
|||
},
|
||||
},
|
||||
];
|
||||
const result = extractErrors(savedObjects);
|
||||
const result = extractErrors(savedObjects, savedObjects);
|
||||
expect(result).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
|
@ -64,6 +70,7 @@ Array [
|
|||
"type": "conflict",
|
||||
},
|
||||
"id": "2",
|
||||
"title": "My Dashboard 2",
|
||||
"type": "dashboard",
|
||||
},
|
||||
Object {
|
||||
|
@ -73,6 +80,7 @@ Array [
|
|||
"type": "unknown",
|
||||
},
|
||||
"id": "3",
|
||||
"title": "My Dashboard 3",
|
||||
"type": "dashboard",
|
||||
},
|
||||
]
|
||||
|
|
|
@ -20,14 +20,29 @@
|
|||
import { SavedObject } from '../service';
|
||||
import { ImportError } from './types';
|
||||
|
||||
export function extractErrors(savedObjects: SavedObject[]) {
|
||||
export function extractErrors(
|
||||
savedObjectResults: SavedObject[],
|
||||
savedObjectsToImport: SavedObject[]
|
||||
) {
|
||||
const errors: ImportError[] = [];
|
||||
for (const savedObject of savedObjects) {
|
||||
const originalSavedObjectsMap = new Map<string, SavedObject>();
|
||||
for (const savedObject of savedObjectsToImport) {
|
||||
originalSavedObjectsMap.set(`${savedObject.type}:${savedObject.id}`, savedObject);
|
||||
}
|
||||
for (const savedObject of savedObjectResults) {
|
||||
if (savedObject.error) {
|
||||
const originalSavedObject = originalSavedObjectsMap.get(
|
||||
`${savedObject.type}:${savedObject.id}`
|
||||
);
|
||||
const title =
|
||||
originalSavedObject &&
|
||||
originalSavedObject.attributes &&
|
||||
originalSavedObject.attributes.title;
|
||||
if (savedObject.error.statusCode === 409) {
|
||||
errors.push({
|
||||
id: savedObject.id,
|
||||
type: savedObject.type,
|
||||
title,
|
||||
error: {
|
||||
type: 'conflict',
|
||||
},
|
||||
|
@ -37,6 +52,7 @@ export function extractErrors(savedObjects: SavedObject[]) {
|
|||
errors.push({
|
||||
id: savedObject.id,
|
||||
type: savedObject.type,
|
||||
title,
|
||||
error: {
|
||||
...savedObject.error,
|
||||
type: 'unknown',
|
||||
|
|
|
@ -26,25 +26,33 @@ describe('importSavedObjects()', () => {
|
|||
{
|
||||
id: '1',
|
||||
type: 'index-pattern',
|
||||
attributes: {},
|
||||
attributes: {
|
||||
title: 'My Index Pattern',
|
||||
},
|
||||
references: [],
|
||||
},
|
||||
{
|
||||
id: '2',
|
||||
type: 'search',
|
||||
attributes: {},
|
||||
attributes: {
|
||||
title: 'My Search',
|
||||
},
|
||||
references: [],
|
||||
},
|
||||
{
|
||||
id: '3',
|
||||
type: 'visualization',
|
||||
attributes: {},
|
||||
attributes: {
|
||||
title: 'My Visualization',
|
||||
},
|
||||
references: [],
|
||||
},
|
||||
{
|
||||
id: '4',
|
||||
type: 'dashboard',
|
||||
attributes: {},
|
||||
attributes: {
|
||||
title: 'My Dashboard',
|
||||
},
|
||||
references: [],
|
||||
},
|
||||
];
|
||||
|
@ -60,13 +68,27 @@ describe('importSavedObjects()', () => {
|
|||
};
|
||||
|
||||
beforeEach(() => {
|
||||
savedObjectsClient.bulkCreate.mockReset();
|
||||
savedObjectsClient.bulkGet.mockReset();
|
||||
savedObjectsClient.create.mockReset();
|
||||
savedObjectsClient.delete.mockReset();
|
||||
savedObjectsClient.find.mockReset();
|
||||
savedObjectsClient.get.mockReset();
|
||||
savedObjectsClient.update.mockReset();
|
||||
jest.resetAllMocks();
|
||||
});
|
||||
|
||||
test('returns early when no objects exist', async () => {
|
||||
const readStream = new Readable({
|
||||
read() {
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
const result = await importSavedObjects({
|
||||
readStream,
|
||||
objectLimit: 1,
|
||||
overwrite: false,
|
||||
savedObjectsClient,
|
||||
});
|
||||
expect(result).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"success": true,
|
||||
"successCount": 0,
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
test('calls bulkCreate without overwrite', async () => {
|
||||
|
@ -98,25 +120,33 @@ Object {
|
|||
Array [
|
||||
Array [
|
||||
Object {
|
||||
"attributes": Object {},
|
||||
"attributes": Object {
|
||||
"title": "My Index Pattern",
|
||||
},
|
||||
"id": "1",
|
||||
"references": Array [],
|
||||
"type": "index-pattern",
|
||||
},
|
||||
Object {
|
||||
"attributes": Object {},
|
||||
"attributes": Object {
|
||||
"title": "My Search",
|
||||
},
|
||||
"id": "2",
|
||||
"references": Array [],
|
||||
"type": "search",
|
||||
},
|
||||
Object {
|
||||
"attributes": Object {},
|
||||
"attributes": Object {
|
||||
"title": "My Visualization",
|
||||
},
|
||||
"id": "3",
|
||||
"references": Array [],
|
||||
"type": "visualization",
|
||||
},
|
||||
Object {
|
||||
"attributes": Object {},
|
||||
"attributes": Object {
|
||||
"title": "My Dashboard",
|
||||
},
|
||||
"id": "4",
|
||||
"references": Array [],
|
||||
"type": "dashboard",
|
||||
|
@ -166,25 +196,33 @@ Object {
|
|||
Array [
|
||||
Array [
|
||||
Object {
|
||||
"attributes": Object {},
|
||||
"attributes": Object {
|
||||
"title": "My Index Pattern",
|
||||
},
|
||||
"id": "1",
|
||||
"references": Array [],
|
||||
"type": "index-pattern",
|
||||
},
|
||||
Object {
|
||||
"attributes": Object {},
|
||||
"attributes": Object {
|
||||
"title": "My Search",
|
||||
},
|
||||
"id": "2",
|
||||
"references": Array [],
|
||||
"type": "search",
|
||||
},
|
||||
Object {
|
||||
"attributes": Object {},
|
||||
"attributes": Object {
|
||||
"title": "My Visualization",
|
||||
},
|
||||
"id": "3",
|
||||
"references": Array [],
|
||||
"type": "visualization",
|
||||
},
|
||||
Object {
|
||||
"attributes": Object {},
|
||||
"attributes": Object {
|
||||
"title": "My Dashboard",
|
||||
},
|
||||
"id": "4",
|
||||
"references": Array [],
|
||||
"type": "dashboard",
|
||||
|
@ -205,7 +243,7 @@ Object {
|
|||
`);
|
||||
});
|
||||
|
||||
test('extracts errors', async () => {
|
||||
test('extracts errors for conflicts', async () => {
|
||||
const readStream = new Readable({
|
||||
read() {
|
||||
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
|
||||
|
@ -237,6 +275,7 @@ Object {
|
|||
"type": "conflict",
|
||||
},
|
||||
"id": "1",
|
||||
"title": "My Index Pattern",
|
||||
"type": "index-pattern",
|
||||
},
|
||||
Object {
|
||||
|
@ -244,6 +283,7 @@ Object {
|
|||
"type": "conflict",
|
||||
},
|
||||
"id": "2",
|
||||
"title": "My Search",
|
||||
"type": "search",
|
||||
},
|
||||
Object {
|
||||
|
@ -251,6 +291,7 @@ Object {
|
|||
"type": "conflict",
|
||||
},
|
||||
"id": "3",
|
||||
"title": "My Visualization",
|
||||
"type": "visualization",
|
||||
},
|
||||
Object {
|
||||
|
@ -258,12 +299,122 @@ Object {
|
|||
"type": "conflict",
|
||||
},
|
||||
"id": "4",
|
||||
"title": "My Dashboard",
|
||||
"type": "dashboard",
|
||||
},
|
||||
],
|
||||
"success": false,
|
||||
"successCount": 0,
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
test('validates references', async () => {
|
||||
const readStream = new Readable({
|
||||
read() {
|
||||
this.push(
|
||||
JSON.stringify({
|
||||
id: '1',
|
||||
type: 'search',
|
||||
attributes: {
|
||||
title: 'My Search',
|
||||
},
|
||||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
type: 'index-pattern',
|
||||
id: '2',
|
||||
},
|
||||
],
|
||||
}) + '\n'
|
||||
);
|
||||
this.push(
|
||||
JSON.stringify({
|
||||
id: '3',
|
||||
type: 'visualization',
|
||||
attributes: {
|
||||
title: 'My Visualization',
|
||||
},
|
||||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
type: 'search',
|
||||
id: '1',
|
||||
},
|
||||
],
|
||||
}) + '\n'
|
||||
);
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
savedObjectsClient.bulkGet.mockResolvedValueOnce({
|
||||
saved_objects: [
|
||||
{
|
||||
type: 'index-pattern',
|
||||
id: '2',
|
||||
error: {
|
||||
statusCode: 404,
|
||||
message: 'Not found',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
const result = await importSavedObjects({
|
||||
readStream,
|
||||
objectLimit: 4,
|
||||
overwrite: false,
|
||||
savedObjectsClient,
|
||||
});
|
||||
expect(result).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"errors": Array [
|
||||
Object {
|
||||
"error": Object {
|
||||
"blocking": Array [
|
||||
Object {
|
||||
"id": "3",
|
||||
"type": "visualization",
|
||||
},
|
||||
],
|
||||
"references": Array [
|
||||
Object {
|
||||
"id": "2",
|
||||
"type": "index-pattern",
|
||||
},
|
||||
],
|
||||
"type": "missing_references",
|
||||
},
|
||||
"id": "1",
|
||||
"title": "My Search",
|
||||
"type": "search",
|
||||
},
|
||||
],
|
||||
"success": false,
|
||||
"successCount": 0,
|
||||
}
|
||||
`);
|
||||
expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(`
|
||||
[MockFunction] {
|
||||
"calls": Array [
|
||||
Array [
|
||||
Array [
|
||||
Object {
|
||||
"fields": Array [
|
||||
"id",
|
||||
],
|
||||
"id": "2",
|
||||
"type": "index-pattern",
|
||||
},
|
||||
],
|
||||
],
|
||||
],
|
||||
"results": Array [
|
||||
Object {
|
||||
"type": "return",
|
||||
"value": Promise {},
|
||||
},
|
||||
],
|
||||
}
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
|
|
@ -43,13 +43,14 @@ export async function importSavedObjects({
|
|||
overwrite,
|
||||
savedObjectsClient,
|
||||
}: ImportSavedObjectsOptions): Promise<ImportResponse> {
|
||||
// Get the objects to import
|
||||
const objectsFromStream = await collectSavedObjects(readStream, objectLimit);
|
||||
|
||||
// Validate references
|
||||
const { filteredObjects, errors: validationErrors } = await validateReferences(
|
||||
objectsFromStream,
|
||||
savedObjectsClient
|
||||
);
|
||||
|
||||
// Exit early if no objects to import
|
||||
if (filteredObjects.length === 0) {
|
||||
return {
|
||||
success: validationErrors.length === 0,
|
||||
|
@ -57,15 +58,18 @@ export async function importSavedObjects({
|
|||
...(validationErrors.length ? { errors: validationErrors } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
// Create objects in bulk
|
||||
const bulkCreateResult = await savedObjectsClient.bulkCreate(filteredObjects, {
|
||||
overwrite,
|
||||
});
|
||||
const errors = [...validationErrors, ...extractErrors(bulkCreateResult.saved_objects)];
|
||||
const errors = [
|
||||
...validationErrors,
|
||||
...extractErrors(bulkCreateResult.saved_objects, filteredObjects),
|
||||
];
|
||||
|
||||
return {
|
||||
success: errors.length === 0,
|
||||
successCount: objectsFromStream.length - errors.length,
|
||||
successCount: bulkCreateResult.saved_objects.filter(obj => !obj.error).length,
|
||||
...(errors.length ? { errors } : {}),
|
||||
};
|
||||
}
|
||||
|
|
|
@ -26,25 +26,33 @@ describe('resolveImportErrors()', () => {
|
|||
{
|
||||
id: '1',
|
||||
type: 'index-pattern',
|
||||
attributes: {},
|
||||
attributes: {
|
||||
title: 'My Index Pattern',
|
||||
},
|
||||
references: [],
|
||||
},
|
||||
{
|
||||
id: '2',
|
||||
type: 'search',
|
||||
attributes: {},
|
||||
attributes: {
|
||||
title: 'My Search',
|
||||
},
|
||||
references: [],
|
||||
},
|
||||
{
|
||||
id: '3',
|
||||
type: 'visualization',
|
||||
attributes: {},
|
||||
attributes: {
|
||||
title: 'My Visualization',
|
||||
},
|
||||
references: [],
|
||||
},
|
||||
{
|
||||
id: '4',
|
||||
type: 'dashboard',
|
||||
attributes: {},
|
||||
attributes: {
|
||||
title: 'My Dashboard',
|
||||
},
|
||||
references: [
|
||||
{
|
||||
name: 'panel_0',
|
||||
|
@ -66,13 +74,7 @@ describe('resolveImportErrors()', () => {
|
|||
};
|
||||
|
||||
beforeEach(() => {
|
||||
savedObjectsClient.bulkCreate.mockReset();
|
||||
savedObjectsClient.bulkGet.mockReset();
|
||||
savedObjectsClient.create.mockReset();
|
||||
savedObjectsClient.delete.mockReset();
|
||||
savedObjectsClient.find.mockReset();
|
||||
savedObjectsClient.get.mockReset();
|
||||
savedObjectsClient.update.mockReset();
|
||||
jest.resetAllMocks();
|
||||
});
|
||||
|
||||
test('works with empty parameters', async () => {
|
||||
|
@ -83,15 +85,13 @@ describe('resolveImportErrors()', () => {
|
|||
},
|
||||
});
|
||||
savedObjectsClient.bulkCreate.mockResolvedValue({
|
||||
saved_objects: savedObjects,
|
||||
saved_objects: [],
|
||||
});
|
||||
const result = await resolveImportErrors({
|
||||
readStream,
|
||||
objectLimit: 4,
|
||||
skips: [],
|
||||
overwrites: [],
|
||||
retries: [],
|
||||
savedObjectsClient,
|
||||
replaceReferences: [],
|
||||
});
|
||||
expect(result).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
|
@ -102,66 +102,28 @@ Object {
|
|||
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`[MockFunction]`);
|
||||
});
|
||||
|
||||
test('works with skips', async () => {
|
||||
test('works with retries', async () => {
|
||||
const readStream = new Readable({
|
||||
read() {
|
||||
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
savedObjectsClient.bulkCreate.mockResolvedValue({
|
||||
saved_objects: savedObjects,
|
||||
savedObjectsClient.bulkCreate.mockResolvedValueOnce({
|
||||
saved_objects: savedObjects.filter(obj => obj.type === 'visualization' && obj.id === '3'),
|
||||
});
|
||||
const result = await resolveImportErrors({
|
||||
readStream,
|
||||
objectLimit: 4,
|
||||
skips: [
|
||||
{
|
||||
type: 'dashboard',
|
||||
id: '4',
|
||||
},
|
||||
],
|
||||
overwrites: [],
|
||||
savedObjectsClient,
|
||||
replaceReferences: [
|
||||
retries: [
|
||||
{
|
||||
type: 'visualization',
|
||||
from: '3',
|
||||
to: '30',
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"success": true,
|
||||
"successCount": 0,
|
||||
}
|
||||
`);
|
||||
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`[MockFunction]`);
|
||||
});
|
||||
|
||||
test('works with overwrites', async () => {
|
||||
const readStream = new Readable({
|
||||
read() {
|
||||
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
savedObjectsClient.bulkCreate.mockResolvedValue({
|
||||
saved_objects: savedObjects,
|
||||
});
|
||||
const result = await resolveImportErrors({
|
||||
readStream,
|
||||
objectLimit: 4,
|
||||
skips: [],
|
||||
overwrites: [
|
||||
{
|
||||
type: 'index-pattern',
|
||||
id: '1',
|
||||
id: '3',
|
||||
replaceReferences: [],
|
||||
overwrite: false,
|
||||
},
|
||||
],
|
||||
savedObjectsClient,
|
||||
replaceReferences: [],
|
||||
});
|
||||
expect(result).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
|
@ -175,7 +137,64 @@ Object {
|
|||
Array [
|
||||
Array [
|
||||
Object {
|
||||
"attributes": Object {},
|
||||
"attributes": Object {
|
||||
"title": "My Visualization",
|
||||
},
|
||||
"id": "3",
|
||||
"references": Array [],
|
||||
"type": "visualization",
|
||||
},
|
||||
],
|
||||
],
|
||||
],
|
||||
"results": Array [
|
||||
Object {
|
||||
"type": "return",
|
||||
"value": Promise {},
|
||||
},
|
||||
],
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
test('works with overwrites', async () => {
|
||||
const readStream = new Readable({
|
||||
read() {
|
||||
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
savedObjectsClient.bulkCreate.mockResolvedValue({
|
||||
saved_objects: savedObjects.filter(obj => obj.type === 'index-pattern' && obj.id === '1'),
|
||||
});
|
||||
const result = await resolveImportErrors({
|
||||
readStream,
|
||||
objectLimit: 4,
|
||||
retries: [
|
||||
{
|
||||
type: 'index-pattern',
|
||||
id: '1',
|
||||
overwrite: true,
|
||||
replaceReferences: [],
|
||||
},
|
||||
],
|
||||
savedObjectsClient,
|
||||
});
|
||||
expect(result).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"success": true,
|
||||
"successCount": 1,
|
||||
}
|
||||
`);
|
||||
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`
|
||||
[MockFunction] {
|
||||
"calls": Array [
|
||||
Array [
|
||||
Array [
|
||||
Object {
|
||||
"attributes": Object {
|
||||
"title": "My Index Pattern",
|
||||
},
|
||||
"id": "1",
|
||||
"references": Array [],
|
||||
"type": "index-pattern",
|
||||
|
@ -204,14 +223,16 @@ Object {
|
|||
},
|
||||
});
|
||||
savedObjectsClient.bulkCreate.mockResolvedValue({
|
||||
saved_objects: savedObjects,
|
||||
saved_objects: savedObjects.filter(obj => obj.type === 'dashboard' && obj.id === '4'),
|
||||
});
|
||||
const result = await resolveImportErrors({
|
||||
readStream,
|
||||
objectLimit: 4,
|
||||
skips: [],
|
||||
overwrites: [],
|
||||
savedObjectsClient,
|
||||
retries: [
|
||||
{
|
||||
type: 'dashboard',
|
||||
id: '4',
|
||||
overwrite: false,
|
||||
replaceReferences: [
|
||||
{
|
||||
type: 'visualization',
|
||||
|
@ -219,6 +240,9 @@ Object {
|
|||
to: '13',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
savedObjectsClient,
|
||||
});
|
||||
expect(result).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
|
@ -232,7 +256,9 @@ Object {
|
|||
Array [
|
||||
Array [
|
||||
Object {
|
||||
"attributes": Object {},
|
||||
"attributes": Object {
|
||||
"title": "My Dashboard",
|
||||
},
|
||||
"id": "4",
|
||||
"references": Array [
|
||||
Object {
|
||||
|
@ -244,10 +270,199 @@ Object {
|
|||
"type": "dashboard",
|
||||
},
|
||||
],
|
||||
],
|
||||
],
|
||||
"results": Array [
|
||||
Object {
|
||||
"overwrite": true,
|
||||
"type": "return",
|
||||
"value": Promise {},
|
||||
},
|
||||
],
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
test('extracts errors for conflicts', async () => {
|
||||
const readStream = new Readable({
|
||||
read() {
|
||||
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
savedObjectsClient.bulkCreate.mockResolvedValue({
|
||||
saved_objects: savedObjects.map(savedObject => ({
|
||||
type: savedObject.type,
|
||||
id: savedObject.id,
|
||||
error: {
|
||||
statusCode: 409,
|
||||
message: 'conflict',
|
||||
},
|
||||
})),
|
||||
});
|
||||
const result = await resolveImportErrors({
|
||||
readStream,
|
||||
objectLimit: 4,
|
||||
retries: savedObjects.map(obj => ({
|
||||
type: obj.type,
|
||||
id: obj.id,
|
||||
overwrite: false,
|
||||
replaceReferences: [],
|
||||
})),
|
||||
savedObjectsClient,
|
||||
});
|
||||
expect(result).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"errors": Array [
|
||||
Object {
|
||||
"error": Object {
|
||||
"type": "conflict",
|
||||
},
|
||||
"id": "1",
|
||||
"title": "My Index Pattern",
|
||||
"type": "index-pattern",
|
||||
},
|
||||
Object {
|
||||
"error": Object {
|
||||
"type": "conflict",
|
||||
},
|
||||
"id": "2",
|
||||
"title": "My Search",
|
||||
"type": "search",
|
||||
},
|
||||
Object {
|
||||
"error": Object {
|
||||
"type": "conflict",
|
||||
},
|
||||
"id": "3",
|
||||
"title": "My Visualization",
|
||||
"type": "visualization",
|
||||
},
|
||||
Object {
|
||||
"error": Object {
|
||||
"type": "conflict",
|
||||
},
|
||||
"id": "4",
|
||||
"title": "My Dashboard",
|
||||
"type": "dashboard",
|
||||
},
|
||||
],
|
||||
"success": false,
|
||||
"successCount": 0,
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
test('validates references', async () => {
|
||||
const readStream = new Readable({
|
||||
read() {
|
||||
this.push(
|
||||
JSON.stringify({
|
||||
id: '1',
|
||||
type: 'search',
|
||||
attributes: {
|
||||
title: 'My Search',
|
||||
},
|
||||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
type: 'index-pattern',
|
||||
id: '2',
|
||||
},
|
||||
],
|
||||
}) + '\n'
|
||||
);
|
||||
this.push(
|
||||
JSON.stringify({
|
||||
id: '3',
|
||||
type: 'visualization',
|
||||
attributes: {
|
||||
title: 'My Visualization',
|
||||
},
|
||||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
type: 'search',
|
||||
id: '1',
|
||||
},
|
||||
],
|
||||
}) + '\n'
|
||||
);
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
savedObjectsClient.bulkGet.mockResolvedValueOnce({
|
||||
saved_objects: [
|
||||
{
|
||||
type: 'index-pattern',
|
||||
id: '2',
|
||||
error: {
|
||||
statusCode: 404,
|
||||
message: 'Not found',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
const result = await resolveImportErrors({
|
||||
readStream,
|
||||
objectLimit: 2,
|
||||
retries: [
|
||||
{
|
||||
type: 'search',
|
||||
id: '1',
|
||||
overwrite: false,
|
||||
replaceReferences: [],
|
||||
},
|
||||
{
|
||||
type: 'visualization',
|
||||
id: '3',
|
||||
overwrite: false,
|
||||
replaceReferences: [],
|
||||
},
|
||||
],
|
||||
savedObjectsClient,
|
||||
});
|
||||
expect(result).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"errors": Array [
|
||||
Object {
|
||||
"error": Object {
|
||||
"blocking": Array [
|
||||
Object {
|
||||
"id": "3",
|
||||
"type": "visualization",
|
||||
},
|
||||
],
|
||||
"references": Array [
|
||||
Object {
|
||||
"id": "2",
|
||||
"type": "index-pattern",
|
||||
},
|
||||
],
|
||||
"type": "missing_references",
|
||||
},
|
||||
"id": "1",
|
||||
"title": "My Search",
|
||||
"type": "search",
|
||||
},
|
||||
],
|
||||
"success": false,
|
||||
"successCount": 0,
|
||||
}
|
||||
`);
|
||||
expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(`
|
||||
[MockFunction] {
|
||||
"calls": Array [
|
||||
Array [
|
||||
Array [
|
||||
Object {
|
||||
"fields": Array [
|
||||
"id",
|
||||
],
|
||||
"id": "2",
|
||||
"type": "index-pattern",
|
||||
},
|
||||
],
|
||||
],
|
||||
],
|
||||
"results": Array [
|
||||
Object {
|
||||
|
|
|
@ -22,25 +22,15 @@ import { SavedObjectsClient } from '../service';
|
|||
import { collectSavedObjects } from './collect_saved_objects';
|
||||
import { createObjectsFilter } from './create_objects_filter';
|
||||
import { extractErrors } from './extract_errors';
|
||||
import { ImportError } from './types';
|
||||
import { splitOverwrites } from './split_overwrites';
|
||||
import { ImportError, Retry } from './types';
|
||||
import { validateReferences } from './validate_references';
|
||||
|
||||
interface ResolveImportErrorsOptions {
|
||||
readStream: Readable;
|
||||
objectLimit: number;
|
||||
savedObjectsClient: SavedObjectsClient;
|
||||
overwrites: Array<{
|
||||
type: string;
|
||||
id: string;
|
||||
}>;
|
||||
replaceReferences: Array<{
|
||||
type: string;
|
||||
from: string;
|
||||
to: string;
|
||||
}>;
|
||||
skips: Array<{
|
||||
type: string;
|
||||
id: string;
|
||||
}>;
|
||||
retries: Retry[];
|
||||
}
|
||||
|
||||
interface ImportResponse {
|
||||
|
@ -52,38 +42,65 @@ interface ImportResponse {
|
|||
export async function resolveImportErrors({
|
||||
readStream,
|
||||
objectLimit,
|
||||
skips,
|
||||
overwrites,
|
||||
retries,
|
||||
savedObjectsClient,
|
||||
replaceReferences,
|
||||
}: ResolveImportErrorsOptions): Promise<ImportResponse> {
|
||||
let successCount = 0;
|
||||
let errors: ImportError[] = [];
|
||||
const filter = createObjectsFilter(skips, overwrites, replaceReferences);
|
||||
const filter = createObjectsFilter(retries);
|
||||
|
||||
// Get the objects to resolve errors
|
||||
const objectsToResolve = await collectSavedObjects(readStream, objectLimit, filter);
|
||||
|
||||
// Replace references
|
||||
const refReplacementsMap: Record<string, string> = {};
|
||||
for (const { type, to, from } of replaceReferences) {
|
||||
refReplacementsMap[`${type}:${from}`] = to;
|
||||
// Create a map of references to replace for each object to avoid iterating through
|
||||
// retries for every object to resolve
|
||||
const retriesReferencesMap = new Map<string, { [key: string]: string }>();
|
||||
for (const retry of retries) {
|
||||
const map: { [key: string]: string } = {};
|
||||
for (const { type, from, to } of retry.replaceReferences) {
|
||||
map[`${type}:${from}`] = to;
|
||||
}
|
||||
retriesReferencesMap.set(`${retry.type}:${retry.id}`, map);
|
||||
}
|
||||
|
||||
// Replace references
|
||||
for (const savedObject of objectsToResolve) {
|
||||
const refMap = retriesReferencesMap.get(`${savedObject.type}:${savedObject.id}`);
|
||||
if (!refMap) {
|
||||
continue;
|
||||
}
|
||||
for (const reference of savedObject.references || []) {
|
||||
if (refReplacementsMap[`${reference.type}:${reference.id}`]) {
|
||||
reference.id = refReplacementsMap[`${reference.type}:${reference.id}`];
|
||||
if (refMap[`${reference.type}:${reference.id}`]) {
|
||||
reference.id = refMap[`${reference.type}:${reference.id}`];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (objectsToResolve.length) {
|
||||
const bulkCreateResult = await savedObjectsClient.bulkCreate(objectsToResolve, {
|
||||
// Validate references
|
||||
const { filteredObjects, errors: validationErrors } = await validateReferences(
|
||||
objectsToResolve,
|
||||
savedObjectsClient
|
||||
);
|
||||
errors = errors.concat(validationErrors);
|
||||
|
||||
// Bulk create in two batches, overwrites and non-overwrites
|
||||
const { objectsToOverwrite, objectsToNotOverwrite } = splitOverwrites(filteredObjects, retries);
|
||||
if (objectsToOverwrite.length) {
|
||||
const bulkCreateResult = await savedObjectsClient.bulkCreate(objectsToOverwrite, {
|
||||
overwrite: true,
|
||||
});
|
||||
errors = extractErrors(bulkCreateResult.saved_objects);
|
||||
errors = errors.concat(extractErrors(bulkCreateResult.saved_objects, objectsToOverwrite));
|
||||
successCount += bulkCreateResult.saved_objects.filter(obj => !obj.error).length;
|
||||
}
|
||||
if (objectsToNotOverwrite.length) {
|
||||
const bulkCreateResult = await savedObjectsClient.bulkCreate(objectsToNotOverwrite);
|
||||
errors = errors.concat(extractErrors(bulkCreateResult.saved_objects, objectsToNotOverwrite));
|
||||
successCount += bulkCreateResult.saved_objects.filter(obj => !obj.error).length;
|
||||
}
|
||||
|
||||
return {
|
||||
successCount,
|
||||
success: errors.length === 0,
|
||||
successCount: objectsToResolve.length - errors.length,
|
||||
...(errors.length ? { errors } : {}),
|
||||
};
|
||||
}
|
||||
|
|
|
@ -0,0 +1,92 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import { splitOverwrites } from './split_overwrites';
|
||||
|
||||
describe('splitOverwrites()', () => {
|
||||
test('should split array accordingly', () => {
|
||||
const retries = [
|
||||
{
|
||||
type: 'a',
|
||||
id: '1',
|
||||
overwrite: true,
|
||||
replaceReferences: [],
|
||||
},
|
||||
{
|
||||
id: '2',
|
||||
type: 'b',
|
||||
overwrite: false,
|
||||
replaceReferences: [],
|
||||
},
|
||||
{
|
||||
type: 'c',
|
||||
id: '3',
|
||||
overwrite: true,
|
||||
replaceReferences: [],
|
||||
},
|
||||
];
|
||||
const savedObjects = [
|
||||
{
|
||||
id: '1',
|
||||
type: 'a',
|
||||
attributes: {},
|
||||
references: [],
|
||||
},
|
||||
{
|
||||
id: '2',
|
||||
type: 'b',
|
||||
attributes: {},
|
||||
references: [],
|
||||
},
|
||||
{
|
||||
id: '3',
|
||||
type: 'c',
|
||||
attributes: {},
|
||||
references: [],
|
||||
},
|
||||
];
|
||||
const result = splitOverwrites(savedObjects, retries);
|
||||
expect(result).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"objectsToNotOverwrite": Array [
|
||||
Object {
|
||||
"attributes": Object {},
|
||||
"id": "2",
|
||||
"references": Array [],
|
||||
"type": "b",
|
||||
},
|
||||
],
|
||||
"objectsToOverwrite": Array [
|
||||
Object {
|
||||
"attributes": Object {},
|
||||
"id": "1",
|
||||
"references": Array [],
|
||||
"type": "a",
|
||||
},
|
||||
Object {
|
||||
"attributes": Object {},
|
||||
"id": "3",
|
||||
"references": Array [],
|
||||
"type": "c",
|
||||
},
|
||||
],
|
||||
}
|
||||
`);
|
||||
});
|
||||
});
|
39
src/legacy/server/saved_objects/import/split_overwrites.ts
Normal file
39
src/legacy/server/saved_objects/import/split_overwrites.ts
Normal file
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import { SavedObject } from '../service';
|
||||
import { Retry } from './types';
|
||||
|
||||
export function splitOverwrites(savedObjects: SavedObject[], retries: Retry[]) {
|
||||
const objectsToOverwrite: SavedObject[] = [];
|
||||
const objectsToNotOverwrite: SavedObject[] = [];
|
||||
const overwrites = retries
|
||||
.filter(retry => retry.overwrite)
|
||||
.map(retry => `${retry.type}:${retry.id}`);
|
||||
|
||||
for (const savedObject of savedObjects) {
|
||||
if (overwrites.includes(`${savedObject.type}:${savedObject.id}`)) {
|
||||
objectsToOverwrite.push(savedObject);
|
||||
} else {
|
||||
objectsToNotOverwrite.push(savedObject);
|
||||
}
|
||||
}
|
||||
|
||||
return { objectsToOverwrite, objectsToNotOverwrite };
|
||||
}
|
|
@ -17,26 +17,42 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
interface ConflictError {
|
||||
export interface Retry {
|
||||
type: string;
|
||||
id: string;
|
||||
overwrite: boolean;
|
||||
replaceReferences: Array<{
|
||||
type: string;
|
||||
from: string;
|
||||
to: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface ConflictError {
|
||||
type: 'conflict';
|
||||
}
|
||||
|
||||
interface UnknownError {
|
||||
export interface UnknownError {
|
||||
type: 'unknown';
|
||||
message: string;
|
||||
statusCode: number;
|
||||
}
|
||||
|
||||
interface MissingReferencesError {
|
||||
export interface MissingReferencesError {
|
||||
type: 'missing_references';
|
||||
references: Array<{
|
||||
type: string;
|
||||
id: string;
|
||||
}>;
|
||||
blocking: Array<{
|
||||
type: string;
|
||||
id: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface ImportError {
|
||||
id: string;
|
||||
type: string;
|
||||
title?: string;
|
||||
error: ConflictError | MissingReferencesError | UnknownError;
|
||||
}
|
||||
|
|
|
@ -299,7 +299,9 @@ Object {
|
|||
{
|
||||
id: '2',
|
||||
type: 'visualization',
|
||||
attributes: {},
|
||||
attributes: {
|
||||
title: 'My Visualization 2',
|
||||
},
|
||||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
|
@ -311,7 +313,9 @@ Object {
|
|||
{
|
||||
id: '4',
|
||||
type: 'visualization',
|
||||
attributes: {},
|
||||
attributes: {
|
||||
title: 'My Visualization 4',
|
||||
},
|
||||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
|
@ -342,6 +346,7 @@ Object {
|
|||
"errors": Array [
|
||||
Object {
|
||||
"error": Object {
|
||||
"blocking": Array [],
|
||||
"references": Array [
|
||||
Object {
|
||||
"id": "3",
|
||||
|
@ -351,10 +356,12 @@ Object {
|
|||
"type": "missing_references",
|
||||
},
|
||||
"id": "2",
|
||||
"title": "My Visualization 2",
|
||||
"type": "visualization",
|
||||
},
|
||||
Object {
|
||||
"error": Object {
|
||||
"blocking": Array [],
|
||||
"references": Array [
|
||||
Object {
|
||||
"id": "5",
|
||||
|
@ -372,6 +379,7 @@ Object {
|
|||
"type": "missing_references",
|
||||
},
|
||||
"id": "4",
|
||||
"title": "My Visualization 4",
|
||||
"type": "visualization",
|
||||
},
|
||||
],
|
||||
|
@ -583,4 +591,36 @@ Object {
|
|||
`);
|
||||
expect(savedObjectsClient.bulkGet).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
|
||||
test('throws when bulkGet fails', async () => {
|
||||
savedObjectsClient.bulkGet.mockResolvedValue({
|
||||
saved_objects: [
|
||||
{
|
||||
id: '1',
|
||||
type: 'index-pattern',
|
||||
error: {
|
||||
statusCode: 400,
|
||||
message: 'Error',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
const savedObjects = [
|
||||
{
|
||||
id: '2',
|
||||
type: 'visualization',
|
||||
attributes: {},
|
||||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
type: 'index-pattern',
|
||||
id: '1',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
await expect(
|
||||
validateReferences(savedObjects, savedObjectsClient)
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(`"Bad Request"`);
|
||||
});
|
||||
});
|
||||
|
|
|
@ -17,37 +17,61 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
import Boom from 'boom';
|
||||
import { SavedObject, SavedObjectsClient } from '../service';
|
||||
import { ImportError } from './types';
|
||||
|
||||
const ENFORCED_TYPES = ['index-pattern', 'search'];
|
||||
const REF_TYPES_TO_VLIDATE = ['index-pattern', 'search'];
|
||||
|
||||
function filterReferencesToValidate({ type }: { type: string }) {
|
||||
return REF_TYPES_TO_VLIDATE.includes(type);
|
||||
}
|
||||
|
||||
export async function getNonExistingReferenceAsKeys(
|
||||
savedObjects: SavedObject[],
|
||||
savedObjectsClient: SavedObjectsClient
|
||||
) {
|
||||
const collector = new Map();
|
||||
// Collect all references within objects
|
||||
for (const savedObject of savedObjects) {
|
||||
for (const { type, id } of savedObject.references || []) {
|
||||
if (!ENFORCED_TYPES.includes(type)) {
|
||||
continue;
|
||||
}
|
||||
const filteredReferences = (savedObject.references || []).filter(filterReferencesToValidate);
|
||||
for (const { type, id } of filteredReferences) {
|
||||
collector.set(`${type}:${id}`, { type, id });
|
||||
}
|
||||
}
|
||||
|
||||
// Remove objects that could be references
|
||||
for (const savedObject of savedObjects) {
|
||||
collector.delete(`${savedObject.type}:${savedObject.id}`);
|
||||
}
|
||||
if (collector.size) {
|
||||
if (collector.size === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Fetch references to see if they exist
|
||||
const bulkGetOpts = Array.from(collector.values()).map(obj => ({ ...obj, fields: ['id'] }));
|
||||
const bulkGetResponse = await savedObjectsClient.bulkGet(bulkGetOpts);
|
||||
|
||||
// Error handling
|
||||
const erroredObjects = bulkGetResponse.saved_objects.filter(
|
||||
obj => obj.error && obj.error.statusCode !== 404
|
||||
);
|
||||
if (erroredObjects.length) {
|
||||
const err = Boom.badRequest();
|
||||
err.output.payload.attributes = {
|
||||
objects: erroredObjects,
|
||||
};
|
||||
throw err;
|
||||
}
|
||||
|
||||
// Cleanup collector
|
||||
for (const savedObject of bulkGetResponse.saved_objects) {
|
||||
if (savedObject.error) {
|
||||
continue;
|
||||
}
|
||||
collector.delete(`${savedObject.type}:${savedObject.id}`);
|
||||
}
|
||||
}
|
||||
|
||||
return [...collector.keys()];
|
||||
}
|
||||
|
||||
|
@ -55,39 +79,59 @@ export async function validateReferences(
|
|||
savedObjects: SavedObject[],
|
||||
savedObjectsClient: SavedObjectsClient
|
||||
) {
|
||||
const errors: ImportError[] = [];
|
||||
|
||||
const errorMap: { [key: string]: ImportError } = {};
|
||||
const nonExistingReferenceKeys = await getNonExistingReferenceAsKeys(
|
||||
savedObjects,
|
||||
savedObjectsClient
|
||||
);
|
||||
|
||||
// Filter out objects with missing references, add to error object
|
||||
const filteredObjects = savedObjects.filter(savedObject => {
|
||||
let filteredObjects = savedObjects.filter(savedObject => {
|
||||
const missingReferences = [];
|
||||
for (const { type: refType, id: refId } of savedObject.references || []) {
|
||||
if (!ENFORCED_TYPES.includes(refType)) {
|
||||
continue;
|
||||
}
|
||||
const enforcedTypeReferences = (savedObject.references || []).filter(
|
||||
filterReferencesToValidate
|
||||
);
|
||||
for (const { type: refType, id: refId } of enforcedTypeReferences) {
|
||||
if (nonExistingReferenceKeys.includes(`${refType}:${refId}`)) {
|
||||
missingReferences.push({ type: refType, id: refId });
|
||||
}
|
||||
}
|
||||
if (missingReferences.length) {
|
||||
errors.push({
|
||||
if (missingReferences.length === 0) {
|
||||
return true;
|
||||
}
|
||||
errorMap[`${savedObject.type}:${savedObject.id}`] = {
|
||||
id: savedObject.id,
|
||||
type: savedObject.type,
|
||||
title: savedObject.attributes && savedObject.attributes.title,
|
||||
error: {
|
||||
type: 'missing_references',
|
||||
references: missingReferences,
|
||||
blocking: [],
|
||||
},
|
||||
};
|
||||
return false;
|
||||
});
|
||||
|
||||
// Filter out objects that reference objects within the import but are missing_references
|
||||
// For example: visualization referencing a search that is missing an index pattern needs to be filtered out
|
||||
filteredObjects = filteredObjects.filter(savedObject => {
|
||||
let isBlocked = false;
|
||||
for (const reference of savedObject.references || []) {
|
||||
const referencedObjectError = errorMap[`${reference.type}:${reference.id}`];
|
||||
if (!referencedObjectError || referencedObjectError.error.type !== 'missing_references') {
|
||||
continue;
|
||||
}
|
||||
return missingReferences.length === 0;
|
||||
referencedObjectError.error.blocking.push({
|
||||
type: savedObject.type,
|
||||
id: savedObject.id,
|
||||
});
|
||||
isBlocked = true;
|
||||
}
|
||||
return !isBlocked;
|
||||
});
|
||||
|
||||
return {
|
||||
errors,
|
||||
errors: Object.values(errorMap),
|
||||
filteredObjects,
|
||||
};
|
||||
}
|
||||
|
|
|
@ -36,13 +36,7 @@ describe('POST /api/saved_objects/_import', () => {
|
|||
|
||||
beforeEach(() => {
|
||||
server = createMockServer();
|
||||
savedObjectsClient.bulkCreate.mockReset();
|
||||
savedObjectsClient.bulkGet.mockReset();
|
||||
savedObjectsClient.create.mockReset();
|
||||
savedObjectsClient.delete.mockReset();
|
||||
savedObjectsClient.find.mockReset();
|
||||
savedObjectsClient.get.mockReset();
|
||||
savedObjectsClient.update.mockReset();
|
||||
jest.resetAllMocks();
|
||||
|
||||
const prereqs = {
|
||||
getSavedObjectsClient: {
|
||||
|
@ -180,6 +174,7 @@ describe('POST /api/saved_objects/_import', () => {
|
|||
{
|
||||
id: 'my-pattern',
|
||||
type: 'index-pattern',
|
||||
title: 'my-pattern-*',
|
||||
error: {
|
||||
type: 'conflict',
|
||||
},
|
||||
|
@ -187,4 +182,88 @@ describe('POST /api/saved_objects/_import', () => {
|
|||
],
|
||||
});
|
||||
});
|
||||
|
||||
test('imports a visualization with missing references', async () => {
|
||||
// NOTE: changes to this scenario should be reflected in the docs
|
||||
const request = {
|
||||
method: 'POST',
|
||||
url: '/api/saved_objects/_import',
|
||||
payload: [
|
||||
'--EXAMPLE',
|
||||
'Content-Disposition: form-data; name="file"; filename="export.ndjson"',
|
||||
'Content-Type: application/ndjson',
|
||||
'',
|
||||
'{"type":"visualization","id":"my-vis","attributes":{"title":"my-vis"},"references":[{"name":"ref_0","type":"index-pattern","id":"my-pattern-*"}]}',
|
||||
'{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"},"references":[{"name":"ref_0","type":"visualization","id":"my-vis"}]}',
|
||||
'--EXAMPLE--',
|
||||
].join('\r\n'),
|
||||
headers: {
|
||||
'content-Type': 'multipart/form-data; boundary=EXAMPLE',
|
||||
},
|
||||
};
|
||||
savedObjectsClient.bulkGet.mockResolvedValueOnce({
|
||||
saved_objects: [
|
||||
{
|
||||
id: 'my-pattern-*',
|
||||
type: 'index-pattern',
|
||||
error: {
|
||||
statusCode: 404,
|
||||
message: 'Not found',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
const { payload, statusCode } = await server.inject(request);
|
||||
const response = JSON.parse(payload);
|
||||
expect(statusCode).toBe(200);
|
||||
expect(response).toEqual({
|
||||
success: false,
|
||||
successCount: 0,
|
||||
errors: [
|
||||
{
|
||||
id: 'my-vis',
|
||||
type: 'visualization',
|
||||
title: 'my-vis',
|
||||
error: {
|
||||
type: 'missing_references',
|
||||
references: [
|
||||
{
|
||||
type: 'index-pattern',
|
||||
id: 'my-pattern-*',
|
||||
},
|
||||
],
|
||||
blocking: [
|
||||
{
|
||||
type: 'dashboard',
|
||||
id: 'my-dashboard',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(`
|
||||
[MockFunction] {
|
||||
"calls": Array [
|
||||
Array [
|
||||
Array [
|
||||
Object {
|
||||
"fields": Array [
|
||||
"id",
|
||||
],
|
||||
"id": "my-pattern-*",
|
||||
"type": "index-pattern",
|
||||
},
|
||||
],
|
||||
],
|
||||
],
|
||||
"results": Array [
|
||||
Object {
|
||||
"type": "return",
|
||||
"value": Promise {},
|
||||
},
|
||||
],
|
||||
}
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
|
|
@ -36,13 +36,7 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => {
|
|||
|
||||
beforeEach(() => {
|
||||
server = createMockServer();
|
||||
savedObjectsClient.bulkCreate.mockReset();
|
||||
savedObjectsClient.bulkGet.mockReset();
|
||||
savedObjectsClient.create.mockReset();
|
||||
savedObjectsClient.delete.mockReset();
|
||||
savedObjectsClient.find.mockReset();
|
||||
savedObjectsClient.get.mockReset();
|
||||
savedObjectsClient.update.mockReset();
|
||||
jest.resetAllMocks();
|
||||
|
||||
const prereqs = {
|
||||
getSavedObjectsClient: {
|
||||
|
@ -66,6 +60,10 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => {
|
|||
'Content-Type: application/ndjson',
|
||||
'',
|
||||
'',
|
||||
'--BOUNDARY',
|
||||
'Content-Disposition: form-data; name="retries"',
|
||||
'',
|
||||
'[]',
|
||||
'--BOUNDARY--',
|
||||
].join('\r\n'),
|
||||
headers: {
|
||||
|
@ -79,7 +77,68 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => {
|
|||
expect(savedObjectsClient.bulkCreate).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
|
||||
test('resolves conflicts for an index pattern and dashboard but skips the index pattern', async () => {
|
||||
test('retries importin a dashboard', async () => {
|
||||
// NOTE: changes to this scenario should be reflected in the docs
|
||||
const request = {
|
||||
method: 'POST',
|
||||
url: '/api/saved_objects/_resolve_import_errors',
|
||||
payload: [
|
||||
'--EXAMPLE',
|
||||
'Content-Disposition: form-data; name="file"; filename="export.ndjson"',
|
||||
'Content-Type: application/ndjson',
|
||||
'',
|
||||
'{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"}}',
|
||||
'--EXAMPLE',
|
||||
'Content-Disposition: form-data; name="retries"',
|
||||
'',
|
||||
'[{"type":"dashboard","id":"my-dashboard"}]',
|
||||
'--EXAMPLE--',
|
||||
].join('\r\n'),
|
||||
headers: {
|
||||
'content-Type': 'multipart/form-data; boundary=EXAMPLE',
|
||||
},
|
||||
};
|
||||
savedObjectsClient.bulkCreate.mockResolvedValueOnce({
|
||||
saved_objects: [
|
||||
{
|
||||
type: 'dashboard',
|
||||
id: 'my-dashboard',
|
||||
attributes: {
|
||||
title: 'Look at my dashboard',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
const { payload, statusCode } = await server.inject(request);
|
||||
const response = JSON.parse(payload);
|
||||
expect(statusCode).toBe(200);
|
||||
expect(response).toEqual({ success: true, successCount: 1 });
|
||||
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`
|
||||
[MockFunction] {
|
||||
"calls": Array [
|
||||
Array [
|
||||
Array [
|
||||
Object {
|
||||
"attributes": Object {
|
||||
"title": "Look at my dashboard",
|
||||
},
|
||||
"id": "my-dashboard",
|
||||
"type": "dashboard",
|
||||
},
|
||||
],
|
||||
],
|
||||
],
|
||||
"results": Array [
|
||||
Object {
|
||||
"type": "return",
|
||||
"value": Promise {},
|
||||
},
|
||||
],
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
test('resolves conflicts for dashboard', async () => {
|
||||
// NOTE: changes to this scenario should be reflected in the docs
|
||||
const request = {
|
||||
method: 'POST',
|
||||
|
@ -92,13 +151,9 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => {
|
|||
'{"type":"index-pattern","id":"my-pattern","attributes":{"title":"my-pattern-*"}}',
|
||||
'{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"}}',
|
||||
'--EXAMPLE',
|
||||
'Content-Disposition: form-data; name="skips"',
|
||||
'Content-Disposition: form-data; name="retries"',
|
||||
'',
|
||||
'[{"type":"index-pattern","id":"my-pattern"}]',
|
||||
'--EXAMPLE',
|
||||
'Content-Disposition: form-data; name="overwrites"',
|
||||
'',
|
||||
'[{"type":"dashboard","id":"my-dashboard"}]',
|
||||
'[{"type":"dashboard","id":"my-dashboard","overwrite":true}]',
|
||||
'--EXAMPLE--',
|
||||
].join('\r\n'),
|
||||
headers: {
|
||||
|
@ -158,12 +213,11 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => {
|
|||
'Content-Disposition: form-data; name="file"; filename="export.ndjson"',
|
||||
'Content-Type: application/ndjson',
|
||||
'',
|
||||
'{"type":"visualization","id":"my-vis","attributes":{"title":"Look at my visualization"}}',
|
||||
'{"type":"dashboard","id":"my-dashboard","attributes":{"title":"Look at my dashboard"},"references":[{"name":"panel_0","type":"visualization","id":"my-vis"}]}',
|
||||
'{"type":"visualization","id":"my-vis","attributes":{"title":"Look at my visualization"},"references":[{"name":"ref_0","type":"index-pattern","id":"missing"}]}',
|
||||
'--EXAMPLE',
|
||||
'Content-Disposition: form-data; name="replaceReferences"',
|
||||
'Content-Disposition: form-data; name="retries"',
|
||||
'',
|
||||
'[{"type":"visualization","from":"my-vis","to":"my-vis-2"}]',
|
||||
'[{"type":"visualization","id":"my-vis","replaceReferences":[{"type":"index-pattern","from":"missing","to":"existing"}]}]',
|
||||
'--EXAMPLE--',
|
||||
].join('\r\n'),
|
||||
headers: {
|
||||
|
@ -173,21 +227,31 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => {
|
|||
savedObjectsClient.bulkCreate.mockResolvedValueOnce({
|
||||
saved_objects: [
|
||||
{
|
||||
type: 'dashboard',
|
||||
id: 'my-dashboard',
|
||||
type: 'visualization',
|
||||
id: 'my-vis',
|
||||
attributes: {
|
||||
title: 'Look at my dashboard',
|
||||
title: 'Look at my visualization',
|
||||
},
|
||||
references: [
|
||||
{
|
||||
name: 'panel_0',
|
||||
type: 'visualization',
|
||||
id: 'my-vis-2',
|
||||
name: 'ref_0',
|
||||
type: 'index-pattern',
|
||||
id: 'existing',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
savedObjectsClient.bulkGet.mockResolvedValueOnce({
|
||||
saved_objects: [
|
||||
{
|
||||
id: 'existing',
|
||||
type: 'index-pattern',
|
||||
attributes: {},
|
||||
references: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
const { payload, statusCode } = await server.inject(request);
|
||||
const response = JSON.parse(payload);
|
||||
expect(statusCode).toBe(200);
|
||||
|
@ -199,24 +263,44 @@ describe('POST /api/saved_objects/_resolve_import_errors', () => {
|
|||
Array [
|
||||
Object {
|
||||
"attributes": Object {
|
||||
"title": "Look at my dashboard",
|
||||
"title": "Look at my visualization",
|
||||
},
|
||||
"id": "my-dashboard",
|
||||
"id": "my-vis",
|
||||
"references": Array [
|
||||
Object {
|
||||
"id": "my-vis-2",
|
||||
"name": "panel_0",
|
||||
"id": "existing",
|
||||
"name": "ref_0",
|
||||
"type": "index-pattern",
|
||||
},
|
||||
],
|
||||
"type": "visualization",
|
||||
},
|
||||
],
|
||||
"type": "dashboard",
|
||||
],
|
||||
],
|
||||
"results": Array [
|
||||
Object {
|
||||
"type": "return",
|
||||
"value": Promise {},
|
||||
},
|
||||
],
|
||||
}
|
||||
`);
|
||||
expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(`
|
||||
[MockFunction] {
|
||||
"calls": Array [
|
||||
Array [
|
||||
Array [
|
||||
Object {
|
||||
"overwrite": true,
|
||||
"fields": Array [
|
||||
"id",
|
||||
],
|
||||
"id": "existing",
|
||||
"type": "index-pattern",
|
||||
},
|
||||
],
|
||||
],
|
||||
],
|
||||
"results": Array [
|
||||
Object {
|
||||
"type": "return",
|
||||
|
|
|
@ -38,18 +38,15 @@ interface ImportRequest extends Hapi.Request {
|
|||
};
|
||||
payload: {
|
||||
file: HapiReadableStream;
|
||||
overwrites: Array<{
|
||||
retries: Array<{
|
||||
type: string;
|
||||
id: string;
|
||||
}>;
|
||||
overwrite: boolean;
|
||||
replaceReferences: Array<{
|
||||
type: string;
|
||||
from: string;
|
||||
to: string;
|
||||
}>;
|
||||
skips: Array<{
|
||||
type: string;
|
||||
id: string;
|
||||
}>;
|
||||
};
|
||||
}
|
||||
|
@ -67,14 +64,12 @@ export const createResolveImportErrorsRoute = (prereqs: Prerequisites, server: H
|
|||
validate: {
|
||||
payload: Joi.object({
|
||||
file: Joi.object().required(),
|
||||
overwrites: Joi.array()
|
||||
retries: Joi.array()
|
||||
.items(
|
||||
Joi.object({
|
||||
type: Joi.string().required(),
|
||||
id: Joi.string().required(),
|
||||
})
|
||||
)
|
||||
.default([]),
|
||||
overwrite: Joi.boolean().default(false),
|
||||
replaceReferences: Joi.array()
|
||||
.items(
|
||||
Joi.object({
|
||||
|
@ -84,14 +79,9 @@ export const createResolveImportErrorsRoute = (prereqs: Prerequisites, server: H
|
|||
})
|
||||
)
|
||||
.default([]),
|
||||
skips: Joi.array()
|
||||
.items(
|
||||
Joi.object({
|
||||
type: Joi.string().required(),
|
||||
id: Joi.string().required(),
|
||||
})
|
||||
)
|
||||
.default([]),
|
||||
.required(),
|
||||
}).default(),
|
||||
},
|
||||
},
|
||||
|
@ -99,16 +89,16 @@ export const createResolveImportErrorsRoute = (prereqs: Prerequisites, server: H
|
|||
const { savedObjectsClient } = request.pre;
|
||||
const { filename } = request.payload.file.hapi;
|
||||
const fileExtension = extname(filename).toLowerCase();
|
||||
|
||||
if (fileExtension !== '.ndjson') {
|
||||
return Boom.badRequest(`Invalid file extension ${fileExtension}`);
|
||||
}
|
||||
|
||||
return await resolveImportErrors({
|
||||
savedObjectsClient,
|
||||
readStream: request.payload.file,
|
||||
retries: request.payload.retries,
|
||||
objectLimit: request.server.config().get('savedObjects.maxImportExportSize'),
|
||||
skips: request.payload.skips,
|
||||
overwrites: request.payload.overwrites,
|
||||
replaceReferences: request.payload.replaceReferences,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
|
|
@ -70,6 +70,7 @@ export default function ({ getService }) {
|
|||
{
|
||||
id: '91200a00-9efd-11e7-acb3-3dab96693fab',
|
||||
type: 'index-pattern',
|
||||
title: 'logstash-*',
|
||||
error: {
|
||||
type: 'conflict',
|
||||
}
|
||||
|
@ -77,6 +78,7 @@ export default function ({ getService }) {
|
|||
{
|
||||
id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab',
|
||||
type: 'visualization',
|
||||
title: 'Count of requests',
|
||||
error: {
|
||||
type: 'conflict',
|
||||
}
|
||||
|
@ -84,6 +86,7 @@ export default function ({ getService }) {
|
|||
{
|
||||
id: 'be3733a0-9efe-11e7-acb3-3dab96693fab',
|
||||
type: 'dashboard',
|
||||
title: 'Requests',
|
||||
error: {
|
||||
type: 'conflict',
|
||||
}
|
||||
|
@ -161,6 +164,7 @@ export default function ({ getService }) {
|
|||
id: '1',
|
||||
error: {
|
||||
type: 'missing_references',
|
||||
blocking: [],
|
||||
references: [
|
||||
{
|
||||
type: 'index-pattern',
|
||||
|
|
|
@ -32,6 +32,7 @@ export default function ({ getService }) {
|
|||
it('should return 200 and import nothing when empty parameters are passed in', async () => {
|
||||
await supertest
|
||||
.post('/api/saved_objects/_resolve_import_errors')
|
||||
.field('retries', '[]')
|
||||
.attach('file', join(__dirname, '../../fixtures/import.ndjson'))
|
||||
.expect(200)
|
||||
.then((resp) => {
|
||||
|
@ -45,18 +46,21 @@ export default function ({ getService }) {
|
|||
it('should return 200 and import everything when overwrite parameters contains all objects', async () => {
|
||||
await supertest
|
||||
.post('/api/saved_objects/_resolve_import_errors')
|
||||
.field('overwrites', JSON.stringify([
|
||||
.field('retries', JSON.stringify([
|
||||
{
|
||||
type: 'index-pattern',
|
||||
id: '91200a00-9efd-11e7-acb3-3dab96693fab',
|
||||
overwrite: true,
|
||||
},
|
||||
{
|
||||
type: 'visualization',
|
||||
id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab',
|
||||
overwrite: true,
|
||||
},
|
||||
{
|
||||
type: 'dashboard',
|
||||
id: 'be3733a0-9efe-11e7-acb3-3dab96693fab',
|
||||
overwrite: true,
|
||||
},
|
||||
]))
|
||||
.attach('file', join(__dirname, '../../fixtures/import.ndjson'))
|
||||
|
@ -72,7 +76,7 @@ export default function ({ getService }) {
|
|||
it('should return 400 when no file passed in', async () => {
|
||||
await supertest
|
||||
.post('/api/saved_objects/_resolve_import_errors')
|
||||
.field('skips', '[]')
|
||||
.field('retries', '[]')
|
||||
.expect(400)
|
||||
.then((resp) => {
|
||||
expect(resp.body).to.eql({
|
||||
|
@ -84,6 +88,142 @@ export default function ({ getService }) {
|
|||
});
|
||||
});
|
||||
|
||||
it('should return 400 when resolving conflicts with a file containing more than 10,000 objects', async () => {
|
||||
const fileChunks = [];
|
||||
for (let i = 0; i < 10001; i++) {
|
||||
fileChunks.push(`{"type":"visualization","id":"${i}","attributes":{},"references":[]}`);
|
||||
}
|
||||
await supertest
|
||||
.post('/api/saved_objects/_resolve_import_errors')
|
||||
.field('retries', '[]')
|
||||
.attach('file', Buffer.from(fileChunks.join('\n'), 'utf8'), 'export.ndjson')
|
||||
.expect(400)
|
||||
.then((resp) => {
|
||||
expect(resp.body).to.eql({
|
||||
statusCode: 400,
|
||||
error: 'Bad Request',
|
||||
message: 'Can\'t import more than 10000 objects',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('should return 200 with errors when missing references', async () => {
|
||||
const objToInsert = {
|
||||
id: '1',
|
||||
type: 'visualization',
|
||||
attributes: {
|
||||
title: 'My favorite vis',
|
||||
},
|
||||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
type: 'index-pattern',
|
||||
id: '2',
|
||||
},
|
||||
],
|
||||
};
|
||||
await supertest
|
||||
.post('/api/saved_objects/_resolve_import_errors')
|
||||
.field('retries', JSON.stringify(
|
||||
[
|
||||
{
|
||||
type: 'visualization',
|
||||
id: '1',
|
||||
},
|
||||
]
|
||||
))
|
||||
.attach('file', Buffer.from(JSON.stringify(objToInsert), 'utf8'), 'export.ndjson')
|
||||
.expect(200)
|
||||
.then((resp) => {
|
||||
expect(resp.body).to.eql({
|
||||
success: false,
|
||||
successCount: 0,
|
||||
errors: [
|
||||
{
|
||||
id: '1',
|
||||
type: 'visualization',
|
||||
title: 'My favorite vis',
|
||||
error: {
|
||||
type: 'missing_references',
|
||||
blocking: [],
|
||||
references: [
|
||||
{
|
||||
type: 'index-pattern',
|
||||
id: '2',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('with kibana index', () => {
|
||||
describe('with basic data existing', () => {
|
||||
before(() => esArchiver.load('saved_objects/basic'));
|
||||
after(() => esArchiver.unload('saved_objects/basic'));
|
||||
|
||||
it('should return 200 when skipping all the records', async () => {
|
||||
await supertest
|
||||
.post('/api/saved_objects/_resolve_import_errors')
|
||||
.field('retries', '[]')
|
||||
.attach('file', join(__dirname, '../../fixtures/import.ndjson'))
|
||||
.expect(200)
|
||||
.then((resp) => {
|
||||
expect(resp.body).to.eql({ success: true, successCount: 0 });
|
||||
});
|
||||
});
|
||||
|
||||
it('should return 200 when manually overwriting each object', async () => {
|
||||
await supertest
|
||||
.post('/api/saved_objects/_resolve_import_errors')
|
||||
.field('retries', JSON.stringify(
|
||||
[
|
||||
{
|
||||
id: '91200a00-9efd-11e7-acb3-3dab96693fab',
|
||||
type: 'index-pattern',
|
||||
overwrite: true,
|
||||
},
|
||||
{
|
||||
id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab',
|
||||
type: 'visualization',
|
||||
overwrite: true,
|
||||
},
|
||||
{
|
||||
id: 'be3733a0-9efe-11e7-acb3-3dab96693fab',
|
||||
type: 'dashboard',
|
||||
overwrite: true,
|
||||
},
|
||||
]
|
||||
))
|
||||
.attach('file', join(__dirname, '../../fixtures/import.ndjson'))
|
||||
.expect(200)
|
||||
.then((resp) => {
|
||||
expect(resp.body).to.eql({ success: true, successCount: 3 });
|
||||
});
|
||||
});
|
||||
|
||||
it('should return 200 with only one record when overwriting 1 and skipping 1', async () => {
|
||||
await supertest
|
||||
.post('/api/saved_objects/_resolve_import_errors')
|
||||
.field('retries', JSON.stringify(
|
||||
[
|
||||
{
|
||||
id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab',
|
||||
type: 'visualization',
|
||||
overwrite: true,
|
||||
},
|
||||
]
|
||||
))
|
||||
.attach('file', join(__dirname, '../../fixtures/import.ndjson'))
|
||||
.expect(200)
|
||||
.then((resp) => {
|
||||
expect(resp.body).to.eql({ success: true, successCount: 1 });
|
||||
});
|
||||
});
|
||||
|
||||
it('should return 200 when replacing references', async () => {
|
||||
const objToInsert = {
|
||||
id: '1',
|
||||
|
@ -94,20 +234,26 @@ export default function ({ getService }) {
|
|||
references: [
|
||||
{
|
||||
name: 'ref_0',
|
||||
type: 'search',
|
||||
id: '1',
|
||||
type: 'index-pattern',
|
||||
id: '2',
|
||||
},
|
||||
]
|
||||
};
|
||||
await supertest
|
||||
.post('/api/saved_objects/_resolve_import_errors')
|
||||
.field('replaceReferences', JSON.stringify(
|
||||
.field('retries', JSON.stringify(
|
||||
[
|
||||
{
|
||||
type: 'search',
|
||||
from: '1',
|
||||
to: '2',
|
||||
}
|
||||
type: 'visualization',
|
||||
id: '1',
|
||||
replaceReferences: [
|
||||
{
|
||||
type: 'index-pattern',
|
||||
from: '2',
|
||||
to: '91200a00-9efd-11e7-acb3-3dab96693fab',
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
))
|
||||
.attach('file', Buffer.from(JSON.stringify(objToInsert), 'utf8'), 'export.ndjson')
|
||||
|
@ -125,114 +271,12 @@ export default function ({ getService }) {
|
|||
expect(resp.body.references).to.eql([
|
||||
{
|
||||
name: 'ref_0',
|
||||
type: 'search',
|
||||
id: '2',
|
||||
type: 'index-pattern',
|
||||
id: '91200a00-9efd-11e7-acb3-3dab96693fab',
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
it('should return 400 when resolving conflicts with a file containing more than 10,000 objects', async () => {
|
||||
const fileChunks = [];
|
||||
for (let i = 0; i < 10001; i++) {
|
||||
fileChunks.push(`{"type":"visualization","id":"${i}","attributes":{},"references":[]}`);
|
||||
}
|
||||
await supertest
|
||||
.post('/api/saved_objects/_resolve_import_errors')
|
||||
.attach('file', Buffer.from(fileChunks.join('\n'), 'utf8'), 'export.ndjson')
|
||||
.expect(400)
|
||||
.then((resp) => {
|
||||
expect(resp.body).to.eql({
|
||||
statusCode: 400,
|
||||
error: 'Bad Request',
|
||||
message: 'Can\'t import more than 10000 objects',
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('with kibana index', () => {
|
||||
describe('with basic data existing', () => {
|
||||
before(() => esArchiver.load('saved_objects/basic'));
|
||||
after(() => esArchiver.unload('saved_objects/basic'));
|
||||
|
||||
it('should return 200 when skipping all the records', async () => {
|
||||
await supertest
|
||||
.post('/api/saved_objects/_resolve_import_errors')
|
||||
.field('skips', JSON.stringify(
|
||||
[
|
||||
{
|
||||
id: '91200a00-9efd-11e7-acb3-3dab96693fab',
|
||||
type: 'index-pattern',
|
||||
},
|
||||
{
|
||||
id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab',
|
||||
type: 'visualization',
|
||||
},
|
||||
{
|
||||
id: 'be3733a0-9efe-11e7-acb3-3dab96693fab',
|
||||
type: 'dashboard',
|
||||
},
|
||||
]
|
||||
))
|
||||
.attach('file', join(__dirname, '../../fixtures/import.ndjson'))
|
||||
.expect(200)
|
||||
.then((resp) => {
|
||||
expect(resp.body).to.eql({ success: true, successCount: 0 });
|
||||
});
|
||||
});
|
||||
|
||||
it('should return 200 when manually overwriting each object', async () => {
|
||||
await supertest
|
||||
.post('/api/saved_objects/_resolve_import_errors')
|
||||
.field('overwrites', JSON.stringify(
|
||||
[
|
||||
{
|
||||
id: '91200a00-9efd-11e7-acb3-3dab96693fab',
|
||||
type: 'index-pattern',
|
||||
},
|
||||
{
|
||||
id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab',
|
||||
type: 'visualization',
|
||||
},
|
||||
{
|
||||
id: 'be3733a0-9efe-11e7-acb3-3dab96693fab',
|
||||
type: 'dashboard',
|
||||
},
|
||||
]
|
||||
))
|
||||
.attach('file', join(__dirname, '../../fixtures/import.ndjson'))
|
||||
.expect(200)
|
||||
.then((resp) => {
|
||||
expect(resp.body).to.eql({ success: true, successCount: 3 });
|
||||
});
|
||||
});
|
||||
|
||||
it('should return 200 with only one record when overwriting 1 and skipping 1', async () => {
|
||||
await supertest
|
||||
.post('/api/saved_objects/_resolve_import_errors')
|
||||
.field('overwrites', JSON.stringify(
|
||||
[
|
||||
{
|
||||
id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab',
|
||||
type: 'visualization',
|
||||
},
|
||||
]
|
||||
))
|
||||
.field('skips', JSON.stringify(
|
||||
[
|
||||
{
|
||||
id: '91200a00-9efd-11e7-acb3-3dab96693fab',
|
||||
type: 'index-pattern',
|
||||
},
|
||||
]
|
||||
))
|
||||
.attach('file', join(__dirname, '../../fixtures/import.ndjson'))
|
||||
.expect(200)
|
||||
.then((resp) => {
|
||||
expect(resp.body).to.eql({ success: true, successCount: 1 });
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
@ -61,6 +61,7 @@ export function importTestSuiteFactory(es: any, esArchiver: any, supertest: Supe
|
|||
{
|
||||
id: '1',
|
||||
type: 'wigwags',
|
||||
title: 'Wigwags title',
|
||||
error: {
|
||||
message: `Unsupported saved object type: 'wigwags': Bad Request`,
|
||||
statusCode: 400,
|
||||
|
|
|
@ -65,6 +65,7 @@ export function resolveImportErrorsTestSuiteFactory(
|
|||
{
|
||||
id: '1',
|
||||
type: 'wigwags',
|
||||
title: 'Wigwags title',
|
||||
error: {
|
||||
message: `Unsupported saved object type: 'wigwags': Bad Request`,
|
||||
statusCode: 400,
|
||||
|
@ -116,11 +117,12 @@ export function resolveImportErrorsTestSuiteFactory(
|
|||
.post(`${getUrlPrefix(spaceId)}/api/saved_objects/_resolve_import_errors`)
|
||||
.auth(user.username, user.password)
|
||||
.field(
|
||||
'overwrites',
|
||||
'retries',
|
||||
JSON.stringify([
|
||||
{
|
||||
type: 'dashboard',
|
||||
id: `${getIdPrefix(spaceId)}a01b2f57-fcfd-4864-b735-09e28f0d815e`,
|
||||
overwrite: true,
|
||||
},
|
||||
])
|
||||
)
|
||||
|
@ -147,15 +149,17 @@ export function resolveImportErrorsTestSuiteFactory(
|
|||
.post(`${getUrlPrefix(spaceId)}/api/saved_objects/_resolve_import_errors`)
|
||||
.auth(user.username, user.password)
|
||||
.field(
|
||||
'overwrites',
|
||||
'retries',
|
||||
JSON.stringify([
|
||||
{
|
||||
type: 'wigwags',
|
||||
id: '1',
|
||||
overwrite: true,
|
||||
},
|
||||
{
|
||||
type: 'dashboard',
|
||||
id: `${getIdPrefix(spaceId)}a01b2f57-fcfd-4864-b735-09e28f0d815e`,
|
||||
overwrite: true,
|
||||
},
|
||||
])
|
||||
)
|
||||
|
|
Loading…
Reference in a new issue