maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Split saved_objects/lib into two folders and separate file per function (#32914)

Browse source
This commit is contained in:
Mike Côté 2019-03-11 16:06:34 -04:00 committed by GitHub
parent 2df42f5829
commit d79f3b5771
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 1571 additions and 1298 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

244
src/legacy/server/saved_objects/export/get_sorted_objects_for_export.test.ts Normal file
View file

@ -0,0 +1,244 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { getSortedObjectsForExport } from './get_sorted_objects_for_export';
describe('getSortedObjectsForExport()', () => {
const savedObjectsClient = {
errors: {} as any,
find: jest.fn(),
bulkGet: jest.fn(),
create: jest.fn(),
bulkCreate: jest.fn(),
delete: jest.fn(),
get: jest.fn(),
update: jest.fn(),
};
afterEach(() => {
savedObjectsClient.find.mockReset();
savedObjectsClient.bulkGet.mockReset();
savedObjectsClient.create.mockReset();
savedObjectsClient.bulkCreate.mockReset();
savedObjectsClient.delete.mockReset();
savedObjectsClient.get.mockReset();
savedObjectsClient.update.mockReset();
});
test('exports selected types and sorts them', async () => {
savedObjectsClient.find.mockResolvedValueOnce({
total: 2,
saved_objects: [
{
id: '2',
type: 'search',
references: [
{
type: 'index-pattern',
id: '1',
},
],
},
{
id: '1',
type: 'index-pattern',
references: [],
},
],
});
const response = await getSortedObjectsForExport({
savedObjectsClient,
exportSizeLimit: 500,
types: ['index-pattern', 'search'],
});
expect(response).toMatchInlineSnapshot(`
Array [
Object {
"id": "1",
"references": Array [],
"type": "index-pattern",
},
Object {
"id": "2",
"references": Array [
Object {
"id": "1",
"type": "index-pattern",
},
],
"type": "search",
},
]
`);
expect(savedObjectsClient.find).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Object {
"perPage": 500,
"sortField": "_id",
"sortOrder": "asc",
"type": Array [
"index-pattern",
"search",
],
},
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
test('export selected types throws error when exceeding exportSizeLimit', async () => {
savedObjectsClient.find.mockResolvedValueOnce({
total: 2,
saved_objects: [
{
id: '2',
type: 'search',
references: [
{
type: 'index-pattern',
id: '1',
},
],
},
{
id: '1',
type: 'index-pattern',
references: [],
},
],
});
await expect(
getSortedObjectsForExport({
savedObjectsClient,
exportSizeLimit: 1,
types: ['index-pattern', 'search'],
})
).rejects.toThrowErrorMatchingInlineSnapshot(`"Can't export more than 1 objects"`);
});
test('exports selected objects and sorts them', async () => {
savedObjectsClient.bulkGet.mockResolvedValueOnce({
saved_objects: [
{
id: '2',
type: 'search',
references: [
{
type: 'index-pattern',
id: '1',
},
],
},
{
id: '1',
type: 'index-pattern',
references: [],
},
],
});
const response = await getSortedObjectsForExport({
exportSizeLimit: 10000,
savedObjectsClient,
types: ['index-pattern', 'search'],
objects: [
{
type: 'index-pattern',
id: '1',
},
{
type: 'search',
id: '2',
},
],
});
expect(response).toMatchInlineSnapshot(`
Array [
Object {
"id": "1",
"references": Array [],
"type": "index-pattern",
},
Object {
"id": "2",
"references": Array [
Object {
"id": "1",
"type": "index-pattern",
},
],
"type": "search",
},
]
`);
expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"id": "1",
"type": "index-pattern",
},
Object {
"id": "2",
"type": "search",
},
],
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
test('export selected objects throws error when exceeding exportSizeLimit', async () => {
const exportOpts = {
exportSizeLimit: 1,
savedObjectsClient,
types: ['index-pattern', 'search'],
objects: [
{
type: 'index-pattern',
id: '1',
},
{
type: 'search',
id: '2',
},
],
};
await expect(getSortedObjectsForExport(exportOpts)).rejects.toThrowErrorMatchingInlineSnapshot(
`"Can't export more than 1 objects"`
);
});
});

36
src/legacy/server/saved_objects/lib/export.ts → src/legacy/server/saved_objects/export/get_sorted_objects_for_export.ts
View file

@ -19,6 +19,7 @@
import Boom from 'boom';
import { SavedObject, SavedObjectsClient } from '../service/saved_objects_client';
import { sortObjects } from './sort_objects';
interface ObjectToExport {
id: string;
@ -66,38 +67,3 @@ export async function getSortedObjectsForExport({
}
return sortObjects(objectsToExport);
}
export function sortObjects(savedObjects: SavedObject[]) {
const path = new Set();
const sorted = new Set();
const objectsByTypeId = new Map(
savedObjects.map(object => [`${object.type}:${object.id}`, object] as [string, SavedObject])
);
function includeObjects(objects: SavedObject[]) {
for (const object of objects) {
if (path.has(object)) {
throw Boom.badRequest(
`circular reference: ${[...path, object]
.map(obj => `[${obj.type}:${obj.id}]`)
.join(' ref-> ')}`
);
}
const refdObjects = object.references
.map(ref => objectsByTypeId.get(`${ref.type}:${ref.id}`))
.filter((ref): ref is SavedObject => !!ref);
if (refdObjects.length) {
path.add(object);
includeObjects(refdObjects);
path.delete(object);
}
sorted.add(object);
}
}
includeObjects(savedObjects);
return [...sorted];
}

3
src/legacy/server/saved_objects/lib/index.ts → src/legacy/server/saved_objects/export/index.ts
View file

@ -17,5 +17,4 @@
* under the License.
*/
export { importSavedObjects, resolveImportConflicts } from './import';
export { getSortedObjectsForExport } from './export';
export { getSortedObjectsForExport } from './get_sorted_objects_for_export';

226
src/legacy/server/saved_objects/lib/export.test.ts → src/legacy/server/saved_objects/export/sort_objects.test.ts
View file

@ -17,231 +17,7 @@
* under the License.
*/
import { getSortedObjectsForExport, sortObjects } from './export';
describe('getSortedObjectsForExport()', () => {
const savedObjectsClient = {
errors: {} as any,
find: jest.fn(),
bulkGet: jest.fn(),
create: jest.fn(),
bulkCreate: jest.fn(),
delete: jest.fn(),
get: jest.fn(),
update: jest.fn(),
};
afterEach(() => {
savedObjectsClient.find.mockReset();
savedObjectsClient.bulkGet.mockReset();
savedObjectsClient.create.mockReset();
savedObjectsClient.bulkCreate.mockReset();
savedObjectsClient.delete.mockReset();
savedObjectsClient.get.mockReset();
savedObjectsClient.update.mockReset();
});
test('exports selected types and sorts them', async () => {
savedObjectsClient.find.mockResolvedValueOnce({
total: 2,
saved_objects: [
{
id: '2',
type: 'search',
references: [
{
type: 'index-pattern',
id: '1',
},
],
},
{
id: '1',
type: 'index-pattern',
references: [],
},
],
});
const response = await getSortedObjectsForExport({
savedObjectsClient,
exportSizeLimit: 500,
types: ['index-pattern', 'search'],
});
expect(response).toMatchInlineSnapshot(`
Array [
Object {
"id": "1",
"references": Array [],
"type": "index-pattern",
},
Object {
"id": "2",
"references": Array [
Object {
"id": "1",
"type": "index-pattern",
},
],
"type": "search",
},
]
`);
expect(savedObjectsClient.find).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Object {
"perPage": 500,
"sortField": "_id",
"sortOrder": "asc",
"type": Array [
"index-pattern",
"search",
],
},
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
test('export selected types throws error when exceeding exportSizeLimit', async () => {
savedObjectsClient.find.mockResolvedValueOnce({
total: 2,
saved_objects: [
{
id: '2',
type: 'search',
references: [
{
type: 'index-pattern',
id: '1',
},
],
},
{
id: '1',
type: 'index-pattern',
references: [],
},
],
});
await expect(
getSortedObjectsForExport({
savedObjectsClient,
exportSizeLimit: 1,
types: ['index-pattern', 'search'],
})
).rejects.toThrowErrorMatchingInlineSnapshot(`"Can't export more than 1 objects"`);
});
test('exports selected objects and sorts them', async () => {
savedObjectsClient.bulkGet.mockResolvedValueOnce({
saved_objects: [
{
id: '2',
type: 'search',
references: [
{
type: 'index-pattern',
id: '1',
},
],
},
{
id: '1',
type: 'index-pattern',
references: [],
},
],
});
const response = await getSortedObjectsForExport({
exportSizeLimit: 10000,
savedObjectsClient,
types: ['index-pattern', 'search'],
objects: [
{
type: 'index-pattern',
id: '1',
},
{
type: 'search',
id: '2',
},
],
});
expect(response).toMatchInlineSnapshot(`
Array [
Object {
"id": "1",
"references": Array [],
"type": "index-pattern",
},
Object {
"id": "2",
"references": Array [
Object {
"id": "1",
"type": "index-pattern",
},
],
"type": "search",
},
]
`);
expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"id": "1",
"type": "index-pattern",
},
Object {
"id": "2",
"type": "search",
},
],
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
test('export selected objects throws error when exceeding exportSizeLimit', async () => {
const exportOpts = {
exportSizeLimit: 1,
savedObjectsClient,
types: ['index-pattern', 'search'],
objects: [
{
type: 'index-pattern',
id: '1',
},
{
type: 'search',
id: '2',
},
],
};
await expect(getSortedObjectsForExport(exportOpts)).rejects.toThrowErrorMatchingInlineSnapshot(
`"Can't export more than 1 objects"`
);
});
});
import { sortObjects } from './sort_objects';
describe('sortObjects()', () => {
test('should return on empty array', () => {

56
src/legacy/server/saved_objects/export/sort_objects.ts Normal file
View file

@ -0,0 +1,56 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import Boom from 'boom';
import { SavedObject } from '../service/saved_objects_client';
export function sortObjects(savedObjects: SavedObject[]) {
const path = new Set();
const sorted = new Set();
const objectsByTypeId = new Map(
savedObjects.map(object => [`${object.type}:${object.id}`, object] as [string, SavedObject])
);
function includeObjects(objects: SavedObject[]) {
for (const object of objects) {
if (path.has(object)) {
throw Boom.badRequest(
`circular reference: ${[...path, object]
.map(obj => `[${obj.type}:${obj.id}]`)
.join(' ref-> ')}`
);
}
const refdObjects = object.references
.map(ref => objectsByTypeId.get(`${ref.type}:${ref.id}`))
.filter((ref): ref is SavedObject => !!ref);
if (refdObjects.length) {
path.add(object);
includeObjects(refdObjects);
path.delete(object);
}
sorted.add(object);
}
}
includeObjects(savedObjects);
return [...sorted];
}

80
src/legacy/server/saved_objects/import/collect_saved_objects.test.ts Normal file
View file

@ -0,0 +1,80 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { Readable } from 'stream';
import { collectSavedObjects } from './collect_saved_objects';
describe('collectSavedObjects()', () => {
test('collects nothing when stream is empty', async () => {
const readStream = new Readable({
read() {
this.push(null);
},
});
const objects = await collectSavedObjects(readStream, 10);
expect(objects).toMatchInlineSnapshot(`Array []`);
});
test('collects objects from stream', async () => {
const readStream = new Readable({
read() {
this.push('{"foo":true}');
this.push(null);
},
});
const objects = await collectSavedObjects(readStream, 1);
expect(objects).toMatchInlineSnapshot(`
Array [
Object {
"foo": true,
},
]
`);
});
test('filters out empty lines', async () => {
const readStream = new Readable({
read() {
this.push('{"foo":true}\n\n');
this.push(null);
},
});
const objects = await collectSavedObjects(readStream, 1);
expect(objects).toMatchInlineSnapshot(`
Array [
Object {
"foo": true,
},
]
`);
});
test('throws error when object limit is reached', async () => {
const readStream = new Readable({
read() {
this.push('{"foo":true}\n');
this.push('{"bar":true}\n');
this.push(null);
},
});
await expect(collectSavedObjects(readStream, 1)).rejects.toThrowErrorMatchingInlineSnapshot(
`"Can't import more than 1 objects"`
);
});
});

49
src/legacy/server/saved_objects/import/collect_saved_objects.ts Normal file
View file

@ -0,0 +1,49 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { Readable } from 'stream';
import {
createConcatStream,
createFilterStream,
createMapStream,
createPromiseFromStreams,
createSplitStream,
} from '../../../utils/streams';
import { SavedObject } from '../service';
import { createLimitStream } from './create_limit_stream';
export async function collectSavedObjects(
readStream: Readable,
objectLimit: number,
filter?: (obj: SavedObject) => boolean
): Promise<SavedObject[]> {
return (await createPromiseFromStreams([
readStream,
createSplitStream('\n'),
createMapStream((str: string) => {
if (str && str !== '') {
return JSON.parse(str);
}
}),
createFilterStream<SavedObject>(obj => !!obj),
createLimitStream(objectLimit),
createFilterStream<SavedObject>(obj => (filter ? filter(obj) : true)),
createConcatStream([]),
])) as SavedObject[];
}

53
src/legacy/server/saved_objects/import/create_limit_stream.test.ts Normal file
View file

@ -0,0 +1,53 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import {
createConcatStream,
createListStream,
createPromiseFromStreams,
} from '../../../utils/streams';
import { createLimitStream } from './create_limit_stream';
describe('createLimitStream()', () => {
test('limit of 5 allows 5 items through', async () => {
await createPromiseFromStreams([createListStream([1, 2, 3, 4, 5]), createLimitStream(5)]);
});
test('limit of 5 errors out when 6 items are through', async () => {
await expect(
createPromiseFromStreams([createListStream([1, 2, 3, 4, 5, 6]), createLimitStream(5)])
).rejects.toThrowErrorMatchingInlineSnapshot(`"Can't import more than 5 objects"`);
});
test('send the values on the output stream', async () => {
const result = await createPromiseFromStreams([
createListStream([1, 2, 3]),
createLimitStream(3),
createConcatStream([]),
]);
expect(result).toMatchInlineSnapshot(`
Array [
1,
2,
3,
]
`);
});
});

35
src/legacy/server/saved_objects/import/create_limit_stream.ts Normal file
View file

@ -0,0 +1,35 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import Boom from 'boom';
import { Transform } from 'stream';
export function createLimitStream(limit: number) {
let counter = 0;
return new Transform({
objectMode: true,
async transform(obj, enc, done) {
if (counter >= limit) {
return done(Boom.badRequest(`Can't import more than ${limit} objects`));
}
counter++;
done(undefined, obj);
},
});
}

185
src/legacy/server/saved_objects/import/create_objects_filter.test.ts Normal file
View file

@ -0,0 +1,185 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { createObjectsFilter } from './create_objects_filter';
describe('createObjectsFilter()', () => {
test('filters should return false when contains empty parameters', () => {
const fn = createObjectsFilter([], [], []);
expect(fn({ type: 'a', id: '1', attributes: {}, references: [] })).toEqual(false);
});
test('filters should exclude skips', () => {
const fn = createObjectsFilter(
[
{
type: 'a',
id: '1',
},
],
[],
[
{
type: 'b',
from: '1',
to: '2',
},
]
);
expect(
fn({
type: 'a',
id: '1',
attributes: {},
references: [{ name: 'ref_0', type: 'b', id: '1' }],
})
).toEqual(false);
expect(
fn({
type: 'a',
id: '2',
attributes: {},
references: [{ name: 'ref_0', type: 'b', id: '1' }],
})
).toEqual(true);
});
test('filter should include references to replace', () => {
const fn = createObjectsFilter(
[],
[],
[
{
type: 'b',
from: '1',
to: '2',
},
]
);
expect(
fn({
type: 'a',
id: '1',
attributes: {},
references: [
{
name: 'ref_0',
type: 'b',
id: '1',
},
],
})
).toEqual(true);
expect(
fn({
type: 'a',
id: '1',
attributes: {},
references: [
{
name: 'ref_0',
type: 'b',
id: '2',
},
],
})
).toEqual(false);
});
test('filter should include objects to overwrite', () => {
const fn = createObjectsFilter(
[],
[
{
type: 'a',
id: '1',
},
],
[]
);
expect(fn({ type: 'a', id: '1', attributes: {}, references: [] })).toEqual(true);
expect(fn({ type: 'a', id: '2', attributes: {}, references: [] })).toEqual(false);
});
test('filter should work with skips, overwrites and replaceReferences', () => {
const fn = createObjectsFilter(
[
{
type: 'a',
id: '1',
},
],
[
{
type: 'a',
id: '2',
},
],
[
{
type: 'b',
from: '1',
to: '2',
},
]
);
expect(
fn({
type: 'a',
id: '1',
attributes: {},
references: [
{
name: 'ref_0',
type: 'b',
id: '1',
},
],
})
).toEqual(false);
expect(
fn({
type: 'a',
id: '2',
attributes: {},
references: [
{
name: 'ref_0',
type: 'b',
id: '2',
},
],
})
).toEqual(true);
expect(
fn({
type: 'a',
id: '3',
attributes: {},
references: [
{
name: 'ref_0',
type: 'b',
id: '1',
},
],
})
).toEqual(true);
});
});

54
src/legacy/server/saved_objects/import/create_objects_filter.ts Normal file
View file

@ -0,0 +1,54 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { SavedObject } from '../service';
export function createObjectsFilter(
skips: Array<{
type: string;
id: string;
}>,
overwrites: Array<{
type: string;
id: string;
}>,
replaceReferences: Array<{
type: string;
from: string;
to: string;
}>
) {
const refReplacements = replaceReferences.map(ref => `${ref.type}:${ref.from}`);
return (obj: SavedObject) => {
if (skips.some(skipObj => skipObj.type === obj.type && skipObj.id === obj.id)) {
return false;
}
if (
overwrites.some(overwriteObj => overwriteObj.type === obj.type && overwriteObj.id === obj.id)
) {
return true;
}
for (const reference of obj.references || []) {
if (refReplacements.includes(`${reference.type}:${reference.id}`)) {
return true;
}
}
return false;
};
}

63
src/legacy/server/saved_objects/import/extract_errors.test.ts Normal file
View file

@ -0,0 +1,63 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { SavedObject } from '../service';
import { extractErrors } from './extract_errors';
describe('extractErrors()', () => {
test('returns empty array when no errors exist', () => {
const savedObjects: SavedObject[] = [];
const result = extractErrors(savedObjects);
expect(result).toMatchInlineSnapshot(`Array []`);
});
test('extracts errors from saved objects', () => {
const savedObjects: SavedObject[] = [
{
id: '1',
type: 'dashboard',
attributes: {},
references: [],
},
{
id: '2',
type: 'dashboard',
attributes: {},
references: [],
error: {
statusCode: 409,
message: 'Conflict',
},
},
];
const result = extractErrors(savedObjects);
expect(result).toMatchInlineSnapshot(`
Array [
Object {
"error": Object {
"message": "Conflict",
"statusCode": 409,
},
"id": "2",
"type": "dashboard",
},
]
`);
});
});

43
src/legacy/server/saved_objects/import/extract_errors.ts Normal file
View file

@ -0,0 +1,43 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { SavedObject } from '../service';
export interface CustomError {
id: string;
type: string;
error: {
message: string;
statusCode: number;
};
}
export function extractErrors(savedObjects: SavedObject[]) {
const errors: CustomError[] = [];
for (const savedObject of savedObjects) {
if (savedObject.error) {
errors.push({
id: savedObject.id,
type: savedObject.type,
error: savedObject.error,
});
}
}
return errors;
}

270
src/legacy/server/saved_objects/import/import_saved_objects.test.ts Normal file
View file

@ -0,0 +1,270 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { Readable } from 'stream';
import { SavedObject } from '../service';
import { importSavedObjects } from './import_saved_objects';
describe('importSavedObjects()', () => {
const savedObjects: SavedObject[] = [
{
id: '1',
type: 'index-pattern',
attributes: {},
references: [],
},
{
id: '2',
type: 'search',
attributes: {},
references: [],
},
{
id: '3',
type: 'visualization',
attributes: {},
references: [],
},
{
id: '4',
type: 'dashboard',
attributes: {},
references: [],
},
];
const savedObjectsClient = {
errors: {} as any,
bulkCreate: jest.fn(),
bulkGet: jest.fn(),
create: jest.fn(),
delete: jest.fn(),
find: jest.fn(),
get: jest.fn(),
update: jest.fn(),
};
beforeEach(() => {
savedObjectsClient.bulkCreate.mockReset();
savedObjectsClient.bulkGet.mockReset();
savedObjectsClient.create.mockReset();
savedObjectsClient.delete.mockReset();
savedObjectsClient.find.mockReset();
savedObjectsClient.get.mockReset();
savedObjectsClient.update.mockReset();
});
test('calls bulkCreate without overwrite', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects,
});
const result = await importSavedObjects({
readStream,
objectLimit: 4,
overwrite: false,
savedObjectsClient,
});
expect(result).toMatchInlineSnapshot(`
Object {
"success": true,
"successCount": 4,
}
`);
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"attributes": Object {},
"id": "1",
"references": Array [],
"type": "index-pattern",
},
Object {
"attributes": Object {},
"id": "2",
"references": Array [],
"type": "search",
},
Object {
"attributes": Object {},
"id": "3",
"references": Array [],
"type": "visualization",
},
Object {
"attributes": Object {},
"id": "4",
"references": Array [],
"type": "dashboard",
},
],
Object {
"overwrite": false,
},
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
test('calls bulkCreate with overwrite', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects,
});
const result = await importSavedObjects({
readStream,
objectLimit: 4,
overwrite: true,
savedObjectsClient,
});
expect(result).toMatchInlineSnapshot(`
Object {
"success": true,
"successCount": 4,
}
`);
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"attributes": Object {},
"id": "1",
"references": Array [],
"type": "index-pattern",
},
Object {
"attributes": Object {},
"id": "2",
"references": Array [],
"type": "search",
},
Object {
"attributes": Object {},
"id": "3",
"references": Array [],
"type": "visualization",
},
Object {
"attributes": Object {},
"id": "4",
"references": Array [],
"type": "dashboard",
},
],
Object {
"overwrite": true,
},
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
test('extracts errors', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects.map(savedObject => ({
type: savedObject.type,
id: savedObject.id,
error: {
statusCode: 409,
message: 'conflict',
},
})),
});
const result = await importSavedObjects({
readStream,
objectLimit: 4,
overwrite: false,
savedObjectsClient,
});
expect(result).toMatchInlineSnapshot(`
Object {
"errors": Array [
Object {
"error": Object {
"message": "conflict",
"statusCode": 409,
},
"id": "1",
"type": "index-pattern",
},
Object {
"error": Object {
"message": "conflict",
"statusCode": 409,
},
"id": "2",
"type": "search",
},
Object {
"error": Object {
"message": "conflict",
"statusCode": 409,
},
"id": "3",
"type": "visualization",
},
Object {
"error": Object {
"message": "conflict",
"statusCode": 409,
},
"id": "4",
"type": "dashboard",
},
],
"success": false,
"successCount": 0,
}
`);
});
});

63
src/legacy/server/saved_objects/import/import_saved_objects.ts Normal file
View file

@ -0,0 +1,63 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { Readable } from 'stream';
import { SavedObjectsClient } from '../service';
import { collectSavedObjects } from './collect_saved_objects';
import { CustomError, extractErrors } from './extract_errors';
interface ImportSavedObjectsOptions {
readStream: Readable;
objectLimit: number;
overwrite: boolean;
savedObjectsClient: SavedObjectsClient;
}
interface ImportResponse {
success: boolean;
successCount: number;
errors?: CustomError[];
}
export async function importSavedObjects({
readStream,
objectLimit,
overwrite,
savedObjectsClient,
}: ImportSavedObjectsOptions): Promise<ImportResponse> {
const objectsToImport = await collectSavedObjects(readStream, objectLimit);
if (objectsToImport.length === 0) {
return {
success: true,
successCount: 0,
};
}
const bulkCreateResult = await savedObjectsClient.bulkCreate(objectsToImport, {
overwrite,
});
const errors = extractErrors(bulkCreateResult.saved_objects);
return {
success: errors.length === 0,
successCount: objectsToImport.length - errors.length,
...(errors.length ? { errors } : {}),
};
}

21
src/legacy/server/saved_objects/import/index.ts Normal file
View file

@ -0,0 +1,21 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export { importSavedObjects } from './import_saved_objects';
export { resolveImportConflicts } from './resolve_import_conflicts';

261
src/legacy/server/saved_objects/import/resolve_import_conflicts.test.ts Normal file
View file

@ -0,0 +1,261 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { Readable } from 'stream';
import { SavedObject } from '../service';
import { resolveImportConflicts } from './resolve_import_conflicts';
describe('resolveImportConflicts()', () => {
const savedObjects: SavedObject[] = [
{
id: '1',
type: 'index-pattern',
attributes: {},
references: [],
},
{
id: '2',
type: 'search',
attributes: {},
references: [],
},
{
id: '3',
type: 'visualization',
attributes: {},
references: [],
},
{
id: '4',
type: 'dashboard',
attributes: {},
references: [
{
name: 'panel_0',
type: 'visualization',
id: '3',
},
],
},
];
const savedObjectsClient = {
errors: {} as any,
bulkCreate: jest.fn(),
bulkGet: jest.fn(),
create: jest.fn(),
delete: jest.fn(),
find: jest.fn(),
get: jest.fn(),
update: jest.fn(),
};
beforeEach(() => {
savedObjectsClient.bulkCreate.mockReset();
savedObjectsClient.bulkGet.mockReset();
savedObjectsClient.create.mockReset();
savedObjectsClient.delete.mockReset();
savedObjectsClient.find.mockReset();
savedObjectsClient.get.mockReset();
savedObjectsClient.update.mockReset();
});
test('works with empty parameters', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects,
});
const result = await resolveImportConflicts({
readStream,
objectLimit: 4,
skips: [],
overwrites: [],
savedObjectsClient,
replaceReferences: [],
});
expect(result).toMatchInlineSnapshot(`
Object {
"success": true,
"successCount": 0,
}
`);
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`[MockFunction]`);
});
test('works with skips', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects,
});
const result = await resolveImportConflicts({
readStream,
objectLimit: 4,
skips: [
{
type: 'dashboard',
id: '4',
},
],
overwrites: [],
savedObjectsClient,
replaceReferences: [
{
type: 'visualization',
from: '3',
to: '30',
},
],
});
expect(result).toMatchInlineSnapshot(`
Object {
"success": true,
"successCount": 0,
}
`);
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`[MockFunction]`);
});
test('works with overwrites', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects,
});
const result = await resolveImportConflicts({
readStream,
objectLimit: 4,
skips: [],
overwrites: [
{
type: 'index-pattern',
id: '1',
},
],
savedObjectsClient,
replaceReferences: [],
});
expect(result).toMatchInlineSnapshot(`
Object {
"success": true,
"successCount": 1,
}
`);
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"attributes": Object {},
"id": "1",
"references": Array [],
"type": "index-pattern",
},
],
Object {
"overwrite": true,
},
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
test('works wtih replaceReferences', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects,
});
const result = await resolveImportConflicts({
readStream,
objectLimit: 4,
skips: [],
overwrites: [],
savedObjectsClient,
replaceReferences: [
{
type: 'visualization',
from: '3',
to: '13',
},
],
});
expect(result).toMatchInlineSnapshot(`
Object {
"success": true,
"successCount": 1,
}
`);
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"attributes": Object {},
"id": "4",
"references": Array [
Object {
"id": "13",
"name": "panel_0",
"type": "visualization",
},
],
"type": "dashboard",
},
],
Object {
"overwrite": true,
},
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
});

88
src/legacy/server/saved_objects/import/resolve_import_conflicts.ts Normal file
View file

@ -0,0 +1,88 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { Readable } from 'stream';
import { SavedObjectsClient } from '../service';
import { collectSavedObjects } from './collect_saved_objects';
import { createObjectsFilter } from './create_objects_filter';
import { CustomError, extractErrors } from './extract_errors';
interface ResolveImportConflictsOptions {
readStream: Readable;
objectLimit: number;
savedObjectsClient: SavedObjectsClient;
overwrites: Array<{
type: string;
id: string;
}>;
replaceReferences: Array<{
type: string;
from: string;
to: string;
}>;
skips: Array<{
type: string;
id: string;
}>;
}
interface ImportResponse {
success: boolean;
successCount: number;
errors?: CustomError[];
}
export async function resolveImportConflicts({
readStream,
objectLimit,
skips,
overwrites,
savedObjectsClient,
replaceReferences,
}: ResolveImportConflictsOptions): Promise<ImportResponse> {
let errors: CustomError[] = [];
const filter = createObjectsFilter(skips, overwrites, replaceReferences);
const objectsToResolve = await collectSavedObjects(readStream, objectLimit, filter);
// Replace references
const refReplacementsMap: Record<string, string> = {};
for (const { type, to, from } of replaceReferences) {
refReplacementsMap[`${type}:${from}`] = to;
}
for (const savedObject of objectsToResolve) {
for (const reference of savedObject.references || []) {
if (refReplacementsMap[`${reference.type}:${reference.id}`]) {
reference.id = refReplacementsMap[`${reference.type}:${reference.id}`];
}
}
}
if (objectsToResolve.length) {
const bulkCreateResult = await savedObjectsClient.bulkCreate(objectsToResolve, {
overwrite: true,
});
errors = extractErrors(bulkCreateResult.saved_objects);
}
return {
success: errors.length === 0,
successCount: objectsToResolve.length - errors.length,
...(errors.length ? { errors } : {}),
};
}

815
src/legacy/server/saved_objects/lib/import.test.ts
View file

@ -1,815 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { Readable } from 'stream';
import {
createConcatStream,
createListStream,
createPromiseFromStreams,
} from '../../../utils/streams';
import { SavedObject } from '../service';
import {
collectSavedObjects,
createLimitStream,
createObjectsFilter,
extractErrors,
importSavedObjects,
resolveImportConflicts,
} from './import';
describe('extractErrors()', () => {
test('returns empty array when no errors exist', () => {
const savedObjects: SavedObject[] = [];
const result = extractErrors(savedObjects);
expect(result).toMatchInlineSnapshot(`Array []`);
});
test('extracts errors from saved objects', () => {
const savedObjects: SavedObject[] = [
{
id: '1',
type: 'dashboard',
attributes: {},
references: [],
},
{
id: '2',
type: 'dashboard',
attributes: {},
references: [],
error: {
statusCode: 409,
message: 'Conflict',
},
},
];
const result = extractErrors(savedObjects);
expect(result).toMatchInlineSnapshot(`
Array [
Object {
"error": Object {
"message": "Conflict",
"statusCode": 409,
},
"id": "2",
"type": "dashboard",
},
]
`);
});
});
describe('createLimitStream()', () => {
test('limit of 5 allows 5 items through', async () => {
await createPromiseFromStreams([createListStream([1, 2, 3, 4, 5]), createLimitStream(5)]);
});
test('limit of 5 errors out when 6 items are through', async () => {
await expect(
createPromiseFromStreams([createListStream([1, 2, 3, 4, 5, 6]), createLimitStream(5)])
).rejects.toThrowErrorMatchingInlineSnapshot(`"Can't import more than 5 objects"`);
});
test('send the values on the output stream', async () => {
const result = await createPromiseFromStreams([
createListStream([1, 2, 3]),
createLimitStream(3),
createConcatStream([]),
]);
expect(result).toMatchInlineSnapshot(`
Array [
1,
2,
3,
]
`);
});
});
describe('collectSavedObjects()', () => {
test('collects nothing when stream is empty', async () => {
const readStream = new Readable({
read() {
this.push(null);
},
});
const objects = await collectSavedObjects(readStream, 10);
expect(objects).toMatchInlineSnapshot(`Array []`);
});
test('collects objects from stream', async () => {
const readStream = new Readable({
read() {
this.push('{"foo":true}');
this.push(null);
},
});
const objects = await collectSavedObjects(readStream, 1);
expect(objects).toMatchInlineSnapshot(`
Array [
Object {
"foo": true,
},
]
`);
});
test('filters out empty lines', async () => {
const readStream = new Readable({
read() {
this.push('{"foo":true}\n\n');
this.push(null);
},
});
const objects = await collectSavedObjects(readStream, 1);
expect(objects).toMatchInlineSnapshot(`
Array [
Object {
"foo": true,
},
]
`);
});
test('throws error when object limit is reached', async () => {
const readStream = new Readable({
read() {
this.push('{"foo":true}\n');
this.push('{"bar":true}\n');
this.push(null);
},
});
await expect(collectSavedObjects(readStream, 1)).rejects.toThrowErrorMatchingInlineSnapshot(
`"Can't import more than 1 objects"`
);
});
});
describe('createObjectsFilter()', () => {
test('filters should return false when contains empty parameters', () => {
const fn = createObjectsFilter([], [], []);
expect(fn({ type: 'a', id: '1', attributes: {}, references: [] })).toEqual(false);
});
test('filters should exclude skips', () => {
const fn = createObjectsFilter(
[
{
type: 'a',
id: '1',
},
],
[],
[
{
type: 'b',
from: '1',
to: '2',
},
]
);
expect(
fn({
type: 'a',
id: '1',
attributes: {},
references: [{ name: 'ref_0', type: 'b', id: '1' }],
})
).toEqual(false);
expect(
fn({
type: 'a',
id: '2',
attributes: {},
references: [{ name: 'ref_0', type: 'b', id: '1' }],
})
).toEqual(true);
});
test('filter should include references to replace', () => {
const fn = createObjectsFilter(
[],
[],
[
{
type: 'b',
from: '1',
to: '2',
},
]
);
expect(
fn({
type: 'a',
id: '1',
attributes: {},
references: [
{
name: 'ref_0',
type: 'b',
id: '1',
},
],
})
).toEqual(true);
expect(
fn({
type: 'a',
id: '1',
attributes: {},
references: [
{
name: 'ref_0',
type: 'b',
id: '2',
},
],
})
).toEqual(false);
});
test('filter should include objects to overwrite', () => {
const fn = createObjectsFilter(
[],
[
{
type: 'a',
id: '1',
},
],
[]
);
expect(fn({ type: 'a', id: '1', attributes: {}, references: [] })).toEqual(true);
expect(fn({ type: 'a', id: '2', attributes: {}, references: [] })).toEqual(false);
});
test('filter should work with skips, overwrites and replaceReferences', () => {
const fn = createObjectsFilter(
[
{
type: 'a',
id: '1',
},
],
[
{
type: 'a',
id: '2',
},
],
[
{
type: 'b',
from: '1',
to: '2',
},
]
);
expect(
fn({
type: 'a',
id: '1',
attributes: {},
references: [
{
name: 'ref_0',
type: 'b',
id: '1',
},
],
})
).toEqual(false);
expect(
fn({
type: 'a',
id: '2',
attributes: {},
references: [
{
name: 'ref_0',
type: 'b',
id: '2',
},
],
})
).toEqual(true);
expect(
fn({
type: 'a',
id: '3',
attributes: {},
references: [
{
name: 'ref_0',
type: 'b',
id: '1',
},
],
})
).toEqual(true);
});
});
describe('importSavedObjects()', () => {
const savedObjects: SavedObject[] = [
{
id: '1',
type: 'index-pattern',
attributes: {},
references: [],
},
{
id: '2',
type: 'search',
attributes: {},
references: [],
},
{
id: '3',
type: 'visualization',
attributes: {},
references: [],
},
{
id: '4',
type: 'dashboard',
attributes: {},
references: [],
},
];
const savedObjectsClient = {
errors: {} as any,
bulkCreate: jest.fn(),
bulkGet: jest.fn(),
create: jest.fn(),
delete: jest.fn(),
find: jest.fn(),
get: jest.fn(),
update: jest.fn(),
};
beforeEach(() => {
savedObjectsClient.bulkCreate.mockReset();
savedObjectsClient.bulkGet.mockReset();
savedObjectsClient.create.mockReset();
savedObjectsClient.delete.mockReset();
savedObjectsClient.find.mockReset();
savedObjectsClient.get.mockReset();
savedObjectsClient.update.mockReset();
});
test('calls bulkCreate without overwrite', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects,
});
const result = await importSavedObjects({
readStream,
objectLimit: 4,
overwrite: false,
savedObjectsClient,
});
expect(result).toMatchInlineSnapshot(`
Object {
"success": true,
"successCount": 4,
}
`);
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"attributes": Object {},
"id": "1",
"references": Array [],
"type": "index-pattern",
},
Object {
"attributes": Object {},
"id": "2",
"references": Array [],
"type": "search",
},
Object {
"attributes": Object {},
"id": "3",
"references": Array [],
"type": "visualization",
},
Object {
"attributes": Object {},
"id": "4",
"references": Array [],
"type": "dashboard",
},
],
Object {
"overwrite": false,
},
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
test('calls bulkCreate with overwrite', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects,
});
const result = await importSavedObjects({
readStream,
objectLimit: 4,
overwrite: true,
savedObjectsClient,
});
expect(result).toMatchInlineSnapshot(`
Object {
"success": true,
"successCount": 4,
}
`);
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"attributes": Object {},
"id": "1",
"references": Array [],
"type": "index-pattern",
},
Object {
"attributes": Object {},
"id": "2",
"references": Array [],
"type": "search",
},
Object {
"attributes": Object {},
"id": "3",
"references": Array [],
"type": "visualization",
},
Object {
"attributes": Object {},
"id": "4",
"references": Array [],
"type": "dashboard",
},
],
Object {
"overwrite": true,
},
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
test('extracts errors', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects.map(savedObject => ({
type: savedObject.type,
id: savedObject.id,
error: {
statusCode: 409,
message: 'conflict',
},
})),
});
const result = await importSavedObjects({
readStream,
objectLimit: 4,
overwrite: false,
savedObjectsClient,
});
expect(result).toMatchInlineSnapshot(`
Object {
"errors": Array [
Object {
"error": Object {
"message": "conflict",
"statusCode": 409,
},
"id": "1",
"type": "index-pattern",
},
Object {
"error": Object {
"message": "conflict",
"statusCode": 409,
},
"id": "2",
"type": "search",
},
Object {
"error": Object {
"message": "conflict",
"statusCode": 409,
},
"id": "3",
"type": "visualization",
},
Object {
"error": Object {
"message": "conflict",
"statusCode": 409,
},
"id": "4",
"type": "dashboard",
},
],
"success": false,
"successCount": 0,
}
`);
});
});
describe('resolveImportConflicts()', () => {
const savedObjects: SavedObject[] = [
{
id: '1',
type: 'index-pattern',
attributes: {},
references: [],
},
{
id: '2',
type: 'search',
attributes: {},
references: [],
},
{
id: '3',
type: 'visualization',
attributes: {},
references: [],
},
{
id: '4',
type: 'dashboard',
attributes: {},
references: [
{
name: 'panel_0',
type: 'visualization',
id: '3',
},
],
},
];
const savedObjectsClient = {
errors: {} as any,
bulkCreate: jest.fn(),
bulkGet: jest.fn(),
create: jest.fn(),
delete: jest.fn(),
find: jest.fn(),
get: jest.fn(),
update: jest.fn(),
};
beforeEach(() => {
savedObjectsClient.bulkCreate.mockReset();
savedObjectsClient.bulkGet.mockReset();
savedObjectsClient.create.mockReset();
savedObjectsClient.delete.mockReset();
savedObjectsClient.find.mockReset();
savedObjectsClient.get.mockReset();
savedObjectsClient.update.mockReset();
});
test('works with empty parameters', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects,
});
const result = await resolveImportConflicts({
readStream,
objectLimit: 4,
skips: [],
overwrites: [],
savedObjectsClient,
replaceReferences: [],
});
expect(result).toMatchInlineSnapshot(`
Object {
"success": true,
"successCount": 0,
}
`);
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`[MockFunction]`);
});
test('works with skips', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects,
});
const result = await resolveImportConflicts({
readStream,
objectLimit: 4,
skips: [
{
type: 'dashboard',
id: '4',
},
],
overwrites: [],
savedObjectsClient,
replaceReferences: [
{
type: 'visualization',
from: '3',
to: '30',
},
],
});
expect(result).toMatchInlineSnapshot(`
Object {
"success": true,
"successCount": 0,
}
`);
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`[MockFunction]`);
});
test('works with overwrites', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects,
});
const result = await resolveImportConflicts({
readStream,
objectLimit: 4,
skips: [],
overwrites: [
{
type: 'index-pattern',
id: '1',
},
],
savedObjectsClient,
replaceReferences: [],
});
expect(result).toMatchInlineSnapshot(`
Object {
"success": true,
"successCount": 1,
}
`);
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"attributes": Object {},
"id": "1",
"references": Array [],
"type": "index-pattern",
},
],
Object {
"overwrite": true,
},
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
test('works wtih replaceReferences', async () => {
const readStream = new Readable({
read() {
savedObjects.forEach(obj => this.push(JSON.stringify(obj) + '\n'));
this.push(null);
},
});
savedObjectsClient.bulkCreate.mockResolvedValue({
saved_objects: savedObjects,
});
const result = await resolveImportConflicts({
readStream,
objectLimit: 4,
skips: [],
overwrites: [],
savedObjectsClient,
replaceReferences: [
{
type: 'visualization',
from: '3',
to: '13',
},
],
});
expect(result).toMatchInlineSnapshot(`
Object {
"success": true,
"successCount": 1,
}
`);
expect(savedObjectsClient.bulkCreate).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"attributes": Object {},
"id": "4",
"references": Array [
Object {
"id": "13",
"name": "panel_0",
"type": "visualization",
},
],
"type": "dashboard",
},
],
Object {
"overwrite": true,
},
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
});

218
src/legacy/server/saved_objects/lib/import.ts
View file

@ -1,218 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import Boom from 'boom';
import { Readable, Transform } from 'stream';
import {
createConcatStream,
createFilterStream,
createMapStream,
createPromiseFromStreams,
createSplitStream,
} from '../../../utils/streams';
import { SavedObject, SavedObjectsClient } from '../service';
interface CustomError {
id: string;
type: string;
error: {
message: string;
statusCode: number;
};
}
interface ImportResponse {
success: boolean;
successCount: number;
errors?: CustomError[];
}
interface ImportSavedObjectsOptions {
readStream: Readable;
objectLimit: number;
overwrite: boolean;
savedObjectsClient: SavedObjectsClient;
}
interface ResolveImportConflictsOptions {
readStream: Readable;
objectLimit: number;
savedObjectsClient: SavedObjectsClient;
overwrites: Array<{
type: string;
id: string;
}>;
replaceReferences: Array<{
type: string;
from: string;
to: string;
}>;
skips: Array<{
type: string;
id: string;
}>;
}
export function extractErrors(savedObjects: SavedObject[]) {
const errors: CustomError[] = [];
for (const savedObject of savedObjects) {
if (savedObject.error) {
errors.push({
id: savedObject.id,
type: savedObject.type,
error: savedObject.error,
});
}
}
return errors;
}
export function createLimitStream(limit: number) {
let counter = 0;
return new Transform({
objectMode: true,
async transform(obj, enc, done) {
if (counter >= limit) {
return done(Boom.badRequest(`Can't import more than ${limit} objects`));
}
counter++;
done(undefined, obj);
},
});
}
export async function collectSavedObjects(
readStream: Readable,
objectLimit: number,
filter?: (obj: SavedObject) => boolean
): Promise<SavedObject[]> {
return (await createPromiseFromStreams([
readStream,
createSplitStream('\n'),
createMapStream((str: string) => {
if (str && str !== '') {
return JSON.parse(str);
}
}),
createFilterStream<SavedObject>(obj => !!obj),
createLimitStream(objectLimit),
createFilterStream<SavedObject>(obj => (filter ? filter(obj) : true)),
createConcatStream([]),
])) as SavedObject[];
}
export function createObjectsFilter(
skips: Array<{
type: string;
id: string;
}>,
overwrites: Array<{
type: string;
id: string;
}>,
replaceReferences: Array<{
type: string;
from: string;
to: string;
}>
) {
const refReplacements = replaceReferences.map(ref => `${ref.type}:${ref.from}`);
return (obj: SavedObject) => {
if (skips.some(skipObj => skipObj.type === obj.type && skipObj.id === obj.id)) {
return false;
}
if (
overwrites.some(overwriteObj => overwriteObj.type === obj.type && overwriteObj.id === obj.id)
) {
return true;
}
for (const reference of obj.references || []) {
if (refReplacements.includes(`${reference.type}:${reference.id}`)) {
return true;
}
}
return false;
};
}
export async function importSavedObjects({
readStream,
objectLimit,
overwrite,
savedObjectsClient,
}: ImportSavedObjectsOptions): Promise<ImportResponse> {
const objectsToImport = await collectSavedObjects(readStream, objectLimit);
if (objectsToImport.length === 0) {
return {
success: true,
successCount: 0,
};
}
const bulkCreateResult = await savedObjectsClient.bulkCreate(objectsToImport, {
overwrite,
});
const errors = extractErrors(bulkCreateResult.saved_objects);
return {
success: errors.length === 0,
successCount: objectsToImport.length - errors.length,
...(errors.length ? { errors } : {}),
};
}
export async function resolveImportConflicts({
readStream,
objectLimit,
skips,
overwrites,
savedObjectsClient,
replaceReferences,
}: ResolveImportConflictsOptions): Promise<ImportResponse> {
let errors: CustomError[] = [];
const filter = createObjectsFilter(skips, overwrites, replaceReferences);
const objectsToResolve = await collectSavedObjects(readStream, objectLimit, filter);
// Replace references
const refReplacementsMap: Record<string, string> = {};
for (const { type, to, from } of replaceReferences) {
refReplacementsMap[`${type}:${from}`] = to;
}
for (const savedObject of objectsToResolve) {
for (const reference of savedObject.references || []) {
if (refReplacementsMap[`${reference.type}:${reference.id}`]) {
reference.id = refReplacementsMap[`${reference.type}:${reference.id}`];
}
}
}
if (objectsToResolve.length) {
const bulkCreateResult = await savedObjectsClient.bulkCreate(objectsToResolve, {
overwrite: true,
});
errors = extractErrors(bulkCreateResult.saved_objects);
}
return {
success: errors.length === 0,
successCount: objectsToResolve.length - errors.length,
...(errors.length ? { errors } : {}),
};
}

2
src/legacy/server/saved_objects/routes/export.ts
View file

@ -21,7 +21,7 @@ import Hapi from 'hapi';
import Joi from 'joi';
import stringify from 'json-stable-stringify';
import { SavedObjectsClient } from '../';
import { getSortedObjectsForExport } from '../lib';
import { getSortedObjectsForExport } from '../export';
import { Prerequisites } from './types';
const ALLOWED_TYPES = ['index-pattern', 'search', 'visualization', 'dashboard'];

2
src/legacy/server/saved_objects/routes/import.ts
View file

@ -23,7 +23,7 @@ import Joi from 'joi';
import { extname } from 'path';
import { Readable } from 'stream';
import { SavedObjectsClient } from '../';
import { importSavedObjects } from '../lib';
import { importSavedObjects } from '../import';
import { Prerequisites, WithoutQueryAndParams } from './types';
interface HapiReadableStream extends Readable {

2
src/legacy/server/saved_objects/routes/resolve_import_conflicts.ts
View file

@ -23,7 +23,7 @@ import Joi from 'joi';
import { extname } from 'path';
import { Readable } from 'stream';
import { SavedObjectsClient } from '../';
import { resolveImportConflicts } from '../lib';
import { resolveImportConflicts } from '../import';
import { Prerequisites } from './types';
interface HapiReadableStream extends Readable {