maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Ingest Manager] check for packages stuck installing and reinstall (#75226)

Browse source 
* check for packages stuck installing and reinstall

* update mock endpoint package

* diferentiate between reinstall and reupdate type of install, remove isUpdate, add integration test

* create new EpmPackageInstallStatus type instead of using InstallStatus

* fix merge conflict

* change EpmPackageInstallStatus to a union type

* change time to install to 1 minute

* used saved object find

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Sandra Gonzales 2020-08-24 09:42:39 -04:00 committed by GitHub
parent fdc93af824
commit ed53ca6b46
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 279 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
x-pack/plugins/ingest_manager/common/constants/epm.ts
View file

@ -7,3 +7,4 @@
export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';
export const INDEX_PATTERN_PLACEHOLDER_SUFFIX = '-index_pattern_placeholder';
export const MAX_TIME_COMPLETE_INSTALL = 60000;

5
x-pack/plugins/ingest_manager/common/types/models/epm.ts
View file

@ -19,6 +19,8 @@ export enum InstallStatus {
uninstalling = 'uninstalling',
}
export type EpmPackageInstallStatus = 'installed' | 'installing';
export type DetailViewPanelName = 'overview' | 'usages' | 'settings';
export type ServiceName = 'kibana' | 'elasticsearch';
export type AssetType = KibanaAssetType | ElasticsearchAssetType | AgentAssetType;
@ -234,6 +236,9 @@ export interface Installation extends SavedObjectAttributes {
es_index_patterns: Record<string, string>;
name: string;
version: string;
install_status: EpmPackageInstallStatus;
install_version: string;
install_started_at: string;
}
export type Installable<T> = Installed<T> | NotInstalled<T>;

1
x-pack/plugins/ingest_manager/server/constants/index.ts
View file

@ -14,6 +14,7 @@ export {
AGENT_POLICY_ROLLOUT_RATE_LIMIT_INTERVAL_MS,
AGENT_UPDATE_ACTIONS_INTERVAL_MS,
INDEX_PATTERN_PLACEHOLDER_SUFFIX,
MAX_TIME_COMPLETE_INSTALL,
// Routes
LIMITED_CONCURRENCY_ROUTE_TAG,
PLUGIN_ID,

3
x-pack/plugins/ingest_manager/server/saved_objects/index.ts
View file

@ -285,6 +285,9 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = {
type: { type: 'keyword' },
},
},
install_started_at: { type: 'date' },
install_version: { type: 'keyword' },
install_status: { type: 'keyword' },
},
},
},

1
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts
View file

@ -22,7 +22,6 @@ import { removeAssetsFromInstalledEsByType, saveInstalledEsRefs } from '../../pa
export const installTemplates = async (
registryPackage: RegistryPackage,
isUpdate: boolean,
callCluster: CallESAsCurrentUser,
paths: string[],
savedObjectsClient: SavedObjectsClientContract

1
x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts
View file

@ -48,7 +48,6 @@ export async function installKibanaAssets(options: {
savedObjectsClient: SavedObjectsClientContract;
pkgName: string;
kibanaAssets: ArchiveAsset[];
isUpdate: boolean;
}): Promise<SavedObject[]> {
const { savedObjectsClient, kibanaAssets } = options;

8
x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { SavedObjectsClientContract } from 'src/core/server';
import { SavedObjectsClientContract, SavedObjectsFindOptions } from 'src/core/server';
import { isPackageLimited } from '../../../../common';
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
import { Installation, InstallationStatus, PackageInfo, KibanaAssetType } from '../../../types';
@ -72,8 +72,12 @@ export async function getLimitedPackages(options: {
return installedPackagesInfo.filter(isPackageLimited).map((pkgInfo) => pkgInfo.name);
}
export async function getPackageSavedObjects(savedObjectsClient: SavedObjectsClientContract) {
export async function getPackageSavedObjects(
savedObjectsClient: SavedObjectsClientContract,
options?: Omit<SavedObjectsFindOptions, 'type'>
) {
return savedObjectsClient.find<Installation>({
...(options || {}),
type: PACKAGES_SAVED_OBJECT_TYPE,
});
}

64
x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts
View file

@ -6,7 +6,7 @@
import { SavedObjectsClientContract } from 'src/core/server';
import semver from 'semver';
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
import { PACKAGES_SAVED_OBJECT_TYPE, MAX_TIME_COMPLETE_INSTALL } from '../../../constants';
import {
AssetReference,
Installation,
@ -33,6 +33,7 @@ import {
import { updateCurrentWriteIndices } from '../elasticsearch/template/template';
import { deleteKibanaSavedObjectsAssets } from './remove';
import { PackageOutdatedError } from '../../../errors';
import { getPackageSavedObjects } from './get';
export async function installLatestPackage(options: {
savedObjectsClient: SavedObjectsClientContract;
@ -107,22 +108,24 @@ export async function installPackage({
// TODO: calls to getInstallationObject, Registry.fetchInfo, and Registry.fetchFindLatestPackge
// and be replaced by getPackageInfo after adjusting for it to not group/use archive assets
const latestPackage = await Registry.fetchFindLatestPackage(pkgName);
if (semver.lt(pkgVersion, latestPackage.version) && !force)
throw new PackageOutdatedError(`${pkgkey} is out-of-date and cannot be installed or updated`);
// get the currently installed package
const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
const reinstall = pkgVersion === installedPkg?.attributes.version;
const reupdate = pkgVersion === installedPkg?.attributes.install_version;
// let the user install if using the force flag or this is a reinstall or reupdate due to intallation interruption
if (semver.lt(pkgVersion, latestPackage.version) && !force && !reinstall && !reupdate) {
throw new PackageOutdatedError(`${pkgkey} is out-of-date and cannot be installed or updated`);
}
const paths = await Registry.getArchiveInfo(pkgName, pkgVersion);
const registryPackageInfo = await Registry.fetchInfo(pkgName, pkgVersion);
// get the currently installed package
const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
const isUpdate = installedPkg && installedPkg.attributes.version < pkgVersion ? true : false;
const reinstall = pkgVersion === installedPkg?.attributes.version;
const removable = !isRequiredPackage(pkgName);
const { internal = false } = registryPackageInfo;
const toSaveESIndexPatterns = generateESIndexPatterns(registryPackageInfo.datasets);
// add the package installation to the saved object
// add the package installation to the saved object.
// if some installation already exists, just update install info
if (!installedPkg) {
await createInstallation({
savedObjectsClient,
@ -134,6 +137,12 @@ export async function installPackage({
installed_es: [],
toSaveESIndexPatterns,
});
} else {
await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
install_version: pkgVersion,
install_status: 'installing',
install_started_at: new Date().toISOString(),
});
}
const installIndexPatternPromise = installIndexPatterns(savedObjectsClient, pkgName, pkgVersion);
const kibanaAssets = await getKibanaAssets(paths);
@ -152,7 +161,6 @@ export async function installPackage({
savedObjectsClient,
pkgName,
kibanaAssets,
isUpdate,
});
// the rest of the installation must happen in sequential order
@ -172,7 +180,6 @@ export async function installPackage({
// install or update the templates referencing the newly installed pipelines
const installedTemplates = await installTemplates(
registryPackageInfo,
isUpdate,
callCluster,
paths,
savedObjectsClient
@ -197,9 +204,14 @@ export async function installPackage({
}));
await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]);
// update to newly installed version when all assets are successfully installed
if (isUpdate) await updateVersion(savedObjectsClient, pkgName, pkgVersion);
if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion);
await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
install_version: pkgVersion,
install_status: 'installed',
});
return [...installedKibanaAssetsRefs, ...installedPipelines, ...installedTemplateRefs];
}
const updateVersion = async (
savedObjectsClient: SavedObjectsClientContract,
pkgName: string,
@ -239,6 +251,9 @@ export async function createInstallation(options: {
version: pkgVersion,
internal,
removable,
install_version: pkgVersion,
install_status: 'installing',
install_started_at: new Date().toISOString(),
},
{ id: pkgName, overwrite: true }
);
@ -286,3 +301,28 @@ export const removeAssetsFromInstalledEsByType = async (
installed_es: installedAssetsToSave,
});
};
export async function ensurePackagesCompletedInstall(
savedObjectsClient: SavedObjectsClientContract,
callCluster: CallESAsCurrentUser
) {
const installingPackages = await getPackageSavedObjects(savedObjectsClient, {
searchFields: ['install_status'],
search: 'installing',
});
const installingPromises = installingPackages.saved_objects.reduce<
Array<Promise<AssetReference[]>>
>((acc, pkg) => {
const startDate = pkg.attributes.install_started_at;
const nowDate = new Date().toISOString();
const elapsedTime = Date.parse(nowDate) - Date.parse(startDate);
const pkgkey = `${pkg.attributes.name}-${pkg.attributes.install_version}`;
// reinstall package
if (elapsedTime > MAX_TIME_COMPLETE_INSTALL) {
acc.push(installPackage({ savedObjectsClient, pkgkey, callCluster }));
}
return acc;
}, []);
await Promise.all(installingPromises);
return installingPackages;
}

6
x-pack/plugins/ingest_manager/server/services/setup.ts
View file

@ -10,7 +10,10 @@ import { SavedObjectsClientContract } from 'src/core/server';
import { CallESAsCurrentUser } from '../types';
import { agentPolicyService } from './agent_policy';
import { outputService } from './output';
import { ensureInstalledDefaultPackages } from './epm/packages/install';
import {
ensureInstalledDefaultPackages,
ensurePackagesCompletedInstall,
} from './epm/packages/install';
import { ensureDefaultIndices } from './epm/kibana/index_pattern/install';
import {
packageToPackagePolicy,
@ -51,6 +54,7 @@ async function createSetupSideEffects(
ensureInstalledDefaultPackages(soClient, callCluster),
outputService.ensureDefaultOutput(soClient),
agentPolicyService.ensureDefaultAgentPolicy(soClient),
ensurePackagesCompletedInstall(soClient, callCluster),
ensureDefaultIndices(callCluster),
settingsService.getSettings(soClient).catch((e: any) => {
if (e.isBoom && e.output.statusCode === 404) {

1
x-pack/plugins/ingest_manager/server/types/index.tsx
View file

@ -37,6 +37,7 @@ export {
EnrollmentAPIKey,
EnrollmentAPIKeySOAttributes,
Installation,
EpmPackageInstallStatus,
InstallationStatus,
PackageInfo,
RegistryVarsEntry,

3
x-pack/plugins/security_solution/common/endpoint/generate_data.ts
View file

@ -1158,6 +1158,9 @@ export class EndpointDocGenerator {
version: '0.5.0',
internal: false,
removable: false,
install_version: '0.5.0',
install_status: 'installed',
install_started_at: '2020-06-24T14:41:23.098Z',
},
references: [],
updated_at: '2020-06-24T14:41:23.098Z',

1
x-pack/test/ingest_manager_api_integration/apis/epm/index.js
View file

@ -17,5 +17,6 @@ export default function loadTests({ loadTestFile }) {
loadTestFile(require.resolve('./install_update'));
loadTestFile(require.resolve('./update_assets'));
loadTestFile(require.resolve('./data_stream'));
loadTestFile(require.resolve('./package_install_complete'));
});
}

3
x-pack/test/ingest_manager_api_integration/apis/epm/install_remove_assets.ts
View file

@ -170,6 +170,9 @@ export default function (providerContext: FtrProviderContext) {
version: '0.1.0',
internal: false,
removable: true,
install_version: '0.1.0',
install_status: 'installed',
install_started_at: res.attributes.install_started_at,
});
});
});

28
x-pack/test/ingest_manager_api_integration/apis/epm/install_update.ts
View file

@ -7,6 +7,10 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
import { skipIfNoDockerRegistry } from '../../helpers';
import {
PACKAGES_SAVED_OBJECT_TYPE,
MAX_TIME_COMPLETE_INSTALL,
} from '../../../../plugins/ingest_manager/common';
export default function (providerContext: FtrProviderContext) {
const { getService } = providerContext;
@ -62,6 +66,12 @@ export default function (providerContext: FtrProviderContext) {
.send({ force: true })
.expect(200);
});
it('should return 200 if trying to reinstall an out-of-date package', async function () {
await supertest
.post(`/api/ingest_manager/epm/packages/multiple_versions-0.1.0`)
.set('kbn-xsrf', 'xxxx')
.expect(200);
});
it('should return 400 if trying to update to an out-of-date package', async function () {
await supertest
.post(`/api/ingest_manager/epm/packages/multiple_versions-0.2.0`)
@ -75,6 +85,24 @@ export default function (providerContext: FtrProviderContext) {
.send({ force: true })
.expect(200);
});
it('should return 200 if trying to reupdate an out-of-date package', async function () {
const previousInstallDate = new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL).toISOString();
// mock package to be stuck installing an update
await kibanaServer.savedObjects.update({
id: 'multiple_versions',
type: PACKAGES_SAVED_OBJECT_TYPE,
attributes: {
install_status: 'installing',
install_started_at: previousInstallDate,
install_version: '0.2.0',
version: '0.1.0',
},
});
await supertest
.post(`/api/ingest_manager/epm/packages/multiple_versions-0.2.0`)
.set('kbn-xsrf', 'xxxx')
.expect(200);
});
it('should return 200 if trying to update to the latest package', async function () {
await supertest
.post(`/api/ingest_manager/epm/packages/multiple_versions-0.3.0`)

167
x-pack/test/ingest_manager_api_integration/apis/epm/package_install_complete.ts Normal file
View file

@ -0,0 +1,167 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import expect from '@kbn/expect';
import {
PACKAGES_SAVED_OBJECT_TYPE,
MAX_TIME_COMPLETE_INSTALL,
} from '../../../../plugins/ingest_manager/common';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
export default function (providerContext: FtrProviderContext) {
const { getService } = providerContext;
const supertest = getService('supertest');
const kibanaServer = getService('kibanaServer');
const pkgName = 'multiple_versions';
const pkgVersion = '0.1.0';
const pkgUpdateVersion = '0.2.0';
describe('setup checks packages completed install', async () => {
describe('package install', async () => {
before(async () => {
await supertest
.post(`/api/ingest_manager/epm/packages/${pkgName}-0.1.0`)
.set('kbn-xsrf', 'xxxx')
.send({ force: true });
});
it('should have not reinstalled if package install completed', async function () {
const packageBeforeSetup = await kibanaServer.savedObjects.get({
type: 'epm-packages',
id: pkgName,
});
const installStartedAtBeforeSetup = packageBeforeSetup.attributes.install_started_at;
await supertest.post(`/api/ingest_manager/setup`).set('kbn-xsrf', 'xxx').send();
const packageAfterSetup = await kibanaServer.savedObjects.get({
type: PACKAGES_SAVED_OBJECT_TYPE,
id: pkgName,
});
const installStartedAfterSetup = packageAfterSetup.attributes.install_started_at;
expect(installStartedAtBeforeSetup).equal(installStartedAfterSetup);
});
it('should have reinstalled if package installing did not complete in elapsed time', async function () {
// change the saved object to installing to mock kibana crashing and not finishing the install
const previousInstallDate = new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL).toISOString();
await kibanaServer.savedObjects.update({
id: pkgName,
type: PACKAGES_SAVED_OBJECT_TYPE,
attributes: {
install_status: 'installing',
install_started_at: previousInstallDate,
},
});
await supertest.post(`/api/ingest_manager/setup`).set('kbn-xsrf', 'xxx').send();
const packageAfterSetup = await kibanaServer.savedObjects.get({
type: PACKAGES_SAVED_OBJECT_TYPE,
id: pkgName,
});
const installStartedAfterSetup = packageAfterSetup.attributes.install_started_at;
expect(Date.parse(installStartedAfterSetup)).greaterThan(Date.parse(previousInstallDate));
expect(packageAfterSetup.attributes.install_status).equal('installed');
});
it('should have not reinstalled if package installing did not surpass elapsed time', async function () {
// change the saved object to installing to mock package still installing, but a time less than the max time allowable
const previousInstallDate = new Date(Date.now()).toISOString();
await kibanaServer.savedObjects.update({
id: pkgName,
type: PACKAGES_SAVED_OBJECT_TYPE,
attributes: {
install_status: 'installing',
install_started_at: previousInstallDate,
},
});
await supertest.post(`/api/ingest_manager/setup`).set('kbn-xsrf', 'xxx').send();
const packageAfterSetup = await kibanaServer.savedObjects.get({
type: PACKAGES_SAVED_OBJECT_TYPE,
id: pkgName,
});
const installStartedAfterSetup = packageAfterSetup.attributes.install_started_at;
expect(Date.parse(installStartedAfterSetup)).equal(Date.parse(previousInstallDate));
expect(packageAfterSetup.attributes.install_status).equal('installing');
});
after(async () => {
await supertest
.delete(`/api/ingest_manager/epm/packages/multiple_versions-0.1.0`)
.set('kbn-xsrf', 'xxxx');
});
});
describe('package update', async () => {
before(async () => {
await supertest
.post(`/api/ingest_manager/epm/packages/${pkgName}-0.1.0`)
.set('kbn-xsrf', 'xxxx')
.send({ force: true });
await supertest
.post(`/api/ingest_manager/epm/packages/${pkgName}-0.2.0`)
.set('kbn-xsrf', 'xxxx')
.send({ force: true });
});
it('should have not reinstalled if package update completed', async function () {
const packageBeforeSetup = await kibanaServer.savedObjects.get({
type: 'epm-packages',
id: pkgName,
});
const installStartedAtBeforeSetup = packageBeforeSetup.attributes.install_started_at;
await supertest.post(`/api/ingest_manager/setup`).set('kbn-xsrf', 'xxx').send();
const packageAfterSetup = await kibanaServer.savedObjects.get({
type: PACKAGES_SAVED_OBJECT_TYPE,
id: pkgName,
});
const installStartedAfterSetup = packageAfterSetup.attributes.install_started_at;
expect(installStartedAtBeforeSetup).equal(installStartedAfterSetup);
});
it('should have reinstalled if package updating did not complete in elapsed time', async function () {
// change the saved object to installing to mock kibana crashing and not finishing the update
const previousInstallDate = new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL).toISOString();
await kibanaServer.savedObjects.update({
id: pkgName,
type: PACKAGES_SAVED_OBJECT_TYPE,
attributes: {
version: pkgVersion,
install_status: 'installing',
install_started_at: previousInstallDate,
install_version: pkgUpdateVersion, // set version back
},
});
await supertest.post(`/api/ingest_manager/setup`).set('kbn-xsrf', 'xxx').send();
const packageAfterSetup = await kibanaServer.savedObjects.get({
type: PACKAGES_SAVED_OBJECT_TYPE,
id: pkgName,
});
const installStartedAfterSetup = packageAfterSetup.attributes.install_started_at;
expect(Date.parse(installStartedAfterSetup)).greaterThan(Date.parse(previousInstallDate));
expect(packageAfterSetup.attributes.install_status).equal('installed');
expect(packageAfterSetup.attributes.version).equal(pkgUpdateVersion);
expect(packageAfterSetup.attributes.install_version).equal(pkgUpdateVersion);
});
it('should have not reinstalled if package updating did not surpass elapsed time', async function () {
// change the saved object to installing to mock package still installing, but a time less than the max time allowable
const previousInstallDate = new Date(Date.now()).toISOString();
await kibanaServer.savedObjects.update({
id: pkgName,
type: PACKAGES_SAVED_OBJECT_TYPE,
attributes: {
install_status: 'installing',
install_started_at: previousInstallDate,
version: pkgVersion, // set version back
},
});
await supertest.post(`/api/ingest_manager/setup`).set('kbn-xsrf', 'xxx').send();
const packageAfterSetup = await kibanaServer.savedObjects.get({
type: PACKAGES_SAVED_OBJECT_TYPE,
id: pkgName,
});
const installStartedAfterSetup = packageAfterSetup.attributes.install_started_at;
expect(Date.parse(installStartedAfterSetup)).equal(Date.parse(previousInstallDate));
expect(packageAfterSetup.attributes.install_status).equal('installing');
expect(packageAfterSetup.attributes.version).equal(pkgVersion);
});
after(async () => {
await supertest
.delete(`/api/ingest_manager/epm/packages/multiple_versions-0.1.0`)
.set('kbn-xsrf', 'xxxx');
});
});
});
}

3
x-pack/test/ingest_manager_api_integration/apis/epm/update_assets.ts
View file

@ -322,6 +322,9 @@ export default function (providerContext: FtrProviderContext) {
version: '0.2.0',
internal: false,
removable: true,
install_version: '0.2.0',
install_status: 'installed',
install_started_at: res.attributes.install_started_at,
});
});
});