maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Fleet] Remove upgradePackage and consolidate it with installPackage, optimize calls to create index patterns (#94490)

Browse source 
* Add data plugin to server app context

* First attempt at switching to indexPatternService for EPM index pattern creation & deletion, instead of interacting directly with index pattern SOs

* Simplify bulk install package, remove upgradePackage() method in favor of consolidating with installPackage(), use installPackage() for bulk install instead

* Update tests

* Change cache logging of objects to trace level

* Install index patterns as a post-package installation operation, for bulk package installation, install index pattern only if one or more packages are actually new installs, add debug logging

* Allow getAsSavedObjectBody to return non-scripted fields when allowNoIndex is true

* Allow `getFieldsForWildcard` to return fields saved on index pattern when allowNoIndices is true

* Bring back passing custom ID for index pattern SO

* Fix tests

* Revert "Merge branch 'index-pattern/allow-no-index' into epm/missing-index-patterns"

This reverts commit 8e712e9c00, reversing
changes made to af0fb0eaa8.

* Allow getAsSavedObjectBody to return non-scripted fields when allowNoIndex is true

(cherry picked from commit 69b93da180)

* Update API docs

* Run post-install ops for install by upload too

* Remove allowedInstallTypes param

* Adjust force install conditions

* Revert "Update API docs"

This reverts commit b9770fdc56.

* Revert "Allow getAsSavedObjectBody to return non-scripted fields when allowNoIndex is true"

This reverts commit afc91ce32f.

* Go back to using SO client for index patterns :(

* Stringify attributes again for SO client saving

* Fix condition for reinstall same pkg version

* Remove unused type

* Adjust comment

* Update snapshot

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Jen Huang 2021-03-23 11:34:20 -07:00 committed by GitHub
parent 2ed2cfe52f
commit d886979e3b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 371 additions and 497 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
x-pack/plugins/fleet/common/constants/epm.ts
View file

@ -7,7 +7,6 @@
export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets';
export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';
export const MAX_TIME_COMPLETE_INSTALL = 60000;
export const FLEET_SERVER_PACKAGE = 'fleet_server';

11
x-pack/plugins/fleet/common/types/rest_spec/epm.ts
View file

@ -82,12 +82,15 @@ export interface IBulkInstallPackageHTTPError {
error: string | Error;
}
export interface InstallResult {
assets: AssetReference[];
status: 'installed' | 'already_installed';
}
export interface BulkInstallPackageInfo {
name: string;
newVersion: string;
// this will be null if no package was present before the upgrade (aka it was an install)
oldVersion: string | null;
assets: AssetReference[];
version: string;
result: InstallResult;
}
export interface BulkInstallPackagesResponse {

1
x-pack/plugins/fleet/server/constants/index.ts
View file

@ -43,7 +43,6 @@ export {
OUTPUT_SAVED_OBJECT_TYPE,
PACKAGES_SAVED_OBJECT_TYPE,
ASSETS_SAVED_OBJECT_TYPE,
INDEX_PATTERN_SAVED_OBJECT_TYPE,
ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
GLOBAL_SETTINGS_SAVED_OBJECT_TYPE,
// Defaults

2
x-pack/plugins/fleet/server/mocks/index.ts
View file

@ -10,6 +10,7 @@ import {
savedObjectsServiceMock,
coreMock,
} from '../../../../../src/core/server/mocks';
import { dataPluginMock } from '../../../../../src/plugins/data/server/mocks';
import { licensingMock } from '../../../../plugins/licensing/server/mocks';
import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks';
import { securityMock } from '../../../security/server/mocks';
@ -23,6 +24,7 @@ export * from '../services/artifacts/mocks';
export const createAppContextStartContractMock = (): FleetAppContext => {
return {
elasticsearch: elasticsearchServiceMock.createStart(),
data: dataPluginMock.createStartContract(),
encryptedSavedObjectsStart: encryptedSavedObjectsMock.createStart(),
savedObjects: savedObjectsServiceMock.createStartContract(),
security: securityMock.createStart(),

4
x-pack/plugins/fleet/server/plugin.ts
View file

@ -23,6 +23,7 @@ import type {
import type { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server';
import type { PluginStart as DataPluginStart } from '../../../../src/plugins/data/server';
import type { LicensingPluginSetup, ILicense } from '../../licensing/server';
import type {
EncryptedSavedObjectsPluginStart,
@ -100,12 +101,14 @@ export interface FleetSetupDeps {
}
export interface FleetStartDeps {
data: DataPluginStart;
encryptedSavedObjects: EncryptedSavedObjectsPluginStart;
security?: SecurityPluginStart;
}
export interface FleetAppContext {
elasticsearch: ElasticsearchServiceStart;
data: DataPluginStart;
encryptedSavedObjectsStart?: EncryptedSavedObjectsPluginStart;
encryptedSavedObjectsSetup?: EncryptedSavedObjectsPluginSetup;
security?: SecurityPluginStart;
@ -293,6 +296,7 @@ export class FleetPlugin
public async start(core: CoreStart, plugins: FleetStartDeps): Promise<FleetStartContract> {
await appContextService.start({
elasticsearch: core.elasticsearch,
data: plugins.data,
encryptedSavedObjectsStart: plugins.encryptedSavedObjects,
encryptedSavedObjectsSetup: this.encryptedSavedObjectsSetup,
security: plugins.security,

31
x-pack/plugins/fleet/server/routes/epm/handlers.ts
View file

@ -40,12 +40,10 @@ import {
getPackages,
getFile,
getPackageInfo,
handleInstallPackageFailure,
isBulkInstallError,
installPackage,
removeInstallation,
getLimitedPackages,
getInstallationObject,
getInstallation,
} from '../../services/epm/packages';
import type { BulkInstallResponse } from '../../services/epm/packages';
@ -228,15 +226,7 @@ export const installPackageFromRegistryHandler: RequestHandler<
const savedObjectsClient = context.core.savedObjects.client;
const esClient = context.core.elasticsearch.client.asCurrentUser;
const { pkgkey } = request.params;
let pkgName: string | undefined;
let pkgVersion: string | undefined;
try {
const parsedPkgKey = splitPkgKey(pkgkey);
pkgName = parsedPkgKey.pkgName;
pkgVersion = parsedPkgKey.pkgVersion;
const res = await installPackage({
installSource: 'registry',
savedObjectsClient,
@ -245,24 +235,11 @@ export const installPackageFromRegistryHandler: RequestHandler<
force: request.body?.force,
});
const body: InstallPackageResponse = {
response: res,
response: res.assets,
};
return response.ok({ body });
} catch (e) {
const defaultResult = await defaultIngestErrorHandler({ error: e, response });
if (pkgName && pkgVersion) {
const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
await handleInstallPackageFailure({
savedObjectsClient,
error: e,
pkgName,
pkgVersion,
installedPkg,
esClient,
});
}
return defaultResult;
return await defaultIngestErrorHandler({ error: e, response });
}
};
@ -291,7 +268,7 @@ export const bulkInstallPackagesFromRegistryHandler: RequestHandler<
const bulkInstalledResponses = await bulkInstallPackages({
savedObjectsClient,
esClient,
packagesToUpgrade: request.body.packages,
packagesToInstall: request.body.packages,
});
const payload = bulkInstalledResponses.map(bulkInstallServiceResponseToHttpEntry);
const body: BulkInstallPackagesResponse = {
@ -324,7 +301,7 @@ export const installPackageByUploadHandler: RequestHandler<
contentType,
});
const body: InstallPackageResponse = {
response: res,
response: res.assets,
};
return response.ok({ body });
} catch (error) {

10
x-pack/plugins/fleet/server/services/app_context.ts
View file

@ -17,6 +17,7 @@ import type {
Logger,
} from 'src/core/server';
import type { PluginStart as DataPluginStart } from '../../../../../src/plugins/data/server';
import type {
EncryptedSavedObjectsClient,
EncryptedSavedObjectsPluginSetup,
@ -29,6 +30,7 @@ import type { CloudSetup } from '../../../cloud/server';
class AppContextService {
private encryptedSavedObjects: EncryptedSavedObjectsClient | undefined;
private encryptedSavedObjectsSetup: EncryptedSavedObjectsPluginSetup | undefined;
private data: DataPluginStart | undefined;
private esClient: ElasticsearchClient | undefined;
private security: SecurityPluginStart | undefined;
private config$?: Observable<FleetConfigType>;
@ -43,6 +45,7 @@ class AppContextService {
private externalCallbacks: ExternalCallbacksStorage = new Map();
public async start(appContext: FleetAppContext) {
this.data = appContext.data;
this.esClient = appContext.elasticsearch.client.asInternalUser;
this.encryptedSavedObjects = appContext.encryptedSavedObjectsStart?.getClient();
this.encryptedSavedObjectsSetup = appContext.encryptedSavedObjectsSetup;
@ -67,6 +70,13 @@ class AppContextService {
this.externalCallbacks.clear();
}
public getData() {
if (!this.data) {
throw new Error('Data start service not set.');
}
return this.data;
}
public getEncryptedSavedObjects() {
if (!this.encryptedSavedObjects) {
throw new Error('Encrypted saved object start service not set.');

18
x-pack/plugins/fleet/server/services/epm/archive/cache.ts
View file

@ -28,13 +28,9 @@ export const getArchiveFilelist = (keyArgs: SharedKey) =>
archiveFilelistCache.get(sharedKey(keyArgs));
export const setArchiveFilelist = (keyArgs: SharedKey, paths: string[]) => {
appContextService
.getLogger()
.debug(
`setting file list to the cache for ${keyArgs.name}-${keyArgs.version}:\n${JSON.stringify(
paths
)}`
);
const logger = appContextService.getLogger();
logger.debug(`setting file list to the cache for ${keyArgs.name}-${keyArgs.version}`);
logger.trace(JSON.stringify(paths));
return archiveFilelistCache.set(sharedKey(keyArgs), paths);
};
@ -63,12 +59,10 @@ export const setPackageInfo = ({
version,
packageInfo,
}: SharedKey & { packageInfo: ArchivePackage | RegistryPackage }) => {
const logger = appContextService.getLogger();
const key = sharedKey({ name, version });
appContextService
.getLogger()
.debug(
`setting package info to the cache for ${name}-${version}:\n${JSON.stringify(packageInfo)}`
);
logger.debug(`setting package info to the cache for ${name}-${version}`);
logger.trace(JSON.stringify(packageInfo));
return packageInfoCache.set(key, packageInfo);
};

218
x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/__snapshots__/install.test.ts.snap generated
View file

File diff suppressed because one or more lines are too long

45
x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.test.ts
View file

@ -11,6 +11,8 @@ import { readFileSync } from 'fs';
import glob from 'glob';
import { safeLoad } from 'js-yaml';
import type { FieldSpec } from 'src/plugins/data/common';
import type { Fields, Field } from '../../fields/field';
import {
@ -22,7 +24,6 @@ import {
createIndexPatternFields,
createIndexPattern,
} from './install';
import type { IndexPatternField } from './install';
import { dupeFields } from './tests/test_data';
// Add our own serialiser to just do JSON.stringify
@ -93,7 +94,6 @@ describe('creating index patterns from yaml fields', () => {
const mergedField = deduped.find((field) => field.name === '1');
expect(mergedField?.searchable).toBe(true);
expect(mergedField?.aggregatable).toBe(true);
expect(mergedField?.analyzed).toBe(true);
expect(mergedField?.count).toBe(0);
});
});
@ -156,7 +156,7 @@ describe('creating index patterns from yaml fields', () => {
{ fields: [{ name: 'testField', type: 'short' }], expect: 'number' },
{ fields: [{ name: 'testField', type: 'byte' }], expect: 'number' },
{ fields: [{ name: 'testField', type: 'keyword' }], expect: 'string' },
{ fields: [{ name: 'testField', type: 'invalidType' }], expect: undefined },
{ fields: [{ name: 'testField', type: 'invalidType' }], expect: 'string' },
{ fields: [{ name: 'testField', type: 'text' }], expect: 'string' },
{ fields: [{ name: 'testField', type: 'date' }], expect: 'date' },
{ fields: [{ name: 'testField', type: 'geo_point' }], expect: 'geo_point' },
@ -171,7 +171,7 @@ describe('creating index patterns from yaml fields', () => {
test('transformField changes values based on other values', () => {
interface TestWithAttr extends Test {
attr: keyof IndexPatternField;
attr: keyof FieldSpec;
}
const tests: TestWithAttr[] = [
@ -211,43 +211,6 @@ describe('creating index patterns from yaml fields', () => {
attr: 'aggregatable',
},
// analyzed
{ fields: [{ name }], expect: false, attr: 'analyzed' },
{ fields: [{ name, analyzed: true }], expect: true, attr: 'analyzed' },
{ fields: [{ name, analyzed: false }], expect: false, attr: 'analyzed' },
{ fields: [{ name, type: 'binary' }], expect: false, attr: 'analyzed' },
{ fields: [{ name, analyzed: true, type: 'binary' }], expect: false, attr: 'analyzed' },
{
fields: [{ name, analyzed: true, type: 'object', enabled: false }],
expect: false,
attr: 'analyzed',
},
// doc_values always set to true except for meta fields
{ fields: [{ name }], expect: true, attr: 'doc_values' },
{ fields: [{ name, doc_values: true }], expect: true, attr: 'doc_values' },
{ fields: [{ name, doc_values: false }], expect: false, attr: 'doc_values' },
{ fields: [{ name, script: 'doc[]' }], expect: false, attr: 'doc_values' },
{ fields: [{ name, doc_values: true, script: 'doc[]' }], expect: false, attr: 'doc_values' },
{ fields: [{ name, type: 'binary' }], expect: false, attr: 'doc_values' },
{ fields: [{ name, doc_values: true, type: 'binary' }], expect: true, attr: 'doc_values' },
{
fields: [{ name, doc_values: true, type: 'object', enabled: false }],
expect: false,
attr: 'doc_values',
},
// enabled - only applies to objects (and only if set)
{ fields: [{ name, type: 'binary', enabled: false }], expect: undefined, attr: 'enabled' },
{ fields: [{ name, type: 'binary', enabled: true }], expect: undefined, attr: 'enabled' },
{ fields: [{ name, type: 'object', enabled: true }], expect: true, attr: 'enabled' },
{ fields: [{ name, type: 'object', enabled: false }], expect: false, attr: 'enabled' },
{
fields: [{ name, type: 'object', enabled: false }],
expect: false,
attr: 'doc_values',
},
// indexed
{ fields: [{ name, type: 'binary' }], expect: false, attr: 'indexed' },
{

122
x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts
View file

@ -5,9 +5,9 @@
* 2.0.
*/
import type { SavedObjectsClientContract } from 'src/core/server';
import type { SavedObjectsClientContract, ElasticsearchClient } from 'src/core/server';
import type { FieldSpec } from 'src/plugins/data/common';
import { INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../../../../constants';
import { loadFieldsFromYaml } from '../../fields/field';
import type { Fields, Field } from '../../fields/field';
import { dataTypes, installationStatuses } from '../../../../../common/constants';
@ -17,6 +17,7 @@ import type {
InstallSource,
ValueOf,
} from '../../../../../common/types';
import { appContextService } from '../../../../services';
import type { RegistryPackage, DataType } from '../../../../types';
import { getInstallation, getPackageFromSource, getPackageSavedObjects } from '../../packages/get';
@ -59,29 +60,29 @@ const typeMap: TypeMap = {
constant_keyword: 'string',
};
export interface IndexPatternField {
name: string;
type?: string;
count: number;
scripted: boolean;
indexed: boolean;
analyzed: boolean;
searchable: boolean;
aggregatable: boolean;
doc_values: boolean;
enabled?: boolean;
script?: string;
lang?: string;
readFromDocValues: boolean;
}
const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';
export const indexPatternTypes = Object.values(dataTypes);
export async function installIndexPatterns(
savedObjectsClient: SavedObjectsClientContract,
pkgName?: string,
pkgVersion?: string,
installSource?: InstallSource
) {
export async function installIndexPatterns({
savedObjectsClient,
pkgName,
pkgVersion,
installSource,
}: {
savedObjectsClient: SavedObjectsClientContract;
esClient: ElasticsearchClient;
pkgName?: string;
pkgVersion?: string;
installSource?: InstallSource;
}) {
const logger = appContextService.getLogger();
logger.debug(
`kicking off installation of index patterns for ${
pkgName && pkgVersion ? `${pkgName}-${pkgVersion}` : 'no specific package'
}`
);
// get all user installed packages
const installedPackagesRes = await getPackageSavedObjects(savedObjectsClient);
const installedPackagesSavedObjects = installedPackagesRes.saved_objects.filter(
@ -115,6 +116,7 @@ export async function installIndexPatterns(
});
}
}
// get each package's registry info
const packagesToFetchPromise = packagesToFetch.map((pkg) =>
getPackageFromSource({
@ -125,27 +127,33 @@ export async function installIndexPatterns(
})
);
const packages = await Promise.all(packagesToFetchPromise);
// for each index pattern type, create an index pattern
indexPatternTypes.forEach(async (indexPatternType) => {
// if this is an update because a package is being uninstalled (no pkgkey argument passed) and no other packages are installed, remove the index pattern
if (!pkgName && installedPackagesSavedObjects.length === 0) {
try {
await savedObjectsClient.delete(INDEX_PATTERN_SAVED_OBJECT_TYPE, `${indexPatternType}-*`);
} catch (err) {
// index pattern was probably deleted by the user already
return Promise.all(
indexPatternTypes.map(async (indexPatternType) => {
// if this is an update because a package is being uninstalled (no pkgkey argument passed) and no other packages are installed, remove the index pattern
if (!pkgName && installedPackagesSavedObjects.length === 0) {
try {
logger.debug(`deleting index pattern ${indexPatternType}-*`);
await savedObjectsClient.delete(INDEX_PATTERN_SAVED_OBJECT_TYPE, `${indexPatternType}-*`);
} catch (err) {
// index pattern was probably deleted by the user already
}
return;
}
return;
}
const packagesWithInfo = packages.map((pkg) => pkg.packageInfo);
// get all data stream fields from all installed packages
const fields = await getAllDataStreamFieldsByType(packagesWithInfo, indexPatternType);
const kibanaIndexPattern = createIndexPattern(indexPatternType, fields);
// create or overwrite the index pattern
await savedObjectsClient.create(INDEX_PATTERN_SAVED_OBJECT_TYPE, kibanaIndexPattern, {
id: `${indexPatternType}-*`,
overwrite: true,
});
});
const packagesWithInfo = packages.map((pkg) => pkg.packageInfo);
// get all data stream fields from all installed packages
const fields = await getAllDataStreamFieldsByType(packagesWithInfo, indexPatternType);
const kibanaIndexPattern = createIndexPattern(indexPatternType, fields);
// create or overwrite the index pattern
await savedObjectsClient.create(INDEX_PATTERN_SAVED_OBJECT_TYPE, kibanaIndexPattern, {
id: `${indexPatternType}-*`,
overwrite: true,
});
logger.debug(`created index pattern ${kibanaIndexPattern.title}`);
})
);
}
// loops through all given packages and returns an array
@ -189,7 +197,7 @@ export const createIndexPattern = (indexPatternType: string, fields: Fields) =>
// and also returns the fieldFormatMap
export const createIndexPatternFields = (
fields: Fields
): { indexPatternFields: IndexPatternField[]; fieldFormatMap: FieldFormatMap } => {
): { indexPatternFields: FieldSpec[]; fieldFormatMap: FieldFormatMap } => {
const flattenedFields = flattenFields(fields);
const fieldFormatMap = createFieldFormatMap(flattenedFields);
const transformedFields = flattenedFields.map(transformField);
@ -198,8 +206,8 @@ export const createIndexPatternFields = (
};
// merges fields that are duplicates with the existing taking precedence
export const dedupeFields = (fields: IndexPatternField[]) => {
const uniqueObj = fields.reduce<{ [name: string]: IndexPatternField }>((acc, field) => {
export const dedupeFields = (fields: FieldSpec[]) => {
const uniqueObj = fields.reduce<{ [name: string]: FieldSpec }>((acc, field) => {
// if field doesn't exist yet
if (!acc[field.name]) {
acc[field.name] = field;
@ -251,34 +259,20 @@ const getField = (fields: Fields, pathNames: string[]): Field | undefined => {
return undefined;
};
export const transformField = (field: Field, i: number, fields: Fields): IndexPatternField => {
const newField: IndexPatternField = {
export const transformField = (field: Field, i: number, fields: Fields): FieldSpec => {
const newField: FieldSpec = {
name: field.name,
type: field.type && typeMap[field.type] ? typeMap[field.type] : 'string',
count: field.count ?? 0,
scripted: false,
indexed: field.index ?? true,
analyzed: field.analyzed ?? false,
searchable: field.searchable ?? true,
aggregatable: field.aggregatable ?? true,
doc_values: field.doc_values ?? true,
readFromDocValues: field.doc_values ?? true,
};
// if type exists, check if it exists in the map
if (field.type) {
// if no type match type is not set (undefined)
if (typeMap[field.type]) {
newField.type = typeMap[field.type];
}
// if type isn't set, default to string
} else {
newField.type = 'string';
}
if (newField.type === 'binary') {
newField.aggregatable = false;
newField.analyzed = false;
newField.doc_values = field.doc_values ?? false;
newField.readFromDocValues = field.doc_values ?? false;
newField.indexed = false;
newField.searchable = false;
@ -286,11 +280,8 @@ export const transformField = (field: Field, i: number, fields: Fields): IndexPa
if (field.type === 'object' && field.hasOwnProperty('enabled')) {
const enabled = field.enabled ?? true;
newField.enabled = enabled;
if (!enabled) {
newField.aggregatable = false;
newField.analyzed = false;
newField.doc_values = false;
newField.readFromDocValues = false;
newField.indexed = false;
newField.searchable = false;
@ -305,7 +296,6 @@ export const transformField = (field: Field, i: number, fields: Fields): IndexPa
newField.scripted = true;
newField.script = field.script;
newField.lang = 'painless';
newField.doc_values = false;
newField.readFromDocValues = false;
}

20
x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/test_data.ts
View file

@ -5,9 +5,9 @@
* 2.0.
*/
import type { IndexPatternField } from '../install';
import type { FieldSpec } from 'src/plugins/data/common';
export const dupeFields: IndexPatternField[] = [
export const dupeFields: FieldSpec[] = [
{
name: '1',
type: 'integer',
@ -15,10 +15,8 @@ export const dupeFields: IndexPatternField[] = [
aggregatable: true,
count: 0,
indexed: true,
doc_values: true,
readFromDocValues: true,
scripted: false,
analyzed: true,
},
{
name: '2',
@ -27,10 +25,8 @@ export const dupeFields: IndexPatternField[] = [
aggregatable: true,
count: 0,
indexed: true,
doc_values: true,
readFromDocValues: true,
scripted: false,
analyzed: true,
},
{
name: '3',
@ -39,10 +35,8 @@ export const dupeFields: IndexPatternField[] = [
aggregatable: true,
count: 0,
indexed: true,
doc_values: true,
readFromDocValues: true,
scripted: false,
analyzed: true,
},
{
name: '1',
@ -51,10 +45,8 @@ export const dupeFields: IndexPatternField[] = [
aggregatable: false,
count: 2,
indexed: true,
doc_values: true,
readFromDocValues: true,
scripted: false,
analyzed: true,
},
{
name: '1.1',
@ -63,10 +55,8 @@ export const dupeFields: IndexPatternField[] = [
aggregatable: false,
count: 0,
indexed: true,
doc_values: true,
readFromDocValues: true,
scripted: false,
analyzed: true,
},
{
name: '4',
@ -75,10 +65,8 @@ export const dupeFields: IndexPatternField[] = [
aggregatable: false,
count: 0,
indexed: true,
doc_values: true,
readFromDocValues: true,
scripted: false,
analyzed: true,
},
{
name: '2',
@ -87,10 +75,8 @@ export const dupeFields: IndexPatternField[] = [
aggregatable: false,
count: 0,
indexed: true,
doc_values: true,
readFromDocValues: true,
scripted: false,
analyzed: true,
},
{
name: '1',
@ -99,9 +85,7 @@ export const dupeFields: IndexPatternField[] = [
aggregatable: false,
count: 1,
indexed: true,
doc_values: true,
readFromDocValues: true,
scripted: false,
analyzed: false,
},
];

15
x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
View file

@ -12,7 +12,6 @@ import type { InstallablePackage, InstallSource, PackageAssetReference } from '.
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
import { ElasticsearchAssetType } from '../../../types';
import type { AssetReference, Installation, InstallType } from '../../../types';
import { installIndexPatterns } from '../kibana/index_pattern/install';
import { installTemplates } from '../elasticsearch/template/install';
import { installPipelines, deletePreviousPipelines } from '../elasticsearch/ingest_pipeline/';
import { installILMPolicy } from '../elasticsearch/ilm/install';
@ -81,11 +80,11 @@ export async function _installPackage({
});
}
// kick off `installIndexPatterns` & `installKibanaAssets` as early as possible because they're the longest running operations
// kick off `installKibanaAssets` as early as possible because they're the longest running operations
// we don't `await` here because we don't want to delay starting the many other `install*` functions
// however, without an `await` or a `.catch` we haven't defined how to handle a promise rejection
// we define it many lines and potentially seconds of wall clock time later in
// `await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]);`
// `await installKibanaAssetsPromise`
// if we encounter an error before we there, we'll have an "unhandled rejection" which causes its own problems
// the program will log something like this _and exit/crash_
// Unhandled Promise rejection detected:
@ -96,13 +95,6 @@ export async function _installPackage({
// add a `.catch` to prevent the "unhandled rejection" case
// in that `.catch`, set something that indicates a failure
// check for that failure later and act accordingly (throw, ignore, return)
let installIndexPatternError;
const installIndexPatternPromise = installIndexPatterns(
savedObjectsClient,
pkgName,
pkgVersion,
installSource
).catch((reason) => (installIndexPatternError = reason));
const kibanaAssets = await getKibanaAssets(paths);
if (installedPkg)
await deleteKibanaSavedObjectsAssets(
@ -184,9 +176,8 @@ export async function _installPackage({
}));
// make sure the assets are installed (or didn't error)
if (installIndexPatternError) throw installIndexPatternError;
if (installKibanaAssetsError) throw installKibanaAssetsError;
await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]);
await installKibanaAssetsPromise;
const packageAssetResults = await saveArchiveEntries({
savedObjectsClient,

88
x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts
View file

@ -7,58 +7,72 @@
import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server';
import { appContextService } from '../../app_context';
import * as Registry from '../registry';
import { installIndexPatterns } from '../kibana/index_pattern/install';
import { getInstallationObject } from './index';
import { upgradePackage } from './install';
import { installPackage } from './install';
import type { BulkInstallResponse, IBulkInstallPackageError } from './install';
interface BulkInstallPackagesParams {
savedObjectsClient: SavedObjectsClientContract;
packagesToUpgrade: string[];
packagesToInstall: string[];
esClient: ElasticsearchClient;
}
export async function bulkInstallPackages({
savedObjectsClient,
packagesToUpgrade,
packagesToInstall,
esClient,
}: BulkInstallPackagesParams): Promise<BulkInstallResponse[]> {
const installedAndLatestPromises = packagesToUpgrade.map((pkgToUpgrade) =>
Promise.all([
getInstallationObject({ savedObjectsClient, pkgName: pkgToUpgrade }),
Registry.fetchFindLatestPackage(pkgToUpgrade),
])
const logger = appContextService.getLogger();
const installSource = 'registry';
const latestPackagesResults = await Promise.allSettled(
packagesToInstall.map((packageName) => Registry.fetchFindLatestPackage(packageName))
);
const installedAndLatestResults = await Promise.allSettled(installedAndLatestPromises);
const installResponsePromises = installedAndLatestResults.map(async (result, index) => {
const pkgToUpgrade = packagesToUpgrade[index];
if (result.status === 'fulfilled') {
const [installedPkg, latestPkg] = result.value;
return upgradePackage({
savedObjectsClient,
esClient,
installedPkg,
latestPkg,
pkgToUpgrade,
});
} else {
return { name: pkgToUpgrade, error: result.reason };
}
});
const installResults = await Promise.allSettled(installResponsePromises);
const installResponses = installResults.map((result, index) => {
const pkgToUpgrade = packagesToUpgrade[index];
if (result.status === 'fulfilled') {
return result.value;
} else {
return { name: pkgToUpgrade, error: result.reason };
}
});
return installResponses;
logger.debug(`kicking off bulk install of ${packagesToInstall.join(', ')} from registry`);
const installResults = await Promise.allSettled(
latestPackagesResults.map(async (result, index) => {
const packageName = packagesToInstall[index];
if (result.status === 'fulfilled') {
const latestPackage = result.value;
return {
name: packageName,
version: latestPackage.version,
result: await installPackage({
savedObjectsClient,
esClient,
pkgkey: Registry.pkgToPkgKey(latestPackage),
installSource,
skipPostInstall: true,
}),
};
}
return { name: packageName, error: result.reason };
})
);
// only install index patterns if we completed install for any package-version for the
// first time, aka fresh installs or upgrades
if (
installResults.find(
(result) => result.status === 'fulfilled' && result.value.result?.status === 'installed'
)
) {
await installIndexPatterns({ savedObjectsClient, esClient, installSource });
}
return installResults.map((result, index) => {
const packageName = packagesToInstall[index];
return result.status === 'fulfilled'
? result.value
: { name: packageName, error: result.reason };
});
}
export function isBulkInstallError(test: any): test is IBulkInstallPackageError {
return 'error' in test && test.error instanceof Error;
export function isBulkInstallError(
installResponse: any
): installResponse is IBulkInstallPackageError {
return 'error' in installResponse && installResponse.error instanceof Error;
}

25
x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts
View file

@ -77,9 +77,8 @@ describe('ensureInstalledDefaultPackages', () => {
return [
{
name: mockInstallation.attributes.name,
assets: [],
newVersion: '',
oldVersion: '',
result: { assets: [], status: 'installed' },
version: '',
statusCode: 200,
},
];
@ -96,16 +95,14 @@ describe('ensureInstalledDefaultPackages', () => {
return [
{
name: 'success one',
assets: [],
newVersion: '',
oldVersion: '',
result: { assets: [], status: 'installed' },
version: '',
statusCode: 200,
},
{
name: 'success two',
assets: [],
newVersion: '',
oldVersion: '',
result: { assets: [], status: 'installed' },
version: '',
statusCode: 200,
},
{
@ -114,9 +111,8 @@ describe('ensureInstalledDefaultPackages', () => {
},
{
name: 'success three',
assets: [],
newVersion: '',
oldVersion: '',
result: { assets: [], status: 'installed' },
version: '',
statusCode: 200,
},
{
@ -138,9 +134,8 @@ describe('ensureInstalledDefaultPackages', () => {
return [
{
name: 'undefined package',
assets: [],
newVersion: '',
oldVersion: '',
result: { assets: [], status: 'installed' },
version: '',
statusCode: 200,
},
];

239
x-pack/plugins/fleet/server/services/epm/packages/install.ts
View file

@ -5,10 +5,8 @@
* 2.0.
*/
import semverGt from 'semver/functions/gt';
import semverLt from 'semver/functions/lt';
import type Boom from '@hapi/boom';
import type { UnwrapPromise } from '@kbn/utility-types';
import type { ElasticsearchClient, SavedObject, SavedObjectsClientContract } from 'src/core/server';
import { generateESIndexPatterns } from '../elasticsearch/template/template';
@ -28,12 +26,14 @@ import type {
AssetType,
EsAssetReference,
InstallType,
InstallResult,
} from '../../../types';
import { appContextService } from '../../app_context';
import * as Registry from '../registry';
import { setPackageInfo, parseAndVerifyArchiveEntries, unpackBufferToCache } from '../archive';
import { toAssetReference } from '../kibana/assets/install';
import type { ArchiveAsset } from '../kibana/assets/install';
import { installIndexPatterns } from '../kibana/index_pattern/install';
import {
isRequiredPackage,
@ -63,7 +63,7 @@ export async function installLatestPackage(options: {
savedObjectsClient,
pkgkey,
esClient,
});
}).then(({ assets }) => assets);
} catch (err) {
throw err;
}
@ -76,7 +76,7 @@ export async function ensureInstalledDefaultPackages(
const installations = [];
const bulkResponse = await bulkInstallPackages({
savedObjectsClient,
packagesToUpgrade: Object.values(defaultPackages),
packagesToInstall: Object.values(defaultPackages),
esClient,
});
@ -164,6 +164,7 @@ export async function handleInstallPackageFailure({
savedObjectsClient,
pkgkey: prevVersion,
esClient,
force: true,
});
}
} catch (e) {
@ -177,64 +178,6 @@ export interface IBulkInstallPackageError {
}
export type BulkInstallResponse = BulkInstallPackageInfo | IBulkInstallPackageError;
interface UpgradePackageParams {
savedObjectsClient: SavedObjectsClientContract;
esClient: ElasticsearchClient;
installedPkg: UnwrapPromise<ReturnType<typeof getInstallationObject>>;
latestPkg: UnwrapPromise<ReturnType<typeof Registry.fetchFindLatestPackage>>;
pkgToUpgrade: string;
}
export async function upgradePackage({
savedObjectsClient,
esClient,
installedPkg,
latestPkg,
pkgToUpgrade,
}: UpgradePackageParams): Promise<BulkInstallResponse> {
if (!installedPkg || semverGt(latestPkg.version, installedPkg.attributes.version)) {
const pkgkey = Registry.pkgToPkgKey({
name: latestPkg.name,
version: latestPkg.version,
});
try {
const assets = await installPackage({
installSource: 'registry',
savedObjectsClient,
pkgkey,
esClient,
});
return {
name: pkgToUpgrade,
newVersion: latestPkg.version,
oldVersion: installedPkg?.attributes.version ?? null,
assets,
};
} catch (installFailed) {
await handleInstallPackageFailure({
savedObjectsClient,
error: installFailed,
pkgName: latestPkg.name,
pkgVersion: latestPkg.version,
installedPkg,
esClient,
});
return { name: pkgToUpgrade, error: installFailed };
}
} else {
// package was already at the latest version
return {
name: pkgToUpgrade,
newVersion: latestPkg.version,
oldVersion: latestPkg.version,
assets: [
...installedPkg.attributes.installed_es,
...installedPkg.attributes.installed_kibana,
],
};
}
}
interface InstallRegistryPackageParams {
savedObjectsClient: SavedObjectsClientContract;
pkgkey: string;
@ -247,32 +190,81 @@ async function installPackageFromRegistry({
pkgkey,
esClient,
force = false,
}: InstallRegistryPackageParams): Promise<AssetReference[]> {
}: InstallRegistryPackageParams): Promise<InstallResult> {
const logger = appContextService.getLogger();
// TODO: change epm API to /packageName/version so we don't need to do this
const { pkgName, pkgVersion } = Registry.splitPkgKey(pkgkey);
// get the currently installed package
const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
const installType = getInstallType({ pkgVersion, installedPkg });
// get latest package version
const latestPackage = await Registry.fetchFindLatestPackage(pkgName);
// let the user install if using the force flag or needing to reinstall or install a previous version due to failed update
const installOutOfDateVersionOk =
installType === 'reinstall' || installType === 'reupdate' || installType === 'rollback';
force || ['reinstall', 'reupdate', 'rollback'].includes(installType);
const latestPackage = await Registry.fetchFindLatestPackage(pkgName);
if (semverLt(pkgVersion, latestPackage.version) && !force && !installOutOfDateVersionOk) {
throw new PackageOutdatedError(`${pkgkey} is out-of-date and cannot be installed or updated`);
// if the requested version is the same as installed version, check if we allow it based on
// current installed package status and force flag, if we don't allow it,
// just return the asset references from the existing installation
if (
installedPkg?.attributes.version === pkgVersion &&
installedPkg?.attributes.install_status === 'installed'
) {
if (!force) {
logger.debug(`${pkgkey} is already installed, skipping installation`);
return {
assets: [
...installedPkg.attributes.installed_es,
...installedPkg.attributes.installed_kibana,
],
status: 'already_installed',
};
}
}
// if the requested version is out-of-date of the latest package version, check if we allow it
// if we don't allow it, return an error
if (semverLt(pkgVersion, latestPackage.version)) {
if (!installOutOfDateVersionOk) {
throw new PackageOutdatedError(`${pkgkey} is out-of-date and cannot be installed or updated`);
}
logger.debug(
`${pkgkey} is out-of-date, installing anyway due to ${
force ? 'force flag' : `install type ${installType}`
}`
);
}
// get package info
const { paths, packageInfo } = await Registry.getRegistryPackage(pkgName, pkgVersion);
return _installPackage({
savedObjectsClient,
esClient,
installedPkg,
paths,
packageInfo,
installType,
installSource: 'registry',
});
// try installing the package, if there was an error, call error handler and rethrow
try {
return _installPackage({
savedObjectsClient,
esClient,
installedPkg,
paths,
packageInfo,
installType,
installSource: 'registry',
}).then((assets) => {
return { assets, status: 'installed' };
});
} catch (e) {
await handleInstallPackageFailure({
savedObjectsClient,
error: e,
pkgName,
pkgVersion,
installedPkg,
esClient,
});
throw e;
}
}
interface InstallUploadedArchiveParams {
@ -282,16 +274,12 @@ interface InstallUploadedArchiveParams {
contentType: string;
}
export type InstallPackageParams =
| ({ installSource: Extract<InstallSource, 'registry'> } & InstallRegistryPackageParams)
| ({ installSource: Extract<InstallSource, 'upload'> } & InstallUploadedArchiveParams);
async function installPackageByUpload({
savedObjectsClient,
esClient,
archiveBuffer,
contentType,
}: InstallUploadedArchiveParams): Promise<AssetReference[]> {
}: InstallUploadedArchiveParams): Promise<InstallResult> {
const { packageInfo } = await parseAndVerifyArchiveEntries(archiveBuffer, contentType);
const installedPkg = await getInstallationObject({
@ -329,32 +317,68 @@ async function installPackageByUpload({
packageInfo,
installType,
installSource,
}).then((assets) => {
return { assets, status: 'installed' };
});
}
export type InstallPackageParams = {
skipPostInstall?: boolean;
} & (
| ({ installSource: Extract<InstallSource, 'registry'> } & InstallRegistryPackageParams)
| ({ installSource: Extract<InstallSource, 'upload'> } & InstallUploadedArchiveParams)
);
export async function installPackage(args: InstallPackageParams) {
if (!('installSource' in args)) {
throw new Error('installSource is required');
}
const logger = appContextService.getLogger();
const { savedObjectsClient, esClient, skipPostInstall = false, installSource } = args;
if (args.installSource === 'registry') {
const { savedObjectsClient, pkgkey, esClient, force } = args;
return installPackageFromRegistry({
const { pkgkey, force } = args;
const { pkgName, pkgVersion } = Registry.splitPkgKey(pkgkey);
logger.debug(`kicking off install of ${pkgkey} from registry`);
const response = installPackageFromRegistry({
savedObjectsClient,
pkgkey,
esClient,
force,
}).then(async (installResult) => {
if (skipPostInstall) {
return installResult;
}
logger.debug(`install of ${pkgkey} finished, running post-install`);
return installIndexPatterns({
savedObjectsClient,
esClient,
pkgName,
pkgVersion,
installSource,
}).then(() => installResult);
});
return response;
} else if (args.installSource === 'upload') {
const { savedObjectsClient, esClient, archiveBuffer, contentType } = args;
return installPackageByUpload({
const { archiveBuffer, contentType } = args;
logger.debug(`kicking off install of uploaded package`);
const response = installPackageByUpload({
savedObjectsClient,
esClient,
archiveBuffer,
contentType,
}).then(async (installResult) => {
if (skipPostInstall) {
return installResult;
}
logger.debug(`install of uploaded package finished, running post-install`);
return installIndexPatterns({
savedObjectsClient,
esClient,
installSource,
}).then(() => installResult);
});
return response;
}
// @ts-expect-error s/b impossibe b/c `never` by this point, but just in case
throw new Error(`Unknown installSource: ${args.installSource}`);
@ -451,26 +475,27 @@ export async function ensurePackagesCompletedInstall(
searchFields: ['install_status'],
search: 'installing',
});
const installingPromises = installingPackages.saved_objects.reduce<
Array<Promise<AssetReference[]>>
>((acc, pkg) => {
const startDate = pkg.attributes.install_started_at;
const nowDate = new Date().toISOString();
const elapsedTime = Date.parse(nowDate) - Date.parse(startDate);
const pkgkey = `${pkg.attributes.name}-${pkg.attributes.install_version}`;
// reinstall package
if (elapsedTime > MAX_TIME_COMPLETE_INSTALL) {
acc.push(
installPackage({
installSource: 'registry',
savedObjectsClient,
pkgkey,
esClient,
})
);
}
return acc;
}, []);
const installingPromises = installingPackages.saved_objects.reduce<Array<Promise<InstallResult>>>(
(acc, pkg) => {
const startDate = pkg.attributes.install_started_at;
const nowDate = new Date().toISOString();
const elapsedTime = Date.parse(nowDate) - Date.parse(startDate);
const pkgkey = `${pkg.attributes.name}-${pkg.attributes.install_version}`;
// reinstall package
if (elapsedTime > MAX_TIME_COMPLETE_INSTALL) {
acc.push(
installPackage({
installSource: 'registry',
savedObjectsClient,
pkgkey,
esClient,
})
);
}
return acc;
},
[]
);
await Promise.all(installingPromises);
return installingPackages;
}

2
x-pack/plugins/fleet/server/services/epm/packages/remove.ts
View file

@ -63,7 +63,7 @@ export async function removeInstallation(options: {
// recreate or delete index patterns when a package is uninstalled
// this must be done after deleting the saved object for the current package otherwise it will retrieve the package
// from the registry again and reinstall the index patterns
await installIndexPatterns(savedObjectsClient);
await installIndexPatterns({ savedObjectsClient, esClient });
// remove the package archive and its contents from the cache so that a reinstall fetches
// a fresh copy from the registry

1
x-pack/plugins/fleet/server/types/index.tsx
View file

@ -72,6 +72,7 @@ export {
SettingsSOAttributes,
InstallType,
InstallSource,
InstallResult,
// Agent Request types
PostAgentEnrollRequest,
PostAgentCheckinRequest,

15
x-pack/test/fleet_api_integration/apis/epm/bulk_upgrade.ts
View file

@ -51,8 +51,7 @@ export default function (providerContext: FtrProviderContext) {
expect(body.response.length).equal(1);
expect(body.response[0].name).equal('multiple_versions');
const entry = body.response[0] as BulkInstallPackageInfo;
expect(entry.oldVersion).equal('0.1.0');
expect(entry.newVersion).equal('0.3.0');
expect(entry.version).equal('0.3.0');
});
it('should return an error for packages that do not exist', async function () {
const { body }: { body: BulkInstallPackagesResponse } = await supertest
@ -63,8 +62,7 @@ export default function (providerContext: FtrProviderContext) {
expect(body.response.length).equal(2);
expect(body.response[0].name).equal('multiple_versions');
const entry = body.response[0] as BulkInstallPackageInfo;
expect(entry.oldVersion).equal('0.1.0');
expect(entry.newVersion).equal('0.3.0');
expect(entry.version).equal('0.3.0');
const err = body.response[1] as IBulkInstallPackageHTTPError;
expect(err.statusCode).equal(404);
@ -79,12 +77,10 @@ export default function (providerContext: FtrProviderContext) {
expect(body.response.length).equal(2);
expect(body.response[0].name).equal('multiple_versions');
let entry = body.response[0] as BulkInstallPackageInfo;
expect(entry.oldVersion).equal('0.1.0');
expect(entry.newVersion).equal('0.3.0');
expect(entry.version).equal('0.3.0');
entry = body.response[1] as BulkInstallPackageInfo;
expect(entry.oldVersion).equal(null);
expect(entry.newVersion).equal('0.1.0');
expect(entry.version).equal('0.1.0');
expect(entry.name).equal('overrides');
});
});
@ -103,8 +99,7 @@ export default function (providerContext: FtrProviderContext) {
expect(body.response.length).equal(1);
expect(body.response[0].name).equal('multiple_versions');
const entry = body.response[0] as BulkInstallPackageInfo;
expect(entry.oldVersion).equal(null);
expect(entry.newVersion).equal('0.3.0');
expect(entry.version).equal('0.3.0');
});
});
});