[Security Solution][Endpoint] Policy creation callback fixes + Improved error handling in user manifest loop (#71269)
* Clean up matcher types * Rework promise and error-handling in ManifestManager * Write tests for ingest callback and ensure policy is returned when errors occur * More tests for ingest callback * Update tests * Fix tests Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Madison Caldwell
GitHub
parent
b24632da54
commit
3fc54e7c55
43
x-pack/plugins/ingest_manager/common/mocks.ts
Normal file
43
x-pack/plugins/ingest_manager/common/mocks.ts
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { NewPackageConfig, PackageConfig } from './types/models/package_config';
|
||||
|
||||
export const createNewPackageConfigMock = () => {
|
||||
return {
|
||||
name: 'endpoint-1',
|
||||
description: '',
|
||||
namespace: 'default',
|
||||
enabled: true,
|
||||
config_id: '93c46720-c217-11ea-9906-b5b8a21b268e',
|
||||
output_id: '',
|
||||
package: {
|
||||
name: 'endpoint',
|
||||
title: 'Elastic Endpoint',
|
||||
version: '0.9.0',
|
||||
},
|
||||
inputs: [],
|
||||
} as NewPackageConfig;
|
||||
};
|
||||
|
||||
export const createPackageConfigMock = () => {
|
||||
const newPackageConfig = createNewPackageConfigMock();
|
||||
return {
|
||||
...newPackageConfig,
|
||||
id: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1',
|
||||
version: 'abcd',
|
||||
revision: 1,
|
||||
updated_at: '2020-06-25T16:03:38.159292',
|
||||
updated_by: 'kibana',
|
||||
created_at: '2020-06-25T16:03:38.159292',
|
||||
created_by: 'kibana',
|
||||
inputs: [
|
||||
{
|
||||
config: {},
|
||||
},
|
||||
],
|
||||
} as PackageConfig;
|
||||
};
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import { loggerMock } from 'src/core/server/logging/logger.mock';
|
||||
import { createNewPackageConfigMock } from '../../../ingest_manager/common/mocks';
|
||||
import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config';
|
||||
import { getManifestManagerMock } from './services/artifacts/manifest_manager/manifest_manager.mock';
|
||||
import { getPackageConfigCreateCallback } from './ingest_integration';
|
||||
|
||||
describe('ingest_integration tests ', () => {
|
||||
describe('ingest_integration sanity checks', () => {
|
||||
test('policy is updated with manifest', async () => {
|
||||
const logger = loggerMock.create();
|
||||
const manifestManager = getManifestManagerMock();
|
||||
const callback = getPackageConfigCreateCallback(logger, manifestManager);
|
||||
const policyConfig = createNewPackageConfigMock();
|
||||
const newPolicyConfig = await callback(policyConfig);
|
||||
expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint');
|
||||
expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory());
|
||||
expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual({
|
||||
artifacts: {
|
||||
'endpoint-exceptionlist-linux-v1': {
|
||||
compression_algorithm: 'zlib',
|
||||
decoded_sha256: '1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc',
|
||||
decoded_size: 287,
|
||||
encoded_sha256: 'c3dec543df1177561ab2aa74a37997ea3c1d748d532a597884f5a5c16670d56c',
|
||||
encoded_size: 133,
|
||||
encryption_algorithm: 'none',
|
||||
relative_url:
|
||||
'/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc',
|
||||
},
|
||||
},
|
||||
manifest_version: 'WzAsMF0=',
|
||||
schema_version: 'v1',
|
||||
});
|
||||
});
|
||||
|
||||
test('policy is returned even if error is encountered during artifact sync', async () => {
|
||||
const logger = loggerMock.create();
|
||||
const manifestManager = getManifestManagerMock();
|
||||
manifestManager.syncArtifacts = jest.fn().mockRejectedValue([new Error('error updating')]);
|
||||
const lastDispatched = await manifestManager.getLastDispatchedManifest();
|
||||
const callback = getPackageConfigCreateCallback(logger, manifestManager);
|
||||
const policyConfig = createNewPackageConfigMock();
|
||||
const newPolicyConfig = await callback(policyConfig);
|
||||
expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint');
|
||||
expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory());
|
||||
expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual(
|
||||
lastDispatched.toEndpointFormat()
|
||||
);
|
||||
});
|
||||
|
||||
test('initial policy creation succeeds if snapshot retrieval fails', async () => {
|
||||
const logger = loggerMock.create();
|
||||
const manifestManager = getManifestManagerMock();
|
||||
const lastDispatched = await manifestManager.getLastDispatchedManifest();
|
||||
manifestManager.getSnapshot = jest.fn().mockResolvedValue(null);
|
||||
const callback = getPackageConfigCreateCallback(logger, manifestManager);
|
||||
const policyConfig = createNewPackageConfigMock();
|
||||
const newPolicyConfig = await callback(policyConfig);
|
||||
expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint');
|
||||
expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory());
|
||||
expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual(
|
||||
lastDispatched.toEndpointFormat()
|
||||
);
|
||||
});
|
||||
|
||||
test('subsequent policy creations succeed', async () => {
|
||||
const logger = loggerMock.create();
|
||||
const manifestManager = getManifestManagerMock();
|
||||
const snapshot = await manifestManager.getSnapshot();
|
||||
manifestManager.getLastDispatchedManifest = jest.fn().mockResolvedValue(snapshot!.manifest);
|
||||
manifestManager.getSnapshot = jest.fn().mockResolvedValue({
|
||||
manifest: snapshot!.manifest,
|
||||
diffs: [],
|
||||
});
|
||||
const callback = getPackageConfigCreateCallback(logger, manifestManager);
|
||||
const policyConfig = createNewPackageConfigMock();
|
||||
const newPolicyConfig = await callback(policyConfig);
|
||||
expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint');
|
||||
expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory());
|
||||
expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual(
|
||||
snapshot!.manifest.toEndpointFormat()
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -8,7 +8,9 @@ import { Logger } from '../../../../../src/core/server';
|
|||
import { NewPackageConfig } from '../../../ingest_manager/common/types/models';
|
||||
import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config';
|
||||
import { NewPolicyData } from '../../common/endpoint/types';
|
||||
import { ManifestManager } from './services/artifacts';
|
||||
import { ManifestManager, ManifestSnapshot } from './services/artifacts';
|
||||
import { reportErrors, ManifestConstants } from './lib/artifacts/common';
|
||||
import { ManifestSchemaVersion } from '../../common/endpoint/schema/common';
|
||||
|
||||
/**
|
||||
* Callback to handle creation of PackageConfigs in Ingest Manager
|
||||
|
|
@ -29,58 +31,83 @@ export const getPackageConfigCreateCallback = (
|
|||
// follow the types/schema expected
|
||||
let updatedPackageConfig = newPackageConfig as NewPolicyData;
|
||||
|
||||
// get snapshot based on exception-list-agnostic SOs
|
||||
// with diffs from last dispatched manifest, if it exists
|
||||
const snapshot = await manifestManager.getSnapshot({ initialize: true });
|
||||
// get current manifest from SO (last dispatched)
|
||||
const manifest = (
|
||||
await manifestManager.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION)
|
||||
)?.toEndpointFormat() ?? {
|
||||
manifest_version: 'default',
|
||||
schema_version: ManifestConstants.SCHEMA_VERSION as ManifestSchemaVersion,
|
||||
artifacts: {},
|
||||
};
|
||||
|
||||
if (snapshot === null) {
|
||||
logger.warn('No manifest snapshot available.');
|
||||
return updatedPackageConfig;
|
||||
}
|
||||
|
||||
if (snapshot.diffs.length > 0) {
|
||||
// create new artifacts
|
||||
await manifestManager.syncArtifacts(snapshot, 'add');
|
||||
|
||||
// Until we get the Default Policy Configuration in the Endpoint package,
|
||||
// we will add it here manually at creation time.
|
||||
// @ts-ignore
|
||||
if (newPackageConfig.inputs.length === 0) {
|
||||
updatedPackageConfig = {
|
||||
...newPackageConfig,
|
||||
inputs: [
|
||||
{
|
||||
type: 'endpoint',
|
||||
enabled: true,
|
||||
streams: [],
|
||||
config: {
|
||||
artifact_manifest: {
|
||||
value: snapshot.manifest.toEndpointFormat(),
|
||||
},
|
||||
policy: {
|
||||
value: policyConfigFactory(),
|
||||
},
|
||||
// Until we get the Default Policy Configuration in the Endpoint package,
|
||||
// we will add it here manually at creation time.
|
||||
if (newPackageConfig.inputs.length === 0) {
|
||||
updatedPackageConfig = {
|
||||
...newPackageConfig,
|
||||
inputs: [
|
||||
{
|
||||
type: 'endpoint',
|
||||
enabled: true,
|
||||
streams: [],
|
||||
config: {
|
||||
artifact_manifest: {
|
||||
value: manifest,
|
||||
},
|
||||
policy: {
|
||||
value: policyConfigFactory(),
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
let snapshot: ManifestSnapshot | null = null;
|
||||
let success = true;
|
||||
try {
|
||||
// Try to get most up-to-date manifest data.
|
||||
|
||||
// get snapshot based on exception-list-agnostic SOs
|
||||
// with diffs from last dispatched manifest, if it exists
|
||||
snapshot = await manifestManager.getSnapshot({ initialize: true });
|
||||
|
||||
if (snapshot && snapshot.diffs.length) {
|
||||
// create new artifacts
|
||||
const errors = await manifestManager.syncArtifacts(snapshot, 'add');
|
||||
if (errors.length) {
|
||||
reportErrors(logger, errors);
|
||||
throw new Error('Error writing new artifacts.');
|
||||
}
|
||||
}
|
||||
|
||||
if (snapshot) {
|
||||
updatedPackageConfig.inputs[0].config.artifact_manifest = {
|
||||
value: snapshot.manifest.toEndpointFormat(),
|
||||
};
|
||||
}
|
||||
|
||||
return updatedPackageConfig;
|
||||
} catch (err) {
|
||||
success = false;
|
||||
logger.error(err);
|
||||
return updatedPackageConfig;
|
||||
} finally {
|
||||
if (snapshot.diffs.length > 0) {
|
||||
// TODO: let's revisit the way this callback happens... use promises?
|
||||
// only commit when we know the package config was created
|
||||
if (success && snapshot !== null) {
|
||||
try {
|
||||
await manifestManager.commit(snapshot.manifest);
|
||||
if (snapshot.diffs.length > 0) {
|
||||
// TODO: let's revisit the way this callback happens... use promises?
|
||||
// only commit when we know the package config was created
|
||||
await manifestManager.commit(snapshot.manifest);
|
||||
|
||||
// clean up old artifacts
|
||||
await manifestManager.syncArtifacts(snapshot, 'delete');
|
||||
// clean up old artifacts
|
||||
await manifestManager.syncArtifacts(snapshot, 'delete');
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error(err);
|
||||
}
|
||||
} else if (snapshot === null) {
|
||||
logger.error('No manifest snapshot available.');
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
import { Logger } from 'src/core/server';
|
||||
|
||||
export const ArtifactConstants = {
|
||||
GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist',
|
||||
|
|
@ -16,3 +17,9 @@ export const ManifestConstants = {
|
|||
SCHEMA_VERSION: 'v1',
|
||||
INITIAL_VERSION: 'WzAsMF0=',
|
||||
};
|
||||
|
||||
export const reportErrors = (logger: Logger, errors: Error[]) => {
|
||||
errors.forEach((err) => {
|
||||
logger.error(err);
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import {
|
|||
TaskManagerStartContract,
|
||||
} from '../../../../../task_manager/server';
|
||||
import { EndpointAppContext } from '../../types';
|
||||
import { reportErrors } from './common';
|
||||
|
||||
export const ManifestTaskConstants = {
|
||||
TIMEOUT: '1m',
|
||||
|
|
@ -88,19 +89,36 @@ export class ManifestTask {
|
|||
return;
|
||||
}
|
||||
|
||||
let errors: Error[] = [];
|
||||
try {
|
||||
// get snapshot based on exception-list-agnostic SOs
|
||||
// with diffs from last dispatched manifest
|
||||
const snapshot = await manifestManager.getSnapshot();
|
||||
if (snapshot && snapshot.diffs.length > 0) {
|
||||
// create new artifacts
|
||||
await manifestManager.syncArtifacts(snapshot, 'add');
|
||||
errors = await manifestManager.syncArtifacts(snapshot, 'add');
|
||||
if (errors.length) {
|
||||
reportErrors(this.logger, errors);
|
||||
throw new Error('Error writing new artifacts.');
|
||||
}
|
||||
// write to ingest-manager package config
|
||||
await manifestManager.dispatch(snapshot.manifest);
|
||||
errors = await manifestManager.dispatch(snapshot.manifest);
|
||||
if (errors.length) {
|
||||
reportErrors(this.logger, errors);
|
||||
throw new Error('Error dispatching manifest.');
|
||||
}
|
||||
// commit latest manifest state to user-artifact-manifest SO
|
||||
await manifestManager.commit(snapshot.manifest);
|
||||
const error = await manifestManager.commit(snapshot.manifest);
|
||||
if (error) {
|
||||
reportErrors(this.logger, [error]);
|
||||
throw new Error('Error committing manifest.');
|
||||
}
|
||||
// clean up old artifacts
|
||||
await manifestManager.syncArtifacts(snapshot, 'delete');
|
||||
errors = await manifestManager.syncArtifacts(snapshot, 'delete');
|
||||
if (errors.length) {
|
||||
reportErrors(this.logger, errors);
|
||||
throw new Error('Error cleaning up outdated artifacts.');
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error(err);
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@
|
|||
|
||||
import { ILegacyScopedClusterClient, SavedObjectsClientContract } from 'kibana/server';
|
||||
import { loggingSystemMock, savedObjectsServiceMock } from 'src/core/server/mocks';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import { loggerMock } from 'src/core/server/logging/logger.mock';
|
||||
import { xpackMocks } from '../../../../mocks';
|
||||
import {
|
||||
AgentService,
|
||||
|
|
@ -63,8 +65,8 @@ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked<
|
|||
> => {
|
||||
return {
|
||||
agentService: createMockAgentService(),
|
||||
logger: loggerMock.create(),
|
||||
savedObjectsStart: savedObjectsServiceMock.createStartContract(),
|
||||
// @ts-ignore
|
||||
manifestManager: getManifestManagerMock(),
|
||||
registerIngestCallback: jest.fn<
|
||||
ReturnType<IngestManagerStartContract['registerExternalCallback']>,
|
||||
|
|
|
|||
|
|
@ -7,38 +7,38 @@
|
|||
import * as t from 'io-ts';
|
||||
import { operator } from '../../../../../lists/common/schemas';
|
||||
|
||||
export const translatedEntryMatchAnyMatcher = t.keyof({
|
||||
exact_cased_any: null,
|
||||
exact_caseless_any: null,
|
||||
});
|
||||
export type TranslatedEntryMatchAnyMatcher = t.TypeOf<typeof translatedEntryMatchAnyMatcher>;
|
||||
|
||||
export const translatedEntryMatchAny = t.exact(
|
||||
t.type({
|
||||
field: t.string,
|
||||
operator,
|
||||
type: t.keyof({
|
||||
exact_cased_any: null,
|
||||
exact_caseless_any: null,
|
||||
}),
|
||||
type: translatedEntryMatchAnyMatcher,
|
||||
value: t.array(t.string),
|
||||
})
|
||||
);
|
||||
export type TranslatedEntryMatchAny = t.TypeOf<typeof translatedEntryMatchAny>;
|
||||
|
||||
export const translatedEntryMatchAnyMatcher = translatedEntryMatchAny.type.props.type;
|
||||
export type TranslatedEntryMatchAnyMatcher = t.TypeOf<typeof translatedEntryMatchAnyMatcher>;
|
||||
export const translatedEntryMatchMatcher = t.keyof({
|
||||
exact_cased: null,
|
||||
exact_caseless: null,
|
||||
});
|
||||
export type TranslatedEntryMatchMatcher = t.TypeOf<typeof translatedEntryMatchMatcher>;
|
||||
|
||||
export const translatedEntryMatch = t.exact(
|
||||
t.type({
|
||||
field: t.string,
|
||||
operator,
|
||||
type: t.keyof({
|
||||
exact_cased: null,
|
||||
exact_caseless: null,
|
||||
}),
|
||||
type: translatedEntryMatchMatcher,
|
||||
value: t.string,
|
||||
})
|
||||
);
|
||||
export type TranslatedEntryMatch = t.TypeOf<typeof translatedEntryMatch>;
|
||||
|
||||
export const translatedEntryMatchMatcher = translatedEntryMatch.type.props.type;
|
||||
export type TranslatedEntryMatchMatcher = t.TypeOf<typeof translatedEntryMatchMatcher>;
|
||||
|
||||
export const translatedEntryMatcher = t.union([
|
||||
translatedEntryMatchMatcher,
|
||||
translatedEntryMatchAnyMatcher,
|
||||
|
|
|
|||
|
|
@ -4,9 +4,11 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
// eslint-disable-next-line max-classes-per-file
|
||||
import { savedObjectsClientMock, loggingSystemMock } from 'src/core/server/mocks';
|
||||
import { Logger } from 'src/core/server';
|
||||
import { createPackageConfigMock } from '../../../../../../ingest_manager/common/mocks';
|
||||
import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server';
|
||||
import { createPackageConfigServiceMock } from '../../../../../../ingest_manager/server/mocks';
|
||||
import { getFoundExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock';
|
||||
import { listMock } from '../../../../../../lists/server/mocks';
|
||||
import {
|
||||
|
|
@ -21,40 +23,6 @@ import { getArtifactClientMock } from '../artifact_client.mock';
|
|||
import { getManifestClientMock } from '../manifest_client.mock';
|
||||
import { ManifestManager } from './manifest_manager';
|
||||
|
||||
function getMockPackageConfig() {
|
||||
return {
|
||||
id: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1',
|
||||
inputs: [
|
||||
{
|
||||
config: {},
|
||||
},
|
||||
],
|
||||
revision: 1,
|
||||
version: 'abcd', // TODO: not yet implemented in ingest_manager (https://github.com/elastic/kibana/issues/69992)
|
||||
updated_at: '2020-06-25T16:03:38.159292',
|
||||
updated_by: 'kibana',
|
||||
created_at: '2020-06-25T16:03:38.159292',
|
||||
created_by: 'kibana',
|
||||
};
|
||||
}
|
||||
|
||||
class PackageConfigServiceMock {
|
||||
public create = jest.fn().mockResolvedValue(getMockPackageConfig());
|
||||
public get = jest.fn().mockResolvedValue(getMockPackageConfig());
|
||||
public getByIds = jest.fn().mockResolvedValue([getMockPackageConfig()]);
|
||||
public list = jest.fn().mockResolvedValue({
|
||||
items: [getMockPackageConfig()],
|
||||
total: 1,
|
||||
page: 1,
|
||||
perPage: 20,
|
||||
});
|
||||
public update = jest.fn().mockResolvedValue(getMockPackageConfig());
|
||||
}
|
||||
|
||||
export function getPackageConfigServiceMock() {
|
||||
return new PackageConfigServiceMock();
|
||||
}
|
||||
|
||||
async function mockBuildExceptionListArtifacts(
|
||||
os: string,
|
||||
schemaVersion: string
|
||||
|
|
@ -66,27 +34,23 @@ async function mockBuildExceptionListArtifacts(
|
|||
return [await buildArtifact(exceptions, os, schemaVersion)];
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
export class ManifestManagerMock extends ManifestManager {
|
||||
// @ts-ignore
|
||||
private buildExceptionListArtifacts = async () => {
|
||||
return mockBuildExceptionListArtifacts('linux', 'v1');
|
||||
};
|
||||
protected buildExceptionListArtifacts = jest
|
||||
.fn()
|
||||
.mockResolvedValue(mockBuildExceptionListArtifacts('linux', 'v1'));
|
||||
|
||||
// @ts-ignore
|
||||
private getLastDispatchedManifest = jest
|
||||
public getLastDispatchedManifest = jest
|
||||
.fn()
|
||||
.mockResolvedValue(new Manifest(new Date(), 'v1', ManifestConstants.INITIAL_VERSION));
|
||||
|
||||
// @ts-ignore
|
||||
private getManifestClient = jest
|
||||
protected getManifestClient = jest
|
||||
.fn()
|
||||
.mockReturnValue(getManifestClientMock(this.savedObjectsClient));
|
||||
}
|
||||
|
||||
export const getManifestManagerMock = (opts?: {
|
||||
cache?: ExceptionsCache;
|
||||
packageConfigService?: PackageConfigServiceMock;
|
||||
packageConfigService?: jest.Mocked<PackageConfigServiceInterface>;
|
||||
savedObjectsClient?: ReturnType<typeof savedObjectsClientMock.create>;
|
||||
}): ManifestManagerMock => {
|
||||
let cache = new ExceptionsCache(5);
|
||||
|
|
@ -94,10 +58,14 @@ export const getManifestManagerMock = (opts?: {
|
|||
cache = opts.cache;
|
||||
}
|
||||
|
||||
let packageConfigService = getPackageConfigServiceMock();
|
||||
let packageConfigService = createPackageConfigServiceMock();
|
||||
if (opts?.packageConfigService !== undefined) {
|
||||
packageConfigService = opts.packageConfigService;
|
||||
}
|
||||
packageConfigService.list = jest.fn().mockResolvedValue({
|
||||
total: 1,
|
||||
items: [createPackageConfigMock()],
|
||||
});
|
||||
|
||||
let savedObjectsClient = savedObjectsClientMock.create();
|
||||
if (opts?.savedObjectsClient !== undefined) {
|
||||
|
|
@ -107,7 +75,6 @@ export const getManifestManagerMock = (opts?: {
|
|||
const manifestManager = new ManifestManagerMock({
|
||||
artifactClient: getArtifactClientMock(savedObjectsClient),
|
||||
cache,
|
||||
// @ts-ignore
|
||||
packageConfigService,
|
||||
exceptionListClient: listMock.getExceptionListClient(),
|
||||
logger: loggingSystemMock.create().get() as jest.Mocked<Logger>,
|
||||
|
|
|
|||
|
|
@ -6,13 +6,14 @@
|
|||
|
||||
import { inflateSync } from 'zlib';
|
||||
import { savedObjectsClientMock } from 'src/core/server/mocks';
|
||||
import { createPackageConfigServiceMock } from '../../../../../../ingest_manager/server/mocks';
|
||||
import {
|
||||
ArtifactConstants,
|
||||
ManifestConstants,
|
||||
Manifest,
|
||||
ExceptionsCache,
|
||||
} from '../../../lib/artifacts';
|
||||
import { getPackageConfigServiceMock, getManifestManagerMock } from './manifest_manager.mock';
|
||||
import { getManifestManagerMock } from './manifest_manager.mock';
|
||||
|
||||
describe('manifest_manager', () => {
|
||||
describe('ManifestManager sanity checks', () => {
|
||||
|
|
@ -73,15 +74,15 @@ describe('manifest_manager', () => {
|
|||
});
|
||||
|
||||
test('ManifestManager can dispatch manifest', async () => {
|
||||
const packageConfigService = getPackageConfigServiceMock();
|
||||
const packageConfigService = createPackageConfigServiceMock();
|
||||
const manifestManager = getManifestManagerMock({ packageConfigService });
|
||||
const snapshot = await manifestManager.getSnapshot();
|
||||
const dispatched = await manifestManager.dispatch(snapshot!.manifest);
|
||||
expect(dispatched).toEqual(true);
|
||||
expect(dispatched).toEqual([]);
|
||||
const entries = snapshot!.manifest.getEntries();
|
||||
const artifact = Object.values(entries)[0].getArtifact();
|
||||
expect(
|
||||
packageConfigService.update.mock.calls[0][2].inputs[0].config.artifact_manifest.value
|
||||
packageConfigService.update.mock.calls[0][2].inputs[0].config!.artifact_manifest.value
|
||||
).toEqual({
|
||||
manifest_version: ManifestConstants.INITIAL_VERSION,
|
||||
schema_version: 'v1',
|
||||
|
|
@ -115,7 +116,7 @@ describe('manifest_manager', () => {
|
|||
snapshot!.diffs.push(diff);
|
||||
|
||||
const dispatched = await manifestManager.dispatch(snapshot!.manifest);
|
||||
expect(dispatched).toEqual(true);
|
||||
expect(dispatched).toEqual([]);
|
||||
|
||||
await manifestManager.commit(snapshot!.manifest);
|
||||
|
||||
|
|
|
|||
|
|
@ -61,19 +61,25 @@ export class ManifestManager {
|
|||
/**
|
||||
* Gets a ManifestClient for the provided schemaVersion.
|
||||
*
|
||||
* @param schemaVersion
|
||||
* @param schemaVersion The schema version of the manifest.
|
||||
* @returns {ManifestClient} A ManifestClient scoped to the provided schemaVersion.
|
||||
*/
|
||||
private getManifestClient(schemaVersion: string) {
|
||||
protected getManifestClient(schemaVersion: string): ManifestClient {
|
||||
return new ManifestClient(this.savedObjectsClient, schemaVersion as ManifestSchemaVersion);
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds an array of artifacts (one per supported OS) based on the current
|
||||
* state of exception-list-agnostic SO's.
|
||||
* state of exception-list-agnostic SOs.
|
||||
*
|
||||
* @param schemaVersion
|
||||
* @param schemaVersion The schema version of the artifact
|
||||
* @returns {Promise<InternalArtifactSchema[]>} An array of uncompressed artifacts built from exception-list-agnostic SOs.
|
||||
* @throws Throws/rejects if there are errors building the list.
|
||||
*/
|
||||
private async buildExceptionListArtifacts(schemaVersion: string) {
|
||||
protected async buildExceptionListArtifacts(
|
||||
schemaVersion: string
|
||||
): Promise<InternalArtifactSchema[]> {
|
||||
// TODO: should wrap in try/catch?
|
||||
return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.reduce(
|
||||
async (acc: Promise<InternalArtifactSchema[]>, os) => {
|
||||
const exceptionList = await getFullEndpointExceptionList(
|
||||
|
|
@ -90,13 +96,75 @@ export class ManifestManager {
|
|||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes new artifact SOs based on provided snapshot.
|
||||
*
|
||||
* @param snapshot A ManifestSnapshot to use for writing the artifacts.
|
||||
* @returns {Promise<Error[]>} Any errors encountered.
|
||||
*/
|
||||
private async writeArtifacts(snapshot: ManifestSnapshot): Promise<Error[]> {
|
||||
const errors: Error[] = [];
|
||||
for (const diff of snapshot.diffs) {
|
||||
const artifact = snapshot.manifest.getArtifact(diff.id);
|
||||
if (artifact === undefined) {
|
||||
throw new Error(
|
||||
`Corrupted manifest detected. Diff contained artifact ${diff.id} not in manifest.`
|
||||
);
|
||||
}
|
||||
|
||||
const compressedArtifact = await compressExceptionList(Buffer.from(artifact.body, 'base64'));
|
||||
artifact.body = compressedArtifact.toString('base64');
|
||||
artifact.encodedSize = compressedArtifact.byteLength;
|
||||
artifact.compressionAlgorithm = 'zlib';
|
||||
artifact.encodedSha256 = createHash('sha256').update(compressedArtifact).digest('hex');
|
||||
|
||||
try {
|
||||
// Write the artifact SO
|
||||
await this.artifactClient.createArtifact(artifact);
|
||||
// Cache the compressed body of the artifact
|
||||
this.cache.set(diff.id, Buffer.from(artifact.body, 'base64'));
|
||||
} catch (err) {
|
||||
if (err.status === 409) {
|
||||
this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`);
|
||||
} else {
|
||||
// TODO: log error here?
|
||||
errors.push(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes old artifact SOs based on provided snapshot.
|
||||
*
|
||||
* @param snapshot A ManifestSnapshot to use for deleting the artifacts.
|
||||
* @returns {Promise<Error[]>} Any errors encountered.
|
||||
*/
|
||||
private async deleteArtifacts(snapshot: ManifestSnapshot): Promise<Error[]> {
|
||||
const errors: Error[] = [];
|
||||
for (const diff of snapshot.diffs) {
|
||||
try {
|
||||
// Delete the artifact SO
|
||||
await this.artifactClient.deleteArtifact(diff.id);
|
||||
// TODO: should we delete the cache entry here?
|
||||
this.logger.info(`Cleaned up artifact ${diff.id}`);
|
||||
} catch (err) {
|
||||
errors.push(err);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the last dispatched manifest based on the current state of the
|
||||
* user-artifact-manifest SO.
|
||||
*
|
||||
* @param schemaVersion
|
||||
* @param schemaVersion The schema version of the manifest.
|
||||
* @returns {Promise<Manifest | null>} The last dispatched manifest, or null if does not exist.
|
||||
* @throws Throws/rejects if there is an unexpected error retrieving the manifest.
|
||||
*/
|
||||
private async getLastDispatchedManifest(schemaVersion: string) {
|
||||
public async getLastDispatchedManifest(schemaVersion: string): Promise<Manifest | null> {
|
||||
try {
|
||||
const manifestClient = this.getManifestClient(schemaVersion);
|
||||
const manifestSo = await manifestClient.getManifest();
|
||||
|
|
@ -127,9 +195,11 @@ export class ManifestManager {
|
|||
/**
|
||||
* Snapshots a manifest based on current state of exception-list-agnostic SOs.
|
||||
*
|
||||
* @param opts TODO
|
||||
* @param opts Optional parameters for snapshot retrieval.
|
||||
* @param opts.initialize Initialize a new Manifest when no manifest SO can be retrieved.
|
||||
* @returns {Promise<ManifestSnapshot | null>} A snapshot of the manifest, or null if not initialized.
|
||||
*/
|
||||
public async getSnapshot(opts?: ManifestSnapshotOpts) {
|
||||
public async getSnapshot(opts?: ManifestSnapshotOpts): Promise<ManifestSnapshot | null> {
|
||||
try {
|
||||
let oldManifest: Manifest | null;
|
||||
|
||||
|
|
@ -176,71 +246,39 @@ export class ManifestManager {
|
|||
* Creates artifacts that do not yet exist and cleans up old artifacts that have been
|
||||
* superceded by this snapshot.
|
||||
*
|
||||
* Can be filtered to apply one or both operations.
|
||||
*
|
||||
* @param snapshot
|
||||
* @param diffType
|
||||
* @param snapshot A ManifestSnapshot to use for sync.
|
||||
* @returns {Promise<Error[]>} Any errors encountered.
|
||||
*/
|
||||
public async syncArtifacts(snapshot: ManifestSnapshot, diffType?: 'add' | 'delete') {
|
||||
const filteredDiffs = snapshot.diffs.reduce((diffs: ManifestDiff[], diff) => {
|
||||
if (diff.type === diffType || diffType === undefined) {
|
||||
diffs.push(diff);
|
||||
} else if (!['add', 'delete'].includes(diff.type)) {
|
||||
// TODO: replace with io-ts schema
|
||||
throw new Error(`Unsupported diff type: ${diff.type}`);
|
||||
}
|
||||
return diffs;
|
||||
}, []);
|
||||
|
||||
const adds = filteredDiffs.filter((diff) => {
|
||||
return diff.type === 'add';
|
||||
public async syncArtifacts(
|
||||
snapshot: ManifestSnapshot,
|
||||
diffType: 'add' | 'delete'
|
||||
): Promise<Error[]> {
|
||||
const filteredDiffs = snapshot.diffs.filter((diff) => {
|
||||
return diff.type === diffType;
|
||||
});
|
||||
|
||||
const deletes = filteredDiffs.filter((diff) => {
|
||||
return diff.type === 'delete';
|
||||
});
|
||||
const tmpSnapshot = { ...snapshot };
|
||||
tmpSnapshot.diffs = filteredDiffs;
|
||||
|
||||
for (const diff of adds) {
|
||||
const artifact = snapshot.manifest.getArtifact(diff.id);
|
||||
if (artifact === undefined) {
|
||||
throw new Error(
|
||||
`Corrupted manifest detected. Diff contained artifact ${diff.id} not in manifest.`
|
||||
);
|
||||
}
|
||||
const compressedArtifact = await compressExceptionList(Buffer.from(artifact.body, 'base64'));
|
||||
artifact.body = compressedArtifact.toString('base64');
|
||||
artifact.encodedSize = compressedArtifact.byteLength;
|
||||
artifact.compressionAlgorithm = 'zlib';
|
||||
artifact.encodedSha256 = createHash('sha256').update(compressedArtifact).digest('hex');
|
||||
|
||||
try {
|
||||
await this.artifactClient.createArtifact(artifact);
|
||||
} catch (err) {
|
||||
if (err.status === 409) {
|
||||
this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
// Cache the body of the artifact
|
||||
this.cache.set(diff.id, Buffer.from(artifact.body, 'base64'));
|
||||
if (diffType === 'add') {
|
||||
return this.writeArtifacts(tmpSnapshot);
|
||||
} else if (diffType === 'delete') {
|
||||
return this.deleteArtifacts(tmpSnapshot);
|
||||
}
|
||||
|
||||
for (const diff of deletes) {
|
||||
await this.artifactClient.deleteArtifact(diff.id);
|
||||
// TODO: should we delete the cache entry here?
|
||||
this.logger.info(`Cleaned up artifact ${diff.id}`);
|
||||
}
|
||||
return [new Error(`Unsupported diff type: ${diffType}`)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatches the manifest by writing it to the endpoint package config.
|
||||
*
|
||||
* @param manifest The Manifest to dispatch.
|
||||
* @returns {Promise<Error[]>} Any errors encountered.
|
||||
*/
|
||||
public async dispatch(manifest: Manifest) {
|
||||
public async dispatch(manifest: Manifest): Promise<Error[]> {
|
||||
let paging = true;
|
||||
let page = 1;
|
||||
let success = true;
|
||||
const errors: Error[] = [];
|
||||
|
||||
while (paging) {
|
||||
const { items, total } = await this.packageConfigService.list(this.savedObjectsClient, {
|
||||
|
|
@ -264,13 +302,10 @@ export class ManifestManager {
|
|||
`Updated package config ${id} with manifest version ${manifest.getVersion()}`
|
||||
);
|
||||
} catch (err) {
|
||||
success = false;
|
||||
this.logger.debug(`Error updating package config ${id}`);
|
||||
this.logger.error(err);
|
||||
errors.push(err);
|
||||
}
|
||||
} else {
|
||||
success = false;
|
||||
this.logger.debug(`Package config ${id} has no config.`);
|
||||
errors.push(new Error(`Package config ${id} has no config.`));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -278,32 +313,38 @@ export class ManifestManager {
|
|||
page++;
|
||||
}
|
||||
|
||||
// TODO: revisit success logic
|
||||
return success;
|
||||
return errors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Commits a manifest to indicate that it has been dispatched.
|
||||
*
|
||||
* @param manifest
|
||||
* @param manifest The Manifest to commit.
|
||||
* @returns {Promise<Error | null>} An error if encountered, or null if successful.
|
||||
*/
|
||||
public async commit(manifest: Manifest) {
|
||||
const manifestClient = this.getManifestClient(manifest.getSchemaVersion());
|
||||
public async commit(manifest: Manifest): Promise<Error | null> {
|
||||
try {
|
||||
const manifestClient = this.getManifestClient(manifest.getSchemaVersion());
|
||||
|
||||
// Commit the new manifest
|
||||
if (manifest.getVersion() === ManifestConstants.INITIAL_VERSION) {
|
||||
await manifestClient.createManifest(manifest.toSavedObject());
|
||||
} else {
|
||||
const version = manifest.getVersion();
|
||||
if (version === ManifestConstants.INITIAL_VERSION) {
|
||||
throw new Error('Updating existing manifest with baseline version. Bad state.');
|
||||
// Commit the new manifest
|
||||
if (manifest.getVersion() === ManifestConstants.INITIAL_VERSION) {
|
||||
await manifestClient.createManifest(manifest.toSavedObject());
|
||||
} else {
|
||||
const version = manifest.getVersion();
|
||||
if (version === ManifestConstants.INITIAL_VERSION) {
|
||||
throw new Error('Updating existing manifest with baseline version. Bad state.');
|
||||
}
|
||||
await manifestClient.updateManifest(manifest.toSavedObject(), {
|
||||
version,
|
||||
});
|
||||
}
|
||||
await manifestClient.updateManifest(manifest.toSavedObject(), {
|
||||
version,
|
||||
});
|
||||
|
||||
this.logger.info(`Committed manifest ${manifest.getVersion()}`);
|
||||
} catch (err) {
|
||||
return err;
|
||||
}
|
||||
|
||||
this.logger.info(`Committed manifest ${manifest.getVersion()}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -116,6 +116,48 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
|
|||
version: policyInfo.packageInfo.version,
|
||||
},
|
||||
},
|
||||
artifact_manifest: {
|
||||
artifacts: {
|
||||
'endpoint-exceptionlist-linux-v1': {
|
||||
compression_algorithm: 'zlib',
|
||||
decoded_sha256:
|
||||
'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
|
||||
decoded_size: 14,
|
||||
encoded_sha256:
|
||||
'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
|
||||
encoded_size: 22,
|
||||
encryption_algorithm: 'none',
|
||||
relative_url:
|
||||
'/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
|
||||
},
|
||||
'endpoint-exceptionlist-macos-v1': {
|
||||
compression_algorithm: 'zlib',
|
||||
decoded_sha256:
|
||||
'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
|
||||
decoded_size: 14,
|
||||
encoded_sha256:
|
||||
'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
|
||||
encoded_size: 22,
|
||||
encryption_algorithm: 'none',
|
||||
relative_url:
|
||||
'/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
|
||||
},
|
||||
'endpoint-exceptionlist-windows-v1': {
|
||||
compression_algorithm: 'zlib',
|
||||
decoded_sha256:
|
||||
'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
|
||||
decoded_size: 14,
|
||||
encoded_sha256:
|
||||
'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
|
||||
encoded_size: 22,
|
||||
encryption_algorithm: 'none',
|
||||
relative_url:
|
||||
'/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
|
||||
},
|
||||
},
|
||||
manifest_version: 'WzEwNSwxXQ==',
|
||||
schema_version: 'v1',
|
||||
},
|
||||
policy: {
|
||||
linux: {
|
||||
events: { file: false, network: true, process: true },
|
||||
|
|
|
|||
Loading…
Reference in a new issue