[Fleet] Configure Fleet packages and integrations through endpoint (#94509)

x-pack/plugins/fleet/common/constants/routes.ts

@@ -119,3 +119,8 @@ export const AGENTS_SETUP_API_ROUTES = {
 export const SETUP_API_ROUTE = `${API_ROOT}/setup`;
 
 export const INSTALL_SCRIPT_API_ROUTES = `${API_ROOT}/install/{osType}`;
+
+// Policy preconfig API routes
+export const PRECONFIGURATION_API_ROUTES = {
+  PUT_PRECONFIG: `${API_ROOT}/setup/preconfiguration`,
+};

x-pack/plugins/fleet/common/services/package_to_package_policy.ts

@@ -11,10 +11,10 @@ import type {
   RegistryVarsEntry,
   RegistryStream,
   PackagePolicyConfigRecord,
-  PackagePolicyConfigRecordEntry,
   NewPackagePolicyInput,
   NewPackagePolicyInputStream,
   NewPackagePolicy,
+  PackagePolicyConfigRecordEntry,
 } from '../types';
 
 const getStreamsForInputType = (

x-pack/plugins/fleet/common/types/models/agent_policy.ts

@@ -21,6 +21,7 @@ export interface NewAgentPolicy {
   is_default_fleet_server?: boolean; // Optional when creating a policy
   is_managed?: boolean; // Optional when creating a policy
   monitoring_enabled?: Array<ValueOf<DataType>>;
+  preconfiguration_id?: string; // Uniqifies preconfigured policies by something other than `name`
 }
 
 export interface AgentPolicy extends NewAgentPolicy {

x-pack/plugins/fleet/common/types/models/index.ts

@@ -14,3 +14,4 @@ export * from './epm';
 export * from './package_spec';
 export * from './enrollment_api_key';
 export * from './settings';
+export * from './preconfiguration';

x-pack/plugins/fleet/common/types/models/preconfiguration.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  PackagePolicyPackage,
+  NewPackagePolicy,
+  NewPackagePolicyInput,
+} from './package_policy';
+import type { NewAgentPolicy } from './agent_policy';
+
+export type InputsOverride = Partial<NewPackagePolicyInput> & {
+  vars?: Array<NewPackagePolicyInput['vars'] & { name: string }>;
+};
+
+export interface PreconfiguredAgentPolicy extends Omit<NewAgentPolicy, 'namespace'> {
+  id: string | number;
+  namespace?: string;
+  package_policies: Array<
+    Partial<Omit<NewPackagePolicy, 'inputs'>> & {
+      name: string;
+      package: Partial<PackagePolicyPackage>;
+      inputs?: InputsOverride[];
+    }
+  >;
+}

x-pack/plugins/fleet/server/constants/index.ts

@@ -33,6 +33,7 @@ export {
   SETUP_API_ROUTE,
   SETTINGS_API_ROUTES,
   APP_API_ROUTES,
+  PRECONFIGURATION_API_ROUTES,
   // Saved object types
   SO_SEARCH_LIMIT,
   AGENT_SAVED_OBJECT_TYPE,

x-pack/plugins/fleet/server/plugin.ts

@@ -64,6 +64,7 @@ import {
   registerOutputRoutes,
   registerSettingsRoutes,
   registerAppRoutes,
+  registerPreconfigurationRoutes,
 } from './routes';
 import type {
   ESIndexPatternService,
@@ -274,6 +275,7 @@ export class FleetPlugin
       registerSettingsRoutes(routerSuperuserOnly);
       registerDataStreamRoutes(routerSuperuserOnly);
       registerEPMRoutes(routerSuperuserOnly);
+      registerPreconfigurationRoutes(routerSuperuserOnly);
 
       // Conditional config routes
       if (config.agents.enabled) {

x-pack/plugins/fleet/server/routes/index.ts

@@ -20,3 +20,4 @@ export { registerRoutes as registerOutputRoutes } from './output';
 export { registerRoutes as registerSettingsRoutes } from './settings';
 export { registerRoutes as registerAppRoutes } from './app';
 export { registerLimitedConcurrencyRoutes } from './limited_concurrency';
+export { registerRoutes as registerPreconfigurationRoutes } from './preconfiguration';

x-pack/plugins/fleet/server/routes/preconfiguration/index.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { IRouter, RequestHandler } from 'src/core/server';
+import type { TypeOf } from '@kbn/config-schema';
+
+import type { PreconfiguredAgentPolicy } from '../../../common';
+
+import { PLUGIN_ID, PRECONFIGURATION_API_ROUTES } from '../../constants';
+import { PutPreconfigurationSchema } from '../../types';
+import { defaultIngestErrorHandler } from '../../errors';
+import { ensurePreconfiguredPackagesAndPolicies, outputService } from '../../services';
+
+export const putPreconfigurationHandler: RequestHandler<
+  undefined,
+  undefined,
+  TypeOf<typeof PutPreconfigurationSchema.body>
+> = async (context, request, response) => {
+  const soClient = context.core.savedObjects.client;
+  const esClient = context.core.elasticsearch.client.asCurrentUser;
+  const defaultOutput = await outputService.ensureDefaultOutput(soClient);
+
+  const { agentPolicies, packages } = request.body;
+
+  try {
+    const body = await ensurePreconfiguredPackagesAndPolicies(
+      soClient,
+      esClient,
+      (agentPolicies as PreconfiguredAgentPolicy[]) ?? [],
+      packages ?? [],
+      defaultOutput
+    );
+    return response.ok({ body });
+  } catch (error) {
+    return defaultIngestErrorHandler({ error, response });
+  }
+};
+
+export const registerRoutes = (router: IRouter) => {
+  router.put(
+    {
+      path: PRECONFIGURATION_API_ROUTES.PUT_PRECONFIG,
+      validate: PutPreconfigurationSchema,
+      options: { tags: [`access:${PLUGIN_ID}-all`] },
+    },
+    putPreconfigurationHandler
+  );
+};

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -178,6 +178,7 @@ const getSavedObjectTypes = (
         updated_by: { type: 'keyword' },
         revision: { type: 'integer' },
         monitoring_enabled: { type: 'keyword', index: false },
+        preconfiguration_id: { type: 'keyword' },
       },
     },
     migrations: {

x-pack/plugins/fleet/server/services/agent_policy.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { uniq } from 'lodash';
+import { uniq, omit } from 'lodash';
 import { safeLoad } from 'js-yaml';
 import uuid from 'uuid/v4';
 import type {
@@ -23,19 +23,28 @@ import {
 import type {
   PackagePolicy,
   NewAgentPolicy,
+  PreconfiguredAgentPolicy,
   AgentPolicy,
   AgentPolicySOAttributes,
   FullAgentPolicy,
   ListWithKuery,
+  NewPackagePolicy,
 } from '../types';
 import {
   agentPolicyStatuses,
   storedPackagePoliciesToAgentInputs,
   dataTypes,
+  packageToPackagePolicy,
   AGENT_POLICY_INDEX,
   DEFAULT_FLEET_SERVER_AGENT_POLICY,
 } from '../../common';
-import type { DeleteAgentPolicyResponse, Settings, FleetServerPolicy } from '../../common';
+import type {
+  DeleteAgentPolicyResponse,
+  Settings,
+  FleetServerPolicy,
+  Installation,
+  Output,
+} from '../../common';
 import {
   AgentPolicyNameExistsError,
   AgentPolicyDeletionError,
@@ -43,6 +52,7 @@ import {
 } from '../errors';
 import { getFullAgentPolicyKibanaConfig } from '../../common/services/full_agent_policy_kibana_config';
 
+import { getPackageInfo } from './epm/packages';
 import { createAgentPolicyAction, getAgentsByKuery } from './agents';
 import { packagePolicyService } from './package_policy';
 import { outputService } from './output';
@@ -106,32 +116,13 @@ class AgentPolicyService {
     esClient: ElasticsearchClient
   ): Promise<{
     created: boolean;
-    defaultAgentPolicy: AgentPolicy;
+    policy: AgentPolicy;
   }> {
-    const agentPolicies = await soClient.find<AgentPolicySOAttributes>({
-      type: AGENT_POLICY_SAVED_OBJECT_TYPE,
+    const searchParams = {
       searchFields: ['is_default'],
       search: 'true',
-    });
-
-    if (agentPolicies.total === 0) {
-      const newDefaultAgentPolicy: NewAgentPolicy = {
-        ...DEFAULT_AGENT_POLICY,
-      };
-
-      return {
-        created: true,
-        defaultAgentPolicy: await this.create(soClient, esClient, newDefaultAgentPolicy),
-      };
-    }
-
-    return {
-      created: false,
-      defaultAgentPolicy: {
-        id: agentPolicies.saved_objects[0].id,
-        ...agentPolicies.saved_objects[0].attributes,
-      },
     };
+    return await this.ensureAgentPolicy(soClient, esClient, DEFAULT_AGENT_POLICY, searchParams);
   }
 
   public async ensureDefaultFleetServerAgentPolicy(
@@ -141,20 +132,68 @@ class AgentPolicyService {
     created: boolean;
     policy: AgentPolicy;
   }> {
-    const agentPolicies = await soClient.find<AgentPolicySOAttributes>({
-      type: AGENT_POLICY_SAVED_OBJECT_TYPE,
+    const searchParams = {
       searchFields: ['is_default_fleet_server'],
       search: 'true',
+    };
+    return await this.ensureAgentPolicy(
+      soClient,
+      esClient,
+      DEFAULT_FLEET_SERVER_AGENT_POLICY,
+      searchParams
+    );
+  }
+
+  public async ensurePreconfiguredAgentPolicy(
+    soClient: SavedObjectsClientContract,
+    esClient: ElasticsearchClient,
+    config: PreconfiguredAgentPolicy
+  ): Promise<{
+    created: boolean;
+    policy: AgentPolicy;
+  }> {
+    const { id, ...preconfiguredAgentPolicy } = omit(config, 'package_policies');
+    const preconfigurationId = String(id);
+    const searchParams = {
+      searchFields: ['preconfiguration_id'],
+      search: escapeSearchQueryPhrase(preconfigurationId),
+    };
+
+    const newAgentPolicyDefaults: Partial<NewAgentPolicy> = {
+      namespace: 'default',
+      monitoring_enabled: ['logs', 'metrics'],
+    };
+
+    const newAgentPolicy = {
+      ...newAgentPolicyDefaults,
+      ...preconfiguredAgentPolicy,
+      preconfiguration_id: preconfigurationId,
+    } as NewAgentPolicy;
+
+    return await this.ensureAgentPolicy(soClient, esClient, newAgentPolicy, searchParams);
+  }
+
+  private async ensureAgentPolicy(
+    soClient: SavedObjectsClientContract,
+    esClient: ElasticsearchClient,
+    newAgentPolicy: NewAgentPolicy,
+    searchParams: {
+      searchFields: string[];
+      search: string;
+    }
+  ): Promise<{
+    created: boolean;
+    policy: AgentPolicy;
+  }> {
+    const agentPolicies = await soClient.find<AgentPolicySOAttributes>({
+      type: AGENT_POLICY_SAVED_OBJECT_TYPE,
+      ...searchParams,
     });
 
     if (agentPolicies.total === 0) {
-      const newDefaultAgentPolicy: NewAgentPolicy = {
-        ...DEFAULT_FLEET_SERVER_AGENT_POLICY,
-      };
-
       return {
         created: true,
-        policy: await this.create(soClient, esClient, newDefaultAgentPolicy),
+        policy: await this.create(soClient, esClient, newAgentPolicy),
       };
     }
 
@@ -514,7 +553,7 @@ class AgentPolicyService {
     }
 
     const {
-      defaultAgentPolicy: { id: defaultAgentPolicyId },
+      policy: { id: defaultAgentPolicyId },
     } = await this.ensureDefaultAgentPolicy(soClient, esClient);
     if (id === defaultAgentPolicyId) {
       throw new Error('The default agent policy cannot be deleted');
@@ -726,3 +765,37 @@ class AgentPolicyService {
 }
 
 export const agentPolicyService = new AgentPolicyService();
+
+export async function addPackageToAgentPolicy(
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient,
+  packageToInstall: Installation,
+  agentPolicy: AgentPolicy,
+  defaultOutput: Output,
+  packagePolicyName?: string,
+  packagePolicyDescription?: string,
+  transformPackagePolicy?: (p: NewPackagePolicy) => NewPackagePolicy
+) {
+  const packageInfo = await getPackageInfo({
+    savedObjectsClient: soClient,
+    pkgName: packageToInstall.name,
+    pkgVersion: packageToInstall.version,
+  });
+
+  const basePackagePolicy = packageToPackagePolicy(
+    packageInfo,
+    agentPolicy.id,
+    defaultOutput.id,
+    agentPolicy.namespace ?? 'default',
+    packagePolicyName,
+    packagePolicyDescription
+  );
+
+  const newPackagePolicy = transformPackagePolicy
+    ? transformPackagePolicy(basePackagePolicy)
+    : basePackagePolicy;
+
+  await packagePolicyService.create(soClient, esClient, newPackagePolicy, {
+    bumpRevision: false,
+  });
+}

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -97,22 +97,54 @@ export async function ensureInstalledDefaultPackages(
   });
 }
 
+export async function isPackageVersionInstalled(options: {
+  savedObjectsClient: SavedObjectsClientContract;
+  pkgName: string;
+  pkgVersion?: string;
+}): Promise<Installation | false> {
+  const { savedObjectsClient, pkgName, pkgVersion } = options;
+  const installedPackage = await getInstallation({ savedObjectsClient, pkgName });
+  if (installedPackage && (!pkgVersion || installedPackage.version === pkgVersion)) {
+    return installedPackage;
+  }
+  return false;
+}
+
 export async function ensureInstalledPackage(options: {
   savedObjectsClient: SavedObjectsClientContract;
   pkgName: string;
   esClient: ElasticsearchClient;
+  pkgVersion?: string;
 }): Promise<Installation> {
-  const { savedObjectsClient, pkgName, esClient } = options;
-  const installedPackage = await getInstallation({ savedObjectsClient, pkgName });
+  const { savedObjectsClient, pkgName, esClient, pkgVersion } = options;
+  const installedPackage = await isPackageVersionInstalled({
+    savedObjectsClient,
+    pkgName,
+    pkgVersion,
+  });
   if (installedPackage) {
     return installedPackage;
   }
   // if the requested packaged was not found to be installed, install
-  await installLatestPackage({
-    savedObjectsClient,
-    pkgName,
-    esClient,
-  });
+  if (pkgVersion) {
+    const pkgkey = Registry.pkgToPkgKey({
+      name: pkgName,
+      version: pkgVersion,
+    });
+    await installPackage({
+      installSource: 'registry',
+      savedObjectsClient,
+      pkgkey,
+      esClient,
+      force: true,
+    });
+  } else {
+    await installLatestPackage({
+      savedObjectsClient,
+      pkgName,
+      esClient,
+    });
+  }
   const installation = await getInstallation({ savedObjectsClient, pkgName });
   if (!installation) throw new Error(`could not get installation ${pkgName}`);
   return installation;

x-pack/plugins/fleet/server/services/index.ts

@@ -83,3 +83,6 @@ export { licenseService } from './license';
 
 // Artifacts services
 export * from './artifacts';
+
+// Policy preconfiguration functions
+export { ensurePreconfiguredPackagesAndPolicies } from './preconfiguration';

x-pack/plugins/fleet/server/services/preconfiguration.test.ts

@@ -0,0 +1,244 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/server/mocks';
+
+import type { PreconfiguredAgentPolicy } from '../../common/types';
+import type { AgentPolicy, NewPackagePolicy, Output } from '../types';
+
+import { ensurePreconfiguredPackagesAndPolicies } from './preconfiguration';
+
+const mockInstalledPackages = new Map();
+const mockConfiguredPolicies = new Map();
+
+const mockDefaultOutput: Output = {
+  id: 'test-id',
+  is_default: true,
+  name: 'default',
+  // @ts-ignore
+  type: 'elasticsearch',
+  hosts: ['http://127.0.0.1:9201'],
+};
+
+function getPutPreconfiguredPackagesMock() {
+  const soClient = savedObjectsClientMock.create();
+  soClient.find.mockImplementation(async ({ type, search }) => {
+    const attributes = mockConfiguredPolicies.get(search!.replace(/"/g, ''));
+    if (attributes) {
+      return {
+        saved_objects: [
+          {
+            id: `mocked-${attributes.preconfiguration_id}`,
+            attributes,
+            type: type as string,
+            score: 1,
+            references: [],
+          },
+        ],
+        total: 1,
+        page: 1,
+        per_page: 1,
+      };
+    } else {
+      return {
+        saved_objects: [],
+        total: 0,
+        page: 1,
+        per_page: 0,
+      };
+    }
+  });
+  soClient.create.mockImplementation(async (type, policy) => {
+    const attributes = policy as AgentPolicy;
+    mockConfiguredPolicies.set(attributes.preconfiguration_id, attributes);
+    return {
+      id: `mocked-${attributes.preconfiguration_id}`,
+      attributes,
+      type,
+      references: [],
+    };
+  });
+  return soClient;
+}
+
+jest.mock('./epm/packages/install', () => ({
+  ensureInstalledPackage({ pkgName, pkgVersion }: { pkgName: string; pkgVersion: string }) {
+    const installedPackage = mockInstalledPackages.get(pkgName);
+    if (installedPackage) return installedPackage;
+
+    const packageInstallation = { name: pkgName, version: pkgVersion, title: pkgName };
+    mockInstalledPackages.set(pkgName, packageInstallation);
+    return packageInstallation;
+  },
+}));
+
+jest.mock('./epm/packages/get', () => ({
+  getPackageInfo({ pkgName }: { pkgName: string }) {
+    const installedPackage = mockInstalledPackages.get(pkgName);
+    if (!installedPackage) return { status: 'not_installed' };
+    return {
+      status: 'installed',
+      ...installedPackage,
+    };
+  },
+  getInstallation({ pkgName }: { pkgName: string }) {
+    return mockInstalledPackages.get(pkgName) ?? false;
+  },
+}));
+
+jest.mock('./package_policy', () => ({
+  packagePolicyService: {
+    create(soClient: any, esClient: any, newPackagePolicy: NewPackagePolicy) {
+      return {
+        id: 'mocked',
+        version: 'mocked',
+        ...newPackagePolicy,
+      };
+    },
+  },
+}));
+
+jest.mock('./agents/setup', () => ({
+  isAgentsSetup() {
+    return false;
+  },
+}));
+
+describe('policy preconfiguration', () => {
+  beforeEach(() => {
+    mockInstalledPackages.clear();
+    mockConfiguredPolicies.clear();
+  });
+
+  it('should perform a no-op when passed no policies or packages', async () => {
+    const soClient = getPutPreconfiguredPackagesMock();
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+
+    const { policies, packages } = await ensurePreconfiguredPackagesAndPolicies(
+      soClient,
+      esClient,
+      [],
+      [],
+      mockDefaultOutput
+    );
+
+    expect(policies.length).toBe(0);
+    expect(packages.length).toBe(0);
+  });
+
+  it('should install packages successfully', async () => {
+    const soClient = getPutPreconfiguredPackagesMock();
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+
+    const { policies, packages } = await ensurePreconfiguredPackagesAndPolicies(
+      soClient,
+      esClient,
+      [],
+      [{ name: 'test-package', version: '3.0.0' }],
+      mockDefaultOutput
+    );
+
+    expect(policies.length).toBe(0);
+    expect(packages).toEqual(expect.arrayContaining(['test-package:3.0.0']));
+  });
+
+  it('should install packages and configure agent policies successfully', async () => {
+    const soClient = getPutPreconfiguredPackagesMock();
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+
+    const { policies, packages } = await ensurePreconfiguredPackagesAndPolicies(
+      soClient,
+      esClient,
+      [
+        {
+          name: 'Test policy',
+          namespace: 'default',
+          id: 'test-id',
+          package_policies: [
+            {
+              package: { name: 'test-package' },
+              name: 'Test package',
+            },
+          ],
+        },
+      ] as PreconfiguredAgentPolicy[],
+      [{ name: 'test-package', version: '3.0.0' }],
+      mockDefaultOutput
+    );
+
+    expect(policies.length).toEqual(1);
+    expect(policies[0].id).toBe('mocked-test-id');
+    expect(packages).toEqual(expect.arrayContaining(['test-package:3.0.0']));
+  });
+
+  it('should throw an error when trying to install duplicate packages', async () => {
+    const soClient = getPutPreconfiguredPackagesMock();
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+
+    await expect(
+      ensurePreconfiguredPackagesAndPolicies(
+        soClient,
+        esClient,
+        [],
+        [
+          { name: 'test-package', version: '3.0.0' },
+          { name: 'test-package', version: '2.0.0' },
+        ],
+        mockDefaultOutput
+      )
+    ).rejects.toThrow(
+      'Duplicate packages specified in configuration: test-package:3.0.0, test-package:2.0.0'
+    );
+  });
+
+  it('should not attempt to recreate or modify an agent policy if its ID is unchanged', async () => {
+    const soClient = getPutPreconfiguredPackagesMock();
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+
+    const { policies: policiesA } = await ensurePreconfiguredPackagesAndPolicies(
+      soClient,
+      esClient,
+      [
+        {
+          name: 'Test policy',
+          namespace: 'default',
+          id: 'test-id',
+          package_policies: [],
+        },
+      ] as PreconfiguredAgentPolicy[],
+      [],
+      mockDefaultOutput
+    );
+
+    expect(policiesA.length).toEqual(1);
+    expect(policiesA[0].id).toBe('mocked-test-id');
+
+    const { policies: policiesB } = await ensurePreconfiguredPackagesAndPolicies(
+      soClient,
+      esClient,
+      [
+        {
+          name: 'Test policy redo',
+          namespace: 'default',
+          id: 'test-id',
+          package_policies: [
+            {
+              package: { name: 'some-uninstalled-package' },
+              name: 'This package is not installed',
+            },
+          ],
+        },
+      ] as PreconfiguredAgentPolicy[],
+      [],
+      mockDefaultOutput
+    );
+
+    expect(policiesB.length).toEqual(1);
+    expect(policiesB[0].id).toBe('mocked-test-id');
+    expect(policiesB[0].updated_at).toEqual(policiesA[0].updated_at);
+  });
+});

x-pack/plugins/fleet/server/services/preconfiguration.ts

@@ -0,0 +1,272 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server';
+import { i18n } from '@kbn/i18n';
+import { groupBy } from 'lodash';
+
+import type {
+  PackagePolicyPackage,
+  NewPackagePolicy,
+  AgentPolicy,
+  Installation,
+  Output,
+  NewPackagePolicyInput,
+  NewPackagePolicyInputStream,
+  PreconfiguredAgentPolicy,
+} from '../../common';
+
+import { getInstallation } from './epm/packages';
+import { ensureInstalledPackage } from './epm/packages/install';
+import { agentPolicyService, addPackageToAgentPolicy } from './agent_policy';
+
+export type InputsOverride = Partial<NewPackagePolicyInput> & {
+  vars?: Array<NewPackagePolicyInput['vars'] & { name: string }>;
+};
+
+export async function ensurePreconfiguredPackagesAndPolicies(
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient,
+  policies: PreconfiguredAgentPolicy[] = [],
+  packages: Array<Omit<PackagePolicyPackage, 'title'>> = [],
+  defaultOutput: Output
+) {
+  // Validate configured packages to ensure there are no version conflicts
+  const packageNames = groupBy(packages, (pkg) => pkg.name);
+  const duplicatePackages = Object.entries(packageNames).filter(
+    ([, versions]) => versions.length > 1
+  );
+  if (duplicatePackages.length) {
+    // List duplicate packages as a comma-separated list of <package-name>:<semver>
+    // If there are multiple packages with duplicate versions, separate them with semicolons, e.g
+    // package-a:1.0.0, package-a:2.0.0; package-b:1.0.0, package-b:2.0.0
+    const duplicateList = duplicatePackages
+      .map(([, versions]) => versions.map((v) => `${v.name}:${v.version}`).join(', '))
+      .join('; ');
+
+    throw new Error(
+      i18n.translate('xpack.fleet.preconfiguration.duplicatePackageError', {
+        defaultMessage: 'Duplicate packages specified in configuration: {duplicateList}',
+        values: {
+          duplicateList,
+        },
+      })
+    );
+  }
+
+  // Preinstall packages specified in Kibana config
+  const preconfiguredPackages = await Promise.all(
+    packages.map(({ name, version }) =>
+      ensureInstalledPreconfiguredPackage(soClient, esClient, name, version)
+    )
+  );
+
+  // Create policies specified in Kibana config
+  const preconfiguredPolicies = await Promise.all(
+    policies.map(async (preconfiguredAgentPolicy) => {
+      const { created, policy } = await agentPolicyService.ensurePreconfiguredAgentPolicy(
+        soClient,
+        esClient,
+        preconfiguredAgentPolicy
+      );
+
+      if (!created) return { created, policy };
+      const { package_policies: packagePolicies } = preconfiguredAgentPolicy;
+
+      const installedPackagePolicies = await Promise.all(
+        packagePolicies.map(async ({ package: pkg, name, ...newPackagePolicy }) => {
+          const installedPackage = await getInstallation({
+            savedObjectsClient: soClient,
+            pkgName: pkg.name,
+          });
+          if (!installedPackage) {
+            throw new Error(
+              i18n.translate('xpack.fleet.preconfiguration.packageMissingError', {
+                defaultMessage:
+                  '{agentPolicyName} could not be added. {pkgName} is not installed, add {pkgName} to `{packagesConfigValue}` or remove it from {packagePolicyName}.',
+                values: {
+                  agentPolicyName: preconfiguredAgentPolicy.name,
+                  packagePolicyName: name,
+                  pkgName: pkg.name,
+                  packagesConfigValue: 'xpack.fleet.packages',
+                },
+              })
+            );
+          }
+          return { name, installedPackage, ...newPackagePolicy };
+        })
+      );
+
+      return { created, policy, installedPackagePolicies };
+    })
+  );
+
+  for (const preconfiguredPolicy of preconfiguredPolicies) {
+    const { created, policy, installedPackagePolicies } = preconfiguredPolicy;
+    if (created) {
+      await addPreconfiguredPolicyPackages(
+        soClient,
+        esClient,
+        policy,
+        installedPackagePolicies!,
+        defaultOutput
+      );
+    }
+  }
+
+  return {
+    policies: preconfiguredPolicies.map((p) => ({
+      id: p.policy.id,
+      updated_at: p.policy.updated_at,
+    })),
+    packages: preconfiguredPackages.map((pkg) => `${pkg.name}:${pkg.version}`),
+  };
+}
+
+async function addPreconfiguredPolicyPackages(
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient,
+  agentPolicy: AgentPolicy,
+  installedPackagePolicies: Array<
+    Partial<Omit<NewPackagePolicy, 'inputs'>> & {
+      name: string;
+      installedPackage: Installation;
+      inputs?: InputsOverride[];
+    }
+  >,
+  defaultOutput: Output
+) {
+  return await Promise.all(
+    installedPackagePolicies.map(async ({ installedPackage, name, description, inputs }) =>
+      addPackageToAgentPolicy(
+        soClient,
+        esClient,
+        installedPackage,
+        agentPolicy,
+        defaultOutput,
+        name,
+        description,
+        (policy) => overridePackageInputs(policy, inputs)
+      )
+    )
+  );
+}
+
+async function ensureInstalledPreconfiguredPackage(
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient,
+  pkgName: string,
+  pkgVersion: string
+) {
+  return ensureInstalledPackage({
+    savedObjectsClient: soClient,
+    pkgName,
+    esClient,
+    pkgVersion,
+  });
+}
+
+function overridePackageInputs(
+  basePackagePolicy: NewPackagePolicy,
+  inputsOverride?: InputsOverride[]
+) {
+  if (!inputsOverride) return basePackagePolicy;
+
+  const inputs = [...basePackagePolicy.inputs];
+  const packageName = basePackagePolicy.package!.name;
+
+  for (const override of inputsOverride) {
+    const originalInput = inputs.find((i) => i.type === override.type);
+    if (!originalInput) {
+      throw new Error(
+        i18n.translate('xpack.fleet.packagePolicyInputOverrideError', {
+          defaultMessage: 'Input type {inputType} does not exist on package {packageName}',
+          values: {
+            inputType: override.type,
+            packageName,
+          },
+        })
+      );
+    }
+
+    if (typeof override.enabled !== 'undefined') originalInput.enabled = override.enabled;
+
+    if (override.vars) {
+      try {
+        deepMergeVars(override, originalInput);
+      } catch (e) {
+        throw new Error(
+          i18n.translate('xpack.fleet.packagePolicyVarOverrideError', {
+            defaultMessage: 'Var {varName} does not exist on {inputType} of package {packageName}',
+            values: {
+              varName: e.message,
+              inputType: override.type,
+              packageName,
+            },
+          })
+        );
+      }
+    }
+
+    if (override.streams) {
+      for (const stream of override.streams) {
+        const originalStream = originalInput.streams.find(
+          (s) => s.data_stream.dataset === stream.data_stream.dataset
+        );
+        if (!originalStream) {
+          throw new Error(
+            i18n.translate('xpack.fleet.packagePolicyStreamOverrideError', {
+              defaultMessage:
+                'Data stream {streamSet} does not exist on {inputType} of package {packageName}',
+              values: {
+                streamSet: stream.data_stream.dataset,
+                inputType: override.type,
+                packageName,
+              },
+            })
+          );
+        }
+
+        if (typeof stream.enabled !== 'undefined') originalStream.enabled = stream.enabled;
+
+        if (stream.vars) {
+          try {
+            deepMergeVars(stream as InputsOverride, originalStream);
+          } catch (e) {
+            throw new Error(
+              i18n.translate('xpack.fleet.packagePolicyStreamVarOverrideError', {
+                defaultMessage:
+                  'Var {varName} does not exist on {streamSet} for {inputType} of package {packageName}',
+                values: {
+                  varName: e.message,
+                  streamSet: stream.data_stream.dataset,
+                  inputType: override.type,
+                  packageName,
+                },
+              })
+            );
+          }
+        }
+      }
+    }
+  }
+
+  return { ...basePackagePolicy, inputs };
+}
+
+function deepMergeVars(
+  override: InputsOverride,
+  original: NewPackagePolicyInput | NewPackagePolicyInputStream
+) {
+  for (const { name, ...val } of override.vars!) {
+    if (!original.vars || !Reflect.has(original.vars, name)) {
+      throw new Error(name);
+    }
+    const originalVar = original.vars[name];
+    Reflect.set(original.vars, name, { ...originalVar, ...val });
+  }
+}

x-pack/plugins/fleet/server/services/setup.ts

@@ -7,26 +7,22 @@
 
 import uuid from 'uuid';
 import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server';
+import { i18n } from '@kbn/i18n';
 
-import {
-  packageToPackagePolicy,
-  DEFAULT_AGENT_POLICIES_PACKAGES,
-  FLEET_SERVER_PACKAGE,
-} from '../../common';
+import { DEFAULT_AGENT_POLICIES_PACKAGES, FLEET_SERVER_PACKAGE } from '../../common';
 
-import type { PackagePolicy, AgentPolicy, Installation, Output } from '../../common';
+import type { PackagePolicy } from '../../common';
 
 import { SO_SEARCH_LIMIT } from '../constants';
 
-import { agentPolicyService } from './agent_policy';
+import { agentPolicyService, addPackageToAgentPolicy } from './agent_policy';
 import { outputService } from './output';
 import {
   ensureInstalledDefaultPackages,
   ensureInstalledPackage,
   ensurePackagesCompletedInstall,
 } from './epm/packages/install';
-import { getPackageInfo } from './epm/packages';
-import { packagePolicyService } from './package_policy';
+
 import { generateEnrollmentAPIKey } from './api_keys';
 import { settingsService } from '.';
 import { awaitIfPending } from './setup_utils';
@@ -55,7 +51,7 @@ async function createSetupSideEffects(
   const [
     installedPackages,
     defaultOutput,
-    { created: defaultAgentPolicyCreated, defaultAgentPolicy },
+    { created: defaultAgentPolicyCreated, policy: defaultAgentPolicy },
     { created: defaultFleetServerPolicyCreated, policy: defaultFleetServerPolicy },
   ] = await Promise.all([
     // packages installed by default
@@ -110,13 +106,21 @@ async function createSetupSideEffects(
       true
     );
     if (!agentPolicyWithPackagePolicies) {
-      throw new Error('Policy not found');
+      throw new Error(
+        i18n.translate('xpack.fleet.setup.policyNotFoundError', {
+          defaultMessage: 'Policy not found',
+        })
+      );
     }
     if (
       agentPolicyWithPackagePolicies.package_policies.length &&
       typeof agentPolicyWithPackagePolicies.package_policies[0] === 'string'
     ) {
-      throw new Error('Policy not found');
+      throw new Error(
+        i18n.translate('xpack.fleet.setup.policyNotFoundError', {
+          defaultMessage: 'Policy not found',
+        })
+      );
     }
 
     for (const installedPackage of installedPackages) {
@@ -210,7 +214,11 @@ export async function setupFleet(
   // save fleet admin user
   const defaultOutputId = await outputService.getDefaultOutputId(soClient);
   if (!defaultOutputId) {
-    throw new Error('Default output does not exist');
+    throw new Error(
+      i18n.translate('xpack.fleet.setup.defaultOutputError', {
+        defaultMessage: 'Default output does not exist',
+      })
+    );
   }
 
   await outputService.updateOutput(soClient, defaultOutputId, {
@@ -242,28 +250,3 @@ export async function setupFleet(
 function generateRandomPassword() {
   return Buffer.from(uuid.v4()).toString('base64');
 }
-
-async function addPackageToAgentPolicy(
-  soClient: SavedObjectsClientContract,
-  esClient: ElasticsearchClient,
-  packageToInstall: Installation,
-  agentPolicy: AgentPolicy,
-  defaultOutput: Output
-) {
-  const packageInfo = await getPackageInfo({
-    savedObjectsClient: soClient,
-    pkgName: packageToInstall.name,
-    pkgVersion: packageToInstall.version,
-  });
-
-  const newPackagePolicy = packageToPackagePolicy(
-    packageInfo,
-    agentPolicy.id,
-    defaultOutput.id,
-    agentPolicy.namespace
-  );
-
-  await packagePolicyService.create(soClient, esClient, newPackagePolicy, {
-    bumpRevision: false,
-  });
-}

x-pack/plugins/fleet/server/types/index.tsx

@@ -32,6 +32,7 @@ export {
   AgentPolicy,
   AgentPolicySOAttributes,
   NewAgentPolicy,
+  PreconfiguredAgentPolicy,
   AgentPolicyStatus,
   DataStream,
   Output,

x-pack/plugins/fleet/server/types/models/agent_policy.ts

@@ -11,7 +11,7 @@ import { agentPolicyStatuses, dataTypes } from '../../../common';
 
 import { PackagePolicySchema, NamespaceSchema } from './package_policy';
 
-const AgentPolicyBaseSchema = {
+export const AgentPolicyBaseSchema = {
   name: schema.string({ minLength: 1 }),
   namespace: NamespaceSchema,
   description: schema.maybe(schema.string()),

x-pack/plugins/fleet/server/types/models/index.ts

@@ -10,3 +10,4 @@ export * from './agent';
 export * from './package_policy';
 export * from './output';
 export * from './enrollment_api_key';
+export * from './preconfiguration';

x-pack/plugins/fleet/server/types/models/preconfiguration.ts

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { i18n } from '@kbn/i18n';
+import { schema } from '@kbn/config-schema';
+import semverValid from 'semver/functions/valid';
+
+import { AgentPolicyBaseSchema } from './agent_policy';
+import { NamespaceSchema } from './package_policy';
+
+const varsSchema = schema.maybe(
+  schema.arrayOf(
+    schema.object({
+      name: schema.string(),
+      type: schema.maybe(schema.string()),
+      value: schema.oneOf([schema.string(), schema.number()]),
+    })
+  )
+);
+
+export const PreconfiguredPackagesSchema = schema.arrayOf(
+  schema.object({
+    name: schema.string(),
+    version: schema.string({
+      validate: (value) => {
+        if (!semverValid(value)) {
+          return i18n.translate('xpack.fleet.config.invalidPackageVersionError', {
+            defaultMessage: 'must be a valid semver',
+          });
+        }
+      },
+    }),
+  })
+);
+
+export const PreconfiguredAgentPoliciesSchema = schema.arrayOf(
+  schema.object({
+    ...AgentPolicyBaseSchema,
+    namespace: schema.maybe(NamespaceSchema),
+    id: schema.oneOf([schema.string(), schema.number()]),
+    package_policies: schema.arrayOf(
+      schema.object({
+        name: schema.string(),
+        package: schema.object({
+          name: schema.string(),
+        }),
+        description: schema.maybe(schema.string()),
+        namespace: schema.maybe(NamespaceSchema),
+        inputs: schema.maybe(
+          schema.arrayOf(
+            schema.object({
+              type: schema.string(),
+              enabled: schema.maybe(schema.boolean()),
+              vars: varsSchema,
+              streams: schema.maybe(
+                schema.arrayOf(
+                  schema.object({
+                    data_stream: schema.object({
+                      type: schema.maybe(schema.string()),
+                      dataset: schema.string(),
+                    }),
+                    enabled: schema.maybe(schema.boolean()),
+                    vars: varsSchema,
+                  })
+                )
+              ),
+            })
+          )
+        ),
+      })
+    ),
+  })
+);

x-pack/plugins/fleet/server/types/rest_spec/index.ts

@@ -13,5 +13,6 @@ export * from './epm';
 export * from './enrollment_api_key';
 export * from './install_script';
 export * from './output';
+export * from './preconfiguration';
 export * from './settings';
 export * from './setup';

x-pack/plugins/fleet/server/types/rest_spec/preconfiguration.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { schema } from '@kbn/config-schema';
+
+import { PreconfiguredAgentPoliciesSchema, PreconfiguredPackagesSchema } from '../models';
+
+export const PutPreconfigurationSchema = {
+  body: schema.object({
+    agentPolicies: schema.maybe(PreconfiguredAgentPoliciesSchema),
+    packages: schema.maybe(PreconfiguredPackagesSchema),
+  }),
+};

x-pack/test/fleet_api_integration/apis/index.js

@@ -47,5 +47,8 @@ export default function ({ loadTestFile }) {
 
     // Settings
     loadTestFile(require.resolve('./settings/index'));
+
+    // Preconfiguration
+    loadTestFile(require.resolve('./preconfiguration/index'));
   });
 }

x-pack/test/fleet_api_integration/apis/preconfiguration/index.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export default function loadTests({ loadTestFile }) {
+  describe('Preconfiguration Endpoints', () => {
+    loadTestFile(require.resolve('./preconfiguration'));
+  });
+}

x-pack/test/fleet_api_integration/apis/preconfiguration/preconfiguration.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { PRECONFIGURATION_API_ROUTES } from '../../../../plugins/fleet/common/constants';
+import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
+import { skipIfNoDockerRegistry } from '../../helpers';
+
+export default function (providerContext: FtrProviderContext) {
+  const { getService } = providerContext;
+  const supertest = getService('supertest');
+
+  // use function () {} and not () => {} here
+  // because `this` has to point to the Mocha context
+  // see https://mochajs.org/#arrow-functions
+
+  describe('Preconfiguration', async () => {
+    skipIfNoDockerRegistry(providerContext);
+    before(async () => {
+      await getService('esArchiver').load('empty_kibana');
+      await getService('esArchiver').load('fleet/empty_fleet_server');
+    });
+
+    after(async () => {
+      await getService('esArchiver').unload('fleet/empty_fleet_server');
+      await getService('esArchiver').unload('empty_kibana');
+    });
+
+    // Basic health check for the API; functionality is covered by the unit tests
+    it('should succeed with an empty payload', async () => {
+      const { body } = await supertest
+        .put(PRECONFIGURATION_API_ROUTES.PUT_PRECONFIG)
+        .set('kbn-xsrf', 'xxxx')
+        .send({})
+        .expect(200);
+
+      expect(body).to.eql({
+        packages: [],
+        policies: [],
+      });
+    });
+  });
+}