Revert "[Fleet][EPM] Revert es-storage-related commits (#85942) (#86017)" (#87183)

This reverts commit 70fbe91fb1.

x-pack/plugins/fleet/common/constants/epm.ts

@@ -3,8 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-
 export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
+export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets';
 export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';
 export const MAX_TIME_COMPLETE_INSTALL = 60000;
 

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -8,6 +8,7 @@
 // TODO: Update when https://github.com/elastic/kibana/issues/53021 is closed
 import { SavedObject, SavedObjectAttributes, SavedObjectReference } from 'src/core/public';
 import {
+  ASSETS_SAVED_OBJECT_TYPE,
   agentAssetTypes,
   dataTypes,
   defaultPackages,
@@ -268,6 +269,7 @@ export type PackageInfo =
 export interface Installation extends SavedObjectAttributes {
   installed_kibana: KibanaAssetReference[];
   installed_es: EsAssetReference[];
+  package_assets: PackageAssetReference[];
   es_index_patterns: Record<string, string>;
   name: string;
   version: string;
@@ -297,6 +299,10 @@ export type EsAssetReference = Pick<SavedObjectReference, 'id'> & {
   type: ElasticsearchAssetType;
 };
 
+export type PackageAssetReference = Pick<SavedObjectReference, 'id'> & {
+  type: typeof ASSETS_SAVED_OBJECT_TYPE;
+};
+
 export type RequiredPackage = typeof requiredPackages;
 
 export type DefaultPackages = typeof defaultPackages;

x-pack/plugins/fleet/server/constants/index.ts

@@ -40,6 +40,7 @@ export {
   PACKAGE_POLICY_SAVED_OBJECT_TYPE,
   OUTPUT_SAVED_OBJECT_TYPE,
   PACKAGES_SAVED_OBJECT_TYPE,
+  ASSETS_SAVED_OBJECT_TYPE,
   INDEX_PATTERN_SAVED_OBJECT_TYPE,
   ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
   GLOBAL_SETTINGS_SAVED_OBJECT_TYPE,

x-pack/plugins/fleet/server/routes/epm/handlers.ts

@@ -41,12 +41,13 @@ import {
   removeInstallation,
   getLimitedPackages,
   getInstallationObject,
+  getInstallation,
 } from '../../services/epm/packages';
 import { defaultIngestErrorHandler, ingestErrorToResponseOptions } from '../../errors';
 import { splitPkgKey } from '../../services/epm/registry';
 import { licenseService } from '../../services';
 import { getArchiveEntry } from '../../services/epm/archive/cache';
-import { bufferToStream } from '../../services/epm/streams';
+import { getAsset } from '../../services/epm/archive/storage';
 
 export const getCategoriesHandler: RequestHandler<
   undefined,
@@ -107,32 +108,51 @@ export const getFileHandler: RequestHandler<TypeOf<typeof GetFileRequestSchema.p
   try {
     const { pkgName, pkgVersion, filePath } = request.params;
     const savedObjectsClient = context.core.savedObjects.client;
-    const savedObject = await getInstallationObject({ savedObjectsClient, pkgName });
-    const pkgInstallSource = savedObject?.attributes.install_source;
-    // TODO: when package storage is available, remove installSource check and check cache and storage, remove registry call
-    if (pkgInstallSource === 'upload' && pkgVersion === savedObject?.attributes.version) {
-      const headerContentType = mime.contentType(path.extname(filePath));
-      if (!headerContentType) {
+    const installation = await getInstallation({ savedObjectsClient, pkgName });
+    const useLocalFile = pkgVersion === installation?.version;
+
+    if (useLocalFile) {
+      const assetPath = `${pkgName}-${pkgVersion}/${filePath}`;
+      const fileBuffer = getArchiveEntry(assetPath);
+      // only pull local installation if we don't have it cached
+      const storedAsset = !fileBuffer && (await getAsset({ savedObjectsClient, path: assetPath }));
+
+      // error, if neither is available
+      if (!fileBuffer && !storedAsset) {
+        return response.custom({
+          body: `installed package file not found: ${filePath}`,
+          statusCode: 404,
+        });
+      }
+
+      // if storedAsset is not available, fileBuffer *must* be
+      // b/c we error if we don't have at least one, and storedAsset is the least likely
+      const { buffer, contentType } = storedAsset
+        ? {
+            contentType: storedAsset.media_type,
+            buffer: storedAsset.data_utf8
+              ? Buffer.from(storedAsset.data_utf8, 'utf8')
+              : Buffer.from(storedAsset.data_base64, 'base64'),
+          }
+        : {
+            contentType: mime.contentType(path.extname(assetPath)),
+            buffer: fileBuffer,
+          };
+
+      if (!contentType) {
         return response.custom({
           body: `unknown content type for file: ${filePath}`,
           statusCode: 400,
         });
       }
-      const archiveFile = getArchiveEntry(`${pkgName}-${pkgVersion}/${filePath}`);
-      if (!archiveFile) {
-        return response.custom({
-          body: `uploaded package file not found: ${filePath}`,
-          statusCode: 404,
-        });
-      }
-      const headers: ResponseHeaders = {
-        'cache-control': 'max-age=10, public',
-        'content-type': headerContentType,
-      };
+
       return response.custom({
-        body: bufferToStream(archiveFile),
+        body: buffer,
         statusCode: 200,
-        headers,
+        headers: {
+          'cache-control': 'max-age=10, public',
+          'content-type': contentType,
+        },
       });
     } else {
       const registryResponse = await getFile(pkgName, pkgVersion, filePath);

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -12,6 +12,7 @@ import {
   AGENT_POLICY_SAVED_OBJECT_TYPE,
   PACKAGE_POLICY_SAVED_OBJECT_TYPE,
   PACKAGES_SAVED_OBJECT_TYPE,
+  ASSETS_SAVED_OBJECT_TYPE,
   AGENT_SAVED_OBJECT_TYPE,
   AGENT_EVENT_SAVED_OBJECT_TYPE,
   AGENT_ACTION_SAVED_OBJECT_TYPE,
@@ -304,6 +305,13 @@ const getSavedObjectTypes = (
             type: { type: 'keyword' },
           },
         },
+        package_assets: {
+          type: 'nested',
+          properties: {
+            id: { type: 'keyword' },
+            type: { type: 'keyword' },
+          },
+        },
         install_started_at: { type: 'date' },
         install_version: { type: 'keyword' },
         install_status: { type: 'keyword' },
@@ -311,6 +319,25 @@ const getSavedObjectTypes = (
       },
     },
   },
+  [ASSETS_SAVED_OBJECT_TYPE]: {
+    name: ASSETS_SAVED_OBJECT_TYPE,
+    hidden: false,
+    namespaceType: 'agnostic',
+    management: {
+      importableAndExportable: false,
+    },
+    mappings: {
+      properties: {
+        package_name: { type: 'keyword' },
+        package_version: { type: 'keyword' },
+        install_source: { type: 'keyword' },
+        asset_path: { type: 'keyword' },
+        media_type: { type: 'keyword' },
+        data_utf8: { type: 'text', index: false },
+        data_base64: { type: 'binary' },
+      },
+    },
+  },
 });
 
 export function registerSavedObjects(

x-pack/plugins/fleet/server/services/epm/archive/storage.ts

@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { extname } from 'path';
+import { isBinaryFile } from 'isbinaryfile';
+import mime from 'mime-types';
+import uuidv5 from 'uuid/v5';
+import { SavedObjectsClientContract, SavedObjectsBulkCreateObject } from 'src/core/server';
+import {
+  ASSETS_SAVED_OBJECT_TYPE,
+  InstallablePackage,
+  InstallSource,
+  PackageAssetReference,
+} from '../../../../common';
+import { getArchiveEntry } from './index';
+
+// could be anything, picked this from https://github.com/elastic/elastic-agent-client/issues/17
+const MAX_ES_ASSET_BYTES = 4 * 1024 * 1024;
+
+export interface PackageAsset {
+  package_name: string;
+  package_version: string;
+  install_source: string;
+  asset_path: string;
+  media_type: string;
+  data_utf8: string;
+  data_base64: string;
+}
+
+export function assetPathToObjectId(assetPath: string): string {
+  // uuid v5 requires a SHA-1 UUID as a namespace
+  // used to ensure same input produces the same id
+  return uuidv5(assetPath, '71403015-cdd5-404b-a5da-6c43f35cad84');
+}
+
+export async function archiveEntryToESDocument(opts: {
+  path: string;
+  buffer: Buffer;
+  name: string;
+  version: string;
+  installSource: InstallSource;
+}): Promise<PackageAsset> {
+  const { path, buffer, name, version, installSource } = opts;
+  const fileExt = extname(path);
+  const contentType = mime.lookup(fileExt);
+  const mediaType = mime.contentType(contentType || fileExt);
+  // can use to create a data URL like `data:${mediaType};base64,${base64Data}`
+
+  const bufferIsBinary = await isBinaryFile(buffer);
+  const dataUtf8 = bufferIsBinary ? '' : buffer.toString('utf8');
+  const dataBase64 = bufferIsBinary ? buffer.toString('base64') : '';
+
+  // validation: filesize? asset type? anything else
+  if (dataUtf8.length > MAX_ES_ASSET_BYTES) {
+    throw new Error(`File at ${path} is larger than maximum allowed size of ${MAX_ES_ASSET_BYTES}`);
+  }
+
+  if (dataBase64.length > MAX_ES_ASSET_BYTES) {
+    throw new Error(
+      `After base64 encoding file at ${path} is larger than maximum allowed size of ${MAX_ES_ASSET_BYTES}`
+    );
+  }
+
+  return {
+    package_name: name,
+    package_version: version,
+    install_source: installSource,
+    asset_path: path,
+    media_type: mediaType || '',
+    data_utf8: dataUtf8,
+    data_base64: dataBase64,
+  };
+}
+
+export async function removeArchiveEntries(opts: {
+  savedObjectsClient: SavedObjectsClientContract;
+  refs: PackageAssetReference[];
+}) {
+  const { savedObjectsClient, refs } = opts;
+  const results = await Promise.all(
+    refs.map((ref) => savedObjectsClient.delete(ASSETS_SAVED_OBJECT_TYPE, ref.id))
+  );
+  return results;
+}
+
+export async function saveArchiveEntries(opts: {
+  savedObjectsClient: SavedObjectsClientContract;
+  paths: string[];
+  packageInfo: InstallablePackage;
+  installSource: InstallSource;
+}) {
+  const { savedObjectsClient, paths, packageInfo, installSource } = opts;
+  const bulkBody = await Promise.all(
+    paths.map((path) => {
+      const buffer = getArchiveEntry(path);
+      if (!buffer) throw new Error(`Could not find ArchiveEntry at ${path}`);
+      const { name, version } = packageInfo;
+      return archiveEntryToBulkCreateObject({ path, buffer, name, version, installSource });
+    })
+  );
+
+  const results = await savedObjectsClient.bulkCreate<PackageAsset>(bulkBody);
+  return results;
+}
+
+export async function archiveEntryToBulkCreateObject(opts: {
+  path: string;
+  buffer: Buffer;
+  name: string;
+  version: string;
+  installSource: InstallSource;
+}): Promise<SavedObjectsBulkCreateObject<PackageAsset>> {
+  const { path, buffer, name, version, installSource } = opts;
+  const doc = await archiveEntryToESDocument({ path, buffer, name, version, installSource });
+  return {
+    id: assetPathToObjectId(doc.asset_path),
+    type: ASSETS_SAVED_OBJECT_TYPE,
+    attributes: doc,
+  };
+}
+
+export async function getAsset(opts: {
+  savedObjectsClient: SavedObjectsClientContract;
+  path: string;
+}) {
+  const { savedObjectsClient, path } = opts;
+  const assetSavedObject = await savedObjectsClient.get<PackageAsset>(
+    ASSETS_SAVED_OBJECT_TYPE,
+    assetPathToObjectId(path)
+  );
+  const storedAsset = assetSavedObject?.attributes;
+  if (!storedAsset) {
+    return;
+  }
+
+  return storedAsset;
+}

x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts

@@ -5,7 +5,13 @@
  */
 
 import { SavedObject, SavedObjectsClientContract } from 'src/core/server';
-import { InstallablePackage, InstallSource, MAX_TIME_COMPLETE_INSTALL } from '../../../../common';
+import {
+  InstallablePackage,
+  InstallSource,
+  PackageAssetReference,
+  MAX_TIME_COMPLETE_INSTALL,
+  ASSETS_SAVED_OBJECT_TYPE,
+} from '../../../../common';
 import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
 import {
   AssetReference,
@@ -24,6 +30,7 @@ import { deleteKibanaSavedObjectsAssets } from './remove';
 import { installTransform } from '../elasticsearch/transform/install';
 import { createInstallation, saveKibanaAssetsRefs, updateVersion } from './install';
 import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install';
+import { saveArchiveEntries } from '../archive/storage';
 import { ConcurrentInstallOperationError } from '../../../errors';
 
 // this is only exported for testing
@@ -188,11 +195,26 @@ export async function _installPackage({
     if (installKibanaAssetsError) throw installKibanaAssetsError;
     await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]);
 
+    const packageAssetResults = await saveArchiveEntries({
+      savedObjectsClient,
+      paths,
+      packageInfo,
+      installSource,
+    });
+    const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map(
+      (result) => ({
+        id: result.id,
+        type: ASSETS_SAVED_OBJECT_TYPE,
+      })
+    );
+
     // update to newly installed version when all assets are successfully installed
     if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion);
+
     await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
       install_version: pkgVersion,
       install_status: 'installed',
+      package_assets: packageAssetRefs,
     });
 
     return [

x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts

@@ -43,6 +43,7 @@ const mockInstallation: SavedObject<Installation> = {
     id: 'test-pkg',
     installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
     installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
+    package_assets: [],
     es_index_patterns: { pattern: 'pattern-name' },
     name: 'test package',
     version: '1.0.0',

x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts

@@ -15,6 +15,7 @@ const mockInstallation: SavedObject<Installation> = {
     id: 'test-pkg',
     installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
     installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
+    package_assets: [],
     es_index_patterns: { pattern: 'pattern-name' },
     name: 'test packagek',
     version: '1.0.0',
@@ -32,6 +33,7 @@ const mockInstallationUpdateFail: SavedObject<Installation> = {
     id: 'test-pkg',
     installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
     installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
+    package_assets: [],
     es_index_patterns: { pattern: 'pattern-name' },
     name: 'test packagek',
     version: '1.0.0',

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -379,6 +379,7 @@ export async function createInstallation(options: {
     {
       installed_kibana: [],
       installed_es: [],
+      package_assets: [],
       es_index_patterns: toSaveESIndexPatterns,
       name: pkgName,
       version: pkgVersion,

x-pack/plugins/fleet/server/services/epm/packages/remove.ts

@@ -24,6 +24,7 @@ import { packagePolicyService, appContextService } from '../..';
 import { splitPkgKey } from '../registry';
 import { deletePackageCache } from '../archive';
 import { deleteIlms } from '../elasticsearch/datastream_ilm/remove';
+import { removeArchiveEntries } from '../archive/storage';
 
 export async function removeInstallation(options: {
   savedObjectsClient: SavedObjectsClientContract;
@@ -49,7 +50,7 @@ export async function removeInstallation(options: {
       `unable to remove package with existing package policy(s) in use by agent(s)`
     );
 
-  // Delete the installed assets
+  // Delete the installed assets. Don't include installation.package_assets. Those are irrelevant to users
   const installedAssets = [...installation.installed_kibana, ...installation.installed_es];
   await deleteAssets(installation, savedObjectsClient, callCluster);
 
@@ -69,6 +70,8 @@ export async function removeInstallation(options: {
     version: pkgVersion,
   });
 
+  await removeArchiveEntries({ savedObjectsClient, refs: installation.package_assets });
+
   // successful delete's in SO client return {}. return something more useful
   return installedAssets;
 }

x-pack/plugins/security_solution/common/endpoint/generate_data.ts

@@ -1332,6 +1332,7 @@ export class EndpointDocGenerator {
             { id: 'logs-endpoint.events.security', type: 'index_template' },
             { id: 'metrics-endpoint.telemetry', type: 'index_template' },
           ] as EsAssetReference[],
+          package_assets: [],
           es_index_patterns: {
             alerts: 'logs-endpoint.alerts-*',
             events: 'events-endpoint-*',

x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts

@@ -433,6 +433,7 @@ const expectAssetsInstalled = ({
       ...res.attributes,
       installed_kibana: sortBy(res.attributes.installed_kibana, (o: AssetReference) => o.type),
       installed_es: sortBy(res.attributes.installed_es, (o: AssetReference) => o.type),
+      package_assets: sortBy(res.attributes.package_assets, (o: AssetReference) => o.type),
     };
     expect(sortedRes).eql({
       installed_kibana: [
@@ -495,6 +496,29 @@ const expectAssetsInstalled = ({
         test_logs: 'logs-all_assets.test_logs-*',
         test_metrics: 'metrics-all_assets.test_metrics-*',
       },
+      package_assets: [
+        { id: '333a22a1-e639-5af5-ae62-907ffc83d603', type: 'epm-packages-assets' },
+        { id: '256f3dad-6870-56c3-80a1-8dfa11e2d568', type: 'epm-packages-assets' },
+        { id: '3fa0512f-bc01-5c2e-9df1-bc2f2a8259c8', type: 'epm-packages-assets' },
+        { id: 'ea334ad8-80c2-5acd-934b-2a377290bf97', type: 'epm-packages-assets' },
+        { id: '96c6eb85-fe2e-56c6-84be-5fda976796db', type: 'epm-packages-assets' },
+        { id: '2d73a161-fa69-52d0-aa09-1bdc691b95bb', type: 'epm-packages-assets' },
+        { id: '0a00c2d2-ce63-5b9c-9aa0-0cf1938f7362', type: 'epm-packages-assets' },
+        { id: '691f0505-18c5-57a6-9f40-06e8affbdf7a', type: 'epm-packages-assets' },
+        { id: 'b36e6dd0-58f7-5dd0-a286-8187e4019274', type: 'epm-packages-assets' },
+        { id: 'f839c76e-d194-555a-90a1-3265a45789e4', type: 'epm-packages-assets' },
+        { id: '9af7bbb3-7d8a-50fa-acc9-9dde6f5efca2', type: 'epm-packages-assets' },
+        { id: '1e97a20f-9d1c-529b-8ff2-da4e8ba8bb71', type: 'epm-packages-assets' },
+        { id: '8cfe0a2b-7016-5522-87e4-6d352360d1fc', type: 'epm-packages-assets' },
+        { id: 'bd5ff3c5-655e-5385-9918-b60ff3040aad', type: 'epm-packages-assets' },
+        { id: '0954ce3b-3165-5c1f-a4c0-56eb5f2fa487', type: 'epm-packages-assets' },
+        { id: '60d6d054-57e4-590f-a580-52bf3f5e7cca', type: 'epm-packages-assets' },
+        { id: '47758dc2-979d-5fbe-a2bd-9eded68a5a43', type: 'epm-packages-assets' },
+        { id: '318959c9-997b-5a14-b328-9fc7355b4b74', type: 'epm-packages-assets' },
+        { id: 'e786cbd9-0f3b-5a0b-82a6-db25145ebf58', type: 'epm-packages-assets' },
+        { id: '53c94591-aa33-591d-8200-cd524c2a0561', type: 'epm-packages-assets' },
+        { id: 'b658d2d4-752e-54b8-afc2-4c76155c1466', type: 'epm-packages-assets' },
+      ],
       name: 'all_assets',
       version: '0.1.0',
       internal: false,

x-pack/test/fleet_api_integration/apis/epm/update_assets.ts

@@ -322,6 +322,26 @@ export default function (providerContext: FtrProviderContext) {
           test_logs: 'logs-all_assets.test_logs-*',
           test_metrics: 'metrics-all_assets.test_metrics-*',
         },
+        package_assets: [
+          { id: '3eb4c54a-638f-51b6-84e2-d53f5a666e37', type: 'epm-packages-assets' },
+          { id: '4acfbf69-7a27-5c58-9c99-7c86843d958f', type: 'epm-packages-assets' },
+          { id: '938655df-b339-523c-a9e4-123c89c0e1e1', type: 'epm-packages-assets' },
+          { id: 'eec4606c-dbfa-565b-8e9c-fce1e641f3fc', type: 'epm-packages-assets' },
+          { id: 'ef67e7e0-dca3-5a62-a42a-745db5ad7c1f', type: 'epm-packages-assets' },
+          { id: '64239d25-be40-5e10-94b5-f6b74b8c5474', type: 'epm-packages-assets' },
+          { id: '071b5113-4c9f-5ee9-aafe-d098a4c066f6', type: 'epm-packages-assets' },
+          { id: '498d8215-2613-5399-9a13-fa4f0bf513e2', type: 'epm-packages-assets' },
+          { id: 'd2f87071-c866-503a-8fcb-7b23a8c7afbf', type: 'epm-packages-assets' },
+          { id: '5a080eba-f482-545c-8695-6ccbd426b2a2', type: 'epm-packages-assets' },
+          { id: '28523a82-1328-578d-84cb-800970560200', type: 'epm-packages-assets' },
+          { id: 'cc1e3e1d-f27b-5d05-86f6-6e4b9a47c7dc', type: 'epm-packages-assets' },
+          { id: '5c3aa147-089c-5084-beca-53c00e72ac80', type: 'epm-packages-assets' },
+          { id: '48e582df-b1d2-5f88-b6ea-ba1fafd3a569', type: 'epm-packages-assets' },
+          { id: 'bf3b0b65-9fdc-53c6-a9ca-e76140e56490', type: 'epm-packages-assets' },
+          { id: '2e56f08b-1d06-55ed-abee-4708e1ccf0aa', type: 'epm-packages-assets' },
+          { id: 'c7bf1a39-e057-58a0-afde-fb4b48751d8c', type: 'epm-packages-assets' },
+          { id: '8c665f28-a439-5f43-b5fd-8fda7b576735', type: 'epm-packages-assets' },
+        ],
         name: 'all_assets',
         version: '0.2.0',
         internal: false,