[Fleet] Have EPR register new categories / Show category counts (#114429)

src/plugins/custom_integrations/common/index.ts

@@ -42,9 +42,6 @@ export const INTEGRATION_CATEGORY_DISPLAY = {
   // Kibana added
   upload_file: 'Upload a file',
   language_client: 'Language client',
-
-  // Internal
-  updates_available: 'Updates available',
 };
 
 /**

src/plugins/custom_integrations/server/index.ts

@@ -19,7 +19,7 @@ export function plugin(initializerContext: PluginInitializerContext) {
 
 export { CustomIntegrationsPluginSetup, CustomIntegrationsPluginStart } from './types';
 
-export type { IntegrationCategory, IntegrationCategoryCount, CustomIntegration } from '../common';
+export type { IntegrationCategory, CustomIntegration } from '../common';
 
 export const config = {
   schema: schema.object({}),

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -227,6 +227,7 @@ export type RegistrySearchResult = Pick<
   | 'internal'
   | 'data_streams'
   | 'policy_templates'
+  | 'categories'
 >;
 
 export type ScreenshotItem = RegistryImage | PackageSpecScreenshot;
@@ -376,6 +377,7 @@ export interface IntegrationCardItem {
   icons: Array<PackageSpecIcon | CustomIntegrationIcon>;
   integration: string;
   id: string;
+  categories: string[];
 }
 
 export type PackagesGroupedByStatus = Record<ValueOf<InstallationStatus>, PackageList>;

x-pack/plugins/fleet/public/applications/integrations/hooks/use_merge_epr_with_replacements.test.ts

@@ -5,10 +5,10 @@
  * 2.0.
  */
 
-import type { PackageListItem } from '../../common/types/models';
-import type { CustomIntegration } from '../../../../../src/plugins/custom_integrations/common';
+import type { PackageListItem } from '../../../../common/types/models';
+import type { CustomIntegration } from '../../../../../../../src/plugins/custom_integrations/common';
 
-import type { IntegrationCategory } from '../../../../../src/plugins/custom_integrations/common';
+import type { IntegrationCategory } from '../../../../../../../src/plugins/custom_integrations/common';
 
 import { useMergeEprPackagesWithReplacements } from './use_merge_epr_with_replacements';
 
@@ -46,7 +46,7 @@ describe('useMergeEprWithReplacements', () => {
       },
     ]);
 
-    expect(useMergeEprPackagesWithReplacements(eprPackages, replacements, '')).toEqual([
+    expect(useMergeEprPackagesWithReplacements(eprPackages, replacements)).toEqual([
       {
         name: 'aws',
         release: 'ga',
@@ -80,7 +80,7 @@ describe('useMergeEprWithReplacements', () => {
       },
     ]);
 
-    expect(useMergeEprPackagesWithReplacements(eprPackages, replacements, '')).toEqual([
+    expect(useMergeEprPackagesWithReplacements(eprPackages, replacements)).toEqual([
       {
         eprOverlap: 'activemq',
         id: 'activemq-logs',
@@ -108,7 +108,7 @@ describe('useMergeEprWithReplacements', () => {
       },
     ]);
 
-    expect(useMergeEprPackagesWithReplacements(eprPackages, replacements, '')).toEqual([
+    expect(useMergeEprPackagesWithReplacements(eprPackages, replacements)).toEqual([
       {
         name: 'activemq',
         release: 'beta',
@@ -120,32 +120,6 @@ describe('useMergeEprWithReplacements', () => {
     ]);
   });
 
-  test('should respect category assignment', () => {
-    const eprPackages: PackageListItem[] = mockEprPackages([
-      {
-        name: 'activemq',
-        release: 'beta',
-      },
-    ]);
-    const replacements: CustomIntegration[] = mockIntegrations([
-      {
-        id: 'prometheus',
-        categories: ['monitoring', 'datastore'],
-      },
-      {
-        id: 'oracle',
-        categories: ['datastore'],
-      },
-    ]);
-
-    expect(useMergeEprPackagesWithReplacements(eprPackages, replacements, 'web')).toEqual([
-      {
-        name: 'activemq',
-        release: 'beta',
-      },
-    ]);
-  });
-
   test('should consists of all 3 types (ga eprs, replacements for non-ga eprs, replacements without epr equivalent', () => {
     const eprPackages: PackageListItem[] = mockEprPackages([
       {
@@ -190,7 +164,7 @@ describe('useMergeEprWithReplacements', () => {
       },
     ]);
 
-    expect(useMergeEprPackagesWithReplacements(eprPackages, replacements, '')).toEqual([
+    expect(useMergeEprPackagesWithReplacements(eprPackages, replacements)).toEqual([
       {
         name: 'aws',
         release: 'ga',

x-pack/plugins/fleet/public/applications/integrations/hooks/use_merge_epr_with_replacements.ts

@@ -5,12 +5,9 @@
  * 2.0.
  */
 
-import type { PackageListItem } from '../../common/types/models';
-import type {
-  CustomIntegration,
-  IntegrationCategory,
-} from '../../../../../src/plugins/custom_integrations/common';
-import { filterCustomIntegrations } from '../../../../../src/plugins/custom_integrations/public';
+import type { PackageListItem } from '../../../../common/types/models';
+import type { CustomIntegration } from '../../../../../../../src/plugins/custom_integrations/common';
+import { filterCustomIntegrations } from '../../../../../../../src/plugins/custom_integrations/public';
 
 // Export this as a utility to find replacements for a package (e.g. in the overview-page for an EPR package)
 function findReplacementsForEprPackage(
@@ -26,17 +23,13 @@ function findReplacementsForEprPackage(
 
 export function useMergeEprPackagesWithReplacements(
   eprPackages: PackageListItem[],
-  replacements: CustomIntegration[],
-  category: IntegrationCategory | ''
+  replacements: CustomIntegration[]
 ): Array<PackageListItem | CustomIntegration> {
   const merged: Array<PackageListItem | CustomIntegration> = [];
-
-  const filteredReplacements = replacements.filter((customIntegration) => {
-    return !category || customIntegration.categories.includes(category);
-  });
+  const filteredReplacements = replacements;
 
   // Either select replacement or select beat
-  eprPackages.forEach((eprPackage) => {
+  eprPackages.forEach((eprPackage: PackageListItem) => {
     const hits = findReplacementsForEprPackage(
       filteredReplacements,
       eprPackage.name,

x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_card.stories.tsx

@@ -32,6 +32,7 @@ const args: Args = {
   url: '/',
   icons: [],
   integration: '',
+  categories: ['foobar'],
 };
 
 const argTypes = {

x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid.stories.tsx

@@ -47,6 +47,7 @@ export const List = (props: Args) => (
         url: 'https://example.com',
         icons: [],
         integration: 'integation',
+        categories: ['web'],
       },
       {
         title: 'Package Two',
@@ -58,6 +59,7 @@ export const List = (props: Args) => (
         url: 'https://example.com',
         icons: [],
         integration: 'integation',
+        categories: ['web'],
       },
       {
         title: 'Package Three',
@@ -69,6 +71,7 @@ export const List = (props: Args) => (
         url: 'https://example.com',
         icons: [],
         integration: 'integation',
+        categories: ['web'],
       },
       {
         title: 'Package Four',
@@ -80,6 +83,7 @@ export const List = (props: Args) => (
         url: 'https://example.com',
         icons: [],
         integration: 'integation',
+        categories: ['web'],
       },
       {
         title: 'Package Five',
@@ -91,6 +95,7 @@ export const List = (props: Args) => (
         url: 'https://example.com',
         icons: [],
         integration: 'integation',
+        categories: ['web'],
       },
       {
         title: 'Package Six',
@@ -102,6 +107,7 @@ export const List = (props: Args) => (
         url: 'https://example.com',
         icons: [],
         integration: 'integation',
+        categories: ['web'],
       },
     ]}
     onSearchChange={action('onSearchChange')}

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/home/available_packages.tsx

@@ -8,6 +8,7 @@
 import React, { memo, useMemo } from 'react';
 import { useLocation, useHistory, useParams } from 'react-router-dom';
 import { i18n } from '@kbn/i18n';
+import _ from 'lodash';
 
 import { pagePathGetters } from '../../../../constants';
 import {
@@ -26,30 +27,53 @@ import type { CustomIntegration } from '../../../../../../../../../../src/plugin
 
 import type { PackageListItem } from '../../../../types';
 
-import type { IntegrationCategory } from '../../../../../../../../../../src/plugins/custom_integrations/common';
+import type { IntegrationCardItem } from '../../../../../../../common/types/models';
 
-import { useMergeEprPackagesWithReplacements } from '../../../../../../hooks/use_merge_epr_with_replacements';
+import { useMergeEprPackagesWithReplacements } from '../../../../hooks/use_merge_epr_with_replacements';
 
-import { mergeAndReplaceCategoryCounts } from './util';
-import { CategoryFacets } from './category_facets';
+import { mergeCategoriesAndCount } from './util';
+import { ALL_CATEGORY, CategoryFacets } from './category_facets';
 import type { CategoryFacet } from './category_facets';
 
 import type { CategoryParams } from '.';
 import { getParams, categoryExists, mapToCard } from '.';
 
+function getAllCategoriesFromIntegrations(pkg: PackageListItem) {
+  if (!doesPackageHaveIntegrations(pkg)) {
+    return pkg.categories;
+  }
+
+  const allCategories = pkg.policy_templates?.reduce((accumulator, integration) => {
+    return [...accumulator, ...(integration.categories || [])];
+  }, pkg.categories || []);
+
+  return _.uniq(allCategories);
+}
+
 // Packages can export multiple integrations, aka `policy_templates`
 // In the case where packages ship >1 `policy_templates`, we flatten out the
 // list of packages by bringing all integrations to top-level so that
 // each integration is displayed as its own tile
 const packageListToIntegrationsList = (packages: PackageList): PackageList => {
   return packages.reduce((acc: PackageList, pkg) => {
-    const { policy_templates: policyTemplates = [], ...restOfPackage } = pkg;
+    const {
+      policy_templates: policyTemplates = [],
+      categories: topCategories = [],
+      ...restOfPackage
+    } = pkg;
+
+    const topPackage = {
+      ...restOfPackage,
+      categories: getAllCategoriesFromIntegrations(pkg),
+    };
+
     return [
       ...acc,
-      restOfPackage,
+      topPackage,
       ...(doesPackageHaveIntegrations(pkg)
         ? policyTemplates.map((integration) => {
-            const { name, title, description, icons } = integration;
+            const { name, title, description, icons, categories = [] } = integration;
+            const allCategories = [...topCategories, ...categories];
             return {
               ...restOfPackage,
               id: `${restOfPackage}-${name}`,
@@ -57,6 +81,7 @@ const packageListToIntegrationsList = (packages: PackageList): PackageList => {
               title,
               description,
               icons: icons || restOfPackage.icons,
+              categories: _.uniq(allCategories),
             };
           })
         : []),
@@ -72,14 +97,11 @@ const title = i18n.translate('xpack.fleet.epmList.allTitle', {
 // or `location` to load data.  Ideally, we'll split this into "connected" and "pure" components.
 export const AvailablePackages: React.FC = memo(() => {
   useBreadcrumbs('integrations_all');
-
   const { selectedCategory, searchParam } = getParams(
     useParams<CategoryParams>(),
     useLocation().search
   );
-
   const history = useHistory();
-
   const { getHref, getAbsolutePath } = useLink();
 
   function setSelectedCategory(categoryId: string) {
@@ -89,7 +111,6 @@ export const AvailablePackages: React.FC = memo(() => {
     })[1];
     history.push(url);
   }
-
   function setSearchTerm(search: string) {
     // Use .replace so the browser's back button is not tied to single keystroke
     history.replace(
@@ -97,84 +118,51 @@ export const AvailablePackages: React.FC = memo(() => {
     );
   }
 
-  const { data: allCategoryPackagesRes, isLoading: isLoadingAllPackages } = useGetPackages({
+  const { data: eprPackages, isLoading: isLoadingAllPackages } = useGetPackages({
     category: '',
   });
-
-  const { data: categoryPackagesRes, isLoading: isLoadingCategoryPackages } = useGetPackages({
-    category: selectedCategory,
-  });
-
-  const { data: categoriesRes, isLoading: isLoadingCategories } = useGetCategories({
-    include_policy_templates: true,
-  });
-
-  const eprPackages = useMemo(
-    () => packageListToIntegrationsList(categoryPackagesRes?.response || []),
-    [categoryPackagesRes]
+  const eprIntegrationList = useMemo(
+    () => packageListToIntegrationsList(eprPackages?.response || []),
+    [eprPackages]
   );
-
-  const allEprPackages = useMemo(
-    () => packageListToIntegrationsList(allCategoryPackagesRes?.response || []),
-    [allCategoryPackagesRes]
-  );
-
   const { value: replacementCustomIntegrations } = useGetReplacementCustomIntegrations();
-
   const mergedEprPackages: Array<PackageListItem | CustomIntegration> =
     useMergeEprPackagesWithReplacements(
-      eprPackages || [],
-      replacementCustomIntegrations || [],
-      selectedCategory as IntegrationCategory
+      eprIntegrationList || [],
+      replacementCustomIntegrations || []
     );
-
   const { loading: isLoadingAppendCustomIntegrations, value: appendCustomIntegrations } =
     useGetAppendCustomIntegrations();
-
-  const filteredAddableIntegrations = appendCustomIntegrations
-    ? appendCustomIntegrations.filter((integration: CustomIntegration) => {
-        if (!selectedCategory) {
-          return true;
-        }
-        return integration.categories.indexOf(selectedCategory as IntegrationCategory) >= 0;
-      })
-    : [];
-
   const eprAndCustomPackages: Array<CustomIntegration | PackageListItem> = [
     ...mergedEprPackages,
-    ...filteredAddableIntegrations,
+    ...(appendCustomIntegrations || []),
   ];
-
-  eprAndCustomPackages.sort((a, b) => {
+  const cards: IntegrationCardItem[] = eprAndCustomPackages.map((item) => {
+    return mapToCard(getAbsolutePath, getHref, item);
+  });
+  cards.sort((a, b) => {
     return a.title.localeCompare(b.title);
   });
 
+  const { data: eprCategories, isLoading: isLoadingCategories } = useGetCategories({
+    include_policy_templates: true,
+  });
   const categories = useMemo(() => {
     const eprAndCustomCategories: CategoryFacet[] =
-      isLoadingCategories ||
-      isLoadingAppendCustomIntegrations ||
-      !appendCustomIntegrations ||
-      !categoriesRes
+      isLoadingCategories || !eprCategories
         ? []
-        : mergeAndReplaceCategoryCounts(
-            categoriesRes.response as CategoryFacet[],
-            appendCustomIntegrations
+        : mergeCategoriesAndCount(
+            eprCategories.response as Array<{ id: string; title: string; count: number }>,
+            cards
           );
-
     return [
       {
-        id: '',
-        count: (allEprPackages?.length || 0) + (appendCustomIntegrations?.length || 0),
+        ...ALL_CATEGORY,
+        count: cards.length,
       },
       ...(eprAndCustomCategories ? eprAndCustomCategories : []),
     ] as CategoryFacet[];
-  }, [
-    allEprPackages?.length,
-    appendCustomIntegrations,
-    categoriesRes,
-    isLoadingAppendCustomIntegrations,
-    isLoadingCategories,
-  ]);
+  }, [cards, eprCategories, isLoadingCategories]);
 
   if (!isLoadingCategories && !categoryExists(selectedCategory, categories)) {
     history.replace(pagePathGetters.integrations_all({ category: '', searchTerm: searchParam })[1]);
@@ -183,7 +171,6 @@ export const AvailablePackages: React.FC = memo(() => {
 
   const controls = categories ? (
     <CategoryFacets
-      showCounts={false}
       isLoading={isLoadingCategories || isLoadingAllPackages || isLoadingAppendCustomIntegrations}
       categories={categories}
       selectedCategory={selectedCategory}
@@ -193,17 +180,20 @@ export const AvailablePackages: React.FC = memo(() => {
     />
   ) : null;
 
-  const cards = eprAndCustomPackages.map((item) => {
-    return mapToCard(getAbsolutePath, getHref, item);
+  const filteredCards = cards.filter((c) => {
+    if (selectedCategory === '') {
+      return true;
+    }
+    return c.categories.includes(selectedCategory);
   });
 
   return (
     <PackageListGrid
-      isLoading={isLoadingCategoryPackages}
+      isLoading={isLoadingAllPackages}
       title={title}
       controls={controls}
       initialSearch={searchParam}
-      list={cards}
+      list={filteredCards}
       setSelectedCategory={setSelectedCategory}
       onSearchChange={setSearchTerm}
       showMissingIntegrationMessage

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/home/category_facets.tsx

@@ -11,18 +11,21 @@ import React from 'react';
 import { i18n } from '@kbn/i18n';
 
 import { Loading } from '../../../../components';
-import type { IntegrationCategoryCount } from '../../../../../../../../../../src/plugins/custom_integrations/common';
-import { INTEGRATION_CATEGORY_DISPLAY } from '../../../../../../../../../../src/plugins/custom_integrations/common';
 
-interface ALL_CATEGORY {
-  id: '';
+export interface CategoryFacet {
   count: number;
+  id: string;
+  title: string;
 }
 
-export type CategoryFacet = IntegrationCategoryCount | ALL_CATEGORY;
+export const ALL_CATEGORY = {
+  id: '',
+  title: i18n.translate('xpack.fleet.epmList.allPackagesFilterLinkText', {
+    defaultMessage: 'All',
+  }),
+};
 
 export interface Props {
-  showCounts: boolean;
   isLoading?: boolean;
   categories: CategoryFacet[];
   selectedCategory: string;
@@ -30,7 +33,6 @@ export interface Props {
 }
 
 export function CategoryFacets({
-  showCounts,
   isLoading,
   categories,
   selectedCategory,
@@ -42,28 +44,15 @@ export function CategoryFacets({
         <Loading />
       ) : (
         categories.map((category) => {
-          let title;
-
-          if (category.id === 'updates_available') {
-            title = i18n.translate('xpack.fleet.epmList.updatesAvailableFilterLinkText', {
-              defaultMessage: 'Updates available',
-            });
-          } else if (category.id === '') {
-            title = i18n.translate('xpack.fleet.epmList.allPackagesFilterLinkText', {
-              defaultMessage: 'All',
-            });
-          } else {
-            title = INTEGRATION_CATEGORY_DISPLAY[category.id];
-          }
           return (
             <EuiFacetButton
               isSelected={category.id === selectedCategory}
               key={category.id}
               id={category.id}
-              quantity={showCounts ? category.count : undefined}
+              quantity={category.count}
               onClick={() => onCategoryChange(category)}
             >
-              {title}
+              {category.title}
             </EuiFacetButton>
           );
         })

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/home/index.tsx

@@ -72,6 +72,7 @@ export const mapToCard = (
     name: 'name' in item ? item.name || '' : '',
     version: 'version' in item ? item.version || '' : '',
     release: 'release' in item ? item.release : undefined,
+    categories: ((item.categories || []) as string[]).filter((c: string) => !!c),
   };
 };
 

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/home/installed_packages.tsx

@@ -23,6 +23,7 @@ import { CategoryFacets } from './category_facets';
 
 import type { CategoryParams } from '.';
 import { getParams, categoryExists, mapToCard } from '.';
+import { ALL_CATEGORY } from './category_facets';
 
 const AnnouncementLink = () => {
   const { docLinks } = useStartServices();
@@ -114,12 +115,15 @@ export const InstalledPackages: React.FC = memo(() => {
   const categories: CategoryFacet[] = useMemo(
     () => [
       {
-        id: '',
+        ...ALL_CATEGORY,
         count: allInstalledPackages.length,
       },
       {
         id: 'updates_available',
         count: updatablePackages.length,
+        title: i18n.translate('xpack.fleet.epmList.updatesAvailableFilterLinkText', {
+          defaultMessage: 'Updates available',
+        }),
       },
     ],
     [allInstalledPackages.length, updatablePackages.length]
@@ -135,7 +139,6 @@ export const InstalledPackages: React.FC = memo(() => {
 
   const controls = (
     <CategoryFacets
-      showCounts={true}
       categories={categories}
       selectedCategory={selectedCategory}
       onCategoryChange={({ id }: CategoryFacet) => setSelectedCategory(id)}

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/home/util.ts

@@ -5,46 +5,57 @@
  * 2.0.
  */
 
-import type {
-  CustomIntegration,
-  IntegrationCategory,
-} from '../../../../../../../../../../src/plugins/custom_integrations/common';
+import type { IntegrationCategory } from '../../../../../../../../../../src/plugins/custom_integrations/common';
+import { INTEGRATION_CATEGORY_DISPLAY } from '../../../../../../../../../../src/plugins/custom_integrations/common';
+
+import type { IntegrationCardItem } from '../../../../../../../common/types/models';
 
 import type { CategoryFacet } from './category_facets';
 
-export function mergeAndReplaceCategoryCounts(
-  eprCounts: CategoryFacet[],
-  addableIntegrations: CustomIntegration[]
+export function mergeCategoriesAndCount(
+  eprCategoryList: Array<{ id: string; title: string; count: number }>, // EPR-categories from backend call to EPR
+  cards: IntegrationCardItem[]
 ): CategoryFacet[] {
-  const merged: CategoryFacet[] = [];
+  const facets: CategoryFacet[] = [];
 
-  const addIfMissing = (category: string, count: number) => {
-    const match = merged.find((c) => {
+  const addIfMissing = (category: string, count: number, title: string) => {
+    const match = facets.find((c) => {
       return c.id === category;
     });
 
     if (match) {
       match.count += count;
     } else {
-      merged.push({
-        id: category as IntegrationCategory,
+      facets.push({
+        id: category,
         count,
+        title,
       });
     }
   };
 
-  eprCounts.forEach((facet) => {
-    addIfMissing(facet.id, facet.count);
+  // Seed the list with the dynamic categories
+  eprCategoryList.forEach((facet) => {
+    addIfMissing(facet.id, 0, facet.title);
   });
-  addableIntegrations.forEach((integration) => {
-    integration.categories.forEach((cat) => {
-      addIfMissing(cat, 1);
+
+  // Count all the categories
+  cards.forEach((integration) => {
+    integration.categories.forEach((cat: string) => {
+      const title = INTEGRATION_CATEGORY_DISPLAY[cat as IntegrationCategory]
+        ? INTEGRATION_CATEGORY_DISPLAY[cat as IntegrationCategory]
+        : cat;
+      addIfMissing(cat, 1, title);
     });
   });
 
-  merged.sort((a, b) => {
+  const filledFacets = facets.filter((facet) => {
+    return facet.count > 0;
+  });
+
+  filledFacets.sort((a, b) => {
     return a.id.localeCompare(b.id);
   });
 
-  return merged;
+  return filledFacets;
 }

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -53,7 +53,6 @@ export async function getPackages(
   });
   // get the installed packages
   const packageSavedObjects = await getPackageSavedObjects(savedObjectsClient);
-
   // filter out any internal packages
   const savedObjectsVisible = packageSavedObjects.saved_objects.filter(
     (o) => !o.attributes.internal