Exclude disabled datasources and streams from agent config (#62869)

x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts

@@ -41,7 +41,7 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
       },
       {
         id: 'test-logs-bar',
-        enabled: false,
+        enabled: true,
         dataset: 'bar',
         config: {
           barVar: { value: 'bar-value' },
@@ -119,7 +119,7 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
             },
             {
               id: 'test-logs-bar',
-              enabled: false,
+              enabled: true,
               dataset: 'bar',
               barVar: 'bar-value',
               barVar2: [1, 2],
@@ -140,6 +140,44 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
     });
   });
 
+  it('returns agent datasource config without disabled streams', () => {
+    expect(
+      storedDatasourceToAgentDatasource({
+        ...mockDatasource,
+        inputs: [
+          {
+            ...mockInput,
+            streams: [{ ...mockInput.streams[0] }, { ...mockInput.streams[1], enabled: false }],
+          },
+        ],
+      })
+    ).toEqual({
+      id: 'mock-datasource',
+      namespace: 'default',
+      enabled: true,
+      use_output: 'default',
+      inputs: [
+        {
+          type: 'test-logs',
+          enabled: true,
+          inputVar: 'input-value',
+          inputVar3: {
+            testField: 'test',
+          },
+          streams: [
+            {
+              id: 'test-logs-foo',
+              enabled: true,
+              dataset: 'foo',
+              fooVar: 'foo-value',
+              fooVar2: [1, 2],
+            },
+          ],
+        },
+      ],
+    });
+  });
+
   it('returns agent datasource config without disabled inputs', () => {
     expect(
       storedDatasourceToAgentDatasource({

x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts

@@ -51,14 +51,16 @@ export const storedDatasourceToAgentDatasource = (
         const fullInput = {
           ...input,
           ...Object.entries(input.config || {}).reduce(configReducer, {}),
-          streams: input.streams.map(stream => {
-            const fullStream = {
-              ...stream,
-              ...Object.entries(stream.config || {}).reduce(configReducer, {}),
-            };
-            delete fullStream.config;
-            return fullStream;
-          }),
+          streams: input.streams
+            .filter(stream => stream.enabled)
+            .map(stream => {
+              const fullStream = {
+                ...stream,
+                ...Object.entries(stream.config || {}).reduce(configReducer, {}),
+              };
+              delete fullStream.config;
+              return fullStream;
+            }),
         };
         delete fullInput.config;
         return fullInput;

x-pack/plugins/ingest_manager/server/services/agent_config.ts

@@ -319,9 +319,9 @@ class AgentConfigService {
           return outputs;
         }, {} as FullAgentConfig['outputs']),
       },
-      datasources: (config.datasources as Datasource[]).map(ds =>
-        storedDatasourceToAgentDatasource(ds)
-      ),
+      datasources: (config.datasources as Datasource[])
+        .filter(datasource => datasource.enabled)
+        .map(ds => storedDatasourceToAgentDatasource(ds)),
       revision: config.revision,
     };