[Ingest Manager] Use optional registryProxyUrl setting when contacting Registry (#78648)

## Summary
If given a `xpack.fleet.registryProxyUrl` setting, Package Manager will use it when contacting the Registry. This only affects the outbound connection Package Manager makes to the Registry to search for available packages, download assets, etc.

### Configuration
<details><summary><strike>Initial PR: common environment variables</strike></summary>

<p>Currently the value must come from a <a href="https://github.com/Rob--W/proxy-from-env#environment-variables">list of popular environment variables</a> which include <code>ALL_PROXY</code>, <code>HTTPS_PROXY</code>, lowercase versions of those, and many more.</p>

<p>Start kibana with a proxy set in an environment variable like: <code>HTTPS_PROXY=https://localhost:8443 yarn start</code></p>

</details>

_update_ based on discussion in the comments, the initial environment variables approach was removed in favor of `xpack.ingestManager.registryProxyUrl`

#### see https://github.com/elastic/kibana/issues/78968 for additional configuration coming later

### Checklist
- [ ] ~~[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials.~~ Created https://github.com/elastic/kibana/issues/78961 to track
- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios

Created https://github.com/elastic/kibana/issues/78968 to track the additional configuration work

refs https://github.com/elastic/kibana/issues/70710

x-pack/plugins/ingest_manager/common/types/index.ts

@@ -9,6 +9,7 @@ export * from './rest_spec';
 export interface IngestManagerConfigType {
   enabled: boolean;
   registryUrl?: string;
+  registryProxyUrl?: string;
   agents: {
     enabled: boolean;
     tlsCheckDisabled: boolean;

x-pack/plugins/ingest_manager/server/index.ts

@@ -31,6 +31,7 @@ export const config: PluginConfigDescriptor = {
   schema: schema.object({
     enabled: schema.boolean({ defaultValue: true }),
     registryUrl: schema.maybe(schema.uri()),
+    registryProxyUrl: schema.maybe(schema.uri()),
     agents: schema.object({
       enabled: schema.boolean({ defaultValue: true }),
       tlsCheckDisabled: schema.boolean({ defaultValue: false }),

x-pack/plugins/ingest_manager/server/services/epm/registry/proxy.test.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import HttpProxyAgent from 'http-proxy-agent';
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import { getProxyAgent, getProxyAgentOptions } from './proxy';
+
+describe('getProxyAgent', () => {
+  test('return HttpsProxyAgent for https proxy url', () => {
+    const agent = getProxyAgent({
+      proxyUrl: 'https://proxyhost',
+      targetUrl: 'https://targethost',
+    });
+    expect(agent instanceof HttpsProxyAgent).toBeTruthy();
+  });
+
+  test('return HttpProxyAgent for http proxy url', () => {
+    const agent = getProxyAgent({
+      proxyUrl: 'http://proxyhost',
+      targetUrl: 'http://targethost',
+    });
+    expect(agent instanceof HttpProxyAgent).toBeTruthy();
+  });
+});
+
+describe('getProxyAgentOptions', () => {
+  test('return url only for https', () => {
+    const httpsProxy = 'https://12.34.56.78:910';
+
+    const optionsA = getProxyAgentOptions({
+      proxyUrl: httpsProxy,
+      targetUrl: 'https://targethost',
+    });
+    expect(optionsA).toEqual({
+      headers: { Host: 'targethost' },
+      host: '12.34.56.78',
+      port: 910,
+      protocol: 'https:',
+      rejectUnauthorized: undefined,
+    });
+
+    const optionsB = getProxyAgentOptions({
+      proxyUrl: httpsProxy,
+      targetUrl: 'https://example.com/?a=b&c=d',
+    });
+    expect(optionsB).toEqual({
+      headers: { Host: 'example.com' },
+      host: '12.34.56.78',
+      port: 910,
+      protocol: 'https:',
+      rejectUnauthorized: undefined,
+    });
+
+    // given http value and https proxy
+    const optionsC = getProxyAgentOptions({
+      proxyUrl: httpsProxy,
+      targetUrl: 'http://example.com/?a=b&c=d',
+    });
+    expect(optionsC).toEqual({
+      headers: { Host: 'example.com' },
+      host: '12.34.56.78',
+      port: 910,
+      protocol: 'https:',
+      rejectUnauthorized: undefined,
+    });
+  });
+});

x-pack/plugins/ingest_manager/server/services/epm/registry/proxy.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import HttpProxyAgent from 'http-proxy-agent';
+import HttpsProxyAgent, {
+  HttpsProxyAgent as IHttpsProxyAgent,
+  HttpsProxyAgentOptions,
+} from 'https-proxy-agent';
+import { appContextService } from '../../index';
+export interface RegistryProxySettings {
+  proxyUrl: string;
+  proxyHeaders?: Record<string, string>;
+  proxyRejectUnauthorizedCertificates?: boolean;
+}
+
+type ProxyAgent = IHttpsProxyAgent | HttpProxyAgent;
+type GetProxyAgentParams = RegistryProxySettings & { targetUrl: string };
+
+export function getRegistryProxyUrl(): string | undefined {
+  const proxyUrl = appContextService.getConfig()?.registryProxyUrl;
+  return proxyUrl;
+}
+
+export function getProxyAgent(options: GetProxyAgentParams): ProxyAgent {
+  const isHttps = options.targetUrl.startsWith('https:');
+  const agentOptions = isHttps && getProxyAgentOptions(options);
+  const agent: ProxyAgent = isHttps
+    ? // @ts-expect-error ts(7009) HttpsProxyAgent isn't a class so TS complains about using `new`
+      new HttpsProxyAgent(agentOptions)
+    : new HttpProxyAgent(options.proxyUrl);
+
+  return agent;
+}
+
+export function getProxyAgentOptions(options: GetProxyAgentParams): HttpsProxyAgentOptions {
+  const endpointParsed = new URL(options.targetUrl);
+  const proxyParsed = new URL(options.proxyUrl);
+
+  return {
+    host: proxyParsed.hostname,
+    port: Number(proxyParsed.port),
+    protocol: proxyParsed.protocol,
+    // The headers to send
+    headers: options.proxyHeaders || {
+      // the proxied URL's host is put in the header instead of the server's actual host
+      Host: endpointParsed.host,
+    },
+    // do not fail on invalid certs if value is false
+    rejectUnauthorized: options.proxyRejectUnauthorizedCertificates,
+  };
+}

x-pack/plugins/ingest_manager/server/services/epm/registry/requests.ts

@@ -4,17 +4,18 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import fetch, { FetchError, Response } from 'node-fetch';
+import fetch, { FetchError, Response, RequestInit } from 'node-fetch';
 import pRetry from 'p-retry';
 import { streamToString } from './streams';
+import { appContextService } from '../../app_context';
 import { RegistryError, RegistryConnectionError, RegistryResponseError } from '../../../errors';
+import { getProxyAgent, getRegistryProxyUrl } from './proxy';
 
 type FailedAttemptErrors = pRetry.FailedAttemptError | FetchError | Error;
 
 // not sure what to call this function, but we're not exporting it
 async function registryFetch(url: string) {
-  const response = await fetch(url);
-
+  const response = await fetch(url, getFetchOptions(url));
   if (response.ok) {
     return response;
   } else {
@@ -81,3 +82,17 @@ function isFetchError(error: FailedAttemptErrors): error is FetchError {
 function isSystemError(error: FailedAttemptErrors): boolean {
   return isFetchError(error) && error.type === 'system';
 }
+
+export function getFetchOptions(targetUrl: string): RequestInit | undefined {
+  const proxyUrl = getRegistryProxyUrl();
+  if (!proxyUrl) {
+    return undefined;
+  }
+
+  const logger = appContextService.getLogger();
+  logger.debug(`Using ${proxyUrl} as proxy for ${targetUrl}`);
+
+  return {
+    agent: getProxyAgent({ proxyUrl, targetUrl }),
+  };
+}