Add support for reading request ID from X-Opaque-Id header (#71019) (#75502)

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

docs/development/core/server/kibana-plugin-core-server.kibanarequest.id.md

@@ -0,0 +1,18 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [KibanaRequest](./kibana-plugin-core-server.kibanarequest.md) &gt; [id](./kibana-plugin-core-server.kibanarequest.id.md)
+
+## KibanaRequest.id property
+
+A identifier to identify this request.
+
+<b>Signature:</b>
+
+```typescript
+readonly id: string;
+```
+
+## Remarks
+
+Depending on the user's configuration, this value may be sourced from the incoming request's `X-Opaque-Id` header which is not guaranteed to be unique per request.
+

docs/development/core/server/kibana-plugin-core-server.kibanarequest.md

@@ -26,6 +26,7 @@ export declare class KibanaRequest<Params = unknown, Query = unknown, Body = unk
 |  [body](./kibana-plugin-core-server.kibanarequest.body.md) |  | <code>Body</code> |  |
 |  [events](./kibana-plugin-core-server.kibanarequest.events.md) |  | <code>KibanaRequestEvents</code> | Request events [KibanaRequestEvents](./kibana-plugin-core-server.kibanarequestevents.md) |
 |  [headers](./kibana-plugin-core-server.kibanarequest.headers.md) |  | <code>Headers</code> | Readonly copy of incoming request headers. |
+|  [id](./kibana-plugin-core-server.kibanarequest.id.md) |  | <code>string</code> | A identifier to identify this request. |
 |  [isSystemRequest](./kibana-plugin-core-server.kibanarequest.issystemrequest.md) |  | <code>boolean</code> | Whether or not the request is a "system request" rather than an application-level request. Can be set on the client using the <code>HttpFetchOptions#asSystemRequest</code> option. |
 |  [params](./kibana-plugin-core-server.kibanarequest.params.md) |  | <code>Params</code> |  |
 |  [query](./kibana-plugin-core-server.kibanarequest.query.md) |  | <code>Query</code> |  |

docs/setup/settings.asciidoc

@@ -476,6 +476,12 @@ identifies this {kib} instance. *Default: `"your-hostname"`*
  | {kib} is served by a back end server. This
 setting specifies the port to use. *Default: `5601`*
 
+| `server.requestId.allowFromAnyIp:`
+ | Sets whether or not the X-Opaque-Id header should be trusted from any IP address for identifying requests in logs and forwarded to Elasticsearch.
+
+| `server.requestId.ipAllowlist:`
+ | A list of IPv4 and IPv6 address which the `X-Opaque-Id` header should be trusted from. Normally this would be set to the IP addresses of the load balancers or reverse-proxy that end users use to access Kibana. If any are set, `server.requestId.allowFromAnyIp` must also be set to `false.`
+
 | `server.rewriteBasePath:`
  | Specifies whether {kib} should
 rewrite requests that are prefixed with `server.basePath` or require that they

packages/kbn-config-schema/src/index.ts

@@ -34,6 +34,8 @@ import {
   ConditionalTypeValue,
   DurationOptions,
   DurationType,
+  IpOptions,
+  IpType,
   LiteralType,
   MapOfOptions,
   MapOfType,
@@ -107,6 +109,10 @@ function never(): Type<never> {
   return new NeverType();
 }
 
+function ip(options?: IpOptions): Type<string> {
+  return new IpType(options);
+}
+
 /**
  * Create an optional type
  */
@@ -207,6 +213,7 @@ export const schema = {
   conditional,
   contextRef,
   duration,
+  ip,
   literal,
   mapOf,
   maybe,

packages/kbn-config-schema/src/types/index.ts

@@ -36,3 +36,4 @@ export { StringOptions, StringType } from './string_type';
 export { UnionType } from './union_type';
 export { URIOptions, URIType } from './uri_type';
 export { NeverType } from './never_type';
+export { IpType, IpOptions } from './ip_type';

packages/kbn-config-schema/src/types/ip_type.test.ts

@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { schema } from '..';
+
+const { ip } = schema;
+
+describe('ip validation', () => {
+  test('accepts ipv4', () => {
+    expect(ip().validate('1.1.1.1')).toEqual('1.1.1.1');
+  });
+  test('accepts ipv6', () => {
+    expect(ip().validate('1200:0000:AB00:1234:0000:2552:7777:1313')).toEqual(
+      '1200:0000:AB00:1234:0000:2552:7777:1313'
+    );
+  });
+  test('rejects ipv6 when not specified', () => {
+    expect(() =>
+      ip({ versions: ['ipv4'] }).validate('1200:0000:AB00:1234:0000:2552:7777:1313')
+    ).toThrowErrorMatchingInlineSnapshot(`"value must be a valid ipv4 address"`);
+  });
+  test('rejects ipv4 when not specified', () => {
+    expect(() => ip({ versions: ['ipv6'] }).validate('1.1.1.1')).toThrowErrorMatchingInlineSnapshot(
+      `"value must be a valid ipv6 address"`
+    );
+  });
+  test('rejects invalid ip addresses', () => {
+    expect(() => ip().validate('1.1.1.1/24')).toThrowErrorMatchingInlineSnapshot(
+      `"value must be a valid ipv4 or ipv6 address"`
+    );
+    expect(() => ip().validate('99999.1.1.1')).toThrowErrorMatchingInlineSnapshot(
+      `"value must be a valid ipv4 or ipv6 address"`
+    );
+    expect(() =>
+      ip().validate('ZZZZ:0000:AB00:1234:0000:2552:7777:1313')
+    ).toThrowErrorMatchingInlineSnapshot(`"value must be a valid ipv4 or ipv6 address"`);
+    expect(() => ip().validate('blah 1234')).toThrowErrorMatchingInlineSnapshot(
+      `"value must be a valid ipv4 or ipv6 address"`
+    );
+  });
+});
+
+test('returns error when not string', () => {
+  expect(() => ip().validate(123)).toThrowErrorMatchingInlineSnapshot(
+    `"expected value of type [string] but got [number]"`
+  );
+
+  expect(() => ip().validate([1, 2, 3])).toThrowErrorMatchingInlineSnapshot(
+    `"expected value of type [string] but got [Array]"`
+  );
+
+  expect(() => ip().validate(/abc/)).toThrowErrorMatchingInlineSnapshot(
+    `"expected value of type [string] but got [RegExp]"`
+  );
+});

packages/kbn-config-schema/src/types/ip_type.ts

@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import typeDetect from 'type-detect';
+import { internals } from '../internals';
+import { Type, TypeOptions } from './type';
+
+export type IpVersion = 'ipv4' | 'ipv6';
+export type IpOptions = TypeOptions<string> & {
+  /**
+   * IP versions to accept, defaults to ['ipv4', 'ipv6'].
+   */
+  versions: IpVersion[];
+};
+
+export class IpType extends Type<string> {
+  constructor(options: IpOptions = { versions: ['ipv4', 'ipv6'] }) {
+    const schema = internals.string().ip({ version: options.versions, cidr: 'forbidden' });
+    super(schema, options);
+  }
+
+  protected handleError(type: string, { value, version }: Record<string, any>) {
+    switch (type) {
+      case 'string.base':
+        return `expected value of type [string] but got [${typeDetect(value)}]`;
+      case 'string.ipVersion':
+        return `value must be a valid ${version.join(' or ')} address`;
+    }
+  }
+}

src/core/server/elasticsearch/client/cluster_client.test.ts

@@ -96,7 +96,7 @@ describe('ClusterClient', () => {
       expect(scopedClusterClient.asCurrentUser).toBe(scopedClient.child.mock.results[0].value);
     });
 
-    it('returns a distinct  scoped cluster client on each call', () => {
+    it('returns a distinct scoped cluster client on each call', () => {
       const clusterClient = new ClusterClient(createConfig(), logger, getAuthHeaders);
       const request = httpServerMock.createKibanaRequest();
 
@@ -127,7 +127,7 @@ describe('ClusterClient', () => {
 
       expect(scopedClient.child).toHaveBeenCalledTimes(1);
       expect(scopedClient.child).toHaveBeenCalledWith({
-        headers: { foo: 'bar' },
+        headers: { foo: 'bar', 'x-opaque-id': expect.any(String) },
       });
     });
 
@@ -147,7 +147,7 @@ describe('ClusterClient', () => {
 
       expect(scopedClient.child).toHaveBeenCalledTimes(1);
       expect(scopedClient.child).toHaveBeenCalledWith({
-        headers: { authorization: 'auth' },
+        headers: { authorization: 'auth', 'x-opaque-id': expect.any(String) },
       });
     });
 
@@ -171,7 +171,7 @@ describe('ClusterClient', () => {
 
       expect(scopedClient.child).toHaveBeenCalledTimes(1);
       expect(scopedClient.child).toHaveBeenCalledWith({
-        headers: { authorization: 'auth' },
+        headers: { authorization: 'auth', 'x-opaque-id': expect.any(String) },
       });
     });
 
@@ -195,6 +195,26 @@ describe('ClusterClient', () => {
         headers: {
           foo: 'bar',
           hello: 'dolly',
+          'x-opaque-id': expect.any(String),
+        },
+      });
+    });
+
+    it('adds the x-opaque-id header based on the request id', () => {
+      const config = createConfig();
+      getAuthHeaders.mockReturnValue({});
+
+      const clusterClient = new ClusterClient(config, logger, getAuthHeaders);
+      const request = httpServerMock.createKibanaRequest({
+        kibanaRequestState: { requestId: 'my-fake-id' },
+      });
+
+      clusterClient.asScoped(request);
+
+      expect(scopedClient.child).toHaveBeenCalledTimes(1);
+      expect(scopedClient.child).toHaveBeenCalledWith({
+        headers: {
+          'x-opaque-id': 'my-fake-id',
         },
       });
     });
@@ -221,6 +241,7 @@ describe('ClusterClient', () => {
         headers: {
           foo: 'auth',
           hello: 'dolly',
+          'x-opaque-id': expect.any(String),
         },
       });
     });
@@ -247,6 +268,31 @@ describe('ClusterClient', () => {
         headers: {
           foo: 'request',
           hello: 'dolly',
+          'x-opaque-id': expect.any(String),
+        },
+      });
+    });
+
+    it('respect the precedence of x-opaque-id header over config headers', () => {
+      const config = createConfig({
+        customHeaders: {
+          'x-opaque-id': 'from config',
+        },
+      });
+      getAuthHeaders.mockReturnValue({});
+
+      const clusterClient = new ClusterClient(config, logger, getAuthHeaders);
+      const request = httpServerMock.createKibanaRequest({
+        headers: { foo: 'request' },
+        kibanaRequestState: { requestId: 'from request' },
+      });
+
+      clusterClient.asScoped(request);
+
+      expect(scopedClient.child).toHaveBeenCalledTimes(1);
+      expect(scopedClient.child).toHaveBeenCalledWith({
+        headers: {
+          'x-opaque-id': 'from request',
         },
       });
     });

src/core/server/elasticsearch/client/cluster_client.ts

@@ -19,7 +19,7 @@
 
 import { Client } from '@elastic/elasticsearch';
 import { Logger } from '../../logging';
-import { GetAuthHeaders, isRealRequest, Headers } from '../../http';
+import { GetAuthHeaders, Headers, isKibanaRequest, isRealRequest } from '../../http';
 import { ensureRawRequest, filterHeaders } from '../../http/router';
 import { ScopeableRequest } from '../types';
 import { ElasticsearchClient } from './types';
@@ -95,12 +95,14 @@ export class ClusterClient implements ICustomClusterClient {
   private getScopedHeaders(request: ScopeableRequest): Headers {
     let scopedHeaders: Headers;
     if (isRealRequest(request)) {
-      const authHeaders = this.getAuthHeaders(request);
       const requestHeaders = ensureRawRequest(request).headers;
-      scopedHeaders = filterHeaders(
-        { ...requestHeaders, ...authHeaders },
-        this.config.requestHeadersWhitelist
-      );
+      const requestIdHeaders = isKibanaRequest(request) ? { 'x-opaque-id': request.id } : {};
+      const authHeaders = this.getAuthHeaders(request);
+
+      scopedHeaders = filterHeaders({ ...requestHeaders, ...requestIdHeaders, ...authHeaders }, [
+        'x-opaque-id',
+        ...this.config.requestHeadersWhitelist,
+      ]);
     } else {
       scopedHeaders = filterHeaders(request?.headers ?? {}, this.config.requestHeadersWhitelist);
     }

src/core/server/elasticsearch/legacy/cluster_client.test.ts

@@ -349,6 +349,20 @@ describe('#asScoped', () => {
     );
   });
 
+  test('passes x-opaque-id header with request id', () => {
+    clusterClient.asScoped(
+      httpServerMock.createKibanaRequest({ kibanaRequestState: { requestId: 'alpha' } })
+    );
+
+    expect(MockScopedClusterClient).toHaveBeenCalledTimes(1);
+    expect(MockScopedClusterClient).toHaveBeenCalledWith(
+      expect.any(Function),
+      expect.any(Function),
+      { 'x-opaque-id': 'alpha' },
+      expect.any(Object)
+    );
+  });
+
   test('both scoped and internal API caller fail if cluster client is closed', async () => {
     clusterClient.asScoped(
       httpServerMock.createRawRequest({ headers: { zero: '0', one: '1', two: '2', three: '3' } })
@@ -482,7 +496,7 @@ describe('#asScoped', () => {
       expect(MockScopedClusterClient).toHaveBeenCalledWith(
         expect.any(Function),
         expect.any(Function),
-        {},
+        expect.objectContaining({ 'x-opaque-id': expect.any(String) }),
         auditor
       );
     });

src/core/server/elasticsearch/legacy/cluster_client.ts

@@ -20,7 +20,7 @@ import { Client } from 'elasticsearch';
 import { get } from 'lodash';
 
 import { LegacyElasticsearchErrorHelpers } from './errors';
-import { GetAuthHeaders, isRealRequest, KibanaRequest } from '../../http';
+import { GetAuthHeaders, KibanaRequest, isKibanaRequest, isRealRequest } from '../../http';
 import { AuditorFactory } from '../../audit_trail';
 import { filterHeaders, ensureRawRequest } from '../../http/router';
 import { Logger } from '../../logging';
@@ -207,7 +207,10 @@ export class LegacyClusterClient implements ILegacyClusterClient {
     return new LegacyScopedClusterClient(
       this.callAsInternalUser,
       this.callAsCurrentUser,
-      filterHeaders(this.getHeaders(request), this.config.requestHeadersWhitelist),
+      filterHeaders(this.getHeaders(request), [
+        'x-opaque-id',
+        ...this.config.requestHeadersWhitelist,
+      ]),
       this.getScopedAuditor(request)
     );
   }
@@ -215,8 +218,7 @@ export class LegacyClusterClient implements ILegacyClusterClient {
   private getScopedAuditor(request?: ScopeableRequest) {
     // TODO: support alternative credential owners from outside of Request context in #39430
     if (request && isRealRequest(request)) {
-      const kibanaRequest =
-        request instanceof KibanaRequest ? request : KibanaRequest.from(request);
+      const kibanaRequest = isKibanaRequest(request) ? request : KibanaRequest.from(request);
       const auditorFactory = this.getAuditorFactory();
       return auditorFactory.asScoped(kibanaRequest);
     }
@@ -256,8 +258,9 @@ export class LegacyClusterClient implements ILegacyClusterClient {
       return request && request.headers ? request.headers : {};
     }
     const authHeaders = this.getAuthHeaders(request);
-    const headers = ensureRawRequest(request).headers;
+    const requestHeaders = ensureRawRequest(request).headers;
+    const requestIdHeaders = isKibanaRequest(request) ? { 'x-opaque-id': request.id } : {};
 
-    return { ...headers, ...authHeaders };
+    return { ...requestHeaders, ...requestIdHeaders, ...authHeaders };
   }
 }

src/core/server/http/__snapshots__/http_config.test.ts.snap

@@ -39,6 +39,10 @@ Object {
   },
   "name": "kibana-hostname",
   "port": 5601,
+  "requestId": Object {
+    "allowFromAnyIp": false,
+    "ipAllowlist": Array [],
+  },
   "rewriteBasePath": false,
   "socketTimeout": 120000,
   "ssl": Object {

src/core/server/http/cookie_session_storage.test.ts

@@ -63,6 +63,10 @@ configService.atPath.mockReturnValue(
       whitelist: [],
     },
     customResponseHeaders: {},
+    requestId: {
+      allowFromAnyIp: true,
+      ipAllowlist: [],
+    },
   } as any)
 );
 

src/core/server/http/http_config.test.ts

@@ -54,6 +54,63 @@ test('throws if invalid hostname', () => {
   expect(() => httpSchema.validate(obj)).toThrowErrorMatchingSnapshot();
 });
 
+describe('requestId', () => {
+  test('accepts valid ip addresses', () => {
+    const {
+      requestId: { ipAllowlist },
+    } = config.schema.validate({
+      requestId: {
+        allowFromAnyIp: false,
+        ipAllowlist: ['0.0.0.0', '123.123.123.123', '1200:0000:AB00:1234:0000:2552:7777:1313'],
+      },
+    });
+    expect(ipAllowlist).toMatchInlineSnapshot(`
+      Array [
+        "0.0.0.0",
+        "123.123.123.123",
+        "1200:0000:AB00:1234:0000:2552:7777:1313",
+      ]
+    `);
+  });
+
+  test('rejects invalid ip addresses', () => {
+    expect(() => {
+      config.schema.validate({
+        requestId: {
+          allowFromAnyIp: false,
+          ipAllowlist: ['1200:0000:AB00:1234:O000:2552:7777:1313', '[2001:db8:0:1]:80'],
+        },
+      });
+    }).toThrowErrorMatchingInlineSnapshot(
+      `"[requestId.ipAllowlist.0]: value must be a valid ipv4 or ipv6 address"`
+    );
+  });
+
+  test('rejects if allowFromAnyIp is `true` and `ipAllowlist` is non-empty', () => {
+    expect(() => {
+      config.schema.validate({
+        requestId: {
+          allowFromAnyIp: true,
+          ipAllowlist: ['0.0.0.0', '123.123.123.123', '1200:0000:AB00:1234:0000:2552:7777:1313'],
+        },
+      });
+    }).toThrowErrorMatchingInlineSnapshot(
+      `"[requestId]: allowFromAnyIp must be set to 'false' if any values are specified in ipAllowlist"`
+    );
+
+    expect(() => {
+      config.schema.validate({
+        requestId: {
+          allowFromAnyIp: true,
+          ipAllowlist: ['0.0.0.0', '123.123.123.123', '1200:0000:AB00:1234:0000:2552:7777:1313'],
+        },
+      });
+    }).toThrowErrorMatchingInlineSnapshot(
+      `"[requestId]: allowFromAnyIp must be set to 'false' if any values are specified in ipAllowlist"`
+    );
+  });
+});
+
 test('can specify max payload as string', () => {
   const obj = {
     maxPayload: '2mb',

src/core/server/http/http_config.ts

@@ -87,6 +87,19 @@ export const config = {
           { defaultValue: [] }
         ),
       }),
+      requestId: schema.object(
+        {
+          allowFromAnyIp: schema.boolean({ defaultValue: false }),
+          ipAllowlist: schema.arrayOf(schema.ip(), { defaultValue: [] }),
+        },
+        {
+          validate(value) {
+            if (value.allowFromAnyIp === true && value.ipAllowlist?.length > 0) {
+              return `allowFromAnyIp must be set to 'false' if any values are specified in ipAllowlist`;
+            }
+          },
+        }
+      ),
     },
     {
       validate: (rawConfig) => {
@@ -130,6 +143,7 @@ export class HttpConfig {
   public compression: { enabled: boolean; referrerWhitelist?: string[] };
   public csp: ICspConfig;
   public xsrf: { disableProtection: boolean; whitelist: string[] };
+  public requestId: { allowFromAnyIp: boolean; ipAllowlist: string[] };
 
   /**
    * @internal
@@ -158,6 +172,7 @@ export class HttpConfig {
     this.compression = rawHttpConfig.compression;
     this.csp = new CspConfig(rawCspConfig);
     this.xsrf = rawHttpConfig.xsrf;
+    this.requestId = rawHttpConfig.requestId;
   }
 }
 

src/core/server/http/http_server.mocks.ts

@@ -29,7 +29,8 @@ import {
   RouteMethod,
   KibanaResponseFactory,
   RouteValidationSpec,
-  KibanaRouteState,
+  KibanaRouteOptions,
+  KibanaRequestState,
 } from './router';
 import { OnPreResponseToolkit } from './lifecycle/on_pre_response';
 import { OnPostAuthToolkit } from './lifecycle/on_post_auth';
@@ -45,7 +46,8 @@ interface RequestFixtureOptions<P = any, Q = any, B = any> {
   method?: RouteMethod;
   socket?: Socket;
   routeTags?: string[];
-  kibanaRouteState?: KibanaRouteState;
+  kibanaRouteOptions?: KibanaRouteOptions;
+  kibanaRequestState?: KibanaRequestState;
   routeAuthRequired?: false;
   validation?: {
     params?: RouteValidationSpec<P>;
@@ -65,13 +67,15 @@ function createKibanaRequestMock<P = any, Q = any, B = any>({
   routeTags,
   routeAuthRequired,
   validation = {},
-  kibanaRouteState = { xsrfRequired: true },
+  kibanaRouteOptions = { xsrfRequired: true },
+  kibanaRequestState = { requestId: '123' },
   auth = { isAuthenticated: true },
 }: RequestFixtureOptions<P, Q, B> = {}) {
   const queryString = stringify(query, { sort: false });
 
   return KibanaRequest.from<P, Q, B>(
     createRawRequestMock({
+      app: kibanaRequestState,
       auth,
       headers,
       params,
@@ -86,7 +90,7 @@ function createKibanaRequestMock<P = any, Q = any, B = any>({
         search: queryString ? `?${queryString}` : queryString,
       },
       route: {
-        settings: { tags: routeTags, auth: routeAuthRequired, app: kibanaRouteState },
+        settings: { tags: routeTags, auth: routeAuthRequired, app: kibanaRouteOptions },
       },
       raw: {
         req: {
@@ -133,6 +137,7 @@ function createRawRequestMock(customization: DeepPartial<Request> = {}) {
       raw: {
         req: {
           url: '/',
+          socket: {},
         },
       },
     },

src/core/server/http/http_server.test.ts

@@ -68,7 +68,11 @@ beforeEach(() => {
     port: 10002,
     ssl: { enabled: false },
     compression: { enabled: true },
-  } as HttpConfig;
+    requestId: {
+      allowFromAnyIp: true,
+      ipAllowlist: [],
+    },
+  } as any;
 
   configWithSSL = {
     ...config,

src/core/server/http/http_server.ts

@@ -22,13 +22,19 @@ import url from 'url';
 
 import { Logger, LoggerFactory } from '../logging';
 import { HttpConfig } from './http_config';
-import { createServer, getListenerOptions, getServerOptions } from './http_tools';
+import { createServer, getListenerOptions, getServerOptions, getRequestId } from './http_tools';
 import { adoptToHapiAuthFormat, AuthenticationHandler } from './lifecycle/auth';
 import { adoptToHapiOnPreAuth, OnPreAuthHandler } from './lifecycle/on_pre_auth';
 import { adoptToHapiOnPostAuthFormat, OnPostAuthHandler } from './lifecycle/on_post_auth';
 import { adoptToHapiOnRequest, OnPreRoutingHandler } from './lifecycle/on_pre_routing';
 import { adoptToHapiOnPreResponseFormat, OnPreResponseHandler } from './lifecycle/on_pre_response';
-import { IRouter, RouteConfigOptions, KibanaRouteState, isSafeMethod } from './router';
+import {
+  IRouter,
+  RouteConfigOptions,
+  KibanaRouteOptions,
+  KibanaRequestState,
+  isSafeMethod,
+} from './router';
 import {
   SessionStorageCookieOptions,
   createCookieSessionStorageFactory,
@@ -115,6 +121,7 @@ export class HttpServer {
     const basePathService = new BasePath(config.basePath);
     this.setupBasePathRewrite(config, basePathService);
     this.setupConditionalCompression(config);
+    this.setupRequestStateAssignment(config);
 
     return {
       registerRouter: this.registerRouter.bind(this),
@@ -164,7 +171,7 @@ export class HttpServer {
         const { authRequired, tags, body = {}, timeout } = route.options;
         const { accepts: allow, maxBytes, output, parse } = body;
 
-        const kibanaRouteState: KibanaRouteState = {
+        const kibanaRouteOptions: KibanaRouteOptions = {
           xsrfRequired: route.options.xsrfRequired ?? !isSafeMethod(route.method),
         };
 
@@ -180,7 +187,7 @@ export class HttpServer {
           path: route.path,
           options: {
             auth: this.getAuthOption(authRequired),
-            app: kibanaRouteState,
+            app: kibanaRouteOptions,
             ext: {
               onPreAuth: {
                 method: (request, h) => {
@@ -303,6 +310,16 @@ export class HttpServer {
     }
   }
 
+  private setupRequestStateAssignment(config: HttpConfig) {
+    this.server!.ext('onRequest', (request, responseToolkit) => {
+      request.app = {
+        ...(request.app ?? {}),
+        requestId: getRequestId(request, config.requestId),
+      } as KibanaRequestState;
+      return responseToolkit.continue;
+    });
+  }
+
   private registerOnPreAuth(fn: OnPreAuthHandler) {
     if (this.server === undefined) {
       throw new Error('Server is not created yet');

src/core/server/http/http_tools.test.ts

@@ -26,11 +26,20 @@ jest.mock('fs', () => {
   };
 });
 
+jest.mock('uuid', () => ({
+  v4: jest.fn().mockReturnValue('xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'),
+}));
+
 import supertest from 'supertest';
 import { Request, ResponseToolkit } from 'hapi';
 import Joi from 'joi';
 
-import { defaultValidationErrorHandler, HapiValidationError, getServerOptions } from './http_tools';
+import {
+  defaultValidationErrorHandler,
+  HapiValidationError,
+  getServerOptions,
+  getRequestId,
+} from './http_tools';
 import { HttpServer } from './http_server';
 import { HttpConfig, config } from './http_config';
 import { Router } from './router';
@@ -94,7 +103,11 @@ describe('timeouts', () => {
       maxPayload: new ByteSizeValue(1024),
       ssl: {},
       compression: { enabled: true },
-    } as HttpConfig);
+      requestId: {
+        allowFromAnyIp: true,
+        ipAllowlist: [],
+      },
+    } as any);
     registerRouter(router);
 
     await server.start();
@@ -173,3 +186,75 @@ describe('getServerOptions', () => {
     `);
   });
 });
+
+describe('getRequestId', () => {
+  describe('when allowFromAnyIp is true', () => {
+    it('generates a UUID if no x-opaque-id header is present', () => {
+      const request = {
+        headers: {},
+        raw: { req: { socket: { remoteAddress: '1.1.1.1' } } },
+      } as any;
+      expect(getRequestId(request, { allowFromAnyIp: true, ipAllowlist: [] })).toEqual(
+        'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
+      );
+    });
+
+    it('uses x-opaque-id header value if present', () => {
+      const request = {
+        headers: {
+          'x-opaque-id': 'id from header',
+          raw: { req: { socket: { remoteAddress: '1.1.1.1' } } },
+        },
+      } as any;
+      expect(getRequestId(request, { allowFromAnyIp: true, ipAllowlist: [] })).toEqual(
+        'id from header'
+      );
+    });
+  });
+
+  describe('when allowFromAnyIp is false', () => {
+    describe('and ipAllowlist is empty', () => {
+      it('generates a UUID even if x-opaque-id header is present', () => {
+        const request = {
+          headers: { 'x-opaque-id': 'id from header' },
+          raw: { req: { socket: { remoteAddress: '1.1.1.1' } } },
+        } as any;
+        expect(getRequestId(request, { allowFromAnyIp: false, ipAllowlist: [] })).toEqual(
+          'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
+        );
+      });
+    });
+
+    describe('and ipAllowlist is not empty', () => {
+      it('uses x-opaque-id header if request comes from trusted IP address', () => {
+        const request = {
+          headers: { 'x-opaque-id': 'id from header' },
+          raw: { req: { socket: { remoteAddress: '1.1.1.1' } } },
+        } as any;
+        expect(getRequestId(request, { allowFromAnyIp: false, ipAllowlist: ['1.1.1.1'] })).toEqual(
+          'id from header'
+        );
+      });
+
+      it('generates a UUID if request comes from untrusted IP address', () => {
+        const request = {
+          headers: { 'x-opaque-id': 'id from header' },
+          raw: { req: { socket: { remoteAddress: '5.5.5.5' } } },
+        } as any;
+        expect(getRequestId(request, { allowFromAnyIp: false, ipAllowlist: ['1.1.1.1'] })).toEqual(
+          'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
+        );
+      });
+
+      it('generates UUID if request comes from trusted IP address but no x-opaque-id header is present', () => {
+        const request = {
+          headers: {},
+          raw: { req: { socket: { remoteAddress: '1.1.1.1' } } },
+        } as any;
+        expect(getRequestId(request, { allowFromAnyIp: false, ipAllowlist: ['1.1.1.1'] })).toEqual(
+          'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
+        );
+      });
+    });
+  });
+});

src/core/server/http/http_tools.ts

@@ -21,6 +21,7 @@ import { Lifecycle, Request, ResponseToolkit, Server, ServerOptions, Util } from
 import Hoek from 'hoek';
 import { ServerOptions as TLSOptions } from 'https';
 import { ValidationError } from 'joi';
+import uuid from 'uuid';
 import { HttpConfig } from './http_config';
 import { validateObject } from './prototype_pollution';
 
@@ -169,3 +170,12 @@ export function defaultValidationErrorHandler(
 
   throw err;
 }
+
+export function getRequestId(request: Request, options: HttpConfig['requestId']): string {
+  return options.allowFromAnyIp ||
+    // socket may be undefined in integration tests that connect via the http listener directly
+    (request.raw.req.socket?.remoteAddress &&
+      options.ipAllowlist.includes(request.raw.req.socket.remoteAddress))
+    ? request.headers['x-opaque-id'] ?? uuid.v4()
+    : uuid.v4();
+}

src/core/server/http/index.ts

@@ -24,6 +24,7 @@ export { AuthStatus, GetAuthState, IsAuthenticated } from './auth_state_storage'
 export {
   CustomHttpResponseOptions,
   IKibanaSocket,
+  isKibanaRequest,
   isRealRequest,
   Headers,
   HttpResponseOptions,

src/core/server/http/integration_tests/core_services.test.ts

@@ -406,7 +406,10 @@ describe('http service', () => {
       // client contains authHeaders for BWC with legacy platform.
       const [client] = MockLegacyScopedClusterClient.mock.calls;
       const [, , clientHeaders] = client;
-      expect(clientHeaders).toEqual(authHeaders);
+      expect(clientHeaders).toEqual({
+        ...authHeaders,
+        'x-opaque-id': expect.any(String),
+      });
     });
 
     it('passes request authorization header to Elasticsearch if registerAuth was not set', async () => {
@@ -430,7 +433,10 @@ describe('http service', () => {
 
       const [client] = MockLegacyScopedClusterClient.mock.calls;
       const [, , clientHeaders] = client;
-      expect(clientHeaders).toEqual({ authorization: authorizationHeader });
+      expect(clientHeaders).toEqual({
+        authorization: authorizationHeader,
+        'x-opaque-id': expect.any(String),
+      });
     });
 
     it('forwards 401 errors returned from elasticsearch', async () => {

src/core/server/http/integration_tests/lifecycle_handlers.test.ts

@@ -62,6 +62,10 @@ describe('core lifecycle handlers', () => {
           'some-header': 'some-value',
         },
         xsrf: { disableProtection: false, whitelist: [whitelistedTestPath] },
+        requestId: {
+          allowFromAnyIp: true,
+          ipAllowlist: [],
+        },
       } as any)
     );
     server = createHttpServer({ configService });

src/core/server/http/integration_tests/request.test.ts

@@ -288,4 +288,24 @@ describe('KibanaRequest', () => {
       });
     });
   });
+
+  describe('request id', () => {
+    it('accepts x-opaque-id header case-insensitively', async () => {
+      const { server: innerServer, createRouter } = await server.setup(setupDeps);
+      const router = createRouter('/');
+      router.get({ path: '/', validate: false }, async (context, req, res) => {
+        return res.ok({ body: { requestId: req.id } });
+      });
+      await server.start();
+
+      const st = supertest(innerServer.listener);
+
+      const resp1 = await st.get('/').set({ 'x-opaque-id': 'alpha' }).expect(200);
+      expect(resp1.body).toEqual({ requestId: 'alpha' });
+      const resp2 = await st.get('/').set({ 'X-Opaque-Id': 'beta' }).expect(200);
+      expect(resp2.body).toEqual({ requestId: 'beta' });
+      const resp3 = await st.get('/').set({ 'X-OPAQUE-ID': 'gamma' }).expect(200);
+      expect(resp3.body).toEqual({ requestId: 'gamma' });
+    });
+  });
 });

src/core/server/http/lifecycle_handlers.test.ts

@@ -24,7 +24,7 @@ import {
 } from './lifecycle_handlers';
 import { httpServerMock } from './http_server.mocks';
 import { HttpConfig } from './http_config';
-import { KibanaRequest, RouteMethod, KibanaRouteState } from './router';
+import { KibanaRequest, RouteMethod, KibanaRouteOptions } from './router';
 
 const createConfig = (partial: Partial<HttpConfig>): HttpConfig => partial as HttpConfig;
 
@@ -32,14 +32,19 @@ const forgeRequest = ({
   headers = {},
   path = '/',
   method = 'get',
-  kibanaRouteState,
+  kibanaRouteOptions,
 }: Partial<{
   headers: Record<string, string>;
   path: string;
   method: RouteMethod;
-  kibanaRouteState: KibanaRouteState;
+  kibanaRouteOptions: KibanaRouteOptions;
 }>): KibanaRequest => {
-  return httpServerMock.createKibanaRequest({ headers, path, method, kibanaRouteState });
+  return httpServerMock.createKibanaRequest({
+    headers,
+    path,
+    method,
+    kibanaRouteOptions,
+  });
 };
 
 describe('xsrf post-auth handler', () => {
@@ -154,7 +159,7 @@ describe('xsrf post-auth handler', () => {
         method: 'post',
         headers: {},
         path: '/some-path',
-        kibanaRouteState: {
+        kibanaRouteOptions: {
           xsrfRequired: false,
         },
       });

src/core/server/http/router/index.ts

@@ -24,7 +24,9 @@ export {
   KibanaRequestEvents,
   KibanaRequestRoute,
   KibanaRequestRouteOptions,
-  KibanaRouteState,
+  KibanaRouteOptions,
+  KibanaRequestState,
+  isKibanaRequest,
   isRealRequest,
   LegacyRequest,
   ensureRawRequest,

src/core/server/http/router/request.test.ts

@@ -16,12 +16,45 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
+jest.mock('uuid', () => ({
+  v4: jest.fn().mockReturnValue('xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'),
+}));
+
 import { RouteOptions } from 'hapi';
 import { KibanaRequest } from './request';
 import { httpServerMock } from '../http_server.mocks';
 import { schema } from '@kbn/config-schema';
 
 describe('KibanaRequest', () => {
+  describe('id property', () => {
+    it('uses the request.app.requestId property if present', () => {
+      const request = httpServerMock.createRawRequest({
+        app: { requestId: 'fakeId' },
+      });
+      const kibanaRequest = KibanaRequest.from(request);
+      expect(kibanaRequest.id).toEqual('fakeId');
+    });
+
+    it('generates a new UUID if request.app property is not present', () => {
+      // Undefined app property
+      const request = httpServerMock.createRawRequest({
+        app: undefined,
+      });
+      const kibanaRequest = KibanaRequest.from(request);
+      expect(kibanaRequest.id).toEqual('xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx');
+    });
+
+    it('generates a new UUID if request.app.requestId property is not present', () => {
+      // Undefined app.requestId property
+      const request = httpServerMock.createRawRequest({
+        app: {},
+      });
+      const kibanaRequest = KibanaRequest.from(request);
+      expect(kibanaRequest.id).toEqual('xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx');
+    });
+  });
+
   describe('get all headers', () => {
     it('returns all headers', () => {
       const request = httpServerMock.createRawRequest({

src/core/server/http/router/request.ts

@@ -18,7 +18,8 @@
  */
 
 import { Url } from 'url';
-import { Request, ApplicationState } from 'hapi';
+import uuid from 'uuid';
+import { Request, RouteOptionsApp, ApplicationState } from 'hapi';
 import { Observable, fromEvent, merge } from 'rxjs';
 import { shareReplay, first, takeUntil } from 'rxjs/operators';
 import { RecursiveReadonly } from '@kbn/utility-types';
@@ -34,9 +35,17 @@ const requestSymbol = Symbol('request');
 /**
  * @internal
  */
-export interface KibanaRouteState extends ApplicationState {
+export interface KibanaRouteOptions extends RouteOptionsApp {
   xsrfRequired: boolean;
 }
+
+/**
+ * @internal
+ */
+export interface KibanaRequestState extends ApplicationState {
+  requestId: string;
+}
+
 /**
  * Route options: If 'GET' or 'OPTIONS' method, body options won't be returned.
  * @public
@@ -134,6 +143,15 @@ export class KibanaRequest<
 
     return { query, params, body };
   }
+  /**
+   * A identifier to identify this request.
+   *
+   * @remarks
+   * Depending on the user's configuration, this value may be sourced from the
+   * incoming request's `X-Opaque-Id` header which is not guaranteed to be unique
+   * per request.
+   */
+  public readonly id: string;
   /** a WHATWG URL standard object. */
   public readonly url: Url;
   /** matched route details */
@@ -171,6 +189,11 @@ export class KibanaRequest<
     // until that time we have to expose all the headers
     private readonly withoutSecretHeaders: boolean
   ) {
+    // The `requestId` property will not be populated for requests that are 'faked' by internal systems that leverage
+    // KibanaRequest in conjunction with scoped Elaticcsearch and SavedObjectsClient in order to pass credentials.
+    // In these cases, the id defaults to a newly generated UUID.
+    this.id = (request.app as KibanaRequestState | undefined)?.requestId ?? uuid.v4();
+
     this.url = request.url;
     this.headers = deepFreeze({ ...request.headers });
     this.isSystemRequest =
@@ -220,7 +243,7 @@ export class KibanaRequest<
     const options = ({
       authRequired: this.getAuthRequired(request),
       // some places in LP call KibanaRequest.from(request) manually. remove fallback to true before v8
-      xsrfRequired: (request.route.settings.app as KibanaRouteState)?.xsrfRequired ?? true,
+      xsrfRequired: (request.route.settings.app as KibanaRouteOptions)?.xsrfRequired ?? true,
       tags: request.route.settings.tags || [],
       timeout: {
         payload: payloadTimeout,
@@ -276,7 +299,11 @@ export class KibanaRequest<
 export const ensureRawRequest = (request: KibanaRequest | LegacyRequest) =>
   isKibanaRequest(request) ? request[requestSymbol] : request;
 
-function isKibanaRequest(request: unknown): request is KibanaRequest {
+/**
+ * Checks if an incoming request is a {@link KibanaRequest}
+ * @internal
+ */
+export function isKibanaRequest(request: unknown): request is KibanaRequest {
   return request instanceof KibanaRequest;
 }
 

src/core/server/http/test_utils.ts

@@ -46,6 +46,10 @@ configService.atPath.mockReturnValue(
       whitelist: [],
     },
     customResponseHeaders: {},
+    requestId: {
+      allowFromAnyIp: true,
+      ipAllowlist: [],
+    },
   } as any)
 );
 

src/core/server/server.api.md

@@ -1059,6 +1059,7 @@ export class KibanaRequest<Params = unknown, Query = unknown, Body = unknown, Me
     // @internal
     static from<P, Q, B>(req: Request, routeSchemas?: RouteValidator<P, Q, B> | RouteValidatorFullConfig<P, Q, B>, withoutSecretHeaders?: boolean): KibanaRequest<P, Q, B, any>;
     readonly headers: Headers;
+    readonly id: string;
     readonly isSystemRequest: boolean;
     // (undocumented)
     readonly params: Params;

x-pack/legacy/plugins/xpack_main/server/lib/__tests__/replace_injected_vars.js

@@ -14,6 +14,7 @@ const buildRequest = (path = '/app/kibana') => {
   const get = sinon.stub();
 
   return {
+    app: {},
     path,
     route: { settings: {} },
     headers: {},

x-pack/plugins/alerts/server/alerts_client_factory.test.ts

@@ -46,6 +46,7 @@ const alertsClientFactoryParams: jest.Mocked<AlertsClientFactoryOpts> = {
   eventLog: eventLogMock.createStart(),
 };
 const fakeRequest = ({
+  app: {},
   headers: {},
   getBasePath: () => '',
   path: '/',

x-pack/plugins/audit_trail/server/client/audit_trail_client.test.ts

@@ -23,7 +23,11 @@ describe('AuditTrailClient', () => {
 
   beforeEach(() => {
     event$ = new Subject();
-    client = new AuditTrailClient(httpServerMock.createKibanaRequest(), event$, deps);
+    client = new AuditTrailClient(
+      httpServerMock.createKibanaRequest({ kibanaRequestState: { requestId: 'request id alpha' } }),
+      event$,
+      deps
+    );
   });
 
   afterEach(() => {
@@ -40,6 +44,15 @@ describe('AuditTrailClient', () => {
       client.add({ message: 'message', type: 'type' });
     });
 
+    it('populates requestId', (done) => {
+      client.withAuditScope('scope_name');
+      event$.subscribe((event) => {
+        expect(event.requestId).toBe('request id alpha');
+        done();
+      });
+      client.add({ message: 'message', type: 'type' });
+    });
+
     it('throws an exception if tries to re-write a scope', () => {
       client.withAuditScope('scope_name');
       expect(() => client.withAuditScope('another_scope_name')).toThrowErrorMatchingInlineSnapshot(

x-pack/plugins/audit_trail/server/client/audit_trail_client.ts

@@ -41,6 +41,7 @@ export class AuditTrailClient implements Auditor {
       user: user?.username,
       space: spaceId,
       scope: this.scope,
+      requestId: this.request.id,
     });
   }
 }

x-pack/plugins/audit_trail/server/types.ts

@@ -13,4 +13,5 @@ export interface AuditEvent {
   scope?: string;
   user?: string;
   space?: string;
+  requestId?: string;
 }

x-pack/plugins/reporting/server/export_types/csv/execute_job.ts

@@ -50,6 +50,7 @@ const getRequest = async (headers: string | undefined, crypto: Crypto, logger: L
     path: '/',
     route: { settings: {} },
     url: { href: '/' },
+    app: {},
     raw: { req: { url: '/' } },
   } as Hapi.Request);
 };