Add support for reading request ID from X-Opaque-Id header (elastic#7…

…1019)

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

docs/development/core/server/kibana-plugin-core-server.kibanarequest.id.md

@@ -0,0 +1,18 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [KibanaRequest](./kibana-plugin-core-server.kibanarequest.md) &gt; [id](./kibana-plugin-core-server.kibanarequest.id.md)
+
+## KibanaRequest.id property
+
+A identifier to identify this request.
+
+<b>Signature:</b>
+
+```typescript
+readonly id: string;
+```
+
+## Remarks
+
+Depending on the user's configuration, this value may be sourced from the incoming request's `X-Opaque-Id` header which is not guaranteed to be unique per request.
+

docs/development/core/server/kibana-plugin-core-server.kibanarequest.md

@@ -26,6 +26,7 @@ export declare class KibanaRequest<Params = unknown, Query = unknown, Body = unk
 |  [body](./kibana-plugin-core-server.kibanarequest.body.md) |  | <code>Body</code> |  |
 |  [events](./kibana-plugin-core-server.kibanarequest.events.md) |  | <code>KibanaRequestEvents</code> | Request events [KibanaRequestEvents](./kibana-plugin-core-server.kibanarequestevents.md) |
 |  [headers](./kibana-plugin-core-server.kibanarequest.headers.md) |  | <code>Headers</code> | Readonly copy of incoming request headers. |
+|  [id](./kibana-plugin-core-server.kibanarequest.id.md) |  | <code>string</code> | A identifier to identify this request. |
 |  [isSystemRequest](./kibana-plugin-core-server.kibanarequest.issystemrequest.md) |  | <code>boolean</code> | Whether or not the request is a "system request" rather than an application-level request. Can be set on the client using the <code>HttpFetchOptions#asSystemRequest</code> option. |
 |  [params](./kibana-plugin-core-server.kibanarequest.params.md) |  | <code>Params</code> |  |
 |  [query](./kibana-plugin-core-server.kibanarequest.query.md) |  | <code>Query</code> |  |

docs/setup/settings.asciidoc

@@ -476,6 +476,12 @@ identifies this {kib} instance. *Default: `"your-hostname"`*
  | {kib} is served by a back end server. This
 setting specifies the port to use. *Default: `5601`*
 
+| `server.requestId.allowFromAnyIp:`
+ | Sets whether or not the X-Opaque-Id header should be trusted from any IP address for identifying requests in logs and forwarded to Elasticsearch.
+
+| `server.requestId.ipAllowlist:`
+ | A list of IPv4 and IPv6 address which the `X-Opaque-Id` header should be trusted from. Normally this would be set to the IP addresses of the load balancers or reverse-proxy that end users use to access Kibana. If any are set, `server.requestId.allowFromAnyIp` must also be set to `false.`
+
 | `server.rewriteBasePath:`
  | Specifies whether {kib} should
 rewrite requests that are prefixed with `server.basePath` or require that they

packages/kbn-config-schema/src/index.ts

@@ -34,6 +34,8 @@ import {
   ConditionalTypeValue,
   DurationOptions,
   DurationType,
+  IpOptions,
+  IpType,
   LiteralType,
   MapOfOptions,
   MapOfType,
@@ -107,6 +109,10 @@ function never(): Type<never> {
   return new NeverType();
 }
 
+function ip(options?: IpOptions): Type<string> {
+  return new IpType(options);
+}
+
 /**
  * Create an optional type
  */
@@ -207,6 +213,7 @@ export const schema = {
   conditional,
   contextRef,
   duration,
+  ip,
   literal,
   mapOf,
   maybe,

packages/kbn-config-schema/src/types/index.ts

@@ -36,3 +36,4 @@ export { StringOptions, StringType } from './string_type';
 export { UnionType } from './union_type';
 export { URIOptions, URIType } from './uri_type';
 export { NeverType } from './never_type';
+export { IpType, IpOptions } from './ip_type';

packages/kbn-config-schema/src/types/ip_type.test.ts

@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { schema } from '..';
+
+const { ip } = schema;
+
+describe('ip validation', () => {
+  test('accepts ipv4', () => {
+    expect(ip().validate('1.1.1.1')).toEqual('1.1.1.1');
+  });
+  test('accepts ipv6', () => {
+    expect(ip().validate('1200:0000:AB00:1234:0000:2552:7777:1313')).toEqual(
+      '1200:0000:AB00:1234:0000:2552:7777:1313'
+    );
+  });
+  test('rejects ipv6 when not specified', () => {
+    expect(() =>
+      ip({ versions: ['ipv4'] }).validate('1200:0000:AB00:1234:0000:2552:7777:1313')
+    ).toThrowErrorMatchingInlineSnapshot(`"value must be a valid ipv4 address"`);
+  });
+  test('rejects ipv4 when not specified', () => {
+    expect(() => ip({ versions: ['ipv6'] }).validate('1.1.1.1')).toThrowErrorMatchingInlineSnapshot(
+      `"value must be a valid ipv6 address"`
+    );
+  });
+  test('rejects invalid ip addresses', () => {
+    expect(() => ip().validate('1.1.1.1/24')).toThrowErrorMatchingInlineSnapshot(
+      `"value must be a valid ipv4 or ipv6 address"`
+    );
+    expect(() => ip().validate('99999.1.1.1')).toThrowErrorMatchingInlineSnapshot(
+      `"value must be a valid ipv4 or ipv6 address"`
+    );
+    expect(() =>
+      ip().validate('ZZZZ:0000:AB00:1234:0000:2552:7777:1313')
+    ).toThrowErrorMatchingInlineSnapshot(`"value must be a valid ipv4 or ipv6 address"`);
+    expect(() => ip().validate('blah 1234')).toThrowErrorMatchingInlineSnapshot(
+      `"value must be a valid ipv4 or ipv6 address"`
+    );
+  });
+});
+
+test('returns error when not string', () => {
+  expect(() => ip().validate(123)).toThrowErrorMatchingInlineSnapshot(
+    `"expected value of type [string] but got [number]"`
+  );
+
+  expect(() => ip().validate([1, 2, 3])).toThrowErrorMatchingInlineSnapshot(
+    `"expected value of type [string] but got [Array]"`
+  );
+
+  expect(() => ip().validate(/abc/)).toThrowErrorMatchingInlineSnapshot(
+    `"expected value of type [string] but got [RegExp]"`
+  );
+});

packages/kbn-config-schema/src/types/ip_type.ts

@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import typeDetect from 'type-detect';
+import { internals } from '../internals';
+import { Type, TypeOptions } from './type';
+
+export type IpVersion = 'ipv4' | 'ipv6';
+export type IpOptions = TypeOptions<string> & {
+  /**
+   * IP versions to accept, defaults to ['ipv4', 'ipv6'].
+   */
+  versions: IpVersion[];
+};
+
+export class IpType extends Type<string> {
+  constructor(options: IpOptions = { versions: ['ipv4', 'ipv6'] }) {
+    const schema = internals.string().ip({ version: options.versions, cidr: 'forbidden' });
+    super(schema, options);
+  }
+
+  protected handleError(type: string, { value, version }: Record<string, any>) {
+    switch (type) {
+      case 'string.base':
+        return `expected value of type [string] but got [${typeDetect(value)}]`;
+      case 'string.ipVersion':
+        return `value must be a valid ${version.join(' or ')} address`;
+    }
+  }
+}

src/core/server/elasticsearch/client/cluster_client.test.ts

@@ -96,7 +96,7 @@ describe('ClusterClient', () => {
       expect(scopedClusterClient.asCurrentUser).toBe(scopedClient.child.mock.results[0].value);
     });
 
-    it('returns a distinct  scoped cluster client on each call', () => {
+    it('returns a distinct scoped cluster client on each call', () => {
       const clusterClient = new ClusterClient(createConfig(), logger, getAuthHeaders);
       const request = httpServerMock.createKibanaRequest();
 
@@ -127,7 +127,7 @@ describe('ClusterClient', () => {
 
       expect(scopedClient.child).toHaveBeenCalledTimes(1);
       expect(scopedClient.child).toHaveBeenCalledWith({
-        headers: { foo: 'bar' },
+        headers: { foo: 'bar', 'x-opaque-id': expect.any(String) },
       });
     });
 
@@ -147,7 +147,7 @@ describe('ClusterClient', () => {
 
       expect(scopedClient.child).toHaveBeenCalledTimes(1);
       expect(scopedClient.child).toHaveBeenCalledWith({
-        headers: { authorization: 'auth' },
+        headers: { authorization: 'auth', 'x-opaque-id': expect.any(String) },
       });
     });
 
@@ -171,7 +171,7 @@ describe('ClusterClient', () => {
 
       expect(scopedClient.child).toHaveBeenCalledTimes(1);
       expect(scopedClient.child).toHaveBeenCalledWith({
-        headers: { authorization: 'auth' },
+        headers: { authorization: 'auth', 'x-opaque-id': expect.any(String) },
       });
     });
 
@@ -195,6 +195,26 @@ describe('ClusterClient', () => {
         headers: {
           foo: 'bar',
           hello: 'dolly',
+          'x-opaque-id': expect.any(String),
+        },
+      });
+    });
+
+    it('adds the x-opaque-id header based on the request id', () => {
+      const config = createConfig();
+      getAuthHeaders.mockReturnValue({});
+
+      const clusterClient = new ClusterClient(config, logger, getAuthHeaders);
+      const request = httpServerMock.createKibanaRequest({
+        kibanaRequestState: { requestId: 'my-fake-id' },
+      });
+
+      clusterClient.asScoped(request);
+
+      expect(scopedClient.child).toHaveBeenCalledTimes(1);
+      expect(scopedClient.child).toHaveBeenCalledWith({
+        headers: {
+          'x-opaque-id': 'my-fake-id',
         },
       });
     });
@@ -221,6 +241,7 @@ describe('ClusterClient', () => {
         headers: {
           foo: 'auth',
           hello: 'dolly',
+          'x-opaque-id': expect.any(String),
         },
       });
     });
@@ -247,6 +268,31 @@ describe('ClusterClient', () => {
         headers: {
           foo: 'request',
           hello: 'dolly',
+          'x-opaque-id': expect.any(String),
+        },
+      });
+    });
+
+    it('respect the precedence of x-opaque-id header over config headers', () => {
+      const config = createConfig({
+        customHeaders: {
+          'x-opaque-id': 'from config',
+        },
+      });
+      getAuthHeaders.mockReturnValue({});
+
+      const clusterClient = new ClusterClient(config, logger, getAuthHeaders);
+      const request = httpServerMock.createKibanaRequest({
+        headers: { foo: 'request' },
+        kibanaRequestState: { requestId: 'from request' },
+      });
+
+      clusterClient.asScoped(request);
+
+      expect(scopedClient.child).toHaveBeenCalledTimes(1);
+      expect(scopedClient.child).toHaveBeenCalledWith({
+        headers: {
+          'x-opaque-id': 'from request',
         },
       });
     });

src/core/server/elasticsearch/client/cluster_client.ts

@@ -19,7 +19,7 @@
 
 import { Client } from '@elastic/elasticsearch';
 import { Logger } from '../../logging';
-import { GetAuthHeaders, isRealRequest, Headers } from '../../http';
+import { GetAuthHeaders, Headers, isKibanaRequest, isRealRequest } from '../../http';
 import { ensureRawRequest, filterHeaders } from '../../http/router';
 import { ScopeableRequest } from '../types';
 import { ElasticsearchClient } from './types';
@@ -95,12 +95,14 @@ export class ClusterClient implements ICustomClusterClient {
   private getScopedHeaders(request: ScopeableRequest): Headers {
     let scopedHeaders: Headers;
     if (isRealRequest(request)) {
-      const authHeaders = this.getAuthHeaders(request);
       const requestHeaders = ensureRawRequest(request).headers;
-      scopedHeaders = filterHeaders(
-        { ...requestHeaders, ...authHeaders },
-        this.config.requestHeadersWhitelist
-      );
+      const requestIdHeaders = isKibanaRequest(request) ? { 'x-opaque-id': request.id } : {};
+      const authHeaders = this.getAuthHeaders(request);
+
+      scopedHeaders = filterHeaders({ ...requestHeaders, ...requestIdHeaders, ...authHeaders }, [
+        'x-opaque-id',
+        ...this.config.requestHeadersWhitelist,
+      ]);
     } else {
       scopedHeaders = filterHeaders(request?.headers ?? {}, this.config.requestHeadersWhitelist);
     }

src/core/server/elasticsearch/legacy/cluster_client.test.ts

@@ -349,6 +349,20 @@ describe('#asScoped', () => {
     );
   });
 
+  test('passes x-opaque-id header with request id', () => {
+    clusterClient.asScoped(
+      httpServerMock.createKibanaRequest({ kibanaRequestState: { requestId: 'alpha' } })
+    );
+
+    expect(MockScopedClusterClient).toHaveBeenCalledTimes(1);
+    expect(MockScopedClusterClient).toHaveBeenCalledWith(
+      expect.any(Function),
+      expect.any(Function),
+      { 'x-opaque-id': 'alpha' },
+      expect.any(Object)
+    );
+  });
+
   test('both scoped and internal API caller fail if cluster client is closed', async () => {
     clusterClient.asScoped(
       httpServerMock.createRawRequest({ headers: { zero: '0', one: '1', two: '2', three: '3' } })
@@ -482,7 +496,7 @@ describe('#asScoped', () => {
       expect(MockScopedClusterClient).toHaveBeenCalledWith(
         expect.any(Function),
         expect.any(Function),
-        {},
+        expect.objectContaining({ 'x-opaque-id': expect.any(String) }),
         auditor
       );
     });

src/core/server/elasticsearch/legacy/cluster_client.ts

@@ -20,7 +20,7 @@ import { Client } from 'elasticsearch';
 import { get } from 'lodash';
 
 import { LegacyElasticsearchErrorHelpers } from './errors';
-import { GetAuthHeaders, isRealRequest, KibanaRequest } from '../../http';
+import { GetAuthHeaders, KibanaRequest, isKibanaRequest, isRealRequest } from '../../http';
 import { AuditorFactory } from '../../audit_trail';
 import { filterHeaders, ensureRawRequest } from '../../http/router';
 import { Logger } from '../../logging';
@@ -207,16 +207,18 @@ export class LegacyClusterClient implements ILegacyClusterClient {
     return new LegacyScopedClusterClient(
       this.callAsInternalUser,
       this.callAsCurrentUser,
-      filterHeaders(this.getHeaders(request), this.config.requestHeadersWhitelist),
+      filterHeaders(this.getHeaders(request), [
+        'x-opaque-id',
+        ...this.config.requestHeadersWhitelist,
+      ]),
       this.getScopedAuditor(request)
     );
   }
 
   private getScopedAuditor(request?: ScopeableRequest) {
     // TODO: support alternative credential owners from outside of Request context in #39430
     if (request && isRealRequest(request)) {
-      const kibanaRequest =
-        request instanceof KibanaRequest ? request : KibanaRequest.from(request);
+      const kibanaRequest = isKibanaRequest(request) ? request : KibanaRequest.from(request);
       const auditorFactory = this.getAuditorFactory();
       return auditorFactory.asScoped(kibanaRequest);
     }
@@ -256,8 +258,9 @@ export class LegacyClusterClient implements ILegacyClusterClient {
       return request && request.headers ? request.headers : {};
     }
     const authHeaders = this.getAuthHeaders(request);
-    const headers = ensureRawRequest(request).headers;
+    const requestHeaders = ensureRawRequest(request).headers;
+    const requestIdHeaders = isKibanaRequest(request) ? { 'x-opaque-id': request.id } : {};
 
-    return { ...headers, ...authHeaders };
+    return { ...requestHeaders, ...requestIdHeaders, ...authHeaders };
   }
 }