[Fleet] Add option to have Kafka dynamic topics in outputs (elastic#1…

…92720) ## Summary Add option to have Kafka dynamic topics in outputs settings. It adds selection with radio buttons under the `topic` section. ### UI changes ![Screenshot 2024-09-13 at 16 32 54](https://github.com/user-attachments/assets/9414f475-c115-4986-a204-eeb79471b7ea) ![Screenshot 2024-09-13 at 16 33 03](https://github.com/user-attachments/assets/ac36b1da-0e81-4f97-9632-ced8b5776af2) ![Screenshot 2024-09-13 at 16 33 12](https://github.com/user-attachments/assets/5ea3cf66-d929-45fb-aa3b-924734c34305) ![Screenshot 2024-09-13 at 16 33 40](https://github.com/user-attachments/assets/62e477c8-5719-41fe-94b1-4bdda0f00007) <details> <summary>Testing</summary> - Create a kafka output, fill the required fields, select "static" topic and fill it out with "default_topic". Payload will be: ![Screenshot 2024-09-13 at 16 36 50](https://github.com/user-attachments/assets/ea3761b8-39cb-40ce-bf1c-85e3af1f9d4b) - The output should be created correctly - Create a kafka output, fill the required fields, select "dynamic" topic and choose one from the dropdown. Payload will be: ![Screenshot 2024-09-13 at 16 36 23](https://github.com/user-attachments/assets/8d0d3ce4-5fb0-4c01-a939-b6a9d50da219) - Create a kafka output, fill the required fields, select "dynamic" topic and input one not present in the dropdown (I had "custom_topic". Payload will be like this: ![Screenshot 2024-09-13 at 16 35 52](https://github.com/user-attachments/assets/27347d01-3efc-4c23-85a8-f3287854de4f) </details> - Try updating an existing Kafka output from one type to another, it should work correctly ### Checklist - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
kibanamachine · Sep 17, 2024 · c0aaada · c0aaada
1 parent c7f3a40
commit c0aaada
Show file tree

Hide file tree

Showing 11 changed files with 461 additions and 79 deletions.
diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml
@@ -68,7 +68,7 @@ pageLoadAssetSize:
   files: 22673
   filesManagement: 18683
   fileUpload: 25664
-  fleet: 174609
+  fleet: 190461
   globalSearch: 29696
   globalSearchBar: 50403
   globalSearchProviders: 25554

diff --git a/x-pack/plugins/fleet/common/constants/output.ts b/x-pack/plugins/fleet/common/constants/output.ts
@@ -66,6 +66,11 @@ export const kafkaPartitionType = {
   Hash: 'hash',
 } as const;
 
+export const kafkaTopicsType = {
+  Static: 'static',
+  Dynamic: 'dynamic',
+} as const;
+
 export const kafkaTopicWhenType = {
   Equals: 'equals',
   Contains: 'contains',
@@ -133,6 +138,27 @@ export const RESERVED_CONFIG_YML_KEYS = [
   'worker',
 ];
 
+export const kafkaTopicsOptions = [
+  {
+    id: kafkaTopicsType.Static,
+    label: 'Static Topic',
+    'data-test-subj': 'kafkaTopicStaticRadioButton',
+  },
+  {
+    id: kafkaTopicsType.Dynamic,
+    label: 'Dynamic Topic',
+    'data-test-subj': 'kafkaTopicDynamicRadioButton',
+  },
+];
+
+export const KAFKA_DYNAMIC_FIELDS = [
+  'data_stream.type',
+  'data_stream.dataset',
+  'data_stream.namespace',
+  '@timestamp',
+  'event.dataset',
+];
+
 export const OUTPUT_TYPES_WITH_PRESET_SUPPORT: Array<ValueOf<OutputType>> = [
   outputType.Elasticsearch,
   outputType.RemoteElasticsearch,

diff --git a/x-pack/plugins/fleet/cypress/screens/fleet.ts b/x-pack/plugins/fleet/cypress/screens/fleet.ts
@@ -163,7 +163,8 @@ export const SETTINGS_OUTPUTS_KAFKA = {
   PARTITIONING_EVENTS_INPUT: 'settingsOutputsFlyout.kafkaPartitionTypeRandomInput',
   PARTITIONING_HASH_INPUT: 'settingsOutputsFlyout.kafkaPartitionTypeHashInput',
   TOPICS_PANEL: 'settingsOutputsFlyout.kafkaTopicsPanel',
-  TOPICS_DEFAULT_TOPIC_INPUT: 'settingsOutputsFlyout.kafkaDefaultTopicInput',
+  TOPICS_DEFAULT_TOPIC_INPUT: 'settingsOutputsFlyout.kafkaStaticTopicInput',
+  TOPICS_DYNAMIC_TOPIC_INPUT: 'settingsOutputsFlyout.kafkaDynamicTopicInput',
   HEADERS_PANEL: 'settingsOutputsFlyout.kafkaHeadersPanel',
   HEADERS_KEY_INPUT: 'settingsOutputsFlyout.kafkaHeadersKeyInput0',
   HEADERS_VALUE_INPUT: 'settingsOutputsFlyout.kafkaHeadersValueInput0',

diff --git a/...ations/fleet/sections/settings/components/edit_output_flyout/output_form_kafka_topics.tsx b/...ations/fleet/sections/settings/components/edit_output_flyout/output_form_kafka_topics.tsx
@@ -5,15 +5,93 @@
  * 2.0.
  */
 
-import { EuiFieldText, EuiFormRow, EuiPanel, EuiSpacer, EuiTitle } from '@elastic/eui';
+import React, { useMemo } from 'react';
+import type { EuiComboBoxOptionOption } from '@elastic/eui';
+import {
+  EuiFieldText,
+  EuiFormRow,
+  EuiPanel,
+  EuiSpacer,
+  EuiTitle,
+  EuiRadioGroup,
+  EuiComboBox,
+} from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n-react';
-import React from 'react';
+import { i18n } from '@kbn/i18n';
+
+import {
+  kafkaTopicsType,
+  KAFKA_DYNAMIC_FIELDS,
+  kafkaTopicsOptions,
+} from '../../../../../../../common/constants';
 
 import type { OutputFormInputsType } from './use_output_form';
 
 export const OutputFormKafkaTopics: React.FunctionComponent<{ inputs: OutputFormInputsType }> = ({
   inputs,
 }) => {
+  const dynamicOptions: Array<EuiComboBoxOptionOption<string>> = useMemo(() => {
+    const options = KAFKA_DYNAMIC_FIELDS.map((option) => ({
+      label: option,
+      value: option,
+    }));
+    return options;
+  }, []);
+
+  const renderTopics = () => {
+    switch (inputs.kafkaTopicsInput.value) {
+      case kafkaTopicsType.Static:
+        return (
+          <EuiFormRow
+            fullWidth
+            label={
+              <FormattedMessage
+                id="xpack.fleet.settings.editOutputFlyout.kafkaTopicsDefaultTopicLabel"
+                defaultMessage="Default topic"
+              />
+            }
+            {...inputs.kafkaStaticTopicInput.formRowProps}
+          >
+            <EuiFieldText
+              data-test-subj="settingsOutputsFlyout.kafkaStaticTopicInput"
+              fullWidth
+              {...inputs.kafkaStaticTopicInput.props}
+            />
+          </EuiFormRow>
+        );
+      case kafkaTopicsType.Dynamic:
+        return (
+          <EuiFormRow
+            fullWidth
+            helpText={i18n.translate(
+              'xpack.fleet.settings.editOutputFlyout.kafkaDynamicTopicHelptext',
+              {
+                defaultMessage:
+                  'Select a topic from the list. If a topic is not available, create a custom one.',
+              }
+            )}
+            label={
+              <FormattedMessage
+                id="xpack.fleet.settings.editOutputFlyout.kafkaDynamicTopicLabel"
+                defaultMessage="Topic from field"
+              />
+            }
+            {...inputs.kafkaDynamicTopicInput.formRowProps}
+          >
+            <EuiComboBox
+              data-test-subj="settingsOutputsFlyout.kafkaDynamicTopicInput"
+              fullWidth
+              isClearable={true}
+              options={dynamicOptions}
+              customOptionText="Use custom field (not recommended)"
+              singleSelection={{ asPlainText: true }}
+              {...inputs.kafkaDynamicTopicInput.props}
+            />
+          </EuiFormRow>
+        );
+    }
+  };
+
   return (
     <EuiPanel
       borderRadius="m"
@@ -39,14 +117,16 @@ export const OutputFormKafkaTopics: React.FunctionComponent<{ inputs: OutputForm
             defaultMessage="Default topic"
           />
         }
-        {...inputs.kafkaDefaultTopicInput.formRowProps}
       >
-        <EuiFieldText
-          data-test-subj="settingsOutputsFlyout.kafkaDefaultTopicInput"
-          fullWidth
-          {...inputs.kafkaDefaultTopicInput.props}
+        <EuiRadioGroup
+          style={{ flexDirection: 'row', flexWrap: 'wrap', columnGap: 30 }}
+          data-test-subj={'editOutputFlyout.kafkaTopicsRadioInput'}
+          options={kafkaTopicsOptions}
+          compressed
+          {...inputs.kafkaTopicsInput.props}
         />
       </EuiFormRow>
+      {renderTopics()}
 
       <EuiSpacer size="m" />
     </EuiPanel>

diff --git a/...ications/fleet/sections/settings/components/edit_output_flyout/output_form_validators.tsx b/...ications/fleet/sections/settings/components/edit_output_flyout/output_form_validators.tsx
@@ -7,6 +7,7 @@
 
 import { i18n } from '@kbn/i18n';
 import { safeLoad } from 'js-yaml';
+import type { EuiComboBoxOptionOption } from '@elastic/eui';
 
 const toSecretValidator =
   (validator: (value: string) => string[] | undefined) =>
@@ -307,7 +308,7 @@ export function validateSSLKey(value: string) {
 
 export const validateSSLKeySecret = toSecretValidator(validateSSLKey);
 
-export function validateKafkaDefaultTopic(value: string) {
+export function validateKafkaStaticTopic(value: string) {
   if (!value || value === '') {
     return [
       i18n.translate('xpack.fleet.settings.outputForm.kafkaDefaultTopicRequiredMessage', {
@@ -317,6 +318,30 @@ export function validateKafkaDefaultTopic(value: string) {
   }
 }
 
+export function validateDynamicKafkaTopics(value: Array<EuiComboBoxOptionOption<string>>) {
+  const res = [];
+  value.forEach((val, idx) => {
+    if (!val) {
+      res.push(
+        i18n.translate('xpack.fleet.settings.outputForm.kafkaTopicFieldRequiredMessage', {
+          defaultMessage: 'Topic is required',
+        })
+      );
+    }
+  });
+
+  if (value.length === 0) {
+    res.push(
+      i18n.translate('xpack.fleet.settings.outputForm.kafkaTopicRequiredMessage', {
+        defaultMessage: 'Topic is required',
+      })
+    );
+  }
+  if (res.length) {
+    return res;
+  }
+}
+
 export function validateKafkaClientId(value: string) {
   const regex = /^[A-Za-z0-9._-]+$/;
   return regex.test(value)
@@ -343,49 +368,6 @@ export function validateKafkaPartitioningGroupEvents(value: string) {
       ];
 }
 
-export function validateKafkaTopics(
-  topics: Array<{
-    topic: string;
-    when?: {
-      condition?: string;
-      type?: string;
-    };
-  }>
-) {
-  const errors: Array<{
-    message: string;
-    index: number;
-    condition?: boolean;
-  }> = [];
-
-  topics.forEach((topic, index) => {
-    if (!topic.topic || topic.topic === '') {
-      errors.push({
-        message: i18n.translate('xpack.fleet.settings.outputForm.kafkaTopicRequiredMessage', {
-          defaultMessage: 'Topic is required',
-        }),
-        index,
-      });
-    }
-    if (
-      !topic.when?.condition ||
-      topic.when.condition === '' ||
-      topic.when.condition.split(':').length - 1 !== 1
-    ) {
-      errors.push({
-        message: i18n.translate('xpack.fleet.settings.outputForm.kafkaTopicConditionRequired', {
-          defaultMessage: 'Must be a key, value pair i.e. "http.response.code: 200"',
-        }),
-        index,
-        condition: true,
-      });
-    }
-  });
-  if (errors.length) {
-    return errors;
-  }
-}
-
 export function validateKafkaHeaders(pairs: Array<{ key: string; value: string }>) {
   const errors: Array<{
     message: string;