From d46f0f2065d000da2d7f300f1e6f9f4994436962 Mon Sep 17 00:00:00 2001
From: martskins <martskins@users.noreply.github.com>
Date: Fri, 5 Jul 2024 08:59:49 +0100
Subject: [PATCH] Add data infra exporter and data infra events topic (#405)

* Add data infra exporter and data infra events topic

* Change retention policy
---
 dev-aws/kafka-shared-msk/cbc/cbc.tf | 45 +++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/dev-aws/kafka-shared-msk/cbc/cbc.tf b/dev-aws/kafka-shared-msk/cbc/cbc.tf
index 58f98192..d10a3547 100644
--- a/dev-aws/kafka-shared-msk/cbc/cbc.tf
+++ b/dev-aws/kafka-shared-msk/cbc/cbc.tf
@@ -16,6 +16,23 @@ resource "kafka_topic" "FraudEvents" {
   }
 }
 
+resource "kafka_topic" "data_product_events_v1" {
+  name = "cbc.DataProductEvents"
+
+  replication_factor = 3
+  partitions         = 15
+
+  config = {
+    "remote.storage.enable" = "true"
+    "retention.bytes"       = "-1"
+    "retention.ms"          = "2629746000" # keep data for 1 month
+    "local.retention.ms"    = "18000000"   # keep data locally for 5 hours
+    "max.message.bytes"     = "2097152"    # allow max 2MB for a message
+    "compression.type"      = "zstd"
+    "cleanup.policy"        = "delete"
+  }
+}
+
 resource "kafka_topic" "rating_events_v3" {
   name = "cbc.RatingEvents_v3"
 
@@ -1250,6 +1267,34 @@ module "cbc_account_events_relay" {
   cert_common_name = "cbc/cbc-account-events-relay-v2"
 }
 
+module "cbc_data_infra_adapter_projector" {
+  source = "../../../modules/tls-app"
+  consume_topics = [
+    kafka_topic.lifecycle_events_v2.name,
+    kafka_topic.legacy_account_events_v2.name
+  ]
+  consume_groups   = ["cbc.cbc-customer-information-projector-v1"]
+  cert_common_name = "cbc/cbc-data-infra-adapter-projector"
+}
+
+module "cbc_data_infra_adapter_consumer" {
+  source = "../../../modules/tls-app"
+  consume_topics = [
+    kafka_topic.transaction_events_v3.name,
+    kafka_topic.lifecycle_events_v2.name
+  ]
+  consume_groups   = ["cbc.cbc-customer-information-consumer-v1"]
+  produce_topics   = [kafka_topic.data_product_events_v1.name]
+  cert_common_name = "cbc/cbc-data-infra-adapter-consumer"
+}
+
+module "cbc_data_infra_exporter" {
+  source           = "../../../modules/tls-app"
+  consume_topics   = [kafka_topic.data_product_events_v1.name, ]
+  consume_groups   = ["cbc.cbc-data-infra-exporter-v1"]
+  cert_common_name = "cbc/cbc-data-infra-exporter"
+}
+
 module "cbc_proximo_tls" {
   source = "../../../modules/tls-app"
   consume_topics = [