Skip to content

Commit

Permalink
Add data-infra topic and modules to prod (#412)
Browse files Browse the repository at this point in the history
  • Loading branch information
martskins committed Jul 8, 2024
1 parent d993e2f commit 78824ff
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 0 deletions.
1 change: 1 addition & 0 deletions prod-aws/kafka-shared-msk/cbc/__env.tf
31 changes: 31 additions & 0 deletions prod-aws/kafka-shared-msk/cbc/cbc.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
resource "kafka_topic" "data_product_events_v1" {
name = "cbc.DataProductEvents"

replication_factor = 3
partitions = 15

config = {
"remote.storage.enable" = "true"
"retention.bytes" = "-1"
"retention.ms" = "2629746000" # keep data for 1 month
"local.retention.ms" = "18000000" # keep data locally for 5 hours
"max.message.bytes" = "2097152" # allow max 2MB for a message
"compression.type" = "zstd"
"cleanup.policy" = "delete"
}
}

module "cbc_data_infra_adapter_consumer" {
source = "../../../modules/tls-app"
consume_topics = []
consume_groups = []
produce_topics = [kafka_topic.data_product_events_v1.name]
cert_common_name = "cbc/cbc-data-infra-adapter-consumer"
}

module "cbc_data_infra_exporter" {
source = "../../../modules/tls-app"
consume_topics = [kafka_topic.data_product_events_v1.name, ]
consume_groups = ["cbc.cbc-data-infra-exporter-v1"]
cert_common_name = "cbc/cbc-data-infra-exporter"
}

0 comments on commit 78824ff

Please sign in to comment.