Adjust retention times too (#552)

* Adjust retention times too * Adjust account_identity_account_management_events config
utilitywarehouse · Sep 24, 2024 · d975c34 · d975c34
1 parent faec101
commit d975c34
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 15 deletions.
diff --git a/prod-aws/kafka-shared-msk/account-identity/account.tf b/prod-aws/kafka-shared-msk/account-identity/account.tf
@@ -61,16 +61,15 @@ resource "kafka_topic" "account_identity_public_account_events" {
 
 resource "kafka_topic" "account_identity_account_management_events" {
   config = {
-    "cleanup.policy"   = "compact"
+    "cleanup.policy"   = "delete"
     "compression.type" = "zstd"
-
-    # compaction lag of 7 days
-    "max.compaction.lag.ms" = "604800000"
-    # infinite retention
-    "retention.ms" = "-1"
+    # keep data in hot storage for 1 day
+    "local.retention.ms" = "86400000"
+    # enable remote storage
+    "remote.storage.enable" = "true"
   }
   name               = "account-identity.account-management-events-green"
-  partitions         = 15
+  partitions         = 1
   replication_factor = 3
 }
 
@@ -124,8 +123,8 @@ resource "kafka_topic" "account_identity_address_lookup_analytics_v1" {
 resource "kafka_topic" "account_identity_analytics_bill_change_events" {
   config = {
     "compression.type" = "zstd"
-    # infinite retention
-    "retention.ms" = "-1"
+    # keep data for 7 days
+    "retention.ms" = "604800000"
     # keep data in hot storage for 1 day
     "local.retention.ms" = "86400000"
     # enable remote storage

diff --git a/prod-aws/kafka-shared-msk/account-identity/exceptions.tf b/prod-aws/kafka-shared-msk/account-identity/exceptions.tf
@@ -36,8 +36,8 @@ resource "kafka_topic" "account_identity_supply_address_debt_exception_check_eve
     "remote.storage.enable" = "true"
     # keep data in hot storage for 1 day
     "local.retention.ms" = "86400000"
-    # infinite retention
-    "retention.ms"     = "-1"
+    # keep data for 7 days
+    "retention.ms"     = "604800000"
     "compression.type" = "zstd"
   }
   name               = "account-identity.supply.address.debt.exception.check.events"
@@ -52,8 +52,8 @@ resource "kafka_topic" "account_identity_correspondence_address_exception_check_
     "remote.storage.enable" = "true"
     # keep data in hot storage for 1 day
     "local.retention.ms" = "86400000"
-    # infinite retention
-    "retention.ms"     = "-1"
+    # keep data for 7 days
+    "retention.ms"     = "604800000"
     "compression.type" = "zstd"
   }
   name               = "account-identity.correspondence.address.exception.check.events"
@@ -83,8 +83,8 @@ resource "kafka_topic" "account_identity_correspondence_address_debt_exception_c
     "remote.storage.enable" = "true"
     # keep data in hot storage for 1 day
     "local.retention.ms" = "86400000"
-    # infinite retention
-    "retention.ms"     = "-1"
+    # keep data for 7 days
+    "retention.ms"     = "604800000"
     "compression.type" = "zstd"
   }
   name               = "account-identity.correspondence.address.debt.exception.check.events"