Add ctlog shards that create their own Cloud SQL instances.

Signed-off-by: Ville Aikas <vaikas@chainguard.dev>

terraform/gcp/modules/ctlog/main.tf

@@ -37,4 +37,40 @@ resource "google_dns_record_set" "A_ctfe" {
   managed_zone = var.dns_zone_name
 
   rrdatas = [var.load_balancer_ipv4]
-}
+}
+
+// For generating random suffix into the Cloud SQL instance name.
+resource "random_id" "db_name_suffix" {
+  byte_length = 4
+}
+
+// MYSQL for this particular CTLog
+// The name of the DB Instance created should match the name of the
+// shard for the CTLog.
+module "mysql" {
+  source = "../mysql"
+
+  # Disable DB create/modifications if enable_ctlog_sql is false
+  count = var.enable_ctlog_sql ? 1 : 0
+
+  region     = var.region
+  project_id = var.project_id
+
+  cluster_name      = var.cluster_name
+  database_version  = var.mysql_db_version
+  tier              = var.mysql_tier
+  availability_type = var.mysql_availability_type
+
+  replica_zones = var.mysql_replica_zones
+  replica_tier  = var.mysql_replica_tier
+
+  network = var.network
+
+  instance_name = var.mysql_instance_name != "" ? var.mysql_instance_name : format("%s-ctlog-mysql-%s", var.cluster_name, random_id.db_name_suffix.hex)
+  db_name       = var.mysql_db_name
+
+  ipv4_enabled              = var.mysql_ipv4_enabled
+  require_ssl               = var.mysql_require_ssl
+  backup_enabled            = var.mysql_backup_enabled
+  binary_log_backup_enabled = var.mysql_binary_log_backup_enabled
+}

terraform/gcp/modules/ctlog/outputs.tf

@@ -0,0 +1,21 @@
+/**
+ * Copyright 2022 The Sigstore Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Used when setting up the GKE cluster to talk to MySQL.
+output "ctlog_mysql_instance" {
+  description = "The generated name of the Cloud SQL instance"
+  value       = var.enable_ctlog_sql ? module.mysql.0.mysql_instance : null
+}

terraform/gcp/modules/ctlog/variables.tf

@@ -36,4 +36,101 @@ variable "dns_domain_name" {
 variable "load_balancer_ipv4" {
   description = "IPv4 adddress of external load balancer"
   type        = string
-}
+}
+
+variable "region" {
+  description = "The region in which to create the VPC network"
+  type        = string
+  default     = "us-west1"
+}
+
+variable "enable_ctlog_sql" {
+  description = "Enable a database module for creating/editing Cloud SQL Instance"
+  type        = bool
+  default     = false
+}
+
+// Optional values that can be overridden or appended to if desired.
+variable "cluster_name" {
+  description = "The name to give the new Kubernetes cluster."
+  type        = string
+  default     = "sigstore-staging"
+}
+
+variable "network" {
+  type        = string
+  description = "Network to connect to."
+  default     = "default"
+}
+
+variable "mysql_instance_name" {
+  type        = string
+  description = "Name for CTLog MySQL instance. If unspecified, will default to '[var.cluster-name]-ctlog-mysql-[random.suffix]'"
+  default     = ""
+}
+
+variable "mysql_db_name" {
+  type        = string
+  description = "Name for CTLog MySQL database name."
+  default     = "trillian"
+}
+
+variable "mysql_db_version" {
+  type        = string
+  description = "CTLog MySQL database version."
+  default     = "MYSQL_5_7"
+}
+
+variable "mysql_tier" {
+  type        = string
+  description = "Machine tier for CTLog MySQL instance."
+  default     = "db-n1-standard-1"
+}
+
+variable "mysql_availability_type" {
+  type        = string
+  description = "Availability tier for CTLog MySQL"
+  default     = "REGIONAL"
+}
+
+variable "mysql_replica_zones" {
+  description = "List of zones for read replicas."
+  type        = list(any)
+  default     = []
+}
+
+variable "mysql_replica_tier" {
+  type        = string
+  description = "Machine tier for CTLog MySQL replica."
+  default     = "db-n1-standard-1"
+}
+
+variable "mysql_ipv4_enabled" {
+  type        = bool
+  description = "Whether to enable ipv4 for CTLog MySQL instance."
+  default     = false
+}
+
+variable "mysql_require_ssl" {
+  type        = bool
+  description = "Whether to require ssl for CTLog MySQL instance."
+  default     = true
+}
+
+variable "mysql_backup_enabled" {
+  type        = bool
+  description = "Whether to enable backup configuration for CTLog MySQL instance."
+  default     = true
+}
+
+variable "mysql_binary_log_backup_enabled" {
+  type        = bool
+  description = "Whether to enable binary log for backup for CTLog MySQL instance."
+  default     = true
+}
+
+variable "mysql_database_version" {
+  type        = string
+  description = "CTLog MySQL database version."
+  default     = "MYSQL_5_7"
+}

terraform/gcp/modules/sigstore/outputs.tf

@@ -31,6 +31,13 @@ output "mysql_instance" {
   value       = module.mysql.mysql_instance
 }
 
+// Used for setting up the GKE cluster to talk to CTLog Shard DBs.
+// Outputs a list of strings for each CTLog Cloud SQL instance.
+output "ctlog_mysql_instances" {
+  description = "Names of the DB instances created for the CTLog shards"
+  value       = [for ctlog_shard in module.ctlog_shards : ctlog_shard.ctlog_mysql_instance]
+}
+
 // Full connection string for the MySQL DB>
 output "mysql_connection" {
   description = "The connection string dynamically generated for storage inside the Kubernetes configmap"

terraform/gcp/modules/sigstore/sigstore.tf

@@ -139,7 +139,9 @@ module "gke-cluster" {
   ]
 }
 
-// MYSQL
+// MYSQL. This is the original DB that was used for both Rekor and CTLog.
+// Newer versions of CTLog create their own database instance, so there's
+// one database instance to a single ctlog shard.
 module "mysql" {
   source = "../mysql"
 
@@ -283,12 +285,20 @@ module "oslogin" {
   ]
 }
 
-// ctlog
+// ctlog. This was the original (pre-ga) ctlog that shared the DB instance
+// with Rekor.
 module "ctlog" {
   source = "../ctlog"
 
   project_id = var.project_id
 
+  // We do not want the old CTLog to have its own Cloud SQL instance. It shares
+  // it with Rekor.
+  enable_ctlog_sql = false
+
+  // Required placeholders even though they are not used for the old ctlog.
+  region = var.region
+
   dns_zone_name      = var.dns_zone_name
   dns_domain_name    = var.dns_domain_name
   load_balancer_ipv4 = module.network.external_ipv4_address
@@ -300,6 +310,47 @@ module "ctlog" {
   ]
 }
 
+// ctlog-shards. This will create CTLog shard that has its own Cloud SQL
+// instance for each shard
+module "ctlog_shards" {
+  source = "../ctlog"
+
+  for_each = toset(var.ctlog_shards)
+  // We want each CTLog shard to have its own Cloud SQL instance
+  enable_ctlog_sql = true
+
+  mysql_instance_name = format("ctlog-%s", each.key)
+
+  project_id = var.project_id
+  region     = var.region
+
+  dns_zone_name      = var.dns_zone_name
+  dns_domain_name    = var.dns_domain_name
+  load_balancer_ipv4 = module.network.external_ipv4_address
+
+  cluster_name            = var.cluster_name
+  mysql_database_version  = var.mysql_db_version
+  mysql_tier              = var.mysql_tier
+  mysql_availability_type = var.mysql_availability_type
+
+  mysql_replica_zones = var.mysql_replica_zones
+  mysql_replica_tier  = var.mysql_replica_tier
+
+  network = module.network.network_self_link
+
+  mysql_db_name = var.mysql_db_name
+
+  mysql_ipv4_enabled              = var.mysql_ipv4_enabled
+  mysql_require_ssl               = var.mysql_require_ssl
+  mysql_backup_enabled            = var.mysql_backup_enabled
+  mysql_binary_log_backup_enabled = var.mysql_binary_log_backup_enabled
+
+  depends_on = [
+    module.gke-cluster,
+    module.network
+  ]
+}
+
 // dex
 module "dex" {
   source = "../dex"

terraform/gcp/modules/sigstore/variables.tf

@@ -253,3 +253,9 @@ variable "static_external_ipv4_address" {
   type        = string
   default     = ""
 }
+
+variable "ctlog_shards" {
+  type        = list(string)
+  description = "Array of CTLog shards to create. Entry should be something like [2021, 2022], which would then have 2 independent CTLog shards backed by ctlog-2021 and ctlog-2022 Cloud SQL instances."
+  default     = []
+}