Ruler statefulsets

production/ksonnet/loki/boltdb_shipper.libsonnet

@@ -40,6 +40,11 @@
     'boltdb.shipper.cache-location': '/data/boltdb-cache',
   } else {},
 
+  ruler_args+:: if $._config.using_boltdb_shipper then {
+    // Use PVC for caching
+    'boltdb.shipper.cache-location': '/data/boltdb-cache',
+  } else {},
+
   // we don't dedupe index writes when using boltdb-shipper so don't deploy a cache for it.
   memcached_index_writes: if $._config.using_boltdb_shipper then {} else super.memcached_index_writes,
 

production/ksonnet/loki/config.libsonnet

@@ -23,6 +23,10 @@
     querier_pvc_size: '10Gi',
     querier_pvc_class: 'fast',
 
+    stateful_rulers: false,
+    ruler_pvc_size: '10Gi',
+    ruler_pvc_class: 'fast',
+
     compactor_pvc_size: '10Gi',
     compactor_pvc_class: 'fast',
 

production/ksonnet/loki/ruler.libsonnet

@@ -1,5 +1,9 @@
 {
   local container = $.core.v1.container,
+  local pvc = $.core.v1.persistentVolumeClaim,
+  local statefulSet = $.apps.v1.statefulSet,
+  local deployment = $.apps.v1.deployment,
+  local volumeMount = $.core.v1.volumeMount,
 
   ruler_args:: $._config.commonArgs {
     target: 'ruler',
@@ -13,13 +17,15 @@
       $.util.resourcesRequests('1', '6Gi') +
       $.util.resourcesLimits('16', '16Gi') +
       $.util.readinessProbe +
-      $.jaeger_mixin
+      $.jaeger_mixin +
+      if $._config.stateful_rulers then
+        container.withVolumeMountsMixin([
+          volumeMount.new('ruler-data', '/data'),
+        ]) else {}
     else {},
 
-  local deployment = $.apps.v1.deployment,
-
   ruler_deployment:
-    if $._config.ruler_enabled then
+    if $._config.ruler_enabled && !$._config.stateful_rulers then
       deployment.new('ruler', 2, [$.ruler_container]) +
       deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(600) +
       $.config_hash_mixin +
@@ -28,10 +34,31 @@
       $.util.antiAffinity
     else {},
 
-  local service = $.core.v1.service,
+  ruler_service: if !$._config.ruler_enabled
+  then {}
+  else
+    if $._config.stateful_rulers
+    then $.util.serviceFor($.ruler_statefulset)
+    else $.util.serviceFor($.ruler_deployment),
 
-  ruler_service:
-    if $._config.ruler_enabled then
-      $.util.serviceFor($.ruler_deployment)
-    else {},
+
+  // PVC for rulers when running as statefulsets
+  ruler_data_pvc:: if $._config.ruler_enabled && $._config.stateful_rulers then
+    pvc.new('ruler-data') +
+    pvc.mixin.spec.resources.withRequests({ storage: $._config.ruler_pvc_size }) +
+    pvc.mixin.spec.withAccessModes(['ReadWriteOnce']) +
+    pvc.mixin.spec.withStorageClassName($._config.ruler_pvc_class)
+  else {},
+
+  ruler_statefulset: if $._config.ruler_enabled && $._config.stateful_rulers then
+    statefulSet.new('ruler', 2, [$.ruler_container], $.ruler_data_pvc) +
+    statefulSet.mixin.spec.withServiceName('ruler') +
+    statefulSet.mixin.spec.withPodManagementPolicy('Parallel') +
+    $.config_hash_mixin +
+    $.util.configVolumeMount('loki', '/etc/loki/config') +
+    $.util.configVolumeMount('overrides', '/etc/loki/overrides') +
+    $.util.antiAffinity +
+    statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') +
+    statefulSet.mixin.spec.template.spec.securityContext.withFsGroup(10001)  // 10001 is the group ID assigned to Loki in the Dockerfile
+  else {},
 }