Merge pull request #832 from WGBH-MLA/sep-workers-by-type-sidekiq-env

sep-workers-by-type-sidekiq-env
WGBH-MLA · Feb 22, 2024 · dc44953 · dc44953
2 parents f99bcb2 + 730346c
commit dc44953
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 13 deletions.
diff --git a/bin/worker b/bin/worker
@@ -10,6 +10,10 @@ if ENV['AAPB_HOST']
   %x{mkdir ~/.ssh && ssh-keyscan $AAPB_HOST >> ~/.ssh/known_hosts}
 end
 
+if ENV['IMPORT_WORKER_SIDEKIQ_CONCURRENCY']
+  ENV['SIDEKIQ_CONCURRENCY'] = ENV.fetch('IMPORT_WORKER_SIDEKIQ_CONCURRENCY', '10')
+end
+
 queues = if !ARGV.empty?
            ' -q ' + ARGV.join(' -q ')
          else

diff --git a/hyrax/templates/import-deployment-worker.yaml b/hyrax/templates/import-deployment-worker.yaml
@@ -1,12 +1,12 @@
-{{- if .Values.worker.enabled }}
+{{- if .Values.importWorker.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: {{ include "hyrax.fullname" . }}-import-worker
   labels:
     {{- include "hyrax.labels" . | nindent 4 }}
 spec:
-  replicas: {{ .Values.worker.importReplicaCount }}
+  replicas: {{ .Values.importWorker.replicaCount }}
   selector:
     matchLabels:
       {{- include "hyrax.workerSelectorLabels" . | nindent 6 }}
@@ -33,8 +33,8 @@ spec:
                 name: app-secrets
             {{- end }}
           env:
-            {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
-            {{- toYaml .Values.importWorker.extraEnvVars | nindent 12 }}
+          {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+          {{- toYaml .Values.importWorker.extraEnvVars | nindent 12 }}
           command:
             - sh
             - -c
@@ -77,8 +77,8 @@ spec:
               {{- toYaml . | nindent 12 }}
             {{- end }}
           env:
-            {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
-            {{- toYaml .Values.importWorker.extraEnvVars | nindent 12 }}
+          {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+          {{- toYaml .Values.importWorker.extraEnvVars | nindent 12 }}
           {{- if .Values.worker.readinessProbe.enabled }}
           readinessProbe:
             exec:

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
@@ -117,7 +117,7 @@ extraEnvVars: &envVars
   - name: SOLR_URL
     value: http://10.0.4.190:8983/solr/ams
   - name: SIDEKIQ_CONCURRENCY
-    value: "10"
+    value: "1"
   - name: AWS_ACCESS_KEY
     value: "AKIAIWFN3WW3WBIMKNCQ"
   - name: MAIL_DELIVERY_METHOD
@@ -141,7 +141,6 @@ extraEnvVars: &envVars
 
 worker:
   replicaCount: 1
-  importReplicaCount: 0
   podSecurityContext:
     runAsUser: 1001
     runAsGroup: 101
@@ -151,9 +150,13 @@ worker:
   extraEnvVars: *envVars
 
 importWorker:
+  enabled: false
+  replicaCount: 1
+  # extraEnvVars will not handle overriden values
+  # only additional to the original workerEnvVars list
   extraEnvVars:
-    - name: SIDEKIQ_CONCURRENCY
-      value: "10"
+    - name: IMPORT_WORKER_SIDEKIQ_CONCURRENCY
+      value: "20"
 
 podSecurityContext:
   runAsUser: 1001

diff --git a/ops/prod-deploy.tmpl.yaml b/ops/prod-deploy.tmpl.yaml
@@ -146,7 +146,6 @@ extraEnvVars: &envVars
 
 worker:
   replicaCount: 1
-  importReplicaCount: 3
   podSecurityContext:
     runAsUser: 1001
     runAsGroup: 101
@@ -156,9 +155,13 @@ worker:
   extraEnvVars: *envVars
 
 importWorker:
+  enabled: true
+  replicaCount: 3
+  # extraEnvVars will not handle overriden values
+  # only additional to the original workerEnvVars list
   extraEnvVars:
-    - name: SIDEKIQ_CONCURRENCY
-      value: "10"
+    - name: IMPORT_WORKER_SIDEKIQ_CONCURRENCY
+      value: "20"
 
 podSecurityContext:
   runAsUser: 1001