Skip to content

Commit

Permalink
feat: add helm config for tasks pods
Browse files Browse the repository at this point in the history
  • Loading branch information
BrendanGalloway committed Mar 7, 2024
1 parent 8ff1194 commit f5f2522
Show file tree
Hide file tree
Showing 5 changed files with 218 additions and 1 deletion.
10 changes: 10 additions & 0 deletions helm-chart/sefaria-project/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -175,3 +175,13 @@ preferredDuringSchedulingIgnoredDuringExecution:
- "true"
{{- end }}
{{- end }}

{{/*
Setup complete tasks queue info
*/}}
{{- define "sefaria.tasks.internalQueues" }}
tasks: {{ .Values.deployEnv }}-tasks
{{- end }}
{{- define "sefaria.tasks.queues" }}
{{- merge (fromYaml (include "sefaria.tasks.internalQueues" . )) .Values.tasks.queues | toYaml }}
{{- end }}
10 changes: 10 additions & 0 deletions helm-chart/sefaria-project/templates/configmap/local-settings.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,13 @@ data:
SENTRY_CODE_VERSION: {{ .Values.web.containerImage.tag }}
FAIL_GRACEFULLY: "{{ .Values.localSettings.FAIL_GRACEFULLY }}"
SEARCH_HOST: {{ .Values.nginx.SEARCH_HOST | quote }}
{{- if .Values.tasks.enabled }}
REDIS_URL: {{ .Values.tasks.redis.url }}
REDIS_PORT: {{ .Values.tasks.redis.port | quote }}
CELERY_REDIS_BROKER_DB_NUM: {{ .Values.tasks.redis.brokerNumber | quote }}
CELERY_REDIS_RESULT_BACKEND_DB_NUM: {{ .Values.tasks.redis.backendNumber | quote }}
CELERY_QUEUES: {{ fromYaml (include "sefaria.tasks.queues" .) | toJson | quote }}
SENTINEL_HEADLESS_URL: {{ .Values.tasks.sentinel.headlessUrl }}
SENTINEL_PORT: {{ .Values.tasks.sentinel.port | quote }}
SENTINEL_TRANSPORT_OPTS: {{ .Values.tasks.sentinel.transportOptions | toJson | quote }}
{{- end }}
158 changes: 158 additions & 0 deletions helm-chart/sefaria-project/templates/rollout/task.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{{- if .Values.tasks.enabled }}
{{- $QueueConfig := (fromYaml (include "sefaria.tasks.queues" . )) -}}
---
# apiVersion: apps/v1
# kind: Deployment
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
name: {{ .Values.deployEnv }}-tasks
labels:
deployEnv: "{{ .Values.deployEnv }}"
app: tasks-{{ .Values.deployEnv }}
# releaseRevision: "{{ .Release.Revision }}"
annotations:
rollout.argoproj.io/revision: "{{ .Release.Revision }}"
spec:
strategy:
blueGreen:
autoPromotionEnabled: true
prePromotionAnalysis:
templates:
# - templateName: rollout-readiness-{{ .Values.deployEnv }}
- templateName: rollout-ready-{{ .Values.deployEnv }}
args:
- name: healthcheck-hostname
value: "nginx-{{ .Values.deployEnv }}-{{ .Release.Revision }}"
{{- if .Values.deploymentMessage }}
postPromotionAnalysis:
templates:
- templateName: rollout-complete-{{ .Values.deployEnv }}
args:
- name: revision
value: "{{ .Release.Revision }}"
- name: deployment
value: {{ .Values.deployEnv }}
- name: version
value: {{ .Values.web.containerImage.tag }}
- name: chartVersion
value: {{ .Chart.Version }}
{{- end }}
selector:
matchLabels:
app: tasks-{{ .Values.deployEnv }}
# releaseRevision: "{{ .Release.Revision }}"
revisionHistoryLimit: 2
replicas: {{ .Values.tasks.replicaCount }}
progressDeadlineSeconds: 1200
template:
metadata:
labels:
app: tasks-{{ .Values.deployEnv }}
tier: application
deployEnv: "{{ .Values.deployEnv }}"
stackRole: celery
releaseRevision: "{{ .Release.Revision }}"
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- mongo
topologyKey: kubernetes.io/hostname
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- "tasks-{{ .Values.deployEnv }}"
topologyKey: kubernetes.io/hostname
nodeAffinity:
{{- include "sefaria.nodeAffinities" . | nindent 10 }}
containers:
- name: tasks
image: "{{ .Values.web.containerImage.imageRegistry }}:{{ .Values.web.containerImage.tag }}"
imagePullPolicy: Always
command: ["celery"]
args: ["-A", "sefaria.celery_setup.app","worker", "-Q", "{{ $QueueConfig.tasks }}", "-l", "INFO"]
env:
- name: ENV_NAME
value: "{{ .Values.deployEnv }}"
- name: STACK_COMPONENT
value: tasks
- name: REDIS_HOST
value: "redis-{{ .Values.deployEnv }}"
- name: NODEJS_HOST
value: "node-{{ .Values.deployEnv }}-{{ .Release.Revision }}"
- name: VARNISH_HOST
value: "varnish-{{ .Values.deployEnv }}-{{ .Release.Revision }}"
- name: HELM_REVISION
value: "{{ .Release.Revision }}"
{{- if .Values.tasks.enabled }}
- name: SENTINEL_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.tasks.sentinel.passwordRef.name }}
key: {{ .Values.tasks.sentinel.passwordRef.key }}
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.tasks.redis.passwordRef.name }}
key: {{ .Values.tasks.redis.passwordRef.key }}
{{- end }}
envFrom:
- secretRef:
name: {{ template "sefaria.secrets.elasticUser" . }}
- secretRef:
name: {{ .Values.secrets.localSettings.ref }}
optional: true
- configMapRef:
name: local-settings-{{ .Values.deployEnv }}
- secretRef:
name: local-settings-secrets-{{ .Values.deployEnv }}
optional: true
- configMapRef:
name: local-settings-web-{{ .Values.deployEnv }}
optional: true
- secretRef:
name: local-settings-web-secrets-{{ .Values.deployEnv }}
optional: true
resources: {{ toYaml .Values.tasks.resources | nindent 10 }}
volumeMounts:
- mountPath: /app/sefaria/local_settings.py
name: local-settings
subPath: local_settings.py
readOnly: true
- mountPath: /client-secret
name: client-secret
readOnly: true
- mountPath: /app/logging-secret.json
name: logging-secret
subPath: logging-secret.json
readOnly: true
- name: elastic-cert
mountPath: /etc/ssl/certs/elastic
readOnly: true
volumes:
- name: local-settings
configMap:
name: local-settings-file-{{ .Values.deployEnv }}
items:
- key: local_settings.py
path: local_settings.py
- name: elastic-cert
secret:
secretName: {{ template "sefaria.secrets.elasticCertificate" . }}
optional: true
- name: client-secret
secret:
secretName: {{ template "sefaria.secrets.googleClient" . }} # needs to be checked if it's a reference object or the data object we created.
- name: logging-secret
secret:
secretName: {{ template "sefaria.secrets.logging" . }}
optional: true
{{- end }}
12 changes: 12 additions & 0 deletions helm-chart/sefaria-project/templates/rollout/web.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,18 @@ spec:
- name: OTEL_RESOURCE_ATTRIBUTES
value: k8s.container.name=app,k8s.deployment.name={{ .Values.deployEnv }}-web,k8s.namespace.name={{ .Release.Namespace }},k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=$(OTEL_RESOURCE_ATTRIBUTES_POD_NAME)
{{- end }}
{{- if .Values.tasks.enabled }}
- name: SENTINEL_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.tasks.sentinel.passwordRef.name }}
key: {{ .Values.tasks.sentinel.passwordRef.key }}
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.tasks.redis.passwordRef.name }}
key: {{ .Values.tasks.redis.passwordRef.key }}
{{- end }}
envFrom:
- secretRef:
name: {{ template "sefaria.secrets.elasticUser" . }}
Expand Down
29 changes: 28 additions & 1 deletion helm-chart/sefaria-project/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,34 @@ varnish:
image: ibmcom/varnish-metrics-exporter
tag: 0.32.0

tasks:
enabled: false
resources:
requests:
memory: "100Mi"
cpu: "100m"
limits:
memory: "200Mi"
cpu: "400m"
replicaCount: 1
redis:
url: redis.redis.svc
port: 6379
passwordRef:
name: redis-password
key: password
brokerNumber: 3
backendNumber: 4
sentinel:
headlessUrl: redis-headless.redis.svc
port: 26379
transportOptions: {}
passwordRef:
name: sentinel-password
key: password
queues:
llm: default-llm

ingress:
# You can set path: serviceName: and port: for each host. By default they are
# set to '/*', 'nginx', '80' respectively for each. Do note that changing this
Expand Down Expand Up @@ -390,7 +418,6 @@ secrets:
ref: elastic-admin
# data:


# Settings for various cronjobs
cronJobs:
# Settings for regenerating long cached data
Expand Down

0 comments on commit f5f2522

Please sign in to comment.