fix(llm): add missing local settings to connect to redis.

On the way, refactor some variable names so they match the llm equivalent variables and refactor config generation to its own file.

helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml

@@ -214,12 +214,15 @@ data:
     }
     
     # Celery
-    CELERY_REDIS_PORT = os.getenv("CELERY_REDIS_PORT")
+    REDIS_PORT = os.getenv("REDIS_PORT")
+    REDIS_PASSWORD = os.getenv("REDIS_PASSWORD")
     CELERY_REDIS_BROKER_DB_NUM = os.getenv("CELERY_REDIS_BROKER_DB_NUM")
     CELERY_REDIS_RESULT_BACKEND_DB_NUM = os.getenv("CELERY_REDIS_RESULT_BACKEND_DB_NUM")
-    # Either define CELERY_SENTINEL_HEADLESS_URL if using sentinel or CELERY_REDIS_URL for a simple redis instance
-    CELERY_SENTINEL_HEADLESS_URL = os.getenv("CELERY_SENTINEL_HEADLESS_URL") 
-    CELERY_REDIS_URL = os.getenv("CELERY_REDIS_URL")
+    # Either define SENTINEL_HEADLESS_URL if using sentinel or REDIS_URL for a simple redis instance
+    SENTINEL_HEADLESS_URL = os.getenv("SENTINEL_HEADLESS_URL") 
+    SENTINEL_TRANSPORT_OPTS = json.loads(os.getenv("SENTINEL_TRANSPORT_OPTS", "{}"))
+    SENTINEL_PASSWORD = os.getenv("SENTINEL_PASSWORD")
+    REDIS_URL = os.getenv("REDIS_URL")
 
     MOBILE_APP_KEY = os.getenv("MOBILE_APP_KEY")
 

helm-chart/sefaria-project/values.yaml

@@ -351,11 +351,14 @@ secrets:
     #   RECAPTCHA_PUBLIC_KEY:
     #   RECAPTCHA_PRIVATE_KEY:
     #   SIMPLE_JWT_SIGNING_KEY:
-    #   CELERY_REDIS_PORT
+    #   REDIS_PORT
+    #   REDIS_PASSWORD
     #   CELERY_REDIS_BROKER_DB_NUM
     #   CELERY_REDIS_RESULT_BACKEND_DB_NUM
-    #   CELERY_SENTINEL_HEADLESS_URL
-    #   CELERY_REDIS_URL
+    #   SENTINEL_HEADLESS_URL
+    #   SENTINEL_TRANSPORT_OPTS
+    #   SENTINEL_PASSWORD
+    #   REDIS_URL
     #   MOBILE_APP_KEY:
   backupManager:
     # If you're using a reference to an existing secret then the data: section

sefaria/celery_setup/app.py

@@ -1,5 +1,6 @@
 from celery import Celery
+from sefaria.celery_setup.config import generate_config_from_env
 
 app = Celery('sefaria')
-app.config_from_object('sefaria.celery_setup.config')
+app.conf.update(**generate_config_from_env())
 app.autodiscover_tasks(packages=['sefaria.helper.llm'])

sefaria/celery_setup/config.py

@@ -1,26 +1,13 @@
-import dns.resolver
-from sefaria.settings import CELERY_REDIS_URL, CELERY_SENTINEL_HEADLESS_URL, \
-    CELERY_REDIS_BROKER_DB_NUM, CELERY_REDIS_RESULT_BACKEND_DB_NUM
-from sefaria.settings import CELERY_REDIS_PORT as CPORT
+from sefaria.settings import (REDIS_URL, REDIS_PASSWORD, REDIS_PORT, CELERY_REDIS_BROKER_DB_NUM,
+                             CELERY_REDIS_RESULT_BACKEND_DB_NUM, SENTINEL_HEADLESS_URL, SENTINEL_PASSWORD,
+                             SENTINEL_TRANSPORT_OPTS)
+from sefaria.celery_setup.generate_config import generate_config, SentinelConfig, RedisConfig
 
 
-def add_db_num_to_url(url, db_num):
-    return url.replace(f':{CPORT}', f':{CPORT}/{db_num}')
+def generate_config_from_env():
+    return generate_config(
+        RedisConfig(REDIS_URL, REDIS_PASSWORD, REDIS_PORT, CELERY_REDIS_BROKER_DB_NUM, CELERY_REDIS_RESULT_BACKEND_DB_NUM),
+        SentinelConfig(SENTINEL_HEADLESS_URL, SENTINEL_PASSWORD, REDIS_PORT, SENTINEL_TRANSPORT_OPTS)
+    )
 
 
-if CELERY_SENTINEL_HEADLESS_URL:
-    redisdns = dns.resolver.resolve(CELERY_SENTINEL_HEADLESS_URL, 'A')
-    addressstring = []
-    for res in redisdns.response.answer:
-        for item in res.items:
-            addressstring.append(f"sentinel://{item.to_text()}:{CPORT}")
-    joined_address = ";".join(addressstring)
-
-    # celery config vars
-    broker_url = add_db_num_to_url(joined_address, CELERY_REDIS_BROKER_DB_NUM)
-    result_backend = add_db_num_to_url(joined_address, CELERY_REDIS_RESULT_BACKEND_DB_NUM)
-    result_backend_transport_options = {}
-    broker_transport_options = {}
-else:
-    broker_url = add_db_num_to_url(f"{CELERY_REDIS_URL}:{CPORT}", CELERY_REDIS_BROKER_DB_NUM)
-    result_backend = add_db_num_to_url(f"{CELERY_REDIS_URL}:{CPORT}", CELERY_REDIS_RESULT_BACKEND_DB_NUM)

sefaria/celery_setup/generate_config.py

@@ -0,0 +1,66 @@
+import re
+import dns.resolver
+from dataclasses import dataclass
+
+
+@dataclass
+class SentinelConfig:
+    url: str
+    password: str
+    port: str
+    transport_opts: dict
+
+    def is_configured(self) -> bool:
+        """
+        Return True if this config has the data it needs to connect to Sentinel
+        :return:
+        """
+        return bool(self.url)
+
+
+@dataclass
+class RedisConfig:
+    url: str
+    password: str
+    port: str
+    broker_db_num: str
+    result_backend_db_num: str
+
+
+def add_db_num_to_url(url, port, db_num):
+    return url.replace(f':{port}', f':{port}/{db_num}')
+
+
+def add_password_to_url(url, password):
+    if len(password) == 0:
+        return url
+    return re.sub(r'((?:redis|sentinel)://)', fr'\1:{password}@', url)
+
+
+def generate_config(redis_config: RedisConfig, sentinel_config: SentinelConfig) -> dict:
+    if sentinel_config.is_configured():
+        redisdns = dns.resolver.resolve(sentinel_config.url, 'A')
+        addressstring = []
+        for res in redisdns.response.answer:
+            for item in res.items:
+                curr_redis_url = f"sentinel://{item.to_text()}:{sentinel_config.port}"
+                curr_redis_url = add_password_to_url(curr_redis_url, redis_config.password)
+                addressstring.append(curr_redis_url)
+        joined_address = ";".join(addressstring)
+        merged_transport_opts = {
+            **sentinel_config.transport_opts,
+            "sentinel_kwargs": {"password": sentinel_config.password}
+        }
+
+        return {
+            "broker_url": add_db_num_to_url(joined_address, sentinel_config.port, redis_config.broker_db_num),
+            "result_backend": add_db_num_to_url(joined_address, sentinel_config.port, redis_config.result_backend_db_num),
+            "result_backend_transport_options": merged_transport_opts,
+            "broker_transport_options": merged_transport_opts,
+        }
+    else:
+        redis_url = add_password_to_url(f"{redis_config.url}:{redis_config.port}", redis_config.password)
+        return {
+            "broker_url": add_db_num_to_url(redis_url, redis_config.port, redis_config.broker_db_num),
+            "result_backend": add_db_num_to_url(redis_url, redis_config.port, redis_config.result_backend_db_num),
+        }

sefaria/local_settings_ci.py

@@ -148,12 +148,15 @@
 }
 
 # Celery
-CELERY_REDIS_PORT = 6379
+REDIS_PORT = 26379
+REDIS_PASSWORD = None
 CELERY_REDIS_BROKER_DB_NUM = 0
 CELERY_REDIS_RESULT_BACKEND_DB_NUM = 1
-# Either define CELERY_SENTINEL_HEADLESS_URL if using sentinel or CELERY_REDIS_URL for a simple redis instance
-CELERY_SENTINEL_HEADLESS_URL = None
-CELERY_REDIS_URL = "redis://127.0.0.1"
+# Either define SENTINEL_HEADLESS_URL if using sentinel or REDIS_URL for a simple redis instance
+SENTINEL_HEADLESS_URL = None
+SENTINEL_TRANSPORT_OPTS = {}
+SENTINEL_PASSWORD = None
+REDIS_URL = "redis://127.0.0.1"
 
 # Key which identifies the Sefaria app as opposed to a user
 # using our API outside of the app. Mainly for registration

sefaria/local_settings_example.py

@@ -256,12 +256,15 @@
 }
 
 # Celery
-CELERY_REDIS_PORT = 6379
+REDIS_PORT = 6379
+REDIS_PASSWORD = None
 CELERY_REDIS_BROKER_DB_NUM = 0
 CELERY_REDIS_RESULT_BACKEND_DB_NUM = 1
-# Either define CELERY_SENTINEL_HEADLESS_URL if using sentinel or CELERY_REDIS_URL for a simple redis instance
-CELERY_SENTINEL_HEADLESS_URL = None
-CELERY_REDIS_URL = "redis://127.0.0.1"
+# Either define SENTINEL_HEADLESS_URL if using sentinel or REDIS_URL for a simple redis instance
+SENTINEL_HEADLESS_URL = None
+SENTINEL_TRANSPORT_OPTS = {}
+SENTINEL_PASSWORD = None
+REDIS_URL = "redis://127.0.0.1"
 
 # Key which identifies the Sefaria app as opposed to a user
 # using our API outside of the app. Mainly for registration