diff --git a/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml b/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml index 04776902e4..c7cdd4fc18 100644 --- a/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml +++ b/helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml @@ -214,12 +214,15 @@ data: } # Celery - CELERY_REDIS_PORT = os.getenv("CELERY_REDIS_PORT") + REDIS_PORT = os.getenv("REDIS_PORT") + REDIS_PASSWORD = os.getenv("REDIS_PASSWORD") CELERY_REDIS_BROKER_DB_NUM = os.getenv("CELERY_REDIS_BROKER_DB_NUM") CELERY_REDIS_RESULT_BACKEND_DB_NUM = os.getenv("CELERY_REDIS_RESULT_BACKEND_DB_NUM") - # Either define CELERY_SENTINEL_HEADLESS_URL if using sentinel or CELERY_REDIS_URL for a simple redis instance - CELERY_SENTINEL_HEADLESS_URL = os.getenv("CELERY_SENTINEL_HEADLESS_URL") - CELERY_REDIS_URL = os.getenv("CELERY_REDIS_URL") + # Either define SENTINEL_HEADLESS_URL if using sentinel or REDIS_URL for a simple redis instance + SENTINEL_HEADLESS_URL = os.getenv("SENTINEL_HEADLESS_URL") + SENTINEL_TRANSPORT_OPTS = json.loads(os.getenv("SENTINEL_TRANSPORT_OPTS", "{}")) + SENTINEL_PASSWORD = os.getenv("SENTINEL_PASSWORD") + REDIS_URL = os.getenv("REDIS_URL") MOBILE_APP_KEY = os.getenv("MOBILE_APP_KEY") diff --git a/helm-chart/sefaria-project/values.yaml b/helm-chart/sefaria-project/values.yaml index dc934b2ee0..f51cd3703e 100644 --- a/helm-chart/sefaria-project/values.yaml +++ b/helm-chart/sefaria-project/values.yaml @@ -351,11 +351,14 @@ secrets: # RECAPTCHA_PUBLIC_KEY: # RECAPTCHA_PRIVATE_KEY: # SIMPLE_JWT_SIGNING_KEY: - # CELERY_REDIS_PORT + # REDIS_PORT + # REDIS_PASSWORD # CELERY_REDIS_BROKER_DB_NUM # CELERY_REDIS_RESULT_BACKEND_DB_NUM - # CELERY_SENTINEL_HEADLESS_URL - # CELERY_REDIS_URL + # SENTINEL_HEADLESS_URL + # SENTINEL_TRANSPORT_OPTS + # SENTINEL_PASSWORD + # REDIS_URL # MOBILE_APP_KEY: backupManager: # If you're using a reference to an existing secret then the data: section diff --git a/sefaria/celery_setup/app.py b/sefaria/celery_setup/app.py index 62d6393281..c866b3318c 100644 --- a/sefaria/celery_setup/app.py +++ b/sefaria/celery_setup/app.py @@ -1,5 +1,6 @@ from celery import Celery +from sefaria.celery_setup.config import generate_config_from_env app = Celery('sefaria') -app.config_from_object('sefaria.celery_setup.config') +app.conf.update(**generate_config_from_env()) app.autodiscover_tasks(packages=['sefaria.helper.llm']) diff --git a/sefaria/celery_setup/config.py b/sefaria/celery_setup/config.py index f404ba4d74..3d93d98ce4 100644 --- a/sefaria/celery_setup/config.py +++ b/sefaria/celery_setup/config.py @@ -1,26 +1,13 @@ -import dns.resolver -from sefaria.settings import CELERY_REDIS_URL, CELERY_SENTINEL_HEADLESS_URL, \ - CELERY_REDIS_BROKER_DB_NUM, CELERY_REDIS_RESULT_BACKEND_DB_NUM -from sefaria.settings import CELERY_REDIS_PORT as CPORT +from sefaria.settings import (REDIS_URL, REDIS_PASSWORD, REDIS_PORT, CELERY_REDIS_BROKER_DB_NUM, + CELERY_REDIS_RESULT_BACKEND_DB_NUM, SENTINEL_HEADLESS_URL, SENTINEL_PASSWORD, + SENTINEL_TRANSPORT_OPTS) +from sefaria.celery_setup.generate_config import generate_config, SentinelConfig, RedisConfig -def add_db_num_to_url(url, db_num): - return url.replace(f':{CPORT}', f':{CPORT}/{db_num}') +def generate_config_from_env(): + return generate_config( + RedisConfig(REDIS_URL, REDIS_PASSWORD, REDIS_PORT, CELERY_REDIS_BROKER_DB_NUM, CELERY_REDIS_RESULT_BACKEND_DB_NUM), + SentinelConfig(SENTINEL_HEADLESS_URL, SENTINEL_PASSWORD, REDIS_PORT, SENTINEL_TRANSPORT_OPTS) + ) -if CELERY_SENTINEL_HEADLESS_URL: - redisdns = dns.resolver.resolve(CELERY_SENTINEL_HEADLESS_URL, 'A') - addressstring = [] - for res in redisdns.response.answer: - for item in res.items: - addressstring.append(f"sentinel://{item.to_text()}:{CPORT}") - joined_address = ";".join(addressstring) - - # celery config vars - broker_url = add_db_num_to_url(joined_address, CELERY_REDIS_BROKER_DB_NUM) - result_backend = add_db_num_to_url(joined_address, CELERY_REDIS_RESULT_BACKEND_DB_NUM) - result_backend_transport_options = {} - broker_transport_options = {} -else: - broker_url = add_db_num_to_url(f"{CELERY_REDIS_URL}:{CPORT}", CELERY_REDIS_BROKER_DB_NUM) - result_backend = add_db_num_to_url(f"{CELERY_REDIS_URL}:{CPORT}", CELERY_REDIS_RESULT_BACKEND_DB_NUM) diff --git a/sefaria/celery_setup/generate_config.py b/sefaria/celery_setup/generate_config.py new file mode 100644 index 0000000000..6d1149571b --- /dev/null +++ b/sefaria/celery_setup/generate_config.py @@ -0,0 +1,66 @@ +import re +import dns.resolver +from dataclasses import dataclass + + +@dataclass +class SentinelConfig: + url: str + password: str + port: str + transport_opts: dict + + def is_configured(self) -> bool: + """ + Return True if this config has the data it needs to connect to Sentinel + :return: + """ + return bool(self.url) + + +@dataclass +class RedisConfig: + url: str + password: str + port: str + broker_db_num: str + result_backend_db_num: str + + +def add_db_num_to_url(url, port, db_num): + return url.replace(f':{port}', f':{port}/{db_num}') + + +def add_password_to_url(url, password): + if len(password) == 0: + return url + return re.sub(r'((?:redis|sentinel)://)', fr'\1:{password}@', url) + + +def generate_config(redis_config: RedisConfig, sentinel_config: SentinelConfig) -> dict: + if sentinel_config.is_configured(): + redisdns = dns.resolver.resolve(sentinel_config.url, 'A') + addressstring = [] + for res in redisdns.response.answer: + for item in res.items: + curr_redis_url = f"sentinel://{item.to_text()}:{sentinel_config.port}" + curr_redis_url = add_password_to_url(curr_redis_url, redis_config.password) + addressstring.append(curr_redis_url) + joined_address = ";".join(addressstring) + merged_transport_opts = { + **sentinel_config.transport_opts, + "sentinel_kwargs": {"password": sentinel_config.password} + } + + return { + "broker_url": add_db_num_to_url(joined_address, sentinel_config.port, redis_config.broker_db_num), + "result_backend": add_db_num_to_url(joined_address, sentinel_config.port, redis_config.result_backend_db_num), + "result_backend_transport_options": merged_transport_opts, + "broker_transport_options": merged_transport_opts, + } + else: + redis_url = add_password_to_url(f"{redis_config.url}:{redis_config.port}", redis_config.password) + return { + "broker_url": add_db_num_to_url(redis_url, redis_config.port, redis_config.broker_db_num), + "result_backend": add_db_num_to_url(redis_url, redis_config.port, redis_config.result_backend_db_num), + } diff --git a/sefaria/local_settings_ci.py b/sefaria/local_settings_ci.py index da136b93b8..12ab5548d0 100644 --- a/sefaria/local_settings_ci.py +++ b/sefaria/local_settings_ci.py @@ -148,12 +148,15 @@ } # Celery -CELERY_REDIS_PORT = 6379 +REDIS_PORT = 26379 +REDIS_PASSWORD = None CELERY_REDIS_BROKER_DB_NUM = 0 CELERY_REDIS_RESULT_BACKEND_DB_NUM = 1 -# Either define CELERY_SENTINEL_HEADLESS_URL if using sentinel or CELERY_REDIS_URL for a simple redis instance -CELERY_SENTINEL_HEADLESS_URL = None -CELERY_REDIS_URL = "redis://127.0.0.1" +# Either define SENTINEL_HEADLESS_URL if using sentinel or REDIS_URL for a simple redis instance +SENTINEL_HEADLESS_URL = None +SENTINEL_TRANSPORT_OPTS = {} +SENTINEL_PASSWORD = None +REDIS_URL = "redis://127.0.0.1" # Key which identifies the Sefaria app as opposed to a user # using our API outside of the app. Mainly for registration diff --git a/sefaria/local_settings_example.py b/sefaria/local_settings_example.py index 907c35a7fd..27ac88de66 100644 --- a/sefaria/local_settings_example.py +++ b/sefaria/local_settings_example.py @@ -256,12 +256,15 @@ } # Celery -CELERY_REDIS_PORT = 6379 +REDIS_PORT = 6379 +REDIS_PASSWORD = None CELERY_REDIS_BROKER_DB_NUM = 0 CELERY_REDIS_RESULT_BACKEND_DB_NUM = 1 -# Either define CELERY_SENTINEL_HEADLESS_URL if using sentinel or CELERY_REDIS_URL for a simple redis instance -CELERY_SENTINEL_HEADLESS_URL = None -CELERY_REDIS_URL = "redis://127.0.0.1" +# Either define SENTINEL_HEADLESS_URL if using sentinel or REDIS_URL for a simple redis instance +SENTINEL_HEADLESS_URL = None +SENTINEL_TRANSPORT_OPTS = {} +SENTINEL_PASSWORD = None +REDIS_URL = "redis://127.0.0.1" # Key which identifies the Sefaria app as opposed to a user # using our API outside of the app. Mainly for registration