-
-
Notifications
You must be signed in to change notification settings - Fork 264
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix(llm): add missing local settings to connect to redis.
On the way, refactor some variable names so they match the llm equivalent variables and refactor config generation to its own file.
- Loading branch information
1 parent
1faa580
commit 02f3c23
Showing
7 changed files
with
104 additions
and
38 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,6 @@ | ||
from celery import Celery | ||
from sefaria.celery_setup.config import generate_config_from_env | ||
|
||
app = Celery('sefaria') | ||
app.config_from_object('sefaria.celery_setup.config') | ||
app.conf.update(**generate_config_from_env()) | ||
app.autodiscover_tasks(packages=['sefaria.helper.llm']) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,26 +1,13 @@ | ||
import dns.resolver | ||
from sefaria.settings import CELERY_REDIS_URL, CELERY_SENTINEL_HEADLESS_URL, \ | ||
CELERY_REDIS_BROKER_DB_NUM, CELERY_REDIS_RESULT_BACKEND_DB_NUM | ||
from sefaria.settings import CELERY_REDIS_PORT as CPORT | ||
from sefaria.settings import (REDIS_URL, REDIS_PASSWORD, REDIS_PORT, CELERY_REDIS_BROKER_DB_NUM, | ||
CELERY_REDIS_RESULT_BACKEND_DB_NUM, SENTINEL_HEADLESS_URL, SENTINEL_PASSWORD, | ||
SENTINEL_TRANSPORT_OPTS) | ||
from sefaria.celery_setup.generate_config import generate_config, SentinelConfig, RedisConfig | ||
|
||
|
||
def add_db_num_to_url(url, db_num): | ||
return url.replace(f':{CPORT}', f':{CPORT}/{db_num}') | ||
def generate_config_from_env(): | ||
return generate_config( | ||
RedisConfig(REDIS_URL, REDIS_PASSWORD, REDIS_PORT, CELERY_REDIS_BROKER_DB_NUM, CELERY_REDIS_RESULT_BACKEND_DB_NUM), | ||
SentinelConfig(SENTINEL_HEADLESS_URL, SENTINEL_PASSWORD, REDIS_PORT, SENTINEL_TRANSPORT_OPTS) | ||
) | ||
|
||
|
||
if CELERY_SENTINEL_HEADLESS_URL: | ||
redisdns = dns.resolver.resolve(CELERY_SENTINEL_HEADLESS_URL, 'A') | ||
addressstring = [] | ||
for res in redisdns.response.answer: | ||
for item in res.items: | ||
addressstring.append(f"sentinel://{item.to_text()}:{CPORT}") | ||
joined_address = ";".join(addressstring) | ||
|
||
# celery config vars | ||
broker_url = add_db_num_to_url(joined_address, CELERY_REDIS_BROKER_DB_NUM) | ||
result_backend = add_db_num_to_url(joined_address, CELERY_REDIS_RESULT_BACKEND_DB_NUM) | ||
result_backend_transport_options = {} | ||
broker_transport_options = {} | ||
else: | ||
broker_url = add_db_num_to_url(f"{CELERY_REDIS_URL}:{CPORT}", CELERY_REDIS_BROKER_DB_NUM) | ||
result_backend = add_db_num_to_url(f"{CELERY_REDIS_URL}:{CPORT}", CELERY_REDIS_RESULT_BACKEND_DB_NUM) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
import re | ||
import dns.resolver | ||
from dataclasses import dataclass | ||
|
||
|
||
@dataclass | ||
class SentinelConfig: | ||
url: str | ||
password: str | ||
port: str | ||
transport_opts: dict | ||
|
||
def is_configured(self) -> bool: | ||
""" | ||
Return True if this config has the data it needs to connect to Sentinel | ||
:return: | ||
""" | ||
return bool(self.url) | ||
|
||
|
||
@dataclass | ||
class RedisConfig: | ||
url: str | ||
password: str | ||
port: str | ||
broker_db_num: str | ||
result_backend_db_num: str | ||
|
||
|
||
def add_db_num_to_url(url, port, db_num): | ||
return url.replace(f':{port}', f':{port}/{db_num}') | ||
|
||
|
||
def add_password_to_url(url, password): | ||
if len(password) == 0: | ||
return url | ||
return re.sub(r'((?:redis|sentinel)://)', fr'\1:{password}@', url) | ||
|
||
|
||
def generate_config(redis_config: RedisConfig, sentinel_config: SentinelConfig) -> dict: | ||
if sentinel_config.is_configured(): | ||
redisdns = dns.resolver.resolve(sentinel_config.url, 'A') | ||
addressstring = [] | ||
for res in redisdns.response.answer: | ||
for item in res.items: | ||
curr_redis_url = f"sentinel://{item.to_text()}:{sentinel_config.port}" | ||
curr_redis_url = add_password_to_url(curr_redis_url, redis_config.password) | ||
addressstring.append(curr_redis_url) | ||
joined_address = ";".join(addressstring) | ||
merged_transport_opts = { | ||
**sentinel_config.transport_opts, | ||
"sentinel_kwargs": {"password": sentinel_config.password} | ||
} | ||
|
||
return { | ||
"broker_url": add_db_num_to_url(joined_address, sentinel_config.port, redis_config.broker_db_num), | ||
"result_backend": add_db_num_to_url(joined_address, sentinel_config.port, redis_config.result_backend_db_num), | ||
"result_backend_transport_options": merged_transport_opts, | ||
"broker_transport_options": merged_transport_opts, | ||
} | ||
else: | ||
redis_url = add_password_to_url(f"{redis_config.url}:{redis_config.port}", redis_config.password) | ||
return { | ||
"broker_url": add_db_num_to_url(redis_url, redis_config.port, redis_config.broker_db_num), | ||
"result_backend": add_db_num_to_url(redis_url, redis_config.port, redis_config.result_backend_db_num), | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters