feat(llm): create basic tasks for celery

sefaria/celery/app.py

@@ -0,0 +1,7 @@
+from celery import Celery
+
+app = Celery('sefaria')
+app.config_from_object('sefaria.celery.config')
+
+# Load task modules from all registered Django apps.
+app.autodiscover_tasks(packages=['sefaria.helper.llm'])

sefaria/celery/config.py

@@ -0,0 +1,26 @@
+import dns.resolver
+from sefaria.settings import CELERY_REDIS_URL, CELERY_SENTINEL_HEADLESS_URL, \
+    CELERY_REDIS_BROKER_DB_NUM, CELERY_REDIS_RESULT_BACKEND_DB_NUM
+from sefaria.settings import CELERY_REDIS_PORT as CPORT
+
+
+def add_db_num_to_url(url, db_num):
+    return url.replace(f':{CPORT}', f':{CPORT}/{db_num}')
+
+
+if CELERY_SENTINEL_HEADLESS_URL:
+    redisdns = dns.resolver.resolve(CELERY_SENTINEL_HEADLESS_URL, 'A')
+    addressstring = []
+    for res in redisdns.response.answer:
+        for item in res.items:
+            addressstring.append(f"sentinel://{item.to_text()}:{CPORT}")
+    joined_address = ";".join(addressstring)
+
+    # celery config vars
+    broker_url = add_db_num_to_url(joined_address, CELERY_REDIS_BROKER_DB_NUM)
+    result_backend = add_db_num_to_url(joined_address, CELERY_REDIS_RESULT_BACKEND_DB_NUM)
+    result_backend_transport_options = {}
+    broker_transport_options = {}
+else:
+    broker_url = add_db_num_to_url(f"{CELERY_REDIS_URL}:{CPORT}", CELERY_REDIS_BROKER_DB_NUM)
+    result_backend = add_db_num_to_url(f"{CELERY_REDIS_URL}:{CPORT}", CELERY_REDIS_RESULT_BACKEND_DB_NUM)

sefaria/helper/llm/tasks.py

@@ -0,0 +1,18 @@
+"""
+Celery tasks for the LLM server
+"""
+from typing import List
+from celery import shared_task
+from sefaria.model.text import Ref
+from sefaria.model.topic import Topic
+from sefaria.helper.llm.topic_prompt import make_topic_prompt_input
+
+
+@shared_task
+def generate_topic_prompts(lang: str, sefaria_topic: Topic, orefs: List[Ref], contexts: List[str]):
+    return make_topic_prompt_input(lang, sefaria_topic, orefs, contexts)
+
+
+@shared_task
+def save_topic_prompts(topic_prompts: List[dict]):
+    pass