Merge pull request #92 from Andrewwango/fastapi-backend

Containerised variant of the backend

.gitignore

@@ -347,3 +347,4 @@ __queuestorage__
 __azurite_db*__.json
 
 *.wav
+temp/

README.md

@@ -94,24 +94,37 @@ The component and demo frontends in this repo are configured to be remotely buil
 
 ### 3.5 Deploy backend
 
-#### 3.5.1 To Deploy Locally
+#### 3.5.1 To Deploy Locally as an Azure Function
 
-Create `local.settings.json` based on the provided `local.settings.json.example` in `backend/`. Probably also create a virtual environment.
+Create `local.settings.json` based on the provided `local.settings.json.example` in `backend/`. Then ensure you have a Python virtual environment with all requirements activated. This can be set up as follows:
 
+```bash
+python3 -m venv venv
+source venv/bin/activate
+pip install -r backend-functions/requirements.txt
 ```
-cd backend
-pip install -r requirements.txt
-func start
+
+Then run the local deployment script.
+
+```
+bash deploy-local.sh
 ```
 
-#### 3.5.2 To Deploy to Azure
+Once finishes, the created `temp/` directory can be safely deleted.
+
+#### 3.5.2 To Deploy to Azure Functions
 
 ```
-cd backend
-func azure functionapp publish shwast-fun-app
+bash deploy-functions.sh
 ```
 
-(if using a different function app, replace `shwast-fun-app` with the new name)
+(if using a different function app, set `FUNCTION_APP` variable with the new name)
+
+#### 3.5.3 To Deploy to Azure Container Instance
+
+```
+bash deploy-container.sh
+```
 
 You must also ensure `shwast-fun-app` resource is configured with the environment variables required (see `local.settings.json.example`).
 
@@ -123,4 +136,4 @@ Use the text client.
 cd text-client
 pip install -r requirements.txt
 python main.py
-```
+```

backend-fastapi/.env.example

@@ -0,0 +1,8 @@
+OPENAI_API_KEY=...
+OPENAI_API_URL=...
+OPENAI_CHATGPT_DEPLOYMENT=...
+OPENAI_GPT_DEPLOYMENT=...
+AZURE_SPEECH_KEY=...
+AZURE_SPEECH_REGION=...
+AZURE_LANGUAGE_KEY=...
+AZURE_LANGUAGE_ENDPOINT=...

backend-fastapi/requirements.txt

@@ -0,0 +1,8 @@
+azure-ai-textanalytics
+azure-cognitiveservices-speech
+fastapi
+openai
+pydantic
+python-dotenv
+python-multipart
+uvicorn

backend-fastapi/server.py

@@ -0,0 +1,78 @@
+if __name__ == "__main__":
+    from dotenv import load_dotenv
+    load_dotenv()
+
+import asyncio
+from collections import defaultdict
+from datetime import datetime, timedelta
+from typing import Annotated
+
+import fastapi
+from fastapi.responses import StreamingResponse
+import uvicorn
+
+from accessible_search import handlers, protocol, services
+
+app = fastapi.FastAPI()
+
+rate_limit_lock = asyncio.Lock()
+rate_limit_data = defaultdict(list)
+rate_limit_times = 10
+rate_limit_window = timedelta(seconds=60)
+
+
+@app.middleware("http")
+async def rate_limit_middleware(request: fastapi.Request, call_next):
+    """Rudimentary in-memory rate limiting solution."""
+    global rate_limit_data
+
+    client_ip = request.client.host
+    current_time = datetime.now()
+
+    async with rate_limit_lock:
+        rate_limit_data[client_ip] = [
+            time for time in rate_limit_data[client_ip]
+            if current_time - time < rate_limit_window
+        ]
+
+        if len(rate_limit_data[client_ip]) >= rate_limit_times:
+            raise fastapi.HTTPException(status_code=429, detail="Too Many Requests")
+
+        rate_limit_data[client_ip].append(current_time)
+
+    return await call_next(request)
+
+
+@app.post("/api/chatgpt", response_model=protocol.TextOutputResponse)
+async def query_chatgpt(parameters: protocol.ChatGPTRequest):
+    response_dict = await handlers.handle_query_chatgpt_async(parameters)
+    return protocol.TextOutputResponse(**response_dict)
+
+
+@app.post("/api/chatgpt-stream")
+async def query_chatgpt_stream(parameters: protocol.ChatGPTRequest):
+    event_stream = handlers.handle_query_chatgpt_stream(parameters)
+    return StreamingResponse(event_stream, media_type="text/event-stream")
+
+
+@app.post("/api/select-relevant-section")
+async def select_relevant_section(parameters: protocol.SelectRelevantSectionRequest):
+    response_dict = await handlers.handle_select_relevant_section_async(parameters)
+    return protocol.TextOutputResponse(**response_dict)
+
+
+@app.post("/api/speech-to-text")
+async def speech_to_text(file: Annotated[bytes, fastapi.File()]):
+    response_dict = services.perform_speech_to_text(content=file)
+    return protocol.TextOutputResponse(**response_dict)
+
+
+@app.post("/api/text-to-speech")
+async def text_to_speech(parameters: protocol.TextToSpeechRequest):
+    response_dict = services.perform_text_to_speech(parameters.text)
+    return protocol.TextOutputResponse(**response_dict)
+
+
+if __name__ == "__main__":
+    # For local testing only
+    uvicorn.run("server:app", port=8000)

backend/backend_function/__init__.py → backend-functions/backend_function/__init__.py

@@ -4,7 +4,7 @@
 
 import azure.functions as func
 
-from backend_function import preprocessing, prompts, services
+from accessible_search import handlers, protocol, services
 from backend_function.exceptions import HTTPException
 
 
@@ -13,6 +13,7 @@ def main(req: func.HttpRequest) -> func.HttpResponse:
 
     action_mapping: dict[str, Callable[[func.HttpRequest], func.HttpResponse]] = {
         "chatgpt": action_query_chatgpt,
+        "chatgpt-stream": action_query_chatgpt_stream,
         "select-relevant-section": action_select_relevant_section,
         "speech-to-text": action_speech_to_text,
         "text-to-speech": action_text_to_speech,
@@ -29,30 +30,18 @@ def main(req: func.HttpRequest) -> func.HttpResponse:
 
 
 def action_query_chatgpt(request: func.HttpRequest) -> func.HttpResponse:
-    parameters = get_request_json(request)
-
-    history = preprocessing.preprocess_history(parameters.get("history", []))
-    query = preprocessing.preprocess_query(parameters["query"])
-    context = preprocessing.preprocess_context(parameters["context"])
-
-    prompt = prompts.construct_query_prompt(context, query)
-
-    response_dict = services.perform_chat_completion(history, prompt, parameters)
-
+    parameters = protocol.ChatGPTRequest(**get_request_json(request))
+    response_dict = handlers.handle_query_chatgpt(parameters)
     return build_json_response(response_dict)
 
 
-def action_select_relevant_section(request: func.HttpRequest) -> func.HttpResponse:
-    parameters = get_request_json(request)
-
-    history = preprocessing.preprocess_history(parameters.get("history", []))
-    query = preprocessing.preprocess_query(parameters["query"])
-    context = preprocessing.preprocess_context(parameters.get("context", ""))
+def action_query_chatgpt_stream(request: func.HttpRequest) -> func.HttpResponse:
+    return func.HttpResponse("Streaming not implemented for Functions backend", status_code=501)
 
-    prompt = prompts.construct_select_prompt(parameters["options"], context, query)
-
-    response_dict = services.perform_chat_completion(history, prompt, parameters, max_tokens=16)
 
+def action_select_relevant_section(request: func.HttpRequest) -> func.HttpResponse:
+    parameters = protocol.SelectRelevantSectionRequest(**get_request_json(request))
+    response_dict = handlers.handle_select_relevant_section(parameters)
     return build_json_response(response_dict)
 
 
@@ -64,14 +53,14 @@ def action_speech_to_text(request: func.HttpRequest) -> func.HttpResponse:
     with open(filename, "wb") as f:
         f.write(content)
 
-    response_dict = services.perform_speech_to_text(filename)
+    response_dict = services.perform_speech_to_text(filename=filename)
 
     return build_json_response(response_dict)
 
 
 def action_text_to_speech(request: func.HttpRequest) -> func.HttpResponse:
-    parameters = get_request_json(request)
-    response_dict = services.perform_text_to_speech(parameters["text"])
+    parameters = protocol.TextToSpeechRequest(**get_request_json(request))
+    response_dict = services.perform_text_to_speech(parameters.text)
     return build_json_response(response_dict)
 
 

backend/requirements.txt → backend-functions/requirements.txt

@@ -6,3 +6,4 @@ azure-ai-textanalytics
 azure-cognitiveservices-speech
 azure-functions
 openai
+pydantic

backend-shared/README.md

@@ -0,0 +1 @@
+Shared code across backends (Azure Functions, FastAPI).

backend-shared/accessible_search/handlers.py

@@ -0,0 +1,48 @@
+"""Request handlers."""
+
+import json
+from typing import AsyncIterable
+
+from accessible_search import preprocessing, prompts, protocol, services
+
+
+def _prepare_query(parameters: protocol.ChatGPTRequest) -> tuple[list[dict], str, float]:
+    history = preprocessing.preprocess_history(parameters.history)
+    query = preprocessing.preprocess_query(parameters.query)
+    context = preprocessing.preprocess_context(parameters.context)
+    prompt = prompts.construct_query_prompt(context, query)
+    return history, prompt, parameters.temperature
+
+
+def handle_query_chatgpt(parameters: protocol.ChatGPTRequest) -> protocol.TextOutputResponse:
+    history, prompt, temperature = _prepare_query(parameters)
+    return services.perform_chat_completion(history, prompt, temperature=temperature)
+
+
+async def handle_query_chatgpt_async(parameters: protocol.ChatGPTRequest) -> protocol.TextOutputResponse:
+    history, prompt, temperature = _prepare_query(parameters)
+    return await services.perform_chat_completion_async(history, prompt, temperature=temperature)
+
+
+async def handle_query_chatgpt_stream(parameters: protocol.ChatGPTRequest) -> AsyncIterable[str]:
+    history, prompt, temperature = _prepare_query(parameters)
+    async for result in services.perform_chat_completion_streaming(history, prompt, temperature=temperature):
+        yield json.dumps({"data": result}) + "\n"
+
+
+def _prepare_select(parameters: protocol.SelectRelevantSectionRequest) -> tuple[list[dict], str]:
+    history = preprocessing.preprocess_history(parameters.history)
+    query = preprocessing.preprocess_query(parameters.query)
+    context = preprocessing.preprocess_context(parameters.context)
+    prompt = prompts.construct_select_prompt(parameters.options, context, query)
+    return history, prompt
+
+
+def handle_select_relevant_section(parameters: protocol.SelectRelevantSectionRequest) -> protocol.TextOutputResponse:
+    history, prompt = _prepare_select(parameters)
+    return services.perform_chat_completion(history, prompt, max_tokens=16)
+
+
+async def handle_select_relevant_section_async(parameters: protocol.SelectRelevantSectionRequest) -> protocol.TextOutputResponse:
+    history, prompt = _prepare_select(parameters)
+    return await services.perform_chat_completion_async(history, prompt, max_tokens=16)

backend-shared/accessible_search/protocol.py

@@ -0,0 +1,23 @@
+import pydantic
+
+
+class ChatGPTRequest(pydantic.BaseModel):
+    query: str
+    context: str
+    history: list[dict] = []
+    temperature: float = 0.0
+
+
+class SelectRelevantSectionRequest(pydantic.BaseModel):
+    query: str
+    context: str = ""
+    options: list[str]
+    history: list[dict] = []
+
+
+class TextToSpeechRequest(pydantic.BaseModel):
+    text: str
+
+
+class TextOutputResponse(pydantic.BaseModel):
+    output: str