Handle concurrent grammar requests

[drbh]

This PR fixes parallel grammar requests, currently grammar states are not concatenated correctly when a new request is added to the batch and this results in incorrect generation. This PR updates the concatenate function to correctly include the previous states.

fixes: #1601

[drbh]

script used during dev

import json
import time
import random
import requests
import sys

from concurrent.futures import ThreadPoolExecutor

## read arguments (number of requests to fire)
n = 1
if len(sys.argv) > 1:
    n = int(sys.argv[1])

print("Firing", n, "requests...")

# build the request
json_schema = {
    "properties": {
        "choose_color": {
            "description": "Choose a color",
            "examples": ["red", "green", "blue"],
            "title": "Choose a color",
            "type": "string",
        }
    },
    "required": ["choose_color"],
}


def get_input():
    return random.choice(
        [
            "the sky",
            "the grass",
            "the sea",
            "the sun",
            "fire",
            "a tomato",
            "a strawberry",
            "a blueberry",
        ]
    )


# prepare function to fire requests
def fire_request():
    try:
        data = {
            "inputs": f"What color is {get_input()}?",
            "parameters": {
                "best_of": None,
                "temperature": 0.3,
                "repetition_penalty": 1.1,
                "frequency_penalty": None,
                "top_k": 30,
                "top_p": 0.95,
                "typical_p": 0.95,
                "do_sample": True,
                "max_new_tokens": 30,
                "return_full_text": False,
                "stop": [],
                "truncate": None,
                "watermark": False,
                "details": True,
                "decoder_input_details": False,
                "seed": None,
                "top_n_tokens": None,
                "grammar": {"type": "json", "value": json_schema},
            },
        }

        headers = {
            "Content-Type": "application/json",
        }

        response = requests.post(
            "http://127.0.0.1:3000/generate", headers=headers, json=data
        )
        # print(response.text)
        response_data = response.json()
        grammar_response = json.loads(response_data["generated_text"])
        print(json.dumps(grammar_response, indent=2))
    except Exception as e:
        # expect that some requests will fail
        print(e, response.text)


## fire N requests in parallel
start = time.time()
with ThreadPoolExecutor() as executor:
    for _ in range(n):
        executor.submit(fire_request)
end = time.time()
print("Time taken:", end - start)

run the following to send 10 requests at the same time

python concurrent.py 10

[OlivierDehaene]

@drbh you can use k6 for this kind of stuff. You have some template load testing scripts in load_tests. If the script go to completion then you can be confident that your model is able to concat/filter correctly.