BabyLlama with CPP backend

cpp/build.sh

@@ -299,6 +299,10 @@ function build() {
     mv $DEPS_DIR/../src/examples/libmnist_handler.so $DEPS_DIR/../../test/resources/torchscript_model/mnist/mnist_handler/libmnist_handler.so
   fi
 
+  if [ -f "$DEPS_DIR/../src/examples/libbabyllama_handler.so" ]; then
+    mv $DEPS_DIR/../src/examples/libbabyllama_handler.so $DEPS_DIR/../../test/resources/torchscript_model/babyllama/babyllama_handler/libbabyllama_handler.so
+  fi
+
   cd $DEPS_DIR/../..
   if [ -f "$DEPS_DIR/../test/torchserve_cpp_test" ]; then
     $DEPS_DIR/../test/torchserve_cpp_test

cpp/src/examples/CMakeLists.txt

@@ -5,3 +5,12 @@ list(APPEND MNIST_SOURCE_FILES ${MNIST_SRC_DIR}/mnist_handler.cc)
 add_library(mnist_handler SHARED ${MNIST_SOURCE_FILES})
 target_include_directories(mnist_handler PUBLIC ${MNIST_SRC_DIR})
 target_link_libraries(mnist_handler PRIVATE ts_backends_torch_scripted ts_utils ${TORCH_LIBRARIES}) 
+
+
+set(BABYLLAMA_SRC_DIR "${torchserve_cpp_SOURCE_DIR}/src/examples/babyllama")
+set(BABYLLAMA_SOURCE_FILES "")
+list(APPEND BABYLLAMA_SOURCE_FILES ${BABYLLAMA_SRC_DIR}/baby_llama_handler.cc)
+add_library(babyllama_handler SHARED ${BABYLLAMA_SOURCE_FILES})
+target_include_directories(babyllama_handler PUBLIC ${BABYLLAMA_SRC_DIR})
+target_link_libraries(babyllama_handler PRIVATE ts_backends_torch_scripted ts_utils ${TORCH_LIBRARIES})
+target_compile_options(babyllama_handler PRIVATE -Wall -Wextra -Ofast)

cpp/src/examples/babyllama/README.md

@@ -0,0 +1,84 @@
+This example is adapted from https://github.com/karpathy/llama2.c 
+
+### Setup
+
+1. Follow the instructions from [README.md](../../../README.md) to build the cpp backend
+2. Download the model and tokenizer using the following command
+
+```
+wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.bin
+```
+Download the [tokenizer.bin](https://github.com/karpathy/llama2.c/blob/master/tokenizer.bin) file from the [llama2.c](https://github.com/karpathy/llama2.c) repo
+
+3. Update [config.json](config.json) with the path of the downloaded model and tokenizer.
+
+For example
+
+```
+{
+"checkpoint_path" : "/home/ubuntu/serve/cpp/stories15M.bin",
+"tokenizer_path" : "/home/ubuntu/serve/cpp/src/examples/babyllama/tokenizer.bin"
+}
+```
+
+4. Run the build
+
+```
+cd serve/cpp
+./builld.sh
+```
+
+Once the build is completed, `libbabyllama_handler.so` file is generated in the [babyllama_handler](../../../test/resources/torchscript_model/babyllama/babyllama_handler) folder
+
+### Generate MAR file
+
+Move to [babyllama_handler](../../../test/resources/torchscript_model/babyllama/babyllama_handler) folder and run the following command to generate mar file
+
+```
+torch-model-archiver --model-name llm --version 1.0 --serialized-file dummy.pt --handler libbabyllama_handler:BabyLlamaHandler --runtime LSP --extra-files config.json
+```
+
+Create model store directory and move the mar file
+
+```
+mkdir model_store
+mv llm.mar model_store/llm.mar
+```
+
+### Inference
+
+Start torchserve using the following command
+
+```
+torchserve --start --ncs --ts-config config.properties --model-store model_store/
+```
+
+Register the model using the following command
+
+```
+curl -v -X POST "http://localhost:8081/models?initial_workers=1&url=llm.mar&batch_size=2&max_batch_delay=5000&initial_workers=3"
+```
+
+Infer the model using the following command
+
+```
+curl http://localhost:8080/predictions/llm -T prompt.txt
+```
+
+This example supports batching. To run batch prediction, run the following command 
+
+```
+curl http://localhost:8080/predictions/llm -T prompt.txt & curl http://localhost:8080/predictions/llm -T prompt1.txt &
+```
+
+Sample Response
+
+```
+Hello my name is Daisy. Daisy is three years old. She loves to play with her toys.
+One day, Daisy's mommy said, "Daisy, it's time to go to the store." Daisy was so excited! She ran to the store with her mommy.
+At the store, Daisy saw a big, red balloon. She wanted it so badly! She asked her mommy, "Can I have the balloon, please?"
+Mommy said, "No, Daisy. We don't have enough money for that balloon."
+Daisy was sad. She wanted the balloon so much. She started to cry.
+Mommy said, "Daisy, don't cry. We can get the balloon. We can buy it and take it home."
+Daisy smiled. She was so happy. She hugged her mommy and said, "Thank you, mommy!"
+```