Skip to content

Commit

Permalink
Add cookbook to run Outlines on Modal
Browse files Browse the repository at this point in the history
  • Loading branch information
rlouf committed Apr 11, 2024
1 parent 868868f commit 938a5fb
Show file tree
Hide file tree
Showing 3 changed files with 191 additions and 0 deletions.
106 changes: 106 additions & 0 deletions docs/cookbook/deploy-using-modal.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
# Run Outlines using Modal

Modal is... In this guide we will show you how you can use Modal to run programs written with Outlines on the cloud.

## Build the image

```python
from modal import Image, Stub, gpu

stub = Stub(name="outlines-app")

outlines_image = Image.debian_slim(python_version="3.11").pip_install(
"outlines==0.0.37",
"transformers==4.38.2",
"datasets==2.18.0",
"accelerate==0.27.2",
)
```

Next we will download the Mistral-7B model from HuggingFace. We will do this as part of the definition of the image so we don't need to define it every time our inference function is run:

```python
def import_model():
import outlines

outlines.models.transformers("mistralai/Mistral-7B-Instruct-v0.2")

outlines_image = outlines_image.run_function(import_model)
```

```python
schema = """{
"title": "Character",
"type": "object",
"properties": {
"name": {
"title": "Name",
"maxLength": 10,
"type": "string"
},
"age": {
"title": "Age",
"type": "integer"
},
"armor": {"$ref": "#/definitions/Armor"},
"weapon": {"$ref": "#/definitions/Weapon"},
"strength": {
"title": "Strength",
"type": "integer"
}
},
"required": ["name", "age", "armor", "weapon", "strength"],
"definitions": {
"Armor": {
"title": "Armor",
"description": "An enumeration.",
"enum": ["leather", "chainmail", "plate"],
"type": "string"
},
"Weapon": {
"title": "Weapon",
"description": "An enumeration.",
"enum": ["sword", "axe", "mace", "spear", "bow", "crossbow"],
"type": "string"
}
}
}"""


@stub.function(image=outlines_image, gpu=gpu.A100(memory=80))
def generate(
prompt: str = "Amiri, a 53 year old warrior woman with a sword and leather armor.",
):
import outlines

model = outlines.models.transformers(
"mistralai/Mistral-7B-v0.1", device="cuda"
)

generator = outlines.generate.json(model, schema)
character = generator(
f"<s>[INST]Give me a character description. Describe {prompt}.[/INST]"
)

print(character)
```

```python
@stub.local_entrypoint()
def main(
prompt: str = "Amiri, a 53 year old warrior woman with a sword and leather armor.",
):
generate.remote(prompt)
```

```bash
pip install modal
```

```bash
modal token set
```

```bash
modal run example.py
```
83 changes: 83 additions & 0 deletions examples/modal_example.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
import modal


stub = modal.Stub(name="outlines-app")


outlines_image = modal.Image.debian_slim(python_version="3.11").pip_install(
"outlines==0.0.37",
"transformers==4.38.2",
"datasets==2.18.0",
"accelerate==0.27.2",
)


def import_model():
import outlines

outlines.models.transformers("mistralai/Mistral-7B-Instruct-v0.2")

outlines_image = outlines_image.run_function(import_model)


schema = """{
"title": "Character",
"type": "object",
"properties": {
"name": {
"title": "Name",
"maxLength": 10,
"type": "string"
},
"age": {
"title": "Age",
"type": "integer"
},
"armor": {"$ref": "#/definitions/Armor"},
"weapon": {"$ref": "#/definitions/Weapon"},
"strength": {
"title": "Strength",
"type": "integer"
}
},
"required": ["name", "age", "armor", "weapon", "strength"],
"definitions": {
"Armor": {
"title": "Armor",
"description": "An enumeration.",
"enum": ["leather", "chainmail", "plate"],
"type": "string"
},
"Weapon": {
"title": "Weapon",
"description": "An enumeration.",
"enum": ["sword", "axe", "mace", "spear", "bow", "crossbow"],
"type": "string"
}
}
}"""


@stub.function(image=outlines_image, gpu=modal.gpu.A100(memory=80))
def generate(
prompt: str = "Amiri, a 53 year old warrior woman with a sword and leather armor.",
):
import outlines

model = outlines.models.transformers(
"mistralai/Mistral-7B-v0.1", device="cuda"
)

generator = outlines.generate.json(model, schema)
character = generator(
f"<s>[INST]Give me a character description. Describe {prompt}.[/INST]"
)

print(character)


@stub.local_entrypoint()
def main(
prompt: str = "Amiri, a 53 year old warrior woman with a sword and leather armor.",
):
generate.remote(prompt)
2 changes: 2 additions & 0 deletions mkdocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ nav:
- Generate synthetic data: cookbook/dating_profiles.md
- Summarize a document: cookbook/chain_of_density.md
- Playing chess: cookbook/models_playing_chess.md
- Run on the cloud:
- Modal: cookbook/deploy-using-modal.md
- Docs:
- reference/index.md
- Generation:
Expand Down

0 comments on commit 938a5fb

Please sign in to comment.