Add coverage checks to CI

.github/workflows/tests.yml

@@ -31,4 +31,55 @@ jobs:
         pip install .[test]
     - name: Run tests
       run: |
-        pytest
+        pytest --cov=outlines
+    - name: Upload coverage data
+      uses: actions/upload-artifact@v3
+      with:
+        name: coverage-data
+        path: .coverage*
+        if-no-files-found: ignore
+
+  coverage:
+    name: Combine & check coverage.
+    needs: tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v4
+        with:
+          cache: pip
+          python-version: "3.11"
+
+      - name: Set up environment
+        run: |
+          pip install --upgrade coverage[toml]
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: coverage-data
+
+      - name: Fetch master for coverage diff
+        run: |
+          git fetch --no-tags --prune origin main
+
+      - name: Combine coverage & fail if it's <100%.
+        run: |
+          # python -m coverage combine
+          python -m coverage html --skip-covered --skip-empty
+
+          # Report and write to summary.
+          python -m coverage report | sed 's/^/    /' >> $GITHUB_STEP_SUMMARY
+
+          # Report again and fail if under 100%.
+          python -m coverage report --fail-under=100
+
+      - name: Upload HTML report if check failed.
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-report
+          path: htmlcov
+        if: ${{ failure() }}

pyproject.toml

@@ -38,6 +38,7 @@ test = [
     "diffusers",
     "pre-commit",
     "pytest",
+    "pytest-cov",
     "torch",
     "transformers"
 ]
@@ -84,3 +85,20 @@ module = [
     "transformers",
 ]
 ignore_missing_imports = true
+
+[tool.coverage.run]
+omit = [
+    "outlines/_version.py",
+    "tests/*",
+]
+branch = true
+
+[tool.coverage.report]
+omit = [
+    "tests/*",
+]
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+]
+show_missing = true