Add text extraction api

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,37 @@
+name: Bug report
+description: Create a report to help us improve
+title: '[BUG] '
+labels: ['bug']
+assignees: []
+body:
+  - type: input
+    attributes:
+      label: Nesis version
+      description: What Nesis version are you using?
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: To reproduce
+      description: Steps to reproduce the behavior
+      placeholder: |
+        1. Go to '...'
+        2. Click on '....'
+  - type: textarea
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen.
+  - type: textarea
+    attributes:
+      label: Screenshots
+      description: If applicable, add screenshots to help explain the bug.
+  - type: textarea
+    attributes:
+      label: Additional context
+      description: Add any other context about the bug here.

.github/ISSUE_TEMPLATE/custom.md

@@ -0,0 +1,10 @@
+---
+name: Custom issue template
+about: Use this template for any issue not covered by the other templates.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,16 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Description**
+A clear and concise description of what you want to happen.
+
+**Detail**
+Add any other context or screenshots about the feature request.
+
+**Acceptance Criteria**

.github/workflows/documentation.yml

@@ -0,0 +1,59 @@
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Build and Deploy Documentation
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["main"]
+    paths:
+      - "docs/**"
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r docs/requirements.txt
+      - name: Build with MkDocs
+        run: |
+          cd docs && mkdocs build --clean --site-dir ../_site && cd -
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/package.yml

@@ -3,84 +3,120 @@ name: Package Nesis
 on:
   workflow_dispatch:
     inputs:
-      branch:
-        description: 'Branch'
-        required: true
-        default: main
       tag:
         description: 'Tag'
         required: true
-        default: latest
+      core:
+        description: 'Core'
+        required: true
+        default: cpu
+        type: choice
+        options:
+          - cpu
+          - cuda
 
 jobs:
   package_api:
-    name: API
+    name: Package ${{ github.event.inputs.tag }} API
     runs-on: ubuntu-latest
+    if: ${{ (github.ref == 'refs/heads/main') || startsWith(github.ref, 'refs/tags/') }}
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.inputs.branch }}
+        uses: actions/checkout@v4
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_PASSWORD }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
           file: ./nesis/api/Dockerfile
-          tags: ametnes/nesis:${{ github.event.inputs.tag }}-api
+          tags: ametnes/nesis:${{ github.event.inputs.tag }}-api,ametnes/nesis:latest-api
+          build-args: |
+            NESIS_VERSION=${{ github.event.inputs.tag }}
 
   package_frontend:
-    name: Frontend
+    name: Package ${{ github.event.inputs.tag }} frontend
     runs-on: ubuntu-latest
+    if: ${{ (github.ref == 'refs/heads/main') || startsWith(github.ref, 'refs/tags/') }}
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.inputs.dockerfile_branch }}
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_PASSWORD }}
 
       - name: Build and push frontend Docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
           file: ./nesis/frontend/Dockerfile
-          tags: ametnes/nesis:${{ github.event.inputs.tag }}-frontend
+          tags: ametnes/nesis:${{ github.event.inputs.tag }}-frontend, ametnes/nesis:latest-frontend
           build-args: |
             PUBLIC_URL=/
             PROFILE=PROD
+            NESIS_VERSION=${{ github.event.inputs.tag }}
 
-  package_rag:
-    name: RAG
+  package_rag_cpu:
+    name: Package ${{ github.event.inputs.tag }}-${{ github.event.inputs.core }} RAG Engine
     runs-on: ubuntu-latest
+    if: ${{ ((github.ref == 'refs/heads/main') || startsWith(github.ref, 'refs/tags/')) && github.event.inputs.core == 'cpu' }}
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.inputs.dockerfile_branch }}
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_PASSWORD }}
 
       - name: Build and push RAG docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          file: ./nesis/rag/Dockerfile
+          tags: ametnes/nesis:${{ github.event.inputs.tag }}-rag, ametnes/nesis:latest-rag
+          build-args: |
+            NESIS_VERSION=${{ github.event.inputs.tag }}
+
+  package_rag_cuda:
+    name: Package ${{ github.event.inputs.tag }}-${{ github.event.inputs.core }} RAG Engine
+    runs-on: ubuntu-latest
+    if: ${{ ((github.ref == 'refs/heads/main') || startsWith(github.ref, 'refs/tags/')) && github.event.inputs.core == 'cuda' }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.dockerfile_branch }}
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: Build and push core tagged RAG docker image
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
           file: ./nesis/rag/Dockerfile
-          tags: ametnes/nesis:${{ github.event.inputs.tag }}-rag
+          tags: ametnes/nesis:${{ github.event.inputs.tag }}-rag-${{ github.event.inputs.core }}, ametnes/nesis:latest-rag-${{ github.event.inputs.core }}
+          build-args: |
+            NESIS_VERSION=${{ github.event.inputs.tag }}
+            CORE=${{ github.event.inputs.core }}

.github/workflows/test_api.yml

@@ -5,12 +5,12 @@ on:
     paths:
       - "nesis/api/core/**"
       - "nesis/api/tests/**"
-      - "nesis/api/core/requirements*"
+      - "nesis/api/requirements*"
   pull_request:
     paths:
       - "nesis/api/core/**"
       - "nesis/api/tests/**"
-      - "nesis/api/core/requirements*"
+      - "nesis/api/requirements*"
 
 jobs:
   linter:
@@ -47,11 +47,11 @@ jobs:
           - 11211:11211
     steps:
       - name: Check out source
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 

.github/workflows/test_rag.yml

@@ -5,12 +5,14 @@ on:
     paths:
       - "nesis/rag/core/**"
       - "nesis/rag/tests/**"
-      - "nesis/rag/core/requirements*"
+      - "nesis/rag/requirements*"
+      - "nesis/rag/settings.yaml"
   pull_request:
     paths:
       - "nesis/rag/core/**"
       - "nesis/rag/tests/**"
-      - "nesis/rag/core/requirements*"
+      - "nesis/rag/requirements*"
+      - "nesis/rag/settings.yaml"
 
 jobs:
   linter:
@@ -21,12 +23,14 @@ jobs:
       - uses: psf/black@stable
         with:
           options: "--check --verbose"
-          src: "./nesis/api/"
+          src: "./nesis/rag/"
           version: "24.3.0"
 
   test:
     runs-on: ubuntu-latest
     name: Test RAG API
+    env:
+      HF_TOKEN: ${{ secrets.HF_TOKEN }}
     services:
       postgres:
         image: ametnes/postgresql:16-debian-12
@@ -48,17 +52,19 @@ jobs:
           - 11211:11211
     steps:
       - name: Check out source
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
+          sudo apt update -y
+          sudo apt install ffmpeg tesseract-ocr poppler-utils -y
           pip install -r nesis/rag/requirements.txt -r nesis/rag/requirements-test.txt -r nesis/rag/requirements-huggingface.txt
       - name: Run unit tests
         env:

.gitignore

@@ -143,4 +143,6 @@ dist
 *.pyc
 videos
 screenshots
-*.mp4
+#*.mp4
+
+site