Update licence pyproject.toml #339
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Fly Deploy | |
on: | |
push: | |
branches: | |
- main | |
env: | |
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} | |
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} | |
HOST: ${{ secrets.HOST }} | |
LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }} | |
LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }} | |
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} | |
POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }} | |
NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }} | |
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }} | |
PORT: ${{ secrets.PORT }} | |
REDIS_URL: ${{ secrets.REDIS_URL }} | |
SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }} | |
SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }} | |
SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }} | |
SUPABASE_URL: ${{ secrets.SUPABASE_URL }} | |
TEST_API_KEY: ${{ secrets.TEST_API_KEY }} | |
PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
NPM_TOKEN: ${{ secrets.NPM_TOKEN }} | |
jobs: | |
pre-deploy-e2e-tests: | |
name: Pre-deploy checks | |
runs-on: ubuntu-latest | |
services: | |
redis: | |
image: redis | |
ports: | |
- 6379:6379 | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Node.js | |
uses: actions/setup-node@v3 | |
with: | |
node-version: "20" | |
- name: Install pnpm | |
run: npm install -g pnpm | |
- name: Install dependencies | |
run: pnpm install | |
working-directory: ./apps/api | |
- name: Start the application | |
run: npm start & | |
working-directory: ./apps/api | |
id: start_app | |
- name: Start workers | |
run: npm run workers & | |
working-directory: ./apps/api | |
id: start_workers | |
- name: Run E2E tests | |
run: | | |
npm run test:prod | |
working-directory: ./apps/api | |
pre-deploy-test-suite: | |
name: Test Suite | |
needs: pre-deploy-e2e-tests | |
runs-on: ubuntu-latest | |
services: | |
redis: | |
image: redis | |
ports: | |
- 6379:6379 | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Node.js | |
uses: actions/setup-node@v3 | |
with: | |
node-version: "20" | |
- name: Install pnpm | |
run: npm install -g pnpm | |
- name: Install dependencies | |
run: pnpm install | |
working-directory: ./apps/api | |
- name: Start the application | |
run: npm start & | |
working-directory: ./apps/api | |
id: start_app | |
- name: Start workers | |
run: npm run workers & | |
working-directory: ./apps/api | |
id: start_workers | |
- name: Install dependencies | |
run: pnpm install | |
working-directory: ./apps/test-suite | |
- name: Run E2E tests | |
run: | | |
npm run test:suite | |
working-directory: ./apps/test-suite | |
python-sdk-tests: | |
name: Python SDK Tests | |
needs: pre-deploy-e2e-tests | |
runs-on: ubuntu-latest | |
services: | |
redis: | |
image: redis | |
ports: | |
- 6379:6379 | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Python | |
uses: actions/setup-python@v4 | |
with: | |
python-version: '3.x' | |
- name: Install pnpm | |
run: npm install -g pnpm | |
- name: Install dependencies | |
run: pnpm install | |
working-directory: ./apps/api | |
- name: Start the application | |
run: npm start & | |
working-directory: ./apps/api | |
id: start_app | |
- name: Start workers | |
run: npm run workers & | |
working-directory: ./apps/api | |
id: start_workers | |
- name: Install Python dependencies | |
run: | | |
python -m pip install --upgrade pip | |
pip install -r requirements.txt | |
working-directory: ./apps/python-sdk | |
- name: Run E2E tests for Python SDK | |
run: | | |
pytest firecrawl/__tests__/e2e_withAuth/test.py | |
working-directory: ./apps/python-sdk | |
js-sdk-tests: | |
name: JavaScript SDK Tests | |
needs: pre-deploy-e2e-tests | |
runs-on: ubuntu-latest | |
services: | |
redis: | |
image: redis | |
ports: | |
- 6379:6379 | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Node.js | |
uses: actions/setup-node@v3 | |
with: | |
node-version: "20" | |
- name: Install pnpm | |
run: npm install -g pnpm | |
- name: Install dependencies | |
run: pnpm install | |
working-directory: ./apps/api | |
- name: Start the application | |
run: npm start & | |
working-directory: ./apps/api | |
id: start_app | |
- name: Start workers | |
run: npm run workers & | |
working-directory: ./apps/api | |
id: start_workers | |
- name: Install dependencies for JavaScript SDK | |
run: pnpm install | |
working-directory: ./apps/js-sdk/firecrawl | |
- name: Run E2E tests for JavaScript SDK | |
run: npm run test | |
working-directory: ./apps/js-sdk/firecrawl | |
deploy: | |
name: Deploy app | |
runs-on: ubuntu-latest | |
needs: [pre-deploy-test-suite, python-sdk-tests, js-sdk-tests] | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Change directory | |
run: cd apps/api | |
- uses: superfly/flyctl-actions/setup-flyctl@master | |
- run: flyctl deploy ./apps/api --remote-only -a firecrawl-scraper-js | |
env: | |
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} | |
build-and-publish-python-sdk: | |
name: Build and publish Python SDK | |
runs-on: ubuntu-latest | |
needs: deploy | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v3 | |
- name: Set up Python | |
uses: actions/setup-python@v4 | |
with: | |
python-version: '3.x' | |
- name: Install dependencies | |
run: | | |
python -m pip install --upgrade pip | |
pip install setuptools wheel twine build requests packaging | |
- name: Run version check script | |
id: version_check_script | |
run: | | |
PYTHON_SDK_VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py) | |
echo "PYTHON_SDK_VERSION_INCREMENTED=$PYTHON_SDK_VERSION_INCREMENTED" >> $GITHUB_ENV | |
- name: Build the package | |
if: ${{ env.PYTHON_SDK_VERSION_INCREMENTED == 'true' }} | |
run: | | |
python -m build | |
working-directory: ./apps/python-sdk | |
- name: Publish to PyPI | |
if: ${{ env.PYTHON_SDK_VERSION_INCREMENTED == 'true' }} | |
env: | |
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
run: | | |
twine upload dist/* | |
working-directory: ./apps/python-sdk | |
build-and-publish-js-sdk: | |
name: Build and publish JavaScript SDK | |
runs-on: ubuntu-latest | |
needs: deploy | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Node.js | |
uses: actions/setup-node@v3 | |
with: | |
node-version: '20' | |
registry-url: 'https://registry.npmjs.org/' | |
scope: '@mendable' | |
always-auth: true | |
- name: Install pnpm | |
run: npm install -g pnpm | |
- name: Install python for running version check script | |
run: | | |
python -m pip install --upgrade pip | |
pip install setuptools wheel requests packaging | |
- name: Install dependencies for JavaScript SDK | |
run: pnpm install | |
working-directory: ./apps/js-sdk/firecrawl | |
- name: Run version check script | |
id: version_check_script | |
run: | | |
VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py js ./apps/js-sdk/firecrawl @mendable/firecrawl-js) | |
echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV | |
- name: Build and publish to npm | |
if: ${{ env.VERSION_INCREMENTED == 'true' }} | |
env: | |
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} | |
run: | | |
npm run build-and-publish | |
working-directory: ./apps/js-sdk/firecrawl | |