Merge branch 'main' of https://github.com/5afe/spellbook

# Conflicts:
#	models/safe/zkevm/safe_zkevm_matic_transfers.sql
#	models/safe/zkevm/safe_zkevm_safes.sql
#	models/safe/zkevm/safe_zkevm_schema.yml
#	models/safe/zkevm/safe_zkevm_singletons.sql

.github/pull_request_template.md

@@ -0,0 +1,52 @@
+## Thank you for contributing to Spellbook 🪄
+
+### Contribution type
+Please check the type of contribution this pull request is for:
+
+- [ ] New spell(s)
+- [ ] Adding to existing spell lineage
+- [ ] Bug fix
+
+**Note:** You can safely discard any section below which doesn't apply based on selection above
+
+---
+
+### For new spell(s)
+If you are building new spell(s), please provide the following information:
+
+- **Spell name(s):** *schema.table_name*
+- **Description:** [Detailed description of the new spell(s) and their purpose]
+- **Who are the new spell(s) for?** [Internal team or general community]
+- **How will the new spell(s) be used downstream?** [Description of downstream usage]
+- **Implementation details:** [Information on how the spell(s) are implemented]
+- **Test instructions:** [How to test the new spell(s)]
+- **Related issue(s):** [Link to related issues, if any]
+
+---
+
+### For adding to existing spell lineage
+If you are adding to an existing spell lineage, please provide the following information:
+
+- **Description:** [Description of the changes made]
+
+---
+
+### For bug fixes
+If you are fixing a bug, please provide the following information:
+
+- **Description:** [Description of the bug fix]
+- **Steps to reproduce:** [How to reproduce the bug]
+- **Implementation details:** [Information on how the bug was fixed]
+- **Test instructions:** [How to test the fix]
+- **Related issue(s):** [Link to related issues, if any]
+
+---
+
+### Additional information
+Please provide any additional information that might help us review your pull request:
+
+- [Any additional information]
+
+---
+
+Thank you for your contribution!

.github/workflows/commit_manifest.yml

@@ -9,13 +9,13 @@ on:
 concurrency:
   group: ${{ github.workflow }}
   cancel-in-progress: true
-      
+
 jobs:
   commit_manifest:
     runs-on: [ self-hosted, linux, spellbook-trino-ci ]
     strategy:
       matrix:
-        project: [ 'tokens', 'spellbook' ]
+        project: [ 'tokens', 'spellbook', 'daily_spellbook', 'nft', 'dex', 'solana']
       max-parallel: 1
 
     steps:
@@ -27,26 +27,25 @@ jobs:
 
     - name: Add git_sha to schema
       run: "/runner/change_schema.sh wizard"
-    
+
     - name: Setup variables
       run: |
         echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV
         echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
-        if [[ "${{ matrix.project }}" == "tokens" ]]; then
-          echo "PROJECT_DIR=tokens" >> $GITHUB_ENV
-          echo "S3_LOCATION=s3://manifest-spellbook-dunesql/tokens" >> $GITHUB_ENV
-        elif [[ "${{ matrix.project }}" == "spellbook" ]]; then
-          echo "PROJECT_DIR=." >> $GITHUB_ENV
-          echo "S3_LOCATION=s3://manifest-spellbook-dunesql/spellbook" >> $GITHUB_ENV
-        fi
+        echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{matrix.project}}" >> $GITHUB_ENV
+          PROJECT_DIR=${{ matrix.project }}
+          if [[ "$PROJECT_DIR" == "spellbook" ]]; then
+            PROJECT_DIR="."
+          fi
+          echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
 
     - name: dbt dependencies
       working-directory: ${{env.PROJECT_DIR}}
       run: "dbt deps"
-    
+
     - name: Run dbt debug on Trino if dunesql engine is used, and retry until cluster is available
       run: "./scripts/ensure_cluster.sh"
-          
+
     - name: dbt compile to create prod manifest from main
       run: "dbt compile --target-path . $PROFILE --project-dir ${PROJECT_DIR}"
 

.github/workflows/daily_spellbook.yml

@@ -0,0 +1,20 @@
+name: DBT Daily Spellbook run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - daily_spellbook/**
+      - .github/workflows/daily_spellbook.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'daily_spellbook'
+

.github/workflows/dbt_run.yml

@@ -0,0 +1,69 @@
+name: DBT Run Reusable Workflow
+
+on:
+  workflow_call:
+    inputs:
+      project:
+        description: 'DBT run for project'
+        required: true
+        type: string
+
+jobs:
+  dbt-test:
+    runs-on: [ self-hosted, linux, spellbook-trino-ci ]
+    timeout-minutes: 90
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Setup variables
+        run: | # Spellbook is a special case because it's not a subdirectory
+          echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV
+          echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
+          echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{inputs.project}}" >> $GITHUB_ENV
+          PROJECT_DIR=${{ inputs.project }}
+          if [[ "$PROJECT_DIR" == "spellbook" ]]; then
+            PROJECT_DIR="."
+          fi
+          echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
+      - name: Add git_sha to schema
+        run: "/runner/change_schema.sh git_dunesql_$GIT_SHA"
+
+      - name: Get latest manifest
+        run: "aws s3 cp $S3_LOCATION/manifest.json $PROJECT_DIR/manifest.json"
+        continue-on-error: true # Temprorary until all projects have manifest
+
+      - name: dbt dependencies
+        working-directory: ${{env.PROJECT_DIR}}
+        run: "dbt deps"
+
+      - name: Activate DuneSQL Cluster
+        run: "./scripts/ensure_cluster.sh"
+
+      - name: dbt compile to create manifest to compare to
+        run: "dbt --warn-error compile --project-dir ${PROJECT_DIR}"
+
+      - name: check schemas
+        run: "./scripts/check_schema.sh"
+
+      - name: dbt seed
+        run: "dbt seed $PROFILE --select @state:modified --exclude tag:prod_exclude tag:remove --state . --project-dir ${PROJECT_DIR}"
+
+      - name: dbt run initial model(s)
+        run: "dbt -x run $PROFILE --select state:modified --exclude tag:prod_exclude tag:remove --defer --state . --project-dir ${PROJECT_DIR}"
+
+      - name: dbt test initial model(s)
+        run: "dbt test $PROFILE --select state:new state:modified --exclude tag:prod_exclude tag:remove --defer --state . --project-dir ${PROJECT_DIR}"
+
+      - name: Set environment variable for incremental model count
+        run: |
+          echo "INC_MODEL_COUNT=$(echo dbt ls $PROFILE --select state:modified,config.materialized:incremental --state . --resource-type model  --project-dir ${PROJECT_DIR} | wc -l)" >> $GITHUB_ENV
+
+      - name: dbt run incremental model(s) if applicable
+        if: env.INC_MODEL_COUNT > 0
+        run: "dbt run $PROFILE --select state:modified,config.materialized:incremental --exclude tag:prod_exclude tag:remove --defer --state . --project-dir ${PROJECT_DIR}"
+
+      - name: dbt test incremental model(s) if applicable
+        if: env.INC_MODEL_COUNT > 0
+        run: "dbt test $PROFILE --select state:modified,config.materialized:incremental --exclude tag:prod_exclude tag:remove --defer --state . --project-dir ${PROJECT_DIR}"

.github/workflows/dex.yml

@@ -0,0 +1,19 @@
+name: DBT DEX sector run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - dex/**
+      - .github/workflows/dex.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'dex'

.github/workflows/nft.yml

@@ -0,0 +1,20 @@
+name: DBT NFT run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - nft/**
+      - .github/workflows/nft.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'nft'
+

.github/workflows/solana.yml

@@ -0,0 +1,19 @@
+name: DBT Solana sector run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - solana/**
+      - .github/workflows/solana.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'solana'

.github/workflows/spellbook.yml

@@ -0,0 +1,24 @@
+name: DBT Spellbook run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths-ignore:
+      - 'scripts/**'
+      - 'Pipfile'
+      - '.gitignore'
+      - 'tokens/**'
+      - 'daily_spellbook/**'
+      - 'nft/**'
+      - 'solana/**'
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'spellbook'
+

.github/workflows/tokens.yml

@@ -0,0 +1,20 @@
+name: DBT Tokens run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - tokens/**
+      - .github/workflows/tokens.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'tokens'
+

daily_spellbook/README.md

@@ -0,0 +1,3 @@
+## Daily Spellbook
+
+This is a DBT sub project for the the main models of Spellbook that runs on a daily candence.