Merge branch 'main' into feature/damian/mpt

.github/workflows/build-nightly.yml

@@ -0,0 +1,22 @@
+name: build-nightly
+run-name: ${{ github.workflow }} is to create nightly wheel file for pypi
+on:
+  push:
+    branches:
+      - 'main'
+  schedule:
+    - cron: '30 0 * * *'
+  workflow_dispatch:
+
+
+jobs:
+
+    BUILD-SPARSEML-NIGHTLY:
+
+      uses: ./.github/workflows/util.yml
+      with:
+        runs_on: ubuntu-22.04
+        run_id: ${{ github.run_id }}
+        build_type: nightly
+        testmo_project_id: 9
+      secrets: inherit

.github/workflows/build-release.yml

@@ -0,0 +1,19 @@
+name: build-release
+run-name: ${{ github.workflow }} is to create release wheel file for pypi
+on:
+  push:
+    branches:
+      - 'release/[0-9]+.[0-9]+'
+    workflow_dispatch:
+
+jobs:
+
+    BUILD-SPARSEML-RELEASE:
+
+      uses: ./.github/workflows/util.yml
+      with:
+        runs_on: ubuntu-22.04
+        run_id: ${{ github.run_id }}
+        build_type: release
+        testmo_project_id: 9
+      secrets: inherit

.github/workflows/linkcheck.yml

@@ -0,0 +1,22 @@
+name: Check Markdown links
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  markdown-link-check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: gaurav-nelson/github-action-markdown-link-check@v1
+      with:
+        use-quiet-mode: 'yes'
+        config-file: '.github/workflows/mlc_config.json'

.github/workflows/mlc-config.json

@@ -0,0 +1,13 @@
+{
+    "ignorePatterns": [
+        {
+            "pattern": ".*localhost.*"
+        },
+        {
+            "pattern": ".*127\\.0\\.0\\.1.*"
+        },
+        {
+            "pattern": ".*0\\.0\\.0\\.0.*"
+        }
+    ]
+}

.github/workflows/util.yml

@@ -0,0 +1,129 @@
+name: report-to-testmo
+on:
+  workflow_call:
+    inputs:
+      runs_on:
+        description: "runner label specifying instance running the job"
+        type: string
+        required: true 
+      run_id:
+        description: "run id provided by GHA"
+        required: true
+        type: string
+      build_type:
+        description: "build type: nightly or release"
+        type: string
+        required: true
+      testmo_project_id:
+        description: "testmo project id"
+        type: string
+        required: true
+
+jobs:
+
+  BUILD:
+    runs-on: ${{ inputs.runs_on }}
+    outputs:
+      status: ${{ steps.build.outputs.status }}
+      commitid: ${{ steps.build.outputs.commitid }}
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+
+      - name: repo checkout
+        uses: actions/checkout@v3
+
+      - name: s3
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.AWS_WEBIDENTITY_FOR_GITHUB_ACTIONS }}
+          aws-region: us-east-1
+
+      - name: build
+        id: build
+        run: |
+             pwd
+             sudo apt-get -y install python3-pip
+             pip3 --version
+             sudo pip3 install virtualenv
+             virtualenv venv
+             source venv/bin/activate
+             pip install -e .
+             if [[ "${{ inputs.build_type }}" = release ]]; then
+                 sed -i 's/is_release = False/is_release = True/g' src/${{ github.event.repository.name }}/version.py
+             fi
+             status=$(make -B build || echo 'FAILED')
+             deactivate
+             echo "=========== Build log ==========="
+             echo "${status}"
+             echo "commitid=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
+             echo "=========== Build status ==========="
+             if [[ "${status}" = "FAILED" ]]; then
+                 echo "${{ github.event.repository.name }} build failed"
+                 echo "status=failed" >> "$GITHUB_OUTPUT"
+                 exit 1
+             else
+                 echo "${{ github.event.repository.name }} build success"
+             fi
+             echo "=========== Generated build ==========="
+             ls dist/
+             echo "=========== Copy build to S3 ==========="
+             aws s3 cp dist/*.whl s3://nm-github-actions/${{ github.event.repository.name }}/
+             if [ $? -eq 0 ]; then
+                 echo "ok: copied to s3://nm-github-actions/${{ github.event.repository.name }}/"
+                 echo "status=success" >> "$GITHUB_OUTPUT"
+             else
+                 echo "failed: copied to s3://nm-github-actions/${{ github.event.repository.name }}/"
+                 echo "status=failed" >> "$GITHUB_OUTPUT"
+                 exit 1
+             fi
+             oldDate=`date --date='-2 month' +%Y%m%d`
+             oldWhl=`(aws s3 ls s3://nm-github-actions/${{ github.event.repository.name }}/ | grep nightly | grep "${oldDate}") || echo "notfound"`
+             if [[ "${oldWhl}" != 'notfound' ]]; then
+                 for oldwhl in $(echo "${oldWhl}" | awk '{print $4}')
+                 do
+                     echo "Remove old build ${oldwhl} in S3"
+                     aws s3 rm s3://nm-github-actions/${{ github.event.repository.name }}/${oldwhl}
+                 done
+             fi
+
+  TESTMO:
+    if: success() || failure()
+    needs: BUILD
+    runs-on: ${{ inputs.runs_on }}
+    steps:
+
+      - id: report
+        run: |
+          echo "node: $(node -v)"
+          echo "npm: $(npm -v)"
+          echo "Installing testmo cli..."
+          sudo npm install -g @testmo/testmo-cli    
+          export TESTMO_TOKEN=${{ secrets.TESTMO_TEST_TOKEN }}
+          TESTMO_URL="https://neuralmagic.testmo.net"
+          todaytime=`date +%Y%m%d`
+          name="${{ github.event.repository.name }} ${{ inputs.build_type }} ${todaytime} ${{ needs.BUILD.outputs.commitid }} RunID:${{ inputs.run_id }}"
+          echo "========== Build info ==========="
+          echo "name: ${name}"
+          echo "build status: ${{ needs.BUILD.outputs.status }}"
+          exit_code=1
+          if [[ "${{ needs.BUILD.outputs.status }}" = "success" ]]; then
+              exit_code=0
+          fi
+          echo "<status>${{ needs.BUILD.outputs.status }}</status>" > result.xml
+          echo "echo \"GHA job ${{ needs.BUILD.outputs.status }}: https://github.com/neuralmagic/${{ github.event.repository.name }}/actions/runs/${{ inputs.run_id }}\"; exit ${exit_code}" > result.sh
+          echo "========== Report to testmo ==========="
+          echo "testmo automation:run:submit \\"
+          echo "  --instance ${TESTMO_URL} \\"
+          echo "  --project-id ${{ inputs.testmo_project_id }} \\"
+          echo "  --name ${name} \\"
+          echo "  --source ${{ github.event.repository.name }} \\"
+          echo "  --results result.xml"
+          testmo automation:run:submit \
+                         --instance "${TESTMO_URL}" \
+                         --project-id ${{ inputs.testmo_project_id }} \
+                         --name "${name}" \
+                         --source "${{ github.event.repository.name }}" \
+                         --results result.xml \
+                         -- bash result.sh

README.md

@@ -164,7 +164,7 @@ sparseml.yolov5.train \
   --hyp hyps/hyp.finetune.yaml --cfg yolov5s.yaml --patience 0
 ```
 
-- Check out the [YOLOv5 CLI example](ultralytics-yolov5/tutorials/sparse-transfer-learning.md) for more details on the YOLOv5 training pipeline
+- Check out the [YOLOv5 CLI example](integrations/ultralytics-yolov5/tutorials/sparse-transfer-learning.md) for more details on the YOLOv5 training pipeline
 - Check out the [Hugging Face CLI example](integrations/huggingface-transformers/tutorials/sparse-transfer-learning-bert.md) for more details on the available NLP training pipelines
 - Check out the [Torchvision CLI example](integrations/torchvision/tutorials/sparse-transfer-learning.md) for more details on the image classification training pipelines
 

integrations/clip/README.md

@@ -16,11 +16,21 @@ limitations under the License.
 
 # CLIP Export Examples
 
-The examples in `clip_onnx_export.py` provide the steps needed to export a CLIP model using sparseml's onnx exporting functionality. The models and pretrained weights are pulled in from [OpenClip](https://github.com/mlfoundations/open_clip/tree/main) and the command line tools provided allow exporting of a given model's Text and Visual branches. See the OpenClip repository for a full list of available models. For the CoCa models available in OpenClip, an additional text-decoder is also exported.
+The examples in `clip_onnx_export.py` provide the steps needed to export a CLIP model using sparseml's onnx exporting functionality. The models and pretrained weights are pulled in from [OpenClip](https://github.com/mlfoundations/open_clip/tree/main) and the command line tools provided allow exporting of a given model's Text and Visual branches. See the OpenClip repository for a full list of available models. For the CoCa/Caption models available in OpenClip, an additional text-decoder is also exported.
 
 ## Installation
 
-The examples provided require torch nighly and `open_clip_torch==2.20.0` to be installed. To work within the `sparseml` environment, be sure to set the environment variable `MAX_TORCH` to your installed version when 
+The examples provided require `open_clip_torch==2.20.0` to be installed along with **torch nighly**. To work within the `sparseml` environment, be sure to set the environment variable `MAX_TORCH` to your installed version when 
 installing torch nightly.
 
-Example: `MAX_TORCH="2.1.0.dev20230613+cpu"`
+Steps:
+- Install `sparseml[clip]`. This will ensure open_clip_torch is installed
+- Uninstall torch by running:  
+```
+   pip uninstall torch
+```
+- Install torch nightly:  
+```   
+   pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/
+```
+- Set your environment variable to the correct torch version: Example: `export MAX_TORCH="2.1.0.dev20230613+cpu"`

integrations/clip/clip_models.py

@@ -16,18 +16,6 @@
 import torch.nn as nn
 
 
-class VisualModel(nn.Module):
-    def __init__(self, visual_model: torch.nn.Module, output_tokens: bool):
-
-        super().__init__()
-
-        self.visual_model = visual_model
-        self.visual_model.output_tokens = output_tokens
-
-    def forward(self, x):
-        return self.visual_model(x)
-
-
 class TextModel(nn.Module):
     def __init__(
         self,

integrations/clip/clip_onnx_export.py

@@ -28,7 +28,7 @@
 import torch
 
 import open_clip
-from clip_models import TextModel, VisualModel
+from clip_models import TextModel
 from sparseml.pytorch.utils import export_onnx
 
 
@@ -61,12 +61,9 @@ def _export_visual(
     **export_kwargs,
 ):
     module_name = "clip_visual.onnx"
-    visual_model = VisualModel(
-        visual_model=model.visual,
-        output_tokens=is_coca,
-    )
+    visual_model = model.visual
 
-    image_shape = visual_model.visual_model.image_size[0]
+    image_shape = visual_model.image_size[0]
     sample_input = torch.randn(1, 3, image_shape, image_shape, requires_grad=True)
 
     visual_model = visual_model.to(device)
@@ -107,10 +104,11 @@ def _export_text(
 
     text_model = text_model.to(device)
     text_model.eval()
-    sample_batch = tokenizer(["a dog"])
 
     if is_coca:
-        sample_batch = sample_batch[:, :-1]
+        sample_batch = torch.ones(6, 15, dtype=torch.long)
+    else:
+        sample_batch = tokenizer(["a dog"]).to(torch.int32)
 
     _export_onnx(
         module=text_model,
@@ -130,9 +128,7 @@ def _export_text_decoder(
 
     sample_batch = OrderedDict()
     sample_batch["image_embs"] = torch.randn(1, 255, model.text.output_dim)
-    sample_batch["text_embs"] = torch.randn(
-        1, model.text.context_length, model.text.output_dim
-    )
+    sample_batch["text_embs"] = torch.randn(1, 15, model.text.output_dim)
 
     _export_onnx(
         module=decoder,
@@ -175,13 +171,13 @@ def main():
     parser.add_argument(
         "--model",
         type=str,
-        default="ViT-B-32",
+        default="coca_ViT-B-32",
         help="Name of CLIP model. See OpenClip docs for a list of available models",
     )
     parser.add_argument(
         "--pretrained",
         type=str,
-        default="laion2b_s34b_b79k",
+        default="mscoco_finetuned_laion2b_s13b_b90k",
         help="Name of the pretraining to use. See OpenClip docs for a list of options.",
     )
     parser.add_argument(

integrations/huggingface-transformers/tutorials/sparse-transfer-learning-bert-python.md

@@ -302,13 +302,13 @@ The command creates a `./deployment` folder in your local directory, which conta
 
 Take a look at the tutorials for more examples in other use cases:
 
-- [Sparse Transfer with GLUE Datasets (SST2) for sentiment analysis](tutorials/sentiment-analysis/docs-sentiment-analysis-python-sst2.ipynb)
-- [Sparse Transfer with Custom Datasets (RottenTomatoes) and Custom Teacher from HF Hub for sentiment analysis](tutorials/sentiment-analysis/docs-sentiment-analysis-python-custom-teacher-rottentomatoes)
-- [Sparse Transfer with GLUE Datasets (QQP) for multi-input text classification](tutorials/text-classification/docs-text-classification-python-qqp.ipynb)
-- [Sparse Transfer with Custom Datasets (SICK) for multi-input text classification](tutorials/text-classification/docs-text-classification-python-sick.ipynb)
-- [Sparse Transfer with Custom Datasets (TweetEval) and Custom Teacher for single input text classificaiton](tutorials/text-classification/docs-text-classification-python-custom-teacher-tweeteval.ipynb)
-- [Sparse Transfer with Custom Datasets (GoEmotions) for multi-label text classification](tutorials/text-classification/docs-text-classification-python-multi-label-go_emotions.ipynb)
-- [Sparse Transfer with Conll2003 for named entity recognition](tutorials/token-classification/docs-token-classification-python-conll2003.ipynb)
-- [Sparse Transfer with Custom Datasets (WNUT) and Custom Teacher for named entity recognition](tutorials/token-classification/docs-token-classification-custom-teacher-wnut.ipynb)
+- [Sparse Transfer with GLUE Datasets (SST2) for sentiment analysis](sentiment-analysis/docs-sentiment-analysis-python-sst2.ipynb)
+- [Sparse Transfer with Custom Datasets (RottenTomatoes) and Custom Teacher from HF Hub for sentiment analysis](sentiment-analysis/docs-sentiment-analysis-python-custom-teacher-rottentomatoes.ipynb)
+- [Sparse Transfer with GLUE Datasets (QQP) for multi-input text classification](text-classification/docs-text-classification-python-qqp.ipynb)
+- [Sparse Transfer with Custom Datasets (SICK) for multi-input text classification](text-classification/docs-text-classification-python-sick.ipynb)
+- [Sparse Transfer with Custom Datasets (TweetEval) and Custom Teacher for single input text classification](text-classification/docs-text-classification-python-custom-teacher-tweeteval.ipynb)
+- [Sparse Transfer with Custom Datasets (GoEmotions) for multi-label text classification](text-classification/docs-text-classification-python-multi-label-go_emotions.ipynb)
+- [Sparse Transfer with Conll2003 for named entity recognition](token-classification/docs-token-classification-python-conll2003.ipynb)
+- [Sparse Transfer with Custom Datasets (WNUT) and Custom Teacher for named entity recognition](token-classification/docs-token-classification-python-custom-teacher-wnut.ipynb)
 - Sparse Transfer with SQuAD (example coming soon!)
 - Sparse Transfer with Squadshifts Amazon (example coming soon!)

integrations/huggingface-transformers/tutorials/token-classification/token-classification-cli.md

@@ -175,7 +175,7 @@ A `deployment` folder is created in your local directory, which has all of the f
 
 ## Sparse Transfer Learning with a Custom Dataset (WNUT_17)
 
-Beyond the Conll2003 dataset, we can also use a dataset from the Hugging Face Hub or from local files. Let's try an example of each for the sentiment analysis using [WNUT 17](wnut_17), which is also a NER task.
+Beyond the Conll2003 dataset, we can also use a dataset from the Hugging Face Hub or from local files. Let's try an example of each for the sentiment analysis using WNUT_17, which is also a NER task.
 
 For simplicity, we will perform the fine-tuning without distillation. Although the transfer learning recipe contains distillation
 modifiers, by setting `--distill_teacher disable` we instruct SparseML to skip distillation.

integrations/old-examples/dbolya-yolact/README.md

@@ -106,7 +106,7 @@ The following table lays out the root-level files and folders along with a descr
 | Folder/File Name                                | Description                                                                                                           |
 |-------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------|
 | [recipes](./recipes)                            | Typical recipes for sparsifying YOLACT models along with any downloaded recipes from the SparseZoo.                   |
-| [yolact](./yolact)                              | Integration repository folder used to train and sparsify YOLACT models (`setup_integration.sh` must run first).       |
+| yolact                                          | Integration repository folder used to train and sparsify YOLACT models (`setup_integration.sh` must run first).       |
 | [README.md](./README.md)                        | Readme file.                                                                                                          |
 | [tutorials](./tutorials)                        | Easy to follow sparsification tutorials for YOLACT  models.                                                            |
 

integrations/old-examples/keras/README.md

@@ -31,7 +31,7 @@ The techniques include, but are not limited to:
 
 ## Tutorials
 
-- [Classification](https://github.com/neuralmagic/sparseml/blob/main/integrations/keras/notebooks/classification.ipynb)
+- [Classification](https://github.com/neuralmagic/sparseml/blob/main/integrations/old-examples/keras/notebooks/classification.ipynb)
 
 ## Installation
 

integrations/old-examples/pytorch/tutorials/classification_sparse_transfer_learning_tutorial.md

@@ -74,7 +74,7 @@ Note: The models above were originally trained and sparsified on the [ImageNet](
 
 - After noting respective SparseZoo model stub, [train.py](../train.py) script can be used to download checkpoint and [Imagenette](https://github.com/fastai/imagenette) and kick-start transfer learning. 
    The transfer learning process itself is guided using recipes; We include example [recipes](../recipes) for classification along with others in the SparseML [GitHub repository](https://github.com/neuralmagic/sparseml). 
-[Learn more about recipes and modifiers](../../../docs/source/recipes.md).
+[Learn more about recipes and modifiers](https://github.com/neuralmagic/sparseml/tree/main/docs/source/recipes.md).
 
 - Run the following example command to kick off transfer learning for [ResNet-50](https://arxiv.org/abs/1512.03385) starting from a moderately pruned checkpoint from [SparseZoo](https://sparsezoo.neuralmagic.com/):
 ```