Faster test, pre-commit formatting, general cleanup (#129)

* add pre commit hooks

* add reqs

* fix configs

* cleanup

* install from reqs file

Former-commit-id: b1cb537

.github/workflows/pre-commit.yml

@@ -0,0 +1,27 @@
+name: Run pre-commit
+
+on:
+  pull_request:
+    branches: [ main ]
+  push:
+    branches: [ main ]
+
+jobs:
+  lint:
+    name: Lint PR or Push to main
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.9]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Run Lint
+        uses: pre-commit/action@v2.0.0

.github/workflows/test.yml

@@ -20,15 +20,8 @@ jobs:
           python-version: '3.9'
       - name: Install python deps
         run: |
-          pip install tensorflow==2.9 setGPU \
-            sklearn matplotlib boost_histogram mplhep pandas scipy uproot \
-            awkward vector pyarrow fastjet keras-tuner networkx \
-            tensorflow-probability tensorflow-addons \
-            tqdm click tensorflow-datasets 'ray[default]'==1.6.0 'ray[tune]==1.6.0' \
-            tf-models-official tensorflow-text \
-            tf2onnx onnxruntime zenodo_get seaborn scikit-optimize nevergrad \
-            tensorflow-estimator keras \
-            notebook papermill ./hep_tfds
+          pip install -r requirements.txt
+          pip install ./hep_tfds
           HOROVOD_WITH_TENSORFLOW=1 pip install horovod[tensorflow,keras]
       - name: Run delphes TF model
         run: ./scripts/local_test_delphes_pipeline.sh
@@ -44,15 +37,8 @@ jobs:
           python-version: '3.9'
       - name: Install python deps
         run: |
-          pip install tensorflow==2.9 setGPU \
-            sklearn matplotlib boost_histogram mplhep pandas scipy uproot \
-            awkward vector pyarrow fastjet keras-tuner networkx \
-            tensorflow-probability tensorflow-addons \
-            tqdm click tensorflow-datasets 'ray[default]'==1.6.0 'ray[tune]==1.6.0' \
-            tf-models-official tensorflow-text \
-            tf2onnx onnxruntime zenodo_get seaborn scikit-optimize nevergrad \
-            tensorflow-estimator keras \
-            notebook papermill ./hep_tfds
+          pip install -r requirements.txt
+          pip install ./hep_tfds
           HOROVOD_WITH_TENSORFLOW=1 pip install horovod[tensorflow,keras]
       - name: Run CMS TF model using the pipeline
         run: ./scripts/local_test_cms_pipeline.sh

.gitignore

@@ -3,6 +3,7 @@
 *.npz
 *.pt
 *.pdf
+*.png
 data/*
 experiments/*
 prp/*
@@ -19,3 +20,6 @@ test/__pycache__/
 
 *playground.py
 nohup.out
+
+*.pkl
+*.pkl.bz2

.pre-commit-config.yaml

@@ -0,0 +1,40 @@
+default_language_version:
+    python: python3.9
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.3.0
+  hooks:
+  - id: trailing-whitespace
+  - id: check-added-large-files
+  - id: check-ast
+  - id: check-json
+  - id: check-merge-conflict
+  - id: check-xml
+  - id: check-yaml
+  - id: debug-statements
+  - id: end-of-file-fixer
+  - id: requirements-txt-fixer
+  - id: mixed-line-ending
+    args: ['--fix=no']
+
+- repo: https://github.com/PyCQA/isort
+  rev: 5.10.1
+  hooks:
+  - id: isort
+    args: ['--profile', 'black', '--filter-files']
+
+- repo: https://github.com/psf/black
+  rev: 22.6.0
+  hooks:
+  - id: black-jupyter
+    language_version: python3
+    args: [--line-length=125]
+
+- repo: https://gitlab.com/pycqa/flake8
+  rev: 3.9.2
+  hooks:
+  - id: flake8
+    # black-compatible flake-8 config
+    args: ['--max-line-length=125',  # github viewer width
+           '--extend-ignore=E203,W605']  # E203 is not PEP8 compliant