Merge branch 'master' into bugfix/early-stopping

.circleci/config.yml

@@ -10,10 +10,11 @@ references:
     run:
       name: Install Dependences
       command: |
+        sudo apt-get update && sudo apt-get install -y cmake
         pip install "$TORCH_VERSION"
         pip install -r requirements.txt -q
         sudo pip install pytest pytest-cov pytest-flake8 -q
-        pip install -r ./tests/requirements.txt -q
+        pip install -r ./tests/requirements-devel.txt -q
 
   tests: &tests
    run:
@@ -63,10 +64,13 @@ references:
      name: Make Documentation
      command: |
        # sudo apt-get install pandoc
+       sudo apt-get update && sudo apt-get install -y cmake
        pip install -r requirements.txt --user
        sudo pip install -r docs/requirements.txt
+       pip install -r requirements-extra.txt --user  # for doctesting loggers etc.
        # sphinx-apidoc -o ./docs/source ./pytorch_lightning **/test_* --force --follow-links
-       cd docs; make clean ; make html --debug --jobs 2 SPHINXOPTS="-W"
+       cd docs; make clean; make html --debug --jobs 2 SPHINXOPTS="-W"
+       make doctest; make coverage
 
 jobs:
 
@@ -76,6 +80,10 @@ jobs:
     steps:
       - checkout
       - *make_docs
+      - store_artifacts:
+          # allows us to preview the generated html pages
+          path: docs/build/html/
+          destination: html
 
   Formatting:
     docker:
@@ -140,6 +148,13 @@ jobs:
       - TORCH_VERSION: "torch>=1.4, <1.5"
     steps: *steps
 
+  PyTorch-v1_5:
+    docker:
+      - image: circleci/python:3.6
+    environment:
+      - TORCH_VERSION: "torch>=1.5, <1.6"
+    steps: *steps
+
   Examples:
     docker:
       - image: circleci/python:3.7
@@ -171,5 +186,6 @@ workflows:
       - PyTorch-v1_2
       - PyTorch-v1_3
       - PyTorch-v1_4
+      - PyTorch-v1_5
       - Install-pkg
       - Examples

.drone.yml

@@ -6,30 +6,40 @@ name: torch-GPU
 
 steps:
 - name: testing
-  image: pytorch/pytorch:1.4-cuda10.1-cudnn7-runtime
+  image: pytorchlightning/pytorch_lightning:devel-pt_1_4
 
   environment:
     SLURM_LOCALID: 0
     CODECOV_TOKEN:
       from_secret: codecov_token
+    HOROVOD_GPU_ALLREDUCE: NCCL
+    HOROVOD_GPU_BROADCAST: NCCL
+    HOROVOD_WITH_PYTORCH: 1
+    HOROVOD_WITHOUT_TENSORFLOW: 1
+    HOROVOD_WITHOUT_MXNET: 1
+    HOROVOD_WITH_GLOO: 1
+    HOROVOD_WITHOUT_MPI: 1
 
   #volumes:
   #  # Mount pip cache from host
   #  - name: pip_cache
   #    path: /opt/conda/lib/python3.7/site-packages
 
   commands:
+    - export PATH="$PATH:/root/.local/bin"
     - python --version
     - pip install pip -U
     - pip --version
     - nvidia-smi
-    - bash ./tests/install_AMP.sh
+    #- bash ./tests/install_AMP.sh
+    - apt-get update && apt-get install -y cmake
     - pip install -r requirements.txt --user -q
-    - pip install coverage pytest pytest-cov pytest-flake8 codecov -q
-    - pip install -r ./tests/requirements.txt --user -q
+    - pip install -r ./tests/requirements-devel.txt --user -q
+    #- pip install -r ./docs/requirements.txt --user -q
     - pip list
     - python -c "import torch ; print(' & '.join([torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())]) if torch.cuda.is_available() else 'only CPU')"
     - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests benchmarks -v --doctest-modules #  --flake8
+    #- cd docs; make doctest; make coverage
     - coverage report
     - codecov --token $CODECOV_TOKEN  # --pr $DRONE_PULL_REQUEST --build $DRONE_BUILD_NUMBER --branch $DRONE_BRANCH --commit $DRONE_COMMIT --tag $DRONE_TAG
     - python tests/collect_env_details.py

.github/CONTRIBUTING.md

@@ -110,13 +110,26 @@ def my_func(param_a: int, param_b: Optional[float] = None) -> str:
     return str(param_a + p)
 ```
 
+When updating the docs make sure to build them first locally and visually inspect the html files (in the browser) for 
+formatting errors. In certain cases, a missing blank line or a wrong indent can lead to a broken layout. 
+Run these commands 
+```bash
+cd docs
+pip install -r requirements.txt
+make html
+```
+and open `docs/build/html/index.html` in your browser.
+
+When you send a PR the continuous integration will run tests and build the docs. You can access a preview of the html pages in the 
+_Artifacts_ tab in CircleCI when you click on the task named _ci/circleci: Build-Docs_ at the bottom of the PR page.
+
 ### Testing
 
 Test your work locally to speed up your work since so you can focus only in particular (failing) test-cases.
  To setup a local development environment, install both local and test dependencies:
 ```bash
 pip install -r requirements.txt
-pip install -r tests/requirements.txt
+pip install -r tests/requirements-devel.txt
 ``` 
 
 You can run the full test-case in your terminal via this bash script: 
@@ -145,3 +158,29 @@ We welcome any useful contribution! For convinece here's a recommended workflow:
 4. Use tags in PR name for following cases:
     - **[blocked by #<number>]** if you work is depending on others changes
     - **[wip]** when you start to re-edit your work, mark it so no one will accidentally merge it in meantime
+
+### Question & Answer
+
+1. **How can I help/contribute?**
+
+    All help is very welcome - reporting bug, solving issues and preparing bug fixes. To solve some issues you can start with label [good first issue](https://github.com/PyTorchLightning/pytorch-lightning/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) or chose something close to your domain with label [help wanted](https://github.com/PyTorchLightning/pytorch-lightning/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22). Before you start to implement anything check that the issue description that it is clear and self-assign the task to you (if it is not possible, just comment that you take it and we assign it to you...).
+
+2. **Is there a recommendation for branch names?**
+
+    We do not rely on the name convention so far you are working with your own fork. Anyway it would be nice to follow this convention `<type>/<issue-id>_<short-name>` where the types are: `bugfix`, `feaure`, `docs`, `tests`, ...
+
+3. **How to rebase my PR?**
+
+    We recommend to create a PR in separate branch different from `master`, especially if you plan to submit several changes and do not want to wait until the fist one is resolved (we can work on them in parallel). Update your master with upstream (assuming you have already set [upstream](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/configuring-a-remote-for-a-fork))
+    ```bash
+    git fetch --all --prune
+    git checkout master
+    git merge upstream/master
+    ```
+    checkout your feature branch
+    ```bash
+    git checkout my-PR-branch
+    git rebase master
+    # follow git instructions to resolve conflists
+    git push -f
+    ```

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,7 +4,7 @@
 - [ ] Did you read the [contributor guideline](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/.github/CONTRIBUTING.md), Pull Request section?
 - [ ] Did you make sure to update the docs?   
 - [ ] Did you write any new necessary tests?  
-- [ ] If you made a notable change (that affects users), did you update the [CHANGELOG](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/.github/CHANGELOG.md)?
+- [ ] If you made a notable change (that affects users), did you update the [CHANGELOG](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/CHANGELOG.md)?
 
 <!-- For CHANGELOG separate each item in unreleased section by blank line to reduce collisions -->
 

.github/workflows/ci-testing.yml

@@ -1,6 +1,6 @@
 name: CI testing
 
-# https://help.github.com/en/actions/reference/events-that-trigger-workflows
+# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on:
   # Trigger the workflow on push or pull request,
   # but only for the master branch
@@ -19,7 +19,7 @@ jobs:
       fail-fast: false
       # max-parallel: 6
       matrix:
-        os: [ubuntu-18.04, windows-2016, macOS-10.15]
+        os: [ubuntu-18.04, windows-2019, macOS-10.15]
         python-version: [3.6, 3.7, 3.8]
         requires: ['minimal', 'latest']
         exclude:
@@ -28,7 +28,7 @@ jobs:
             requires: 'minimal'
 
     # Timeout: https://stackoverflow.com/a/59076067/4521646
-    timeout-minutes: 30
+    timeout-minutes: 15
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -41,54 +41,83 @@ jobs:
       if: runner.os == 'macOS'
       run: |
         brew install libomp  # https://github.com/pytorch/pytorch/issues/20030
+        brew install openmpi  # Horovod on macOS requires OpenMPI, Gloo not currently supported
+
+    - name: Setup Windows
+      if: runner.os == 'windows'
+      run: |
+        python -c "lines = [line for line in open('requirements-extra.txt').readlines() if not line.startswith('horovod')] ; open('requirements-extra.txt', 'w').writelines(lines)"
+
+    # TODO: remove after https://github.com/pytorch/pytorch/issues/32186 is resolved
+    - name: Setup Windows on Latest
+      if: runner.os == 'windows' && matrix.requires == 'latest'
+      run: |
+        python -c "req = open('requirements.txt').read().replace('torch>=1.1', 'torch<1.5') ; open('requirements.txt', 'w').write(req)"
 
     - name: Set min. dependencies
       if: matrix.requires == 'minimal'
       run: |
         python -c "req = open('requirements.txt').read().replace('>', '=') ; open('requirements.txt', 'w').write(req)"
         python -c "req = open('requirements-extra.txt').read().replace('>', '=') ; open('requirements-extra.txt', 'w').write(req)"
 
+    # Note: This uses an internal pip API and may not always work
+    # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
+    - name: Get pip cache
+      id: pip-cache
+      run: |
+        python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
+
     - name: Cache pip
       uses: actions/cache@v1
       with:
-        path: ~/.cache/pip # This path is specific to Ubuntu
-        # Look to see if there is a cache hit for the corresponding requirements file
-        key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-pip-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-extra.txt') }}
         restore-keys: |
-          ${{ runner.os }}-pip-${{ matrix.python-version }}-
-          ${{ runner.os }}-${{ matrix.python-version }}-
-
-    - name: Cache datasets
-      uses: actions/cache@v1
-      with:
-        path: tests/Datasets # This path is specific to Ubuntu
-        # Look to see if there is a cache hit for the corresponding requirements file
-        key: mnist-dataset
+          ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-pip-
 
     - name: Install dependencies
       run: |
         # python -m pip install --upgrade --user pip
         pip install -r requirements.txt -U -f https://download.pytorch.org/whl/torch_stable.html -q
-        pip install -r ./tests/requirements.txt -q
+        HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install -r ./tests/requirements-devel.txt -q
         # pip install tox coverage
         python --version
         pip --version
         pip list
+      shell: bash
+
+    - name: Reinstall Horovod if necessary
+      if: runner.os != 'windows' && matrix.python-version != '3.8'
+      run: |
+        HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')")
+        if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then
+          pip uninstall -y horovod
+          HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install --no-cache-dir $(grep "horovod" requirements-extra.txt)
+        fi
+        horovodrun --check-build
+      shell: bash
+
+    - name: Cache datasets
+      uses: actions/cache@v1
+      with:
+        path: tests/Datasets # This path is specific to Ubuntu
+        # Look to see if there is a cache hit for the corresponding requirements file
+        key: mnist-dataset
 
     - name: Tests
       # env:
       #   TOXENV: py${{ matrix.python-version }}
       run: |
         # tox --sitepackages
         # flake8 .
-        coverage run --source pytorch_lightning -m py.test pytorch_lightning tests -v --doctest-modules --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}.xml
+        coverage run --source pytorch_lightning -m py.test pytorch_lightning tests -v --doctest-modules --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
         coverage report
 
     - name: Upload pytest test results
       uses: actions/upload-artifact@master
       with:
-        name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}
-        path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}.xml
+        name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}
+        path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
       # Use always() to always run this step to publish test results when there are test failures
       if: always()
 
@@ -98,7 +127,6 @@ jobs:
         python setup.py check --metadata --strict
         python setup.py sdist
         twine check dist/*
-
     #- name: Try install package
     #  if: ! startsWith(matrix.os, 'windows')
     #  run: |
@@ -109,4 +137,4 @@ jobs:
     - name: Statistics
       if: success()
       run: |
-         coverage report
+         coverage report

.github/workflows/docker-builds.yml

@@ -0,0 +1,50 @@
+name: Publish Docker Releases
+# https://www.docker.com/blog/first-docker-github-action-is-here
+on:
+  push:
+    branches: 
+      - master
+  release:
+    types:
+       - created
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python_version: [3.6, 3.7, 3.8]
+        pytorch_version: [1.1, 1.2, 1.3, 1.4, 1.5]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Publish Master to Docker
+        # publish master
+        uses: docker/build-push-action@v1.1.0
+        if: github.event_name == 'push'
+        with:
+          repository: pytorchlightning/pytorch_lightning
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          dockerfile: docker/Dockerfile
+          buildargs: PYTHON_VERSION=${{ matrix.python_version }},PYTORCH_VERSION=${{ matrix.pytorch_version }}
+          tags: "nightly-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}"
+        timeout-minutes: 30
+
+      - name: Get release version
+        if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'release'
+        id: get_version
+        run: echo ::set-env name=RELEASE_VERSION::$(echo ${GITHUB_REF##*/})
+
+      - name: Publish Releases to Docker
+        # only on releases
+        uses: docker/build-push-action@v1.1.0
+        if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'release'
+        with:
+          repository: pytorchlightning/pytorch_lightning
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          dockerfile: docker/Dockerfile
+          buildargs: PYTHON_VERSION=${{ matrix.python_version }},PYTORCH_VERSION=${{ matrix.pytorch_version }},LIGHTNING_VERSION=${{ env.RELEASE_VERSION }}
+          tags: "${{ env.RELEASE_VERSION }}-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }},latest-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}"
+        timeout-minutes: 30

.github/workflows/docs-check.yml

@@ -11,7 +11,7 @@ jobs:
     - uses: actions/checkout@v2
     - uses: ammaraskar/sphinx-action@master
       with:
-        # git is requried to clone the docs theme
+        # git is required to clone the docs theme
         pre-build-command: "apt-get update -y && apt-get install -y git"
         docs-folder: "docs/"
         repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/pypi-release.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v2
     - name: Set up Python 3.7
       uses: actions/setup-python@v1
       with:
@@ -30,7 +30,7 @@ jobs:
     - name: Build
       run: >-
         python setup.py sdist bdist_wheel
-    
+
     # We do this, since failures on test.pypi aren't that bad
     - name: Publish to Test PyPI
       if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'