Added ccache to CI and fixed PyTorch caching. (facebookresearch#161)

* fixed PyTorch caching
* added ccache to CI
* broke habitat api step on installation and testing to save extra 5 min on saving cache in parallel
* testing time drops from first run to second run from 17.5 minutes to 9 minutes (2x times) for full testing.

.circleci/config.yml

@@ -89,7 +89,7 @@ jobs:
                 export PATH=$HOME/miniconda/bin:$PATH
                 conda create -y -n habitat python=3.6
                 . activate habitat
-                conda install -q -y -c conda-forge ninja numpy pytest
+                conda install -q -y -c conda-forge ninja ccache numpy pytest
               fi
       - run:
           name: Install pytorch
@@ -107,6 +107,19 @@ jobs:
       - restore_cache:
           keys:
             - habitat-sim-{{ checksum "./hsim_sha" }}
+      - restore_cache:
+          keys:
+            - ccache-{{ arch }}-master
+          paths:
+            - /home/circleci/.ccache
+      - run:
+          name: CCache initialization
+          command: |
+            export PATH=$HOME/miniconda/bin:$PATH
+            . activate habitat;
+            ccache --show-stats
+            ccache --zero-stats
+            ccache --max-size=10.0G
       - run:
           name: Build, install habitat-sim and run benchmark
           no_output_timeout: 20m
@@ -127,11 +140,13 @@ jobs:
                   python setup.py install --headless
                 fi
               fi
-      - save_cache:
-          key: conda-{{ checksum "habitat-api/.circleci/config.yml" }}
-          background: true
-          paths:
-            - ~/miniconda
+      - run:
+          name: Ccache stats
+          when: always
+          command: |
+            export PATH=$HOME/miniconda/bin:$PATH
+            . activate habitat;
+            ccache --show-stats
       - run:
           name: Download test data
           command: |
@@ -152,24 +167,37 @@ jobs:
               python examples/example.py --scene data/scene_datasets/habitat-test-scenes/van-gogh-room.glb --silent --test_fps_regression $FPS_THRESHOLD
       - save_cache:
           key: habitat-sim-{{ checksum "./hsim_sha" }}
+          background: true
           paths:
             - ./habitat-sim
+      - save_cache:
+          key: ccache-{{ arch }}-master
+          background: true
+          paths:
+            - /home/circleci/.ccache
       - run:
-          name: Run api tests
+          name: Install api
           command: |
               export PATH=$HOME/miniconda/bin:$PATH
               . activate habitat; cd habitat-api
               while [ ! -f ~/miniconda/pytorch_installed ]; do sleep 2; done # wait for Pytorch
               ln -s ../habitat-sim/data data
               pip install -r requirements.txt --progress-bar off
               touch ~/miniconda/pip_deps_installed
-              python setup.py test
-              python setup.py develop --all
-              python -u habitat_baselines/train_ppo.py --log-file "train.log" --checkpoint-folder "data/checkpoints" --sim-gpu-id 0 --pth-gpu-id 0 --num-processes 1 --num-mini-batch 1 --num-updates 10
       - save_cache:
           key: conda-{{ checksum "habitat-api/.circleci/config.yml" }}
+          background: true
           paths:
             - ~/miniconda
+      - run:
+          name: Run api tests
+          command: |
+              export PATH=$HOME/miniconda/bin:$PATH
+              . activate habitat; cd habitat-api
+              python setup.py test
+              python setup.py develop --all
+              python -u habitat_baselines/train_ppo.py --log-file "train.log" --checkpoint-folder "data/checkpoints" --sim-gpu-id 0 --pth-gpu-id 0 --num-processes 1 --num-mini-batch 1 --num-updates 10
+
 
 workflows:
   version: 2