Flake8 + Bugfixes + Linter (facebookresearch#459)

* Ran autoflake * flake8 examples * flake8 habitat * Flake8 Habitat_baselines * flake8 tests * Add flake8 linter * Fix eqa __init__ * Fix vln datasets.py * fix nav __init__ * Fix .circleci linter * Fix EQA __init__.py * Re-enable pre-commit hook * Refix vln * More __init__ repair * Fix two inits * Fix typo * Finalize __init__s and tests fixes * Update deprecated isort pre-commit hook * Add missing auotflake flag * clean up errors on pyrobot import * Apparently seed-isort pre-commit is deprecated * More bugfixes * reuse __init__ in vocabdict and fix m_docstring * Fix typo * Fix typo * Address comments * Add back known_first_party * Fix isort tutorial
SgtVincent · Aug 22, 2020 · 60dab1d · 60dab1d
1 parent 1e435b1
commit 60dab1d
Show file tree

Hide file tree

Showing 53 changed files with 108 additions and 181 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -15,7 +15,7 @@ jobs:
       - run:
           name: setup
           command: |
-              sudo pip install black "isort[pyproject]" numpy --progress-bar off
+              sudo pip install black flake8 "isort[pyproject]" numpy --progress-bar off
               sudo pip install -r requirements.txt --progress-bar off
       - run:
           name: run black
@@ -26,9 +26,13 @@ jobs:
           name: run isort
           command: |
               isort --version
-              isort -rc habitat/. habitat_baselines/. examples/. test/. setup.py --diff
-              isort -rc habitat/. habitat_baselines/. examples/. test/. setup.py --check-only
-
+              isort habitat/. habitat_baselines/. examples/. test/. setup.py --diff
+              isort habitat/. habitat_baselines/. examples/. test/. setup.py --check-only
+      - run:
+          name: run flake8
+          command: |
+              flake8 --version
+              flake8 habitat/. habitat_baselines/. examples/. tests/. setup.py
   install_and_test_ubuntu:
     <<: *gpu
     steps:

diff --git a/.editorconfig b/.editorconfig
@@ -20,4 +20,3 @@ include_trailing_comma = true
 ensure_newline_before_comments=true
 use_parentheses = true
 known_first_party = habitat,habitat_sim,habitat_baselines,version
-known_third_party = PIL,attr,conf,cv2,git,gym,ifcfg,imageio,magnum,matplotlib,mock,numba,numpy,orbslam2,pyrobot,pytest,quaternion,requests,scipy,setuptools,torch,torchvision,tqdm,yacs
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -20,14 +20,8 @@ repos:
     -   id: mixed-line-ending
         args: ['--fix=lf']
 
--   repo: https://github.com/asottile/seed-isort-config
-    rev: v2.2.0
-    hooks:
-    -   id: seed-isort-config
-        language_version: python3
-
--   repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.0.7
+-   repo: https://github.com/timothycrosley/isort
+    rev: 5.4.2
     hooks:
     -   id: isort
         exclude: docs/
@@ -43,9 +37,14 @@ repos:
     rev: master
     hooks:
     -   id: autoflake
-        args: ['--expand-star-imports', '--ignore-init-module-imports', '--in-place']
+        args: ['--expand-star-imports', '--ignore-init-module-imports', '--in-place', '-c']
         exclude: docs/
 
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.3
+    hooks:
+    -   id: flake8
+
 -   repo: https://github.com/kynan/nbstripout
     rev: master
     hooks:

diff --git a/examples/example.py b/examples/example.py
@@ -14,12 +14,12 @@ def example():
         config=habitat.get_config("configs/tasks/pointnav.yaml")
     ) as env:
         print("Environment creation successful")
-        observations = env.reset()
+        observations = env.reset()  # noqa: F841
 
         print("Agent stepping around inside environment.")
         count_steps = 0
         while not env.episode_over:
-            observations = env.step(env.action_space.sample())
+            observations = env.step(env.action_space.sample())  # noqa: F841
             count_steps += 1
         print("Episode finished after {} steps.".format(count_steps))
 

diff --git a/examples/tutorials/colabs/Habitat_Interactive_Tasks.ipynb b/examples/tutorials/colabs/Habitat_Interactive_Tasks.ipynb
@@ -68,9 +68,7 @@
     "import gzip\n",
     "import json\n",
     "import os\n",
-    "import random\n",
     "import sys\n",
-    "import time\n",
     "from typing import Any, Dict, List, Optional, Type\n",
     "\n",
     "import attr\n",
@@ -87,7 +85,6 @@
     "import habitat_sim\n",
     "from habitat.config import Config\n",
     "from habitat.core.registry import registry\n",
-    "from habitat_sim.utils import common as ut\n",
     "from habitat_sim.utils import viz_utils as vut\n",
     "\n",
     "if \"google.colab\" in sys.modules:\n",
@@ -145,10 +142,6 @@
     "    video_file = output_path + prefix + \".mp4\"\n",
     "    print(\"Encoding the video: %s \" % video_file)\n",
     "    writer = vut.get_fast_video_writer(video_file, fps=fps)\n",
-    "    thumb_size = (int(videodims[0] / 5), int(videodims[1] / 5))\n",
-    "    outline_frame = (\n",
-    "        np.ones((thumb_size[1] + 2, thumb_size[0] + 2, 3), np.uint8) * 150\n",
-    "    )\n",
     "    for ob in observations:\n",
     "        # If in RGB/RGBA format, remove the alpha channel\n",
     "        rgb_im_1st_person = cv2.cvtColor(ob[\"rgb\"], cv2.COLOR_RGBA2RGB)\n",
@@ -815,7 +808,7 @@
     "        sim, \"rgb\", crosshair_pos=[128, 190], max_distance=1.0\n",
     "    )\n",
     "    print(f\"Closest Object ID: {closest_object} using 1.0 threshold\")\n",
-    "    assert closest_object == -1, f\"Agent shoud not be able to pick any object\""
+    "    assert closest_object == -1, \"Agent shoud not be able to pick any object\""
    ]
   },
   {
@@ -1129,7 +1122,6 @@
     "from habitat.core.embodied_task import Measure\n",
     "from habitat.core.simulator import Observations, Sensor, SensorTypes, Simulator\n",
     "from habitat.tasks.nav.nav import PointGoalSensor\n",
-    "from habitat_sim.utils.common import quat_from_magnum\n",
     "\n",
     "\n",
     "@registry.register_sensor\n",
@@ -1493,7 +1485,6 @@
     "# @markdown - The agent gets a slack penalty of -0.01 for every action it takes in the environment.\n",
     "# @markdown - Finally the agent gets a large success reward when the episode is completed successfully.\n",
     "\n",
-    "from collections import defaultdict\n",
     "from typing import Optional, Type\n",
     "\n",
     "import numpy as np\n",
@@ -1680,15 +1671,12 @@
     "from typing import Any, Dict, List, Optional\n",
     "\n",
     "import numpy as np\n",
-    "import torch\n",
     "from torch.optim.lr_scheduler import LambdaLR\n",
     "\n",
     "from habitat import Config, logger\n",
-    "from habitat.core.vector_env import ThreadedVectorEnv\n",
     "from habitat.utils.visualizations.utils import observations_to_image\n",
-    "from habitat_baselines.common.base_trainer import BaseRLTrainer\n",
     "from habitat_baselines.common.baseline_registry import baseline_registry\n",
-    "from habitat_baselines.common.env_utils import construct_envs, make_env_fn\n",
+    "from habitat_baselines.common.env_utils import make_env_fn\n",
     "from habitat_baselines.common.environments import get_env_class\n",
     "from habitat_baselines.common.rollout_storage import RolloutStorage\n",
     "from habitat_baselines.common.tensorboard_utils import TensorboardWriter\n",
@@ -1698,7 +1686,7 @@
     "    linear_decay,\n",
     ")\n",
     "from habitat_baselines.rl.models.rnn_state_encoder import RNNStateEncoder\n",
-    "from habitat_baselines.rl.ppo import PPO, PointNavBaselinePolicy\n",
+    "from habitat_baselines.rl.ppo import PPO\n",
     "from habitat_baselines.rl.ppo.policy import Net, Policy\n",
     "from habitat_baselines.rl.ppo.ppo_trainer import PPOTrainer\n",
     "\n",
@@ -1721,7 +1709,7 @@
     "    num_processes = config.NUM_PROCESSES\n",
     "    configs = []\n",
     "    env_classes = [env_class for _ in range(num_processes)]\n",
-    "    dataset = make_dataset(config.TASK_CONFIG.DATASET.TYPE)\n",
+    "    dataset = habitat.datasets.make_dataset(config.TASK_CONFIG.DATASET.TYPE)\n",
     "    scenes = config.TASK_CONFIG.DATASET.CONTENT_SCENES\n",
     "    if \"*\" in config.TASK_CONFIG.DATASET.CONTENT_SCENES:\n",
     "        scenes = dataset.get_scenes_to_load(config.TASK_CONFIG.DATASET)\n",
@@ -2160,8 +2148,7 @@
     "import torch\n",
     "\n",
     "import habitat\n",
-    "from habitat import Config, Env, RLEnv, VectorEnv, make_dataset\n",
-    "from habitat.config import get_config\n",
+    "from habitat import Config\n",
     "from habitat_baselines.config.default import get_config as get_baseline_config\n",
     "\n",
     "baseline_config = get_baseline_config(\n",

diff --git a/examples/tutorials/nb_python/Habitat_Interactive_Tasks.py b/examples/tutorials/nb_python/Habitat_Interactive_Tasks.py
@@ -71,9 +71,7 @@
 import gzip
 import json
 import os
-import random
 import sys
-import time
 from typing import Any, Dict, List, Optional, Type
 
 import attr
@@ -90,7 +88,6 @@
 import habitat_sim
 from habitat.config import Config
 from habitat.core.registry import registry
-from habitat_sim.utils import common as ut
 from habitat_sim.utils import viz_utils as vut
 
 if "google.colab" in sys.modules:
@@ -141,10 +138,6 @@ def make_video_cv2(
     video_file = output_path + prefix + ".mp4"
     print("Encoding the video: %s " % video_file)
     writer = vut.get_fast_video_writer(video_file, fps=fps)
-    thumb_size = (int(videodims[0] / 5), int(videodims[1] / 5))
-    outline_frame = (
-        np.ones((thumb_size[1] + 2, thumb_size[0] + 2, 3), np.uint8) * 150
-    )
     for ob in observations:
         # If in RGB/RGBA format, remove the alpha channel
         rgb_im_1st_person = cv2.cvtColor(ob["rgb"], cv2.COLOR_RGBA2RGB)
@@ -749,7 +742,7 @@ def raycast(sim, sensor_name, crosshair_pos=[128, 128], max_distance=2.0):
         sim, "rgb", crosshair_pos=[128, 190], max_distance=1.0
     )
     print(f"Closest Object ID: {closest_object} using 1.0 threshold")
-    assert closest_object == -1, f"Agent shoud not be able to pick any object"
+    assert closest_object == -1, "Agent shoud not be able to pick any object"
 
 
 # %%
@@ -1039,7 +1032,6 @@ def step(self, action: int):
 from habitat.core.embodied_task import Measure
 from habitat.core.simulator import Observations, Sensor, SensorTypes, Simulator
 from habitat.tasks.nav.nav import PointGoalSensor
-from habitat_sim.utils.common import quat_from_magnum
 
 
 @registry.register_sensor
@@ -1383,7 +1375,6 @@ def print_info(obs, metrics):
 # @markdown - The agent gets a slack penalty of -0.01 for every action it takes in the environment.
 # @markdown - Finally the agent gets a large success reward when the episode is completed successfully.
 
-from collections import defaultdict
 from typing import Optional, Type
 
 import numpy as np
@@ -1565,15 +1556,12 @@ def get_info(self, observations):
 from typing import Any, Dict, List, Optional
 
 import numpy as np
-import torch
 from torch.optim.lr_scheduler import LambdaLR
 
 from habitat import Config, logger
-from habitat.core.vector_env import ThreadedVectorEnv
 from habitat.utils.visualizations.utils import observations_to_image
-from habitat_baselines.common.base_trainer import BaseRLTrainer
 from habitat_baselines.common.baseline_registry import baseline_registry
-from habitat_baselines.common.env_utils import construct_envs, make_env_fn
+from habitat_baselines.common.env_utils import make_env_fn
 from habitat_baselines.common.environments import get_env_class
 from habitat_baselines.common.rollout_storage import RolloutStorage
 from habitat_baselines.common.tensorboard_utils import TensorboardWriter
@@ -1583,7 +1571,7 @@ def get_info(self, observations):
     linear_decay,
 )
 from habitat_baselines.rl.models.rnn_state_encoder import RNNStateEncoder
-from habitat_baselines.rl.ppo import PPO, PointNavBaselinePolicy
+from habitat_baselines.rl.ppo import PPO
 from habitat_baselines.rl.ppo.policy import Net, Policy
 from habitat_baselines.rl.ppo.ppo_trainer import PPOTrainer
 
@@ -1606,7 +1594,7 @@ def construct_envs(
     num_processes = config.NUM_PROCESSES
     configs = []
     env_classes = [env_class for _ in range(num_processes)]
-    dataset = make_dataset(config.TASK_CONFIG.DATASET.TYPE)
+    dataset = habitat.datasets.make_dataset(config.TASK_CONFIG.DATASET.TYPE)
     scenes = config.TASK_CONFIG.DATASET.CONTENT_SCENES
     if "*" in config.TASK_CONFIG.DATASET.CONTENT_SCENES:
         scenes = dataset.get_scenes_to_load(config.TASK_CONFIG.DATASET)
@@ -2034,8 +2022,7 @@ def eval(self) -> None:
 import torch
 
 import habitat
-from habitat import Config, Env, RLEnv, VectorEnv, make_dataset
-from habitat.config import get_config
+from habitat import Config
 from habitat_baselines.config.default import get_config as get_baseline_config
 
 baseline_config = get_baseline_config(

diff --git a/examples/vln_benchmark.py b/examples/vln_benchmark.py
@@ -6,6 +6,7 @@
 
 import argparse
 from collections import defaultdict
+from typing import Dict
 
 import habitat
 from habitat.config.default import get_config

diff --git a/habitat/__init__.py b/habitat/__init__.py
@@ -12,7 +12,7 @@
 from habitat.core.embodied_task import EmbodiedTask, Measure, Measurements
 from habitat.core.env import Env, RLEnv
 from habitat.core.logging import logger
-from habitat.core.registry import registry
+from habitat.core.registry import registry  # noqa : F401
 from habitat.core.simulator import Sensor, SensorSuite, SensorTypes, Simulator
 from habitat.core.vector_env import ThreadedVectorEnv, VectorEnv
 from habitat.datasets import make_dataset

diff --git a/habitat/core/benchmark.py b/habitat/core/benchmark.py
@@ -46,7 +46,7 @@ def remote_evaluate(
         import pickle
         import time
 
-        import evalai_environment_habitat
+        import evalai_environment_habitat  # noqa: F401
         import evaluation_pb2
         import evaluation_pb2_grpc
         import grpc

diff --git a/habitat/core/embodied_task.py b/habitat/core/embodied_task.py
@@ -320,7 +320,7 @@ def step(self, action: Union[int, Dict[str, Any]], episode: Type[Episode]):
 
     def get_action_name(self, action_index: int):
         if action_index >= len(self.actions):
-            raise ValueError(f"Action index '{action}' is out of range.")
+            raise ValueError(f"Action index '{action_index}' is out of range.")
         return self._action_keys[action_index]
 
     @property

diff --git a/habitat/core/utils.py b/habitat/core/utils.py
@@ -4,11 +4,12 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
+import cmath
 import json
 from typing import List
 
 import numpy as np
-import quaternion
+import quaternion  # noqa: F401
 
 from habitat.utils.geometry_utils import quaternion_to_list
 
@@ -148,7 +149,7 @@ def floatstr(
             _inf=float("inf"),
             _neginf=-float("inf"),
         ):
-            if o != o:
+            if cmath.isnan(o):
                 text = "NaN"
             elif o == _inf:
                 text = "Infinity"

diff --git a/habitat/datasets/__init__.py b/habitat/datasets/__init__.py
@@ -4,4 +4,4 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-from habitat.datasets.registration import make_dataset
+from habitat.datasets.registration import make_dataset  # noqa: F401 .
diff --git a/habitat/datasets/eqa/__init__.py b/habitat/datasets/eqa/__init__.py
@@ -10,17 +10,12 @@
 
 def _try_register_mp3d_eqa_dataset():
     try:
-        from habitat.datasets.eqa.mp3d_eqa_dataset import Matterport3dDatasetV1
-
-        has_mp3deqa = True
+        from habitat.datasets.eqa.mp3d_eqa_dataset import (  # noqa: F401 isort:skip
+            Matterport3dDatasetV1,
+        )
     except ImportError as e:
-        has_mp3deqa = False
         mp3deqa_import_error = e
 
-    if has_mp3deqa:
-        from habitat.datasets.eqa.mp3d_eqa_dataset import Matterport3dDatasetV1
-    else:
-
         @registry.register_dataset(name="MP3DEQA-v1")
         class Matterport3dDatasetImportError(Dataset):
             def __init__(self, *args, **kwargs):

diff --git a/habitat/datasets/eqa/mp3d_eqa_dataset.py b/habitat/datasets/eqa/mp3d_eqa_dataset.py
@@ -13,7 +13,7 @@
 from habitat.core.dataset import Dataset
 from habitat.core.registry import registry
 from habitat.core.simulator import AgentState
-from habitat.datasets.utils import VocabDict, VocabFromText
+from habitat.datasets.utils import VocabDict
 from habitat.tasks.eqa.eqa import EQAEpisode, QuestionData
 from habitat.tasks.nav.nav import ShortestPathPoint
 from habitat.tasks.nav.object_nav_task import ObjectGoal

diff --git a/habitat/datasets/object_nav/__init__.py b/habitat/datasets/object_nav/__init__.py
@@ -17,21 +17,13 @@
 # Post that PR we would no longer need try register blocks.
 def _try_register_objectnavdatasetv1():
     try:
-        from habitat.datasets.object_nav.object_nav_dataset import (
+        from habitat.datasets.object_nav.object_nav_dataset import (  # noqa: F401
             ObjectNavDatasetV1,
         )
 
-        has_pointnav = True
     except ImportError as e:
-        has_pointnav = False
         pointnav_import_error = e
 
-    if has_pointnav:
-        from habitat.datasets.object_nav.object_nav_dataset import (
-            ObjectNavDatasetV1,
-        )
-    else:
-
         @registry.register_dataset(name="ObjectNav-v1")
         class ObjectNavDatasetImportError(Dataset):
             def __init__(self, *args, **kwargs):