PR: Add setuptools setup file, add command to compile CUDA/C libraries and general package file refactoring

.pylintrc

@@ -1,6 +1,5 @@
 [MASTER]
 extension-pkg-whitelist=numpy,torch,cv2
-init-hook="sys.path.insert(0, './tools'); import _init_paths"
 
 [MESSAGES CONTROL]
 disable=wrong-import-position

MANIFEST.in

@@ -0,0 +1,6 @@
+include CHANGELOG.md
+include CONTRIBUTORS.md
+include LICENSE.txt
+include README.rst
+include setupbase.py
+recursive-include detectron_pytorch/ *.cu *.c *.cpp *.h *.o *.so *.a *.yml *.yaml *.pyx

README.md

@@ -78,11 +78,11 @@ Tested under python3.
 Compile the CUDA code:
 
 ```
-cd lib  # please change to this directory
-sh make.sh
+export CUDA_PATH=/usr/local/cuda  # Or any other custom location
+python setup.py build_static
 ```
 
-If your are using Volta GPUs, uncomment this [line](https://github.com/roytseng-tw/mask-rcnn.pytorch/tree/master/lib/make.sh#L15) in `lib/mask.sh` and remember to postpend a backslash at the line above. `CUDA_PATH` defaults to `/usr/loca/cuda`. If you want to use a CUDA library on different path, change this [line](https://github.com/roytseng-tw/mask-rcnn.pytorch/tree/master/lib/make.sh#L3) accordingly.
+If your are using Volta GPUs, uncomment this [line](https://github.com/roytseng-tw/mask-rcnn.pytorch/tree/master/lib/make.sh#L15) in `lib/mask.sh` and remember to postpend a backslash at the line above. `CUDA_PATH` defaults to `/usr/loca/cuda`.
 
 It will compile all the modules you need, including NMS, ROI_Pooing, ROI_Crop and ROI_Align. (Actually gpu nms is never used ...)
 
@@ -144,7 +144,7 @@ You can the following command to download them all:
 - extra required packages: `argparse_color_formater`, `colorama`, `requests`
 
 ```
-python tools/download_imagenet_weights.py
+python -m detectron_pytorch.tools.download_imagenet_weights
 ```
 
 **NOTE**: Caffe pretrained weights have slightly better performance than Pytorch pretrained. Suggest to use Caffe pretrained models from the above link to reproduce the results. By the way, Detectron also use pretrained weights from Caffe.
@@ -193,7 +193,7 @@ Following config options will be adjusted **automatically** according to actual
 ### Train from scratch
 Take mask-rcnn with res50 backbone for example.
 ```
-python tools/train_net_step.py --dataset coco2017 --cfg configs/baselines/e2e_mask_rcnn_R-50-C4.yml --use_tfboard --bs {batch_size} --nw {num_workers}
+python -m detectron_pytorch.tools.train_net_step --dataset coco2017 --cfg configs/baselines/e2e_mask_rcnn_R-50-C4.yml --use_tfboard --bs {batch_size} --nw {num_workers}
 ```
 
 Use `--bs` to overwrite the default batch size to a proper value that fits into your GPUs. Simliar for `--nw`, number of data loader threads defaults to 4 in config.py.
@@ -206,17 +206,17 @@ Specify `—-use_tfboard` to log the losses on Tensorboard.
 As in Caffe, update network once (`optimizer.step()`) every `iter_size` iterations (forward + backward). This way to have a larger effective batch size for training. Notice that, step count is only increased after network update.
 
 ```
-python tools/train_net_step.py --dataset coco2017 --cfg configs/baselines/e2e_mask_rcnn_R-50-C4.yml --bs 4 --iter_size 4
+python -m detectron_pytorch.tools.train_net_step --dataset coco2017 --cfg configs/baselines/e2e_mask_rcnn_R-50-C4.yml --bs 4 --iter_size 4
 ```
 `iter_size` defaults to 1.
 
 ### Finetune from a pretrained checkpoint
 ```
-python tools/train_net_step.py ... --load_ckpt {path/to/the/checkpoint}
+python -m detectron_pytorch.tools.train_net_step ... --load_ckpt {path/to/the/checkpoint}
 ```
 or using Detectron's checkpoint file
 ```
-python tools/train_net_step.py ... --load_detectron {path/to/the/checkpoint}
+python -m detectron_pytorch.tools.train_net_step ... --load_detectron {path/to/the/checkpoint}
 ```
 
 ### Resume training with the same dataset and batch size
@@ -229,24 +229,24 @@ When resume the training, **step count** and **optimizer state** will also be re
 
 ### Set config options in command line
 ```
-  python tools/train_net_step.py ... --no_save --set {config.name1} {value1} {config.name2} {value2} ...
+  python -m detectron_pytorch.tools.train_net_step ... --no_save --set {config.name1} {value1} {config.name2} {value2} ...
 ```
 - For Example, run for debugging.
   ```
-  python tools/train_net_step.py ... --no_save --set DEBUG True
+  python -m detectron_pytorch.tools.train_net_step ... --no_save --set DEBUG True
   ```
   Load less annotations to accelarate training progress. Add `--no_save` to avoid saving any checkpoint or logging.
 
 ### Show command line help messages
 ```
-python train_net_step.py --help
+python -m detectron_pytorch.tools.train_net_step --help
 ```
 
 ### Two Training Scripts
 
-In short, use `train_net_step.py`.
+In short, use `detectron_pytorch.tools.train_net_step`.
 
-In `train_net_step.py`:
+In `detectron_pytorch.tools.train_net_step`:
 - `SOLVER.LR_POLICY: steps_with_decay` is supported.
 - Training warm up in [Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour](https://arxiv.org/abs/1706.02677) is supported.
 
@@ -268,15 +268,15 @@ In `train_net_step.py`:
 ### Evaluate the training results
 For example, test mask-rcnn on coco2017 val set
 ```
-python tools/test_net.py --dataset coco2017 --cfg config/baselines/e2e_mask_rcnn_R-50-FPN_1x.yaml --load_ckpt {path/to/your/checkpoint}
+python -m detectron_pytorch.tools.test_net --dataset coco2017 --cfg config/baselines/e2e_mask_rcnn_R-50-FPN_1x.yaml --load_ckpt {path/to/your/checkpoint}
 ```
 Use `--load_detectron` to load Detectron's checkpoint. If multiple gpus are available, add `--multi-gpu-testing`.
 
 Specify a different output directry, use `--output_dir {...}`. Defaults to `{the/parent/dir/of/checkpoint}/test`
 
 ### Visualize the training results on images
 ```
-python tools/infer_simple.py --dataset coco --cfg cfgs/baselines/e2e_mask_rcnn_R-50-C4.yml --load_ckpt {path/to/your/checkpoint} --image_dir {dir/of/input/images}  --output_dir {dir/to/save/visualizations}
+python -m detectron_pytorch.tools.infer_simple --dataset coco --cfg cfgs/baselines/e2e_mask_rcnn_R-50-C4.yml --load_ckpt {path/to/your/checkpoint} --image_dir {dir/of/input/images}  --output_dir {dir/to/save/visualizations}
 ```
 `--output_dir` defaults to `infer_outputs`.
 

detectron_pytorch/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------------
+# Copyright (c) Roy Tseng
+#
+# Licensed under the terms of the MIT License
+# (see LICENSE.txt for details)
+# -----------------------------------------------------------------------------
+"""Detectron PyTorch"""
+
+VERSION_INFO = (0, 1, 0, 'dev0')
+__version__ = '.'.join(map(str, VERSION_INFO))

lib/core/config.py → detectron_pytorch/core/config.py

@@ -16,8 +16,8 @@
 from torch.nn import init
 import yaml
 
-import nn as mynn
-from utils.collections import AttrDict
+import detectron_pytorch.nn as mynn
+from detectron_pytorch.utils.collections import AttrDict
 
 __C = AttrDict()
 # Consumers can get config by:

lib/core/test.py → detectron_pytorch/core/test.py

@@ -38,13 +38,13 @@
 from torch.autograd import Variable
 import torch
 
-from core.config import cfg
-from utils.timer import Timer
-import utils.boxes as box_utils
-import utils.blob as blob_utils
-import utils.fpn as fpn_utils
-import utils.image as image_utils
-import utils.keypoints as keypoint_utils
+from detectron_pytorch.core.config import cfg
+from detectron_pytorch.utils.timer import Timer
+import detectron_pytorch.utils.boxes as box_utils
+import detectron_pytorch.utils.blob as blob_utils
+import detectron_pytorch.utils.fpn as fpn_utils
+import detectron_pytorch.utils.image as image_utils
+import detectron_pytorch.utils.keypoints as keypoint_utils
 
 
 def im_detect_all(model, im, box_proposals=None, timers=None):

lib/core/test_engine.py → detectron_pytorch/core/test_engine.py

@@ -30,21 +30,21 @@
 
 import torch
 
-from core.config import cfg
-# from core.rpn_generator import generate_rpn_on_dataset  #TODO: for rpn only case
-# from core.rpn_generator import generate_rpn_on_range
-from core.test import im_detect_all
-from datasets import task_evaluation
-from datasets.json_dataset import JsonDataset
-from modeling import model_builder
-import nn as mynn
-from utils.detectron_weight_helper import load_detectron_weight
-import utils.env as envu
-import utils.net as net_utils
-import utils.subprocess as subprocess_utils
-import utils.vis as vis_utils
-from utils.io import save_object
-from utils.timer import Timer
+from detectron_pytorch.core.config import cfg
+# from detectron_pytorch.core.rpn_generator import generate_rpn_on_dataset  #TODO: for rpn only case
+# from detectron_pytorch.core.rpn_generator import generate_rpn_on_range
+from detectron_pytorch.core.test import im_detect_all
+from detectron_pytorch.datasets import task_evaluation
+from detectron_pytorch.datasets.json_dataset import JsonDataset
+from detectron_pytorch.modeling import model_builder
+import detectron_pytorch.nn as mynn
+from detectron_pytorch.utils.detectron_weight_helper import load_detectron_weight
+import detectron_pytorch.utils.env as envu
+import detectron_pytorch.utils.net as net_utils
+import detectron_pytorch.utils.subprocess as subprocess_utils
+import detectron_pytorch.utils.vis as vis_utils
+from detectron_pytorch.utils.io import save_object
+from detectron_pytorch.utils.timer import Timer
 
 logger = logging.getLogger(__name__)
 

lib/datasets/cityscapes/tools/convert_cityscapes_to_coco.py → detectron_pytorch/datasets/cityscapes/tools/convert_cityscapes_to_coco.py

@@ -12,8 +12,8 @@
 
 import cityscapesscripts.evaluation.instances2dict_with_polygons as cs
 
-import utils.segms as segms_util
-import utils.boxes as bboxs_util
+import detectron_pytorch.utils.segms as segms_util
+import detectron_pytorch.utils.boxes as bboxs_util
 
 
 def parse_args():

lib/datasets/cityscapes/tools/convert_coco_model_to_cityscapes.py → detectron_pytorch/datasets/cityscapes/tools/convert_coco_model_to_cityscapes.py

@@ -14,7 +14,7 @@
 import sys
 import numpy as np
 
-import datasets.cityscapes.coco_to_cityscapes_id as cs
+import detectron_pytorch.datasets.cityscapes.coco_to_cityscapes_id as cs
 
 NUM_CS_CLS = 9
 NUM_COCO_CLS = 81

lib/datasets/cityscapes_json_dataset_evaluator.py → detectron_pytorch/datasets/cityscapes_json_dataset_evaluator.py

@@ -27,9 +27,9 @@
 
 import pycocotools.mask as mask_util
 
-from core.config import cfg
-from datasets.dataset_catalog import DATASETS
-from datasets.dataset_catalog import RAW_DIR
+from detectron_pytorch.core.config import cfg
+from detectron_pytorch.datasets.dataset_catalog import DATASETS
+from detectron_pytorch.datasets.dataset_catalog import RAW_DIR
 
 logger = logging.getLogger(__name__)
 

lib/datasets/dataset_catalog.py → detectron_pytorch/datasets/dataset_catalog.py

@@ -22,7 +22,7 @@
 
 import os
 
-from core.config import cfg
+from detectron_pytorch.core.config import cfg
 
 # Path to data dir
 _DATA_DIR = cfg.DATA_DIR

lib/datasets/dummy_datasets.py → detectron_pytorch/datasets/dummy_datasets.py

@@ -22,7 +22,7 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from utils.collections import AttrDict
+from detectron_pytorch.utils.collections import AttrDict
 
 
 def get_coco_dataset():

lib/datasets/json_dataset.py → detectron_pytorch/datasets/json_dataset.py

@@ -33,15 +33,15 @@
 import scipy.sparse
 
 # Must happen before importing COCO API (which imports matplotlib)
-import utils.env as envu
+import detectron_pytorch.utils.env as envu
 envu.set_up_matplotlib()
 # COCO API
 from pycocotools import mask as COCOmask
 from pycocotools.coco import COCO
 
-import utils.boxes as box_utils
-from core.config import cfg
-from utils.timer import Timer
+import detectron_pytorch.utils.boxes as box_utils
+from detectron_pytorch.core.config import cfg
+from detectron_pytorch.utils.timer import Timer
 from .dataset_catalog import ANN_FN
 from .dataset_catalog import DATASETS
 from .dataset_catalog import IM_DIR

lib/datasets/json_dataset_evaluator.py → detectron_pytorch/datasets/json_dataset_evaluator.py

@@ -28,9 +28,9 @@
 
 from pycocotools.cocoeval import COCOeval
 
-from core.config import cfg
-from utils.io import save_object
-import utils.boxes as box_utils
+from detectron_pytorch.core.config import cfg
+from detectron_pytorch.utils.io import save_object
+import detectron_pytorch.utils.boxes as box_utils
 
 logger = logging.getLogger(__name__)
 

lib/datasets/roidb.py → detectron_pytorch/datasets/roidb.py

@@ -24,11 +24,11 @@
 import logging
 import numpy as np
 
-import utils.boxes as box_utils
-import utils.keypoints as keypoint_utils
-import utils.segms as segm_utils
-import utils.blob as blob_utils
-from core.config import cfg
+import detectron_pytorch.utils.boxes as box_utils
+import detectron_pytorch.utils.keypoints as keypoint_utils
+import detectron_pytorch.utils.segms as segm_utils
+import detectron_pytorch.utils.blob as blob_utils
+from detectron_pytorch.core.config import cfg
 from .json_dataset import JsonDataset
 
 logger = logging.getLogger(__name__)

lib/datasets/task_evaluation.py → detectron_pytorch/datasets/task_evaluation.py

@@ -40,11 +40,11 @@
 import os
 import pprint
 
-from core.config import cfg
-from utils.logging import send_email
-import datasets.cityscapes_json_dataset_evaluator as cs_json_dataset_evaluator
-import datasets.json_dataset_evaluator as json_dataset_evaluator
-import datasets.voc_dataset_evaluator as voc_dataset_evaluator
+from detectron_pytorch.core.config import cfg
+from detectron_pytorch.utils.logging import send_email
+import detectron_pytorch.datasets.cityscapes_json_dataset_evaluator as cs_json_dataset_evaluator
+import detectron_pytorch.datasets.json_dataset_evaluator as json_dataset_evaluator
+import detectron_pytorch.datasets.voc_dataset_evaluator as voc_dataset_evaluator
 
 logger = logging.getLogger(__name__)
 

lib/datasets/voc_dataset_evaluator.py → detectron_pytorch/datasets/voc_dataset_evaluator.py

@@ -26,11 +26,11 @@
 import shutil
 import uuid
 
-from core.config import cfg
-from datasets.dataset_catalog import DATASETS
-from datasets.dataset_catalog import DEVKIT_DIR
-from datasets.voc_eval import voc_eval
-from utils.io import save_object
+from detectron_pytorch.core.config import cfg
+from detectron_pytorch.datasets.dataset_catalog import DATASETS
+from detectron_pytorch.datasets.dataset_catalog import DEVKIT_DIR
+from detectron_pytorch.datasets.voc_eval import voc_eval
+from detectron_pytorch.utils.io import save_object
 
 logger = logging.getLogger(__name__)
 

lib/make.sh → detectron_pytorch/make.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
-CUDA_PATH=/usr/local/cuda/
-
+cd detectron_pytorch
 python setup.py build_ext --inplace
 rm -rf build
 

detectron_pytorch/model/nms/.gitignore

@@ -0,0 +1 @@
+*.so

lib/model/nms/nms_wrapper.py → detectron_pytorch/model/nms/nms_wrapper.py

@@ -5,8 +5,8 @@
 # Written by Ross Girshick
 # --------------------------------------------------------
 import torch
-from core.config import cfg
-from model.nms.nms_gpu import nms_gpu
+from detectron_pytorch.core.config import cfg
+from detectron_pytorch.model.nms.nms_gpu import nms_gpu
 
 def nms(dets, thresh, force_cpu=False):
     """Dispatch to either CPU or GPU NMS implementations."""

lib/model/roi_align/make.sh → detectron_pytorch/model/roi_align/make.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-CUDA_PATH=/usr/local/cuda/
+# CUDA_PATH=/usr/local/cuda/
 
 cd src
 echo "Compiling my_lib kernels by nvcc..."