You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by jx...@apache.org on 2017/12/18 20:28:26 UTC
[incubator-mxnet] branch master updated: Fix compatibility of
examples with py3 (#8993)
This is an automated email from the ASF dual-hosted git repository.
jxie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new df55829 Fix compatibility of examples with py3 (#8993)
df55829 is described below
commit df558290930f9f3ed6941c306b5cc650505f7481
Author: Rahul Huilgol <ra...@gmail.com>
AuthorDate: Mon Dec 18 12:28:23 2017 -0800
Fix compatibility of examples with py3 (#8993)
* usability improvements
support for py3 and mac
Signed-off-by: Rahul <ra...@gmail.com>
* more py3 changes
* fix cpickle changes py3
Signed-off-by: Rahul <ra...@gmail.com>
* fix cpickle changes py3
Signed-off-by: Rahul <ra...@gmail.com>
* fix cpickle changes py3
Signed-off-by: Rahul <ra...@gmail.com>
* change float used to index to int
Signed-off-by: Rahul <ra...@gmail.com>
* change float used to index to int
Signed-off-by: Rahul <ra...@gmail.com>
* change float used to index to int
Signed-off-by: Rahul <ra...@gmail.com>
* fix xrange
Signed-off-by: Rahul <ra...@gmail.com>
* import compatibility
Signed-off-by: Rahul <ra...@gmail.com>
* int numpy deprecation
Signed-off-by: Rahul <ra...@gmail.com>
* add note about future
Signed-off-by: Rahul <ra...@gmail.com>
* cpu and windows messages
Signed-off-by: Rahul <ra...@gmail.com>
* change cpickle to pickle for both py2 and py3
Signed-off-by: Rahul <ra...@gmail.com>
* remove six dependency
Signed-off-by: Rahul <ra...@gmail.com>
---
example/gluon/tree_lstm/main.py | 16 ++++++----
example/rcnn/README.md | 22 +++++++------
example/rcnn/rcnn/core/tester.py | 18 ++++++-----
example/rcnn/rcnn/cython/setup.py | 2 +-
example/rcnn/rcnn/dataset/__init__.py | 6 ++--
example/rcnn/rcnn/dataset/coco.py | 18 ++++++-----
example/rcnn/rcnn/dataset/ds_utils.py | 2 +-
example/rcnn/rcnn/dataset/imdb.py | 7 +++--
example/rcnn/rcnn/dataset/pascal_voc.py | 19 +++++++-----
example/rcnn/rcnn/dataset/pascal_voc_eval.py | 10 +++---
example/rcnn/rcnn/io/rcnn.py | 6 ++--
example/rcnn/rcnn/processing/bbox_regression.py | 2 +-
example/rcnn/rcnn/processing/generate_anchor.py | 4 +--
example/rcnn/rcnn/pycocotools/cocoeval.py | 4 +--
example/rcnn/rcnn/pycocotools/mask.py | 2 +-
example/rcnn/rcnn/symbol/__init__.py | 4 +--
example/rcnn/rcnn/symbol/proposal_target.py | 2 +-
example/rcnn/rcnn/symbol/symbol_resnet.py | 4 +--
example/rcnn/rcnn/symbol/symbol_vgg.py | 5 ++-
example/rcnn/rcnn/tools/reeval.py | 7 +++--
example/rcnn/script/additional_deps.sh | 14 +--------
.../linear_classification/weighted_softmax_ce.py | 2 +-
example/sparse/matrix_factorization/data.py | 18 ++++++-----
example/sparse/matrix_factorization/train.py | 9 +++---
python/mxnet/test_utils.py | 36 ++++++++++++++++++----
25 files changed, 139 insertions(+), 100 deletions(-)
diff --git a/example/gluon/tree_lstm/main.py b/example/gluon/tree_lstm/main.py
index f04a69f..67644f9 100644
--- a/example/gluon/tree_lstm/main.py
+++ b/example/gluon/tree_lstm/main.py
@@ -16,7 +16,11 @@
# under the License.
# This example is inspired by https://github.com/dasguptar/treelstm.pytorch
-import argparse, cPickle, math, os, random
+import argparse, math, os, random
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
import logging
logging.basicConfig(level=logging.INFO)
import numpy as np
@@ -66,9 +70,9 @@ random.seed(opt.seed)
batch_size = opt.batch_size
# read dataset
-if os.path.exists('dataset.cPickle'):
- with open('dataset.cPickle', 'rb') as f:
- train_iter, dev_iter, test_iter, vocab = cPickle.load(f)
+if os.path.exists('dataset.pickle'):
+ with open('dataset.pickle', 'rb') as f:
+ train_iter, dev_iter, test_iter, vocab = pickle.load(f)
else:
root_dir = opt.data
segments = ['train', 'dev', 'test']
@@ -80,8 +84,8 @@ else:
train_iter, dev_iter, test_iter = [SICKDataIter(os.path.join(root_dir, segment), vocab, num_classes)
for segment in segments]
- with open('dataset.cPickle', 'wb') as f:
- cPickle.dump([train_iter, dev_iter, test_iter, vocab], f)
+ with open('dataset.pickle', 'wb') as f:
+ pickle.dump([train_iter, dev_iter, test_iter, vocab], f)
logging.info('==> SICK vocabulary size : %d ' % vocab.size)
logging.info('==> Size of train data : %d ' % len(train_iter))
diff --git a/example/rcnn/README.md b/example/rcnn/README.md
index 282a1ae..dbf2a42 100644
--- a/example/rcnn/README.md
+++ b/example/rcnn/README.md
@@ -29,10 +29,8 @@ MXNet engines and parallelization for object detection.
* If you value simplicity. Technical details are *very complicated* in MXNet.
This is by design to attain maximum possible performance instead of patching fixes after fixes.
Performance and parallelization are more than a change of parameter.
-* If you want to do CPU training, be advised that it has not been verified yet.
- You will not encounter NOT_IMPLEMENTED_ERROR so it is still possible.
-* If you are on Windows or Python3, some people reported it was possible with some modifications.
- But they have disappeared.
+* If you want to do CPU training, be advised that it has not been verified properly yet. You can change the `ctx` variable in `train_end2end.py` or `train_alternate.py` scripts to `mx.cpu` and run these scripts directly to test it.
+* If you are on Windows some people reported it was possible with some modifications. But they have disappeared.
## Experiments
| Method | Network | Training Data | Testing Data | Reference | Result |
@@ -48,18 +46,22 @@ MXNet engines and parallelization for object detection.
The above experiments were conducted at [mx-rcnn](https://github.com/precedenceguo/mx-rcnn/tree/6a1ab0eec5035a10a1efb5fc8c9d6c54e101b4d0)
using [a MXNet fork, based on MXNet 0.9.1 nnvm pre-release](https://github.com/precedenceguo/mxnet/tree/simple).
-## I'm Feeling Lucky
+## Quickstart
* Prepare: `bash script/additional_deps.sh`
* Download training data: `bash script/get_voc.sh`
* Download pretrained model: `bash script/get_pretrained_model.sh`
-* Training and testing: `bash script/vgg_voc07.sh 0,1` (use gpu 0 and 1)
+* Training and testing: `bash script/vgg_voc07.sh 0,1` (this means to use gpu 0 and 1)
+
+## Prerequisites
+* Pip, Python-dev, Unzip
+* Some python packages are required: Cython, Scikit-image, Easydict, Matplot, OpenCV, Future
+* On debian, you can usually run `sudo apt install python-pip python-dev unzip`
+* And the python packages can be installed by running `sudo pip install cython scikit-image easydict matplotlib opencv-python future`. Note that you may have to remove sudo depending on how your mxnet package is installed.
+* MXNet version v0.9.5 or higher with Python interface installed. Open `python` type `import mxnet` to confirm.
## Getting started
-See if `bash script/additional_deps.sh` will do the following for you.
* Suppose `HOME` represents where this file is located. All commands, unless stated otherwise, should be started from `HOME`.
-* Install python package `cython easydict matplotlib scikit-image`.
-* Install MXNet version v0.9.5 or higher and MXNet Python Interface. Open `python` type `import mxnet` to confirm.
-* Run `make` in `HOME`.
+* Ensure that `bash script/additional_deps.sh` installs all prerequisites listed above. If you're not using this script, ensure above prerequisities are present on your system and then run `make` from `HOME`. This builds the cython extensions and installs python bindings for them.
Command line arguments have the same meaning as in mxnet/example/image-classification.
* `prefix` refers to the first part of a saved model file name and `epoch` refers to a number in this file name.
diff --git a/example/rcnn/rcnn/core/tester.py b/example/rcnn/rcnn/core/tester.py
index 651b2a9..a451883 100644
--- a/example/rcnn/rcnn/core/tester.py
+++ b/example/rcnn/rcnn/core/tester.py
@@ -15,13 +15,17 @@
# specific language governing permissions and limitations
# under the License.
-import cPickle
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
import os
import time
import mxnet as mx
import numpy as np
+from builtins import range
-from module import MutableModule
+from .module import MutableModule
from rcnn.logger import logger
from rcnn.config import config
from rcnn.io import image
@@ -110,12 +114,12 @@ def generate_proposals(predictor, test_data, imdb, vis=False, thresh=0.):
rpn_file = os.path.join(rpn_folder, imdb.name + '_rpn.pkl')
with open(rpn_file, 'wb') as f:
- cPickle.dump(imdb_boxes, f, cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(imdb_boxes, f, pickle.HIGHEST_PROTOCOL)
if thresh > 0:
full_rpn_file = os.path.join(rpn_folder, imdb.name + '_full_rpn.pkl')
with open(full_rpn_file, 'wb') as f:
- cPickle.dump(original_boxes, f, cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(original_boxes, f, pickle.HIGHEST_PROTOCOL)
logger.info('wrote rpn proposals to %s' % rpn_file)
return imdb_boxes
@@ -168,8 +172,8 @@ def pred_eval(predictor, test_data, imdb, vis=False, thresh=1e-3):
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
- all_boxes = [[[] for _ in xrange(num_images)]
- for _ in xrange(imdb.num_classes)]
+ all_boxes = [[[] for _ in range(num_images)]
+ for _ in range(imdb.num_classes)]
i = 0
t = time.time()
@@ -211,7 +215,7 @@ def pred_eval(predictor, test_data, imdb, vis=False, thresh=1e-3):
det_file = os.path.join(imdb.cache_path, imdb.name + '_detections.pkl')
with open(det_file, 'wb') as f:
- cPickle.dump(all_boxes, f, protocol=cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(all_boxes, f, protocol=pickle.HIGHEST_PROTOCOL)
imdb.evaluate_detections(all_boxes)
diff --git a/example/rcnn/rcnn/cython/setup.py b/example/rcnn/rcnn/cython/setup.py
index e50478b..4ab6f75 100644
--- a/example/rcnn/rcnn/cython/setup.py
+++ b/example/rcnn/rcnn/cython/setup.py
@@ -67,7 +67,7 @@ def locate_cuda():
cudaconfig = {'home':home, 'nvcc':nvcc,
'include': pjoin(home, 'include'),
'lib64': pjoin(home, 'lib64')}
- for k, v in cudaconfig.iteritems():
+ for k, v in cudaconfig.items():
if not os.path.exists(v):
raise EnvironmentError('The CUDA %s path could not be located in %s' % (k, v))
diff --git a/example/rcnn/rcnn/dataset/__init__.py b/example/rcnn/rcnn/dataset/__init__.py
index 1a706e9..80fcc32 100644
--- a/example/rcnn/rcnn/dataset/__init__.py
+++ b/example/rcnn/rcnn/dataset/__init__.py
@@ -15,6 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-from imdb import IMDB
-from pascal_voc import PascalVOC
-from coco import coco
+from .imdb import IMDB
+from .pascal_voc import PascalVOC
+from .coco import coco
diff --git a/example/rcnn/rcnn/dataset/coco.py b/example/rcnn/rcnn/dataset/coco.py
index 9ca5a74..1ec7567 100644
--- a/example/rcnn/rcnn/dataset/coco.py
+++ b/example/rcnn/rcnn/dataset/coco.py
@@ -15,14 +15,18 @@
# specific language governing permissions and limitations
# under the License.
-import cPickle
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
import cv2
import os
import json
import numpy as np
+from builtins import range
from ..logger import logger
-from imdb import IMDB
+from .imdb import IMDB
# coco api
from ..pycocotools.coco import COCO
@@ -47,7 +51,7 @@ class coco(IMDB):
cats = [cat['name'] for cat in self.coco.loadCats(self.coco.getCatIds())]
self.classes = ['__background__'] + cats
self.num_classes = len(self.classes)
- self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes)))
+ self._class_to_ind = dict(zip(self.classes, range(self.num_classes)))
self._class_to_coco_ind = dict(zip(cats, self.coco.getCatIds()))
self._coco_ind_to_class_ind = dict([(self._class_to_coco_ind[cls], self._class_to_ind[cls])
for cls in self.classes[1:]])
@@ -84,13 +88,13 @@ class coco(IMDB):
cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
if os.path.exists(cache_file):
with open(cache_file, 'rb') as fid:
- roidb = cPickle.load(fid)
+ roidb = pickle.load(fid)
logger.info('%s gt roidb loaded from %s' % (self.name, cache_file))
return roidb
gt_roidb = [self._load_coco_annotation(index) for index in self.image_set_index]
with open(cache_file, 'wb') as fid:
- cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
logger.info('%s wrote gt roidb to %s' % (self.name, cache_file))
return gt_roidb
@@ -193,7 +197,7 @@ class coco(IMDB):
result = [{'image_id': index,
'category_id': cat_id,
'bbox': [xs[k], ys[k], ws[k], hs[k]],
- 'score': scores[k]} for k in xrange(dets.shape[0])]
+ 'score': scores[k]} for k in range(dets.shape[0])]
results.extend(result)
return results
@@ -208,7 +212,7 @@ class coco(IMDB):
eval_file = os.path.join(res_folder, 'detections_%s_results.pkl' % self.image_set)
with open(eval_file, 'wb') as f:
- cPickle.dump(coco_eval, f, cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(coco_eval, f, pickle.HIGHEST_PROTOCOL)
logger.info('eval results saved to %s' % eval_file)
def _print_detection_metrics(self, coco_eval):
diff --git a/example/rcnn/rcnn/dataset/ds_utils.py b/example/rcnn/rcnn/dataset/ds_utils.py
index e6f839b..8f90e8d 100644
--- a/example/rcnn/rcnn/dataset/ds_utils.py
+++ b/example/rcnn/rcnn/dataset/ds_utils.py
@@ -21,7 +21,7 @@ import numpy as np
def unique_boxes(boxes, scale=1.0):
""" return indices of unique boxes """
v = np.array([1, 1e3, 1e6, 1e9])
- hashes = np.round(boxes * scale).dot(v)
+ hashes = np.round(boxes * scale).dot(v).astype(np.int)
_, index = np.unique(hashes, return_index=True)
return np.sort(index)
diff --git a/example/rcnn/rcnn/dataset/imdb.py b/example/rcnn/rcnn/dataset/imdb.py
index b9038c5..5908cc3 100644
--- a/example/rcnn/rcnn/dataset/imdb.py
+++ b/example/rcnn/rcnn/dataset/imdb.py
@@ -28,7 +28,10 @@ basic format [image_index]
from ..logger import logger
import os
-import cPickle
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
import numpy as np
from ..processing.bbox_transform import bbox_overlaps
@@ -90,7 +93,7 @@ class IMDB(object):
assert os.path.exists(rpn_file), '%s rpn data not found at %s' % (self.name, rpn_file)
logger.info('%s loading rpn data from %s' % (self.name, rpn_file))
with open(rpn_file, 'rb') as f:
- box_list = cPickle.load(f)
+ box_list = pickle.load(f)
return box_list
def load_rpn_roidb(self, gt_roidb):
diff --git a/example/rcnn/rcnn/dataset/pascal_voc.py b/example/rcnn/rcnn/dataset/pascal_voc.py
index 091c4e8..753f703 100644
--- a/example/rcnn/rcnn/dataset/pascal_voc.py
+++ b/example/rcnn/rcnn/dataset/pascal_voc.py
@@ -23,15 +23,18 @@ function. Results are written as the Pascal VOC format. Evaluation is based on m
criterion.
"""
-import cPickle
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
import cv2
import os
import numpy as np
from ..logger import logger
-from imdb import IMDB
-from pascal_voc_eval import voc_eval
-from ds_utils import unique_boxes, filter_small_boxes
+from .imdb import IMDB
+from .pascal_voc_eval import voc_eval
+from .ds_utils import unique_boxes, filter_small_boxes
class PascalVOC(IMDB):
@@ -94,13 +97,13 @@ class PascalVOC(IMDB):
cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
if os.path.exists(cache_file):
with open(cache_file, 'rb') as fid:
- roidb = cPickle.load(fid)
+ roidb = pickle.load(fid)
logger.info('%s gt roidb loaded from %s' % (self.name, cache_file))
return roidb
gt_roidb = [self.load_pascal_annotation(index) for index in self.image_set_index]
with open(cache_file, 'wb') as fid:
- cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
logger.info('%s wrote gt roidb to %s' % (self.name, cache_file))
return gt_roidb
@@ -184,7 +187,7 @@ class PascalVOC(IMDB):
cache_file = os.path.join(self.cache_path, self.name + '_ss_roidb.pkl')
if os.path.exists(cache_file):
with open(cache_file, 'rb') as fid:
- roidb = cPickle.load(fid)
+ roidb = pickle.load(fid)
logger.info('%s ss roidb loaded from %s' % (self.name, cache_file))
return roidb
@@ -195,7 +198,7 @@ class PascalVOC(IMDB):
else:
roidb = self.load_selective_search_roidb(gt_roidb)
with open(cache_file, 'wb') as fid:
- cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(roidb, fid, pickle.HIGHEST_PROTOCOL)
logger.info('%s wrote ss roidb to %s' % (self.name, cache_file))
return roidb
diff --git a/example/rcnn/rcnn/dataset/pascal_voc_eval.py b/example/rcnn/rcnn/dataset/pascal_voc_eval.py
index e584ed7..2583aed 100644
--- a/example/rcnn/rcnn/dataset/pascal_voc_eval.py
+++ b/example/rcnn/rcnn/dataset/pascal_voc_eval.py
@@ -22,8 +22,10 @@ given a pascal voc imdb, compute mAP
from ..logger import logger
import numpy as np
import os
-import cPickle
-
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
def parse_voc_rec(filename):
"""
@@ -106,10 +108,10 @@ def voc_eval(detpath, annopath, imageset_file, classname, annocache, ovthresh=0.
logger.info('reading annotations for %d/%d' % (ind + 1, len(image_filenames)))
logger.info('saving annotations cache to %s' % annocache)
with open(annocache, 'wb') as f:
- cPickle.dump(recs, f, protocol=cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(recs, f, protocol=pickle.HIGHEST_PROTOCOL)
else:
with open(annocache, 'rb') as f:
- recs = cPickle.load(f)
+ recs = pickle.load(f)
# extract objects in :param classname:
class_recs = {}
diff --git a/example/rcnn/rcnn/io/rcnn.py b/example/rcnn/rcnn/io/rcnn.py
index f9613d6..d11c7ca 100644
--- a/example/rcnn/rcnn/io/rcnn.py
+++ b/example/rcnn/rcnn/io/rcnn.py
@@ -75,7 +75,7 @@ def get_rcnn_batch(roidb):
assert config.TRAIN.BATCH_ROIS % config.TRAIN.BATCH_IMAGES == 0, \
'BATCHIMAGES {} must divide BATCH_ROIS {}'.format(config.TRAIN.BATCH_IMAGES, config.TRAIN.BATCH_ROIS)
rois_per_image = config.TRAIN.BATCH_ROIS / config.TRAIN.BATCH_IMAGES
- fg_rois_per_image = np.round(config.TRAIN.FG_FRACTION * rois_per_image).astype(int)
+ fg_rois_per_image = np.round(config.TRAIN.FG_FRACTION * rois_per_image).astype(np.int)
rois_array = list()
labels_array = list()
@@ -147,7 +147,7 @@ def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes,
# foreground RoI with FG_THRESH overlap
fg_indexes = np.where(overlaps >= config.TRAIN.FG_THRESH)[0]
# guard against the case when an image has fewer than fg_rois_per_image foreground RoIs
- fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size)
+ fg_rois_per_this_image = int(np.minimum(fg_rois_per_image, fg_indexes.size))
# Sample foreground regions without replacement
if len(fg_indexes) > fg_rois_per_this_image:
fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False)
@@ -156,7 +156,7 @@ def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes,
bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI) & (overlaps >= config.TRAIN.BG_THRESH_LO))[0]
# Compute number of background RoIs to take from this image (guarding against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
- bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size)
+ bg_rois_per_this_image = int(np.minimum(bg_rois_per_this_image, bg_indexes.size))
# Sample foreground regions without replacement
if len(bg_indexes) > bg_rois_per_this_image:
bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False)
diff --git a/example/rcnn/rcnn/processing/bbox_regression.py b/example/rcnn/rcnn/processing/bbox_regression.py
index d5330f4..24812ac 100644
--- a/example/rcnn/rcnn/processing/bbox_regression.py
+++ b/example/rcnn/rcnn/processing/bbox_regression.py
@@ -22,7 +22,7 @@ This file has functions about generating bounding box regression targets
import numpy as np
from ..logger import logger
-from bbox_transform import bbox_overlaps, bbox_transform
+from .bbox_transform import bbox_overlaps, bbox_transform
from rcnn.config import config
diff --git a/example/rcnn/rcnn/processing/generate_anchor.py b/example/rcnn/rcnn/processing/generate_anchor.py
index 0e97d6e..53c280d 100644
--- a/example/rcnn/rcnn/processing/generate_anchor.py
+++ b/example/rcnn/rcnn/processing/generate_anchor.py
@@ -18,7 +18,7 @@
"""
Generate base anchors on index 0
"""
-
+from builtins import range
import numpy as np
@@ -32,7 +32,7 @@ def generate_anchors(base_size=16, ratios=[0.5, 1, 2],
base_anchor = np.array([1, 1, base_size, base_size]) - 1
ratio_anchors = _ratio_enum(base_anchor, ratios)
anchors = np.vstack([_scale_enum(ratio_anchors[i, :], scales)
- for i in xrange(ratio_anchors.shape[0])])
+ for i in range(ratio_anchors.shape[0])])
return anchors
diff --git a/example/rcnn/rcnn/pycocotools/cocoeval.py b/example/rcnn/rcnn/pycocotools/cocoeval.py
index 8b78026..e1d181b 100644
--- a/example/rcnn/rcnn/pycocotools/cocoeval.py
+++ b/example/rcnn/rcnn/pycocotools/cocoeval.py
@@ -21,7 +21,7 @@ import numpy as np
import datetime
import time
from collections import defaultdict
-import mask as maskUtils
+from .mask import *
import copy
class COCOeval:
@@ -204,7 +204,7 @@ class COCOeval:
# compute iou between each dt and gt region
iscrowd = [int(o['iscrowd']) for o in gt]
- ious = maskUtils.iou(d,g,iscrowd)
+ ious = iou(d,g,iscrowd)
return ious
def computeOks(self, imgId, catId):
diff --git a/example/rcnn/rcnn/pycocotools/mask.py b/example/rcnn/rcnn/pycocotools/mask.py
index 48c050c..2122468 100644
--- a/example/rcnn/rcnn/pycocotools/mask.py
+++ b/example/rcnn/rcnn/pycocotools/mask.py
@@ -17,7 +17,7 @@
__author__ = 'tsungyi'
-import _mask
+from rcnn.pycocotools import _mask
# Interface for manipulating masks stored in RLE format.
#
diff --git a/example/rcnn/rcnn/symbol/__init__.py b/example/rcnn/rcnn/symbol/__init__.py
index 113b52c..7547122 100644
--- a/example/rcnn/rcnn/symbol/__init__.py
+++ b/example/rcnn/rcnn/symbol/__init__.py
@@ -15,5 +15,5 @@
# specific language governing permissions and limitations
# under the License.
-from symbol_vgg import *
-from symbol_resnet import *
+from .symbol_vgg import *
+from .symbol_resnet import *
diff --git a/example/rcnn/rcnn/symbol/proposal_target.py b/example/rcnn/rcnn/symbol/proposal_target.py
index e0444f9..0af19a9 100644
--- a/example/rcnn/rcnn/symbol/proposal_target.py
+++ b/example/rcnn/rcnn/symbol/proposal_target.py
@@ -45,7 +45,7 @@ class ProposalTargetOperator(mx.operator.CustomOp):
assert self._batch_rois % self._batch_images == 0, \
'BATCHIMAGES {} must devide BATCH_ROIS {}'.format(self._batch_images, self._batch_rois)
rois_per_image = self._batch_rois / self._batch_images
- fg_rois_per_image = np.round(self._fg_fraction * rois_per_image).astype(int)
+ fg_rois_per_image = np.round(self._fg_fraction * rois_per_image).astype(np.int)
all_rois = in_data[0].asnumpy()
gt_boxes = in_data[1].asnumpy()
diff --git a/example/rcnn/rcnn/symbol/symbol_resnet.py b/example/rcnn/rcnn/symbol/symbol_resnet.py
index 4a9677d..f772136 100644
--- a/example/rcnn/rcnn/symbol/symbol_resnet.py
+++ b/example/rcnn/rcnn/symbol/symbol_resnet.py
@@ -16,9 +16,9 @@
# under the License.
import mxnet as mx
-import proposal
-import proposal_target
from rcnn.config import config
+from . import proposal
+from . import proposal_target
eps = 2e-5
use_global_stats = True
diff --git a/example/rcnn/rcnn/symbol/symbol_vgg.py b/example/rcnn/rcnn/symbol/symbol_vgg.py
index 00ba15e..33fbede 100644
--- a/example/rcnn/rcnn/symbol/symbol_vgg.py
+++ b/example/rcnn/rcnn/symbol/symbol_vgg.py
@@ -16,10 +16,9 @@
# under the License.
import mxnet as mx
-import proposal
-import proposal_target
from rcnn.config import config
-
+from . import proposal
+from . import proposal_target
def get_vgg_conv(data):
"""
diff --git a/example/rcnn/rcnn/tools/reeval.py b/example/rcnn/rcnn/tools/reeval.py
index a7ae898..1e5c0aa 100644
--- a/example/rcnn/rcnn/tools/reeval.py
+++ b/example/rcnn/rcnn/tools/reeval.py
@@ -16,7 +16,10 @@
# under the License.
import argparse
-import cPickle
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
import os
import mxnet as mx
@@ -32,7 +35,7 @@ def reeval(args):
# load detection results
cache_file = os.path.join(imdb.cache_path, imdb.name, 'detections.pkl')
with open(cache_file) as f:
- detections = cPickle.load(f)
+ detections = pickle.load(f)
# eval
imdb.evaluate_detections(detections)
diff --git a/example/rcnn/script/additional_deps.sh b/example/rcnn/script/additional_deps.sh
index 0e6599c..cddc391 100755
--- a/example/rcnn/script/additional_deps.sh
+++ b/example/rcnn/script/additional_deps.sh
@@ -20,19 +20,7 @@
# install additional depts
sudo apt install python-pip python-dev unzip python-matplotlib
-sudo pip install cython scikit-image easydict
-
-# install a forked MXNet
-pushd ../../
-cp make/config.mk ./
-echo "USE_CUDA=1" >>config.mk
-echo "USE_CUDA_PATH=/usr/local/cuda" >>config.mk
-echo "USE_CUDNN=1" >>config.mk
-make -j$(nproc)
-pushd python
-python setup.py install --user
-popd
-popd
+sudo pip install cython scikit-image easydict opencv-python
# build cython extension
make
diff --git a/example/sparse/linear_classification/weighted_softmax_ce.py b/example/sparse/linear_classification/weighted_softmax_ce.py
index a40ece6..f781e6a 100644
--- a/example/sparse/linear_classification/weighted_softmax_ce.py
+++ b/example/sparse/linear_classification/weighted_softmax_ce.py
@@ -61,7 +61,7 @@ class WeightedSoftmaxCrossEntropyLossProp(mx.operator.CustomOpProp):
def __init__(self, positive_cls_weight):
super(WeightedSoftmaxCrossEntropyLossProp, self).__init__(True)
self.positive_cls_weight = positive_cls_weight
- assert(positive_cls_weight > 0)
+ assert(float(positive_cls_weight) > 0)
def list_arguments(self):
return ['data', 'label']
diff --git a/example/sparse/matrix_factorization/data.py b/example/sparse/matrix_factorization/data.py
index fae2c23..c897165 100644
--- a/example/sparse/matrix_factorization/data.py
+++ b/example/sparse/matrix_factorization/data.py
@@ -15,17 +15,17 @@
# specific language governing permissions and limitations
# under the License.
-import os, gzip
-import sys
+import os
import mxnet as mx
from mxnet.test_utils import DummyIter
-def get_movielens_data(prefix):
- if not os.path.exists("%s.zip" % prefix):
- print("Dataset MovieLens 10M not present. Downloading now ...")
- os.system("wget http://files.grouplens.org/datasets/movielens/%s.zip" % prefix)
- os.system("unzip %s.zip" % prefix)
- os.system("cd ml-10M100K; sh split_ratings.sh; cd -;")
+def get_movielens_data(data_dir, prefix):
+ if not os.path.exists(os.path.join(data_dir, "ml-10M100K")):
+ mx.test_utils.get_zip_data(data_dir,
+ "http://files.grouplens.org/datasets/movielens/%s.zip" % prefix,
+ prefix + ".zip")
+ assert os.path.exists(os.path.join(data_dir, "ml-10M100K"))
+ os.system("cd data/ml-10M100K; chmod +x allbut.pl; sh split_ratings.sh; cd -;")
def get_movielens_iter(filename, batch_size, dummy_iter):
"""Not particularly fast code to parse the text file and load into NDArrays.
@@ -58,3 +58,5 @@ def get_movielens_iter(filename, batch_size, dummy_iter):
batch_size=batch_size, shuffle=True)
iter_train = DummyIter(iter_train) if dummy_iter else iter_train
return mx.io.PrefetchingIter(iter_train)
+
+
diff --git a/example/sparse/matrix_factorization/train.py b/example/sparse/matrix_factorization/train.py
index 14c6ca1..0db58ad 100644
--- a/example/sparse/matrix_factorization/train.py
+++ b/example/sparse/matrix_factorization/train.py
@@ -17,11 +17,11 @@
import argparse
import logging
-import time
import mxnet as mx
import numpy as np
from data import get_movielens_iter, get_movielens_data
from model import matrix_fact_net
+import os
logging.basicConfig(level=logging.DEBUG)
@@ -44,8 +44,8 @@ parser.add_argument('--dummy-iter', action='store_true',
MOVIELENS = {
'dataset': 'ml-10m',
- 'train': './ml-10M100K/r1.train',
- 'val': './ml-10M100K/r1.test',
+ 'train': './data/ml-10M100K/r1.train',
+ 'val': './data/ml-10M100K/r1.test',
'max_user': 71569,
'max_movie': 65135,
}
@@ -72,7 +72,8 @@ if __name__ == '__main__':
# prepare dataset and iterators
max_user = MOVIELENS['max_user']
max_movies = MOVIELENS['max_movie']
- get_movielens_data(MOVIELENS['dataset'])
+ data_dir = os.path.join(os.getcwd(), 'data')
+ get_movielens_data(data_dir, MOVIELENS['dataset'])
train_iter = get_movielens_iter(MOVIELENS['train'], batch_size, dummy_iter)
val_iter = get_movielens_iter(MOVIELENS['val'], batch_size, dummy_iter)
diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py
index 0dfeec5..53814b7 100644
--- a/python/mxnet/test_utils.py
+++ b/python/mxnet/test_utils.py
@@ -29,6 +29,7 @@ import os
import errno
import logging
import bz2
+import zipfile
from contextlib import contextmanager
import numpy as np
import numpy.testing as npt
@@ -1440,6 +1441,31 @@ def get_mnist():
return {'train_data':train_img, 'train_label':train_lbl,
'test_data':test_img, 'test_label':test_lbl}
+def get_zip_data(data_dir, url, data_origin_name):
+ """Download and extract zip data.
+
+ Parameters
+ ----------
+
+ data_dir : str
+ Absolute or relative path of the directory name to store zip files
+ url : str
+ URL to download data from
+ data_origin_name : str
+ Name of the downloaded zip file
+
+ Examples
+ --------
+ >>> get_zip_data("data_dir",
+ "http://files.grouplens.org/datasets/movielens/ml-10m.zip",
+ "ml-10m.zip")
+ """
+ data_origin_name = os.path.join(data_dir, data_origin_name)
+ if not os.path.exists(data_origin_name):
+ download(url, dirname=data_dir, overwrite=False)
+ zip_file = zipfile.ZipFile(data_origin_name)
+ zip_file.extractall(path=data_dir)
+
def get_bz2_data(data_dir, data_name, url, data_origin_name):
"""Download and extract bz2 data.
@@ -1465,14 +1491,12 @@ def get_bz2_data(data_dir, data_name, url, data_origin_name):
data_name = os.path.join(data_dir, data_name)
data_origin_name = os.path.join(data_dir, data_origin_name)
if not os.path.exists(data_name):
- download(url, dirname=data_dir, overwrite=False)
+ download(url, fname=data_origin_name, dirname=data_dir, overwrite=False)
bz_file = bz2.BZ2File(data_origin_name, 'rb')
with open(data_name, 'wb') as fout:
- try:
- content = bz_file.read()
- fout.write(content)
- finally:
- bz_file.close()
+ for line in bz_file:
+ fout.write(line)
+ bz_file.close()
os.remove(data_origin_name)
def set_env_var(key, val, default_val=""):
--
To stop receiving notification emails like this one, please contact
['"commits@mxnet.apache.org" <co...@mxnet.apache.org>'].