You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2017/12/18 20:28:27 UTC
[GitHub] piiswrong closed pull request #8993: Fix compatibility of examples with py3
piiswrong closed pull request #8993: Fix compatibility of examples with py3
URL: https://github.com/apache/incubator-mxnet/pull/8993
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/example/gluon/tree_lstm/main.py b/example/gluon/tree_lstm/main.py
index f04a69f267..67644f97d3 100644
--- a/example/gluon/tree_lstm/main.py
+++ b/example/gluon/tree_lstm/main.py
@@ -16,7 +16,11 @@
# under the License.
# This example is inspired by https://github.com/dasguptar/treelstm.pytorch
-import argparse, cPickle, math, os, random
+import argparse, math, os, random
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
import logging
logging.basicConfig(level=logging.INFO)
import numpy as np
@@ -66,9 +70,9 @@
batch_size = opt.batch_size
# read dataset
-if os.path.exists('dataset.cPickle'):
- with open('dataset.cPickle', 'rb') as f:
- train_iter, dev_iter, test_iter, vocab = cPickle.load(f)
+if os.path.exists('dataset.pickle'):
+ with open('dataset.pickle', 'rb') as f:
+ train_iter, dev_iter, test_iter, vocab = pickle.load(f)
else:
root_dir = opt.data
segments = ['train', 'dev', 'test']
@@ -80,8 +84,8 @@
train_iter, dev_iter, test_iter = [SICKDataIter(os.path.join(root_dir, segment), vocab, num_classes)
for segment in segments]
- with open('dataset.cPickle', 'wb') as f:
- cPickle.dump([train_iter, dev_iter, test_iter, vocab], f)
+ with open('dataset.pickle', 'wb') as f:
+ pickle.dump([train_iter, dev_iter, test_iter, vocab], f)
logging.info('==> SICK vocabulary size : %d ' % vocab.size)
logging.info('==> Size of train data : %d ' % len(train_iter))
diff --git a/example/rcnn/README.md b/example/rcnn/README.md
index 282a1aebe9..dbf2a423c3 100644
--- a/example/rcnn/README.md
+++ b/example/rcnn/README.md
@@ -29,10 +29,8 @@ MXNet engines and parallelization for object detection.
* If you value simplicity. Technical details are *very complicated* in MXNet.
This is by design to attain maximum possible performance instead of patching fixes after fixes.
Performance and parallelization are more than a change of parameter.
-* If you want to do CPU training, be advised that it has not been verified yet.
- You will not encounter NOT_IMPLEMENTED_ERROR so it is still possible.
-* If you are on Windows or Python3, some people reported it was possible with some modifications.
- But they have disappeared.
+* If you want to do CPU training, be advised that it has not been verified properly yet. You can change the `ctx` variable in `train_end2end.py` or `train_alternate.py` scripts to `mx.cpu` and run these scripts directly to test it.
+* If you are on Windows some people reported it was possible with some modifications. But they have disappeared.
## Experiments
| Method | Network | Training Data | Testing Data | Reference | Result |
@@ -48,18 +46,22 @@ MXNet engines and parallelization for object detection.
The above experiments were conducted at [mx-rcnn](https://github.com/precedenceguo/mx-rcnn/tree/6a1ab0eec5035a10a1efb5fc8c9d6c54e101b4d0)
using [a MXNet fork, based on MXNet 0.9.1 nnvm pre-release](https://github.com/precedenceguo/mxnet/tree/simple).
-## I'm Feeling Lucky
+## Quickstart
* Prepare: `bash script/additional_deps.sh`
* Download training data: `bash script/get_voc.sh`
* Download pretrained model: `bash script/get_pretrained_model.sh`
-* Training and testing: `bash script/vgg_voc07.sh 0,1` (use gpu 0 and 1)
+* Training and testing: `bash script/vgg_voc07.sh 0,1` (this means to use gpu 0 and 1)
+
+## Prerequisites
+* Pip, Python-dev, Unzip
+* Some python packages are required: Cython, Scikit-image, Easydict, Matplot, OpenCV, Future
+* On debian, you can usually run `sudo apt install python-pip python-dev unzip`
+* And the python packages can be installed by running `sudo pip install cython scikit-image easydict matplotlib opencv-python future`. Note that you may have to remove sudo depending on how your mxnet package is installed.
+* MXNet version v0.9.5 or higher with Python interface installed. Open `python` type `import mxnet` to confirm.
## Getting started
-See if `bash script/additional_deps.sh` will do the following for you.
* Suppose `HOME` represents where this file is located. All commands, unless stated otherwise, should be started from `HOME`.
-* Install python package `cython easydict matplotlib scikit-image`.
-* Install MXNet version v0.9.5 or higher and MXNet Python Interface. Open `python` type `import mxnet` to confirm.
-* Run `make` in `HOME`.
+* Ensure that `bash script/additional_deps.sh` installs all prerequisites listed above. If you're not using this script, ensure above prerequisities are present on your system and then run `make` from `HOME`. This builds the cython extensions and installs python bindings for them.
Command line arguments have the same meaning as in mxnet/example/image-classification.
* `prefix` refers to the first part of a saved model file name and `epoch` refers to a number in this file name.
diff --git a/example/rcnn/rcnn/core/tester.py b/example/rcnn/rcnn/core/tester.py
index 651b2a945e..a451883f58 100644
--- a/example/rcnn/rcnn/core/tester.py
+++ b/example/rcnn/rcnn/core/tester.py
@@ -15,13 +15,17 @@
# specific language governing permissions and limitations
# under the License.
-import cPickle
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
import os
import time
import mxnet as mx
import numpy as np
+from builtins import range
-from module import MutableModule
+from .module import MutableModule
from rcnn.logger import logger
from rcnn.config import config
from rcnn.io import image
@@ -110,12 +114,12 @@ def generate_proposals(predictor, test_data, imdb, vis=False, thresh=0.):
rpn_file = os.path.join(rpn_folder, imdb.name + '_rpn.pkl')
with open(rpn_file, 'wb') as f:
- cPickle.dump(imdb_boxes, f, cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(imdb_boxes, f, pickle.HIGHEST_PROTOCOL)
if thresh > 0:
full_rpn_file = os.path.join(rpn_folder, imdb.name + '_full_rpn.pkl')
with open(full_rpn_file, 'wb') as f:
- cPickle.dump(original_boxes, f, cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(original_boxes, f, pickle.HIGHEST_PROTOCOL)
logger.info('wrote rpn proposals to %s' % rpn_file)
return imdb_boxes
@@ -168,8 +172,8 @@ def pred_eval(predictor, test_data, imdb, vis=False, thresh=1e-3):
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
- all_boxes = [[[] for _ in xrange(num_images)]
- for _ in xrange(imdb.num_classes)]
+ all_boxes = [[[] for _ in range(num_images)]
+ for _ in range(imdb.num_classes)]
i = 0
t = time.time()
@@ -211,7 +215,7 @@ def pred_eval(predictor, test_data, imdb, vis=False, thresh=1e-3):
det_file = os.path.join(imdb.cache_path, imdb.name + '_detections.pkl')
with open(det_file, 'wb') as f:
- cPickle.dump(all_boxes, f, protocol=cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(all_boxes, f, protocol=pickle.HIGHEST_PROTOCOL)
imdb.evaluate_detections(all_boxes)
diff --git a/example/rcnn/rcnn/cython/setup.py b/example/rcnn/rcnn/cython/setup.py
index e50478b2d9..4ab6f75cc6 100644
--- a/example/rcnn/rcnn/cython/setup.py
+++ b/example/rcnn/rcnn/cython/setup.py
@@ -67,7 +67,7 @@ def locate_cuda():
cudaconfig = {'home':home, 'nvcc':nvcc,
'include': pjoin(home, 'include'),
'lib64': pjoin(home, 'lib64')}
- for k, v in cudaconfig.iteritems():
+ for k, v in cudaconfig.items():
if not os.path.exists(v):
raise EnvironmentError('The CUDA %s path could not be located in %s' % (k, v))
diff --git a/example/rcnn/rcnn/dataset/__init__.py b/example/rcnn/rcnn/dataset/__init__.py
index 1a706e9e0c..80fcc32c21 100644
--- a/example/rcnn/rcnn/dataset/__init__.py
+++ b/example/rcnn/rcnn/dataset/__init__.py
@@ -15,6 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-from imdb import IMDB
-from pascal_voc import PascalVOC
-from coco import coco
+from .imdb import IMDB
+from .pascal_voc import PascalVOC
+from .coco import coco
diff --git a/example/rcnn/rcnn/dataset/coco.py b/example/rcnn/rcnn/dataset/coco.py
index 9ca5a74cc4..1ec7567958 100644
--- a/example/rcnn/rcnn/dataset/coco.py
+++ b/example/rcnn/rcnn/dataset/coco.py
@@ -15,14 +15,18 @@
# specific language governing permissions and limitations
# under the License.
-import cPickle
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
import cv2
import os
import json
import numpy as np
+from builtins import range
from ..logger import logger
-from imdb import IMDB
+from .imdb import IMDB
# coco api
from ..pycocotools.coco import COCO
@@ -47,7 +51,7 @@ def __init__(self, image_set, root_path, data_path):
cats = [cat['name'] for cat in self.coco.loadCats(self.coco.getCatIds())]
self.classes = ['__background__'] + cats
self.num_classes = len(self.classes)
- self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes)))
+ self._class_to_ind = dict(zip(self.classes, range(self.num_classes)))
self._class_to_coco_ind = dict(zip(cats, self.coco.getCatIds()))
self._coco_ind_to_class_ind = dict([(self._class_to_coco_ind[cls], self._class_to_ind[cls])
for cls in self.classes[1:]])
@@ -84,13 +88,13 @@ def gt_roidb(self):
cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
if os.path.exists(cache_file):
with open(cache_file, 'rb') as fid:
- roidb = cPickle.load(fid)
+ roidb = pickle.load(fid)
logger.info('%s gt roidb loaded from %s' % (self.name, cache_file))
return roidb
gt_roidb = [self._load_coco_annotation(index) for index in self.image_set_index]
with open(cache_file, 'wb') as fid:
- cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
logger.info('%s wrote gt roidb to %s' % (self.name, cache_file))
return gt_roidb
@@ -193,7 +197,7 @@ def _coco_results_one_category(self, boxes, cat_id):
result = [{'image_id': index,
'category_id': cat_id,
'bbox': [xs[k], ys[k], ws[k], hs[k]],
- 'score': scores[k]} for k in xrange(dets.shape[0])]
+ 'score': scores[k]} for k in range(dets.shape[0])]
results.extend(result)
return results
@@ -208,7 +212,7 @@ def _do_python_eval(self, res_file, res_folder):
eval_file = os.path.join(res_folder, 'detections_%s_results.pkl' % self.image_set)
with open(eval_file, 'wb') as f:
- cPickle.dump(coco_eval, f, cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(coco_eval, f, pickle.HIGHEST_PROTOCOL)
logger.info('eval results saved to %s' % eval_file)
def _print_detection_metrics(self, coco_eval):
diff --git a/example/rcnn/rcnn/dataset/ds_utils.py b/example/rcnn/rcnn/dataset/ds_utils.py
index e6f839b8fd..8f90e8d390 100644
--- a/example/rcnn/rcnn/dataset/ds_utils.py
+++ b/example/rcnn/rcnn/dataset/ds_utils.py
@@ -21,7 +21,7 @@
def unique_boxes(boxes, scale=1.0):
""" return indices of unique boxes """
v = np.array([1, 1e3, 1e6, 1e9])
- hashes = np.round(boxes * scale).dot(v)
+ hashes = np.round(boxes * scale).dot(v).astype(np.int)
_, index = np.unique(hashes, return_index=True)
return np.sort(index)
diff --git a/example/rcnn/rcnn/dataset/imdb.py b/example/rcnn/rcnn/dataset/imdb.py
index b9038c5da0..5908cc3358 100644
--- a/example/rcnn/rcnn/dataset/imdb.py
+++ b/example/rcnn/rcnn/dataset/imdb.py
@@ -28,7 +28,10 @@
from ..logger import logger
import os
-import cPickle
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
import numpy as np
from ..processing.bbox_transform import bbox_overlaps
@@ -90,7 +93,7 @@ def load_rpn_data(self, full=False):
assert os.path.exists(rpn_file), '%s rpn data not found at %s' % (self.name, rpn_file)
logger.info('%s loading rpn data from %s' % (self.name, rpn_file))
with open(rpn_file, 'rb') as f:
- box_list = cPickle.load(f)
+ box_list = pickle.load(f)
return box_list
def load_rpn_roidb(self, gt_roidb):
diff --git a/example/rcnn/rcnn/dataset/pascal_voc.py b/example/rcnn/rcnn/dataset/pascal_voc.py
index 091c4e8ea1..753f7038aa 100644
--- a/example/rcnn/rcnn/dataset/pascal_voc.py
+++ b/example/rcnn/rcnn/dataset/pascal_voc.py
@@ -23,15 +23,18 @@
criterion.
"""
-import cPickle
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
import cv2
import os
import numpy as np
from ..logger import logger
-from imdb import IMDB
-from pascal_voc_eval import voc_eval
-from ds_utils import unique_boxes, filter_small_boxes
+from .imdb import IMDB
+from .pascal_voc_eval import voc_eval
+from .ds_utils import unique_boxes, filter_small_boxes
class PascalVOC(IMDB):
@@ -94,13 +97,13 @@ def gt_roidb(self):
cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
if os.path.exists(cache_file):
with open(cache_file, 'rb') as fid:
- roidb = cPickle.load(fid)
+ roidb = pickle.load(fid)
logger.info('%s gt roidb loaded from %s' % (self.name, cache_file))
return roidb
gt_roidb = [self.load_pascal_annotation(index) for index in self.image_set_index]
with open(cache_file, 'wb') as fid:
- cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
logger.info('%s wrote gt roidb to %s' % (self.name, cache_file))
return gt_roidb
@@ -184,7 +187,7 @@ def selective_search_roidb(self, gt_roidb, append_gt=False):
cache_file = os.path.join(self.cache_path, self.name + '_ss_roidb.pkl')
if os.path.exists(cache_file):
with open(cache_file, 'rb') as fid:
- roidb = cPickle.load(fid)
+ roidb = pickle.load(fid)
logger.info('%s ss roidb loaded from %s' % (self.name, cache_file))
return roidb
@@ -195,7 +198,7 @@ def selective_search_roidb(self, gt_roidb, append_gt=False):
else:
roidb = self.load_selective_search_roidb(gt_roidb)
with open(cache_file, 'wb') as fid:
- cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(roidb, fid, pickle.HIGHEST_PROTOCOL)
logger.info('%s wrote ss roidb to %s' % (self.name, cache_file))
return roidb
diff --git a/example/rcnn/rcnn/dataset/pascal_voc_eval.py b/example/rcnn/rcnn/dataset/pascal_voc_eval.py
index e584ed7503..2583aed166 100644
--- a/example/rcnn/rcnn/dataset/pascal_voc_eval.py
+++ b/example/rcnn/rcnn/dataset/pascal_voc_eval.py
@@ -22,8 +22,10 @@
from ..logger import logger
import numpy as np
import os
-import cPickle
-
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
def parse_voc_rec(filename):
"""
@@ -106,10 +108,10 @@ def voc_eval(detpath, annopath, imageset_file, classname, annocache, ovthresh=0.
logger.info('reading annotations for %d/%d' % (ind + 1, len(image_filenames)))
logger.info('saving annotations cache to %s' % annocache)
with open(annocache, 'wb') as f:
- cPickle.dump(recs, f, protocol=cPickle.HIGHEST_PROTOCOL)
+ pickle.dump(recs, f, protocol=pickle.HIGHEST_PROTOCOL)
else:
with open(annocache, 'rb') as f:
- recs = cPickle.load(f)
+ recs = pickle.load(f)
# extract objects in :param classname:
class_recs = {}
diff --git a/example/rcnn/rcnn/io/rcnn.py b/example/rcnn/rcnn/io/rcnn.py
index f9613d68bd..d11c7cadac 100644
--- a/example/rcnn/rcnn/io/rcnn.py
+++ b/example/rcnn/rcnn/io/rcnn.py
@@ -75,7 +75,7 @@ def get_rcnn_batch(roidb):
assert config.TRAIN.BATCH_ROIS % config.TRAIN.BATCH_IMAGES == 0, \
'BATCHIMAGES {} must divide BATCH_ROIS {}'.format(config.TRAIN.BATCH_IMAGES, config.TRAIN.BATCH_ROIS)
rois_per_image = config.TRAIN.BATCH_ROIS / config.TRAIN.BATCH_IMAGES
- fg_rois_per_image = np.round(config.TRAIN.FG_FRACTION * rois_per_image).astype(int)
+ fg_rois_per_image = np.round(config.TRAIN.FG_FRACTION * rois_per_image).astype(np.int)
rois_array = list()
labels_array = list()
@@ -147,7 +147,7 @@ def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes,
# foreground RoI with FG_THRESH overlap
fg_indexes = np.where(overlaps >= config.TRAIN.FG_THRESH)[0]
# guard against the case when an image has fewer than fg_rois_per_image foreground RoIs
- fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size)
+ fg_rois_per_this_image = int(np.minimum(fg_rois_per_image, fg_indexes.size))
# Sample foreground regions without replacement
if len(fg_indexes) > fg_rois_per_this_image:
fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False)
@@ -156,7 +156,7 @@ def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes,
bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI) & (overlaps >= config.TRAIN.BG_THRESH_LO))[0]
# Compute number of background RoIs to take from this image (guarding against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
- bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size)
+ bg_rois_per_this_image = int(np.minimum(bg_rois_per_this_image, bg_indexes.size))
# Sample foreground regions without replacement
if len(bg_indexes) > bg_rois_per_this_image:
bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False)
diff --git a/example/rcnn/rcnn/processing/bbox_regression.py b/example/rcnn/rcnn/processing/bbox_regression.py
index d5330f4098..24812ac2bd 100644
--- a/example/rcnn/rcnn/processing/bbox_regression.py
+++ b/example/rcnn/rcnn/processing/bbox_regression.py
@@ -22,7 +22,7 @@
import numpy as np
from ..logger import logger
-from bbox_transform import bbox_overlaps, bbox_transform
+from .bbox_transform import bbox_overlaps, bbox_transform
from rcnn.config import config
diff --git a/example/rcnn/rcnn/processing/generate_anchor.py b/example/rcnn/rcnn/processing/generate_anchor.py
index 0e97d6ef2b..53c280dd45 100644
--- a/example/rcnn/rcnn/processing/generate_anchor.py
+++ b/example/rcnn/rcnn/processing/generate_anchor.py
@@ -18,7 +18,7 @@
"""
Generate base anchors on index 0
"""
-
+from builtins import range
import numpy as np
@@ -32,7 +32,7 @@ def generate_anchors(base_size=16, ratios=[0.5, 1, 2],
base_anchor = np.array([1, 1, base_size, base_size]) - 1
ratio_anchors = _ratio_enum(base_anchor, ratios)
anchors = np.vstack([_scale_enum(ratio_anchors[i, :], scales)
- for i in xrange(ratio_anchors.shape[0])])
+ for i in range(ratio_anchors.shape[0])])
return anchors
diff --git a/example/rcnn/rcnn/pycocotools/cocoeval.py b/example/rcnn/rcnn/pycocotools/cocoeval.py
index 8b78026d39..e1d181b5bc 100644
--- a/example/rcnn/rcnn/pycocotools/cocoeval.py
+++ b/example/rcnn/rcnn/pycocotools/cocoeval.py
@@ -21,7 +21,7 @@
import datetime
import time
from collections import defaultdict
-import mask as maskUtils
+from .mask import *
import copy
class COCOeval:
@@ -204,7 +204,7 @@ def computeIoU(self, imgId, catId):
# compute iou between each dt and gt region
iscrowd = [int(o['iscrowd']) for o in gt]
- ious = maskUtils.iou(d,g,iscrowd)
+ ious = iou(d,g,iscrowd)
return ious
def computeOks(self, imgId, catId):
diff --git a/example/rcnn/rcnn/pycocotools/mask.py b/example/rcnn/rcnn/pycocotools/mask.py
index 48c050c594..2122468f68 100644
--- a/example/rcnn/rcnn/pycocotools/mask.py
+++ b/example/rcnn/rcnn/pycocotools/mask.py
@@ -17,7 +17,7 @@
__author__ = 'tsungyi'
-import _mask
+from rcnn.pycocotools import _mask
# Interface for manipulating masks stored in RLE format.
#
diff --git a/example/rcnn/rcnn/symbol/__init__.py b/example/rcnn/rcnn/symbol/__init__.py
index 113b52c98a..7547122dd5 100644
--- a/example/rcnn/rcnn/symbol/__init__.py
+++ b/example/rcnn/rcnn/symbol/__init__.py
@@ -15,5 +15,5 @@
# specific language governing permissions and limitations
# under the License.
-from symbol_vgg import *
-from symbol_resnet import *
+from .symbol_vgg import *
+from .symbol_resnet import *
diff --git a/example/rcnn/rcnn/symbol/proposal_target.py b/example/rcnn/rcnn/symbol/proposal_target.py
index e0444f978b..0af19a9cf3 100644
--- a/example/rcnn/rcnn/symbol/proposal_target.py
+++ b/example/rcnn/rcnn/symbol/proposal_target.py
@@ -45,7 +45,7 @@ def forward(self, is_train, req, in_data, out_data, aux):
assert self._batch_rois % self._batch_images == 0, \
'BATCHIMAGES {} must devide BATCH_ROIS {}'.format(self._batch_images, self._batch_rois)
rois_per_image = self._batch_rois / self._batch_images
- fg_rois_per_image = np.round(self._fg_fraction * rois_per_image).astype(int)
+ fg_rois_per_image = np.round(self._fg_fraction * rois_per_image).astype(np.int)
all_rois = in_data[0].asnumpy()
gt_boxes = in_data[1].asnumpy()
diff --git a/example/rcnn/rcnn/symbol/symbol_resnet.py b/example/rcnn/rcnn/symbol/symbol_resnet.py
index 4a9677d440..f7721366c1 100644
--- a/example/rcnn/rcnn/symbol/symbol_resnet.py
+++ b/example/rcnn/rcnn/symbol/symbol_resnet.py
@@ -16,9 +16,9 @@
# under the License.
import mxnet as mx
-import proposal
-import proposal_target
from rcnn.config import config
+from . import proposal
+from . import proposal_target
eps = 2e-5
use_global_stats = True
diff --git a/example/rcnn/rcnn/symbol/symbol_vgg.py b/example/rcnn/rcnn/symbol/symbol_vgg.py
index 00ba15ed8e..33fbede2df 100644
--- a/example/rcnn/rcnn/symbol/symbol_vgg.py
+++ b/example/rcnn/rcnn/symbol/symbol_vgg.py
@@ -16,10 +16,9 @@
# under the License.
import mxnet as mx
-import proposal
-import proposal_target
from rcnn.config import config
-
+from . import proposal
+from . import proposal_target
def get_vgg_conv(data):
"""
diff --git a/example/rcnn/rcnn/tools/reeval.py b/example/rcnn/rcnn/tools/reeval.py
index a7ae898f41..1e5c0aa5a8 100644
--- a/example/rcnn/rcnn/tools/reeval.py
+++ b/example/rcnn/rcnn/tools/reeval.py
@@ -16,7 +16,10 @@
# under the License.
import argparse
-import cPickle
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
import os
import mxnet as mx
@@ -32,7 +35,7 @@ def reeval(args):
# load detection results
cache_file = os.path.join(imdb.cache_path, imdb.name, 'detections.pkl')
with open(cache_file) as f:
- detections = cPickle.load(f)
+ detections = pickle.load(f)
# eval
imdb.evaluate_detections(detections)
diff --git a/example/rcnn/script/additional_deps.sh b/example/rcnn/script/additional_deps.sh
index 0e6599c77f..cddc391b13 100755
--- a/example/rcnn/script/additional_deps.sh
+++ b/example/rcnn/script/additional_deps.sh
@@ -20,19 +20,7 @@
# install additional depts
sudo apt install python-pip python-dev unzip python-matplotlib
-sudo pip install cython scikit-image easydict
-
-# install a forked MXNet
-pushd ../../
-cp make/config.mk ./
-echo "USE_CUDA=1" >>config.mk
-echo "USE_CUDA_PATH=/usr/local/cuda" >>config.mk
-echo "USE_CUDNN=1" >>config.mk
-make -j$(nproc)
-pushd python
-python setup.py install --user
-popd
-popd
+sudo pip install cython scikit-image easydict opencv-python
# build cython extension
make
diff --git a/example/sparse/linear_classification/weighted_softmax_ce.py b/example/sparse/linear_classification/weighted_softmax_ce.py
index a40ece658e..f781e6ae38 100644
--- a/example/sparse/linear_classification/weighted_softmax_ce.py
+++ b/example/sparse/linear_classification/weighted_softmax_ce.py
@@ -61,7 +61,7 @@ class WeightedSoftmaxCrossEntropyLossProp(mx.operator.CustomOpProp):
def __init__(self, positive_cls_weight):
super(WeightedSoftmaxCrossEntropyLossProp, self).__init__(True)
self.positive_cls_weight = positive_cls_weight
- assert(positive_cls_weight > 0)
+ assert(float(positive_cls_weight) > 0)
def list_arguments(self):
return ['data', 'label']
diff --git a/example/sparse/matrix_factorization/data.py b/example/sparse/matrix_factorization/data.py
index fae2c237c8..c8971651b9 100644
--- a/example/sparse/matrix_factorization/data.py
+++ b/example/sparse/matrix_factorization/data.py
@@ -15,17 +15,17 @@
# specific language governing permissions and limitations
# under the License.
-import os, gzip
-import sys
+import os
import mxnet as mx
from mxnet.test_utils import DummyIter
-def get_movielens_data(prefix):
- if not os.path.exists("%s.zip" % prefix):
- print("Dataset MovieLens 10M not present. Downloading now ...")
- os.system("wget http://files.grouplens.org/datasets/movielens/%s.zip" % prefix)
- os.system("unzip %s.zip" % prefix)
- os.system("cd ml-10M100K; sh split_ratings.sh; cd -;")
+def get_movielens_data(data_dir, prefix):
+ if not os.path.exists(os.path.join(data_dir, "ml-10M100K")):
+ mx.test_utils.get_zip_data(data_dir,
+ "http://files.grouplens.org/datasets/movielens/%s.zip" % prefix,
+ prefix + ".zip")
+ assert os.path.exists(os.path.join(data_dir, "ml-10M100K"))
+ os.system("cd data/ml-10M100K; chmod +x allbut.pl; sh split_ratings.sh; cd -;")
def get_movielens_iter(filename, batch_size, dummy_iter):
"""Not particularly fast code to parse the text file and load into NDArrays.
@@ -58,3 +58,5 @@ def get_movielens_iter(filename, batch_size, dummy_iter):
batch_size=batch_size, shuffle=True)
iter_train = DummyIter(iter_train) if dummy_iter else iter_train
return mx.io.PrefetchingIter(iter_train)
+
+
diff --git a/example/sparse/matrix_factorization/train.py b/example/sparse/matrix_factorization/train.py
index 14c6ca188f..0db58ad524 100644
--- a/example/sparse/matrix_factorization/train.py
+++ b/example/sparse/matrix_factorization/train.py
@@ -17,11 +17,11 @@
import argparse
import logging
-import time
import mxnet as mx
import numpy as np
from data import get_movielens_iter, get_movielens_data
from model import matrix_fact_net
+import os
logging.basicConfig(level=logging.DEBUG)
@@ -44,8 +44,8 @@
MOVIELENS = {
'dataset': 'ml-10m',
- 'train': './ml-10M100K/r1.train',
- 'val': './ml-10M100K/r1.test',
+ 'train': './data/ml-10M100K/r1.train',
+ 'val': './data/ml-10M100K/r1.test',
'max_user': 71569,
'max_movie': 65135,
}
@@ -72,7 +72,8 @@
# prepare dataset and iterators
max_user = MOVIELENS['max_user']
max_movies = MOVIELENS['max_movie']
- get_movielens_data(MOVIELENS['dataset'])
+ data_dir = os.path.join(os.getcwd(), 'data')
+ get_movielens_data(data_dir, MOVIELENS['dataset'])
train_iter = get_movielens_iter(MOVIELENS['train'], batch_size, dummy_iter)
val_iter = get_movielens_iter(MOVIELENS['val'], batch_size, dummy_iter)
diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py
index 0dfeec56c1..53814b766f 100644
--- a/python/mxnet/test_utils.py
+++ b/python/mxnet/test_utils.py
@@ -29,6 +29,7 @@
import errno
import logging
import bz2
+import zipfile
from contextlib import contextmanager
import numpy as np
import numpy.testing as npt
@@ -1440,6 +1441,31 @@ def read_data(label_url, image_url):
return {'train_data':train_img, 'train_label':train_lbl,
'test_data':test_img, 'test_label':test_lbl}
+def get_zip_data(data_dir, url, data_origin_name):
+ """Download and extract zip data.
+
+ Parameters
+ ----------
+
+ data_dir : str
+ Absolute or relative path of the directory name to store zip files
+ url : str
+ URL to download data from
+ data_origin_name : str
+ Name of the downloaded zip file
+
+ Examples
+ --------
+ >>> get_zip_data("data_dir",
+ "http://files.grouplens.org/datasets/movielens/ml-10m.zip",
+ "ml-10m.zip")
+ """
+ data_origin_name = os.path.join(data_dir, data_origin_name)
+ if not os.path.exists(data_origin_name):
+ download(url, dirname=data_dir, overwrite=False)
+ zip_file = zipfile.ZipFile(data_origin_name)
+ zip_file.extractall(path=data_dir)
+
def get_bz2_data(data_dir, data_name, url, data_origin_name):
"""Download and extract bz2 data.
@@ -1465,14 +1491,12 @@ def get_bz2_data(data_dir, data_name, url, data_origin_name):
data_name = os.path.join(data_dir, data_name)
data_origin_name = os.path.join(data_dir, data_origin_name)
if not os.path.exists(data_name):
- download(url, dirname=data_dir, overwrite=False)
+ download(url, fname=data_origin_name, dirname=data_dir, overwrite=False)
bz_file = bz2.BZ2File(data_origin_name, 'rb')
with open(data_name, 'wb') as fout:
- try:
- content = bz_file.read()
- fout.write(content)
- finally:
- bz_file.close()
+ for line in bz_file:
+ fout.write(line)
+ bz_file.close()
os.remove(data_origin_name)
def set_env_var(key, val, default_val=""):
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services