You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/10/16 17:16:17 UTC
[GitHub] marcoabreu closed pull request #11952: [MXNET-707] Add unit test
for mxnet to coreml converter
marcoabreu closed pull request #11952: [MXNET-707] Add unit test for mxnet to coreml converter
URL: https://github.com/apache/incubator-mxnet/pull/11952
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.travis.yml b/.travis.yml
index d6b38f4ac8d..b5c323f3aa1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -34,7 +34,9 @@ script:
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- mv make/osx.mk config.mk
- make -j 2
+
# We ignore several tests to avoid possible timeouts on large PRs.
# This lowers our test coverage, but is required for consistent Travis runs.
# These tests will be tested in a variety of environments in Jenkins based tests.
- python -m nose --with-timer --exclude-test=test_sparse_operator.test_elemwise_binary_ops --exclude-test=test_gluon_model_zoo.test_models --exclude-test=test_random.test_shuffle --exclude-test=test_operator.test_broadcast_binary_op --exclude-test=test_operator.test_pick --exclude-test=test_profiler.test_continuous_profile_and_instant_marker --exclude-test=test_metric_perf.test_metric_performance --exclude-test=test_operator.test_order --verbose tests/python/unittest/
+ - python2 -m nose --verbose tools/coreml/test --exclude-test=test_mxnet_image
diff --git a/ci/travis/install.sh b/ci/travis/install.sh
index ae959767133..64128495b48 100644
--- a/ci/travis/install.sh
+++ b/ci/travis/install.sh
@@ -22,5 +22,5 @@ export HOMEBREW_NO_AUTO_UPDATE=1
if [ ${TRAVIS_OS_NAME} == "osx" ]; then
brew install opencv
- python -m pip install --user nose numpy cython scipy requests mock nose-timer nose-exclude
+ python -m pip install --user nose numpy cython scipy requests mock nose-timer nose-exclude mxnet-to-coreml
fi
diff --git a/tools/coreml/test/test_mxnet_converter.py b/tools/coreml/test/test_mxnet_converter.py
index 6020041ac53..5d26c5faf75 100644
--- a/tools/coreml/test/test_mxnet_converter.py
+++ b/tools/coreml/test/test_mxnet_converter.py
@@ -18,15 +18,12 @@
import unittest
import mxnet as mx
import numpy as np
-import sys
-import os
-current_working_directory = os.getcwd()
-sys.path.append(current_working_directory + "/..")
-sys.path.append(current_working_directory + "/../converter/")
-import _mxnet_converter as mxnet_converter
+
+from converter._mxnet_converter import convert
from collections import namedtuple
from converter import utils
+
def _mxnet_remove_batch(input_data):
for blob in input_data:
input_data[blob] = np.reshape(input_data[blob], input_data[blob].shape[1:])
@@ -39,7 +36,8 @@ def _get_mxnet_module(net, data_shapes, mode, label_names, input_names=None):
"""
mx.random.seed(1993)
- mod = utils.create_module(sym=net, data_shapes=data_shapes, label_shapes=input_names, label_names=label_names)
+ mod = utils.create_module(sym=net, data_shapes=data_shapes, label_shapes=input_names,
+ label_names=label_names)
if mode == 'random':
mod.init_params(
@@ -62,11 +60,14 @@ def _get_mxnet_module(net, data_shapes, mode, label_names, input_names=None):
class SingleLayerTest(unittest.TestCase):
"""
Unit test class for testing where converter is able to convert individual layers or not.
- In order to do so, it converts model and generates preds on both CoreML and MXNet and check they are the same.
+ In order to do so, it converts model and generates preds on both CoreML and MXNet and check
+ they are the same.
"""
- def _test_mxnet_model(self, net, input_shape, mode, class_labels=None, coreml_mode=None, label_names=None, delta=1e-3,
+ def _test_mxnet_model(self, net, input_shape, mode, class_labels=None,
+ coreml_mode=None, label_names=None, delta=1e-2,
pre_processing_args=None, input_name='data'):
- """ Helper method that convert the CoreML model into CoreML and compares the predictions over random data.
+ """ Helper method that convert the CoreML model into CoreML and compares the predictions
+ over random data.
Parameters
----------
@@ -89,7 +90,7 @@ def _test_mxnet_model(self, net, input_shape, mode, class_labels=None, coreml_mo
The name of the input variable to the symbolic graph.
"""
- data_shapes=[(input_name, input_shape)]
+ data_shapes = [(input_name, input_shape)]
mod = _get_mxnet_module(net, data_shapes, mode, label_names)
@@ -100,7 +101,7 @@ def _test_mxnet_model(self, net, input_shape, mode, class_labels=None, coreml_mo
mxnet_preds = mod.get_outputs()[0].asnumpy().flatten()
# Get predictions from coreml
- coreml_model = mxnet_converter.convert(
+ coreml_model = convert(
model=mod,
class_labels=class_labels,
mode=coreml_mode,
@@ -112,7 +113,7 @@ def _test_mxnet_model(self, net, input_shape, mode, class_labels=None, coreml_mo
# Check prediction accuracy
self.assertEquals(len(mxnet_preds), len(coreml_preds))
for i in range(len(mxnet_preds)):
- self.assertAlmostEquals(mxnet_preds[i], coreml_preds[i], delta = delta)
+ self.assertAlmostEquals(mxnet_preds[i], coreml_preds[i], delta=delta)
def test_tiny_inner_product_zero_input(self):
np.random.seed(1988)
@@ -140,7 +141,7 @@ def test_tiny_inner_product_ones_input(self):
input_shape = (1, 10)
net = mx.sym.Variable('data')
net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
- self._test_mxnet_model(net, input_shape=input_shape, mode='ones')
+ self._test_mxnet_model(net, input_shape=input_shape, mode='ones', delta=0.05)
def test_tiny_inner_product_random_input(self):
np.random.seed(1988)
@@ -162,7 +163,8 @@ def test_tiny_softmax_random_input(self):
net = mx.sym.Variable('data')
net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
net = mx.sym.SoftmaxOutput(net, name='softmax')
- self._test_mxnet_model(net, input_shape=input_shape, mode='random', label_names=['softmax_label'])
+ self._test_mxnet_model(net, input_shape=input_shape, mode='random',
+ label_names=['softmax_label'])
def test_tiny_relu_activation_random_input(self):
np.random.seed(1988)
@@ -228,7 +230,7 @@ def test_tiny_conv_ones_input(self):
pad=pad,
name='conv_1'
)
- self._test_mxnet_model(net, input_shape=input_shape, mode='ones')
+ self._test_mxnet_model(net, input_shape=input_shape, mode='ones', delta=0.05)
def test_tiny_conv_random_input(self):
np.random.seed(1988)
@@ -481,7 +483,8 @@ def test_flatten(self):
net = mx.sym.Flatten(data=net, name='flatten1')
net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
net = mx.sym.SoftmaxOutput(net, name='softmax')
- self._test_mxnet_model(net, input_shape=input_shape, mode='random', label_names=['softmax_label'])
+ self._test_mxnet_model(net, input_shape=input_shape, mode='random',
+ label_names=['softmax_label'])
def test_transpose(self):
np.random.seed(1988)
@@ -516,10 +519,8 @@ def test_tiny_synset_random_input(self):
net = mx.sym.Variable('data')
net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
net = mx.sym.SoftmaxOutput(net, name='softmax')
- mod = _get_mxnet_module(net,
- data_shapes=[('data', input_shape)],
- mode='random',
- label_names=['softmax_label'])
+ mod = _get_mxnet_module(net, data_shapes=[('data', input_shape)],
+ mode='random', label_names=['softmax_label'])
# Generate some dummy data
input_data = np.random.uniform(-0.1, 0.1, input_shape)
@@ -529,14 +530,15 @@ def test_tiny_synset_random_input(self):
kwargs = {'input_shape': {'data': input_shape}}
# Get predictions from coreml
- coreml_model = mxnet_converter.convert(
+ coreml_model = convert(
model=mod,
- class_labels=['Category1','Category2','Category3','Category4','Category5'],
+ class_labels=['Category1', 'Category2', 'Category3', 'Category4', 'Category5'],
mode='classifier',
**kwargs
)
- prediction = coreml_model.predict(_mxnet_remove_batch({'data': input_data}))
+ prediction = coreml_model.predict(
+ _mxnet_remove_batch({'data': input_data}))
self.assertEqual(prediction['classLabel'], 'Category3')
def test_really_tiny_deconv_random_input(self):
@@ -579,7 +581,7 @@ def test_tiny_deconv_ones_input(self):
name='deconv_1'
)
# Test the mxnet model
- self._test_mxnet_model(net, input_shape=input_shape, mode='ones')
+ self._test_mxnet_model(net, input_shape=input_shape, mode='ones', delta=0.05)
def test_tiny_deconv_random_input(self):
np.random.seed(1988)
@@ -915,7 +917,6 @@ def test_batch_norm(self):
def test_batch_norm_no_global_stats(self):
""" This test should throw an exception since converter doesn't support
- conversion of MXNet models that use local batch stats (i.e.
use_global_stats=False). The reason for this is CoreML doesn't support
local batch stats.
"""
@@ -943,12 +944,17 @@ def test_pre_processing_args(self):
net = mx.sym.Variable('data')
net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
net = mx.sym.SoftmaxOutput(net, name='softmax')
- self._test_mxnet_model(net, input_shape=input_shape, mode='random', label_names=['softmax_label'],
- pre_processing_args={'red_bias':0, 'blue_bias':0, 'green_bias':0, 'image_scale':1})
+ self._test_mxnet_model(net, input_shape=input_shape, mode='random',
+ label_names=['softmax_label'],
+ pre_processing_args={'red_bias': 0,
+ 'blue_bias': 0,
+ 'green_bias': 0,
+ 'image_scale': 1})
def test_different_input_variables(self):
"""
- Verifying the behavior when input variable name is different than the standard name - 'data'.
+ Verifying the behavior when input variable name is different than the
+ standard name - 'data'.
"""
np.random.seed(1988)
input_shape = (1, 10)
@@ -958,12 +964,13 @@ def test_different_input_variables(self):
def test_really_tiny_conv_optional_params(self):
"""
- Verifying the behavior of a convolutional layer when stride and pad are not provided.
+ Verifying the behavior of a convolutional layer when stride and pad
+ are not provided.
"""
np.random.seed(1988)
input_shape = (1, 1, 10, 10)
num_filter = 1
- kernel = (1 ,1)
+ kernel = (1, 1)
# Define a model
net = mx.sym.Variable('data')
diff --git a/tools/coreml/test/test_mxnet_image.py b/tools/coreml/test/test_mxnet_image.py
index e75451961a9..b3165f0e09a 100644
--- a/tools/coreml/test/test_mxnet_image.py
+++ b/tools/coreml/test/test_mxnet_image.py
@@ -1,4 +1,4 @@
-from __future__ import print_function
+
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,19 +16,13 @@
# specific language governing permissions and limitations
# under the License.
+import unittest
import mxnet as mx
import numpy as np
-import unittest
-import sys
-import os
from six.moves import xrange
-
-current_working_directory = os.getcwd()
-sys.path.append(current_working_directory + "/..")
-sys.path.append(current_working_directory + "/../converter/")
-import _mxnet_converter as mxnet_converter
-from converter.utils import load_model
+from converter._mxnet_converter import convert
+from converter import utils
VAL_DATA = 'data/val-5k-256.rec'
URL = 'http://data.mxnet.io/data/val-5k-256.rec'
@@ -44,7 +38,7 @@ def read_image(data_val, label_name):
label_width=1,
preprocess_threads=4,
batch_size=32,
- data_shape=(3,224,224),
+ data_shape=(3, 224, 224),
label_name=label_name,
rand_corp=False,
rand_mirror=False,
@@ -80,11 +74,11 @@ def _test_image_prediction(self, model_name, epoch, label_name):
epoch_num=epoch,
data_shapes=data.provide_data,
label_shapes=data.provide_label,
- label_names=[label_name,]
+ label_names=[label_name, ]
)
input_shape = (1, 3, 224, 224)
- coreml_model = mxnet_converter.convert(mod, input_shape={'data': input_shape})
+ coreml_model = convert(mod, input_shape={'data': input_shape})
mxnet_acc = []
mxnet_top_5_acc = []
@@ -104,34 +98,43 @@ def _test_image_prediction(self, model_name, epoch, label_name):
mxnet_predict = mxnet_preds[i]
label = label_numpy[i]
mxnet_acc.append(is_correct_top_one(mxnet_predict, label))
- mxnet_top_5_acc.append(is_correct_top_five(mxnet_predict, label))
+ mxnet_top_5_acc.append(is_correct_top_five(mxnet_predict,
+ label))
coreml_acc.append(is_correct_top_one(coreml_predict, label))
- coreml_top_5_acc.append(is_correct_top_five(coreml_predict, label))
+ coreml_top_5_acc.append(is_correct_top_five(coreml_predict,
+ label))
num_batch += 1
- if (num_batch == 5): break # we only use a subset of the batches.
+ if (num_batch == 5):
+ break # we only use a subset of the batches.
print("MXNet acc %s" % np.mean(mxnet_acc))
print("Coreml acc %s" % np.mean(coreml_acc))
print("MXNet top 5 acc %s" % np.mean(mxnet_top_5_acc))
print("Coreml top 5 acc %s" % np.mean(coreml_top_5_acc))
self.assertAlmostEqual(np.mean(mxnet_acc), np.mean(coreml_acc), delta=1e-4)
- self.assertAlmostEqual(np.mean(mxnet_top_5_acc), np.mean(coreml_top_5_acc), delta=1e-4)
+ self.assertAlmostEqual(np.mean(mxnet_top_5_acc),
+ np.mean(coreml_top_5_acc),
+ delta=1e-4)
def test_squeezenet(self):
print("Testing Image Classification with Squeezenet")
- self._test_image_prediction(model_name='squeezenet_v1.1', epoch=0, label_name='prob_label')
+ self._test_image_prediction(model_name='squeezenet_v1.1', epoch=0,
+ label_name='prob_label')
def test_inception_with_batch_normalization(self):
print("Testing Image Classification with Inception/BatchNorm")
- self._test_image_prediction(model_name='Inception-BN', epoch=126, label_name='softmax_label')
+ self._test_image_prediction(model_name='Inception-BN', epoch=126,
+ label_name='softmax_label')
def test_resnet18(self):
print("Testing Image Classification with ResNet18")
- self._test_image_prediction(model_name='resnet-18', epoch=0, label_name='softmax_label')
+ self._test_image_prediction(model_name='resnet-18', epoch=0,
+ label_name='softmax_label')
def test_vgg16(self):
print("Testing Image Classification with vgg16")
- self._test_image_prediction(model_name='vgg16', epoch=0, label_name='prob_label')
+ self._test_image_prediction(model_name='vgg16', epoch=0,
+ label_name='prob_label')
if __name__ == '__main__':
diff --git a/tools/coreml/test/test_mxnet_models.py b/tools/coreml/test/test_mxnet_models.py
index 7c294db8611..8dd319ab4e0 100644
--- a/tools/coreml/test/test_mxnet_models.py
+++ b/tools/coreml/test/test_mxnet_models.py
@@ -19,21 +19,16 @@
import unittest
import mxnet as mx
import numpy as np
-import sys
-import os
-from collections import namedtuple
+from collections import namedtuple
+from converter._mxnet_converter import convert
from six.moves import xrange
-current_working_directory = os.getcwd()
-sys.path.append(current_working_directory + "/..")
-sys.path.append(current_working_directory + "/../converter/")
-import _mxnet_converter as mxnet_converter
-
def _mxnet_remove_batch(input_data):
for blob in input_data:
- input_data[blob] = np.reshape(input_data[blob], input_data[blob].shape[1:])
+ input_data[blob] = np.reshape(input_data[blob], input_data[blob]
+ .shape[1:])
return input_data
@@ -53,9 +48,10 @@ def _kl_divergence(distribution1, distribution2):
class ModelsTest(unittest.TestCase):
"""
- Unit test class that tests converter on entire MXNet models .
- In order to test each unit test converts MXNet model into CoreML model using the converter, generate predictions
- on both MXNet and CoreML and verifies that predictions are same (or similar).
+ Unit test class that tests converter on entire MXNet models.
+ In order to test each unit test converts MXNet model into CoreML model using the converter,
+ generate predictions on both MXNet and CoreML and verifies that predictions are same
+ (or similar).
"""
def _load_model(self, model_name, epoch_num, input_shape):
sym, arg_params, aux_params = mx.model.load_checkpoint(model_name, epoch_num)
@@ -76,8 +72,10 @@ def _load_model(self, model_name, epoch_num, input_shape):
)
return mod
- def _test_model(self, model_name, epoch_num, input_shape=(1, 3, 224, 224), files=None):
- """ Tests whether the converted CoreML model's preds are equal to MXNet preds for a given model or not.
+ def _test_model(self, model_name, epoch_num, input_shape=(1, 3, 224, 224),
+ files=None):
+ """ Tests whether the converted CoreML model's preds are equal to MXNet
+ preds for a given model or not.
Parameters
----------
@@ -88,10 +86,12 @@ def _test_model(self, model_name, epoch_num, input_shape=(1, 3, 224, 224), files
Epoch number of model we would like to load.
input_shape: tuple
- The shape of the input data in the form of (batch_size, channels, height, width)
+ The shape of the input data in the form of (batch_size, channels,
+ height, width)
files: list of strings
- List of URLs pertaining to files that need to be downloaded in order to use the model.
+ List of URLs pertaining to files that need to be downloaded in
+ order to use the model.
"""
if files is not None:
@@ -106,17 +106,22 @@ def _test_model(self, model_name, epoch_num, input_shape=(1, 3, 224, 224), files
input_shape=input_shape
)
- coreml_model = mxnet_converter.convert(module, input_shape={'data': input_shape})
+ coreml_model = convert(module, input_shape={'data': input_shape})
# Get predictions from MXNet and coreml
- div=[] # For storing KL divergence for each input.
+ div = [] # For storing KL divergence for each input.
for _ in xrange(1):
np.random.seed(1993)
- input_data = {'data': np.random.uniform(0, 1, input_shape).astype(np.float32)}
+ input_data = {'data': np.random.uniform(0, 1, input_shape)
+ .astype(np.float32)}
Batch = namedtuple('Batch', ['data'])
- module.forward(Batch([mx.nd.array(input_data['data'])]), is_train=False)
+ module.forward(Batch([mx.nd.array(input_data['data'])]),
+ is_train=False)
mxnet_pred = module.get_outputs()[0].asnumpy().flatten()
- coreml_pred = coreml_model.predict(_mxnet_remove_batch(input_data)).values()[0].flatten()
+ coreml_pred = coreml_model \
+ .predict(_mxnet_remove_batch(input_data)) \
+ .values()[0] \
+ .flatten()
self.assertEqual(len(mxnet_pred), len(coreml_pred))
div.append(_kl_divergence(mxnet_pred, coreml_pred))
@@ -138,6 +143,7 @@ def test_pred_resnet_50(self):
files=["http://data.mxnet.io/models/imagenet/resnet/50-layers/resnet-50-symbol.json",
"http://data.mxnet.io/models/imagenet/resnet/50-layers/resnet-50-0000.params"])
+ @unittest.skip("Model is too big for unit test")
def test_pred_vgg16(self):
self._test_model(model_name='vgg16', epoch_num=0,
files=["http://data.mxnet.io/models/imagenet/vgg/vgg16-symbol.json",
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services