You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2017/12/12 22:17:04 UTC
[GitHub] piiswrong closed pull request #8313: Ci test randomness
piiswrong closed pull request #8313: Ci test randomness
URL: https://github.com/apache/incubator-mxnet/pull/8313
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py
index 3e667364cd..5148f9fab0 100644
--- a/python/mxnet/test_utils.py
+++ b/python/mxnet/test_utils.py
@@ -44,8 +44,12 @@
from .ndarray import array
from .symbol import Symbol
-_rng = np.random.RandomState(1234)
-
+# spawn off a new seeded random number generator, by default based on the
+# global random number generator's current state.
+def get_rng(seed=None):
+ if seed is None:
+ seed = np.random.randint(0, np.iinfo(np.uint32).max)
+ return np.random.RandomState(seed)
def default_context():
"""Get default context for regression test."""
@@ -830,6 +834,8 @@ def check_numeric_gradient(sym, location, aux_states=None, numeric_eps=1e-3, rto
if ctx is None:
ctx = default_context()
+ _rng = get_rng()
+
def random_projection(shape):
"""Get a random weight matrix with not too small elements
@@ -1063,6 +1069,7 @@ def check_symbolic_backward(sym, location, out_grads, expected, rtol=1e-5, atol=
if isinstance(expected, (list, tuple)):
expected = {k:v for k, v in zip(sym.list_arguments(), expected)}
+ _rng = get_rng()
args_grad_npy = {k:_rng.normal(size=v.shape) for k, v in expected.items()}
args_grad_data = {}
for k, v in args_grad_npy.items():
@@ -1155,6 +1162,7 @@ def check_speed(sym, location=None, ctx=None, N=20, grad_req=None, typ="whole",
if grad_req is None:
grad_req = 'write'
+ _rng = get_rng()
if location is None:
exe = sym.simple_bind(grad_req=grad_req, ctx=ctx, **kwargs)
location = {k: _rng.normal(size=arr.shape, scale=1.0) for k, arr in
@@ -1538,3 +1546,53 @@ def discard_stderr():
finally:
os.dup2(old_stderr, stderr_fileno)
bit_bucket.close()
+
+@contextmanager
+def np_random_seed(seed=None):
+ """
+ Runs a code block with a new state of np.random.
+ To impose rng determinism, invoke e.g. as in:
+
+ with np_random_seed(1234):
+ ...
+
+ To impose rng non-determinism, invoke as in:
+
+ with np_random_seed():
+ ...
+
+ """
+
+ try:
+ saved_rng_state = np.random.get_state()
+ np.random.seed(seed)
+ yield
+ finally:
+ # Reinstate prior state of np.random
+ np.random.set_state(saved_rng_state)
+
+# Set seed and output to stderr (to avoid default nosetests filtering of stdout)
+def set_np_random_seed(seed=None, ostream=sys.stderr):
+ """Set the np.random seed and announce the value to an output stream
+
+ Parameters
+ ----------
+
+ seed: int
+ Seed to pass to np.random.seed(). Should be None to set and output
+ a randomly chosen value.
+ ostream :
+ Stream to announce the new seed value to.
+
+ The expected use of this function is to set the seed globally before
+ a suite of tests and output the set value to help reproduce any failures
+ that are dependent on the random data. To fix the seed for a single test
+ without modifying the randomness of subsequent tests in the same file,
+ use 'np_random_seed'.
+ """
+ if seed is None:
+ seed = np.random.randint(0, np.iinfo(np.uint32).max)
+ if ostream is not None:
+ ostream.write('Setting np.random seed to %s.\n' % seed)
+ ostream.flush()
+ np.random.seed(seed)
diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py
index b1f43f3105..f5843b533d 100644
--- a/tests/python/gpu/test_operator_gpu.py
+++ b/tests/python/gpu/test_operator_gpu.py
@@ -81,18 +81,18 @@ def check_countsketch(in_dim,out_dim,n):
assert_almost_equal(a,arr_grad[0].asnumpy(),rtol=1e-3, atol=1e-12)
def test_countsketch():
- np.random.seed(0)
- nrepeat = 2
- minindim = 40
- maxindim = 100
- minoutdim = 5
- maxoutdim = 30
- maxn = 200
- for repeat in range(nrepeat):
- in_dim = np.random.randint(minindim, maxindim)
- out_dim = np.random.randint(minoutdim, maxoutdim)
- n = np.random.randint(1,maxn)
- check_countsketch(in_dim, out_dim, n)
+ with np_random_seed(0):
+ nrepeat = 2
+ minindim = 40
+ maxindim = 100
+ minoutdim = 5
+ maxoutdim = 30
+ maxn = 200
+ for repeat in range(nrepeat):
+ in_dim = np.random.randint(minindim, maxindim)
+ out_dim = np.random.randint(minoutdim, maxoutdim)
+ n = np.random.randint(1,maxn)
+ check_countsketch(in_dim, out_dim, n)
def check_ifft(shape):
shape_old = shape
@@ -167,13 +167,13 @@ def check_ifft(shape):
def test_ifft():
- np.random.seed(0)
- nrepeat = 2
- maxdim = 10
- for repeat in range(nrepeat):
- for order in [2,4]:
- shape = tuple(np.random.randint(1, maxdim, size=order))
- check_ifft(shape)
+ with np_random_seed(0):
+ nrepeat = 2
+ maxdim = 10
+ for repeat in range(nrepeat):
+ for order in [2,4]:
+ shape = tuple(np.random.randint(1, maxdim, size=order))
+ check_ifft(shape)
def check_fft(shape):
sym = mx.sym.contrib.fft(name='fft', compute_size = 128)
@@ -254,13 +254,13 @@ def check_fft(shape):
assert_almost_equal(a.real, exe.grad_arrays[0].asnumpy()/shape[3],rtol=1e-3, atol=1e-6)
def test_fft():
- np.random.seed(0)
- nrepeat = 2
- maxdim = 10
- for repeat in range(nrepeat):
- for order in [2,4]:
- shape = tuple(np.random.randint(1, maxdim, size=order))
- check_fft(shape)
+ with np_random_seed(0):
+ nrepeat = 2
+ maxdim = 10
+ for repeat in range(nrepeat):
+ for order in [2,4]:
+ shape = tuple(np.random.randint(1, maxdim, size=order))
+ check_fft(shape)
def test_batchnorm_with_type():
ctx_list_v1_2D = [
@@ -409,38 +409,38 @@ def test_3d_batchnorm(fix_gamma, use_global_stats):
def test_convolution_with_type():
- np.random.seed(1234)
- sym1 = mx.sym.Convolution(num_filter=3, kernel=(3,3), name='conv')
-
- data = mx.sym.Variable('conv_data')
- w = mx.sym.Variable('conv_weight')
- b = mx.sym.Variable('conv_bias')
- w = mx.sym.transpose(w, axes=(0,2,3,1))
- sym2 = mx.sym.transpose(data, axes=(0,2,3,1))
- sym2 = mx.sym.Convolution(sym2, w, b, layout='NHWC', num_filter=3, kernel=(3,3))
- sym2 = mx.sym.transpose(sym2, axes=(0,3,1,2), name='conv')
-
- sym = [sym1, sym1, sym1, sym1, sym1, sym2, sym2]
- ctx_list = [{'ctx': mx.gpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float64}},
- {'ctx': mx.gpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float32}},
- {'ctx': mx.gpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float16}},
- {'ctx': mx.cpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float64}},
- {'ctx': mx.cpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float32}},
- # NHWC
- {'ctx': mx.gpu(0), 'conv_data': (2, 2, 10, 10), 'conv_weight': (3, 2, 3, 3),
- 'type_dict': {'conv_data': np.float32, 'conv_weight': np.float32}},
- {'ctx': mx.gpu(0), 'conv_data': (2, 2, 10, 10), 'conv_weight': (3, 2, 3, 3),
- 'type_dict': {'conv_data': np.float16, 'conv_weight': np.float16}}
- ]
- # wider tolerance needed for true-fp16 NCHW test above
- tol = {np.dtype(np.float16): 0.5,
- np.dtype(np.float32): 1e-3,
- np.dtype(np.float64): 1e-5,
- np.dtype(np.uint8): 0,
- np.dtype(np.int32): 0}
- check_consistency(sym, ctx_list, tol=tol)
- # test ability to turn off training on bias
- check_consistency(sym, ctx_list, grad_req={'conv_data': 'write', 'conv_weight': 'write', 'conv_bias': 'null'}, tol=tol)
+ with np_random_seed(1234):
+ sym1 = mx.sym.Convolution(num_filter=3, kernel=(3,3), name='conv')
+
+ data = mx.sym.Variable('conv_data')
+ w = mx.sym.Variable('conv_weight')
+ b = mx.sym.Variable('conv_bias')
+ w = mx.sym.transpose(w, axes=(0,2,3,1))
+ sym2 = mx.sym.transpose(data, axes=(0,2,3,1))
+ sym2 = mx.sym.Convolution(sym2, w, b, layout='NHWC', num_filter=3, kernel=(3,3))
+ sym2 = mx.sym.transpose(sym2, axes=(0,3,1,2), name='conv')
+
+ sym = [sym1, sym1, sym1, sym1, sym1, sym2, sym2]
+ ctx_list = [{'ctx': mx.gpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float64}},
+ {'ctx': mx.gpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float32}},
+ {'ctx': mx.gpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float16}},
+ {'ctx': mx.cpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float64}},
+ {'ctx': mx.cpu(0), 'conv_data': (2, 2, 10, 10), 'type_dict': {'conv_data': np.float32}},
+ # NHWC
+ {'ctx': mx.gpu(0), 'conv_data': (2, 2, 10, 10), 'conv_weight': (3, 2, 3, 3),
+ 'type_dict': {'conv_data': np.float32, 'conv_weight': np.float32}},
+ {'ctx': mx.gpu(0), 'conv_data': (2, 2, 10, 10), 'conv_weight': (3, 2, 3, 3),
+ 'type_dict': {'conv_data': np.float16, 'conv_weight': np.float16}}
+ ]
+ # wider tolerance needed for true-fp16 NCHW test above
+ tol = {np.dtype(np.float16): 0.5,
+ np.dtype(np.float32): 1e-3,
+ np.dtype(np.float64): 1e-5,
+ np.dtype(np.uint8): 0,
+ np.dtype(np.int32): 0}
+ check_consistency(sym, ctx_list, tol=tol)
+ # test ability to turn off training on bias
+ check_consistency(sym, ctx_list, grad_req={'conv_data': 'write', 'conv_weight': 'write', 'conv_bias': 'null'}, tol=tol)
# Apply N symbols against each of M contexts, checking that all NxM combinations match.
def check_consistency_NxM(sym_list, ctx_list):
@@ -624,22 +624,22 @@ def test_deconvolution_options():
# check_consistency_NxM([sym, sym_no_cudnn], ctx_list)
def test_bilinear_sampler_with_type():
- np.random.seed(1234)
- data = mx.sym.Variable('data')
- grid = mx.sym.Variable('grid')
- sym = mx.sym.BilinearSampler(data=data, grid=grid)
- ctx_list = [{'ctx': mx.gpu(0), 'data': (1, 5, 10, 10), 'grid': (1, 2, 10, 10),
- 'type_dict': {'data': np.float64}},
- {'ctx': mx.gpu(0), 'data': (1, 5, 10, 10), 'grid': (1, 2, 10, 10),
- 'type_dict': {'data': np.float32}},
- {'ctx': mx.gpu(0), 'data': (1, 5, 10, 10), 'grid': (1, 2, 10, 10),
- 'type_dict': {'data': np.float16}},
- {'ctx': mx.cpu(0), 'data': (1, 5, 10, 10), 'grid': (1, 2, 10, 10),
- 'type_dict': {'data': np.float64}},
- {'ctx': mx.cpu(0), 'data': (1, 5, 10, 10), 'grid': (1, 2, 10, 10),
- 'type_dict': {'data': np.float32}}]
- check_consistency(sym, ctx_list)
- check_consistency(sym, ctx_list, grad_req="add")
+ with np_random_seed(1234):
+ data = mx.sym.Variable('data')
+ grid = mx.sym.Variable('grid')
+ sym = mx.sym.BilinearSampler(data=data, grid=grid)
+ ctx_list = [{'ctx': mx.gpu(0), 'data': (1, 5, 10, 10), 'grid': (1, 2, 10, 10),
+ 'type_dict': {'data': np.float64}},
+ {'ctx': mx.gpu(0), 'data': (1, 5, 10, 10), 'grid': (1, 2, 10, 10),
+ 'type_dict': {'data': np.float32}},
+ {'ctx': mx.gpu(0), 'data': (1, 5, 10, 10), 'grid': (1, 2, 10, 10),
+ 'type_dict': {'data': np.float16}},
+ {'ctx': mx.cpu(0), 'data': (1, 5, 10, 10), 'grid': (1, 2, 10, 10),
+ 'type_dict': {'data': np.float64}},
+ {'ctx': mx.cpu(0), 'data': (1, 5, 10, 10), 'grid': (1, 2, 10, 10),
+ 'type_dict': {'data': np.float32}}]
+ check_consistency(sym, ctx_list)
+ check_consistency(sym, ctx_list, grad_req="add")
def test_grid_generator_with_type():
data = mx.sym.Variable('data')
@@ -656,42 +656,42 @@ def test_grid_generator_with_type():
@unittest.skip("test fails intermittently. temporarily disabled till it gets fixed. tracked at https://github.com/apache/incubator-mxnet/issues/7645")
def test_spatial_transformer_with_type():
- np.random.seed(1234)
- data = mx.sym.Variable('data')
- loc = mx.sym.Flatten(data)
- loc = mx.sym.FullyConnected(data=loc, num_hidden=10)
- loc = mx.sym.Activation(data=loc, act_type='relu')
- loc = mx.sym.FullyConnected(data=loc, num_hidden=6)
- sym = mx.sym.SpatialTransformer(data=data, loc=loc, target_shape=(10, 10),
- transform_type="affine", sampler_type="bilinear")
- ctx_list = [{'ctx': mx.gpu(0), 'data': (1, 5, 10, 10), 'type_dict': {'data': np.float32}},
- {'ctx': mx.cpu(0), 'data': (1, 5, 10, 10), 'type_dict': {'data': np.float32}}]
- check_consistency(sym, ctx_list)
- check_consistency(sym, ctx_list, grad_req="add")
+ with np_random_seed(1234):
+ data = mx.sym.Variable('data')
+ loc = mx.sym.Flatten(data)
+ loc = mx.sym.FullyConnected(data=loc, num_hidden=10)
+ loc = mx.sym.Activation(data=loc, act_type='relu')
+ loc = mx.sym.FullyConnected(data=loc, num_hidden=6)
+ sym = mx.sym.SpatialTransformer(data=data, loc=loc, target_shape=(10, 10),
+ transform_type="affine", sampler_type="bilinear")
+ ctx_list = [{'ctx': mx.gpu(0), 'data': (1, 5, 10, 10), 'type_dict': {'data': np.float32}},
+ {'ctx': mx.cpu(0), 'data': (1, 5, 10, 10), 'type_dict': {'data': np.float32}}]
+ check_consistency(sym, ctx_list)
+ check_consistency(sym, ctx_list, grad_req="add")
# Checking max pooling consistency over the data sets of different float types is problematic
# as one max value in a float32 data set may not be the max value in a float16 data set.
# This function will not be called.
def test_pooling_with_type():
- np.random.seed(1234)
- ctx_list = [{'ctx': mx.gpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float64}},
- {'ctx': mx.gpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float32}},
- {'ctx': mx.gpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float16}},
- {'ctx': mx.cpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float64}},
- {'ctx': mx.cpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float32}}]
-
- sym = mx.sym.Pooling(name='pool', kernel=(3,3), stride=(2,2), pool_type='max')
- check_consistency(sym, ctx_list)
+ with np_random_seed(1234):
+ ctx_list = [{'ctx': mx.gpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float64}},
+ {'ctx': mx.gpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float32}},
+ {'ctx': mx.gpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float16}},
+ {'ctx': mx.cpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float64}},
+ {'ctx': mx.cpu(0), 'pool_data': (10, 2, 10, 10), 'type_dict': {'pool_data': np.float32}}]
- sym = mx.sym.Pooling(name='pool', kernel=(3,3), pad=(1,1), pool_type='avg')
- check_consistency(sym, ctx_list)
+ sym = mx.sym.Pooling(name='pool', kernel=(3,3), stride=(2,2), pool_type='max')
+ check_consistency(sym, ctx_list)
- # this is unstable
- # sym = mx.sym.Pooling(name='pool', kernel=(5,5), pad=(2,2), pool_type='max')
- # check_consistency(sym, ctx_list)
+ sym = mx.sym.Pooling(name='pool', kernel=(3,3), pad=(1,1), pool_type='avg')
+ check_consistency(sym, ctx_list)
- sym = mx.sym.Pooling(name='pool', kernel=(3,3), pad=(1,1), pool_type='sum')
- check_consistency(sym, ctx_list)
+ # this is unstable
+ # sym = mx.sym.Pooling(name='pool', kernel=(5,5), pad=(2,2), pool_type='max')
+ # check_consistency(sym, ctx_list)
+
+ sym = mx.sym.Pooling(name='pool', kernel=(3,3), pad=(1,1), pool_type='sum')
+ check_consistency(sym, ctx_list)
def test_pooling_versions():
@@ -1151,91 +1151,92 @@ def test_unfuse():
check_rnn_consistency(stack, fused)
def test_psroipooling_with_type():
- np.random.seed(1234)
- arg_params = {
- 'psroipool_rois': np.array([[0, 10, 22, 161, 173], [0, 20, 15, 154, 160]])}
-
- # plain psroipooling
- sym = mx.sym.contrib.PSROIPooling(spatial_scale=0.0625, output_dim=2, pooled_size=3, name='psroipool')
- ctx_list = [{'ctx': mx.gpu(0),
- 'psroipool_data': (1, 18, 14, 14),
- 'psroipool_rois': (2, 5),
- 'type_dict': {'psroipool_data': np.float64, 'psroipool_rois': np.float64}},
- {'ctx': mx.gpu(0),
- 'psroipool_data': (1, 18, 14, 14),
- 'psroipool_rois': (2, 5),
- 'type_dict': {'psroipool_data': np.float32, 'psroipool_rois': np.float32}},
- {'ctx': mx.gpu(0),
- 'psroipool_data': (1, 18, 14, 14),
- 'psroipool_rois': (2, 5),
- 'type_dict': {'psroipool_data': np.float16, 'psroipool_rois': np.float16}},
- ]
-
- check_consistency(sym, ctx_list, grad_req={'psroipool_data': 'write',
- 'psroipool_rois': 'null'}, arg_params=arg_params)
+ with np_random_seed(1234):
+ arg_params = {
+ 'psroipool_rois': np.array([[0, 10, 22, 161, 173], [0, 20, 15, 154, 160]])}
+
+ # plain psroipooling
+ sym = mx.sym.contrib.PSROIPooling(spatial_scale=0.0625, output_dim=2, pooled_size=3, name='psroipool')
+ ctx_list = [{'ctx': mx.gpu(0),
+ 'psroipool_data': (1, 18, 14, 14),
+ 'psroipool_rois': (2, 5),
+ 'type_dict': {'psroipool_data': np.float64, 'psroipool_rois': np.float64}},
+ {'ctx': mx.gpu(0),
+ 'psroipool_data': (1, 18, 14, 14),
+ 'psroipool_rois': (2, 5),
+ 'type_dict': {'psroipool_data': np.float32, 'psroipool_rois': np.float32}},
+ {'ctx': mx.gpu(0),
+ 'psroipool_data': (1, 18, 14, 14),
+ 'psroipool_rois': (2, 5),
+ 'type_dict': {'psroipool_data': np.float16, 'psroipool_rois': np.float16}},
+ ]
+
+ check_consistency(sym, ctx_list, grad_req={'psroipool_data': 'write',
+ 'psroipool_rois': 'null'}, arg_params=arg_params)
def test_deformable_psroipooling_with_type():
- np.random.seed(1234)
- arg_params = {
- 'deformable_psroipool_rois': np.array([[0, 10, 22, 161, 173], [0, 20, 15, 154, 160]])}
-
- # deformable psroipooling
- sym = mx.sym.contrib.DeformablePSROIPooling(spatial_scale=0.0625, sample_per_part=4, group_size=3, pooled_size=3,
- output_dim=2, trans_std=0.1, no_trans=False, name='deformable_psroipool')
-
- ctx_list = [{'ctx': mx.gpu(0),
- 'deformable_psroipool_data': (1, 18, 14, 14),
- 'deformable_psroipool_rois': (2, 5),
- 'deformable_psroipool_trans': (2, 4, 3, 3),
- 'type_dict': {'deformable_psroipool_data': np.float64, 'deformable_psroipool_rois': np.float64,
- 'deformable_psroipool_trans': np.float64}},
- {'ctx': mx.gpu(0),
- 'deformable_psroipool_data': (1, 18, 14, 14),
- 'deformable_psroipool_rois': (2, 5),
- 'deformable_psroipool_trans': (2, 4, 3, 3),
- 'type_dict': {'deformable_psroipool_data': np.float32, 'deformable_psroipool_rois': np.float32,
- 'deformable_psroipool_trans': np.float32}},
- {'ctx': mx.gpu(0),
- 'deformable_psroipool_data': (1, 18, 14, 14),
- 'deformable_psroipool_rois': (2, 5),
- 'deformable_psroipool_trans': (2, 4, 3, 3),
- 'type_dict': {'deformable_psroipool_data': np.float16, 'deformable_psroipool_rois': np.float16,
- 'deformable_psroipool_trans': np.float16}},
- ]
-
- check_consistency(sym, ctx_list, grad_req={'deformable_psroipool_data': 'write',
- 'deformable_psroipool_rois': 'null',
- 'deformable_psroipool_trans': 'write'}, arg_params=arg_params)
+ with np_random_seed(1234):
+ arg_params = {
+ 'deformable_psroipool_rois': np.array([[0, 10, 22, 161, 173], [0, 20, 15, 154, 160]])}
+
+ # deformable psroipooling
+ sym = mx.sym.contrib.DeformablePSROIPooling(spatial_scale=0.0625, sample_per_part=4, group_size=3, pooled_size=3,
+ output_dim=2, trans_std=0.1, no_trans=False, name='deformable_psroipool')
+
+ ctx_list = [{'ctx': mx.gpu(0),
+ 'deformable_psroipool_data': (1, 18, 14, 14),
+ 'deformable_psroipool_rois': (2, 5),
+ 'deformable_psroipool_trans': (2, 4, 3, 3),
+ 'type_dict': {'deformable_psroipool_data': np.float64, 'deformable_psroipool_rois': np.float64,
+ 'deformable_psroipool_trans': np.float64}},
+ {'ctx': mx.gpu(0),
+ 'deformable_psroipool_data': (1, 18, 14, 14),
+ 'deformable_psroipool_rois': (2, 5),
+ 'deformable_psroipool_trans': (2, 4, 3, 3),
+ 'type_dict': {'deformable_psroipool_data': np.float32, 'deformable_psroipool_rois': np.float32,
+ 'deformable_psroipool_trans': np.float32}},
+ {'ctx': mx.gpu(0),
+ 'deformable_psroipool_data': (1, 18, 14, 14),
+ 'deformable_psroipool_rois': (2, 5),
+ 'deformable_psroipool_trans': (2, 4, 3, 3),
+ 'type_dict': {'deformable_psroipool_data': np.float16, 'deformable_psroipool_rois': np.float16,
+ 'deformable_psroipool_trans': np.float16}},
+ ]
+
+ check_consistency(sym, ctx_list, grad_req={'deformable_psroipool_data': 'write',
+ 'deformable_psroipool_rois': 'null',
+ 'deformable_psroipool_trans': 'write'}, arg_params=arg_params)
def test_deformable_convolution_with_type():
- np.random.seed(1234)
- sym = mx.sym.contrib.DeformableConvolution(num_filter=3, kernel=(3,3), name='deformable_conv')
- # since atomicAdd does not support fp16 (which deformable conv uses in backward), we do not test fp16 here
- ctx_list = [{'ctx': mx.gpu(0),
- 'deformable_conv_data': (2, 2, 10, 10),
- 'deformable_conv_offset': (2, 18, 8, 8),
- 'type_dict': {'deformable_conv_data': np.float64, 'deformable_conv_offset': np.float64}},
- {'ctx': mx.gpu(0),
- 'deformable_conv_data': (2, 2, 10, 10),
- 'deformable_conv_offset': (2, 18, 8, 8),
- 'type_dict': {'deformable_conv_data': np.float32, 'deformable_conv_offset': np.float32}},
- # {'ctx': mx.gpu(0),
- # 'deformable_conv_data': (2, 2, 10, 10),
- # 'deformable_conv_offset': (2, 18, 8, 8),
- # 'type_dict': {'deformable_conv_data': np.float16, 'deformable_conv_offset': np.float16}},
- ]
- # wider tolerance needed for true-fp16 NCHW test above
- tol = {np.dtype(np.float16): 0.5,
- np.dtype(np.float32): 1e-3,
- np.dtype(np.float64): 1e-5,
- np.dtype(np.uint8): 0,
- np.dtype(np.int32): 0}
- check_consistency(sym, ctx_list, tol=tol)
- # test ability to turn off training on bias
- check_consistency(sym, ctx_list, grad_req={'deformable_conv_data': 'write',
- 'deformable_conv_offset': 'write',
- 'deformable_conv_weight': 'write',
- 'deformable_conv_bias': 'null'}, tol=tol)
+ with np_random_seed(1234):
+ sym = mx.sym.contrib.DeformableConvolution(num_filter=3, kernel=(3,3), name='deformable_conv')
+ # since atomicAdd does not support fp16 (which deformable conv uses in backward), we do not test fp16 here
+ ctx_list = [{'ctx': mx.gpu(0),
+ 'deformable_conv_data': (2, 2, 10, 10),
+ 'deformable_conv_offset': (2, 18, 8, 8),
+ 'type_dict': {'deformable_conv_data': np.float64, 'deformable_conv_offset': np.float64}},
+ {'ctx': mx.gpu(0),
+ 'deformable_conv_data': (2, 2, 10, 10),
+ 'deformable_conv_offset': (2, 18, 8, 8),
+ 'type_dict': {'deformable_conv_data': np.float32, 'deformable_conv_offset': np.float32}},
+ # {'ctx': mx.gpu(0),
+ # 'deformable_conv_data': (2, 2, 10, 10),
+ # 'deformable_conv_offset': (2, 18, 8, 8),
+ # 'type_dict': {'deformable_conv_data': np.float16, 'deformable_conv_offset': np.float16}},
+ ]
+ # wider tolerance needed for true-fp16 NCHW test above
+ tol = {np.dtype(np.float16): 0.5,
+ np.dtype(np.float32): 1e-3,
+ np.dtype(np.float64): 1e-5,
+ np.dtype(np.uint8): 0,
+ np.dtype(np.int32): 0}
+ check_consistency(sym, ctx_list, tol=tol)
+ # test ability to turn off training on bias
+ check_consistency(sym, ctx_list, grad_req={'deformable_conv_data': 'write',
+ 'deformable_conv_offset': 'write',
+ 'deformable_conv_weight': 'write',
+ 'deformable_conv_bias': 'null'}, tol=tol)
+
def test_deformable_convolution_options():
# 2D convolution
diff --git a/tests/python/unittest/test_executor.py b/tests/python/unittest/test_executor.py
index e3d977df65..e0ca78270e 100644
--- a/tests/python/unittest/test_executor.py
+++ b/tests/python/unittest/test_executor.py
@@ -17,7 +17,9 @@
import numpy as np
import mxnet as mx
+from mxnet.test_utils import np_random_seed, set_np_random_seed
+set_np_random_seed()
def reldiff(a, b):
diff = np.sum(np.abs(a - b))
@@ -82,56 +84,56 @@ def test_bind(disable_bulk_exec=False):
prev_bulk_inf_val = mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_INFERENCE", "0", "1")
prev_bulk_train_val = mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_TRAIN", "0", "1")
- np.random.seed(0)
- nrepeat = 10
- maxdim = 4
- for repeat in range(nrepeat):
- for dim in range(1, maxdim):
- check_bind_with_uniform(lambda x, y: x + y,
- lambda g, x, y: (g, g),
- dim)
- check_bind_with_uniform(lambda x, y: x - y,
- lambda g, x, y: (g, -g),
- dim)
- check_bind_with_uniform(lambda x, y: x * y,
- lambda g, x, y: (y * g, x * g),
- dim)
- check_bind_with_uniform(lambda x, y: x / y,
- lambda g, x, y: (g / y, -x * g/ (y**2)),
- dim)
-
- check_bind_with_uniform(lambda x, y: np.maximum(x, y),
- lambda g, x, y: (g * (x>y), g * (y>x)),
- dim,
- sf=mx.symbol.maximum)
- check_bind_with_uniform(lambda x, y: np.minimum(x, y),
- lambda g, x, y: (g * (x<y), g * (y<x)),
- dim,
- sf=mx.symbol.minimum)
- if disable_bulk_exec:
- mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_INFERENCE", prev_bulk_inf_val)
- mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_TRAIN", prev_bulk_train_val)
+ with np_random_seed(0):
+ nrepeat = 10
+ maxdim = 4
+ for repeat in range(nrepeat):
+ for dim in range(1, maxdim):
+ check_bind_with_uniform(lambda x, y: x + y,
+ lambda g, x, y: (g, g),
+ dim)
+ check_bind_with_uniform(lambda x, y: x - y,
+ lambda g, x, y: (g, -g),
+ dim)
+ check_bind_with_uniform(lambda x, y: x * y,
+ lambda g, x, y: (y * g, x * g),
+ dim)
+ check_bind_with_uniform(lambda x, y: x / y,
+ lambda g, x, y: (g / y, -x * g/ (y**2)),
+ dim)
+
+ check_bind_with_uniform(lambda x, y: np.maximum(x, y),
+ lambda g, x, y: (g * (x>y), g * (y>x)),
+ dim,
+ sf=mx.symbol.maximum)
+ check_bind_with_uniform(lambda x, y: np.minimum(x, y),
+ lambda g, x, y: (g * (x<y), g * (y<x)),
+ dim,
+ sf=mx.symbol.minimum)
+ if disable_bulk_exec:
+ mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_INFERENCE", prev_bulk_inf_val)
+ mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_TRAIN", prev_bulk_train_val)
def test_dot():
- np.random.seed(0)
- nrepeat = 10
- maxdim = 4
- for repeat in range(nrepeat):
- s =tuple(np.random.randint(1, 500, size=3))
- check_bind_with_uniform(lambda x, y: np.dot(x, y),
- lambda g, x, y: (np.dot(g, y.T), np.dot(x.T, g)),
- 2,
- lshape=(s[0], s[1]),
- rshape=(s[1], s[2]),
- sf = mx.symbol.dot)
- for repeat in range(nrepeat):
- s =tuple(np.random.randint(1, 500, size=1))
- check_bind_with_uniform(lambda x, y: np.dot(x, y),
- lambda g, x, y: (g * y, g * x),
- 2,
- lshape=(s[0],),
- rshape=(s[0],),
- sf = mx.symbol.dot)
+ with np_random_seed(0):
+ nrepeat = 10
+ maxdim = 4
+ for repeat in range(nrepeat):
+ s =tuple(np.random.randint(1, 500, size=3))
+ check_bind_with_uniform(lambda x, y: np.dot(x, y),
+ lambda g, x, y: (np.dot(g, y.T), np.dot(x.T, g)),
+ 2,
+ lshape=(s[0], s[1]),
+ rshape=(s[1], s[2]),
+ sf = mx.symbol.dot)
+ for repeat in range(nrepeat):
+ s =tuple(np.random.randint(1, 500, size=1))
+ check_bind_with_uniform(lambda x, y: np.dot(x, y),
+ lambda g, x, y: (g * y, g * x),
+ 2,
+ lshape=(s[0],),
+ rshape=(s[0],),
+ sf = mx.symbol.dot)
def test_reshape():
diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py
index 8ee4bfae00..8c20a62881 100644
--- a/tests/python/unittest/test_loss.py
+++ b/tests/python/unittest/test_loss.py
@@ -18,8 +18,9 @@
import mxnet as mx
import numpy as np
from mxnet import gluon
-from mxnet.test_utils import assert_almost_equal, default_context
+from mxnet.test_utils import assert_almost_equal, default_context, np_random_seed, set_np_random_seed
+set_np_random_seed()
def test_loss_ndarray():
output = mx.nd.array([1, 2, 3, 4])
@@ -63,39 +64,39 @@ def get_net(num_hidden, flatten=True):
def test_ce_loss():
- np.random.seed(1234)
- nclass = 10
- N = 20
- data = mx.random.uniform(-1, 1, shape=(N, nclass))
- label = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32')
- data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label')
- output = get_net(nclass)
- l = mx.symbol.Variable('label')
- Loss = gluon.loss.SoftmaxCrossEntropyLoss()
- loss = Loss(output, l)
- loss = mx.sym.make_loss(loss)
- mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
- mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
- eval_metric=mx.metric.Loss(), optimizer='adam')
- assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
+ with np_random_seed(1234):
+ nclass = 10
+ N = 20
+ data = mx.random.uniform(-1, 1, shape=(N, nclass))
+ label = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32')
+ data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label')
+ output = get_net(nclass)
+ l = mx.symbol.Variable('label')
+ Loss = gluon.loss.SoftmaxCrossEntropyLoss()
+ loss = Loss(output, l)
+ loss = mx.sym.make_loss(loss)
+ mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
+ mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
+ eval_metric=mx.metric.Loss(), optimizer='adam')
+ assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
def test_bce_loss():
- np.random.seed(1234)
- N = 20
- data = mx.random.uniform(-1, 1, shape=(N, 20))
- label = mx.nd.array(np.random.randint(2, size=(N,)), dtype='float32')
- data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label')
- output = get_net(1)
- l = mx.symbol.Variable('label')
- Loss = gluon.loss.SigmoidBinaryCrossEntropyLoss()
- loss = Loss(output, l)
- loss = mx.sym.make_loss(loss)
- mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
- mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
- eval_metric=mx.metric.Loss(), optimizer='adam',
- initializer=mx.init.Xavier(magnitude=2))
- assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.01
+ with np_random_seed(1234):
+ N = 20
+ data = mx.random.uniform(-1, 1, shape=(N, 20))
+ label = mx.nd.array(np.random.randint(2, size=(N,)), dtype='float32')
+ data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label')
+ output = get_net(1)
+ l = mx.symbol.Variable('label')
+ Loss = gluon.loss.SigmoidBinaryCrossEntropyLoss()
+ loss = Loss(output, l)
+ loss = mx.sym.make_loss(loss)
+ mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
+ mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
+ eval_metric=mx.metric.Loss(), optimizer='adam',
+ initializer=mx.init.Xavier(magnitude=2))
+ assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.01
def test_bce_equal_ce2():
N = 100
@@ -108,56 +109,56 @@ def test_bce_equal_ce2():
def test_kl_loss():
- np.random.seed(1234)
- N = 20
- data = mx.random.uniform(-1, 1, shape=(N, 10))
- label = mx.nd.softmax(mx.random.uniform(0, 1, shape=(N, 2)))
- data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label')
- output = mx.sym.log_softmax(get_net(2))
- l = mx.symbol.Variable('label')
- Loss = gluon.loss.KLDivLoss()
- loss = Loss(output, l)
- loss = mx.sym.make_loss(loss)
- mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
- mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
- eval_metric=mx.metric.Loss(), optimizer='adam')
- assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
+ with np_random_seed(1234):
+ N = 20
+ data = mx.random.uniform(-1, 1, shape=(N, 10))
+ label = mx.nd.softmax(mx.random.uniform(0, 1, shape=(N, 2)))
+ data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label')
+ output = mx.sym.log_softmax(get_net(2))
+ l = mx.symbol.Variable('label')
+ Loss = gluon.loss.KLDivLoss()
+ loss = Loss(output, l)
+ loss = mx.sym.make_loss(loss)
+ mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
+ mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
+ eval_metric=mx.metric.Loss(), optimizer='adam')
+ assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
def test_l2_loss():
- np.random.seed(1234)
- N = 20
- data = mx.random.uniform(-1, 1, shape=(N, 10))
- label = mx.random.uniform(-1, 1, shape=(N, 1))
- data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True)
- output = get_net(1)
- l = mx.symbol.Variable('label')
- Loss = gluon.loss.L2Loss()
- loss = Loss(output, l)
- loss = mx.sym.make_loss(loss)
- mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
- mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
- initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
- optimizer='adam')
- assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
+ with np_random_seed(1234):
+ N = 20
+ data = mx.random.uniform(-1, 1, shape=(N, 10))
+ label = mx.random.uniform(-1, 1, shape=(N, 1))
+ data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True)
+ output = get_net(1)
+ l = mx.symbol.Variable('label')
+ Loss = gluon.loss.L2Loss()
+ loss = Loss(output, l)
+ loss = mx.sym.make_loss(loss)
+ mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
+ mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
+ initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
+ optimizer='adam')
+ assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
def test_l1_loss():
- np.random.seed(1234)
- N = 20
- data = mx.random.uniform(-1, 1, shape=(N, 10))
- label = mx.random.uniform(-1, 1, shape=(N, 1))
- data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True)
- output = get_net(1)
- l = mx.symbol.Variable('label')
- Loss = gluon.loss.L1Loss()
- loss = Loss(output, l)
- loss = mx.sym.make_loss(loss)
- mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
- mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
- initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
- optimizer='adam')
- assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.1
+ with np_random_seed(1234):
+ N = 20
+ data = mx.random.uniform(-1, 1, shape=(N, 10))
+ label = mx.random.uniform(-1, 1, shape=(N, 1))
+ data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True)
+ output = get_net(1)
+ l = mx.symbol.Variable('label')
+ Loss = gluon.loss.L1Loss()
+ loss = Loss(output, l)
+ loss = mx.sym.make_loss(loss)
+ mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
+ mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
+ initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
+ optimizer='adam')
+ assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.1
def test_ctc_loss():
@@ -187,144 +188,144 @@ def test_ctc_loss():
def test_ctc_loss_train():
- np.random.seed(1234)
- N = 20
- data = mx.random.uniform(-1, 1, shape=(N, 20, 10))
- label = mx.nd.arange(4, repeat=N).reshape((N, 4))
- data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True)
- output = get_net(5, False)
- l = mx.symbol.Variable('label')
- Loss = gluon.loss.CTCLoss(layout='NTC', label_layout='NT')
- loss = Loss(output, l)
- loss = mx.sym.make_loss(loss)
- mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
- mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.},
- initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
- optimizer='adam')
- assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 10
+ with np_random_seed(1234):
+ N = 20
+ data = mx.random.uniform(-1, 1, shape=(N, 20, 10))
+ label = mx.nd.arange(4, repeat=N).reshape((N, 4))
+ data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True)
+ output = get_net(5, False)
+ l = mx.symbol.Variable('label')
+ Loss = gluon.loss.CTCLoss(layout='NTC', label_layout='NT')
+ loss = Loss(output, l)
+ loss = mx.sym.make_loss(loss)
+ mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
+ mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.},
+ initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
+ optimizer='adam')
+ assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 10
def test_sample_weight_loss():
- np.random.seed(1234)
- nclass = 10
- N = 20
- data = mx.random.uniform(-1, 1, shape=(N, nclass))
- label = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32')
- weight = mx.nd.array([1 for i in range(10)] + [0 for i in range(10)])
- data_iter = mx.io.NDArrayIter(data, {'label': label, 'w': weight}, batch_size=10)
- output = get_net(nclass)
- l = mx.symbol.Variable('label')
- w = mx.symbol.Variable('w')
- Loss = gluon.loss.SoftmaxCrossEntropyLoss()
- loss = Loss(output, l, w)
- loss = mx.sym.make_loss(loss)
- mod = mx.mod.Module(loss, data_names=('data',), label_names=('label', 'w'))
- mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
- eval_metric=mx.metric.Loss(), optimizer='adam')
- data_iter = mx.io.NDArrayIter(data[10:], {'label': label, 'w': weight}, batch_size=10)
- score = mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1]
- assert score > 1
- data_iter = mx.io.NDArrayIter(data[:10], {'label': label, 'w': weight}, batch_size=10)
- score = mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1]
- assert score < 0.05
+ with np_random_seed(1234):
+ nclass = 10
+ N = 20
+ data = mx.random.uniform(-1, 1, shape=(N, nclass))
+ label = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32')
+ weight = mx.nd.array([1 for i in range(10)] + [0 for i in range(10)])
+ data_iter = mx.io.NDArrayIter(data, {'label': label, 'w': weight}, batch_size=10)
+ output = get_net(nclass)
+ l = mx.symbol.Variable('label')
+ w = mx.symbol.Variable('w')
+ Loss = gluon.loss.SoftmaxCrossEntropyLoss()
+ loss = Loss(output, l, w)
+ loss = mx.sym.make_loss(loss)
+ mod = mx.mod.Module(loss, data_names=('data',), label_names=('label', 'w'))
+ mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
+ eval_metric=mx.metric.Loss(), optimizer='adam')
+ data_iter = mx.io.NDArrayIter(data[10:], {'label': label, 'w': weight}, batch_size=10)
+ score = mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1]
+ assert score > 1
+ data_iter = mx.io.NDArrayIter(data[:10], {'label': label, 'w': weight}, batch_size=10)
+ score = mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1]
+ assert score < 0.05
def test_saveload():
mx.random.seed(1234)
- np.random.seed(1234)
- nclass = 10
- N = 20
- data = mx.random.uniform(-1, 1, shape=(N, nclass))
- label = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32')
- data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label')
- output = get_net(nclass)
- l = mx.symbol.Variable('label')
- Loss = gluon.loss.SoftmaxCrossEntropyLoss()
- loss = Loss(output, l)
- loss = mx.sym.make_loss(loss)
- mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
- mod.fit(data_iter, num_epoch=100, optimizer_params={'learning_rate': 1.},
- eval_metric=mx.metric.Loss())
- mod.save_checkpoint('test', 100, save_optimizer_states=True)
- mod = mx.mod.Module.load('test', 100, load_optimizer_states=True,
- data_names=('data',), label_names=('label',))
- mod.fit(data_iter, num_epoch=100, optimizer_params={'learning_rate': 1.},
- eval_metric=mx.metric.Loss())
- assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
+ with np_random_seed(1234):
+ nclass = 10
+ N = 20
+ data = mx.random.uniform(-1, 1, shape=(N, nclass))
+ label = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32')
+ data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label')
+ output = get_net(nclass)
+ l = mx.symbol.Variable('label')
+ Loss = gluon.loss.SoftmaxCrossEntropyLoss()
+ loss = Loss(output, l)
+ loss = mx.sym.make_loss(loss)
+ mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
+ mod.fit(data_iter, num_epoch=100, optimizer_params={'learning_rate': 1.},
+ eval_metric=mx.metric.Loss())
+ mod.save_checkpoint('test', 100, save_optimizer_states=True)
+ mod = mx.mod.Module.load('test', 100, load_optimizer_states=True,
+ data_names=('data',), label_names=('label',))
+ mod.fit(data_iter, num_epoch=100, optimizer_params={'learning_rate': 1.},
+ eval_metric=mx.metric.Loss())
+ assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
def test_huber_loss():
- np.random.seed(1234)
- N = 20
- data = mx.random.uniform(-1, 1, shape=(N, 10))
- label = mx.random.uniform(-1, 1, shape=(N, 1))
- data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True)
- output = get_net(1)
- l = mx.symbol.Variable('label')
- Loss = gluon.loss.HuberLoss()
- loss = Loss(output, l)
- loss = mx.sym.make_loss(loss)
- mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
- mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
- initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
- optimizer='adam')
- assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
+ with np_random_seed(1234):
+ N = 20
+ data = mx.random.uniform(-1, 1, shape=(N, 10))
+ label = mx.random.uniform(-1, 1, shape=(N, 1))
+ data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True)
+ output = get_net(1)
+ l = mx.symbol.Variable('label')
+ Loss = gluon.loss.HuberLoss()
+ loss = Loss(output, l)
+ loss = mx.sym.make_loss(loss)
+ mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
+ mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
+ initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
+ optimizer='adam')
+ assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
def test_hinge_loss():
- np.random.seed(1234)
- N = 20
- data = mx.random.uniform(-1, 1, shape=(N, 10))
- label = mx.nd.sign(mx.random.uniform(-1, 1, shape=(N, 1)))
- data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True)
- output = get_net(1)
- l = mx.symbol.Variable('label')
- Loss = gluon.loss.HingeLoss()
- loss = Loss(output, l)
- loss = mx.sym.make_loss(loss)
- mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
- mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
- initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
- optimizer='adam')
- assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
+ with np_random_seed(1234):
+ N = 20
+ data = mx.random.uniform(-1, 1, shape=(N, 10))
+ label = mx.nd.sign(mx.random.uniform(-1, 1, shape=(N, 1)))
+ data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True)
+ output = get_net(1)
+ l = mx.symbol.Variable('label')
+ Loss = gluon.loss.HingeLoss()
+ loss = Loss(output, l)
+ loss = mx.sym.make_loss(loss)
+ mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
+ mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
+ initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
+ optimizer='adam')
+ assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
def test_squared_hinge_loss():
- np.random.seed(1234)
- N = 20
- data = mx.random.uniform(-1, 1, shape=(N, 10))
- label = mx.nd.sign(mx.random.uniform(-1, 1, shape=(N, 1)))
- data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True)
- output = get_net(1)
- l = mx.symbol.Variable('label')
- Loss = gluon.loss.SquaredHingeLoss()
- loss = Loss(output, l)
- loss = mx.sym.make_loss(loss)
- mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
- mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
- initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
- optimizer='adam')
- assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
+ with np_random_seed(1234):
+ N = 20
+ data = mx.random.uniform(-1, 1, shape=(N, 10))
+ label = mx.nd.sign(mx.random.uniform(-1, 1, shape=(N, 1)))
+ data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True)
+ output = get_net(1)
+ l = mx.symbol.Variable('label')
+ Loss = gluon.loss.SquaredHingeLoss()
+ loss = Loss(output, l)
+ loss = mx.sym.make_loss(loss)
+ mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',))
+ mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
+ initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
+ optimizer='adam')
+ assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
def test_triplet_loss():
- np.random.seed(1234)
- N = 20
- data = mx.random.uniform(-1, 1, shape=(N, 10))
- pos = mx.random.uniform(-1, 1, shape=(N, 10))
- neg = mx.random.uniform(-1, 1, shape=(N, 10))
- data_iter = mx.io.NDArrayIter(data, {'pos': pos, 'neg': neg}, batch_size=10,
- label_name='label', shuffle=True)
- output = get_net(10)
- pos = mx.symbol.Variable('pos')
- neg = mx.symbol.Variable('neg')
- Loss = gluon.loss.TripletLoss()
- loss = Loss(output, pos, neg)
- loss = mx.sym.make_loss(loss)
- mod = mx.mod.Module(loss, data_names=('data',), label_names=('pos','neg'))
- mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
- initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
- optimizer='adam')
- assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
+ with np_random_seed(1234):
+ N = 20
+ data = mx.random.uniform(-1, 1, shape=(N, 10))
+ pos = mx.random.uniform(-1, 1, shape=(N, 10))
+ neg = mx.random.uniform(-1, 1, shape=(N, 10))
+ data_iter = mx.io.NDArrayIter(data, {'pos': pos, 'neg': neg}, batch_size=10,
+ label_name='label', shuffle=True)
+ output = get_net(10)
+ pos = mx.symbol.Variable('pos')
+ neg = mx.symbol.Variable('neg')
+ Loss = gluon.loss.TripletLoss()
+ loss = Loss(output, pos, neg)
+ loss = mx.sym.make_loss(loss)
+ mod = mx.mod.Module(loss, data_names=('data',), label_names=('pos','neg'))
+ mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01},
+ initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(),
+ optimizer='adam')
+ assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
if __name__ == '__main__':
diff --git a/tests/python/unittest/test_module.py b/tests/python/unittest/test_module.py
index bfe129e883..c0a73daf48 100644
--- a/tests/python/unittest/test_module.py
+++ b/tests/python/unittest/test_module.py
@@ -24,8 +24,7 @@
from common import assertRaises
from collections import namedtuple
-import numpy.random as rnd
-
+set_np_random_seed()
def test_module_dtype():
dtype = np.float16
@@ -465,7 +464,6 @@ def test_factorization_machine_module():
def check_factorization_machine_module(optimizer=None, num_epochs=None):
print("check_factorization_machine_module( {} )".format(optimizer))
mx.random.seed(11)
- rnd.seed(11)
def fm(factor_size, feature_dim, init):
x = mx.symbol.Variable("data", stype='csr')
@@ -493,70 +491,71 @@ def fm(factor_size, feature_dim, init):
model = mx.symbol.LinearRegressionOutput(data=model, label=y)
return model
- # model
- init = mx.initializer.Normal(sigma=0.01)
- factor_size = 4
- feature_dim = 10000
- model = fm(factor_size, feature_dim, init)
-
- # data iter
- num_batches = 5
- batch_size = 64
- num_samples = batch_size * num_batches
- # generate some random csr data
- csr_nd = rand_ndarray((num_samples, feature_dim), 'csr', 0.1)
- label = mx.nd.ones((num_samples,1))
- # the alternative is to use LibSVMIter
- train_iter = mx.io.NDArrayIter(data=csr_nd,
- label={'label':label},
- batch_size=batch_size,
- last_batch_handle='discard')
- # create module
- mod = mx.mod.Module(symbol=model, data_names=['data'], label_names=['label'])
- # allocate memory by given the input data and lable shapes
- mod.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label)
- # initialize parameters by uniform random numbers
- mod.init_params(initializer=init)
- if optimizer == 'sgd':
- # use Sparse SGD with learning rate 0.1 to train
- sgd = mx.optimizer.SGD(momentum=0.1, clip_gradient=5.0, learning_rate=0.01,
- rescale_grad=1.0/batch_size)
- mod.init_optimizer(optimizer=sgd)
- if num_epochs is None:
- num_epochs = 10
- expected_accuracy = 0.02
- elif optimizer == 'adam':
- # use Sparse Adam to train
- adam = mx.optimizer.Adam(clip_gradient=5.0, learning_rate=0.001,
- rescale_grad=1.0/batch_size)
- mod.init_optimizer(optimizer=adam)
- if num_epochs is None:
- num_epochs = 10
- expected_accuracy = 0.05
- elif optimizer == 'adagrad':
- # use Sparse AdaGrad with learning rate 0.1 to train
- adagrad = mx.optimizer.AdaGrad(clip_gradient=5.0, learning_rate=0.01,
- rescale_grad=1.0/batch_size)
- mod.init_optimizer(optimizer=adagrad)
- if num_epochs is None:
- num_epochs = 20
- expected_accuracy = 0.09
- else:
- raise AssertionError("Unsupported optimizer type '" + optimizer + "' specified")
- # use accuracy as the metric
- metric = mx.metric.create('MSE')
- # train 'num_epochs' epoch
- for epoch in range(num_epochs):
- train_iter.reset()
- metric.reset()
- for batch in train_iter:
- mod.forward(batch, is_train=True) # compute predictions
- mod.update_metric(metric, batch.label) # accumulate prediction accuracy
- mod.backward() # compute gradients
- mod.update() # update parameters
- print('Epoch %d, Training %s' % (epoch, metric.get()))
- if num_epochs > 1:
- assert(metric.get()[1] < expected_accuracy)
+ with np_random_seed(11):
+ # model
+ init = mx.initializer.Normal(sigma=0.01)
+ factor_size = 4
+ feature_dim = 10000
+ model = fm(factor_size, feature_dim, init)
+
+ # data iter
+ num_batches = 5
+ batch_size = 64
+ num_samples = batch_size * num_batches
+ # generate some random csr data
+ csr_nd = rand_ndarray((num_samples, feature_dim), 'csr', 0.1)
+ label = mx.nd.ones((num_samples,1))
+ # the alternative is to use LibSVMIter
+ train_iter = mx.io.NDArrayIter(data=csr_nd,
+ label={'label':label},
+ batch_size=batch_size,
+ last_batch_handle='discard')
+ # create module
+ mod = mx.mod.Module(symbol=model, data_names=['data'], label_names=['label'])
+ # allocate memory by given the input data and lable shapes
+ mod.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label)
+ # initialize parameters by uniform random numbers
+ mod.init_params(initializer=init)
+ if optimizer == 'sgd':
+ # use Sparse SGD with learning rate 0.1 to train
+ sgd = mx.optimizer.SGD(momentum=0.1, clip_gradient=5.0, learning_rate=0.01,
+ rescale_grad=1.0/batch_size)
+ mod.init_optimizer(optimizer=sgd)
+ if num_epochs is None:
+ num_epochs = 10
+ expected_accuracy = 0.02
+ elif optimizer == 'adam':
+ # use Sparse Adam to train
+ adam = mx.optimizer.Adam(clip_gradient=5.0, learning_rate=0.001,
+ rescale_grad=1.0/batch_size)
+ mod.init_optimizer(optimizer=adam)
+ if num_epochs is None:
+ num_epochs = 10
+ expected_accuracy = 0.05
+ elif optimizer == 'adagrad':
+ # use Sparse AdaGrad with learning rate 0.1 to train
+ adagrad = mx.optimizer.AdaGrad(clip_gradient=5.0, learning_rate=0.01,
+ rescale_grad=1.0/batch_size)
+ mod.init_optimizer(optimizer=adagrad)
+ if num_epochs is None:
+ num_epochs = 20
+ expected_accuracy = 0.09
+ else:
+ raise AssertionError("Unsupported optimizer type '" + optimizer + "' specified")
+ # use accuracy as the metric
+ metric = mx.metric.create('MSE')
+ # train 'num_epochs' epoch
+ for epoch in range(num_epochs):
+ train_iter.reset()
+ metric.reset()
+ for batch in train_iter:
+ mod.forward(batch, is_train=True) # compute predictions
+ mod.update_metric(metric, batch.label) # accumulate prediction accuracy
+ mod.backward() # compute gradients
+ mod.update() # update parameters
+ print('Epoch %d, Training %s' % (epoch, metric.get()))
+ if num_epochs > 1:
+ assert(metric.get()[1] < expected_accuracy)
check_factorization_machine_module('adam')
check_factorization_machine_module('sgd')
diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py
index e5dddeb95a..ef100e235e 100644
--- a/tests/python/unittest/test_ndarray.py
+++ b/tests/python/unittest/test_ndarray.py
@@ -25,6 +25,8 @@
from numpy.testing import assert_allclose
import unittest
+set_np_random_seed()
+
def check_with_uniform(uf, arg_shapes, dim=None, npuf=None, rmin=-10, type_list=[np.float32]):
"""check function consistency with uniform random numbers"""
if isinstance(arg_shapes, int):
@@ -112,21 +114,21 @@ def test_ndarray_setitem():
def test_ndarray_elementwise():
- np.random.seed(0)
- nrepeat = 10
- maxdim = 4
- all_type = [np.float32, np.float64, np.float16, np.uint8, np.int32]
- real_type = [np.float32, np.float64, np.float16]
- for repeat in range(nrepeat):
- for dim in range(1, maxdim):
- check_with_uniform(lambda x, y: x + y, 2, dim, type_list=all_type)
- check_with_uniform(lambda x, y: x - y, 2, dim, type_list=all_type)
- check_with_uniform(lambda x, y: x * y, 2, dim, type_list=all_type)
- check_with_uniform(lambda x, y: x / y, 2, dim, type_list=real_type)
- check_with_uniform(lambda x, y: x / y, 2, dim, rmin=1, type_list=all_type)
- check_with_uniform(mx.nd.sqrt, 1, dim, np.sqrt, rmin=0)
- check_with_uniform(mx.nd.square, 1, dim, np.square, rmin=0)
- check_with_uniform(lambda x: mx.nd.norm(x).asscalar(), 1, dim, np.linalg.norm)
+ with np_random_seed(0):
+ nrepeat = 10
+ maxdim = 4
+ all_type = [np.float32, np.float64, np.float16, np.uint8, np.int32]
+ real_type = [np.float32, np.float64, np.float16]
+ for repeat in range(nrepeat):
+ for dim in range(1, maxdim):
+ check_with_uniform(lambda x, y: x + y, 2, dim, type_list=all_type)
+ check_with_uniform(lambda x, y: x - y, 2, dim, type_list=all_type)
+ check_with_uniform(lambda x, y: x * y, 2, dim, type_list=all_type)
+ check_with_uniform(lambda x, y: x / y, 2, dim, type_list=real_type)
+ check_with_uniform(lambda x, y: x / y, 2, dim, rmin=1, type_list=all_type)
+ check_with_uniform(mx.nd.sqrt, 1, dim, np.sqrt, rmin=0)
+ check_with_uniform(mx.nd.square, 1, dim, np.square, rmin=0)
+ check_with_uniform(lambda x: mx.nd.norm(x).asscalar(), 1, dim, np.linalg.norm)
def test_ndarray_elementwisesum():
ones = mx.nd.ones((10,), dtype=np.int32)
@@ -220,52 +222,52 @@ def test_ndarray_scalar():
assert(np.sum(d.asnumpy()) < 1e-5)
def test_ndarray_pickle():
- np.random.seed(0)
- maxdim = 5
- nrepeat = 10
- for repeat in range(nrepeat):
- for dim in range(1, maxdim):
- a = random_ndarray(dim)
- b = mx.nd.empty(a.shape)
- a[:] = np.random.uniform(-10, 10, a.shape)
- b[:] = np.random.uniform(-10, 10, a.shape)
- a = a + b
- data = pkl.dumps(a)
- a2 = pkl.loads(data)
- assert np.sum(a.asnumpy() != a2.asnumpy()) == 0
+ with np_random_seed(0):
+ maxdim = 5
+ nrepeat = 10
+ for repeat in range(nrepeat):
+ for dim in range(1, maxdim):
+ a = random_ndarray(dim)
+ b = mx.nd.empty(a.shape)
+ a[:] = np.random.uniform(-10, 10, a.shape)
+ b[:] = np.random.uniform(-10, 10, a.shape)
+ a = a + b
+ data = pkl.dumps(a)
+ a2 = pkl.loads(data)
+ assert np.sum(a.asnumpy() != a2.asnumpy()) == 0
def test_ndarray_saveload():
- np.random.seed(0)
- nrepeat = 10
- fname = 'tmp_list.bin'
- for repeat in range(nrepeat):
- data = []
- # test save/load as list
- for i in range(10):
- data.append(random_ndarray(np.random.randint(1, 5)))
- mx.nd.save(fname, data)
- data2 = mx.nd.load(fname)
- assert len(data) == len(data2)
- for x, y in zip(data, data2):
- assert np.sum(x.asnumpy() != y.asnumpy()) == 0
- # test save/load as dict
- dmap = {'ndarray xx %s' % i : x for i, x in enumerate(data)}
- mx.nd.save(fname, dmap)
- dmap2 = mx.nd.load(fname)
- assert len(dmap2) == len(dmap)
- for k, x in dmap.items():
- y = dmap2[k]
- assert np.sum(x.asnumpy() != y.asnumpy()) == 0
- # test save/load as ndarray
- # we expect the single ndarray to be converted into a list containing the ndarray
- single_ndarray = data[0]
- mx.nd.save(fname, single_ndarray)
- single_ndarray_loaded = mx.nd.load(fname)
- assert len(single_ndarray_loaded) == 1
- single_ndarray_loaded = single_ndarray_loaded[0]
- assert np.sum(single_ndarray.asnumpy() != single_ndarray_loaded.asnumpy()) == 0
- os.remove(fname)
+ with np_random_seed(0):
+ nrepeat = 10
+ fname = 'tmp_list.bin'
+ for repeat in range(nrepeat):
+ data = []
+ # test save/load as list
+ for i in range(10):
+ data.append(random_ndarray(np.random.randint(1, 5)))
+ mx.nd.save(fname, data)
+ data2 = mx.nd.load(fname)
+ assert len(data) == len(data2)
+ for x, y in zip(data, data2):
+ assert np.sum(x.asnumpy() != y.asnumpy()) == 0
+ # test save/load as dict
+ dmap = {'ndarray xx %s' % i : x for i, x in enumerate(data)}
+ mx.nd.save(fname, dmap)
+ dmap2 = mx.nd.load(fname)
+ assert len(dmap2) == len(dmap)
+ for k, x in dmap.items():
+ y = dmap2[k]
+ assert np.sum(x.asnumpy() != y.asnumpy()) == 0
+ # test save/load as ndarray
+ # we expect the single ndarray to be converted into a list containing the ndarray
+ single_ndarray = data[0]
+ mx.nd.save(fname, single_ndarray)
+ single_ndarray_loaded = mx.nd.load(fname)
+ assert len(single_ndarray_loaded) == 1
+ single_ndarray_loaded = single_ndarray_loaded[0]
+ assert np.sum(single_ndarray.asnumpy() != single_ndarray_loaded.asnumpy()) == 0
+ os.remove(fname)
def test_ndarray_legacy_load():
data = []
@@ -345,6 +347,7 @@ def test_clip():
assert B1[i] <= 2
def test_dot():
+ atol = 1e-5
# Test normal dot
a = np.random.uniform(-3, 3, (3, 4))
b = np.random.uniform(-3, 3, (4, 5))
@@ -352,7 +355,7 @@ def test_dot():
A = mx.nd.array(a)
B = mx.nd.array(b)
C = mx.nd.dot(A, B)
- assert_almost_equal(c, C.asnumpy())
+ assert_almost_equal(c, C.asnumpy(), atol=atol)
# Test dot with transpose kargs
a = np.random.uniform(-3, 3, (3, 4))
b = np.random.uniform(-3, 3, (3, 5))
@@ -360,7 +363,7 @@ def test_dot():
A = mx.nd.array(a)
B = mx.nd.array(b)
C = mx.nd.dot(A, B, transpose_a=True)
- assert_almost_equal(c, C.asnumpy())
+ assert_almost_equal(c, C.asnumpy(), atol=atol)
# Test dot with transpose kargs
a = np.random.uniform(-3, 3, (3, 4))
b = np.random.uniform(-3, 3, (5, 4))
@@ -368,7 +371,7 @@ def test_dot():
A = mx.nd.array(a)
B = mx.nd.array(b)
C = mx.nd.dot(A, B, transpose_b=True)
- assert_almost_equal(c, C.asnumpy())
+ assert_almost_equal(c, C.asnumpy(), atol=atol)
# Test dot with transpose kargs
a = np.random.uniform(-3, 3, (4, 3))
b = np.random.uniform(-3, 3, (5, 4))
@@ -376,7 +379,7 @@ def test_dot():
A = mx.nd.array(a)
B = mx.nd.array(b)
C = mx.nd.dot(A, B, transpose_a=True, transpose_b=True)
- assert_almost_equal(c, C.asnumpy())
+ assert_almost_equal(c, C.asnumpy(), atol=atol)
diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py
index 105d1ce2c1..0985e51dd0 100644
--- a/tests/python/unittest/test_operator.py
+++ b/tests/python/unittest/test_operator.py
@@ -25,6 +25,7 @@
from mxnet.test_utils import *
import unittest
+set_np_random_seed()
def np_softmax(x, axis=-1):
# fix for old numpy on Travis not supporting keepdims
@@ -62,13 +63,15 @@ def check_elementwise_sum_with_shape(shape, n):
def test_elementwise_sum():
- np.random.seed(0)
- nrepeat = 2
- maxdim = 4
- for repeat in range(nrepeat):
- for dim in range(1, maxdim):
- shape = tuple(np.random.randint(1, int(1000**(1.0/dim)), size=dim))
- check_elementwise_sum_with_shape(shape, np.random.randint(1, 8))
+ # Comparing random sums that are close to zero is troublesome.
+ # Pick a specific seed without altering the determinism of the other tests.
+ with np_random_seed(0):
+ nrepeat = 2
+ maxdim = 4
+ for repeat in range(nrepeat):
+ for dim in range(1, maxdim):
+ shape = tuple(np.random.randint(1, int(1000**(1.0/dim)), size=dim))
+ check_elementwise_sum_with_shape(shape, np.random.randint(1, 8))
def check_concat_with_shape(shapes, dimension, skip_second):
@@ -489,13 +492,13 @@ def test_embedding():
arg_map["data"][:] = np_data
arg_map["embed_weight"][:] = np_weight
exe_test.forward(is_train=True)
- assert_almost_equal(exe_test.outputs[0].asnumpy(), np.dot(np_onehot, np_weight))
+ assert_almost_equal(exe_test.outputs[0].asnumpy(), np.dot(np_onehot, np_weight), rtol=1e-5, atol=1e-5)
# backward
np_grad = np.random.uniform(-1, 1, exe_test.outputs[0].shape)
grad = mx.nd.zeros(np_grad.shape)
grad[:] = np_grad
exe_test.backward([grad])
- assert_almost_equal(grad_map["embed_weight"].asnumpy(), np.dot(np_onehot.T, np_grad))
+ assert_almost_equal(grad_map["embed_weight"].asnumpy(), np.dot(np_onehot.T, np_grad), rtol=1e-5, atol=1e-5)
# check ops handle duplicate input correctly.
@@ -1038,10 +1041,10 @@ def test_depthwise_convolution():
exe2.backward(exe2.outputs[0])
for arr1, arr2 in zip(exe1.outputs + exe1.grad_arrays, exe2.outputs + exe2.grad_arrays):
- np.testing.assert_allclose(arr1.asnumpy(), arr2.asnumpy(), rtol=1e-3, atol=1e-4)
+ np.testing.assert_allclose(arr1.asnumpy(), arr2.asnumpy(), rtol=1e-3, atol=1e-3)
-def gen_broadcast_data(idx):
+def gen_broadcast_data(idx=None):
# Manually set test cases
binary_op_data_shape = np.array(
[[[2, 5, 1, 30, 7], [1, 5, 448, 30, 1]],
@@ -1075,7 +1078,7 @@ def gen_broadcast_data(idx):
[[1, 24, 103, 17, 18], [1, 24, 1, 1, 1]],
[[1, 1, 1, 1, 2], [1, 24, 194, 50, 1]],
[[1, 1, 107, 84, 9], [1, 1, 1, 1, 1]]])
- if idx < binary_op_data_shape.shape[0]:
+ if idx != None and idx < binary_op_data_shape.shape[0]:
l_shape = binary_op_data_shape[idx][0]
r_shape = binary_op_data_shape[idx][1]
else:
@@ -1196,8 +1199,10 @@ def test_bdiv(a, b):
def test_bmod(a, b):
c = a % b
- check_binary_op_forward(c, lambda a, b: a % b, gen_binary_data)
- check_binary_op_backward(c, lambda g_out, a, b: (g_out, - g_out * (a // b)), gen_binary_data)
+ # '%' is sensitive to the precision of the calculation. Force numpy to match mxnet's float32.
+ check_binary_op_forward(c, lambda a, b: np.float32(a) % np.float32(b), gen_binary_data)
+ check_binary_op_backward(c,
+ lambda g_out, a, b: (g_out, - g_out * (np.float32(a) // np.float32(b))), gen_binary_data)
def test_bmod_int(a, b):
c = mx.sym.cast(a, dtype='int32') % mx.sym.cast(b, dtype='int32')
@@ -1271,17 +1276,34 @@ def test_bequal(a, b):
mx_nd_func=mx.nd.equal)
check_binary_op_backward(c, lambda g_out, a, b: (np.zeros_like(a), np.zeros_like(b)), gen_broadcast_data_int)
+ # What is the closest pair of elements in the two input arrays? This is the minimum
+ # workable eps for the finite difference method of gradient calculation.
+ def required_eps(input_data):
+ return abs(input_data[0] - input_data[1]).min()
+
+ # The finite difference method may calculate an incorrect gradient for z = max(x,y) if
+ # the x and y values are closer than eps. One approach to dealing with this would be to
+ # lower the eps based on the data, but eps values smaller than 2**(-18) were seen to be
+ # problematic because of other multipliers involved in the calculation. Instead, we simply
+ # regenerate the dataset if it is incompatible with the desired eps.
+ def gen_min_max_input_data(desired_eps):
+ input_data = gen_broadcast_data()
+ while required_eps(input_data) < desired_eps:
+ print('Note: random input values unsuitable for the finite difference method detected, regenerating.')
+ input_data = gen_broadcast_data()
+ return input_data
+
def test_bmax(a, b):
c = mx.sym.broadcast_maximum(a, b)
check_binary_op_forward(c, lambda x, y: np.maximum(x, y), gen_broadcast_data, mx_nd_func=mx.nd.maximum)
- # pass idx=200 to gen_broadcast_data so that generated ndarrays' sizes are not too big
- check_numeric_gradient(c, gen_broadcast_data(idx=200), rtol=1e-2, atol=1e-3)
+ eps = 2**(-12)
+ check_numeric_gradient(c, gen_min_max_input_data(eps), numeric_eps=eps, rtol=1e-2, atol=1e-3)
def test_bmin(a, b):
c = mx.sym.broadcast_minimum(a, b)
check_binary_op_forward(c, lambda x, y: np.minimum(x, y), gen_broadcast_data, mx_nd_func=mx.nd.minimum)
- # pass idx=200 to gen_broadcast_data so that generated ndarrays' sizes are not too big
- check_numeric_gradient(c, gen_broadcast_data(idx=200), rtol=1e-2, atol=1e-3)
+ eps = 2**(-12)
+ check_numeric_gradient(c, gen_min_max_input_data(eps), numeric_eps=eps, rtol=1e-2, atol=1e-3)
test_bplus(a, b)
test_bminus(a, b)
@@ -1709,67 +1731,67 @@ def test_stn():
def test_dot(ctx=default_context()):
- np.random.seed(1234)
- dtypes = ['float32', 'float64']
-
- # Test normal dot.
- for data_type in dtypes:
- for m in range(1, 5):
- for k in range(1, 5):
- for n in range(1, 5):
- a_npy = np.random.normal(0, 1, (m, k))
- a_npy = a_npy.astype(data_type)
- b_npy = np.random.normal(0, 1, (k, n))
- b_npy = b_npy.astype(data_type)
- c_npy = np.empty((m, n), dtype=data_type)
- ograd_npy = np.random.normal(0, 1, (m, n))
- ograd_npy = ograd_npy.astype(data_type)
- agrad_npy = np.empty((m, k), dtype=data_type)
- bgrad_npy = np.empty((k, n), dtype=data_type)
- c_npy[:, :] = np.dot(a_npy[:, :], b_npy[:, :])
- bgrad_npy[:, :] = np.dot(a_npy[:, :].T, ograd_npy[:, :])
- agrad_npy[:, :] = np.dot(ograd_npy[:, :], b_npy[:, :].T)
- a = mx.sym.Variable('a', dtype=data_type)
- b = mx.sym.Variable('b', dtype=data_type)
- c = mx.sym.dot(a, b)
- exe = c.simple_bind(ctx=ctx, a=a_npy.shape, b=b_npy.shape)
- outputs = exe.forward(is_train=True, a=a_npy, b=b_npy)
- assert_almost_equal(outputs[0].asnumpy(), c_npy, rtol=1e-3)
- exe.backward(out_grads=[mx.nd.array(ograd_npy, mx.cpu())])
- assert_almost_equal(exe.grad_dict['a'].asnumpy(), agrad_npy, rtol=1e-3)
- assert_almost_equal(exe.grad_dict['b'].asnumpy(), bgrad_npy, rtol=1e-3)
-
- # Test dot with transpose flag using gradient checker.
- def dot_sym(data_type):
- x = mx.sym.Variable('x', dtype=data_type)
- y = mx.sym.Variable('y', dtype=data_type)
- return mx.sym.dot(x, y)
-
- def dot_sym_xT(data_type):
- x = mx.sym.Variable('x', dtype=data_type)
- y = mx.sym.Variable('y', dtype=data_type)
- return mx.sym.dot(x, y, transpose_a=True)
-
- def dot_sym_yT(data_type):
- x = mx.sym.Variable('x', dtype=data_type)
- y = mx.sym.Variable('y', dtype=data_type)
- return mx.sym.dot(x, y, transpose_b=True)
-
- def dot_sym_xT_yT(data_type):
- x = mx.sym.Variable('x', dtype=data_type)
- y = mx.sym.Variable('y', dtype=data_type)
- return mx.sym.dot(x, y, transpose_a=True, transpose_b=True)
+ with np_random_seed(1234):
+ dtypes = ['float32', 'float64']
- for data_type in dtypes:
- for ashape, bshape in [((3, 4), (4, 5)), ((2, 3, 4), (4, 5, 6))]:
- m1_npy = np.random.uniform(-1, 1, ashape)
- m1_npy = m1_npy.astype(data_type)
- m2_npy = np.random.uniform(-1, 1, bshape)
- m2_npy = m2_npy.astype(data_type)
- check_numeric_gradient(dot_sym(data_type), [m1_npy, m2_npy], numeric_eps=1e-1, rtol=2e-2, atol=1e-3)
- check_numeric_gradient(dot_sym_xT(data_type), [m1_npy.T, m2_npy], numeric_eps=1e-1, rtol=2e-2, atol=1e-3)
- check_numeric_gradient(dot_sym_yT(data_type), [m1_npy, m2_npy.T], numeric_eps=1e-1, rtol=2e-2, atol=1e-3)
- check_numeric_gradient(dot_sym_xT_yT(data_type), [m1_npy.T, m2_npy.T], numeric_eps=1e-1, rtol=2e-2, atol=1e-3)
+ # Test normal dot.
+ for data_type in dtypes:
+ for m in range(1, 5):
+ for k in range(1, 5):
+ for n in range(1, 5):
+ a_npy = np.random.normal(0, 1, (m, k))
+ a_npy = a_npy.astype(data_type)
+ b_npy = np.random.normal(0, 1, (k, n))
+ b_npy = b_npy.astype(data_type)
+ c_npy = np.empty((m, n), dtype=data_type)
+ ograd_npy = np.random.normal(0, 1, (m, n))
+ ograd_npy = ograd_npy.astype(data_type)
+ agrad_npy = np.empty((m, k), dtype=data_type)
+ bgrad_npy = np.empty((k, n), dtype=data_type)
+ c_npy[:, :] = np.dot(a_npy[:, :], b_npy[:, :])
+ bgrad_npy[:, :] = np.dot(a_npy[:, :].T, ograd_npy[:, :])
+ agrad_npy[:, :] = np.dot(ograd_npy[:, :], b_npy[:, :].T)
+ a = mx.sym.Variable('a', dtype=data_type)
+ b = mx.sym.Variable('b', dtype=data_type)
+ c = mx.sym.dot(a, b)
+ exe = c.simple_bind(ctx=ctx, a=a_npy.shape, b=b_npy.shape)
+ outputs = exe.forward(is_train=True, a=a_npy, b=b_npy)
+ assert_almost_equal(outputs[0].asnumpy(), c_npy, rtol=1e-3)
+ exe.backward(out_grads=[mx.nd.array(ograd_npy, mx.cpu())])
+ assert_almost_equal(exe.grad_dict['a'].asnumpy(), agrad_npy, rtol=1e-3)
+ assert_almost_equal(exe.grad_dict['b'].asnumpy(), bgrad_npy, rtol=1e-3)
+
+ # Test dot with transpose flag using gradient checker.
+ def dot_sym(data_type):
+ x = mx.sym.Variable('x', dtype=data_type)
+ y = mx.sym.Variable('y', dtype=data_type)
+ return mx.sym.dot(x, y)
+
+ def dot_sym_xT(data_type):
+ x = mx.sym.Variable('x', dtype=data_type)
+ y = mx.sym.Variable('y', dtype=data_type)
+ return mx.sym.dot(x, y, transpose_a=True)
+
+ def dot_sym_yT(data_type):
+ x = mx.sym.Variable('x', dtype=data_type)
+ y = mx.sym.Variable('y', dtype=data_type)
+ return mx.sym.dot(x, y, transpose_b=True)
+
+ def dot_sym_xT_yT(data_type):
+ x = mx.sym.Variable('x', dtype=data_type)
+ y = mx.sym.Variable('y', dtype=data_type)
+ return mx.sym.dot(x, y, transpose_a=True, transpose_b=True)
+
+ for data_type in dtypes:
+ for ashape, bshape in [((3, 4), (4, 5)), ((2, 3, 4), (4, 5, 6))]:
+ m1_npy = np.random.uniform(-1, 1, ashape)
+ m1_npy = m1_npy.astype(data_type)
+ m2_npy = np.random.uniform(-1, 1, bshape)
+ m2_npy = m2_npy.astype(data_type)
+ check_numeric_gradient(dot_sym(data_type), [m1_npy, m2_npy], numeric_eps=1e-1, rtol=2e-2, atol=1e-3)
+ check_numeric_gradient(dot_sym_xT(data_type), [m1_npy.T, m2_npy], numeric_eps=1e-1, rtol=2e-2, atol=1e-3)
+ check_numeric_gradient(dot_sym_yT(data_type), [m1_npy, m2_npy.T], numeric_eps=1e-1, rtol=2e-2, atol=1e-3)
+ check_numeric_gradient(dot_sym_xT_yT(data_type), [m1_npy.T, m2_npy.T], numeric_eps=1e-1, rtol=2e-2, atol=1e-3)
def test_batch_dot():
@@ -2040,21 +2062,20 @@ def test_support_vector_machine_l2_svm():
def test_roipooling():
- np.random.seed(1234)
+ with np_random_seed(1234):
+ data = mx.symbol.Variable(name='data')
+ rois = mx.symbol.Variable(name='rois')
+ test = mx.symbol.ROIPooling(data=data, rois=rois, pooled_size=(4, 4), spatial_scale=1)
- data = mx.symbol.Variable(name='data')
- rois = mx.symbol.Variable(name='rois')
- test = mx.symbol.ROIPooling(data=data, rois=rois, pooled_size=(4, 4), spatial_scale=1)
+ x1 = np.random.rand(4, 3, 12, 8).astype('float32')
+ x2 = np.array([[0, 1.1, 1.1, 6.2, 6.2], [2, 6.1, 2.1, 8.2, 11.2], [1, 3.1, 1.1, 5.2, 10.2], [0, 3, 3, 3, 3]], dtype='float32')
- x1 = np.random.rand(4, 3, 12, 8).astype('float32')
- x2 = np.array([[0, 1.1, 1.1, 6.2, 6.2], [2, 6.1, 2.1, 8.2, 11.2], [1, 3.1, 1.1, 5.2, 10.2], [0, 3, 3, 3, 3]], dtype='float32')
-
- check_numeric_gradient(sym=test, location=[x1, x2],
- grad_nodes={'data':'write', 'rois':'null'},
- numeric_eps=1e-4, rtol=1e-1, atol=1e-4)
- check_numeric_gradient(sym=test, location=[x1, x2],
- grad_nodes={'data':'add', 'rois':'null'},
- numeric_eps=1e-4, rtol=1e-1, atol=1E-4)
+ check_numeric_gradient(sym=test, location=[x1, x2],
+ grad_nodes={'data':'write', 'rois':'null'},
+ numeric_eps=1e-4, rtol=1e-1, atol=1e-4)
+ check_numeric_gradient(sym=test, location=[x1, x2],
+ grad_nodes={'data':'add', 'rois':'null'},
+ numeric_eps=1e-4, rtol=1e-1, atol=1E-4)
def check_pad_with_shape(shape, xpu, pad_width, mode):
@@ -2138,34 +2159,34 @@ def check_l2_normalization(in_shape, mode, ctx=default_context(), norm_eps=1e-10
data = mx.symbol.Variable('data')
out = mx.symbol.L2Normalization(data=data, mode=mode, eps=norm_eps)
# TODO(szha): Seeding this masks failures. We need to do a deep dive for failures without this seed.
- np.random.seed(1234)
- in_data = np.random.uniform(-1, 1, in_shape)
- # calculate numpy results
- if mode == 'channel':
- assert in_data.ndim > 2
- np_norm = np.linalg.norm(in_data, axis=1) + norm_eps
- np_norm = np.repeat(1. / np.expand_dims(np_norm, axis=1), in_data.shape[1], axis=1)
- np_out = np.multiply(in_data, np_norm)
- elif mode == 'spatial':
- assert in_data.ndim > 2
- s = in_data.shape
- np_norm = np.linalg.norm(in_data.reshape((s[0], s[1], -1)), axis=2) + norm_eps
- np_norm = np.repeat(1. / np_norm[:, np.newaxis], in_data.size / s[0] / s[1], axis=2)
- np_out = np.multiply(in_data, np_norm.reshape(s))
- elif mode == 'instance':
- assert in_data.ndim > 1
- s = in_data.shape
- np_norm = np.linalg.norm(in_data.reshape((s[0], -1)), axis=1) + norm_eps
- np_norm = np.repeat(1. / np_norm[:, np.newaxis], in_data.size / s[0], axis=1)
- np_out = np.multiply(in_data, np_norm.reshape(s))
- else:
- raise RuntimeError('Unknown l2 normalization mode')
- exe = out.simple_bind(ctx=ctx, data=in_data.shape)
- output = exe.forward(is_train=True, data=in_data)
- # compare numpy + mxnet
- assert_almost_equal(exe.outputs[0].asnumpy(), np_out, rtol=1e-5)
- # check gradient
- check_numeric_gradient(out, [in_data], numeric_eps=1e-3, rtol=1e-2, atol=1e-3)
+ with np_random_seed(1234):
+ in_data = np.random.uniform(-1, 1, in_shape)
+ # calculate numpy results
+ if mode == 'channel':
+ assert in_data.ndim > 2
+ np_norm = np.linalg.norm(in_data, axis=1) + norm_eps
+ np_norm = np.repeat(1. / np.expand_dims(np_norm, axis=1), in_data.shape[1], axis=1)
+ np_out = np.multiply(in_data, np_norm)
+ elif mode == 'spatial':
+ assert in_data.ndim > 2
+ s = in_data.shape
+ np_norm = np.linalg.norm(in_data.reshape((s[0], s[1], -1)), axis=2) + norm_eps
+ np_norm = np.repeat(1. / np_norm[:, np.newaxis], in_data.size / s[0] / s[1], axis=2)
+ np_out = np.multiply(in_data, np_norm.reshape(s))
+ elif mode == 'instance':
+ assert in_data.ndim > 1
+ s = in_data.shape
+ np_norm = np.linalg.norm(in_data.reshape((s[0], -1)), axis=1) + norm_eps
+ np_norm = np.repeat(1. / np_norm[:, np.newaxis], in_data.size / s[0], axis=1)
+ np_out = np.multiply(in_data, np_norm.reshape(s))
+ else:
+ raise RuntimeError('Unknown l2 normalization mode')
+ exe = out.simple_bind(ctx=ctx, data=in_data.shape)
+ output = exe.forward(is_train=True, data=in_data)
+ # compare numpy + mxnet
+ assert_almost_equal(exe.outputs[0].asnumpy(), np_out, rtol=1e-5)
+ # check gradient
+ check_numeric_gradient(out, [in_data], numeric_eps=1e-3, rtol=1e-2, atol=1e-3)
def test_l2_normalization():
@@ -2743,7 +2764,6 @@ def test_grid_generator():
def test_bilinear_sampler():
- np.random.seed(1234)
from math import floor
def between(x, lowerbound, upperbound):
@@ -2860,46 +2880,47 @@ def bilinear_backward_numpy(out_grad, data, grid):
return data_grad, grid_grad
- data = mx.sym.Variable('data')
- grid = mx.sym.Variable('grid')
- net = mx.sym.BilinearSampler(data=data,grid=grid)
-
- test_case = [[(1,3,15,16),(1,2,10,10)],
- [(1,6,7,16),(1,2,10,4)],
- [(1,7,3,16),(1,2,8,11)],
- [(1,9,50,50),(1,2,50,50)]]
-
- for ctx in [default_context()]:
- for item in test_case:
- data_shape, grid_shape = item
- exe = net.simple_bind(data=data_shape,grid=grid_shape,ctx=ctx,grad_req='write')
- # check forward
- exe.arg_dict['data'][:] = np.random.uniform(low=-0.1, high=0.1,size=data_shape).astype(np.float32)
- exe.arg_dict['grid'][:] = np.random.uniform(low=-2, high=2, size=grid_shape).astype(np.float32)
- exe.forward(is_train=True)
- out = bilinear_forward_numpy(exe.arg_dict['data'].asnumpy(), exe.arg_dict['grid'].asnumpy())
- assert_almost_equal(exe.outputs[0].asnumpy(), out, rtol=1e-3,atol=1e-5)
-
- # check backward
- out_grad = np.random.uniform(low=-0.01, high=0.01,size=data_shape[:2] + grid_shape[2:]).astype(np.float32)
- exe.backward(mx.nd.array(out_grad))
- data_grad, grid_grad = bilinear_backward_numpy(out_grad,exe.arg_dict['data'].asnumpy(),
- exe.arg_dict['grid'].asnumpy())
- assert_almost_equal(exe.grad_dict['data'].asnumpy(), data_grad, rtol=1e-3, atol=1e-5)
- assert_almost_equal(exe.grad_dict['grid'].asnumpy(), grid_grad, rtol=1e-3, atol=1e-5)
-
- # check kAddTo
- exe_addto = net.simple_bind(data=data_shape, grid=grid_shape, ctx=ctx, grad_req='add')
- data_initial_grid = np.random.normal(size=exe_addto.grad_dict['data'].shape).astype(np.float32)
- grid_initial_grid = np.random.normal(size=exe_addto.grad_dict['grid'].shape).astype(np.float32)
- exe_addto.arg_dict['data'][:] = exe.arg_dict['data'][:]
- exe_addto.arg_dict['grid'][:] = exe.arg_dict['grid'][:]
- exe_addto.grad_dict['data'][:] = data_initial_grid
- exe_addto.grad_dict['grid'][:] = grid_initial_grid
- exe_addto.forward(is_train=True)
- exe_addto.backward(mx.nd.array(out_grad))
- assert_almost_equal(exe_addto.grad_dict['data'].asnumpy(), data_grad + data_initial_grid, rtol=1e-3,atol=1e-5)
- assert_almost_equal(exe_addto.grad_dict['grid'].asnumpy(), grid_grad + grid_initial_grid, rtol=1e-3,atol=1e-5)
+ with np_random_seed(1234):
+ data = mx.sym.Variable('data')
+ grid = mx.sym.Variable('grid')
+ net = mx.sym.BilinearSampler(data=data,grid=grid)
+
+ test_case = [[(1,3,15,16),(1,2,10,10)],
+ [(1,6,7,16),(1,2,10,4)],
+ [(1,7,3,16),(1,2,8,11)],
+ [(1,9,50,50),(1,2,50,50)]]
+
+ for ctx in [default_context()]:
+ for item in test_case:
+ data_shape, grid_shape = item
+ exe = net.simple_bind(data=data_shape,grid=grid_shape,ctx=ctx,grad_req='write')
+ # check forward
+ exe.arg_dict['data'][:] = np.random.uniform(low=-0.1, high=0.1,size=data_shape).astype(np.float32)
+ exe.arg_dict['grid'][:] = np.random.uniform(low=-2, high=2, size=grid_shape).astype(np.float32)
+ exe.forward(is_train=True)
+ out = bilinear_forward_numpy(exe.arg_dict['data'].asnumpy(), exe.arg_dict['grid'].asnumpy())
+ assert_almost_equal(exe.outputs[0].asnumpy(), out, rtol=1e-3,atol=1e-5)
+
+ # check backward
+ out_grad = np.random.uniform(low=-0.01, high=0.01,size=data_shape[:2] + grid_shape[2:]).astype(np.float32)
+ exe.backward(mx.nd.array(out_grad))
+ data_grad, grid_grad = bilinear_backward_numpy(out_grad,exe.arg_dict['data'].asnumpy(),
+ exe.arg_dict['grid'].asnumpy())
+ assert_almost_equal(exe.grad_dict['data'].asnumpy(), data_grad, rtol=1e-3, atol=1e-5)
+ assert_almost_equal(exe.grad_dict['grid'].asnumpy(), grid_grad, rtol=1e-3, atol=1e-5)
+
+ # check kAddTo
+ exe_addto = net.simple_bind(data=data_shape, grid=grid_shape, ctx=ctx, grad_req='add')
+ data_initial_grid = np.random.normal(size=exe_addto.grad_dict['data'].shape).astype(np.float32)
+ grid_initial_grid = np.random.normal(size=exe_addto.grad_dict['grid'].shape).astype(np.float32)
+ exe_addto.arg_dict['data'][:] = exe.arg_dict['data'][:]
+ exe_addto.arg_dict['grid'][:] = exe.arg_dict['grid'][:]
+ exe_addto.grad_dict['data'][:] = data_initial_grid
+ exe_addto.grad_dict['grid'][:] = grid_initial_grid
+ exe_addto.forward(is_train=True)
+ exe_addto.backward(mx.nd.array(out_grad))
+ assert_almost_equal(exe_addto.grad_dict['data'].asnumpy(), data_grad + data_initial_grid, rtol=1e-3,atol=1e-5)
+ assert_almost_equal(exe_addto.grad_dict['grid'].asnumpy(), grid_grad + grid_initial_grid, rtol=1e-3,atol=1e-5)
def test_index2d():
@@ -3497,31 +3518,39 @@ def test_quantization_op():
def test_reciprocal_op():
+ eps = 2**(-11)
data_tmp = np.random.rand(3, 4) * 10 - 5
- # Avoid possible division by 0 errors
- data_tmp[data_tmp == 0] = 1.0
+ # Avoid possible division by 0 errors and finite difference method inaccuracies.
+ # Factor of 6 below set empirically, depends on eps.
+ data_tmp[abs(data_tmp) < 6*eps] = 1.0
data = mx.symbol.Variable('data')
test = mx.sym.reciprocal(data)
- check_numeric_gradient(test, [data_tmp])
+ check_numeric_gradient(test, [data_tmp], numeric_eps = eps)
check_symbolic_forward(test, [data_tmp], [np.reciprocal(data_tmp)])
def test_cbrt_op():
+ eps = 2**(-11)
data_tmp = np.random.rand(3, 4) * 10 - 5
+ # Avoid finite difference method inaccuracies due to infinite gradient
+ # at the origin. Factor of 4 below set empirically, depends on eps.
+ data_tmp[abs(data_tmp) < 4*eps] = 1.0
data = mx.symbol.Variable('data')
test = mx.sym.cbrt(data)
- check_numeric_gradient(test, [data_tmp])
+ check_numeric_gradient(test, [data_tmp], numeric_eps=eps)
check_symbolic_forward(test, [data_tmp], [np.cbrt(data_tmp)])
def test_rcbrt_op():
+ eps = 2**(-11)
data_tmp = np.random.rand(3, 4) * 10 - 5
- # Avoid possible division by 0 errors
- data_tmp[data_tmp == 0] = 1.0
+ # Avoid possible division by 0 errors and finite difference method inaccuracies.
+ # Factor of 4 below set empirically, depends on eps.
+ data_tmp[abs(data_tmp) < 4*eps] = 1.0
data = mx.symbol.Variable('data')
test = mx.sym.rcbrt(data)
- check_numeric_gradient(test, [data_tmp])
+ check_numeric_gradient(test, [data_tmp], numeric_eps = eps)
check_symbolic_forward(test, [data_tmp], [1/np.cbrt(data_tmp)])
def test_custom_op():
@@ -3656,13 +3685,13 @@ def test_deformable_psroipooling():
pooled_size=num_group, output_dim=num_classes,
trans_std=0.1, no_trans=False, name='test_op')
if grad_nodes[0] == 'offset_data':
- # wider tolerance needed for coordinate differential
- rtol, atol = 1.0, 1e-2
+ # wider tolerance needed for coordinate differential, smaller eps
+ rtol, atol, eps = 1.0, 1e-2, 2**(-14)
else:
- rtol, atol = 1e-2, 1e-3
+ rtol, atol, eps = 1e-2, 1e-3, 1e-3
# By now we only have gpu implementation
if mx.Context.default_ctx.device_type == 'gpu':
- check_numeric_gradient(op, [im_data, rois_data, offset_data], rtol=rtol, atol=atol,
+ check_numeric_gradient(op, [im_data, rois_data, offset_data], numeric_eps=eps, rtol=rtol, atol=atol,
grad_nodes=grad_nodes, ctx=mx.gpu(0))
@@ -3719,225 +3748,225 @@ def test_laop():
shape2 = (3, 2)
shape3 = (3, 3)
shape4 = (2, 2)
- #Ensure that ithis tests don't get changed by other calls to random.
- np.random.seed(42)
- data_in1 = np.random.uniform(1, 10, shape1)
- data_in2 = np.random.uniform(1, 10, shape2)
- data_in3 = np.random.uniform(1, 10, shape3)
- data_in4 = np.random.uniform(1, 10, shape4)
- # Check all transpositions of gemm operator.
- data_in1_t = np.transpose(data_in1)
- data_in2_t = np.transpose(data_in2)
- res_gemm = 4. * np.dot(data_in1, data_in2) + 7. * data_in4
- test_gemm = mx.sym.linalg.gemm(data1, data2, data3, alpha=4., beta=7.)
- check_fw(test_gemm, [data_in1, data_in2, data_in4], [res_gemm])
- if grad_check == 1:
- check_grad(test_gemm, [data_in1, data_in2, data_in4])
- res_gemm = 4. * np.dot(data_in1_t, data_in2_t) + 7. * data_in3
- test_gemm = mx.sym.linalg.gemm(data1, data2, data3, alpha=4., beta=7.,
- transpose_a=True, transpose_b=True)
- check_fw(test_gemm, [data_in1, data_in2, data_in3], [res_gemm])
- if grad_check == 1:
- check_grad(test_gemm, [data_in1, data_in2, data_in3])
- res_gemm = 4. * np.dot(data_in1_t, data_in1) + 7. * data_in3
- test_gemm = mx.sym.linalg.gemm(data1, data2, data3, alpha=4., beta=7.,
- transpose_a=True)
- check_fw(test_gemm, [data_in1, data_in1, data_in3], [res_gemm])
- if grad_check == 1:
- check_grad(test_gemm, [data_in1, data_in1, data_in3])
- res_gemm = 4. * np.dot(data_in1, data_in1_t) + 7. * data_in4
- test_gemm = mx.sym.linalg.gemm(data1, data2, data3, alpha=4., beta=7.,
- transpose_b=True)
- check_fw(test_gemm, [data_in1, data_in1, data_in4], [res_gemm])
- if grad_check == 1:
- check_grad(test_gemm, [data_in1, data_in1, data_in4])
-
- # Check batch of gemm.
- a = rep_3x(data_in1, 2, 3)
- b = rep_3x(data_in2, 3, 2)
- c = rep_3x(data_in4, 2, 2)
- r = 4. * np.dot(data_in1, data_in2) + 7. * data_in4
- r = rep_3x(r, 2, 2)
- test_gemm = mx.sym.linalg.gemm(data1, data2, data3, alpha=4., beta=7.)
- check_fw(test_gemm, [a, b, c], [r])
- if grad_check == 1:
- check_grad(test_gemm, [a, b, c])
-
- # Check gemm2 operator same way as gemm.
- res_gemm = 4. * np.dot(data_in1, data_in2)
- test_gemm = mx.sym.linalg.gemm2(data1, data2, alpha=4.)
- check_fw(test_gemm, [data_in1, data_in2], [res_gemm])
- if grad_check == 1:
- check_grad(test_gemm, [data_in1, data_in2])
- res_gemm = 4. * np.dot(data_in1_t, data_in2_t)
- test_gemm = mx.sym.linalg.gemm2(data1, data2, alpha=4., transpose_a=True,
- transpose_b=True)
- check_fw(test_gemm, [data_in1, data_in2], [res_gemm])
- if grad_check == 1:
- check_grad(test_gemm, [data_in1, data_in2])
- res_gemm = 4. * np.dot(data_in1_t, data_in1)
- test_gemm = mx.sym.linalg.gemm2(data1, data2, alpha=4., transpose_a=True)
- check_fw(test_gemm, [data_in1, data_in1], [res_gemm])
- if grad_check == 1:
- check_grad(test_gemm, [data_in1, data_in1])
- res_gemm = 4. * np.dot(data_in1, data_in1_t)
- test_gemm = mx.sym.linalg.gemm2(data1, data2, alpha=4., transpose_b=True)
- check_fw(test_gemm, [data_in1, data_in1], [res_gemm])
- if grad_check == 1:
- check_grad(test_gemm, [data_in1, data_in1])
-
- # Check batch of gemm2.
- a = rep_3x(data_in1, 2, 3)
- b = rep_3x(data_in2, 3, 2)
- r = rep_3x(4. * np.dot(data_in1, data_in2), 2, 2)
- test_gemm = mx.sym.linalg.gemm2(data1, data2, alpha=4.)
- check_fw(test_gemm, [a, b], [r])
- if grad_check == 1:
- check_grad(test_gemm, [a, b])
-
- # Now test all the other operators.
-
- # Tests with trivial 1x1 matrices.
- shape = (4, 4, 1, 1)
- data_in = np.random.uniform(1, 10, shape)
- # test potrf
- # Note: Have to symmetrize input, for gradient test to work
- res_potrf = np.sqrt(data_in)
- test_potrf = mx.sym.linalg.potrf(data1)
- check_fw(test_potrf, [data_in], [res_potrf])
- if grad_check == 1:
- check_grad(test_potrf, [data_in])
- # test potri
- ones = mx.nd.ones(shape).asnumpy()
- res_potri = np.divide(ones, data_in * data_in)
- test_potri = mx.sym.linalg.potri(data1)
- check_fw(test_potri, [data_in], [res_potri])
- if grad_check == 1:
- check_grad(test_potri, [data_in])
- # test trsm
- trian_in = data_in * 7.
- test_trsm = mx.sym.linalg.trsm(data1, data2, alpha=7.)
- check_fw(test_trsm, [trian_in, data_in], [ones])
- if grad_check == 1:
- check_grad(test_trsm, [trian_in,data_in])
- # test trmm
- trian_in = np.divide(ones, trian_in)
- test_trmm = mx.sym.linalg.trmm(data1, data2, alpha=7., transpose=True,
- rightside=True)
- check_fw(test_trmm, [trian_in, data_in], [ones])
- if grad_check == 1:
- check_grad(test_trmm, [trian_in, data_in])
- # test sumlogdiag
- res_sumlogdiag = np.reshape(np.log(data_in), (4, 4))
- test_sumlogdiag = mx.sym.linalg.sumlogdiag(data1)
- check_fw(test_sumlogdiag, [data_in], [res_sumlogdiag])
- if grad_check == 1:
- check_grad(test_sumlogdiag, [data_in])
-
- # more elaborate example of Cholesky factorization
- matrix = np.array([[9., 3., -6., 12.],
- [3., 26., -7., -11.],
- [-6., -7., 9., 7.],
- [12., -11., 7., 65.]])
- trian = np.array([[3., 0., 0., 0.],
- [1., 5., 0., 0.],
- [-2., -1., 2., 0.],
- [4., -3., 6., 2.]])
- pow = np.array([[2., 1., 1., 1.],
- [1., 4., 1., 1.],
- [1., 1., 8., 1.],
- [1., 1., 1., 16.]])
- inv = np.array([[8.95/3., 0.05/3., 2.65, -2.5/3.],
- [0.05/3., 0.05, 0.05, 0.],
- [2.65, 0.05, 2.5, -0.75],
- [-2.5/3., 0., -0.75, 0.25]])
- ident = np.eye(4)
-
- # test potrf
- test_potrf = mx.sym.linalg.potrf(_make_symm_symbol(data1, ndims=4))
- a = rep_3x(matrix, 4, 4)
- r = rep_3x(trian, 4, 4)
- check_fw(test_potrf, [a], [r])
- if grad_check == 1:
- check_grad(test_potrf, [a])
-
- #test potri
- data1_ltri = _make_lower_triangle_symm(
- data1, ndims=4, m=4, dtype=dtype)
- test_potri = mx.sym.linalg.potri(data1_ltri)
- a = rep_3x(trian, 4, 4)
- r = rep_3x(inv, 4, 4)
- check_fw(test_potri, [a], [r])
- if grad_check == 1:
- check_grad(test_potri, [a])
-
- # test trsm
- test_trsm = mx.sym.linalg.trsm(data1_ltri, data2, alpha=7.)
- a = rep_3x(trian, 4, 4)
- b = rep_3x(matrix, 4, 4)
- r = rep_3x(7. * np.transpose(trian), 4, 4)
- check_fw(test_trsm, [a, b], [r])
- if grad_check == 1:
- check_grad(test_trsm, [a, b])
-
- test_trsm2 = mx.sym.linalg.trsm(
- data1_ltri, data2, alpha=-2., rightside=True, transpose=True)
- r = rep_3x(-2. * trian, 4, 4)
- check_fw(test_trsm2, [a, b], [r])
- if grad_check == 1:
- check_grad(test_trsm2, [a, b])
-
- test_trsm3 = mx.sym.linalg.trsm(
- data1_ltri, data2, alpha=0.5, transpose=True)
- b = rep_3x(np.transpose(trian), 4, 4)
- r = rep_3x(0.5 * ident, 4, 4)
- check_fw(test_trsm3, [a, b], [r])
- if grad_check == 1:
- check_grad(test_trsm3, [a, b])
-
- test_trsm4 = mx.sym.linalg.trsm(
- data1_ltri, data2, alpha=-0.5, rightside=True)
- b = rep_3x(trian, 4, 4)
- r = rep_3x(-0.5 * ident, 4, 4)
- check_fw(test_trsm4, [a, b], [r])
- if grad_check == 1:
- check_grad(test_trsm4, [a, b])
-
- # test trmm
- test_trmm = mx.sym.linalg.trmm(
- data1_ltri, data2, alpha=7., transpose=True, rightside=True)
- a = rep_3x(trian, 4, 4)
- b = rep_3x(matrix, 4, 4)
- r = rep_3x(7. * np.dot(matrix, trian.T), 4, 4)
- check_fw(test_trmm, [a, b], [r])
- if grad_check == 1:
- check_grad(test_trmm, [a, b])
-
- test_trmm2 = mx.sym.linalg.trmm(data1_ltri, data2, alpha=-2.)
- r = rep_3x(-2. * np.dot(trian, matrix), 4, 4)
- check_fw(test_trmm2, [a, b], [r])
- if grad_check == 1:
- check_grad(test_trmm2, [a, b])
-
- test_trmm3 = mx.sym.linalg.trmm(data1_ltri, data2, rightside=True)
- r = rep_3x(np.dot(matrix, trian), 4, 4)
- check_fw(test_trmm3, [a, b], [r])
- if grad_check == 1:
- check_grad(test_trmm3, [a, b])
-
- test_trmm4 = mx.sym.linalg.trmm(
- data1_ltri, data2, alpha=1.2, transpose=True)
- r = rep_3x(1.2 * np.dot(trian.T, matrix), 4, 4)
- check_fw(test_trmm4, [a, b], [r])
- if grad_check == 1:
- check_grad(test_trmm4, [a, b])
-
- # test sumlogdiag
- a = rep_3x(pow, 4, 4)
- r = np.reshape(np.tile(10. * np.log(np.array([2.])), 3), (3,))
- check_fw(test_sumlogdiag, [a], [r])
- if grad_check == 1:
- check_grad(test_sumlogdiag, [a])
+ # Ensure this test doesn't get changed by other calls to random.
+ with np_random_seed(42):
+ data_in1 = np.random.uniform(1, 10, shape1)
+ data_in2 = np.random.uniform(1, 10, shape2)
+ data_in3 = np.random.uniform(1, 10, shape3)
+ data_in4 = np.random.uniform(1, 10, shape4)
+ # Check all transpositions of gemm operator.
+ data_in1_t = np.transpose(data_in1)
+ data_in2_t = np.transpose(data_in2)
+ res_gemm = 4. * np.dot(data_in1, data_in2) + 7. * data_in4
+ test_gemm = mx.sym.linalg.gemm(data1, data2, data3, alpha=4., beta=7.)
+ check_fw(test_gemm, [data_in1, data_in2, data_in4], [res_gemm])
+ if grad_check == 1:
+ check_grad(test_gemm, [data_in1, data_in2, data_in4])
+ res_gemm = 4. * np.dot(data_in1_t, data_in2_t) + 7. * data_in3
+ test_gemm = mx.sym.linalg.gemm(data1, data2, data3, alpha=4., beta=7.,
+ transpose_a=True, transpose_b=True)
+ check_fw(test_gemm, [data_in1, data_in2, data_in3], [res_gemm])
+ if grad_check == 1:
+ check_grad(test_gemm, [data_in1, data_in2, data_in3])
+ res_gemm = 4. * np.dot(data_in1_t, data_in1) + 7. * data_in3
+ test_gemm = mx.sym.linalg.gemm(data1, data2, data3, alpha=4., beta=7.,
+ transpose_a=True)
+ check_fw(test_gemm, [data_in1, data_in1, data_in3], [res_gemm])
+ if grad_check == 1:
+ check_grad(test_gemm, [data_in1, data_in1, data_in3])
+ res_gemm = 4. * np.dot(data_in1, data_in1_t) + 7. * data_in4
+ test_gemm = mx.sym.linalg.gemm(data1, data2, data3, alpha=4., beta=7.,
+ transpose_b=True)
+ check_fw(test_gemm, [data_in1, data_in1, data_in4], [res_gemm])
+ if grad_check == 1:
+ check_grad(test_gemm, [data_in1, data_in1, data_in4])
+
+ # Check batch of gemm.
+ a = rep_3x(data_in1, 2, 3)
+ b = rep_3x(data_in2, 3, 2)
+ c = rep_3x(data_in4, 2, 2)
+ r = 4. * np.dot(data_in1, data_in2) + 7. * data_in4
+ r = rep_3x(r, 2, 2)
+ test_gemm = mx.sym.linalg.gemm(data1, data2, data3, alpha=4., beta=7.)
+ check_fw(test_gemm, [a, b, c], [r])
+ if grad_check == 1:
+ check_grad(test_gemm, [a, b, c])
+
+ # Check gemm2 operator same way as gemm.
+ res_gemm = 4. * np.dot(data_in1, data_in2)
+ test_gemm = mx.sym.linalg.gemm2(data1, data2, alpha=4.)
+ check_fw(test_gemm, [data_in1, data_in2], [res_gemm])
+ if grad_check == 1:
+ check_grad(test_gemm, [data_in1, data_in2])
+ res_gemm = 4. * np.dot(data_in1_t, data_in2_t)
+ test_gemm = mx.sym.linalg.gemm2(data1, data2, alpha=4., transpose_a=True,
+ transpose_b=True)
+ check_fw(test_gemm, [data_in1, data_in2], [res_gemm])
+ if grad_check == 1:
+ check_grad(test_gemm, [data_in1, data_in2])
+ res_gemm = 4. * np.dot(data_in1_t, data_in1)
+ test_gemm = mx.sym.linalg.gemm2(data1, data2, alpha=4., transpose_a=True)
+ check_fw(test_gemm, [data_in1, data_in1], [res_gemm])
+ if grad_check == 1:
+ check_grad(test_gemm, [data_in1, data_in1])
+ res_gemm = 4. * np.dot(data_in1, data_in1_t)
+ test_gemm = mx.sym.linalg.gemm2(data1, data2, alpha=4., transpose_b=True)
+ check_fw(test_gemm, [data_in1, data_in1], [res_gemm])
+ if grad_check == 1:
+ check_grad(test_gemm, [data_in1, data_in1])
+
+ # Check batch of gemm2.
+ a = rep_3x(data_in1, 2, 3)
+ b = rep_3x(data_in2, 3, 2)
+ r = rep_3x(4. * np.dot(data_in1, data_in2), 2, 2)
+ test_gemm = mx.sym.linalg.gemm2(data1, data2, alpha=4.)
+ check_fw(test_gemm, [a, b], [r])
+ if grad_check == 1:
+ check_grad(test_gemm, [a, b])
+
+ # Now test all the other operators.
+
+ # Tests with trivial 1x1 matrices.
+ shape = (4, 4, 1, 1)
+ data_in = np.random.uniform(1, 10, shape)
+ # test potrf
+ # Note: Have to symmetrize input, for gradient test to work
+ res_potrf = np.sqrt(data_in)
+ test_potrf = mx.sym.linalg.potrf(data1)
+ check_fw(test_potrf, [data_in], [res_potrf])
+ if grad_check == 1:
+ check_grad(test_potrf, [data_in])
+ # test potri
+ ones = mx.nd.ones(shape).asnumpy()
+ res_potri = np.divide(ones, data_in * data_in)
+ test_potri = mx.sym.linalg.potri(data1)
+ check_fw(test_potri, [data_in], [res_potri])
+ if grad_check == 1:
+ check_grad(test_potri, [data_in])
+ # test trsm
+ trian_in = data_in * 7.
+ test_trsm = mx.sym.linalg.trsm(data1, data2, alpha=7.)
+ check_fw(test_trsm, [trian_in, data_in], [ones])
+ if grad_check == 1:
+ check_grad(test_trsm, [trian_in,data_in])
+ # test trmm
+ trian_in = np.divide(ones, trian_in)
+ test_trmm = mx.sym.linalg.trmm(data1, data2, alpha=7., transpose=True,
+ rightside=True)
+ check_fw(test_trmm, [trian_in, data_in], [ones])
+ if grad_check == 1:
+ check_grad(test_trmm, [trian_in, data_in])
+ # test sumlogdiag
+ res_sumlogdiag = np.reshape(np.log(data_in), (4, 4))
+ test_sumlogdiag = mx.sym.linalg.sumlogdiag(data1)
+ check_fw(test_sumlogdiag, [data_in], [res_sumlogdiag])
+ if grad_check == 1:
+ check_grad(test_sumlogdiag, [data_in])
+
+ # more elaborate example of Cholesky factorization
+ matrix = np.array([[9., 3., -6., 12.],
+ [3., 26., -7., -11.],
+ [-6., -7., 9., 7.],
+ [12., -11., 7., 65.]])
+ trian = np.array([[3., 0., 0., 0.],
+ [1., 5., 0., 0.],
+ [-2., -1., 2., 0.],
+ [4., -3., 6., 2.]])
+ pow = np.array([[2., 1., 1., 1.],
+ [1., 4., 1., 1.],
+ [1., 1., 8., 1.],
+ [1., 1., 1., 16.]])
+ inv = np.array([[8.95/3., 0.05/3., 2.65, -2.5/3.],
+ [0.05/3., 0.05, 0.05, 0.],
+ [2.65, 0.05, 2.5, -0.75],
+ [-2.5/3., 0., -0.75, 0.25]])
+ ident = np.eye(4)
+
+ # test potrf
+ test_potrf = mx.sym.linalg.potrf(_make_symm_symbol(data1, ndims=4))
+ a = rep_3x(matrix, 4, 4)
+ r = rep_3x(trian, 4, 4)
+ check_fw(test_potrf, [a], [r])
+ if grad_check == 1:
+ check_grad(test_potrf, [a])
+
+ #test potri
+ data1_ltri = _make_lower_triangle_symm(
+ data1, ndims=4, m=4, dtype=dtype)
+ test_potri = mx.sym.linalg.potri(data1_ltri)
+ a = rep_3x(trian, 4, 4)
+ r = rep_3x(inv, 4, 4)
+ check_fw(test_potri, [a], [r])
+ if grad_check == 1:
+ check_grad(test_potri, [a])
+
+ # test trsm
+ test_trsm = mx.sym.linalg.trsm(data1_ltri, data2, alpha=7.)
+ a = rep_3x(trian, 4, 4)
+ b = rep_3x(matrix, 4, 4)
+ r = rep_3x(7. * np.transpose(trian), 4, 4)
+ check_fw(test_trsm, [a, b], [r])
+ if grad_check == 1:
+ check_grad(test_trsm, [a, b])
+
+ test_trsm2 = mx.sym.linalg.trsm(
+ data1_ltri, data2, alpha=-2., rightside=True, transpose=True)
+ r = rep_3x(-2. * trian, 4, 4)
+ check_fw(test_trsm2, [a, b], [r])
+ if grad_check == 1:
+ check_grad(test_trsm2, [a, b])
+
+ test_trsm3 = mx.sym.linalg.trsm(
+ data1_ltri, data2, alpha=0.5, transpose=True)
+ b = rep_3x(np.transpose(trian), 4, 4)
+ r = rep_3x(0.5 * ident, 4, 4)
+ check_fw(test_trsm3, [a, b], [r])
+ if grad_check == 1:
+ check_grad(test_trsm3, [a, b])
+
+ test_trsm4 = mx.sym.linalg.trsm(
+ data1_ltri, data2, alpha=-0.5, rightside=True)
+ b = rep_3x(trian, 4, 4)
+ r = rep_3x(-0.5 * ident, 4, 4)
+ check_fw(test_trsm4, [a, b], [r])
+ if grad_check == 1:
+ check_grad(test_trsm4, [a, b])
+
+ # test trmm
+ test_trmm = mx.sym.linalg.trmm(
+ data1_ltri, data2, alpha=7., transpose=True, rightside=True)
+ a = rep_3x(trian, 4, 4)
+ b = rep_3x(matrix, 4, 4)
+ r = rep_3x(7. * np.dot(matrix, trian.T), 4, 4)
+ check_fw(test_trmm, [a, b], [r])
+ if grad_check == 1:
+ check_grad(test_trmm, [a, b])
+
+ test_trmm2 = mx.sym.linalg.trmm(data1_ltri, data2, alpha=-2.)
+ r = rep_3x(-2. * np.dot(trian, matrix), 4, 4)
+ check_fw(test_trmm2, [a, b], [r])
+ if grad_check == 1:
+ check_grad(test_trmm2, [a, b])
+
+ test_trmm3 = mx.sym.linalg.trmm(data1_ltri, data2, rightside=True)
+ r = rep_3x(np.dot(matrix, trian), 4, 4)
+ check_fw(test_trmm3, [a, b], [r])
+ if grad_check == 1:
+ check_grad(test_trmm3, [a, b])
+
+ test_trmm4 = mx.sym.linalg.trmm(
+ data1_ltri, data2, alpha=1.2, transpose=True)
+ r = rep_3x(1.2 * np.dot(trian.T, matrix), 4, 4)
+ check_fw(test_trmm4, [a, b], [r])
+ if grad_check == 1:
+ check_grad(test_trmm4, [a, b])
+
+ # test sumlogdiag
+ a = rep_3x(pow, 4, 4)
+ r = np.reshape(np.tile(10. * np.log(np.array([2.])), 3), (3,))
+ check_fw(test_sumlogdiag, [a], [r])
+ if grad_check == 1:
+ check_grad(test_sumlogdiag, [a])
# Tests for operators linalg.syrk, linalg.gelqf
@@ -3970,84 +3999,84 @@ def _syevd_combined_symbol(a):
return mx.sym.Group([u_ut, ut_lam_u])
def test_laop_2():
- np.random.seed(1896893923)
- dtype = np.float64
- rtol_fw = 1e-7
- atol_fw = 1e-9
- num_eps = 1e-6
- rtol_bw = 1e-5
- atol_bw = 1e-6
- # enable numerical checking of gradients
- grad_check = 1
-
- data1 = mx.symbol.Variable('data1')
-
- check_fw = lambda sym, location, expected :\
- check_symbolic_forward(sym, location, expected, rtol=rtol_fw,
- atol=atol_fw, dtype=dtype)
- check_grad = lambda sym, location:\
- check_numeric_gradient(sym, location, numeric_eps=num_eps, rtol=rtol_bw,
- atol=atol_bw, dtype=dtype)
- rep_3x = lambda a, m, n :\
- np.reshape(np.tile(np.array(a).flatten(), 3), (3, 1, m, n))
-
- # Tests for linalg.syrk
- mnalpha_lst = [(2, 3, 1.), (5, 3, -2.), (1, 6, 5.), (3, 3, 0.5), (4, 1, 10.), (1, 1, 1.)]
- for m, n, alpha in mnalpha_lst:
- #print('syrk: m={}, n={}, alpha={}'.format(m, n, alpha))
- data_in1 = np.random.uniform(1, 10, (m, n))
- res_syrk1 = alpha * np.dot(data_in1, data_in1.T)
- test_syrk1 = mx.sym.linalg.syrk(data1, transpose=False, alpha=alpha)
- check_fw(test_syrk1, [data_in1], [res_syrk1])
- if grad_check == 1:
- check_grad(test_syrk1, [data_in1])
- res_syrk2 = alpha * np.dot(data_in1.T, data_in1)
- test_syrk2 = mx.sym.linalg.syrk(data1, transpose=True, alpha=alpha)
- check_fw(test_syrk2, [data_in1], [res_syrk2])
- if grad_check == 1:
- check_grad(test_syrk2, [data_in1])
- # Batch mode (3x the same thing)
- a_batch = rep_3x(data_in1, m, n)
- r1_batch = rep_3x(res_syrk1, m, m)
- check_fw(test_syrk1, [a_batch], [r1_batch])
- if grad_check == 1:
- check_grad(test_syrk1, [a_batch])
- r2_batch = rep_3x(res_syrk2, n, n)
- check_fw(test_syrk2, [a_batch], [r2_batch])
- if grad_check == 1:
- check_grad(test_syrk2, [a_batch])
-
- # Tests for linalg.gelqf
- # Currently disabled on GPU as they need cuda8
- # and MxNet builds use cuda 7.5
- if not (default_context() == mx.cpu()):
- return
-
- test_gelqf2 = _gelqf_combined_symbol(data1) # Outputs (dot(Q, Q.T), dot(L, Q))
- test_gelqf_q = _gelqf_first_output(data1) # Output Q (L is not dangling)
- test_gelqf_l = _gelqf_second_output(data1) # Output L (Q is not dangling)
- mn_lst = [(4, 4), (1, 1), (5, 20), (1, 10), (15, 50)]
- for m, n in mn_lst:
- #print('gelqf: m={}, n={}'.format(m, n))
- data_in1 = np.random.normal(0., 10., (m, n))
- res_eye = np.eye(m)
- res_a = data_in1
- check_fw(test_gelqf2, [data_in1], [res_eye, res_a])
- if grad_check == 1:
- # A => Q
- check_grad(test_gelqf_q, [data_in1])
- # A => L
- check_grad(test_gelqf_l, [data_in1])
- # Batch mode (3x the same thing)
- a_batch = rep_3x(data_in1, m, n)
- reye_batch = rep_3x(res_eye, m, m)
- ra_batch = a_batch
- check_fw(test_gelqf2, [a_batch], [reye_batch, ra_batch])
- if grad_check == 1:
- # A => Q
- check_grad(test_gelqf_q, [a_batch])
- # A => L
- check_grad(test_gelqf_l, [a_batch])
+ with np_random_seed(1896893923):
+ dtype = np.float64
+ rtol_fw = 1e-7
+ atol_fw = 1e-9
+ num_eps = 1e-6
+ rtol_bw = 1e-5
+ atol_bw = 1e-6
+ # enable numerical checking of gradients
+ grad_check = 1
+
+ data1 = mx.symbol.Variable('data1')
+
+ check_fw = lambda sym, location, expected :\
+ check_symbolic_forward(sym, location, expected, rtol=rtol_fw,
+ atol=atol_fw, dtype=dtype)
+ check_grad = lambda sym, location:\
+ check_numeric_gradient(sym, location, numeric_eps=num_eps, rtol=rtol_bw,
+ atol=atol_bw, dtype=dtype)
+ rep_3x = lambda a, m, n :\
+ np.reshape(np.tile(np.array(a).flatten(), 3), (3, 1, m, n))
+
+ # Tests for linalg.syrk
+ mnalpha_lst = [(2, 3, 1.), (5, 3, -2.), (1, 6, 5.), (3, 3, 0.5), (4, 1, 10.), (1, 1, 1.)]
+ for m, n, alpha in mnalpha_lst:
+ #print('syrk: m={}, n={}, alpha={}'.format(m, n, alpha))
+ data_in1 = np.random.uniform(1, 10, (m, n))
+ res_syrk1 = alpha * np.dot(data_in1, data_in1.T)
+ test_syrk1 = mx.sym.linalg.syrk(data1, transpose=False, alpha=alpha)
+ check_fw(test_syrk1, [data_in1], [res_syrk1])
+ if grad_check == 1:
+ check_grad(test_syrk1, [data_in1])
+ res_syrk2 = alpha * np.dot(data_in1.T, data_in1)
+ test_syrk2 = mx.sym.linalg.syrk(data1, transpose=True, alpha=alpha)
+ check_fw(test_syrk2, [data_in1], [res_syrk2])
+ if grad_check == 1:
+ check_grad(test_syrk2, [data_in1])
+ # Batch mode (3x the same thing)
+ a_batch = rep_3x(data_in1, m, n)
+ r1_batch = rep_3x(res_syrk1, m, m)
+ check_fw(test_syrk1, [a_batch], [r1_batch])
+ if grad_check == 1:
+ check_grad(test_syrk1, [a_batch])
+ r2_batch = rep_3x(res_syrk2, n, n)
+ check_fw(test_syrk2, [a_batch], [r2_batch])
+ if grad_check == 1:
+ check_grad(test_syrk2, [a_batch])
+
+ # Tests for linalg.gelqf
+ # Currently disabled on GPU as they need cuda8
+ # and MxNet builds use cuda 7.5
+ if not (default_context() == mx.cpu()):
+ return
+
+ test_gelqf2 = _gelqf_combined_symbol(data1) # Outputs (dot(Q, Q.T), dot(L, Q))
+ test_gelqf_q = _gelqf_first_output(data1) # Output Q (L is not dangling)
+ test_gelqf_l = _gelqf_second_output(data1) # Output L (Q is not dangling)
+ mn_lst = [(4, 4), (1, 1), (5, 20), (1, 10), (15, 50)]
+ for m, n in mn_lst:
+ #print('gelqf: m={}, n={}'.format(m, n))
+ data_in1 = np.random.normal(0., 10., (m, n))
+ res_eye = np.eye(m)
+ res_a = data_in1
+ check_fw(test_gelqf2, [data_in1], [res_eye, res_a])
+ if grad_check == 1:
+ # A => Q
+ check_grad(test_gelqf_q, [data_in1])
+ # A => L
+ check_grad(test_gelqf_l, [data_in1])
+ # Batch mode (3x the same thing)
+ a_batch = rep_3x(data_in1, m, n)
+ reye_batch = rep_3x(res_eye, m, m)
+ ra_batch = a_batch
+ check_fw(test_gelqf2, [a_batch], [reye_batch, ra_batch])
+ if grad_check == 1:
+ # A => Q
+ check_grad(test_gelqf_q, [a_batch])
+ # A => L
+ check_grad(test_gelqf_l, [a_batch])
# Tests for operator linalg.syevd
@@ -4096,67 +4125,67 @@ def test_laop_3():
# Operators implemented for CPU only currently
if not (default_context() == mx.cpu()):
return
- np.random.seed(1896893923)
- dtype = np.float64
- rtol_fw = 1e-6
- atol_fw = 1e-6
- num_eps = 1e-4
- rtol_bw = 1e-2
- atol_bw = 1e-2
- # enable numerical checking of gradients
- grad_check = 1
-
- data1 = mx.symbol.Variable('data1')
-
- check_fw = lambda sym, location, expected :\
- check_symbolic_forward(sym, location, expected, rtol=rtol_fw,
- atol=atol_fw, dtype=dtype)
- check_grad = lambda sym, location:\
- check_numeric_gradient(sym, location, numeric_eps=num_eps, rtol=rtol_bw,
- atol=atol_bw, dtype=dtype)
- rep_3x = lambda a, m, n :\
- np.reshape(np.tile(np.array(a).flatten(), 3), (3, 1, m, n))
- check_bw = lambda sym, location, out_grads, expected :\
- check_symbolic_backward(sym, location, out_grads, expected,
- rtol=rtol_fw, atol=atol_fw, dtype=dtype)
-
- # Tests for linalg.syevd
- test_syevd2 = _syevd_combined_symbol(data1) # Outputs (U U^T, U^T (diag L) U)
- data1_s2 = _make_symm_symbol(data1, ndims=2)
- test_syevd_u_2 = _syevd_first_output(data1_s2)
- test_syevd_l_2 = _syevd_second_output(data1_s2)
- data1_s4 = _make_symm_symbol(data1, ndims=4)
- test_syevd_u_4 = _syevd_first_output(data1_s4)
- test_syevd_l_4 = _syevd_second_output(data1_s4)
- n_lst = [4, 1, 2, 10, 14]
- for n in n_lst:
- #print('\n** syevd: n={}'.format(n))
- data_in1 = np.random.normal(0., 10., (n, n))
- data_in1 = 0.5 * (data_in1 + data_in1.T)
- res_eye = np.eye(n)
- res_a = data_in1
- check_fw(test_syevd2, [data_in1], [res_eye, res_a])
- # Check backward
- grad_u = np.random.normal(0., 2., (n, n))
- grad_l = np.random.normal(0., 2., (n,))
- bw_u, bw_l = _syevd_forward(data_in1)
- grad_a = _syevd_backward(grad_u, grad_l, bw_u, bw_l)
- check_bw(mx.sym.linalg.syevd(data1), [data_in1], [grad_u, grad_l], [grad_a])
- if grad_check == 1:
- # A => U
- check_grad(test_syevd_u_2, [data_in1])
- # A => L
- check_grad(test_syevd_l_2, [data_in1])
- # Batch mode (3x the same thing)
- a_batch = rep_3x(data_in1, n, n)
- reye_batch = rep_3x(res_eye, n, n)
- ra_batch = a_batch
- check_fw(test_syevd2, [a_batch], [reye_batch, ra_batch])
- if grad_check == 1:
- # A => U
- check_grad(test_syevd_u_4, [a_batch])
- # A => L
- check_grad(test_syevd_l_4, [a_batch])
+ with np_random_seed(1896893923):
+ dtype = np.float64
+ rtol_fw = 1e-6
+ atol_fw = 1e-6
+ num_eps = 1e-4
+ rtol_bw = 1e-2
+ atol_bw = 1e-2
+ # enable numerical checking of gradients
+ grad_check = 1
+
+ data1 = mx.symbol.Variable('data1')
+
+ check_fw = lambda sym, location, expected :\
+ check_symbolic_forward(sym, location, expected, rtol=rtol_fw,
+ atol=atol_fw, dtype=dtype)
+ check_grad = lambda sym, location:\
+ check_numeric_gradient(sym, location, numeric_eps=num_eps, rtol=rtol_bw,
+ atol=atol_bw, dtype=dtype)
+ rep_3x = lambda a, m, n :\
+ np.reshape(np.tile(np.array(a).flatten(), 3), (3, 1, m, n))
+ check_bw = lambda sym, location, out_grads, expected :\
+ check_symbolic_backward(sym, location, out_grads, expected,
+ rtol=rtol_fw, atol=atol_fw, dtype=dtype)
+
+ # Tests for linalg.syevd
+ test_syevd2 = _syevd_combined_symbol(data1) # Outputs (U U^T, U^T (diag L) U)
+ data1_s2 = _make_symm_symbol(data1, ndims=2)
+ test_syevd_u_2 = _syevd_first_output(data1_s2)
+ test_syevd_l_2 = _syevd_second_output(data1_s2)
+ data1_s4 = _make_symm_symbol(data1, ndims=4)
+ test_syevd_u_4 = _syevd_first_output(data1_s4)
+ test_syevd_l_4 = _syevd_second_output(data1_s4)
+ n_lst = [4, 1, 2, 10, 14]
+ for n in n_lst:
+ #print('\n** syevd: n={}'.format(n))
+ data_in1 = np.random.normal(0., 10., (n, n))
+ data_in1 = 0.5 * (data_in1 + data_in1.T)
+ res_eye = np.eye(n)
+ res_a = data_in1
+ check_fw(test_syevd2, [data_in1], [res_eye, res_a])
+ # Check backward
+ grad_u = np.random.normal(0., 2., (n, n))
+ grad_l = np.random.normal(0., 2., (n,))
+ bw_u, bw_l = _syevd_forward(data_in1)
+ grad_a = _syevd_backward(grad_u, grad_l, bw_u, bw_l)
+ check_bw(mx.sym.linalg.syevd(data1), [data_in1], [grad_u, grad_l], [grad_a])
+ if grad_check == 1:
+ # A => U
+ check_grad(test_syevd_u_2, [data_in1])
+ # A => L
+ check_grad(test_syevd_l_2, [data_in1])
+ # Batch mode (3x the same thing)
+ a_batch = rep_3x(data_in1, n, n)
+ reye_batch = rep_3x(res_eye, n, n)
+ ra_batch = a_batch
+ check_fw(test_syevd2, [a_batch], [reye_batch, ra_batch])
+ if grad_check == 1:
+ # A => U
+ check_grad(test_syevd_u_4, [a_batch])
+ # A => L
+ check_grad(test_syevd_l_4, [a_batch])
# Note: Currently, linalg.syevd is activated for float64 only, due to the issues
@@ -4166,26 +4195,26 @@ def test_laop_4():
# Operators implemented for CPU only currently
if not(default_context() == mx.cpu()):
return
- np.random.seed(1896893923)
- rtol_fw = 1e-6
- atol_fw = 1e-6
-
- data1 = mx.symbol.Variable('data1')
-
- check_fw = lambda sym, location, expected, dtype :\
- check_symbolic_forward(sym, location, expected, rtol=rtol_fw,
- atol=atol_fw, dtype=dtype)
-
- a_np = np.array([[1., 2.], [2., 4.]])
- u_np = np.array([[0.89442718, -0.44721359], [0.44721359, 0.89442718]])
- l_np = np.array([0., 5.])
- test_syevd = mx.sym.linalg.syevd(data1)
- # float64
- #print('float64')
- check_fw(test_syevd, [a_np], [u_np, l_np], np.float64)
- # float32
- #print('float32')
- #check_fw(test_syevd, [a_np], [u_np, l_np], np.float32)
+ with np_random_seed(1896893923):
+ rtol_fw = 1e-6
+ atol_fw = 1e-6
+
+ data1 = mx.symbol.Variable('data1')
+
+ check_fw = lambda sym, location, expected, dtype :\
+ check_symbolic_forward(sym, location, expected, rtol=rtol_fw,
+ atol=atol_fw, dtype=dtype)
+
+ a_np = np.array([[1., 2.], [2., 4.]])
+ u_np = np.array([[0.89442718, -0.44721359], [0.44721359, 0.89442718]])
+ l_np = np.array([0., 5.])
+ test_syevd = mx.sym.linalg.syevd(data1)
+ # float64
+ #print('float64')
+ check_fw(test_syevd, [a_np], [u_np, l_np], np.float64)
+ # float32
+ #print('float32')
+ #check_fw(test_syevd, [a_np], [u_np, l_np], np.float32)
def test_stack():
diff --git a/tests/python/unittest/test_sparse_ndarray.py b/tests/python/unittest/test_sparse_ndarray.py
index cb3f97921c..032c5a7c13 100644
--- a/tests/python/unittest/test_sparse_ndarray.py
+++ b/tests/python/unittest/test_sparse_ndarray.py
@@ -24,6 +24,7 @@
from mxnet.ndarray.sparse import RowSparseNDArray, CSRNDArray
+set_np_random_seed()
def sparse_nd_ones(shape, stype):
return mx.nd.ones(shape).tostype(stype)
@@ -378,11 +379,11 @@ def test_sparse_nd_random():
rsp_out = mx.nd.zeros(shape=shape, stype='row_sparse')
dns_out = mx.nd.zeros(shape=shape, stype='default')
mx.random.seed(0)
- np.random.seed(0)
- fn(shape=shape, out=dns_out)
+ with np_random_seed(0):
+ fn(shape=shape, out=dns_out)
mx.random.seed(0)
- np.random.seed(0)
- fn(shape=shape, out=rsp_out)
+ with np_random_seed(0):
+ fn(shape=shape, out=rsp_out)
assert_almost_equal(dns_out.asnumpy(), rsp_out.asnumpy())
@@ -395,56 +396,56 @@ def test_sparse_nd_astype():
def test_sparse_nd_pickle():
- np.random.seed(0)
- repeat = 1
- dim0 = 40
- dim1 = 40
- stypes = ['row_sparse', 'csr']
- densities = [0, 0.5]
- stype_dict = {'row_sparse': RowSparseNDArray, 'csr': CSRNDArray}
- for _ in range(repeat):
- shape = rand_shape_2d(dim0, dim1)
- for stype in stypes:
- for density in densities:
- a, _ = rand_sparse_ndarray(shape, stype, density)
- assert isinstance(a, stype_dict[stype])
- data = pkl.dumps(a)
- b = pkl.loads(data)
- assert isinstance(b, stype_dict[stype])
- assert same(a.asnumpy(), b.asnumpy())
+ with np_random_seed(0):
+ repeat = 1
+ dim0 = 40
+ dim1 = 40
+ stypes = ['row_sparse', 'csr']
+ densities = [0, 0.5]
+ stype_dict = {'row_sparse': RowSparseNDArray, 'csr': CSRNDArray}
+ for _ in range(repeat):
+ shape = rand_shape_2d(dim0, dim1)
+ for stype in stypes:
+ for density in densities:
+ a, _ = rand_sparse_ndarray(shape, stype, density)
+ assert isinstance(a, stype_dict[stype])
+ data = pkl.dumps(a)
+ b = pkl.loads(data)
+ assert isinstance(b, stype_dict[stype])
+ assert same(a.asnumpy(), b.asnumpy())
def test_sparse_nd_save_load():
- np.random.seed(0)
- repeat = 1
- stypes = ['default', 'row_sparse', 'csr']
- stype_dict = {'default': NDArray, 'row_sparse': RowSparseNDArray, 'csr': CSRNDArray}
- num_data = 20
- densities = [0, 0.5]
- fname = 'tmp_list.bin'
- for _ in range(repeat):
- data_list1 = []
- for i in range(num_data):
- stype = stypes[np.random.randint(0, len(stypes))]
- shape = rand_shape_2d(dim0=40, dim1=40)
- density = densities[np.random.randint(0, len(densities))]
- data_list1.append(rand_ndarray(shape, stype, density))
- assert isinstance(data_list1[-1], stype_dict[stype])
- mx.nd.save(fname, data_list1)
-
- data_list2 = mx.nd.load(fname)
- assert len(data_list1) == len(data_list2)
- for x, y in zip(data_list1, data_list2):
- assert same(x.asnumpy(), y.asnumpy())
-
- data_map1 = {'ndarray xx %s' % i: x for i, x in enumerate(data_list1)}
- mx.nd.save(fname, data_map1)
- data_map2 = mx.nd.load(fname)
- assert len(data_map1) == len(data_map2)
- for k, x in data_map1.items():
- y = data_map2[k]
- assert same(x.asnumpy(), y.asnumpy())
- os.remove(fname)
+ with np_random_seed(0):
+ repeat = 1
+ stypes = ['default', 'row_sparse', 'csr']
+ stype_dict = {'default': NDArray, 'row_sparse': RowSparseNDArray, 'csr': CSRNDArray}
+ num_data = 20
+ densities = [0, 0.5]
+ fname = 'tmp_list.bin'
+ for _ in range(repeat):
+ data_list1 = []
+ for i in range(num_data):
+ stype = stypes[np.random.randint(0, len(stypes))]
+ shape = rand_shape_2d(dim0=40, dim1=40)
+ density = densities[np.random.randint(0, len(densities))]
+ data_list1.append(rand_ndarray(shape, stype, density))
+ assert isinstance(data_list1[-1], stype_dict[stype])
+ mx.nd.save(fname, data_list1)
+
+ data_list2 = mx.nd.load(fname)
+ assert len(data_list1) == len(data_list2)
+ for x, y in zip(data_list1, data_list2):
+ assert same(x.asnumpy(), y.asnumpy())
+
+ data_map1 = {'ndarray xx %s' % i: x for i, x in enumerate(data_list1)}
+ mx.nd.save(fname, data_map1)
+ data_map2 = mx.nd.load(fname)
+ assert len(data_map1) == len(data_map2)
+ for k, x in data_map1.items():
+ y = data_map2[k]
+ assert same(x.asnumpy(), y.asnumpy())
+ os.remove(fname)
def test_sparse_nd_unsupported():
nd = mx.nd.zeros((2,2), stype='row_sparse')
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services