You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2020/08/08 17:10:20 UTC
[GitHub] [incubator-mxnet] eric-haibin-lin opened a new issue #18881: test_npx_batch_dot flaky test
eric-haibin-lin opened a new issue #18881:
URL: https://github.com/apache/incubator-mxnet/issues/18881
https://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/mxnet-validation%2Funix-gpu/detail/PR-18880/2/pipeline
```
[2020-08-08T15:33:27.895Z] =================================== FAILURES ===================================
[2020-08-08T15:33:27.895Z] ______________________________ test_npx_batch_dot ______________________________
[2020-08-08T15:33:27.895Z] [gw3] linux -- Python 3.6.9 /usr/bin/python3
[2020-08-08T15:33:27.895Z]
[2020-08-08T15:33:27.895Z] @with_seed()
[2020-08-08T15:33:27.895Z] @use_np
[2020-08-08T15:33:27.895Z] def test_npx_batch_dot():
[2020-08-08T15:33:27.895Z] ctx = mx.context.current_context()
[2020-08-08T15:33:27.895Z] dtypes = ['float32', 'float64']
[2020-08-08T15:33:27.895Z] if ctx.device_type == 'gpu':
[2020-08-08T15:33:27.895Z] dtypes += ['float16']
[2020-08-08T15:33:27.895Z] eps_dict = {'float32': 1E-4, 'float64': 1E-4, 'float16': 1E-3}
[2020-08-08T15:33:27.895Z] class TestBatchDot(HybridBlock):
[2020-08-08T15:33:27.895Z] def __init__(self, transpose_a, transpose_b):
[2020-08-08T15:33:27.895Z] super(TestBatchDot, self).__init__()
[2020-08-08T15:33:27.895Z] self._transpose_a = transpose_a
[2020-08-08T15:33:27.895Z] self._transpose_b = transpose_b
[2020-08-08T15:33:27.895Z]
[2020-08-08T15:33:27.895Z] def hybrid_forward(self, F, lhs, rhs):
[2020-08-08T15:33:27.895Z] return F.npx.batch_dot(lhs, rhs,
[2020-08-08T15:33:27.895Z] transpose_a=self._transpose_a,
[2020-08-08T15:33:27.895Z] transpose_b=self._transpose_b)
[2020-08-08T15:33:27.895Z]
[2020-08-08T15:33:27.895Z] def batch_dot_numpy(lhs, rhs, transpose_a, transpose_b):
[2020-08-08T15:33:27.895Z] assert lhs.ndim == rhs.ndim >= 3
[2020-08-08T15:33:27.895Z] if transpose_a:
[2020-08-08T15:33:27.895Z] lhs = lhs.swapaxes(-1, -2)
[2020-08-08T15:33:27.895Z] if transpose_b:
[2020-08-08T15:33:27.895Z] rhs = rhs.swapaxes(-1, -2)
[2020-08-08T15:33:27.895Z] return _np.matmul(lhs, rhs)
[2020-08-08T15:33:27.895Z]
[2020-08-08T15:33:27.895Z] def gt_grad_batch_dot_numpy(lhs, rhs, ograd, transpose_a, transpose_b, lhs_req, rhs_req,
[2020-08-08T15:33:27.895Z] init_lhs_grad, init_rhs_grad):
[2020-08-08T15:33:27.895Z]
[2020-08-08T15:33:27.895Z] if transpose_a and transpose_b:
[2020-08-08T15:33:27.895Z] # Gradient of z = dot(x.T, y.T)
[2020-08-08T15:33:27.895Z] # dx = dot(dz, y).T = dot(y.T, dz.T)
[2020-08-08T15:33:27.895Z] # dy = dot(x, dz).T = dot(dz.T, x.T)
[2020-08-08T15:33:27.895Z] lhs_grad = batch_dot_numpy(rhs, ograd, transpose_a=True, transpose_b=True)
[2020-08-08T15:33:27.895Z] rhs_grad = batch_dot_numpy(ograd, lhs, transpose_a=True, transpose_b=True)
[2020-08-08T15:33:27.895Z] elif not transpose_a and transpose_b:
[2020-08-08T15:33:27.895Z] # Gradient of z = dot(x, y.T)
[2020-08-08T15:33:27.896Z] # dx = dot(dz, y)
[2020-08-08T15:33:27.896Z] # dy = dot(x.T, dz).T = dot(dz.T, x)
[2020-08-08T15:33:27.896Z] lhs_grad = batch_dot_numpy(ograd, rhs, transpose_a=False, transpose_b=False)
[2020-08-08T15:33:27.896Z] rhs_grad = batch_dot_numpy(ograd, lhs, transpose_a=True, transpose_b=False)
[2020-08-08T15:33:27.896Z] elif transpose_a and not transpose_b:
[2020-08-08T15:33:27.896Z] # Gradient of z = dot(x.T, y)
[2020-08-08T15:33:27.896Z] # dx = dot(dz, y.T).T = dot(y, dz.T)
[2020-08-08T15:33:27.896Z] # dy = dot(x, dz)
[2020-08-08T15:33:27.896Z] lhs_grad = batch_dot_numpy(rhs, ograd, transpose_a=False, transpose_b=True)
[2020-08-08T15:33:27.896Z] rhs_grad = batch_dot_numpy(lhs, ograd, transpose_a=False, transpose_b=False)
[2020-08-08T15:33:27.896Z] else:
[2020-08-08T15:33:27.896Z] # Gradient of z = dot(x, y)
[2020-08-08T15:33:27.896Z] # dx = dot(dz, y.T)
[2020-08-08T15:33:27.896Z] # dy = dot(x.T, dz)
[2020-08-08T15:33:27.896Z] lhs_grad = batch_dot_numpy(ograd, rhs, transpose_a=False, transpose_b=True)
[2020-08-08T15:33:27.896Z] rhs_grad = batch_dot_numpy(lhs, ograd, transpose_a=True, transpose_b=False)
[2020-08-08T15:33:27.896Z] if lhs_req == 'add':
[2020-08-08T15:33:27.896Z] lhs_grad += init_lhs_grad
[2020-08-08T15:33:27.896Z] if rhs_req == 'add':
[2020-08-08T15:33:27.896Z] rhs_grad += init_rhs_grad
[2020-08-08T15:33:27.896Z] return lhs_grad, rhs_grad
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] configs = [
[2020-08-08T15:33:27.896Z] ((2, 3, 0), (2, 4, 0), False, True),
[2020-08-08T15:33:27.896Z] ((2, 4, 3), (2, 4, 3), True, False),
[2020-08-08T15:33:27.896Z] ((0, 3, 0), (0, 0, 2), False, False),
[2020-08-08T15:33:27.896Z] ((3, 2, 3, 2), (3, 2, 2, 3), True, True),
[2020-08-08T15:33:27.896Z] ((3, 1, 5, 2), (3, 1, 2, 1), False, False)
[2020-08-08T15:33:27.896Z] ]
[2020-08-08T15:33:27.896Z] bad_configs = [
[2020-08-08T15:33:27.896Z] ((5, 3, 2), (5, 1, 3), False, False),
[2020-08-08T15:33:27.896Z] ((2, 5, 3, 1), (2, 4, 3, 1), True, False)
[2020-08-08T15:33:27.896Z] ]
[2020-08-08T15:33:27.896Z] for hybridize in [True, False]:
[2020-08-08T15:33:27.896Z] for lhs_shape, rhs_shape, transpose_a, transpose_b in configs:
[2020-08-08T15:33:27.896Z] for dtype in dtypes:
[2020-08-08T15:33:27.896Z] eps = eps_dict[dtype]
[2020-08-08T15:33:27.896Z] for lhs_grad_req in ['write', 'add']:
[2020-08-08T15:33:27.896Z] for rhs_grad_req in ['write', 'add']:
[2020-08-08T15:33:27.896Z] f_batch_dot = TestBatchDot(transpose_a=transpose_a,
[2020-08-08T15:33:27.896Z] transpose_b=transpose_b)
[2020-08-08T15:33:27.896Z] if hybridize:
[2020-08-08T15:33:27.896Z] f_batch_dot.hybridize()
[2020-08-08T15:33:27.896Z] lhs_val = mx.np.array(_np.random.uniform(-1.0, 1.0, lhs_shape), dtype=dtype)
[2020-08-08T15:33:27.896Z] rhs_val = mx.np.array(_np.random.uniform(-1.0, 1.0, rhs_shape), dtype=dtype)
[2020-08-08T15:33:27.896Z] lhs_val.attach_grad(grad_req=lhs_grad_req)
[2020-08-08T15:33:27.896Z] rhs_val.attach_grad(grad_req=rhs_grad_req)
[2020-08-08T15:33:27.896Z] gt_out = batch_dot_numpy(lhs_val.asnumpy(), rhs_val.asnumpy(),
[2020-08-08T15:33:27.896Z] transpose_a, transpose_b)
[2020-08-08T15:33:27.896Z] init_lhs_grad = mx.np.random.uniform(-1.0, 1.0, lhs_shape, dtype=dtype)
[2020-08-08T15:33:27.896Z] init_rhs_grad = mx.np.random.uniform(-1.0, 1.0, rhs_shape, dtype=dtype)
[2020-08-08T15:33:27.896Z] o_grad = mx.np.random.uniform(-1.0, 1.0, gt_out.shape, dtype=dtype)
[2020-08-08T15:33:27.896Z] if lhs_grad_req == 'add':
[2020-08-08T15:33:27.896Z] lhs_val.grad[:] = init_lhs_grad
[2020-08-08T15:33:27.896Z] if rhs_grad_req == 'add':
[2020-08-08T15:33:27.896Z] rhs_val.grad[:] = init_rhs_grad
[2020-08-08T15:33:27.896Z] with mx.autograd.record():
[2020-08-08T15:33:27.896Z] out = f_batch_dot(lhs_val, rhs_val)
[2020-08-08T15:33:27.896Z] out.backward(o_grad)
[2020-08-08T15:33:27.896Z] assert_almost_equal(out.asnumpy(), gt_out, rtol=eps, atol=eps)
[2020-08-08T15:33:27.896Z] gt_lhs_grad, gt_rhs_grad = gt_grad_batch_dot_numpy(lhs_val.asnumpy(),
[2020-08-08T15:33:27.896Z] rhs_val.asnumpy(),
[2020-08-08T15:33:27.896Z] o_grad.asnumpy(),
[2020-08-08T15:33:27.896Z] transpose_a=transpose_a,
[2020-08-08T15:33:27.896Z] transpose_b=transpose_b,
[2020-08-08T15:33:27.896Z] lhs_req=lhs_grad_req,
[2020-08-08T15:33:27.896Z] rhs_req=rhs_grad_req,
[2020-08-08T15:33:27.896Z] init_lhs_grad=init_lhs_grad.asnumpy(),
[2020-08-08T15:33:27.896Z] init_rhs_grad=init_rhs_grad.asnumpy())
[2020-08-08T15:33:27.896Z] assert_almost_equal(lhs_val.grad.asnumpy(), gt_lhs_grad, rtol=eps, atol=eps)
[2020-08-08T15:33:27.896Z] > assert_almost_equal(rhs_val.grad.asnumpy(), gt_rhs_grad, rtol=eps, atol=eps)
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] tests/python/unittest/test_numpy_op.py:1752:
[2020-08-08T15:33:27.896Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] a = array([[[[ 0.3486],
[2020-08-08T15:33:27.896Z] [-0.2471]]],
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] [[[-0.7656],
[2020-08-08T15:33:27.896Z] [-0.8223]]],
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] [[[-0.271 ],
[2020-08-08T15:33:27.896Z] [-1.143 ]]]], dtype=float16)
[2020-08-08T15:33:27.896Z] b = array([[[[ 0.3506],
[2020-08-08T15:33:27.896Z] [-0.2476]]],
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] [[[-0.7646],
[2020-08-08T15:33:27.896Z] [-0.8223]]],
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] [[[-0.271 ],
[2020-08-08T15:33:27.896Z] [-1.143 ]]]], dtype=float16)
[2020-08-08T15:33:27.896Z] rtol = 0.001, atol = 0.001, names = ('a', 'b'), equal_nan = False
[2020-08-08T15:33:27.896Z] use_broadcast = True, mismatches = (10, 10)
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] def assert_almost_equal(a, b, rtol=None, atol=None, names=('a', 'b'), equal_nan=False,
[2020-08-08T15:33:27.896Z] use_broadcast=True, mismatches=(10, 10)):
[2020-08-08T15:33:27.896Z] """Test that two numpy arrays are almost equal. Raise exception message if not.
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] Parameters
[2020-08-08T15:33:27.896Z] ----------
[2020-08-08T15:33:27.896Z] a : np.ndarray or mx.nd.array
[2020-08-08T15:33:27.896Z] b : np.ndarray or mx.nd.array
[2020-08-08T15:33:27.896Z] rtol : None or float or dict of dtype -> float
[2020-08-08T15:33:27.896Z] The relative threshold. Default threshold will be used if set to ``None``.
[2020-08-08T15:33:27.896Z] atol : None or float or dict of dtype -> float
[2020-08-08T15:33:27.896Z] The absolute threshold. Default threshold will be used if set to ``None``.
[2020-08-08T15:33:27.896Z] names : tuple of names, optional
[2020-08-08T15:33:27.896Z] The names used in error message when an exception occurs
[2020-08-08T15:33:27.896Z] equal_nan : boolean, optional
[2020-08-08T15:33:27.896Z] The flag determining how to treat NAN values in comparison
[2020-08-08T15:33:27.896Z] mismatches : tuple of mismatches
[2020-08-08T15:33:27.896Z] Maximum number of mismatches to be printed (mismatches[0]) and determine (mismatches[1])
[2020-08-08T15:33:27.896Z] """
[2020-08-08T15:33:27.896Z] if not use_broadcast:
[2020-08-08T15:33:27.896Z] checkShapes(a, b)
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] rtol, atol = get_tols(a, b, rtol, atol)
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] if isinstance(a, mx.numpy.ndarray):
[2020-08-08T15:33:27.896Z] a = a.asnumpy()
[2020-08-08T15:33:27.896Z] if isinstance(b, mx.numpy.ndarray):
[2020-08-08T15:33:27.896Z] b = b.asnumpy()
[2020-08-08T15:33:27.896Z] use_np_allclose = isinstance(a, np.ndarray) and isinstance(b, np.ndarray)
[2020-08-08T15:33:27.896Z] if not use_np_allclose:
[2020-08-08T15:33:27.896Z] if not (hasattr(a, 'ctx') and hasattr(b, 'ctx') and a.ctx == b.ctx and a.dtype == b.dtype):
[2020-08-08T15:33:27.896Z] use_np_allclose = True
[2020-08-08T15:33:27.896Z] if isinstance(a, mx.nd.NDArray):
[2020-08-08T15:33:27.896Z] a = a.asnumpy()
[2020-08-08T15:33:27.896Z] if isinstance(b, mx.nd.NDArray):
[2020-08-08T15:33:27.896Z] b = b.asnumpy()
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] if use_np_allclose:
[2020-08-08T15:33:27.896Z] if hasattr(a, 'dtype') and a.dtype == np.bool_ and hasattr(b, 'dtype') and b.dtype == np.bool_:
[2020-08-08T15:33:27.896Z] np.testing.assert_equal(a, b)
[2020-08-08T15:33:27.896Z] return
[2020-08-08T15:33:27.896Z] if almost_equal(a, b, rtol, atol, equal_nan=equal_nan):
[2020-08-08T15:33:27.896Z] return
[2020-08-08T15:33:27.896Z] else:
[2020-08-08T15:33:27.896Z] output = mx.nd.contrib.allclose(a, b, rtol, atol, equal_nan)
[2020-08-08T15:33:27.896Z] if output.asnumpy() == 1:
[2020-08-08T15:33:27.896Z] return
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] a = a.asnumpy()
[2020-08-08T15:33:27.896Z] b = b.asnumpy()
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] index, rel = _find_max_violation(a, b, rtol, atol)
[2020-08-08T15:33:27.896Z] if index != ():
[2020-08-08T15:33:27.896Z] # a, b are the numpy arrays
[2020-08-08T15:33:27.896Z] indexErr = index
[2020-08-08T15:33:27.896Z] relErr = rel
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] print('\n*** Maximum errors for vector of size {}: rtol={}, atol={}\n'.format(a.size, rtol, atol))
[2020-08-08T15:33:27.896Z] aTmp = a.copy()
[2020-08-08T15:33:27.896Z] bTmp = b.copy()
[2020-08-08T15:33:27.896Z] i = 1
[2020-08-08T15:33:27.896Z] while i <= a.size:
[2020-08-08T15:33:27.896Z] if i <= mismatches[0]:
[2020-08-08T15:33:27.896Z] print("%3d: Error %f %s" %(i, rel, locationError(a, b, index, names)))
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] aTmp[index] = bTmp[index] = 0
[2020-08-08T15:33:27.896Z] if almost_equal(aTmp, bTmp, rtol, atol, equal_nan=equal_nan):
[2020-08-08T15:33:27.896Z] break
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] i += 1
[2020-08-08T15:33:27.896Z] if i <= mismatches[1] or mismatches[1] <= 0:
[2020-08-08T15:33:27.896Z] index, rel = _find_max_violation(aTmp, bTmp, rtol, atol)
[2020-08-08T15:33:27.896Z] else:
[2020-08-08T15:33:27.896Z] break
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] mismatchDegree = "at least " if mismatches[1] > 0 and i > mismatches[1] else ""
[2020-08-08T15:33:27.896Z] errMsg = "Error %f exceeds tolerance rtol=%e, atol=%e (mismatch %s%f%%).\n%s" % \
[2020-08-08T15:33:27.896Z] (relErr, rtol, atol, mismatchDegree, 100*i/a.size, \
[2020-08-08T15:33:27.896Z] locationError(a, b, indexErr, names, maxError=True))
[2020-08-08T15:33:27.896Z] else:
[2020-08-08T15:33:27.896Z] errMsg = "Error %f exceeds tolerance rtol=%e, atol=%e.\n" % (rel, rtol, atol)
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] np.set_printoptions(threshold=4, suppress=True)
[2020-08-08T15:33:27.896Z] msg = npt.build_err_msg([a, b], err_msg=errMsg)
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] > raise AssertionError(msg)
[2020-08-08T15:33:27.896Z] E AssertionError:
[2020-08-08T15:33:27.896Z] E Items are not equal:
[2020-08-08T15:33:27.896Z] E Error 1.445312 exceeds tolerance rtol=1.000000e-03, atol=1.000000e-03 (mismatch 16.666667%).
[2020-08-08T15:33:27.896Z] E Location of maximum error: (0, 0, 0, 0), a=0.34863281, b=0.35058594
[2020-08-08T15:33:27.896Z] E ACTUAL: array([[[[ 0.3486],
[2020-08-08T15:33:27.896Z] E [-0.2471]]],
[2020-08-08T15:33:27.896Z] E ...
[2020-08-08T15:33:27.896Z] E DESIRED: array([[[[ 0.3506],
[2020-08-08T15:33:27.896Z] E [-0.2476]]],
[2020-08-08T15:33:27.896Z] E ...
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] python/mxnet/test_utils.py:735: AssertionError
[2020-08-08T15:33:27.896Z] ----------------------------- Captured stdout call -----------------------------
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] *** Maximum errors for vector of size 6: rtol=0.001, atol=0.001
[2020-08-08T15:33:27.896Z]
[2020-08-08T15:33:27.896Z] 1: Error 1.445312 Location of error: (0, 0, 0, 0), a=0.34863281, b=0.35058594
[2020-08-08T15:33:27.896Z] ----------------------------- Captured stderr call -----------------------------
[2020-08-08T15:33:27.896Z] [WARNING] Error seen with seeded test, use MXNET_TEST_SEED=2074472578 to reproduce.
[2020-08-08T15:33:27.896Z] WARNING:common:Error seen with seeded test, use MXNET_TEST_SEED=2074472578 to reproduce.
[2020-08-08T15:33:27.896Z] ------------------------------ Captured log call -------------------------------
[2020-08-08T15:33:27.896Z] WARNING common:common.py:230 Error seen with seeded test, use MXNET_TEST_SEED=2074472578 to reproduce.
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org