[incubator-mxnet] branch master updated: float32 -> float16 cast consistency across implementations (#13857)

[zh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zhasheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
     new c939c2d  float32 -> float16 cast consistency across implementations (#13857)
c939c2d is described below

commit c939c2d55e9789a367ffb95b5a78ec5aa2fd35c0
Author: Dick Carter <di...@comcast.net>
AuthorDate: Wed Jan 30 13:04:36 2019 -0800

    float32 -> float16 cast consistency across implementations (#13857)
    
    * Added test showing float32->float16 discrepancy when mshadow float2half() is used.
    
    * Temp update mshadow submodule SHA to point to PR368 (b211cb7).
    
    * Temp switch to url = https://github.com/DickJC123/mshadow.git
    
    * Updata mshadow submodule SHA.
    
    * Improve code style per reviewer comments.
    
    * Move back to dmlc/mshadow.git, now with float->half rounding.
    
    * Expand test_operator.py:test_cast_float32_to_float16 to test np.nan.
---
 3rdparty/mshadow                       |  2 +-
 tests/python/unittest/test_operator.py | 42 ++++++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/3rdparty/mshadow b/3rdparty/mshadow
index 6dc04f7..3dc8081 160000
--- a/3rdparty/mshadow
+++ b/3rdparty/mshadow
@@ -1 +1 @@
-Subproject commit 6dc04f7c729cd5c6c6210d5d4d2026a26ce0bfbf
+Subproject commit 3dc80815d965b56b9a975dc27229361955bf66fe
diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py
index ce61beb..146836c 100644
--- a/tests/python/unittest/test_operator.py
+++ b/tests/python/unittest/test_operator.py
@@ -4000,6 +4000,48 @@ def test_cast():
             assert_almost_equal(exe.grad_arrays[0].asnumpy(), X.astype(dsttype).astype(srctype), rtol=1e-3, atol=1e-5)
 
 
+# Test requires all platforms to round float32->float16 with same round-to-nearest-even policy.
+@with_seed()
+def test_cast_float32_to_float16():
+    FP16_FRACTION_BITS = 10
+    FP32_FRACTION_BITS = 23
+    FP32_EXP_MIN = -126
+    FP32_EXP_MAX = 127
+    # generate test cases in the vicinity of representable float16 mantissas
+    # and mid-way between them, but over the full range of float32 exponents.
+    def get_data():
+        for sign_bit in [0, 1]:
+            for exponent in range(FP32_EXP_MIN - FP32_FRACTION_BITS - 1, FP32_EXP_MAX + 2):
+                denominator = 2**(FP16_FRACTION_BITS + 1)
+                for numerator in range(0, denominator):
+                    fraction = numerator / float(denominator)
+                    for y in [-1.0, 0.0, 1.0]:
+                        small_delta = y / 2**FP32_FRACTION_BITS
+                        val = (-1.0)**sign_bit * 2.0**exponent * (1.0 + fraction + small_delta)
+                        yield val
+        # Add np.nan as a final data value to process
+        yield np.nan
+
+    input_np = np.array(list(get_data())).astype(np.float32)
+    # The intermediate cast to np.float64 below gets around a numpy rounding bug that is fixed
+    # as of numpy 1.17 by PR https://github.com/numpy/numpy/pull/12722
+    expected_output = input_np.astype(np.float64).astype(np.float16)
+
+    x = mx.sym.Variable('x', dtype=np.float32)
+    sym = mx.sym.Cast(x, dtype=np.float16)
+    ctx = default_context()
+    exe = sym.bind(ctx, {'x' : mx.nd.array(input_np, dtype=np.float32, ctx=ctx)})
+    assert exe.arg_arrays[0].dtype == np.float32
+    assert exe.outputs[0].dtype == np.float16
+    exe.forward(is_train=False)
+    sym_output = exe.outputs[0].asnumpy()
+    for fp32_val, model_fp16_val, np_fp16_val in zip(input_np, sym_output, expected_output):
+        assert (model_fp16_val == np_fp16_val) or \
+               (np.isnan(model_fp16_val) and np.isnan(np_fp16_val)), \
+            'fp32->fp16 cast mismatch: with fp32 value {}, model_fp16 = {}, numpy_fp16 = {}'.format(
+                fp32_val, model_fp16_val, np_fp16_val)
+
+
 @with_seed()
 def test_repeat():
     def test_repeat_forward():