You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/02/05 19:03:13 UTC
[GitHub] piiswrong closed pull request #9697: fix NAG if multi_precision = true
piiswrong closed pull request #9697: fix NAG if multi_precision = true
URL: https://github.com/apache/incubator-mxnet/pull/9697
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/python/mxnet/optimizer.py b/python/mxnet/optimizer.py
index d3eeba1b50..0856a7f894 100644
--- a/python/mxnet/optimizer.py
+++ b/python/mxnet/optimizer.py
@@ -129,10 +129,11 @@ def register(klass):
assert(isinstance(klass, type))
name = klass.__name__.lower()
if name in Optimizer.opt_registry:
- warnings.warn('WARNING: New optimizer %s.%s is overriding existing '
- 'optimizer %s.%s', klass.__module__, klass.__name__,
- Optimizer.opt_registry[name].__module__,
- Optimizer.opt_registry[name].__name__)
+ warnings.warn('WARNING: New optimizer %s.%s is overriding '
+ 'existing optimizer %s.%s' %
+ (klass.__module__, klass.__name__,
+ Optimizer.opt_registry[name].__module__,
+ Optimizer.opt_registry[name].__name__))
Optimizer.opt_registry[name] = klass
return klass
@@ -892,7 +893,8 @@ def update(self, index, weight, grad, state):
weight[:] += mom
@register
-class NAG(SGD):
+@register
+class NAG(Optimizer):
"""Nesterov accelerated SGD.
This optimizer updates each weight by::
@@ -900,10 +902,26 @@ class NAG(SGD):
state = momentum * state + grad + wd * weight
weight = weight - (lr * (grad + momentum * state))
- This optimizer accepts the same arguments as :class:`.SGD`.
+ Parameters
+ ----------
+ momentum : float, optional
+ The momentum value.
+ multi_precision: bool, optional
+ Flag to control the internal precision of the optimizer.
+ ``False`` results in using the same precision as the weights (default),
+ ``True`` makes internal 32-bit copy of the weights and applies gradients \
+ in 32-bit precision even if actual weights used in the model have lower precision.\
+ Turning this on can improve convergence and accuracy when training with float16.
"""
- def __init__(self, **kwargs):
+ def __init__(self, momentum=0.0, **kwargs):
super(NAG, self).__init__(**kwargs)
+ self.momentum = momentum
+
+ def create_state(self, index, weight):
+ momentum = None
+ if self.momentum != 0.0:
+ momentum = zeros(weight.shape, weight.context, dtype=weight.dtype)
+ return momentum
def update(self, index, weight, grad, state):
assert(isinstance(weight, NDArray))
diff --git a/tests/python/unittest/test_optimizer.py b/tests/python/unittest/test_optimizer.py
index 26ff48babc..f7ee6d0653 100644
--- a/tests/python/unittest/test_optimizer.py
+++ b/tests/python/unittest/test_optimizer.py
@@ -357,6 +357,118 @@ def test_std_sparse_sgd():
w_stype='row_sparse', g_stype='row_sparse')
+class PyNAG(PySGD):
+ def __init__(self, **kwargs):
+ super(PyNAG, self).__init__(**kwargs)
+
+ def create_state(self, index, weight):
+ """Create additional optimizer state: momentum
+
+ Parameters
+ ----------
+ weight : NDArray
+ The weight data
+
+ """
+ momentum = None
+ weight_master_copy = None
+ do_multi_precision = self.multi_precision and weight.dtype == np.float16
+ if do_multi_precision:
+ if self.momentum != 0.0:
+ momentum = mx.nd.zeros(weight.shape, weight.context, dtype=np.float32)
+ weight_master_copy = array(weight, ctx=weight.context, dtype=np.float32)
+ return (weight_master_copy, momentum)
+ else:
+ if self.momentum != 0.0:
+ momentum = mx.nd.zeros(weight.shape, weight.context, dtype=weight.dtype)
+ return momentum
+
+ def create_state_multi_precision(self, index, weight):
+ return self.create_state(index, weight)
+
+ def update(self, index, weight, grad, state):
+ """Update the parameters.
+
+ Parameters
+ ----------
+ index : int
+ An unique integer key used to index the parameters
+
+ weight : NDArray
+ weight ndarray
+
+ grad : NDArray
+ grad ndarray
+
+ state : NDArray or other objects returned by init_state
+ The auxiliary state used in optimization.
+ """
+ lr = self._get_lr(index)
+ wd = self._get_wd(index)
+ self._update_count(index)
+ use_multi_precision = isinstance(state, list) or isinstance(state, tuple)
+ if not use_multi_precision:
+ grad = grad * self.rescale_grad
+ if self.clip_gradient is not None:
+ grad = mx.nd.clip(grad, -self.clip_gradient, self.clip_gradient)
+ if self.momentum == 0.0:
+ weight[:] += -lr * (grad + wd * weight)
+ else:
+ mom = state
+ mom[:] *= self.momentum
+ grad += wd * weight
+ mom[:] += grad
+ grad[:] += self.momentum * mom
+ weight[:] += -lr * grad
+ else:
+ grad32 = array(grad, ctx=grad.context, dtype=np.float32)
+ grad32 = grad32 * self.rescale_grad
+ if self.clip_gradient is not None:
+ grad32 = mx.nd.clip(grad32, -self.clip_gradient, self.clip_gradient)
+ mom = state[1]
+ weight32 = state[0]
+ if self.momentum == 0.0:
+ weight32[:] += -lr * (grad32 + wd * weight32)
+ else:
+ mom[:] *= self.momentum
+ grad32 += wd * weight32
+ mom[:] += grad32
+ grad32[:] += self.momentum * mom
+ weight32[:] += -lr * grad32
+ tmp = weight32.astype(weight.dtype)
+ tmp.copyto(weight)
+
+
+def test_nag():
+ mx.random.seed(0)
+ opt1 = PyNAG
+ opt2 = mx.optimizer.NAG
+ shape = (3, 4, 5)
+ mom_options = [{}, {'momentum': 0.9}]
+ cg_options = [{}, {'clip_gradient': 0.4}, {'clip_gradient': 0.5}]
+ rg_options = [{}, {'rescale_grad': 0.14}, {'rescale_grad': 0.8}]
+ wd_options = [{}, {'wd': 0.03}, {'wd': 0.05}, {'wd': 0.07}]
+ mp_options = [{}, {'multi_precision': False}, {'multi_precision': True}]
+ for dtype in [np.float16, np.float32, np.float64]:
+ for mom_option in mom_options:
+ for cg_option in cg_options:
+ for rg_option in rg_options:
+ for wd_option in wd_options:
+ for mp_option in mp_options:
+ kwarg = {}
+ kwarg.update(mom_option)
+ kwarg.update(cg_option)
+ kwarg.update(rg_option)
+ kwarg.update(wd_option)
+ kwarg.update(mp_option)
+ if (dtype == np.float16 and
+ ('multi_precision' not in kwarg or
+ not kwarg['multi_precision'])):
+ continue
+ compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape, dtype)
+
+
+
# FTML
class PyFTML(mx.optimizer.Optimizer):
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services