You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by cj...@apache.org on 2017/10/17 15:51:29 UTC
[incubator-mxnet] 03/04: Enable smoothing in softmax operator
(#8125)
This is an automated email from the ASF dual-hosted git repository.
cjolivier01 pushed a commit to branch v0.12.0
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
commit a631c59cbbd58788907476f8084555bb33d5d1ed
Author: Kellen Sunderland <ke...@gmail.com>
AuthorDate: Tue Oct 17 17:00:52 2017 +0200
Enable smoothing in softmax operator (#8125)
---
src/operator/softmax_output-inl.h | 21 ++++++++++--
tests/python/unittest/test_operator.py | 61 ++++++++++++++++++++++++++++++----
2 files changed, 74 insertions(+), 8 deletions(-)
diff --git a/src/operator/softmax_output-inl.h b/src/operator/softmax_output-inl.h
index 5f8203e..7216c76 100644
--- a/src/operator/softmax_output-inl.h
+++ b/src/operator/softmax_output-inl.h
@@ -53,6 +53,7 @@ struct SoftmaxOutputParam : public dmlc::Parameter<SoftmaxOutputParam> {
bool preserve_shape;
int normalization;
bool out_grad;
+ float smooth_alpha;
DMLC_DECLARE_PARAMETER(SoftmaxOutputParam) {
DMLC_DECLARE_FIELD(grad_scale).set_default(1.0f)
.describe("Scales the gradient by a float factor.");
@@ -78,6 +79,13 @@ struct SoftmaxOutputParam : public dmlc::Parameter<SoftmaxOutputParam> {
DMLC_DECLARE_FIELD(out_grad)
.set_default(false)
.describe("Multiplies gradient with output gradient element-wise.");
+ DMLC_DECLARE_FIELD(smooth_alpha)
+ .set_default(0.0f)
+ .set_range(0.0f, 1.0f)
+ .describe("Constant for computing a label smoothed version of cross-entropy"
+ "for the backwards pass. This constant gets subtracted from the"
+ "one-hot encoding of the gold label and distributed uniformly to"
+ "all other labels.");
};
};
@@ -215,9 +223,18 @@ class SoftmaxOutputOp : public Operator {
in_grad[softmaxout_enum::kData].get_with_shape<xpu, 2, DType>(data_shape, s);
index_t valid_cnt = label.shape_.Size();
if (param_.use_ignore) {
- SoftmaxGrad(grad, out, label, static_cast<DType>(param_.ignore_label));
+ if (param_.smooth_alpha == 0.0f) {
+ SoftmaxGrad(grad, out, label, static_cast<DType>(param_.ignore_label));
+ } else {
+ SmoothSoftmaxGrad(grad, out, label, static_cast<DType>(param_.ignore_label),
+ param_.smooth_alpha);
+ }
} else {
- SoftmaxGrad(grad, out, label);
+ if (param_.smooth_alpha == 0.0f) {
+ SoftmaxGrad(grad, out, label);
+ } else {
+ SmoothSoftmaxGrad(grad, out, label, param_.smooth_alpha);
+ }
}
if (param_.normalization == softmaxout_enum::kBatch) {
valid_cnt = label.size(0);
diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py
index 105d1ce..024e089 100644
--- a/tests/python/unittest/test_operator.py
+++ b/tests/python/unittest/test_operator.py
@@ -236,6 +236,53 @@ def test_regression():
lambda x, y : x - y)
+def check_softmax_grad(xpu):
+ x = mx.sym.Variable('x')
+ label = mx.sym.Variable('label')
+ x_nd = mx.nd.array([[1, 6, 4, 2]], ctx=xpu)
+ grad_x = mx.nd.zeros((1,4), ctx=xpu)
+ label_nd = mx.nd.array([1], ctx=xpu)
+
+ sym = mx.sym.SoftmaxOutput(data=x, label=label, ignore_label=0, use_ignore=False)
+ ex = sym.bind(ctx=xpu, args={'x': x_nd, 'label': label_nd}, args_grad={'x': grad_x})
+
+ ex.forward(is_train=True)
+ softmax_out = ex.outputs[0].asnumpy()
+ expected_softmax_out = [[0.005806628, 0.861780069, 0.116629249, 0.015784052]]
+ assert np.isclose(softmax_out, expected_softmax_out).all()
+
+ ex.backward(is_train=True)
+ grad_out = ex.grad_arrays[0].asnumpy()
+ k = int(label_nd[0].asscalar())
+ expected_grad_out = np.zeros((1,4))
+ expected_grad_out[0, k] = -1
+ assert np.isclose(grad_out - softmax_out, expected_grad_out).all()
+
+
+def check_smoothed_softmax_grad(xpu):
+ alpha = 0.2
+ x = mx.sym.Variable('x')
+ label = mx.sym.Variable('label')
+ x_nd = mx.nd.array([[1, 6, 4, 2]], ctx=xpu)
+ grad_x = mx.nd.zeros((1,4), ctx=xpu)
+ label_nd = mx.nd.array([1], ctx=xpu)
+
+ sym = mx.sym.SoftmaxOutput(data=x, label=label, ignore_label=0, use_ignore=False, smooth_alpha=alpha)
+ ex = sym.bind(ctx=xpu, args={'x': x_nd, 'label': label_nd}, args_grad={'x': grad_x})
+
+ ex.forward(is_train=True)
+ softmax_out = ex.outputs[0].asnumpy()
+ expected_softmax_out = [[0.005806628, 0.861780069, 0.116629249, 0.015784052]]
+ assert np.isclose(softmax_out, expected_softmax_out).all()
+
+ ex.backward(is_train=True)
+ grad_out = ex.grad_arrays[0].asnumpy()
+ k = int(label_nd[0].asscalar())
+ expected_grad_out = np.full((1,4), fill_value=-alpha/float(4-1))
+ expected_grad_out[0, k] = - (1 - alpha)
+ assert np.isclose(grad_out - softmax_out, expected_grad_out).all()
+
+
def check_softmax_with_ignore_label(xpu):
X = mx.symbol.Variable('X')
L = mx.symbol.Variable('L')
@@ -286,12 +333,6 @@ def check_softmax_with_shape(shape, xpu, preserve_shape=False):
assert_almost_equal(grad.asnumpy(), np_softmax(x.asnumpy()) - l.asnumpy(), rtol=1e-4)
-def test_softmax():
- check_softmax_with_shape((3, 4), default_context(), preserve_shape=False)
- check_softmax_with_shape((3, 4), default_context(), preserve_shape=True)
- check_softmax_with_shape((3, 4, 2), default_context(), preserve_shape=True)
-
-
def test_python_op():
X = mx.symbol.Variable('X')
op = mx.operator.NumpyOp()
@@ -4542,6 +4583,14 @@ def test_binary_math_operators():
num_eps)
+def test_softmax():
+ check_softmax_with_shape((3, 4), default_context(), preserve_shape=False)
+ check_softmax_with_shape((3, 4), default_context(), preserve_shape=True)
+ check_softmax_with_shape((3, 4, 2), default_context(), preserve_shape=True)
+ check_softmax_grad(default_context())
+ check_smoothed_softmax_grad(default_context())
+
+
if __name__ == '__main__':
import nose
nose.runmodule()
--
To stop receiving notification emails like this one, please contact
"commits@mxnet.apache.org" <co...@mxnet.apache.org>.