You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by jx...@apache.org on 2017/10/31 18:11:24 UTC
[incubator-mxnet] branch master updated: Simplified unary/binary
math operators (#8361) (#8361)
This is an automated email from the ASF dual-hosted git repository.
jxie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new c1846ce Simplified unary/binary math operators (#8361) (#8361)
c1846ce is described below
commit c1846ce2ca5003cf613c8bdcf5b0c89d8e0b0d67
Author: mseeger <ms...@gmail.com>
AuthorDate: Tue Oct 31 19:11:21 2017 +0100
Simplified unary/binary math operators (#8361) (#8361)
---
src/operator/math_functions-inl.h | 55 +--
src/operator/mshadow_op.h | 811 +++++++--------------------------
tests/python/unittest/test_operator.py | 20 +-
3 files changed, 221 insertions(+), 665 deletions(-)
diff --git a/src/operator/math_functions-inl.h b/src/operator/math_functions-inl.h
index 2c3aad0..a5b83ea 100644
--- a/src/operator/math_functions-inl.h
+++ b/src/operator/math_functions-inl.h
@@ -35,47 +35,29 @@ namespace op {
namespace math {
// Wrappers for math.h unary and binary functions
-// - For DType != double: math::name(a) does computation in float,
-// casting back to DType afterwards
+// - For DType != double: math::name(a) does computation in float
+// and returns float
// - For DType == double: math::name(a) does computation in double
-// - math::namef(a) does computation in float and returns float value
-// (no casting back). Use this in combined expressions (for example,
-// math::log1p(math::expf(a)) in softrelu (mshadow_op.h))
+// and returns double
#define MXNET_UNARY_MATH_FUNC(name) \
template<typename DType> MSHADOW_XINLINE \
-DType name(DType a) { \
- return DType(::name##f(static_cast<float>(a))); \
-} \
-template<> MSHADOW_XINLINE \
-float name(float a) { \
- return ::name##f(a); \
+float name(DType a) { \
+ return ::name##f(static_cast<float>(a)); \
} \
-template<> MSHADOW_XINLINE \
+MSHADOW_XINLINE \
double name(double a) { \
return ::name(a); \
-} \
-template<typename DType> MSHADOW_XINLINE \
-float name##f(DType a) { \
- return ::name##f(static_cast<float>(a)); \
}
#define MXNET_BINARY_MATH_FUNC(name) \
template<typename DType> MSHADOW_XINLINE \
-DType name(DType a, DType b) { \
- return DType(::name##f(static_cast<float>(a), static_cast<float>(b))); \
+float name(DType a, DType b) { \
+ return ::name##f(static_cast<float>(a), static_cast<float>(b)); \
} \
-template<> MSHADOW_XINLINE \
+MSHADOW_XINLINE \
double name(double a, double b) { \
return ::name(a, b); \
-} \
-template<> MSHADOW_XINLINE \
-float name(float a, float b) { \
- return ::name##f(a, b); \
-} \
-template<typename DType> MSHADOW_XINLINE \
-float name##f(DType a, DType b) { \
- return ::name##f(static_cast<float>(a), static_cast<float>(b)); \
}
MXNET_UNARY_MATH_FUNC(exp)
@@ -136,6 +118,25 @@ MXNET_BINARY_MATH_FUNC(hypot)
MXNET_BINARY_MATH_FUNC(pow)
+template<typename DType> MSHADOW_XINLINE
+float id(DType a) {
+ return static_cast<float>(a);
+}
+MSHADOW_XINLINE
+double id(double a) {
+ return a;
+}
+
+template<typename DType> MSHADOW_XINLINE
+float sqr(DType a) {
+ float af(static_cast<float>(a));
+ return af * af;
+}
+MSHADOW_XINLINE
+double sqr(double a) {
+ return a * a;
+}
+
} // namespace math
} // namespace op
} // namespace mxnet
diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h
index 3acc675..a0d6ee3 100644
--- a/src/operator/mshadow_op.h
+++ b/src/operator/mshadow_op.h
@@ -47,147 +47,77 @@ using std::isnan;
using std::enable_if;
using std::is_unsigned;
-#define MXNET_UNARY_MATH_OP(name) \
+#define MXNET_UNARY_MATH_OP(name, expr) \
struct name { \
template<typename DType> \
MSHADOW_XINLINE static DType Map(DType a) { \
- return math::name(a); \
+ return DType(expr); \
} \
}
-#define MXNET_BINARY_MATH_OP(name) \
+#define MXNET_UNARY_MATH_OP_NC(name, expr) \
+struct name { \
+ template<typename DType> \
+ MSHADOW_XINLINE static DType Map(DType a) { \
+ return (expr); \
+ } \
+}
+
+#define MXNET_BINARY_MATH_OP(name, expr) \
struct name { \
template<typename DType> \
MSHADOW_XINLINE static DType Map(DType a, DType b) { \
- return math::name(a, b); \
+ return DType(expr); \
} \
}
-/*! \brief identity Operation */
-struct identity {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return a;
- }
-};
+#define MXNET_BINARY_MATH_OP_NC(name, expr) \
+struct name { \
+ template<typename DType> \
+ MSHADOW_XINLINE static DType Map(DType a, DType b) { \
+ return (expr); \
+ } \
+}
-struct identity_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(1.0f);
- }
-};
+#define MXNET_SIMPLE_UNARY_MATH_OP(name) MXNET_UNARY_MATH_OP(name, math::name(a))
-struct left {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return a;
- }
-};
+#define MXNET_SIMPLE_BINARY_MATH_OP(name) MXNET_BINARY_MATH_OP(name, math::name(a, b))
-struct right {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return b;
- }
-};
+MXNET_UNARY_MATH_OP_NC(identity, a);
-struct negation {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(-a);
- }
-};
+MXNET_UNARY_MATH_OP(identity_grad, 1);
-struct reciprocal {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(1.0f/a);
- }
-};
+MXNET_BINARY_MATH_OP_NC(left, a);
-struct reciprocal_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(-(DType(1.0f) / (a * a)));
- }
-};
+MXNET_BINARY_MATH_OP_NC(right, b);
-/*! \brief sigmoid unit */
-struct sigmoid {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- float af(static_cast<float>(a));
- return DType(1.0f / (1.0f + math::exp(-af)));
- }
-};
+MXNET_UNARY_MATH_OP(negation, -a);
-template<>
-MSHADOW_XINLINE double sigmoid::Map<double>(double a) {
- return 1.0 / (1.0 + math::exp(-a));
-}
+MXNET_UNARY_MATH_OP(reciprocal, 1.0f / math::id(a));
-struct sigmoid_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(a * (DType(1.0f) - a));
- }
-};
+MXNET_UNARY_MATH_OP(reciprocal_grad, -1.0f / math::sqr(a));
-/*! \brief Rectified Linear Operation */
-struct relu {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return a > DType(0.0f) ? a : DType(0.0f);
- }
-};
+MXNET_UNARY_MATH_OP(sigmoid, 1.0f / (1.0f + math::exp(-a)));
-struct relu_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return a > DType(0.0f) ? DType(1.0f) : DType(0.0f);
- }
-};
+MXNET_UNARY_MATH_OP(sigmoid_grad, math::id(a) * (1.0f - math::id(a)));
-/*! \brief Leaky ReLU Operation */
-struct xelu {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return DType(a > DType(0.0f) ? a : a * b);
- }
-};
+MXNET_UNARY_MATH_OP_NC(relu, a > DType(0) ? a : DType(0));
-struct xelu_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return DType(a > DType(0.0f) ? DType(1.0f) : b);
- }
-};
+MXNET_UNARY_MATH_OP_NC(relu_grad, a > DType(0) ? DType(1) : DType(0));
-/*! \brief Exponential Linear Unit */
-struct elu {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType x, DType a) {
- float af(static_cast<float>(a));
- return x > DType(0.0f) ? x : DType(af * math::expm1f(x));
- }
-};
+MXNET_BINARY_MATH_OP(xelu, a > DType(0) ? math::id(a) :
+ math::id(a) * math::id(b));
-struct elu_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType x, DType a) {
- return x > DType(0.0f) ? DType(1.0f) : DType(a + x);
- }
-};
+MXNET_BINARY_MATH_OP_NC(xelu_grad, a > DType(0) ? DType(1) : b);
-MXNET_UNARY_MATH_OP(tanh);
+MXNET_BINARY_MATH_OP(elu, a > DType(0) ? math::id(a) :
+ math::id(b) * math::expm1(a));
-struct tanh_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(DType(1.0f) - a * a);
- }
-};
+MXNET_BINARY_MATH_OP_NC(elu_grad, a > DType(0) ? DType(1) : DType(b + a));
+
+MXNET_SIMPLE_UNARY_MATH_OP(tanh);
+
+MXNET_UNARY_MATH_OP(tanh_grad, 1.0f - math::sqr(a));
/*! \brief SoftReLU, also known as softplus activation */
struct softrelu {
@@ -195,557 +125,191 @@ struct softrelu {
MSHADOW_XINLINE static DType Map(DType a) {
// Avoid overflow of exp for large inputs.
// Thresholds 20.0 is chosen such that softrelu(a) = a
- // for a > 20 using floating precision.
+ // for a > 20 using floating precision
if (a > DType(20.0f)) {
return a;
} else {
- return DType(math::log1p(math::expf(a)));
+ return DType(math::log1p(math::exp(a)));
}
}
};
-template<>
-MSHADOW_XINLINE double softrelu::Map<double>(double a) {
- if (a > 20.0) {
- return a;
- } else {
- return math::log1p(math::exp(a));
- }
-}
+MXNET_UNARY_MATH_OP(softrelu_grad, -math::expm1(-a));
-struct softrelu_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return -math::expm1(-a);
- }
-};
-
-MXNET_UNARY_MATH_OP(exp);
+MXNET_SIMPLE_UNARY_MATH_OP(exp);
-MXNET_UNARY_MATH_OP(expm1);
+MXNET_SIMPLE_UNARY_MATH_OP(expm1);
-MXNET_UNARY_MATH_OP(log);
+MXNET_SIMPLE_UNARY_MATH_OP(log);
-struct log_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(DType(1.0f) / a);
- }
-};
+MXNET_UNARY_MATH_OP(log_grad, 1.0f / math::id(a));
-MXNET_UNARY_MATH_OP(log10);
+MXNET_SIMPLE_UNARY_MATH_OP(log10);
// Constant is 1 / log(10)
-struct log10_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(0.43429448190325182765f / static_cast<float>(a));
- }
-};
-
-template<>
-MSHADOW_XINLINE double log10_grad::Map<double>(double a) {
- return 0.43429448190325182765 / a;
-}
+MXNET_UNARY_MATH_OP(log10_grad, 0.43429448190325182765 / math::id(a));
-MXNET_UNARY_MATH_OP(log2);
+MXNET_SIMPLE_UNARY_MATH_OP(log2);
// Constant is 1 / log(2)
-struct log2_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(1.44269504088896340737f / static_cast<float>(a));
- }
-};
-
-template<>
-MSHADOW_XINLINE double log2_grad::Map<double>(double a) {
- return 1.44269504088896340737 / a;
-}
+MXNET_UNARY_MATH_OP(log2_grad, 1.44269504088896340737 / math::id(a));
-MXNET_UNARY_MATH_OP(sin);
+MXNET_SIMPLE_UNARY_MATH_OP(sin);
-struct sin_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::cos(a);
- }
-};
+MXNET_UNARY_MATH_OP(sin_grad, math::cos(a));
-MXNET_UNARY_MATH_OP(log1p);
+MXNET_SIMPLE_UNARY_MATH_OP(log1p);
-struct log1p_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(DType(1.0f) / (DType(1.0f) + a));
- }
-};
+MXNET_UNARY_MATH_OP(log1p_grad, 1.0f / (1.0f + math::id(a)));
-MXNET_UNARY_MATH_OP(cos);
+MXNET_SIMPLE_UNARY_MATH_OP(cos);
-struct cos_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return -math::sin(a);
- }
-};
+MXNET_UNARY_MATH_OP(cos_grad, -math::sin(a));
-MXNET_UNARY_MATH_OP(tan);
+MXNET_SIMPLE_UNARY_MATH_OP(tan);
-struct tan_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(a * a + DType(1.0f));
- }
-};
+MXNET_UNARY_MATH_OP(tan_grad, math::sqr(a) + 1.0f);
-struct arcsin {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::asin(a);
- }
-};
+MXNET_UNARY_MATH_OP(arcsin, math::asin(a));
-struct arcsin_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- float af(static_cast<float>(a));
- return DType(1.0f / math::sqrt(1.0f - af * af));
- }
-};
+MXNET_UNARY_MATH_OP(arcsin_grad, 1.0f / math::sqrt(1.0f - math::sqr(a)));
-template<>
-MSHADOW_XINLINE double arcsin_grad::Map<double>(double a) {
- return 1.0 / math::sqrt(1.0 - a * a);
-}
+MXNET_UNARY_MATH_OP(arccos, math::acos(a));
-struct arccos {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::acos(a);
- }
-};
+MXNET_UNARY_MATH_OP(arccos_grad, -1.0f / math::sqrt(1.0f - math::sqr(a)));
-struct arccos_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- float af(static_cast<float>(a));
- return DType(-1.0f / math::sqrt(1.0f - af * af));
- }
-};
+MXNET_UNARY_MATH_OP(arctan, math::atan(a));
-template<>
-MSHADOW_XINLINE double arccos_grad::Map<double>(double a) {
- return -1.0 / math::sqrt(1.0 - a * a);
-}
+MXNET_UNARY_MATH_OP(arctan_grad, 1.0f / (math::sqr(a) + 1.0f));
-struct arctan {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::atan(a);
- }
-};
+MXNET_SIMPLE_BINARY_MATH_OP(hypot);
-struct arctan_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(DType(1.0f) / (a * a + DType(1.0f)));
- }
-};
+MXNET_BINARY_MATH_OP(hypot_grad_left, math::id(a) / math::hypot(a, b));
-MXNET_BINARY_MATH_OP(hypot);
+MXNET_BINARY_MATH_OP(hypot_grad_right, math::id(b) / math::hypot(a, b));
-struct hypot_grad_left {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- float af(static_cast<float>(a));
- return DType(af / math::hypotf(a, b));
- }
-};
+MXNET_UNARY_MATH_OP(degrees, 180.0f / PI * math::id(a));
-template<>
-MSHADOW_XINLINE double hypot_grad_left::Map<double>(double a, double b) {
- return a / math::hypot(a, b);
-}
+MXNET_UNARY_MATH_OP(degrees_grad, 180.0f / PI);
-struct hypot_grad_right {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- float bf(static_cast<float>(b));
- return DType(bf / math::hypotf(a, b));
- }
-};
+MXNET_UNARY_MATH_OP(radians, PI / 180.0f * math::id(a));
-template<>
-MSHADOW_XINLINE double hypot_grad_right::Map<double>(double a, double b) {
- return b / math::hypot(a, b);
-}
+MXNET_UNARY_MATH_OP(radians_grad, PI / 180.0f);
-struct degrees {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(180. / PI * a);
- }
-};
+MXNET_SIMPLE_UNARY_MATH_OP(sinh);
-struct degrees_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(180. / PI);
- }
-};
+MXNET_UNARY_MATH_OP(sinh_grad, math::cosh(a));
-struct radians {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(PI /180. * a);
- }
-};
+MXNET_SIMPLE_UNARY_MATH_OP(cosh);
-struct radians_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(PI / 180.);
- }
-};
+MXNET_UNARY_MATH_OP(cosh_grad, math::sinh(a));
-MXNET_UNARY_MATH_OP(sinh);
+MXNET_UNARY_MATH_OP(arcsinh, math::asinh(a));
-struct sinh_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::cosh(a);
- }
-};
+MXNET_UNARY_MATH_OP(arcsinh_grad, 1.0f / math::hypot(a, DType(1)));
-MXNET_UNARY_MATH_OP(cosh);
+MXNET_UNARY_MATH_OP(arccosh, math::acosh(a));
-struct cosh_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::sinh(a);
- }
-};
+MXNET_UNARY_MATH_OP(arccosh_grad, 1.0f / math::sqrt(math::sqr(a) - 1.0f));
-struct arcsinh {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::asinh(a);
- }
-};
+MXNET_UNARY_MATH_OP(arctanh, math::atanh(a));
-struct arcsinh_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- float af(static_cast<float>(a));
- return DType(1.0f / math::sqrt(1.0f + af * af));
- }
-};
+MXNET_UNARY_MATH_OP(arctanh_grad, 1.0f / (1.0f - math::sqr(a)));
-template<>
-MSHADOW_XINLINE double arcsinh_grad::Map<double>(double a) {
- return 1.0 / math::sqrt(1.0 + a * a);
-}
+MXNET_UNARY_MATH_OP(square, math::sqr(a));
-struct arccosh {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::acosh(a);
- }
-};
-
-struct arccosh_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- float af(static_cast<float>(a));
- return DType(1.0f / math::sqrt(af * af - 1.0f));
- }
-};
-
-template<>
-MSHADOW_XINLINE double arccosh_grad::Map<double>(double a) {
- return 1.0 / math::sqrt(a * a - 1.0);
-}
-
-struct arctanh {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::atanh(a);
- }
-};
-
-struct arctanh_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(DType(1.0f) / (DType(1.0f) - a * a));
- }
-};
-
-struct square {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(a * a);
- }
-};
-
-struct square_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(DType(2.0f) * a);
- }
-};
+MXNET_UNARY_MATH_OP(square_grad, 2.0f * math::id(a));
/*! \brief used for generate Bernoulli mask */
-struct threshold {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return a < b ? DType(1.0f) : DType(0.0f);
- }
-};
+MXNET_BINARY_MATH_OP_NC(threshold, a < b ? DType(1) : DType(0));
/*! \brief used for generate element of abs */
-struct abs {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::fabs(a); // NOLINT(*)
- }
-};
+MXNET_UNARY_MATH_OP(abs, math::fabs(a)); // NOLINT(*)
/*! \brief used for generate element of sign */
struct sign {
template<typename DType>
MSHADOW_XINLINE static typename enable_if<!is_unsigned<DType>::value, DType>::type
Map(DType a) {
- if (a < DType(0.0f)) return DType(-DType(1.0f));
- if (a > DType(0.0f)) return DType(1.0f);
- return DType(0.0f);
+ if (a < DType(0)) return DType(-DType(1));
+ if (a > DType(0)) return DType(1);
+ return DType(0);
}
template<typename DType>
MSHADOW_XINLINE static typename enable_if<is_unsigned<DType>::value, DType>::type
Map(DType a) {
- if (a > DType(0.0f)) return DType(1.0f);
- return DType(0.0f);
+ if (a > DType(0)) return DType(1);
+ return DType(0);
}
};
-struct sign_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(0.0f);
- }
-};
+MXNET_UNARY_MATH_OP_NC(sign_grad, DType(0));
/*! \brief used for generate element of power */
-struct power {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return math::pow(a, b);
- }
-};
+MXNET_BINARY_MATH_OP(power, math::pow(a, b));
-struct power_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- float bf(static_cast<float>(b));
- return DType(math::pow(static_cast<float>(a), bf - 1.0f) * bf);
- }
-};
+MXNET_BINARY_MATH_OP(power_grad, math::pow(a, b - DType(1)) * math::id(b));
-template<>
-MSHADOW_XINLINE double power_grad::Map<double>(double a, double b) {
- return math::pow(a, b - 1.0) * b;
-}
+MXNET_BINARY_MATH_OP(power_rgrad, math::pow(a, b) * math::log(a));
-struct power_rgrad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return DType(math::powf(a, b) * math::logf(a));
- }
-};
-
-template<>
-MSHADOW_XINLINE double power_rgrad::Map<double>(double a, double b) {
- return math::pow(a, b) * math::log(a);
-}
-
-struct rpower {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return math::pow(b, a);
- }
-};
-
-struct rpower_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return DType(static_cast<float>(a) * math::logf(b));
- }
-};
+MXNET_BINARY_MATH_OP(rpower, math::pow(b, a));
-template<>
-MSHADOW_XINLINE double rpower_grad::Map<double>(double a, double b) {
- return a * math::log(b);
-}
+MXNET_BINARY_MATH_OP(rpower_grad, math::id(a) * math::log(b));
/*! \brief used for generate element of maximum */
-struct maximum {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return a > b ? a : b;
- }
-};
+MXNET_BINARY_MATH_OP(maximum, a > b ? a : b);
/*! \brief used for generate element of minimum */
-struct minimum {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return a < b ? a : b;
- }
-};
+MXNET_BINARY_MATH_OP_NC(minimum, a < b ? a : b);
-struct ge {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return a >= b ? DType(1.0f) : DType(0.0f);
- }
-};
+MXNET_BINARY_MATH_OP_NC(ge, a >= b ? DType(1) : DType(0));
-struct gt {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return a > b ? DType(1.0f) : DType(0.0f);
- }
-};
+MXNET_BINARY_MATH_OP_NC(gt, a > b ? DType(1) : DType(0));
-struct lt {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return a < b ? DType(1.0f) : DType(0.0f);
- }
-};
+MXNET_BINARY_MATH_OP_NC(lt, a < b ? DType(1) : DType(0));
-struct le {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return a <= b ? DType(1.0f) : DType(0.0f);
- }
-};
+MXNET_BINARY_MATH_OP_NC(le, a <= b ? DType(1) : DType(0));
-struct eq {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return a == b ? DType(1.0f) : DType(0.0f);
- }
-};
+MXNET_BINARY_MATH_OP_NC(eq, a == b ? DType(1) : DType(0));
-struct ne {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return a != b ? DType(1.0f) : DType(0.0f);
- }
-};
+MXNET_BINARY_MATH_OP_NC(ne, a != b ? DType(1) : DType(0));
-/*!\ \brief used for generate element sqrt */
-struct square_root {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::sqrt(a);
- }
-};
+MXNET_UNARY_MATH_OP(square_root, math::sqrt(a));
-struct square_root_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(DType(0.5f) / a);
- }
-};
+MXNET_UNARY_MATH_OP(square_root_grad, 0.5f / math::id(a));
-/*!\ \brief used for generate element rsqrt */
-struct reciprocal_square_root {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(1.0f / math::sqrtf(a));
- }
-};
+MXNET_UNARY_MATH_OP(reciprocal_square_root, 1.0f / math::sqrt(a));
-template<>
-MSHADOW_XINLINE double reciprocal_square_root::Map<double>(double a) {
- return 1.0 / math::sqrt(a);
-}
+MXNET_UNARY_MATH_OP(reciprocal_square_root_grad, -0.5f / (math::sqrt(a) * math::id(a)));
-struct reciprocal_square_root_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- float af(static_cast<float>(a));
- return DType(-0.5f / (af * math::sqrt(af)));
- }
-};
+MXNET_UNARY_MATH_OP(cube_root, math::cbrt(a));
-template<>
-MSHADOW_XINLINE double reciprocal_square_root_grad::Map<double>(double a) {
- return -0.5 / (a * math::sqrt(a));
-}
+MXNET_UNARY_MATH_OP(cube_root_grad, 1.0f / (3.0f * math::sqr(a)));
-/*!\ \brief used for generate element cbrt */
-struct cube_root {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::cbrt(a);
- }
-};
+MXNET_UNARY_MATH_OP(reciprocal_cube_root, 1.0f / math::cbrt(a));
-struct cube_root_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(DType(1.0f) / (DType(3.0f) * a * a));
- }
-};
-
-/*!\ \brief used for generate element rcbrt */
-struct reciprocal_cube_root {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return DType(1.0f / math::cbrtf(a));
- }
-};
-
-template<>
-MSHADOW_XINLINE double reciprocal_cube_root::Map<double>(double a) {
- return 1.0 / math::cbrt(a);
-}
-
-struct reciprocal_cube_root_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- float af(static_cast<float>(a));
- return DType(-1.0f / (3.0f * af * math::cbrt(af)));
- }
-};
-
-template<>
-MSHADOW_XINLINE double reciprocal_cube_root_grad::Map<double>(double a) {
- return -1.0 / (3.0 * a * math::cbrt(a));
-}
+MXNET_UNARY_MATH_OP(reciprocal_cube_root_grad, -1.0f / (3.0f * math::cbrt(a) * math::id(a)));
/*! \brief used for generate element of round */
-MXNET_UNARY_MATH_OP(round);
+MXNET_SIMPLE_UNARY_MATH_OP(round);
/*! \brief used for generate element of ceil */
-MXNET_UNARY_MATH_OP(ceil);
+MXNET_SIMPLE_UNARY_MATH_OP(ceil);
/*! \brief used for generate element of floor */
-MXNET_UNARY_MATH_OP(floor);
+MXNET_SIMPLE_UNARY_MATH_OP(floor);
/*! \brief used to round towards zero */
-MXNET_UNARY_MATH_OP(trunc);
+MXNET_SIMPLE_UNARY_MATH_OP(trunc);
/*! \brief used to round number to nearest integer */
struct rint {
template<typename DType>
MSHADOW_XINLINE static DType Map(DType a) {
- float floor = math::floorf(a);
- float ceil = math::ceilf(a);
- return DType((a - floor) <= (ceil - a) ? floor : ceil);
+ auto floor = math::floor(a);
+ auto ceil = math::ceil(a);
+ auto af = math::id(a);
+ return DType((af - floor) <= (ceil - af) ? floor : ceil);
}
};
@@ -753,54 +317,38 @@ struct rint {
struct fix {
template<typename DType>
MSHADOW_XINLINE static DType Map(DType a) {
- float floor = math::floorf(a);
- float ceil = math::ceilf(a);
+ auto floor = math::floor(a);
+ auto ceil = math::ceil(a);
return DType((floor > 0 ? floor : -floor) < (ceil > 0 ? ceil : -ceil) ? floor : ceil);
}
};
/*! \brief used for generate gradient of MAE loss*/
-struct minus_sign {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return DType(a-b > DType(0.0f) ? DType(1.0f) : -DType(1.0f));
- }
-};
+MXNET_BINARY_MATH_OP_NC(minus_sign, a - b > DType(0) ? DType(1) : -DType(1));
-struct rminus {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return DType(b - a);
- }
-};
+MXNET_BINARY_MATH_OP(rminus, b - a);
-struct div_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return DType(DType(1.0f) / b);
- }
-};
+MXNET_BINARY_MATH_OP(div_grad, 1.0f / math::id(b));
-struct div_rgrad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return DType(-a / (b * b));
- }
-};
+template<>
+MSHADOW_XINLINE mshadow::half::half2_t div_grad::Map<mshadow::half::half2_t>
+ (mshadow::half::half2_t a,
+ mshadow::half::half2_t b) {
+ return mshadow::half::half2_t(1) / b;
+}
-struct rdiv {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return DType(b / a);
- }
-};
+MXNET_BINARY_MATH_OP(div_rgrad, -math::id(a) / math::sqr(b));
-struct rdiv_grad {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a, DType b) {
- return DType(-b / (a * a));
- }
-};
+template<>
+MSHADOW_XINLINE mshadow::half::half2_t div_rgrad::Map<mshadow::half::half2_t>
+ (mshadow::half::half2_t a,
+ mshadow::half::half2_t b) {
+ return -a / (b * b);
+}
+
+MXNET_BINARY_MATH_OP(rdiv, math::id(b) / math::id(a));
+
+MXNET_BINARY_MATH_OP(rdiv_grad, -math::id(b) / math::sqr(a));
struct mod {
template<typename DType>
@@ -836,14 +384,13 @@ struct mod {
}
}
};
-#ifdef __CUDACC__
+
template<>
MSHADOW_XINLINE mshadow::half::half2_t mod::Map<mshadow::half::half2_t>
(mshadow::half::half2_t a,
mshadow::half::half2_t b) {
return a%b;
}
-#endif
struct mod_grad {
template<typename DType>
@@ -859,7 +406,7 @@ template<>
MSHADOW_XINLINE float mod_grad::Map<float>(float a, float b) {
return 1.0f;
}
-#ifdef __CUDACC__
+
template<>
MSHADOW_XINLINE mshadow::half::half_t mod_grad::Map<mshadow::half::half_t>
(mshadow::half::half_t a,
@@ -871,7 +418,7 @@ MSHADOW_XINLINE mshadow::half::half2_t mod_grad::Map<mshadow::half::half2_t>
(mshadow::half::half2_t a,
mshadow::half::half2_t b) {
mshadow::half::half2_t result = mshadow::half::half2_t();
-#if MSHADOW_CUDA_HALF2
+#if (defined(__CUDACC__) && MSHADOW_CUDA_HALF2)
result.half2_ = ::__float2half2_rn(1.0f);
#else
result.half_t2[0] = mshadow::half::half_t(0.0f);
@@ -879,7 +426,6 @@ MSHADOW_XINLINE mshadow::half::half2_t mod_grad::Map<mshadow::half::half2_t>
#endif
return result;
}
-#endif
struct mod_rgrad {
template<typename DType>
@@ -895,7 +441,7 @@ template<>
MSHADOW_XINLINE float mod_rgrad::Map<float>(float a, float b) {
return -::floorf(a/b);
}
-#ifdef __CUDACC__
+
template<>
MSHADOW_XINLINE mshadow::half::half_t mod_rgrad::Map<mshadow::half::half_t>
(mshadow::half::half_t a,
@@ -906,7 +452,7 @@ template<>
MSHADOW_XINLINE mshadow::half::half2_t mod_rgrad::Map<mshadow::half::half2_t>
(mshadow::half::half2_t a,
mshadow::half::half2_t b) {
-#if MSHADOW_CUDA_HALF2
+#if (defined(__CUDACC__) && MSHADOW_CUDA_HALF2)
return mshadow::half::half2_t(__hneg2(::h2floor((a/b).half2_)));
#else
return mshadow::half::half2_t(mshadow::half::half_t(-::floorf(
@@ -915,7 +461,6 @@ MSHADOW_XINLINE mshadow::half::half2_t mod_rgrad::Map<mshadow::half::half2_t>
static_cast<float>(a.half_t2[1]/b.half_t2[1]))));
#endif
}
-#endif
struct rmod {
template<typename DType>
@@ -951,14 +496,13 @@ struct rmod {
}
}
};
-#ifdef __CUDACC__
+
template<>
MSHADOW_XINLINE mshadow::half::half2_t rmod::Map<mshadow::half::half2_t>
(mshadow::half::half2_t a,
mshadow::half::half2_t b) {
return b%a;
}
-#endif
struct rmod_grad {
template<typename DType>
@@ -974,7 +518,7 @@ template<>
MSHADOW_XINLINE float rmod_grad::Map<float>(float a, float b) {
return -::floorf(b/a);
}
-#ifdef __CUDACC__
+
template<>
MSHADOW_XINLINE mshadow::half::half_t rmod_grad::Map<mshadow::half::half_t>
(mshadow::half::half_t a,
@@ -985,7 +529,7 @@ template<>
MSHADOW_XINLINE mshadow::half::half2_t rmod_grad::Map<mshadow::half::half2_t>
(mshadow::half::half2_t a,
mshadow::half::half2_t b) {
-#if MSHADOW_CUDA_HALF2
+#if (defined(__CUDACC__) && MSHADOW_CUDA_HALF2)
return mshadow::half::half2_t(::__hneg2(::h2floor((b/a).half2_)));
#else
return mshadow::half::half2_t(mshadow::half::half_t(-::floorf(
@@ -994,7 +538,6 @@ MSHADOW_XINLINE mshadow::half::half2_t rmod_grad::Map<mshadow::half::half2_t>
static_cast<float>(b.half_t2[1]/a.half_t2[1]))));
#endif
}
-#endif
struct clip {
template<typename DType>
@@ -1011,12 +554,7 @@ struct clip {
/***** gamma ******/
-struct gamma {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::tgamma(a);
- }
-};
+MXNET_UNARY_MATH_OP(gamma, math::tgamma(a));
struct gamma_grad {
template<typename DType>
@@ -1034,12 +572,7 @@ MSHADOW_XINLINE double gamma_grad::Map<double>(double a) {
/***** gammaln ******/
-struct gammaln {
- template<typename DType>
- MSHADOW_XINLINE static DType Map(DType a) {
- return math::lgamma(a);
- }
-};
+MXNET_UNARY_MATH_OP(gammaln, math::lgamma(a));
struct gammaln_grad {
template<typename DType>
@@ -1055,25 +588,27 @@ MSHADOW_XINLINE double gammaln_grad::Map<double>(double a) {
}
/* Smooth L1 Loss is a loss specific for R-CNN franchise training
- * Smooth L1 Loss function
+ * Smooth L1 Loss function:
* f(x) = 0.5 * (sigma * x) ^ 2, |x| < 1 / sigma^2
* = |x| - 0.5 / sigma / sigma, otherwise
- * When sigma = 1, it is equivalent to Huber Loss evaluated at
+ * When sigma = 1, it is equivalent to the Huber loss, evaluated at
* delta = 1.
* smooth_l1_loss = w_out * f(w_in * x)
* with w_in, w_out provided by input_data.
*/
struct smooth_l1_loss {
- // a is x, b is sigma2
+ // a is x, b is sigma
template<typename DType>
MSHADOW_XINLINE static DType Map(DType a, DType b) {
- b *= b;
- if (a > DType(1.0f) / b) {
- return a - DType(0.5f) / b;
- } else if (a < DType(-1.0f) / b) {
- return -a - DType(0.5f) / b;
+ auto bsq = math::sqr(b);
+ auto ibsq = 1.0f / bsq;
+ auto af = math::id(a);
+ if (af > ibsq) {
+ return DType(af - 0.5f * ibsq);
+ } else if (af < -ibsq) {
+ return DType(-af - 0.5f * ibsq);
} else {
- return DType(0.5f) * a * a * b;
+ return DType(0.5f * af * af * bsq);
}
}
}; // struct smooth_l1_loss
@@ -1086,13 +621,15 @@ struct smooth_l1_gradient {
// a is x, b is sigma2
template<typename DType>
MSHADOW_XINLINE static DType Map(DType a, DType b) {
- b *= b;
- if (a > DType(1.0f) / b) {
- return DType(1.0f);
- } else if (a < DType(-1.0f) / b) {
- return DType(-1.0f);
+ auto bsq = math::sqr(b);
+ auto ibsq = 1.0f / bsq;
+ auto af = math::id(a);
+ if (af > ibsq) {
+ return DType(1);
+ } else if (af < -ibsq) {
+ return DType(-1);
} else {
- return b * a;
+ return DType(bsq * af);
}
}
}; // struct smooth_l1_derivative
diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py
index 1ad10c8..11a3366 100644
--- a/tests/python/unittest/test_operator.py
+++ b/tests/python/unittest/test_operator.py
@@ -4350,6 +4350,16 @@ def finite_diff_unary_op(
name=op_name)
check_grad(op_ex, [data_np])
+def np_smooth_l1(x, sigma):
+ issq = 1. / sigma / sigma
+ absx = np.abs(x)
+ temp = x * sigma
+ return np.where(absx < issq, 0.5 * (temp ** 2), absx - 0.5 * issq)
+
+def np_smooth_l1_grad(x, sigma):
+ ssq = sigma * sigma
+ return np.where(np.abs(x) < 1. / ssq, x * ssq, np.sign(x))
+
# Tests for unary operators (basic mathematical functions):
# - Forward: Comparison to NumPy (several dtype)
# - Backward: Comparison to NumPy (several dtype)
@@ -4471,7 +4481,15 @@ def test_unary_math_operators():
'tanh': [lambda x: mx.sym.tanh(x),
lambda x: np.tanh(x),
lambda x: 1. - np.tanh(x) ** 2,
- -4.0, 4.0]
+ -4.0, 4.0],
+ 'smooth_l1_sig1': [lambda x: mx.sym.smooth_l1(x, scalar=1.),
+ lambda x: np_smooth_l1(x, 1.),
+ lambda x: np_smooth_l1_grad(x, 1.),
+ -2.0, 2.0],
+ 'smooth_l1_sig2': [lambda x: mx.sym.smooth_l1(x, scalar=2.),
+ lambda x: np_smooth_l1(x, 2.),
+ lambda x: np_smooth_l1_grad(x, 2.),
+ -1.0, 1.0]
}
if have_scipy:
unary_ops['gamma'] = [lambda x: mx.sym.gamma(x),
--
To stop receiving notification emails like this one, please contact
['"commits@mxnet.apache.org" <co...@mxnet.apache.org>'].