You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2017/12/12 22:01:06 UTC
[GitHub] piiswrong closed pull request #6586: Implementation for concat relu operator
piiswrong closed pull request #6586: Implementation for concat relu operator
URL: https://github.com/apache/incubator-mxnet/pull/6586
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/src/operator/contrib/crelu-inl.h b/src/operator/contrib/crelu-inl.h
new file mode 100644
index 0000000000..d764658dd8
--- /dev/null
+++ b/src/operator/contrib/crelu-inl.h
@@ -0,0 +1,185 @@
+
+/*!
+ * Copyright (c) 2017 by Contributors
+ * \file crelu-inl.h
+ * \brief concat relu
+ * \author Yijie Zhuang
+*/
+
+#ifndef MXNET_OPERATOR_CONTRIB_CRELU_INL_H_
+#define MXNET_OPERATOR_CONTRIB_CRELU_INL_H_
+
+#include <dmlc/logging.h>
+#include <dmlc/parameter.h>
+#include <mxnet/operator.h>
+#include <algorithm>
+#include <map>
+#include <vector>
+#include <string>
+#include <utility>
+#include "../operator_common.h"
+#include "../channel_op_common.h"
+#include "../mshadow_op.h"
+
+namespace mxnet {
+namespace op {
+
+namespace concat_relu {
+enum CReluOpInputs {kData};
+enum CReluOpResource {kTempSpace};
+enum CReluOpOutputs {kOut};
+}
+
+struct CReluParam : public dmlc::Parameter<CReluParam> {
+ DMLC_DECLARE_PARAMETER(CReluParam) {}
+};
+
+template<typename xpu, typename DType>
+class CReluOp : public Operator {
+ public:
+ explicit CReluOp(CReluParam p) { this->param_ = p; }
+
+ virtual void Forward(const OpContext &ctx,
+ const std::vector<TBlob> &in_data,
+ const std::vector<OpReqType> &req,
+ const std::vector<TBlob> &out_data,
+ const std::vector<TBlob> &aux_args) {
+ using namespace mshadow;
+ using namespace mshadow::expr;
+ CHECK_EQ(in_data.size(), 1U);
+ CHECK_EQ(out_data.size(), 1U);
+ Stream<xpu> *s = ctx.get_stream<xpu>();
+
+ Tensor<xpu, 2, DType> data = in_data[concat_relu::kData].FlatTo2D<xpu, DType>(s);
+ Tensor<xpu, 2, DType> out = out_data[concat_relu::kOut].FlatTo2D<xpu, DType>(s);
+ std::vector<Tensor<xpu, 2, DType> > concat_data(2);
+
+ concat_data[0] = Tensor<xpu, 2, DType>(out.dptr_, data.shape_, s);
+ concat_data[1] = ctx.requested[concat_relu::kTempSpace]
+ .get_space_typed<xpu, 2, DType>(data.shape_, s);
+
+ concat_data[0] = F<mshadow_op::relu>(data);
+ concat_data[1] = F<mshadow_op::relu>(F<mshadow_op::negation>(data));
+
+ Concatenate(concat_data, &out, 1, req[concat_relu::kOut]);
+ }
+
+ virtual void Backward(const OpContext &ctx,
+ const std::vector<TBlob> &out_grad,
+ const std::vector<TBlob> &in_data,
+ const std::vector<TBlob> &out_data,
+ const std::vector<OpReqType> &req,
+ const std::vector<TBlob> &in_grad,
+ const std::vector<TBlob> &aux_args) {
+ using namespace mshadow;
+ using namespace mshadow::expr;
+ CHECK_EQ(out_grad.size(), 1U);
+ CHECK(in_data.size() == 1 && in_grad.size() == 1);
+ CHECK_EQ(req.size(), 1U);
+ Stream<xpu> *s = ctx.get_stream<xpu>();
+
+ using namespace std;
+
+ Tensor<xpu, 2, DType> m_in_grad = in_grad[concat_relu::kData].FlatTo2D<xpu, DType>(s);
+ Tensor<xpu, 2, DType> m_in_data = in_data[concat_relu::kData].FlatTo2D<xpu, DType>(s);
+ Tensor<xpu, 2, DType> m_out_grad = out_grad[concat_relu::kOut].FlatTo2D<xpu, DType>(s);
+ Tensor<xpu, 2, DType> m_out_data = out_data[concat_relu::kOut].FlatTo2D<xpu, DType>(s);
+
+ m_out_grad = F<mshadow_op::relu_grad>(m_out_data) * m_out_grad;
+
+ Assign(m_in_grad, req[concat_relu::kData], slice<1>(m_out_grad, 0, m_out_grad.size(1)/2)
+ * F<mshadow_op::sign>(m_in_data)
+ + slice<1>(m_out_grad, m_out_grad.size(1)/2, m_out_grad.size(1))
+ * F<mshadow_op::sign>(m_in_data));
+ }
+
+ private:
+ CReluParam param_;
+}; // class crelu
+
+// Decalre Factory function, used for dispatch specialization
+template<typename xpu>
+Operator* CreateOp(CReluParam param, int dtype);
+
+#if DMLC_USE_CXX11
+class CReluProp : public OperatorProperty {
+ public:
+ void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {
+ param_.Init(kwargs);
+ }
+
+ std::map<std::string, std::string> GetParams() const override {
+ return param_.__DICT__();
+ }
+
+ bool InferShape(std::vector<TShape> *in_shape,
+ std::vector<TShape> *out_shape,
+ std::vector<TShape> *aux_shape) const override {
+ using namespace mshadow;
+ CHECK_EQ(in_shape->size(), 1U) << "Input:[data]";
+ TShape dshape = in_shape->at(concat_relu::kData);
+ if (dshape.ndim() == 0) return false;
+ dshape[dshape.ndim()-1] = dshape[dshape.ndim()-1] * 2;
+ out_shape->clear();
+ out_shape->push_back(dshape);
+ return true;
+ }
+
+ bool InferType(std::vector<int> *in_type,
+ std::vector<int> *out_type,
+ std::vector<int> *aux_type) const override {
+ CHECK_GE(in_type->size(), 1U);
+ int dtype = (*in_type)[0];
+ CHECK_NE(dtype, -1) << "First input must have specified type";
+ for (index_t i = 0; i < in_type->size(); ++i) {
+ if ((*in_type)[i] == -1) {
+ (*in_type)[i] = dtype;
+ } else {
+ CHECK_EQ((*in_type)[i], dtype) << "This layer requires uniform type. "
+ << "Expected " << dtype << " v.s. given "
+ << (*in_type)[i] << " at " << ListArguments()[i];
+ }
+ }
+ out_type->clear();
+ out_type->push_back(dtype);
+ return true;
+ }
+
+ OperatorProperty* Copy() const override {
+ auto ptr = new CReluProp();
+ ptr->param_ = param_;
+ return ptr;
+ }
+
+ std::string TypeString() const override {
+ return "_contrib_CRelu";
+ }
+
+ // decalre dependency and inplace optimization options
+ std::vector<int> DeclareBackwardDependency(
+ const std::vector<int> &out_grad,
+ const std::vector<int> &in_data,
+ const std::vector<int> &out_data) const override {
+ return {out_grad[concat_relu::kOut], out_data[concat_relu::kOut], in_data[concat_relu::kData]};
+ }
+
+ std::vector<ResourceRequest> ForwardResource(
+ const std::vector<TShape> &in_shape) const override {
+ return {ResourceRequest::kTempSpace};
+ }
+
+ Operator* CreateOperator(Context ctx) const override {
+ LOG(FATAL) << "Not Implemented.";
+ return NULL;
+ }
+
+ Operator* CreateOperatorEx(Context ctx, std::vector<TShape> *in_shape,
+ std::vector<int> *in_type) const override;
+
+ private:
+ CReluParam param_;
+};
+#endif // DMLC_USE_CXX11
+} // namespace op
+} // namespace mxnet
+#endif // MXNET_OPERATOR_CONTRIB_CRELU_INL_H_
diff --git a/src/operator/contrib/crelu.cc b/src/operator/contrib/crelu.cc
new file mode 100644
index 0000000000..8063aeedac
--- /dev/null
+++ b/src/operator/contrib/crelu.cc
@@ -0,0 +1,46 @@
+/*!
+ * Copyright (c) 2017 by Contributors
+ * \file crelu.cc
+ * \brief crelu op
+ * \author Yijie Zhuang
+*/
+
+#include "./crelu-inl.h"
+
+namespace mxnet {
+namespace op {
+template<>
+Operator *CreateOp<cpu>(CReluParam param, int dtype) {
+ Operator *op = NULL;
+
+ MSHADOW_REAL_TYPE_SWITCH(dtype, DType, {
+ op = new CReluOp<cpu, DType>(param);
+ })
+ return op;
+}
+
+// DO_BIND_DISPATCH comes from operator_common.h
+Operator *CReluProp::CreateOperatorEx(Context ctx, std::vector<TShape> *in_shape,
+ std::vector<int> *in_type) const {
+ std::vector<TShape> out_shape, aux_shape;
+ std::vector<int> out_type, aux_type;
+ CHECK(InferType(in_type, &out_type, &aux_type));
+ CHECK(InferShape(in_shape, &out_shape, &aux_shape));
+ DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]);
+}
+
+DMLC_REGISTER_PARAMETER(CReluParam);
+
+MXNET_REGISTER_OP_PROPERTY(_contrib_CRelu, CReluProp)
+.describe(R"code(Applies concatenate relu function to the input.
+
+- `crelu`: Concatenate Rectified Linear Unit, :math:`y = (max(x, 0), max(-x,0))`
+
+)code" ADD_FILELINE)
+.add_argument("data", "NDArray-or-Symbol", "Input array to activation function.")
+.add_arguments(CReluParam::__FIELDS__());
+
+NNVM_REGISTER_OP(_contrib_CRelu).add_alias("_contrib_crelu");
+
+} // namespace op
+} // namespace mxnet
diff --git a/src/operator/contrib/crelu.cu b/src/operator/contrib/crelu.cu
new file mode 100644
index 0000000000..5eb3b870f7
--- /dev/null
+++ b/src/operator/contrib/crelu.cu
@@ -0,0 +1,25 @@
+
+/*!
+ * Copyright (c) 2017 by Contributors
+ * \file crelu.cu
+ * \brief crelu op
+ * \author Yijie Zhuang
+*/
+
+#include "./crelu-inl.h"
+
+namespace mxnet {
+namespace op {
+template<>
+Operator *CreateOp<gpu>(CReluParam param, int dtype) {
+ Operator *op = NULL;
+
+ MSHADOW_REAL_TYPE_SWITCH(dtype, DType, {
+ op = new CReluOp<gpu, DType>(param);
+ })
+
+ return op;
+}
+
+} // namespace op
+} // namespace mxnet
diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py
index 8129a41ee7..d609dfb8ae 100644
--- a/tests/python/unittest/test_operator.py
+++ b/tests/python/unittest/test_operator.py
@@ -3178,6 +3178,27 @@ def test_quantization_op():
assert same(a_.asnumpy(), a_real.asnumpy())
+def test_crelu():
+ def fcrelu(x):
+ return np.maximum(np.concatenate((x, -x), axis = x.ndim-1), 0.0)
+ def fcrelu_grad(x, y):
+ tmp = (1.0 * (y > 0.0))
+ y1,y2 = np.split(tmp, 2, y.ndim-1)
+ y1 = y1 * np.sign(x)
+ y2 = y2 * np.sign(x)
+ return y1+y2
+
+ shape = (2, 3)
+ x = mx.symbol.Variable("x")
+ y = mx.contrib.sym.crelu(x)
+ xa = np.random.uniform(low=-1.0,high=1.0,size=shape)
+ ya = fcrelu(xa)
+ ga = fcrelu_grad(xa, ya)
+ check_numeric_gradient(y, [xa], numeric_eps=1E-3)
+ check_symbolic_forward(y, [xa], [ya])
+ check_symbolic_backward(y, [xa], [np.ones((2,6))], [ga])
+
+
def test_custom_op():
class Sqr(mx.operator.CustomOp):
def forward(self, is_train, req, in_data, out_data, aux):
@@ -3565,3 +3586,72 @@ def test_laop():
if __name__ == '__main__':
import nose
nose.runmodule()
+ test_custom_op()
+ test_log_softmax()
+ test_new_softmax()
+ test_pick()
+ test_l2_normalization()
+ test_sequence_mask()
+ test_roipooling()
+ test_batchnorm_training()
+ test_order()
+ test_grid_generator()
+ test_dot()
+ test_cast()
+ test_clip()
+ test_index2d()
+ test_scalarop()
+ test_reduce()
+ test_init()
+ test_expand_dims()
+ test_slice_axis()
+ test_softmax()
+ test_broadcast_binary_op()
+ test_flip()
+ test_crop()
+ test_transpose()
+ test_convolution_grouping()
+ test_nearest_upsampling()
+ test_binary_op_duplicate_input()
+ test_elementwise_sum()
+ test_concat()
+ test_slice_channel()
+ test_regression()
+ test_python_op()
+ test_swapaxes()
+ test_scalar_pow()
+ test_symbol_pow()
+ test_pow_fn()
+ test_embedding()
+ test_rsqrt_cos_sin()
+ test_maximum_minimum()
+ test_maximum_minimum_scalar()
+ test_abs()
+ test_round_ceil_floor()
+ test_deconvolution()
+ check_softmax_with_ignore_label(default_context())
+ test_convolution_dilated_impulse_response()
+ test_reshape()
+ test_broadcast()
+ test_stn()
+ test_batch_dot()
+ test_correlation()
+ test_support_vector_machine_l1_svm()
+ test_support_vector_machine_l2_svm()
+ test_pad()
+ test_instance_normalization()
+ test_mathematical()
+ test_special_functions_using_scipy()
+ test_blockgrad()
+ test_take()
+ test_bilinear_sampler()
+ test_binary_logic()
+ test_repeat()
+ test_tile()
+ test_one_hot()
+ test_where()
+ test_ctc_loss()
+ test_quantization_op()
+ test_relu()
+ test_sigmoid()
+ test_crelu()
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services