[GitHub] [incubator-mxnet] apeforest commented on a change in pull request #15480: [MXNET-978] Higher Order Gradient Support `elemwise_mul`, `elemwise_add`.

[GitBox <gi...@apache.org>]

apeforest commented on a change in pull request #15480: [MXNET-978] Higher Order Gradient Support `elemwise_mul`, `elemwise_add`.
URL: https://github.com/apache/incubator-mxnet/pull/15480#discussion_r300884763
 
 

 ##########
 File path: src/operator/tensor/elemwise_binary_op_basic.cc
 ##########
 @@ -274,7 +298,43 @@ NNVM_REGISTER_OP(_backward_div)
 .set_attr<FCompute>("FCompute<cpu>", ElemwiseBinaryOp::BackwardUseIn<
   cpu, mshadow_op::div_grad, mshadow_op::div_rgrad>)
 .set_attr<FComputeEx>("FComputeEx<cpu>", ElemwiseBinaryOp::BackwardUseInEx<
-  cpu, mshadow_op::div_grad, mshadow_op::div_rgrad>);
+  cpu, mshadow_op::div_grad, mshadow_op::div_rgrad>)
+.set_attr<nnvm::FGradient>("FGradient",
+  [](const nnvm::NodePtr& n, const std::vector<nnvm::NodeEntry>& ograds) {
+    // z = x / y
+    // NodeEntry{n, 0, 0} : z_grad * (1/y)
+    // NodeEntry{n, 1, 0} : z_grad * (-x/y^2)
+    // n->inputs[0] : z_grad
+    // n->inputs[1] : x
+    // n->inputs[1] : y
+    // ograds[0] : head_grads
+    // f(x, y) = x / y
+    // dx = z_grad * (1/y), dy = z_grad * (-x/y^2)
+    // d2x = 0, d2y = dy * (-2/x) = (2x/y^3), dz_grad = (dx + dy) / (z_grad)
+    auto dx = nnvm::NodeEntry{n, 0, 0};
+    auto dy = nnvm::NodeEntry{n, 1, 0};
+    auto dx_add_dy = MakeNode("elemwise_add", n->attrs.name + "_x_add_y",
+                              {dx, dy}, nullptr, &n);
+    auto dz_grad = MakeNode("elemwise_div", n->attrs.name + "_x_add_y",
+                                         {nnvm::NodeEntry{dx_add_dy}, n->inputs[0]}, nullptr, &n);
+
+    const std::unordered_map<std::string, std::string> two = {{"scalar", "-2.0"}};
+    auto y = n->inputs[2];
+    auto r_y = MakeNode("reciprocal", n->attrs.name + "_r_y", {y}, nullptr, &n);
+    auto neg_two_r_y = MakeNode("_mul_scalar", n->attrs.name + "_neg_two_r_y",
+                            {nnvm::NodeEntry{r_y}}, &two, &n);
+    auto d2y = MakeNode("elemwise_mul", n->attrs.name + "_d2y",
+                            {nnvm::NodeEntry{neg_two_r_y}, dy}, &two, &n);
+
+    std::vector<nnvm::NodeEntry> ret;
+    ret.emplace_back(MakeNode("elemwise_mul", n->attrs.name + "_backward_grad_grad",
+                              {ograds[0], nnvm::NodeEntry{dz_grad}}, nullptr, &n));
+    ret.emplace_back(MakeNode("zeros_like", n->attrs.name + "_backward_grad_grad_x",
 
 Review comment:
   Really?  Have you tested y = x * x?

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services