You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ha...@apache.org on 2018/05/11 17:40:51 UTC
[incubator-mxnet] branch master updated: [MXNET-329] deprecate
contrib.SparseEmbedding (#10877)
This is an automated email from the ASF dual-hosted git repository.
haibin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 74f104a [MXNET-329] deprecate contrib.SparseEmbedding (#10877)
74f104a is described below
commit 74f104ae9906958b067171d8d639e922c33aef69
Author: Haibin Lin <li...@gmail.com>
AuthorDate: Fri May 11 10:40:45 2018 -0700
[MXNET-329] deprecate contrib.SparseEmbedding (#10877)
* deprecate contrib.SparseEmbedding
* update doc
* remove sparse alias for softsign
* remove contrib.SparseEmbedding
* update doc
---
docs/api/python/ndarray/sparse.md | 2 +-
docs/api/python/symbol/sparse.md | 2 +-
example/rnn/large_word_lm/model.py | 17 +++--
example/sparse/factorization_machine/README.md | 2 +-
example/sparse/matrix_factorization/README.md | 2 +-
example/sparse/matrix_factorization/model.py | 47 ++++++--------
example/sparse/wide_deep/model.py | 4 +-
src/operator/tensor/dot-inl.cuh | 2 -
src/operator/tensor/dot.cc | 20 +++---
src/operator/tensor/elemwise_unary_op_basic.cc | 1 -
src/operator/tensor/indexing_op.cc | 16 ++++-
src/operator/tensor/indexing_op.cu | 10 +--
src/operator/tensor/indexing_op.h | 87 +++++++++++++++++++++++++-
tests/python/unittest/test_sparse_operator.py | 50 ++++++++++++++-
14 files changed, 198 insertions(+), 64 deletions(-)
diff --git a/docs/api/python/ndarray/sparse.md b/docs/api/python/ndarray/sparse.md
index 28fb386..581a74f 100644
--- a/docs/api/python/ndarray/sparse.md
+++ b/docs/api/python/ndarray/sparse.md
@@ -499,7 +499,7 @@ We summarize the interface for each class in the following sections.
make_loss
stop_gradient
- mxnet.ndarray.contrib.SparseEmbedding
+ Embedding
LinearRegressionOutput
LogisticRegressionOutput
```
diff --git a/docs/api/python/symbol/sparse.md b/docs/api/python/symbol/sparse.md
index c4c8e92..86191e3 100644
--- a/docs/api/python/symbol/sparse.md
+++ b/docs/api/python/symbol/sparse.md
@@ -197,7 +197,7 @@ In the rest of this document, we list sparse related routines provided by the
make_loss
stop_gradient
- mxnet.symbol.contrib.SparseEmbedding
+ Embedding
LinearRegressionOutput
LogisticRegressionOutput
```
diff --git a/example/rnn/large_word_lm/model.py b/example/rnn/large_word_lm/model.py
index 7ee010e..3d3c83b 100644
--- a/example/rnn/large_word_lm/model.py
+++ b/example/rnn/large_word_lm/model.py
@@ -46,12 +46,11 @@ def cross_entropy_loss(inputs, labels, rescale_loss=1):
def rnn(bptt, vocab_size, num_embed, nhid, num_layers, dropout, num_proj, batch_size):
""" word embedding + LSTM Projected """
- embed = mx.sym.contrib.SparseEmbedding
state_names = []
data = S.var('data')
weight = S.var("encoder_weight", stype='row_sparse')
- embed = embed(data=data, weight=weight, input_dim=vocab_size,
- output_dim=num_embed, name='embed', deterministic=True)
+ embed = S.sparse.Embedding(data=data, weight=weight, input_dim=vocab_size,
+ output_dim=num_embed, name='embed', sparse_grad=True)
states = []
outputs = S.Dropout(embed, p=dropout)
for i in range(num_layers):
@@ -78,7 +77,6 @@ def sampled_softmax(num_classes, num_samples, in_dim, inputs, weight, bias,
This under-estimates the full softmax and is only used for training.
"""
# inputs = (n, in_dim)
- embed = mx.sym.contrib.SparseEmbedding
sample, prob_sample, prob_target = sampled_values
# (num_samples, )
@@ -90,12 +88,13 @@ def sampled_softmax(num_classes, num_samples, in_dim, inputs, weight, bias,
sample_label = S.concat(sample, label, dim=0)
# lookup weights and biases
# (num_samples+n, dim)
- sample_target_w = embed(data=sample_label, weight=weight,
- input_dim=num_classes, output_dim=in_dim,
- deterministic=True)
+ sample_target_w = S.sparse.Embedding(data=sample_label, weight=weight,
+ input_dim=num_classes, output_dim=in_dim,
+ sparse_grad=True)
# (num_samples+n, 1)
- sample_target_b = embed(data=sample_label, weight=bias,
- input_dim=num_classes, output_dim=1, deterministic=True)
+ sample_target_b = S.sparse.Embedding(data=sample_label, weight=bias,
+ input_dim=num_classes, output_dim=1,
+ sparse_grad=True)
# (num_samples, dim)
sample_w = S.slice(sample_target_w, begin=(0, 0), end=(num_samples, None))
target_w = S.slice(sample_target_w, begin=(num_samples, 0), end=(None, None))
diff --git a/example/sparse/factorization_machine/README.md b/example/sparse/factorization_machine/README.md
index 7609f31..32b956e 100644
--- a/example/sparse/factorization_machine/README.md
+++ b/example/sparse/factorization_machine/README.md
@@ -11,6 +11,6 @@ It takes more than 30 GB to download and extract the dataset.
## Train the Model
-- python train.py --train-data /path/to/criteo.kaggle2014.test.svm --test-data /path/to/criteo.kaggle2014.test.svm
+- python train.py --data-train /path/to/criteo.kaggle2014.test.svm --data-test /path/to/criteo.kaggle2014.test.svm
[Rendle, Steffen. "Factorization machines." In Data Mining (ICDM), 2010 IEEE 10th International Conference on, pp. 995-1000. IEEE, 2010. ](https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf)
diff --git a/example/sparse/matrix_factorization/README.md b/example/sparse/matrix_factorization/README.md
index 5c4beef..ddbf662 100644
--- a/example/sparse/matrix_factorization/README.md
+++ b/example/sparse/matrix_factorization/README.md
@@ -1,6 +1,6 @@
Matrix Factorization w/ Sparse Embedding
===========
-The example demonstrates the basic usage of the SparseEmbedding operator in MXNet, adapted based on @leopd's recommender examples.
+The example demonstrates the basic usage of the sparse.Embedding operator in MXNet, adapted based on @leopd's recommender examples.
This is for demonstration purpose only.
```
diff --git a/example/sparse/matrix_factorization/model.py b/example/sparse/matrix_factorization/model.py
index 672c392..68dd4d6 100644
--- a/example/sparse/matrix_factorization/model.py
+++ b/example/sparse/matrix_factorization/model.py
@@ -19,37 +19,30 @@ import mxnet as mx
def matrix_fact_net(factor_size, num_hidden, max_user, max_item, dense):
# input
- user = mx.symbol.Variable('user')
- item = mx.symbol.Variable('item')
- score = mx.symbol.Variable('score')
+ user = mx.sym.Variable('user')
+ item = mx.sym.Variable('item')
+ score = mx.sym.Variable('score')
stype = 'default' if dense else 'row_sparse'
- user_weight = mx.symbol.Variable('user_weight', stype=stype)
- item_weight = mx.symbol.Variable('item_weight', stype=stype)
- if not dense:
- embed = mx.symbol.contrib.SparseEmbedding
- # user feature lookup
- user = embed(data=user, weight=user_weight,
- input_dim=max_user, output_dim=factor_size, deterministic=True)
- # item feature lookup
- item = embed(data=item, weight=item_weight,
- input_dim=max_item, output_dim=factor_size, deterministic=True)
- else:
- # user feature lookup
- user = mx.symbol.Embedding(data=user, weight=user_weight,
- input_dim=max_user, output_dim=factor_size)
- # item feature lookup
- item = mx.symbol.Embedding(data=item, weight=item_weight,
- input_dim=max_item, output_dim=factor_size)
+ sparse_grad = not dense
+ user_weight = mx.sym.Variable('user_weight', stype=stype)
+ item_weight = mx.sym.Variable('item_weight', stype=stype)
+ # user feature lookup
+ user = mx.sym.Embedding(data=user, weight=user_weight, sparse_grad=sparse_grad,
+ input_dim=max_user, output_dim=factor_size)
+ # item feature lookup
+ item = mx.sym.Embedding(data=item, weight=item_weight, sparse_grad=sparse_grad,
+ input_dim=max_item, output_dim=factor_size)
+
# non-linear transformation of user features
- user = mx.symbol.Activation(data=user, act_type='relu')
- user_act = mx.symbol.FullyConnected(data=user, num_hidden=num_hidden)
+ user = mx.sym.Activation(data=user, act_type='relu')
+ user_act = mx.sym.FullyConnected(data=user, num_hidden=num_hidden)
# non-linear transformation of item features
- item = mx.symbol.Activation(data=item, act_type='relu')
- item_act = mx.symbol.FullyConnected(data=item, num_hidden=num_hidden)
+ item = mx.sym.Activation(data=item, act_type='relu')
+ item_act = mx.sym.FullyConnected(data=item, num_hidden=num_hidden)
# predict by the inner product, which is elementwise product and then sum
pred = user_act * item_act
- pred = mx.symbol.sum(data=pred, axis=1)
- pred = mx.symbol.Flatten(data=pred)
+ pred = mx.sym.sum(data=pred, axis=1)
+ pred = mx.sym.Flatten(data=pred)
# loss layer
- pred = mx.symbol.LinearRegressionOutput(data=pred, label=score)
+ pred = mx.sym.LinearRegressionOutput(data=pred, label=score)
return pred
diff --git a/example/sparse/wide_deep/model.py b/example/sparse/wide_deep/model.py
index b907455..37c83fb 100644
--- a/example/sparse/wide_deep/model.py
+++ b/example/sparse/wide_deep/model.py
@@ -44,8 +44,8 @@ def wide_deep_model(num_linear_features, num_embed_features, num_cont_features,
for i, embed in enumerate(embeds):
embed_weight = mx.symbol.Variable('embed_%d_weight' % i, stype='row_sparse')
- features.append(mx.symbol.contrib.SparseEmbedding(data=embed, weight=embed_weight,
- input_dim=input_dims[i], output_dim=hidden_units[0]))
+ features.append(mx.symbol.sparse.Embedding(data=embed, weight=embed_weight,
+ input_dim=input_dims[i], output_dim=hidden_units[0], sparse_grad=True))
hidden = mx.symbol.concat(*features, dim=1)
hidden = mx.symbol.FullyConnected(data=hidden, num_hidden=hidden_units[1])
diff --git a/src/operator/tensor/dot-inl.cuh b/src/operator/tensor/dot-inl.cuh
index e3e0b28..c507a9a 100644
--- a/src/operator/tensor/dot-inl.cuh
+++ b/src/operator/tensor/dot-inl.cuh
@@ -31,8 +31,6 @@
#include "./sort_op.h"
#include "./util/tensor_util-inl.h"
#include "./util/tensor_util-inl.cuh"
-#include "./indexing_op.h"
-
namespace mxnet {
namespace op {
diff --git a/src/operator/tensor/dot.cc b/src/operator/tensor/dot.cc
index 9d62d0d..11a8b27 100644
--- a/src/operator/tensor/dot.cc
+++ b/src/operator/tensor/dot.cc
@@ -51,19 +51,19 @@ NNVM_REGISTER_OP(dot)
dot(x,y)[0,0,1,1] = 0
sum(x[0,0,:]*y[:,1,1]) = 0
-The storage type of ``dot`` output depends on storage types of inputs, transpose options and given
-hint for output storage type:
+The storage type of ``dot`` output depends on storage types of inputs, transpose option and
+forward_stype option for output storage type. Implemented sparse operations include:
-Implemented sprase operations include:
-- dot(csr, default) = default
+- dot(default, default, transpose_a=True/False, transpose_b=True/False) = default
+- dot(csr, default, transpose_a=True) = default
- dot(csr, default, transpose_a=True) = row_sparse
- dot(csr, row_sparse) = default
-- dot(default, csr) = csr on CPU only
-- dot(default, csr, forward_stype='default') = default on GPU only
-- dot(default, csr, transpose_b=True, forward_stype='default') = default on GPU only
-- if the combination of input storage types and forward_stype_hint
-- does not match any of the above patterns,
-- dot will generate output with default storage
+- dot(default, csr) = csr (CPU only)
+- dot(default, csr, forward_stype='default') = default (GPU only)
+- dot(default, csr, transpose_b=True, forward_stype='default') = default (GPU only)
+
+If the combination of input storage types and forward_stype does not match any of the
+above patterns, ``dot`` will fallback and generate output with default storage.
)doc" ADD_FILELINE)
.set_num_inputs(2)
diff --git a/src/operator/tensor/elemwise_unary_op_basic.cc b/src/operator/tensor/elemwise_unary_op_basic.cc
index 93519d3..46f6265 100644
--- a/src/operator/tensor/elemwise_unary_op_basic.cc
+++ b/src/operator/tensor/elemwise_unary_op_basic.cc
@@ -138,7 +138,6 @@ NNVM_REGISTER_OP(_backward_hard_sigmoid)
// softsign
MXNET_OPERATOR_REGISTER_UNARY(softsign)
-MXNET_ADD_SPARSE_OP_ALIAS(softsign)
.describe(R"code(Computes softsign of x element-wise.
.. math::
diff --git a/src/operator/tensor/indexing_op.cc b/src/operator/tensor/indexing_op.cc
index fbb94b2..e5ba058 100644
--- a/src/operator/tensor/indexing_op.cc
+++ b/src/operator/tensor/indexing_op.cc
@@ -70,7 +70,7 @@ void SparseEmbeddingOpForwardRspImpl<cpu>(const OpContext& ctx,
template<>
-inline void SparseEmbeddingOpBackwardRspImpl<cpu>(const SparseEmbeddingParam& param,
+inline void SparseEmbeddingOpBackwardRspImpl<cpu>(const bool deterministic,
const OpContext& ctx,
const TBlob& ograd,
const TBlob& data,
@@ -185,6 +185,7 @@ DMLC_REGISTER_PARAMETER(OneHotParam);
DMLC_REGISTER_PARAMETER(ScatterNDParam);
NNVM_REGISTER_OP(Embedding)
+MXNET_ADD_SPARSE_OP_ALIAS(Embedding)
.describe(R"code(Maps integer indices to vector representations (embeddings).
This operator maps words to real-valued vectors in a high-dimensional space,
@@ -224,6 +225,10 @@ Examples::
[[ 0., 1., 2., 3., 4.],
[ 10., 11., 12., 13., 14.]]]
+
+The storage type of weight can be either row_sparse or default, while
+the storage type of weight's grad depends on the value of "sparse_grad".
+
)code" ADD_FILELINE)
.set_num_inputs(2)
.set_num_outputs(1)
@@ -234,11 +239,13 @@ Examples::
})
.set_attr<nnvm::FInferShape>("FInferShape", EmbeddingOpShape<EmbeddingParam>)
.set_attr<nnvm::FInferType>("FInferType", EmbeddingOpType<EmbeddingParam>)
+.set_attr<FInferStorageType>("FInferStorageType", EmbeddingOpForwardStorageType)
.set_attr<FResourceRequest>("FResourceRequest",
[](const NodeAttrs& attrs) {
return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
})
.set_attr<FCompute>("FCompute<cpu>", EmbeddingOpForward<cpu>)
+.set_attr<FComputeEx>("FComputeEx<cpu>", SparseEmbeddingOpForwardEx<cpu>)
.set_attr<nnvm::FGradient>("FGradient",
[](const nnvm::NodePtr& n, const std::vector<nnvm::NodeEntry>& ograds) {
return MakeNonlossGradNode("_backward_Embedding", n, ograds,
@@ -251,6 +258,8 @@ Examples::
NNVM_REGISTER_OP(_contrib_SparseEmbedding)
.describe(R"code(Maps integer indices to vector representations (embeddings).
+note:: ``contrib.SparseEmbedding`` is deprecated, use ``Embedding`` instead.
+
This operator maps words to real-valued vectors in a high-dimensional space,
called word embeddings. These embeddings can capture semantic and syntactic properties of the words.
For example, it has been noted that in the learned embedding spaces, similar words tend
@@ -324,12 +333,15 @@ Examples::
NNVM_REGISTER_OP(_backward_Embedding)
.set_num_inputs(2)
.set_num_outputs(2)
+.set_attr_parser(ParamParser<EmbeddingParam>)
.set_attr<FResourceRequest>("FResourceRequest",
[](const NodeAttrs& attrs) {
return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
})
+.set_attr<FInferStorageType>("FInferStorageType", EmbeddingOpBackwardStorageType)
.set_attr<nnvm::TIsBackward>("TIsBackward", true)
-.set_attr<FCompute>("FCompute<cpu>", EmbeddingOpBackward<cpu>);
+.set_attr<FCompute>("FCompute<cpu>", EmbeddingOpBackward<cpu>)
+.set_attr<FComputeEx>("FComputeEx<cpu>", EmbeddingOpBackwardEx<cpu>);
NNVM_REGISTER_OP(_backward_SparseEmbedding)
.set_attr_parser(ParamParser<SparseEmbeddingParam>)
diff --git a/src/operator/tensor/indexing_op.cu b/src/operator/tensor/indexing_op.cu
index 8d4fefb..593593a 100644
--- a/src/operator/tensor/indexing_op.cu
+++ b/src/operator/tensor/indexing_op.cu
@@ -281,13 +281,13 @@ inline void SparseEmbeddingOpBackwardDeterministicRspImpl(const OpContext& ctx,
template<>
-inline void SparseEmbeddingOpBackwardRspImpl<gpu>(const SparseEmbeddingParam& param,
+inline void SparseEmbeddingOpBackwardRspImpl<gpu>(const bool deterministic,
const OpContext& ctx,
const TBlob& ograd,
const TBlob& data,
const OpReqType req,
const NDArray& output) {
- if (param.deterministic) {
+ if (deterministic) {
SparseEmbeddingOpBackwardDeterministicRspImpl(ctx, ograd, data, req, output);
return;
}
@@ -386,13 +386,15 @@ inline void GatherNDBackwardImpl(int N, int M, int K,
}
NNVM_REGISTER_OP(Embedding)
-.set_attr<FCompute>("FCompute<gpu>", EmbeddingOpForward<gpu>);
+.set_attr<FCompute>("FCompute<gpu>", EmbeddingOpForward<gpu>)
+.set_attr<FComputeEx>("FComputeEx<gpu>", SparseEmbeddingOpForwardEx<gpu>);
NNVM_REGISTER_OP(_contrib_SparseEmbedding)
.set_attr<FComputeEx>("FComputeEx<gpu>", SparseEmbeddingOpForwardEx<gpu>);
NNVM_REGISTER_OP(_backward_Embedding)
-.set_attr<FCompute>("FCompute<gpu>", EmbeddingOpBackward<gpu>);
+.set_attr<FCompute>("FCompute<gpu>", EmbeddingOpBackward<gpu>)
+.set_attr<FComputeEx>("FComputeEx<gpu>", EmbeddingOpBackwardEx<gpu>);
NNVM_REGISTER_OP(_backward_SparseEmbedding)
.set_attr<FComputeEx>("FComputeEx<gpu>", SparseEmbeddingOpBackwardEx<gpu>);
diff --git a/src/operator/tensor/indexing_op.h b/src/operator/tensor/indexing_op.h
index f824add..28827db 100644
--- a/src/operator/tensor/indexing_op.h
+++ b/src/operator/tensor/indexing_op.h
@@ -83,6 +83,7 @@ struct EmbeddingParam: public dmlc::Parameter<EmbeddingParam> {
int input_dim;
int output_dim;
int dtype;
+ bool sparse_grad;
DMLC_DECLARE_PARAMETER(EmbeddingParam) {
DMLC_DECLARE_FIELD(input_dim).set_lower_bound(1)
.describe("Vocabulary size of the input indices.");
@@ -91,6 +92,9 @@ struct EmbeddingParam: public dmlc::Parameter<EmbeddingParam> {
DMLC_DECLARE_FIELD(dtype).set_default(mshadow::kFloat32)
MXNET_ADD_ALL_TYPES
.describe("Data type of weight.");
+ DMLC_DECLARE_FIELD(sparse_grad).set_default(false)
+ .describe("Compute row sparse gradient in the backward calculation. If set to True, "
+ "the grad's storage type is row_sparse.");
}
};
@@ -196,6 +200,32 @@ inline bool EmbeddingOpType(const nnvm::NodeAttrs& attrs,
return true;
}
+// storage type inference function for Embedding
+inline bool EmbeddingOpForwardStorageType(const nnvm::NodeAttrs& attrs,
+ const int dev_mask,
+ DispatchMode* dispatch_mode,
+ std::vector<int>* in_attrs,
+ std::vector<int>* out_attrs) {
+ CHECK_EQ(in_attrs->size(), 2U);
+ CHECK_EQ(out_attrs->size(), 1U);
+ const int& data_stype = in_attrs->at(embedding::kData);
+ const int& weight_stype = in_attrs->at(embedding::kWeight);
+ int& out_stype = out_attrs->at(embedding::kOut);
+ bool dispatched = false;
+ if (!dispatched && data_stype == kDefaultStorage && weight_stype == kDefaultStorage) {
+ // dns, dns -> dns
+ dispatched = storage_type_assign(&out_stype, kDefaultStorage,
+ dispatch_mode, DispatchMode::kFCompute);
+ }
+ if (!dispatched && data_stype == kDefaultStorage && weight_stype == kRowSparseStorage) {
+ // dns, rsp -> dns
+ dispatched = storage_type_assign(&out_stype, kDefaultStorage,
+ dispatch_mode, DispatchMode::kFComputeEx);
+ }
+ return dispatched;
+}
+
+// storage type inference function for SparseEmbedding
inline bool SparseEmbeddingOpForwardStorageType(const nnvm::NodeAttrs& attrs,
const int dev_mask,
DispatchMode* dispatch_mode,
@@ -216,7 +246,34 @@ inline bool SparseEmbeddingOpForwardStorageType(const nnvm::NodeAttrs& attrs,
return dispatched;
}
+// storage type inference function for _backward_Embedding
+inline bool EmbeddingOpBackwardStorageType(const nnvm::NodeAttrs& attrs,
+ const int dev_mask,
+ DispatchMode* dispatch_mode,
+ std::vector<int>* in_attrs,
+ std::vector<int>* out_attrs) {
+ CHECK_EQ(in_attrs->size(), 2U);
+ CHECK_EQ(out_attrs->size(), 2U);
+ const bool sparse_grad = nnvm::get<EmbeddingParam>(attrs.parsed).sparse_grad;
+ const NDArrayStorageType target_stype = sparse_grad ? kRowSparseStorage : kDefaultStorage;
+ const auto target_mode = sparse_grad ? DispatchMode::kFComputeEx : DispatchMode::kFCompute;
+ const int ograd_stype = in_attrs->at(0);
+ const int data_stype = in_attrs->at(1);
+ int& data_grad_stype = out_attrs->at(0);
+ int& weight_grad_stype = out_attrs->at(1);
+ bool dispatched = false;
+ if (!dispatched && ograd_stype == kDefaultStorage && data_stype == kDefaultStorage) {
+ // dns, dns -> dns, dns/rsp
+ if (type_assign(&data_grad_stype, kDefaultStorage) &&
+ type_assign(&weight_grad_stype, target_stype)) {
+ dispatched = dispatch_mode_assign(dispatch_mode, target_mode);
+ }
+ }
+ return dispatched;
+}
+
+// storage type inference function for _backward_SparseEmbedding
inline bool SparseEmbeddingOpBackwardStorageType(const nnvm::NodeAttrs& attrs,
const int dev_mask,
DispatchMode* dispatch_mode,
@@ -245,6 +302,7 @@ inline bool SparseEmbeddingOpBackwardStorageType(const nnvm::NodeAttrs& attrs,
}
return dispatched;
}
+
/*! \brief name the struct Take instead of take
* to avoid conflict with the take function in mshadow
*/
@@ -590,7 +648,7 @@ struct AddTakeGradRspKernel {
};
template<typename xpu>
-inline void SparseEmbeddingOpBackwardRspImpl(const SparseEmbeddingParam& param,
+inline void SparseEmbeddingOpBackwardRspImpl(const bool deterministic,
const OpContext& ctx,
const TBlob& ograd,
const TBlob& data,
@@ -598,6 +656,31 @@ inline void SparseEmbeddingOpBackwardRspImpl(const SparseEmbeddingParam& param,
const NDArray& output);
template<typename xpu>
+void EmbeddingOpBackwardEx(const nnvm::NodeAttrs& attrs,
+ const OpContext& ctx,
+ const std::vector<NDArray>& inputs,
+ const std::vector<OpReqType>& req,
+ const std::vector<NDArray>& outputs) {
+ CHECK_EQ(inputs.size(), 2U);
+ CHECK_EQ(outputs.size(), 2U);
+ const NDArray& weight_grad = outputs[1];
+ const NDArray& ograd = inputs[0];
+ const NDArray& data = inputs[1];
+ // check dtype
+ CHECK_EQ(weight_grad.dtype(), ograd.dtype());
+ // check req
+ CHECK_EQ(req[embedding::kData], kNullOp)
+ << "Embedding layer doesn't support calculate data gradient";
+ if (data.storage_type() == kDefaultStorage && ograd.storage_type() == kDefaultStorage &&
+ weight_grad.storage_type() == kRowSparseStorage) {
+ SparseEmbeddingOpBackwardRspImpl<xpu>(true, ctx, ograd.data(), data.data(),
+ req[embedding::kWeight], weight_grad);
+ } else {
+ LogUnimplementedOp(attrs, ctx, inputs, req, outputs);
+ }
+}
+
+template<typename xpu>
void SparseEmbeddingOpBackwardEx(const nnvm::NodeAttrs& attrs,
const OpContext& ctx,
const std::vector<NDArray>& inputs,
@@ -616,7 +699,7 @@ void SparseEmbeddingOpBackwardEx(const nnvm::NodeAttrs& attrs,
const SparseEmbeddingParam& param = nnvm::get<SparseEmbeddingParam>(attrs.parsed);
if (data.storage_type() == kDefaultStorage && ograd.storage_type() == kDefaultStorage &&
weight_grad.storage_type() == kRowSparseStorage) {
- SparseEmbeddingOpBackwardRspImpl<xpu>(param, ctx, ograd.data(), data.data(),
+ SparseEmbeddingOpBackwardRspImpl<xpu>(param.deterministic, ctx, ograd.data(), data.data(),
req[embedding::kWeight], weight_grad);
} else {
LogUnimplementedOp(attrs, ctx, inputs, req, outputs);
diff --git a/tests/python/unittest/test_sparse_operator.py b/tests/python/unittest/test_sparse_operator.py
index f21334e..1b3c128 100644
--- a/tests/python/unittest/test_sparse_operator.py
+++ b/tests/python/unittest/test_sparse_operator.py
@@ -1718,7 +1718,7 @@ def test_sparse_elementwise_sum():
@with_seed()
-def test_sparse_embedding():
+def test_contrib_sparse_embedding():
''' test sparse embedding operator '''
def check_sparse_embedding(in_dim, out_dim, batch, densities, deterministic, weight_stype):
# init executor
@@ -1769,6 +1769,54 @@ def test_sparse_embedding():
check_sparse_embedding(in_dim, out_dim, batch, densities, deterministic, stype)
@with_seed()
+def test_sparse_embedding():
+ ''' test sparse embedding operator '''
+ def check_sparse_embedding(in_dim, out_dim, batch, densities, sparse_grad, weight_stype):
+ target_stype = 'row_sparse' if sparse_grad else 'default'
+ # init executor
+ data = mx.sym.Variable("data")
+ weight = mx.sym.Variable("embed_weight", stype=weight_stype)
+ embed = mx.sym.sparse.Embedding(data=data, weight=weight, input_dim=in_dim,
+ sparse_grad=sparse_grad, output_dim=out_dim, name='embed')
+ grad_req = {'data': 'null', 'embed_weight': 'write'}
+ exe_test = embed.simple_bind(default_context(), grad_req=grad_req, data=(batch,))
+ arg_map = dict(zip(embed.list_arguments(), exe_test.arg_arrays))
+ grad_map = dict(zip(embed.list_arguments(), exe_test.grad_arrays))
+ # init data
+ np_data = np.random.randint(low=0, high=in_dim, size=batch)
+ np_onehot = np.zeros((batch, in_dim)).astype(np.float32)
+ np_onehot[np.arange(batch), np_data] = 1.0
+ arg_map["data"][:] = np_data
+ # init grad
+ np_grad = np.random.uniform(-1, 1, exe_test.outputs[0].shape)
+ grad = mx.nd.zeros(np_grad.shape)
+ grad[:] = np_grad
+ # weight
+ weight = arg_map["embed_weight"]
+ for density in densities:
+ # update weight based on density
+ weight[:] = rand_ndarray(weight.shape, weight_stype, density=density)
+ # check forward
+ exe_test.forward(is_train=True)
+ assert_almost_equal(exe_test.outputs[0].asnumpy(), np.dot(np_onehot, weight.asnumpy()), atol=1e-4)
+ # check backward
+ exe_test.backward([grad])
+ assert_almost_equal(grad_map["embed_weight"].asnumpy(), np.dot(np_onehot.T, grad.asnumpy()), atol=1e-4)
+ # check grad stype
+ assert(grad_map["embed_weight"].stype == target_stype)
+
+ densities = [0, 0.5, 1]
+ in_dim = 50
+ out_dim = 3
+ batch = 8
+ weight_stypes = ['default', 'row_sparse']
+ sparse_grads = [True, False]
+ for weight_stype in weight_stypes:
+ for sparse_grad in sparse_grads:
+ check_sparse_embedding(in_dim, out_dim, batch, densities, sparse_grad, weight_stype)
+ check_sparse_embedding(in_dim, out_dim, batch, densities, sparse_grad, weight_stype)
+
+@with_seed()
def test_sparse_broadcast_add_sub():
def check_broadcast_add(mx_lhs, mx_rhs, np_lhs, np_rhs, dtype):
assert_almost_equal(mx.nd.sparse.add(mx_lhs, mx_rhs).asnumpy(), np.add(np_lhs, np_rhs), atol=1e-4)
--
To stop receiving notification emails like this one, please contact
haibin@apache.org.