You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2023/02/22 12:56:32 UTC
[tvm] branch unity updated: [Unity][Op] `log_softmax` and `cross_entropy_with_logits` (#14083)
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch unity
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/unity by this push:
new 4f9cd5a0fd [Unity][Op] `log_softmax` and `cross_entropy_with_logits` (#14083)
4f9cd5a0fd is described below
commit 4f9cd5a0fd96dcfa53dbc36df039efc529f7d60f
Author: Chaosfan <17...@qq.com>
AuthorDate: Wed Feb 22 20:56:24 2023 +0800
[Unity][Op] `log_softmax` and `cross_entropy_with_logits` (#14083)
This PR introduces two high-level operators log_softmax and cross_entropy_with_logits, which are important when we are calculating CrossEntropyLoss (in torch).
Co-authored-by: Yixin Dong <ub...@gmail.com>
---
python/tvm/relax/op/nn/nn.py | 55 +++++
python/tvm/relax/transform/legalize_ops/nn.py | 20 ++
src/relax/op/nn/nn.cc | 74 ++++++
src/relax/op/nn/nn.h | 6 +
tests/python/relax/test_op_nn.py | 173 ++++++++++++-
.../python/relax/test_transform_legalize_ops_nn.py | 268 +++++++++++++++++++++
tests/python/relax/test_tvmscript_parser_op_nn.py | 33 +++
7 files changed, 621 insertions(+), 8 deletions(-)
diff --git a/python/tvm/relax/op/nn/nn.py b/python/tvm/relax/op/nn/nn.py
index cdf0e96464..0ff143fd04 100644
--- a/python/tvm/relax/op/nn/nn.py
+++ b/python/tvm/relax/op/nn/nn.py
@@ -348,6 +348,34 @@ def softmax(data: Expr, axis: int = -1) -> Expr:
return _ffi_api.softmax(data, axis) # type: ignore
+def log_softmax(data: Expr, axis: int = -1) -> Expr:
+ r"""Computes log softmax.
+
+ .. math::
+
+ \text{log\_softmax}(x_i) = \log\left( \frac{\exp(x_i)}{\sum_j \exp(x_j)}\right)
+
+ .. note::
+ This operator can be optimized away for inference.
+
+ Parameters
+ ----------
+ data: relax.Expr
+ The input data to the operator.
+
+ axis: int
+ The axis to sum over when computing log softmax.
+ If not specified, it is by default the last axis of the input tensor.
+ Supports negative indexing.
+
+ Returns
+ -------
+ result : relax.Expr
+ The computed result.
+ """
+ return _ffi_api.log_softmax(data, axis) # type: ignore
+
+
def batch_norm(
data: Expr,
gamma: Expr,
@@ -522,3 +550,30 @@ def dropout(data: Expr, rate: float = 0.5) -> Expr:
mask tensor (1.0 where element not dropped, 0.0 where dropped)
"""
return _ffi_api.dropout(data, rate) # type: ignore
+
+
+def cross_entropy_with_logits(predictions: Expr, labels: Expr) -> Expr:
+ r"""CrossEntropy with logits between the predictions and labels.
+
+ The shape of predictions and labels must be the same. And when ndim >= 2,
+ the first dimension is regarded as the batch_size N. In this case the
+ computed result will divide by N to perform a mean reduction.
+
+ .. math::
+
+ \text{cross\_entropy\_with\_logits}(x_i, y_i) = \frac{\sum_i -x_i \cdot y_i}{N}
+
+ Parameters
+ ----------
+ predictions : relax.Expr
+ The predictions.
+
+ labels : relax.Expr
+ The labels (the ground truth values).
+
+ Returns
+ -------
+ result : relax.Expr
+ The computed result.
+ """
+ return _ffi_api.cross_entropy_with_logits(predictions, labels) # type: ignore
diff --git a/python/tvm/relax/transform/legalize_ops/nn.py b/python/tvm/relax/transform/legalize_ops/nn.py
index 49f198306d..31c6eb04cd 100644
--- a/python/tvm/relax/transform/legalize_ops/nn.py
+++ b/python/tvm/relax/transform/legalize_ops/nn.py
@@ -144,6 +144,26 @@ def _nn_softmax(bb: BlockBuilder, call: Call) -> Expr:
return bb.call_te(topi.nn.softmax, call.args[0], call.attrs.axis)
+@register_legalize("relax.nn.log_softmax")
+def _nn_log_softmax(bb: BlockBuilder, call: Call):
+ return bb.call_te(topi.nn.log_softmax, call.args[0], call.attrs.axis)
+
+
+@register_legalize("relax.nn.cross_entropy_with_logits")
+def _nn_cross_entropy_with_logits(bb: BlockBuilder, call: Call):
+ def te_cross_entropy_with_logits(x, y):
+ if len(x.shape) > 1:
+ return -topi.sum(x * y) / x.shape[0]
+ return -topi.sum(x * y)
+
+ return bb.call_te(
+ te_cross_entropy_with_logits,
+ call.args[0],
+ call.args[1],
+ primfunc_name_hint="cross_entropy_with_logits",
+ )
+
+
@register_legalize("relax.nn.batch_norm")
def _nn_batch_norm(bb: BlockBuilder, call: Call) -> Expr:
return bb.call_te(
diff --git a/src/relax/op/nn/nn.cc b/src/relax/op/nn/nn.cc
index 66ae10fe6c..e63b3306f2 100644
--- a/src/relax/op/nn/nn.cc
+++ b/src/relax/op/nn/nn.cc
@@ -68,6 +68,22 @@ TVM_REGISTER_OP("relax.nn.softmax")
.set_attrs_type<SoftmaxAttrs>()
.set_attr<FInferStructInfo>("FInferStructInfo", InferStructInfoSoftmax);
+/* relax.nn.log_softmax */
+Expr log_softmax(Expr data, int axis) {
+ auto attrs = make_object<SoftmaxAttrs>();
+ attrs->axis = axis;
+ static const Op& op = Op::Get("relax.nn.log_softmax");
+ return Call(op, {data}, Attrs(attrs), {});
+}
+
+TVM_REGISTER_GLOBAL("relax.op.nn.log_softmax").set_body_typed(log_softmax);
+
+TVM_REGISTER_OP("relax.nn.log_softmax")
+ .set_num_inputs(1)
+ .add_argument("data", "Tensor", "The input tensor.")
+ .set_attrs_type<SoftmaxAttrs>()
+ .set_attr<FInferStructInfo>("FInferStructInfo", InferStructInfoSoftmax);
+
bool NormCheckDtypeAndShape(const Call& call, const BlockBuilder& ctx,
const Array<TensorStructInfo>& input_sinfo, Array<Integer> axes) {
Op op = Downcast<Op>(call->op);
@@ -241,5 +257,63 @@ TVM_REGISTER_OP("relax.nn.dropout")
.add_argument("data", "Tensor", "Input to which dropout will be applied.")
.set_attr<FInferStructInfo>("FInferStructInfo", InferStructInfoDropout);
+/* relax.nn.cross_entropy_with_logits */
+StructInfo InferStructInfoCrossEntropy(const Call& call, const BlockBuilder& ctx) {
+ Array<TensorStructInfo> input_sinfo = GetInputTensorStructInfo(call, ctx);
+ TensorStructInfo pred_sinfo = input_sinfo[0];
+ TensorStructInfo label_sinfo = input_sinfo[1];
+
+ // infer dtype
+ DataType dtype = InferBinaryArithOpOutDtype(call, ctx, pred_sinfo, label_sinfo);
+
+ // infer ndim
+ if (!pred_sinfo->IsUnknownNdim() && !label_sinfo->IsUnknownNdim() &&
+ pred_sinfo->ndim != label_sinfo->ndim) {
+ ctx->ReportFatal(Diagnostic::Error(call)
+ << "CrossEntropy requires predictions and labels to have the same ndim. "
+ "However, the ndim of predictions is "
+ << pred_sinfo->ndim << " while the ndim of labels is " << label_sinfo->ndim);
+ }
+
+ Optional<Array<PrimExpr>> pred_shape_value;
+ if (pred_sinfo->shape.defined()) {
+ pred_shape_value = GetStructInfoAs<ShapeStructInfoNode>(pred_sinfo->shape.value())->values;
+ }
+
+ Optional<Array<PrimExpr>> label_shape_value;
+ if (label_sinfo->shape.defined()) {
+ label_shape_value = GetStructInfoAs<ShapeStructInfoNode>(label_sinfo->shape.value())->values;
+ }
+
+ if (pred_shape_value.defined() && label_shape_value.defined()) {
+ arith::Analyzer* analyzer = ctx->GetAnalyzer();
+ for (size_t i = 0; i < pred_shape_value.value().size(); ++i) {
+ if (analyzer->CanProve(pred_shape_value.value()[i] != label_shape_value.value()[i])) {
+ ctx->ReportFatal(Diagnostic::Error(call)
+ << "CrossEntropy requires the predictions and labels to have "
+ "the same shape. However, the shape of predictions at dim "
+ << i << " is" << pred_shape_value.value()[i]
+ << " while the shape of labels at this dim is "
+ << label_shape_value.value()[i]);
+ }
+ }
+ }
+ return TensorStructInfo(ShapeExpr(Array<PrimExpr>()), dtype);
+}
+
+Expr cross_entropy_with_logits(Expr predictions, Expr labels) {
+ static const Op& op = Op::Get("relax.nn.cross_entropy_with_logits");
+ return Call(op, {std::move(predictions), std::move(labels)}, {}, {});
+}
+
+TVM_REGISTER_GLOBAL("relax.op.nn.cross_entropy_with_logits")
+ .set_body_typed(cross_entropy_with_logits);
+
+TVM_REGISTER_OP("relax.nn.cross_entropy_with_logits")
+ .set_num_inputs(2)
+ .add_argument("predictions", "Tensor", "The predictions.")
+ .add_argument("labels", "Tensor", "The labels.")
+ .set_attr<FInferStructInfo>("FInferStructInfo", InferStructInfoCrossEntropy);
+
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/nn/nn.h b/src/relax/op/nn/nn.h
index df2b978fc2..f13b930fc2 100644
--- a/src/relax/op/nn/nn.h
+++ b/src/relax/op/nn/nn.h
@@ -57,6 +57,9 @@ Expr silu(Expr data);
/*! \brief Softmax function. */
Expr softmax(Expr data, int axis);
+/*! \brief LogSoftmax function. */
+Expr log_softmax(Expr data, int axis);
+
/*! \brief Compute batch normalization. */
Expr batch_norm(Expr data, Expr gamma, Expr beta, Expr moving_mean, Expr moving_var, //
int axis, double epsilon, bool center, bool scale);
@@ -75,6 +78,9 @@ Expr layer_norm(Expr data, Expr gamma, Expr beta, Array<Integer> axes, double ep
*/
Expr dropout(Expr data, double rate);
+/*! \brief CrossEntropy with logits. */
+Expr cross_entropy_with_logits(Expr predictions, Expr labels);
+
} // namespace relax
} // namespace tvm
diff --git a/tests/python/relax/test_op_nn.py b/tests/python/relax/test_op_nn.py
index d047448309..5294596cee 100644
--- a/tests/python/relax/test_op_nn.py
+++ b/tests/python/relax/test_op_nn.py
@@ -29,6 +29,7 @@ def test_op_correctness():
assert relax.op.nn.gelu(x).op == Op.get("relax.nn.gelu")
assert relax.op.nn.silu(x).op == Op.get("relax.nn.silu")
assert relax.op.nn.softmax(x).op == Op.get("relax.nn.softmax")
+ assert relax.op.nn.log_softmax(x).op == Op.get("relax.nn.log_softmax")
assert relax.op.nn.dropout(x).op == Op.get("relax.nn.dropout")
x = relax.Var("x", R.Tensor((2, 3, 32, 32), "float32"))
@@ -41,6 +42,12 @@ def test_op_correctness():
)
assert relax.op.nn.layer_norm(x, gamma, beta, axes=1).op == Op.get("relax.nn.layer_norm")
+ x = relax.Var("x", R.Tensor((2, 3), "float32"))
+ y = relax.Var("y", R.Tensor((2, 3), "float32"))
+ assert relax.op.nn.cross_entropy_with_logits(x, y).op == Op.get(
+ "relax.nn.cross_entropy_with_logits"
+ )
+
def _check_inference(bb: relax.BlockBuilder, call: relax.Call, expected_sinfo: relax.StructInfo):
ret = bb.normalize(call)
@@ -117,7 +124,7 @@ def test_linear_unit_infer_struct_info_wrong_input_type():
bb.normalize(relax.op.nn.silu(x1))
-def test_softmax_infer_struct_info():
+def test_softmax_log_softmax_infer_struct_info():
bb = relax.BlockBuilder()
x0 = relax.Var("x", R.Tensor((2, 3), "float32"))
x1 = relax.Var("x", R.Tensor("float32", ndim=3))
@@ -133,8 +140,20 @@ def test_softmax_infer_struct_info():
_check_inference(bb, relax.op.nn.softmax(x3, axis=-1), relax.TensorStructInfo((2, 3), dtype=""))
_check_inference(bb, relax.op.nn.softmax(x4, axis=-2), relax.TensorStructInfo(dtype=""))
+ _check_inference(bb, relax.op.nn.log_softmax(x0), relax.TensorStructInfo((2, 3), "float32"))
+ _check_inference(
+ bb, relax.op.nn.log_softmax(x1, axis=0), relax.TensorStructInfo(dtype="float32", ndim=3)
+ )
+ _check_inference(
+ bb, relax.op.nn.log_softmax(x2, axis=1), relax.TensorStructInfo(dtype="float32")
+ )
+ _check_inference(
+ bb, relax.op.nn.log_softmax(x3, axis=-1), relax.TensorStructInfo((2, 3), dtype="")
+ )
+ _check_inference(bb, relax.op.nn.log_softmax(x4, axis=-2), relax.TensorStructInfo(dtype=""))
+
-def test_softmax_infer_struct_info_shape_symbolic():
+def test_softmax_log_softmax_infer_struct_info_shape_symbolic():
bb = relax.BlockBuilder()
m = tir.Var("m", "int64")
n = tir.Var("n", "int64")
@@ -144,8 +163,13 @@ def test_softmax_infer_struct_info_shape_symbolic():
_check_inference(bb, relax.op.nn.softmax(x0), relax.TensorStructInfo((m, n), "float32"))
_check_inference(bb, relax.op.nn.softmax(x1, axis=0), relax.TensorStructInfo((4, n), "float32"))
+ _check_inference(bb, relax.op.nn.log_softmax(x0), relax.TensorStructInfo((m, n), "float32"))
+ _check_inference(
+ bb, relax.op.nn.log_softmax(x1, axis=0), relax.TensorStructInfo((4, n), "float32")
+ )
-def test_softmax_infer_struct_info_shape_var():
+
+def test_softmax_log_softmax_infer_struct_info_shape_var():
bb = relax.BlockBuilder()
s0 = relax.Var("s", relax.ShapeStructInfo(ndim=2))
s1 = relax.Var("s", relax.ShapeStructInfo())
@@ -155,8 +179,11 @@ def test_softmax_infer_struct_info_shape_var():
_check_inference(bb, relax.op.nn.softmax(x0), relax.TensorStructInfo(s0, "float32"))
_check_inference(bb, relax.op.nn.softmax(x1), relax.TensorStructInfo(s1, "float32"))
+ _check_inference(bb, relax.op.nn.log_softmax(x0), relax.TensorStructInfo(s0, "float32"))
+ _check_inference(bb, relax.op.nn.log_softmax(x1), relax.TensorStructInfo(s1, "float32"))
+
-def test_softmax_infer_struct_info_more_input_dtype():
+def test_softmax_log_softmax_infer_struct_info_more_input_dtype():
bb = relax.BlockBuilder()
x0 = relax.Var("x", R.Tensor((2, 3), "float16"))
x1 = relax.Var("x", R.Tensor((2, 3), "float64"))
@@ -164,8 +191,11 @@ def test_softmax_infer_struct_info_more_input_dtype():
_check_inference(bb, relax.op.nn.softmax(x0), relax.TensorStructInfo((2, 3), "float16"))
_check_inference(bb, relax.op.nn.softmax(x1), relax.TensorStructInfo((2, 3), "float64"))
+ _check_inference(bb, relax.op.nn.log_softmax(x0), relax.TensorStructInfo((2, 3), "float16"))
+ _check_inference(bb, relax.op.nn.log_softmax(x1), relax.TensorStructInfo((2, 3), "float64"))
+
-def test_softmax_infer_struct_info_invalid_input_dtype():
+def test_softmax_log_softmax_infer_struct_info_invalid_input_dtype():
bb = relax.BlockBuilder()
x0 = relax.Var("x", R.Tensor((2, 3), "int8"))
x1 = relax.Var("x", R.Tensor((2, 3), "int64"))
@@ -174,26 +204,40 @@ def test_softmax_infer_struct_info_invalid_input_dtype():
bb.normalize(relax.op.nn.softmax(x0))
with pytest.raises(TVMError):
bb.normalize(relax.op.nn.softmax(x1))
+ with pytest.raises(TVMError):
+ bb.normalize(relax.op.nn.log_softmax(x0))
+ with pytest.raises(TVMError):
+ bb.normalize(relax.op.nn.log_softmax(x1))
-def test_softmax_infer_struct_info_axis_out_of_range():
+def test_softmax_log_softmax_infer_struct_info_axis_out_of_range():
bb = relax.BlockBuilder()
x = relax.Var("x", R.Tensor((2, 3, 4), "float32"))
+
with pytest.raises(TVMError):
bb.normalize(relax.op.nn.softmax(x, axis=3))
with pytest.raises(TVMError):
bb.normalize(relax.op.nn.softmax(x, axis=-4))
+ with pytest.raises(TVMError):
+ bb.normalize(relax.op.nn.log_softmax(x, axis=3))
+ with pytest.raises(TVMError):
+ bb.normalize(relax.op.nn.log_softmax(x, axis=-4))
-def test_softmax_wrong_with_multiple_axes():
+def test_softmax_log_softmax_wrong_with_multiple_axes():
x = relax.Var("x", R.Tensor((2, 3, 4), "float32"))
+
with pytest.raises(TVMError):
relax.op.nn.softmax(x, axis=[1, 2])
with pytest.raises(TVMError):
relax.op.nn.softmax(x, axis=[-1, -2, -3])
+ with pytest.raises(TVMError):
+ relax.op.nn.log_softmax(x, axis=[1, 2])
+ with pytest.raises(TVMError):
+ relax.op.nn.log_softmax(x, axis=[-1, -2, -3])
-def test_softmax_infer_struct_info_wrong_input_type():
+def test_softmax_log_softmax_infer_struct_info_wrong_input_type():
bb = relax.BlockBuilder()
x0 = relax.Var("x", relax.ShapeStructInfo((2, 3)))
x1 = relax.Var("x", relax.FuncStructInfo([], R.Tensor((2, 3), "float32")))
@@ -202,6 +246,10 @@ def test_softmax_infer_struct_info_wrong_input_type():
bb.normalize(relax.op.nn.softmax(x0))
with pytest.raises(TVMError):
bb.normalize(relax.op.nn.softmax(x1))
+ with pytest.raises(TVMError):
+ bb.normalize(relax.op.nn.log_softmax(x0))
+ with pytest.raises(TVMError):
+ bb.normalize(relax.op.nn.log_softmax(x1))
def test_batch_norm_infer_struct_info():
@@ -925,5 +973,114 @@ def test_dropout_infer_struct_info_wrong_input_type():
bb.normalize(relax.op.nn.dropout(x1))
+def test_cross_entropy_infer_struct_info():
+ bb = relax.BlockBuilder()
+ x = relax.Var("x", R.Tensor((2, 3), "float32"))
+ y0 = relax.Var("y", R.Tensor((2, 3), "float32"))
+ y1 = relax.Var("y", R.Tensor("float32", ndim=2))
+ y2 = relax.Var("y", R.Tensor((2, 3)))
+ y3 = relax.Var("y", R.Tensor(ndim=2))
+
+ _check_inference(
+ bb, relax.op.nn.cross_entropy_with_logits(x, y0), relax.TensorStructInfo((), "float32")
+ )
+ _check_inference(
+ bb,
+ relax.op.nn.cross_entropy_with_logits(x, y1),
+ relax.TensorStructInfo((), dtype="float32"),
+ )
+ _check_inference(
+ bb, relax.op.nn.cross_entropy_with_logits(x, y2), relax.TensorStructInfo((), dtype="")
+ )
+ _check_inference(
+ bb, relax.op.nn.cross_entropy_with_logits(x, y3), relax.TensorStructInfo((), dtype="")
+ )
+
+
+def test_cross_entropy_infer_struct_info_shape_symbolic():
+ bb = relax.BlockBuilder()
+ m0 = tir.Var("m", "int64")
+ m1 = tir.Var("m", "int64")
+ n = tir.Var("n", "int64")
+ x0 = relax.Var("x", R.Tensor((m0, n), "float32"))
+ x1 = relax.Var("x", R.Tensor((m1, n), "float32"))
+ y = relax.Var("y", R.Tensor((m0, n), "float32"))
+
+ _check_inference(
+ bb, relax.op.nn.cross_entropy_with_logits(x0, y), relax.TensorStructInfo((), "float32")
+ )
+ _check_inference(
+ bb, relax.op.nn.cross_entropy_with_logits(x1, y), relax.TensorStructInfo((), "float32")
+ )
+
+
+def test_cross_entropy_infer_struct_info_shape_var():
+ bb = relax.BlockBuilder()
+ s0 = relax.Var("s", relax.ShapeStructInfo(ndim=2))
+ s1 = relax.Var("s", relax.ShapeStructInfo(ndim=2))
+ x = relax.Var("x", relax.TensorStructInfo(s0, "float32"))
+ y0 = relax.Var("x", relax.TensorStructInfo(s0, "float32"))
+ y1 = relax.Var("x", relax.TensorStructInfo(s1, "float32"))
+
+ _check_inference(
+ bb, relax.op.nn.cross_entropy_with_logits(x, y0), relax.TensorStructInfo((), "float32")
+ )
+ _check_inference(
+ bb, relax.op.nn.cross_entropy_with_logits(x, y1), relax.TensorStructInfo((), "float32")
+ )
+
+
+def test_cross_entropy_infer_struct_info_more_input_dtype():
+ bb = relax.BlockBuilder()
+ x0 = relax.Var("x", R.Tensor((2, 3), "float16"))
+ y0 = relax.Var("y", R.Tensor((2, 3), "float16"))
+ x1 = relax.Var("x", R.Tensor((2, 3), "int8"))
+ y1 = relax.Var("y", R.Tensor((2, 3), "int8"))
+ x2 = relax.Var("x", R.Tensor((2, 3), "int32"))
+ y2 = relax.Var("y", R.Tensor((2, 3), "int32"))
+
+ _check_inference(
+ bb, relax.op.nn.cross_entropy_with_logits(x0, y0), relax.TensorStructInfo((), "float16")
+ )
+ _check_inference(
+ bb, relax.op.nn.cross_entropy_with_logits(x1, y1), relax.TensorStructInfo((), "int8")
+ )
+
+
+def test_cross_entropy_infer_struct_info_wrong_ndim():
+ bb = relax.BlockBuilder()
+ x0 = relax.Var("x", R.Tensor((2, 3), "float32"))
+ x1 = relax.Var("x", R.Tensor((2, 3, 4), "float32"))
+ y0 = relax.Var("y", R.Tensor((2, 3), "float32"))
+ y1 = relax.Var("y", R.Tensor("float32", ndim=4))
+
+ with pytest.raises(TVMError):
+ bb.normalize(relax.op.nn.cross_entropy_with_logits(x1, y0))
+ with pytest.raises(TVMError):
+ bb.normalize(relax.op.nn.cross_entropy_with_logits(x0, y1))
+
+
+def test_cross_entropy_infer_struct_info_shape_mismatch():
+ bb = relax.BlockBuilder()
+ m = tir.Var("m", "int64")
+ x0 = relax.Var("x", R.Tensor((2, 3), "float32"))
+ y0 = relax.Var("y", R.Tensor((2, 4), "float32"))
+
+ with pytest.raises(TVMError):
+ bb.normalize(relax.op.nn.cross_entropy_with_logits(x0, y0))
+
+
+def test_cross_entropy_infer_struct_info_wrong_input_type():
+ bb = relax.BlockBuilder()
+ x0 = relax.Var("x", relax.ShapeStructInfo((2, 3)))
+ x1 = relax.Var("x", relax.FuncStructInfo([], R.Tensor((2, 3), "float32")))
+ y = relax.Var("y", R.Tensor((2, 3), "float32"))
+
+ with pytest.raises(TVMError):
+ bb.normalize(relax.op.nn.cross_entropy_with_logits(x0, y))
+ with pytest.raises(TVMError):
+ bb.normalize(relax.op.nn.cross_entropy_with_logits(x1, y))
+
+
if __name__ == "__main__":
tvm.testing.main()
diff --git a/tests/python/relax/test_transform_legalize_ops_nn.py b/tests/python/relax/test_transform_legalize_ops_nn.py
index 729368b82a..07d414980e 100644
--- a/tests/python/relax/test_transform_legalize_ops_nn.py
+++ b/tests/python/relax/test_transform_legalize_ops_nn.py
@@ -851,6 +851,274 @@ def test_softmax_symbolic():
tvm.ir.assert_structural_equal(mod, Expected)
+def test_log_softmax():
+ # fmt: off
+ @tvm.script.ir_module
+ class LogSoftmax:
+ @R.function
+ def main(x: R.Tensor((2, 3, 16, 32), "float32")) -> R.Tensor(None, "float32", ndim=4):
+ gv: R.Tensor((2, 3, 16, 32), "float32") = R.nn.log_softmax(x, axis=-2)
+ return gv
+
+ @tvm.script.ir_module
+ class Expected:
+ @R.function
+ def main(x: R.Tensor((2, 3, 16, 32), dtype="float32")) -> R.Tensor((2, 3, 16, 32), dtype="float32"):
+ gv = R.call_tir(log_softmax, (x,), R.Tensor((2, 3, 16, 32), dtype="float32"))
+ return gv
+
+ @T.prim_func
+ def log_softmax(rxplaceholder: T.Buffer[(T.int64(2), T.int64(3), T.int64(16), T.int64(32)), "float32"], compute: T.Buffer[(T.int64(2), T.int64(3), T.int64(16), T.int64(32)), "float32"],):
+ T.func_attr({"tir.noalias": True})
+ T_softmax_maxelem = T.alloc_buffer([T.int64(2), T.int64(3), T.int64(32)], dtype="float32")
+ compute_1 = T.alloc_buffer([T.int64(2), T.int64(3), T.int64(32)], dtype="float32")
+ for i0, i1, i2, i3 in T.grid(T.int64(2), T.int64(3), T.int64(32), T.int64(16)):
+ with T.block("T_softmax_maxelem"):
+ i0_1, i1_1, i2_1, k = T.axis.remap("SSSR", [i0, i1, i2, i3])
+ T.reads(rxplaceholder[i0_1, i1_1, k, i2_1])
+ T.writes(T_softmax_maxelem[i0_1, i1_1, i2_1])
+ with T.init():
+ T_softmax_maxelem[i0_1, i1_1, i2_1] = T.float32(-3.4028234663852886e38)
+ T_softmax_maxelem[i0_1, i1_1, i2_1] = T.max(T_softmax_maxelem[i0_1, i1_1, i2_1], rxplaceholder[i0_1, i1_1, k, i2_1])
+ for i0, i1, i2, i3 in T.grid(T.int64(2), T.int64(3), T.int64(32), T.int64(16)):
+ with T.block("compute"):
+ i0_2, i1_2, i2_2, k = T.axis.remap("SSSR", [i0, i1, i2, i3])
+ T.reads(rxplaceholder[i0_2, i1_2, k, i2_2], T_softmax_maxelem[i0_2, i1_2, i2_2])
+ T.writes(compute_1[i0_2, i1_2, i2_2])
+ with T.init():
+ compute_1[i0_2, i1_2, i2_2] = T.float32(0)
+ compute_1[i0_2, i1_2, i2_2] = compute_1[i0_2, i1_2, i2_2] + T.exp(rxplaceholder[i0_2, i1_2, k, i2_2] - T_softmax_maxelem[i0_2, i1_2, i2_2], dtype="float32")
+ for i0_3, i1_3, i2_3, i3 in T.grid(T.int64(2), T.int64(3), T.int64(16), T.int64(32)):
+ with T.block("compute_1"):
+ i0_4, i1_4, i2_4, i3_1 = T.axis.remap("SSSS", [i0_3, i1_3, i2_3, i3])
+ T.reads(rxplaceholder[i0_4, i1_4, i2_4, i3_1], T_softmax_maxelem[i0_4, i1_4, i3_1], compute_1[i0_4, i1_4, i3_1])
+ T.writes(compute[i0_4, i1_4, i2_4, i3_1])
+ T.block_attr({"axis": 2})
+ compute[i0_4, i1_4, i2_4, i3_1] = (rxplaceholder[i0_4, i1_4, i2_4, i3_1] - T_softmax_maxelem[i0_4, i1_4, i3_1] - T.log(compute_1[i0_4, i1_4, i3_1], dtype="float32"))
+ # fmt: on
+
+ mod = LegalizeOps()(LogSoftmax)
+ tvm.ir.assert_structural_equal(mod, Expected)
+
+
+def test_log_softmax_symbolic():
+ # fmt: off
+ @tvm.script.ir_module
+ class LogSoftmax:
+ @R.function
+ def main(x: R.Tensor(("a", "b", "c"), "float32")) -> R.Tensor(("a", "b", "c"), "float32"):
+ a = T.var("int64")
+ b = T.var("int64")
+ c = T.var("int64")
+ gv: R.Tensor((a, b, c), "float32") = R.nn.log_softmax(x)
+ return gv
+
+ @tvm.script.ir_module
+ class Expected:
+ @R.function
+ def main(x: R.Tensor(("a", "b", "c"), dtype="float32")) -> R.Tensor(("a", "b", "c"), dtype="float32"):
+ a = T.var("int64")
+ b = T.var("int64")
+ c = T.var("int64")
+ # block 0
+ gv = R.call_tir(log_softmax, (x,), R.Tensor((a, b, c), dtype="float32"))
+ return gv
+
+ @T.prim_func
+ def log_softmax(var_rxplaceholder: T.handle, var_compute: T.handle):
+ T.func_attr({"tir.noalias": True})
+ a = T.var("int64")
+ b = T.var("int64")
+ c = T.var("int64")
+ rxplaceholder = T.match_buffer(var_rxplaceholder, [a, b, c], dtype="float32")
+ compute = T.match_buffer(var_compute, [a, b, c], dtype="float32")
+ T_softmax_maxelem = T.alloc_buffer([a, b], dtype="float32")
+ compute_1 = T.alloc_buffer([a, b], dtype="float32")
+ for i0, i1, k in T.grid(a, b, c):
+ with T.block("T_softmax_maxelem"):
+ v_i0, v_i1, v_k = T.axis.remap("SSR", [i0, i1, k])
+ T.reads(rxplaceholder[v_i0, v_i1, v_k])
+ T.writes(T_softmax_maxelem[v_i0, v_i1])
+ with T.init():
+ T_softmax_maxelem[v_i0, v_i1] = T.float32(-3.4028234663852886e38)
+ T_softmax_maxelem[v_i0, v_i1] = T.max(T_softmax_maxelem[v_i0, v_i1], rxplaceholder[v_i0, v_i1, v_k])
+ for i0, i1, k in T.grid(a, b, c):
+ with T.block("compute"):
+ v_i0, v_i1, v_k = T.axis.remap("SSR", [i0, i1, k])
+ T.reads(rxplaceholder[v_i0, v_i1, v_k], T_softmax_maxelem[v_i0, v_i1])
+ T.writes(compute_1[v_i0, v_i1])
+ with T.init():
+ compute_1[v_i0, v_i1] = T.float32(0)
+ compute_1[v_i0, v_i1] = compute_1[v_i0, v_i1] + T.exp(rxplaceholder[v_i0, v_i1, v_k] - T_softmax_maxelem[v_i0, v_i1], dtype="float32")
+ for i0, i1, i2 in T.grid(a, b, c):
+ with T.block("compute_1"):
+ v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
+ T.reads(rxplaceholder[v_i0, v_i1, v_i2], T_softmax_maxelem[v_i0, v_i1], compute_1[v_i0, v_i1],)
+ T.writes(compute[v_i0, v_i1, v_i2])
+ T.block_attr({"axis": 2})
+ compute[v_i0, v_i1, v_i2] = (rxplaceholder[v_i0, v_i1, v_i2] - T_softmax_maxelem[v_i0, v_i1] - T.log(compute_1[v_i0, v_i1], dtype="float32"))
+ # fmt: on
+
+ mod = LegalizeOps()(LogSoftmax)
+ tvm.ir.assert_structural_equal(mod, Expected)
+
+
+def test_cross_entropy_with_logits():
+ # fmt: off
+ @tvm.script.ir_module
+ class CrossEntropyWithLogits:
+ @R.function
+ def main(x: R.Tensor((3,), "float32"), y: R.Tensor((3,), "float32")) -> R.Tensor(None, "float32", ndim=2):
+ gv: R.Tensor((), "float32") = R.nn.cross_entropy_with_logits(x, y)
+ return gv
+
+ @tvm.script.ir_module
+ class Expected:
+ @R.function
+ def main(x: R.Tensor((3,), dtype="float32"), y: R.Tensor((3,), dtype="float32")) -> R.Tensor(dtype="float32", ndim=2):
+ gv = R.call_tir(cross_entropy_with_logits, (x, y), R.Tensor((), dtype="float32"))
+ return gv
+
+ @T.prim_func
+ def cross_entropy_with_logits(rxplaceholder: T.Buffer[T.int64(3), "float32"], rxplaceholder_1: T.Buffer[T.int64(3), "float32"], T_multiply: T.Buffer[(), "float32"]):
+ T.func_attr({"tir.noalias": True})
+ T_multiply_1 = T.alloc_buffer([T.int64(3)], dtype="float32")
+ T_multiply_red = T.alloc_buffer([], dtype="float32")
+ for i0 in T.serial(T.int64(3)):
+ with T.block("T_multiply"):
+ ax0 = T.axis.spatial(T.int64(3), i0)
+ T.reads(rxplaceholder[ax0], rxplaceholder_1[ax0])
+ T.writes(T_multiply_1[ax0])
+ T_multiply_1[ax0] = rxplaceholder[ax0] * rxplaceholder_1[ax0]
+ for i0 in T.serial(T.int64(3)):
+ with T.block("T_multiply_red"):
+ k0 = T.axis.reduce(T.int64(3), i0)
+ T.reads(T_multiply_1[k0])
+ T.writes(T_multiply_red[()])
+ with T.init():
+ T_multiply_red[()] = T.float32(0)
+ T_multiply_red[()] = T_multiply_red[()] + T_multiply_1[k0]
+ with T.block("T_multiply_1"):
+ vi = T.axis.spatial(1, T.int64(0))
+ T.reads(T_multiply_red[()])
+ T.writes(T_multiply[()])
+ T_multiply[()] = T_multiply_red[()] * T.float32(-1)
+ # fmt: on
+
+ mod = LegalizeOps()(CrossEntropyWithLogits)
+ tvm.ir.assert_structural_equal(mod, Expected)
+
+
+def test_cross_entropy_with_logits_batch():
+ # fmt: off
+ @tvm.script.ir_module
+ class CrossEntropyWithLogits:
+ @R.function
+ def main(x: R.Tensor((2, 3), "float32"), y: R.Tensor((2, 3), "float32")) -> R.Tensor(None, "float32", ndim=2):
+ gv: R.Tensor((), "float32") = R.nn.cross_entropy_with_logits(x, y)
+ return gv
+
+ @tvm.script.ir_module
+ class Expected:
+ @R.function
+ def main(x: R.Tensor((2, 3), dtype="float32"), y: R.Tensor((2, 3), dtype="float32")) -> R.Tensor(dtype="float32", ndim=2):
+ gv = R.call_tir(cross_entropy_with_logits, (x, y), R.Tensor((), dtype="float32"))
+ return gv
+
+ @T.prim_func
+ def cross_entropy_with_logits(rxplaceholder: T.Buffer[(T.int64(2), T.int64(3)), "float32"], rxplaceholder_1: T.Buffer[(T.int64(2), T.int64(3)), "float32"], T_divide: T.Buffer[(), "float32"]):
+ T.func_attr({"tir.noalias": True})
+ T_multiply = T.alloc_buffer([T.int64(2), T.int64(3)], dtype="float32")
+ T_multiply_red = T.alloc_buffer([], dtype="float32")
+ T_multiply_1 = T.alloc_buffer([], dtype="float32")
+ for i0, i1 in T.grid(T.int64(2), T.int64(3)):
+ with T.block("T_multiply"):
+ ax0, ax1 = T.axis.remap("SS", [i0, i1])
+ T.reads(rxplaceholder[ax0, ax1], rxplaceholder_1[ax0, ax1])
+ T.writes(T_multiply[ax0, ax1])
+ T_multiply[ax0, ax1] = rxplaceholder[ax0, ax1] * rxplaceholder_1[ax0, ax1]
+ for i0, i1 in T.grid(T.int64(2), T.int64(3)):
+ with T.block("T_multiply_red"):
+ k0, k1 = T.axis.remap("RR", [i0, i1])
+ T.reads(T_multiply[k0, k1])
+ T.writes(T_multiply_red[()])
+ with T.init():
+ T_multiply_red[()] = T.float32(0)
+ T_multiply_red[()] = T_multiply_red[()] + T_multiply[k0, k1]
+ with T.block("T_multiply_1"):
+ vi = T.axis.spatial(1, T.int64(0))
+ T.reads(T_multiply_red[()])
+ T.writes(T_multiply_1[()])
+ T_multiply_1[()] = T_multiply_red[()] * T.float32(-1)
+ with T.block("T_divide"):
+ vi = T.axis.spatial(1, T.int64(0))
+ T.reads(T_multiply_1[()])
+ T.writes(T_divide[()])
+ T_divide[()] = T_multiply_1[()] * T.float32(0.5)
+ # fmt: on
+
+ mod = LegalizeOps()(CrossEntropyWithLogits)
+ tvm.ir.assert_structural_equal(mod, Expected)
+
+
+def test_cross_entropy_with_logits_batch_symbolic():
+ # fmt: off
+ @tvm.script.ir_module
+ class CrossEntropyWithLogits:
+ @R.function
+ def main(x: R.Tensor(("n", "m"), "float32"), y: R.Tensor(("n", "m"), "float32")) -> R.Tensor(None, "float32", ndim=2):
+ n = T.var("int64")
+ m = T.var("int64")
+ gv: R.Tensor((), "float32") = R.nn.cross_entropy_with_logits(x, y)
+ return gv
+
+ @tvm.script.ir_module
+ class Expected:
+ @R.function
+ def main(x: R.Tensor(("n", "m"), dtype="float32"), y: R.Tensor(("n", "m"), dtype="float32")) -> R.Tensor(dtype="float32", ndim=2):
+ gv = R.call_tir(cross_entropy_with_logits, (x, y), R.Tensor((), dtype="float32"))
+ return gv
+
+ @T.prim_func
+ def cross_entropy_with_logits(var_rxplaceholder: T.handle, var_rxplaceholder_1: T.handle, T_divide: T.Buffer[(), "float32"]):
+ T.func_attr({"tir.noalias": True})
+ m = T.var("int64")
+ n = T.var("int64")
+ rxplaceholder = T.match_buffer(var_rxplaceholder, [n, m], dtype="float32")
+ rxplaceholder_1 = T.match_buffer(var_rxplaceholder_1, [n, m], dtype="float32")
+ T_multiply = T.alloc_buffer([n, m], dtype="float32")
+ T_multiply_red = T.alloc_buffer([], dtype="float32")
+ T_multiply_1 = T.alloc_buffer([], dtype="float32")
+ for ax0, ax1 in T.grid(n, m):
+ with T.block("T_multiply"):
+ v_ax0, v_ax1 = T.axis.remap("SS", [ax0, ax1])
+ T.reads(rxplaceholder[v_ax0, v_ax1], rxplaceholder_1[v_ax0, v_ax1])
+ T.writes(T_multiply[v_ax0, v_ax1])
+ T_multiply[v_ax0, v_ax1] = rxplaceholder[v_ax0, v_ax1] * rxplaceholder_1[v_ax0, v_ax1]
+ for k0, k1 in T.grid(n, m):
+ with T.block("T_multiply_red"):
+ v_k0, v_k1 = T.axis.remap("RR", [k0, k1])
+ T.reads(T_multiply[v_k0, v_k1])
+ T.writes(T_multiply_red[()])
+ with T.init():
+ T_multiply_red[()] = T.float32(0)
+ T_multiply_red[()] = T_multiply_red[()] + T_multiply[v_k0, v_k1]
+ with T.block("T_multiply_1"):
+ vi = T.axis.spatial(1, T.int64(0))
+ T.reads(T_multiply_red[()])
+ T.writes(T_multiply_1[()])
+ T_multiply_1[()] = T_multiply_red[()] * T.float32(-1)
+ with T.block("T_divide"):
+ vi = T.axis.spatial(1, T.int64(0))
+ T.reads(T_multiply_1[()])
+ T.writes(T_divide[()])
+ T_divide[()] = T_multiply_1[()] / T.Cast("float32", n)
+ # fmt: on
+
+ mod = LegalizeOps()(CrossEntropyWithLogits)
+ tvm.ir.assert_structural_equal(mod, Expected)
+
+
def test_batch_norm():
# fmt: off
@tvm.script.ir_module
diff --git a/tests/python/relax/test_tvmscript_parser_op_nn.py b/tests/python/relax/test_tvmscript_parser_op_nn.py
index 4e52bccb86..781700af7b 100644
--- a/tests/python/relax/test_tvmscript_parser_op_nn.py
+++ b/tests/python/relax/test_tvmscript_parser_op_nn.py
@@ -115,6 +115,21 @@ def test_softmax():
_check(foo, bb.get()["foo"])
+def test_log_softmax():
+ @R.function
+ def foo(x: R.Tensor((2, 3), "float32")) -> R.Tensor((2, 3), "float32"):
+ gv: R.Tensor((2, 3), "float32") = R.nn.log_softmax(x)
+ return gv
+
+ x = relax.Var("x", R.Tensor((2, 3), "float32"))
+ bb = relax.BlockBuilder()
+ with bb.function("foo", [x]):
+ gv = bb.emit(relax.op.nn.log_softmax(x))
+ bb.emit_func_output(gv)
+
+ _check(foo, bb.get()["foo"])
+
+
def test_batch_norm():
@R.function
def foo(
@@ -189,5 +204,23 @@ def test_dropout():
_check(foo, bb.get()["foo"])
+def test_cross_entropy_with_logits():
+ @R.function
+ def foo(
+ predictions: R.Tensor((2, 3), "float32"), labels: R.Tensor((2, 3), "float32")
+ ) -> R.Tensor((), "float32"):
+ gv: R.Tensor((), "float32") = R.nn.cross_entropy_with_logits(predictions, labels)
+ return gv
+
+ predictions = relax.Var("predictions", R.Tensor((2, 3), "float32"))
+ labels = relax.Var("labels", R.Tensor((2, 3), "float32"))
+ bb = relax.BlockBuilder()
+ with bb.function("foo", [predictions, labels]):
+ gv = bb.emit(relax.op.nn.cross_entropy_with_logits(predictions, labels))
+ bb.emit_func_output(gv)
+
+ _check(foo, bb.get()["foo"])
+
+
if __name__ == "__main__":
tvm.testing.main()