You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2023/02/22 12:56:32 UTC

[tvm] branch unity updated: [Unity][Op] `log_softmax` and `cross_entropy_with_logits` (#14083)

This is an automated email from the ASF dual-hosted git repository.

tqchen pushed a commit to branch unity
in repository https://gitbox.apache.org/repos/asf/tvm.git


The following commit(s) were added to refs/heads/unity by this push:
     new 4f9cd5a0fd [Unity][Op] `log_softmax` and `cross_entropy_with_logits` (#14083)
4f9cd5a0fd is described below

commit 4f9cd5a0fd96dcfa53dbc36df039efc529f7d60f
Author: Chaosfan <17...@qq.com>
AuthorDate: Wed Feb 22 20:56:24 2023 +0800

    [Unity][Op] `log_softmax` and `cross_entropy_with_logits` (#14083)
    
    This PR introduces two high-level operators log_softmax and cross_entropy_with_logits, which are important when we are calculating CrossEntropyLoss (in torch).
    
    Co-authored-by: Yixin Dong <ub...@gmail.com>
---
 python/tvm/relax/op/nn/nn.py                       |  55 +++++
 python/tvm/relax/transform/legalize_ops/nn.py      |  20 ++
 src/relax/op/nn/nn.cc                              |  74 ++++++
 src/relax/op/nn/nn.h                               |   6 +
 tests/python/relax/test_op_nn.py                   | 173 ++++++++++++-
 .../python/relax/test_transform_legalize_ops_nn.py | 268 +++++++++++++++++++++
 tests/python/relax/test_tvmscript_parser_op_nn.py  |  33 +++
 7 files changed, 621 insertions(+), 8 deletions(-)

diff --git a/python/tvm/relax/op/nn/nn.py b/python/tvm/relax/op/nn/nn.py
index cdf0e96464..0ff143fd04 100644
--- a/python/tvm/relax/op/nn/nn.py
+++ b/python/tvm/relax/op/nn/nn.py
@@ -348,6 +348,34 @@ def softmax(data: Expr, axis: int = -1) -> Expr:
     return _ffi_api.softmax(data, axis)  # type: ignore
 
 
+def log_softmax(data: Expr, axis: int = -1) -> Expr:
+    r"""Computes log softmax.
+
+    .. math::
+
+        \text{log\_softmax}(x_i) = \log\left( \frac{\exp(x_i)}{\sum_j \exp(x_j)}\right)
+
+    .. note::
+        This operator can be optimized away for inference.
+
+    Parameters
+    ----------
+    data: relax.Expr
+        The input data to the operator.
+
+    axis: int
+        The axis to sum over when computing log softmax.
+        If not specified, it is by default the last axis of the input tensor.
+        Supports negative indexing.
+
+    Returns
+    -------
+    result : relax.Expr
+        The computed result.
+    """
+    return _ffi_api.log_softmax(data, axis)  # type: ignore
+
+
 def batch_norm(
     data: Expr,
     gamma: Expr,
@@ -522,3 +550,30 @@ def dropout(data: Expr, rate: float = 0.5) -> Expr:
         mask tensor (1.0 where element not dropped, 0.0 where dropped)
     """
     return _ffi_api.dropout(data, rate)  # type: ignore
+
+
+def cross_entropy_with_logits(predictions: Expr, labels: Expr) -> Expr:
+    r"""CrossEntropy with logits between the predictions and labels.
+
+    The shape of predictions and labels must be the same. And when ndim >= 2,
+    the first dimension is regarded as the batch_size N. In this case the
+    computed result will divide by N to perform a mean reduction.
+
+    .. math::
+
+        \text{cross\_entropy\_with\_logits}(x_i, y_i) = \frac{\sum_i -x_i \cdot y_i}{N}
+
+    Parameters
+    ----------
+    predictions : relax.Expr
+      The predictions.
+
+    labels : relax.Expr
+      The labels (the ground truth values).
+
+    Returns
+    -------
+    result : relax.Expr
+      The computed result.
+    """
+    return _ffi_api.cross_entropy_with_logits(predictions, labels)  # type: ignore
diff --git a/python/tvm/relax/transform/legalize_ops/nn.py b/python/tvm/relax/transform/legalize_ops/nn.py
index 49f198306d..31c6eb04cd 100644
--- a/python/tvm/relax/transform/legalize_ops/nn.py
+++ b/python/tvm/relax/transform/legalize_ops/nn.py
@@ -144,6 +144,26 @@ def _nn_softmax(bb: BlockBuilder, call: Call) -> Expr:
     return bb.call_te(topi.nn.softmax, call.args[0], call.attrs.axis)
 
 
+@register_legalize("relax.nn.log_softmax")
+def _nn_log_softmax(bb: BlockBuilder, call: Call):
+    return bb.call_te(topi.nn.log_softmax, call.args[0], call.attrs.axis)
+
+
+@register_legalize("relax.nn.cross_entropy_with_logits")
+def _nn_cross_entropy_with_logits(bb: BlockBuilder, call: Call):
+    def te_cross_entropy_with_logits(x, y):
+        if len(x.shape) > 1:
+            return -topi.sum(x * y) / x.shape[0]
+        return -topi.sum(x * y)
+
+    return bb.call_te(
+        te_cross_entropy_with_logits,
+        call.args[0],
+        call.args[1],
+        primfunc_name_hint="cross_entropy_with_logits",
+    )
+
+
 @register_legalize("relax.nn.batch_norm")
 def _nn_batch_norm(bb: BlockBuilder, call: Call) -> Expr:
     return bb.call_te(
diff --git a/src/relax/op/nn/nn.cc b/src/relax/op/nn/nn.cc
index 66ae10fe6c..e63b3306f2 100644
--- a/src/relax/op/nn/nn.cc
+++ b/src/relax/op/nn/nn.cc
@@ -68,6 +68,22 @@ TVM_REGISTER_OP("relax.nn.softmax")
     .set_attrs_type<SoftmaxAttrs>()
     .set_attr<FInferStructInfo>("FInferStructInfo", InferStructInfoSoftmax);
 
+/* relax.nn.log_softmax */
+Expr log_softmax(Expr data, int axis) {
+  auto attrs = make_object<SoftmaxAttrs>();
+  attrs->axis = axis;
+  static const Op& op = Op::Get("relax.nn.log_softmax");
+  return Call(op, {data}, Attrs(attrs), {});
+}
+
+TVM_REGISTER_GLOBAL("relax.op.nn.log_softmax").set_body_typed(log_softmax);
+
+TVM_REGISTER_OP("relax.nn.log_softmax")
+    .set_num_inputs(1)
+    .add_argument("data", "Tensor", "The input tensor.")
+    .set_attrs_type<SoftmaxAttrs>()
+    .set_attr<FInferStructInfo>("FInferStructInfo", InferStructInfoSoftmax);
+
 bool NormCheckDtypeAndShape(const Call& call, const BlockBuilder& ctx,
                             const Array<TensorStructInfo>& input_sinfo, Array<Integer> axes) {
   Op op = Downcast<Op>(call->op);
@@ -241,5 +257,63 @@ TVM_REGISTER_OP("relax.nn.dropout")
     .add_argument("data", "Tensor", "Input to which dropout will be applied.")
     .set_attr<FInferStructInfo>("FInferStructInfo", InferStructInfoDropout);
 
+/* relax.nn.cross_entropy_with_logits */
+StructInfo InferStructInfoCrossEntropy(const Call& call, const BlockBuilder& ctx) {
+  Array<TensorStructInfo> input_sinfo = GetInputTensorStructInfo(call, ctx);
+  TensorStructInfo pred_sinfo = input_sinfo[0];
+  TensorStructInfo label_sinfo = input_sinfo[1];
+
+  // infer dtype
+  DataType dtype = InferBinaryArithOpOutDtype(call, ctx, pred_sinfo, label_sinfo);
+
+  // infer ndim
+  if (!pred_sinfo->IsUnknownNdim() && !label_sinfo->IsUnknownNdim() &&
+      pred_sinfo->ndim != label_sinfo->ndim) {
+    ctx->ReportFatal(Diagnostic::Error(call)
+                     << "CrossEntropy requires predictions and labels to have the same ndim. "
+                        "However, the ndim of predictions is "
+                     << pred_sinfo->ndim << " while the ndim of labels is " << label_sinfo->ndim);
+  }
+
+  Optional<Array<PrimExpr>> pred_shape_value;
+  if (pred_sinfo->shape.defined()) {
+    pred_shape_value = GetStructInfoAs<ShapeStructInfoNode>(pred_sinfo->shape.value())->values;
+  }
+
+  Optional<Array<PrimExpr>> label_shape_value;
+  if (label_sinfo->shape.defined()) {
+    label_shape_value = GetStructInfoAs<ShapeStructInfoNode>(label_sinfo->shape.value())->values;
+  }
+
+  if (pred_shape_value.defined() && label_shape_value.defined()) {
+    arith::Analyzer* analyzer = ctx->GetAnalyzer();
+    for (size_t i = 0; i < pred_shape_value.value().size(); ++i) {
+      if (analyzer->CanProve(pred_shape_value.value()[i] != label_shape_value.value()[i])) {
+        ctx->ReportFatal(Diagnostic::Error(call)
+                         << "CrossEntropy requires the predictions and labels to have "
+                            "the same shape. However, the shape of predictions at dim "
+                         << i << " is" << pred_shape_value.value()[i]
+                         << " while the shape of labels at this dim is "
+                         << label_shape_value.value()[i]);
+      }
+    }
+  }
+  return TensorStructInfo(ShapeExpr(Array<PrimExpr>()), dtype);
+}
+
+Expr cross_entropy_with_logits(Expr predictions, Expr labels) {
+  static const Op& op = Op::Get("relax.nn.cross_entropy_with_logits");
+  return Call(op, {std::move(predictions), std::move(labels)}, {}, {});
+}
+
+TVM_REGISTER_GLOBAL("relax.op.nn.cross_entropy_with_logits")
+    .set_body_typed(cross_entropy_with_logits);
+
+TVM_REGISTER_OP("relax.nn.cross_entropy_with_logits")
+    .set_num_inputs(2)
+    .add_argument("predictions", "Tensor", "The predictions.")
+    .add_argument("labels", "Tensor", "The labels.")
+    .set_attr<FInferStructInfo>("FInferStructInfo", InferStructInfoCrossEntropy);
+
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/nn/nn.h b/src/relax/op/nn/nn.h
index df2b978fc2..f13b930fc2 100644
--- a/src/relax/op/nn/nn.h
+++ b/src/relax/op/nn/nn.h
@@ -57,6 +57,9 @@ Expr silu(Expr data);
 /*! \brief Softmax function. */
 Expr softmax(Expr data, int axis);
 
+/*! \brief LogSoftmax function. */
+Expr log_softmax(Expr data, int axis);
+
 /*! \brief Compute batch normalization. */
 Expr batch_norm(Expr data, Expr gamma, Expr beta, Expr moving_mean, Expr moving_var,  //
                 int axis, double epsilon, bool center, bool scale);
@@ -75,6 +78,9 @@ Expr layer_norm(Expr data, Expr gamma, Expr beta, Array<Integer> axes, double ep
  */
 Expr dropout(Expr data, double rate);
 
+/*! \brief CrossEntropy with logits. */
+Expr cross_entropy_with_logits(Expr predictions, Expr labels);
+
 }  // namespace relax
 }  // namespace tvm
 
diff --git a/tests/python/relax/test_op_nn.py b/tests/python/relax/test_op_nn.py
index d047448309..5294596cee 100644
--- a/tests/python/relax/test_op_nn.py
+++ b/tests/python/relax/test_op_nn.py
@@ -29,6 +29,7 @@ def test_op_correctness():
     assert relax.op.nn.gelu(x).op == Op.get("relax.nn.gelu")
     assert relax.op.nn.silu(x).op == Op.get("relax.nn.silu")
     assert relax.op.nn.softmax(x).op == Op.get("relax.nn.softmax")
+    assert relax.op.nn.log_softmax(x).op == Op.get("relax.nn.log_softmax")
     assert relax.op.nn.dropout(x).op == Op.get("relax.nn.dropout")
 
     x = relax.Var("x", R.Tensor((2, 3, 32, 32), "float32"))
@@ -41,6 +42,12 @@ def test_op_correctness():
     )
     assert relax.op.nn.layer_norm(x, gamma, beta, axes=1).op == Op.get("relax.nn.layer_norm")
 
+    x = relax.Var("x", R.Tensor((2, 3), "float32"))
+    y = relax.Var("y", R.Tensor((2, 3), "float32"))
+    assert relax.op.nn.cross_entropy_with_logits(x, y).op == Op.get(
+        "relax.nn.cross_entropy_with_logits"
+    )
+
 
 def _check_inference(bb: relax.BlockBuilder, call: relax.Call, expected_sinfo: relax.StructInfo):
     ret = bb.normalize(call)
@@ -117,7 +124,7 @@ def test_linear_unit_infer_struct_info_wrong_input_type():
         bb.normalize(relax.op.nn.silu(x1))
 
 
-def test_softmax_infer_struct_info():
+def test_softmax_log_softmax_infer_struct_info():
     bb = relax.BlockBuilder()
     x0 = relax.Var("x", R.Tensor((2, 3), "float32"))
     x1 = relax.Var("x", R.Tensor("float32", ndim=3))
@@ -133,8 +140,20 @@ def test_softmax_infer_struct_info():
     _check_inference(bb, relax.op.nn.softmax(x3, axis=-1), relax.TensorStructInfo((2, 3), dtype=""))
     _check_inference(bb, relax.op.nn.softmax(x4, axis=-2), relax.TensorStructInfo(dtype=""))
 
+    _check_inference(bb, relax.op.nn.log_softmax(x0), relax.TensorStructInfo((2, 3), "float32"))
+    _check_inference(
+        bb, relax.op.nn.log_softmax(x1, axis=0), relax.TensorStructInfo(dtype="float32", ndim=3)
+    )
+    _check_inference(
+        bb, relax.op.nn.log_softmax(x2, axis=1), relax.TensorStructInfo(dtype="float32")
+    )
+    _check_inference(
+        bb, relax.op.nn.log_softmax(x3, axis=-1), relax.TensorStructInfo((2, 3), dtype="")
+    )
+    _check_inference(bb, relax.op.nn.log_softmax(x4, axis=-2), relax.TensorStructInfo(dtype=""))
+
 
-def test_softmax_infer_struct_info_shape_symbolic():
+def test_softmax_log_softmax_infer_struct_info_shape_symbolic():
     bb = relax.BlockBuilder()
     m = tir.Var("m", "int64")
     n = tir.Var("n", "int64")
@@ -144,8 +163,13 @@ def test_softmax_infer_struct_info_shape_symbolic():
     _check_inference(bb, relax.op.nn.softmax(x0), relax.TensorStructInfo((m, n), "float32"))
     _check_inference(bb, relax.op.nn.softmax(x1, axis=0), relax.TensorStructInfo((4, n), "float32"))
 
+    _check_inference(bb, relax.op.nn.log_softmax(x0), relax.TensorStructInfo((m, n), "float32"))
+    _check_inference(
+        bb, relax.op.nn.log_softmax(x1, axis=0), relax.TensorStructInfo((4, n), "float32")
+    )
 
-def test_softmax_infer_struct_info_shape_var():
+
+def test_softmax_log_softmax_infer_struct_info_shape_var():
     bb = relax.BlockBuilder()
     s0 = relax.Var("s", relax.ShapeStructInfo(ndim=2))
     s1 = relax.Var("s", relax.ShapeStructInfo())
@@ -155,8 +179,11 @@ def test_softmax_infer_struct_info_shape_var():
     _check_inference(bb, relax.op.nn.softmax(x0), relax.TensorStructInfo(s0, "float32"))
     _check_inference(bb, relax.op.nn.softmax(x1), relax.TensorStructInfo(s1, "float32"))
 
+    _check_inference(bb, relax.op.nn.log_softmax(x0), relax.TensorStructInfo(s0, "float32"))
+    _check_inference(bb, relax.op.nn.log_softmax(x1), relax.TensorStructInfo(s1, "float32"))
+
 
-def test_softmax_infer_struct_info_more_input_dtype():
+def test_softmax_log_softmax_infer_struct_info_more_input_dtype():
     bb = relax.BlockBuilder()
     x0 = relax.Var("x", R.Tensor((2, 3), "float16"))
     x1 = relax.Var("x", R.Tensor((2, 3), "float64"))
@@ -164,8 +191,11 @@ def test_softmax_infer_struct_info_more_input_dtype():
     _check_inference(bb, relax.op.nn.softmax(x0), relax.TensorStructInfo((2, 3), "float16"))
     _check_inference(bb, relax.op.nn.softmax(x1), relax.TensorStructInfo((2, 3), "float64"))
 
+    _check_inference(bb, relax.op.nn.log_softmax(x0), relax.TensorStructInfo((2, 3), "float16"))
+    _check_inference(bb, relax.op.nn.log_softmax(x1), relax.TensorStructInfo((2, 3), "float64"))
+
 
-def test_softmax_infer_struct_info_invalid_input_dtype():
+def test_softmax_log_softmax_infer_struct_info_invalid_input_dtype():
     bb = relax.BlockBuilder()
     x0 = relax.Var("x", R.Tensor((2, 3), "int8"))
     x1 = relax.Var("x", R.Tensor((2, 3), "int64"))
@@ -174,26 +204,40 @@ def test_softmax_infer_struct_info_invalid_input_dtype():
         bb.normalize(relax.op.nn.softmax(x0))
     with pytest.raises(TVMError):
         bb.normalize(relax.op.nn.softmax(x1))
+    with pytest.raises(TVMError):
+        bb.normalize(relax.op.nn.log_softmax(x0))
+    with pytest.raises(TVMError):
+        bb.normalize(relax.op.nn.log_softmax(x1))
 
 
-def test_softmax_infer_struct_info_axis_out_of_range():
+def test_softmax_log_softmax_infer_struct_info_axis_out_of_range():
     bb = relax.BlockBuilder()
     x = relax.Var("x", R.Tensor((2, 3, 4), "float32"))
+
     with pytest.raises(TVMError):
         bb.normalize(relax.op.nn.softmax(x, axis=3))
     with pytest.raises(TVMError):
         bb.normalize(relax.op.nn.softmax(x, axis=-4))
+    with pytest.raises(TVMError):
+        bb.normalize(relax.op.nn.log_softmax(x, axis=3))
+    with pytest.raises(TVMError):
+        bb.normalize(relax.op.nn.log_softmax(x, axis=-4))
 
 
-def test_softmax_wrong_with_multiple_axes():
+def test_softmax_log_softmax_wrong_with_multiple_axes():
     x = relax.Var("x", R.Tensor((2, 3, 4), "float32"))
+
     with pytest.raises(TVMError):
         relax.op.nn.softmax(x, axis=[1, 2])
     with pytest.raises(TVMError):
         relax.op.nn.softmax(x, axis=[-1, -2, -3])
+    with pytest.raises(TVMError):
+        relax.op.nn.log_softmax(x, axis=[1, 2])
+    with pytest.raises(TVMError):
+        relax.op.nn.log_softmax(x, axis=[-1, -2, -3])
 
 
-def test_softmax_infer_struct_info_wrong_input_type():
+def test_softmax_log_softmax_infer_struct_info_wrong_input_type():
     bb = relax.BlockBuilder()
     x0 = relax.Var("x", relax.ShapeStructInfo((2, 3)))
     x1 = relax.Var("x", relax.FuncStructInfo([], R.Tensor((2, 3), "float32")))
@@ -202,6 +246,10 @@ def test_softmax_infer_struct_info_wrong_input_type():
         bb.normalize(relax.op.nn.softmax(x0))
     with pytest.raises(TVMError):
         bb.normalize(relax.op.nn.softmax(x1))
+    with pytest.raises(TVMError):
+        bb.normalize(relax.op.nn.log_softmax(x0))
+    with pytest.raises(TVMError):
+        bb.normalize(relax.op.nn.log_softmax(x1))
 
 
 def test_batch_norm_infer_struct_info():
@@ -925,5 +973,114 @@ def test_dropout_infer_struct_info_wrong_input_type():
         bb.normalize(relax.op.nn.dropout(x1))
 
 
+def test_cross_entropy_infer_struct_info():
+    bb = relax.BlockBuilder()
+    x = relax.Var("x", R.Tensor((2, 3), "float32"))
+    y0 = relax.Var("y", R.Tensor((2, 3), "float32"))
+    y1 = relax.Var("y", R.Tensor("float32", ndim=2))
+    y2 = relax.Var("y", R.Tensor((2, 3)))
+    y3 = relax.Var("y", R.Tensor(ndim=2))
+
+    _check_inference(
+        bb, relax.op.nn.cross_entropy_with_logits(x, y0), relax.TensorStructInfo((), "float32")
+    )
+    _check_inference(
+        bb,
+        relax.op.nn.cross_entropy_with_logits(x, y1),
+        relax.TensorStructInfo((), dtype="float32"),
+    )
+    _check_inference(
+        bb, relax.op.nn.cross_entropy_with_logits(x, y2), relax.TensorStructInfo((), dtype="")
+    )
+    _check_inference(
+        bb, relax.op.nn.cross_entropy_with_logits(x, y3), relax.TensorStructInfo((), dtype="")
+    )
+
+
+def test_cross_entropy_infer_struct_info_shape_symbolic():
+    bb = relax.BlockBuilder()
+    m0 = tir.Var("m", "int64")
+    m1 = tir.Var("m", "int64")
+    n = tir.Var("n", "int64")
+    x0 = relax.Var("x", R.Tensor((m0, n), "float32"))
+    x1 = relax.Var("x", R.Tensor((m1, n), "float32"))
+    y = relax.Var("y", R.Tensor((m0, n), "float32"))
+
+    _check_inference(
+        bb, relax.op.nn.cross_entropy_with_logits(x0, y), relax.TensorStructInfo((), "float32")
+    )
+    _check_inference(
+        bb, relax.op.nn.cross_entropy_with_logits(x1, y), relax.TensorStructInfo((), "float32")
+    )
+
+
+def test_cross_entropy_infer_struct_info_shape_var():
+    bb = relax.BlockBuilder()
+    s0 = relax.Var("s", relax.ShapeStructInfo(ndim=2))
+    s1 = relax.Var("s", relax.ShapeStructInfo(ndim=2))
+    x = relax.Var("x", relax.TensorStructInfo(s0, "float32"))
+    y0 = relax.Var("x", relax.TensorStructInfo(s0, "float32"))
+    y1 = relax.Var("x", relax.TensorStructInfo(s1, "float32"))
+
+    _check_inference(
+        bb, relax.op.nn.cross_entropy_with_logits(x, y0), relax.TensorStructInfo((), "float32")
+    )
+    _check_inference(
+        bb, relax.op.nn.cross_entropy_with_logits(x, y1), relax.TensorStructInfo((), "float32")
+    )
+
+
+def test_cross_entropy_infer_struct_info_more_input_dtype():
+    bb = relax.BlockBuilder()
+    x0 = relax.Var("x", R.Tensor((2, 3), "float16"))
+    y0 = relax.Var("y", R.Tensor((2, 3), "float16"))
+    x1 = relax.Var("x", R.Tensor((2, 3), "int8"))
+    y1 = relax.Var("y", R.Tensor((2, 3), "int8"))
+    x2 = relax.Var("x", R.Tensor((2, 3), "int32"))
+    y2 = relax.Var("y", R.Tensor((2, 3), "int32"))
+
+    _check_inference(
+        bb, relax.op.nn.cross_entropy_with_logits(x0, y0), relax.TensorStructInfo((), "float16")
+    )
+    _check_inference(
+        bb, relax.op.nn.cross_entropy_with_logits(x1, y1), relax.TensorStructInfo((), "int8")
+    )
+
+
+def test_cross_entropy_infer_struct_info_wrong_ndim():
+    bb = relax.BlockBuilder()
+    x0 = relax.Var("x", R.Tensor((2, 3), "float32"))
+    x1 = relax.Var("x", R.Tensor((2, 3, 4), "float32"))
+    y0 = relax.Var("y", R.Tensor((2, 3), "float32"))
+    y1 = relax.Var("y", R.Tensor("float32", ndim=4))
+
+    with pytest.raises(TVMError):
+        bb.normalize(relax.op.nn.cross_entropy_with_logits(x1, y0))
+    with pytest.raises(TVMError):
+        bb.normalize(relax.op.nn.cross_entropy_with_logits(x0, y1))
+
+
+def test_cross_entropy_infer_struct_info_shape_mismatch():
+    bb = relax.BlockBuilder()
+    m = tir.Var("m", "int64")
+    x0 = relax.Var("x", R.Tensor((2, 3), "float32"))
+    y0 = relax.Var("y", R.Tensor((2, 4), "float32"))
+
+    with pytest.raises(TVMError):
+        bb.normalize(relax.op.nn.cross_entropy_with_logits(x0, y0))
+
+
+def test_cross_entropy_infer_struct_info_wrong_input_type():
+    bb = relax.BlockBuilder()
+    x0 = relax.Var("x", relax.ShapeStructInfo((2, 3)))
+    x1 = relax.Var("x", relax.FuncStructInfo([], R.Tensor((2, 3), "float32")))
+    y = relax.Var("y", R.Tensor((2, 3), "float32"))
+
+    with pytest.raises(TVMError):
+        bb.normalize(relax.op.nn.cross_entropy_with_logits(x0, y))
+    with pytest.raises(TVMError):
+        bb.normalize(relax.op.nn.cross_entropy_with_logits(x1, y))
+
+
 if __name__ == "__main__":
     tvm.testing.main()
diff --git a/tests/python/relax/test_transform_legalize_ops_nn.py b/tests/python/relax/test_transform_legalize_ops_nn.py
index 729368b82a..07d414980e 100644
--- a/tests/python/relax/test_transform_legalize_ops_nn.py
+++ b/tests/python/relax/test_transform_legalize_ops_nn.py
@@ -851,6 +851,274 @@ def test_softmax_symbolic():
     tvm.ir.assert_structural_equal(mod, Expected)
 
 
+def test_log_softmax():
+    # fmt: off
+    @tvm.script.ir_module
+    class LogSoftmax:
+        @R.function
+        def main(x: R.Tensor((2, 3, 16, 32), "float32")) -> R.Tensor(None, "float32", ndim=4):
+            gv: R.Tensor((2, 3, 16, 32), "float32") = R.nn.log_softmax(x, axis=-2)
+            return gv
+
+    @tvm.script.ir_module
+    class Expected:
+        @R.function
+        def main(x: R.Tensor((2, 3, 16, 32), dtype="float32")) -> R.Tensor((2, 3, 16, 32), dtype="float32"):
+            gv = R.call_tir(log_softmax, (x,), R.Tensor((2, 3, 16, 32), dtype="float32"))
+            return gv
+
+        @T.prim_func
+        def log_softmax(rxplaceholder: T.Buffer[(T.int64(2), T.int64(3), T.int64(16), T.int64(32)), "float32"], compute: T.Buffer[(T.int64(2), T.int64(3), T.int64(16), T.int64(32)), "float32"],):
+            T.func_attr({"tir.noalias": True})
+            T_softmax_maxelem = T.alloc_buffer([T.int64(2), T.int64(3), T.int64(32)], dtype="float32")
+            compute_1 = T.alloc_buffer([T.int64(2), T.int64(3), T.int64(32)], dtype="float32")
+            for i0, i1, i2, i3 in T.grid(T.int64(2), T.int64(3), T.int64(32), T.int64(16)):
+                with T.block("T_softmax_maxelem"):
+                    i0_1, i1_1, i2_1, k = T.axis.remap("SSSR", [i0, i1, i2, i3])
+                    T.reads(rxplaceholder[i0_1, i1_1, k, i2_1])
+                    T.writes(T_softmax_maxelem[i0_1, i1_1, i2_1])
+                    with T.init():
+                        T_softmax_maxelem[i0_1, i1_1, i2_1] = T.float32(-3.4028234663852886e38)
+                    T_softmax_maxelem[i0_1, i1_1, i2_1] = T.max(T_softmax_maxelem[i0_1, i1_1, i2_1], rxplaceholder[i0_1, i1_1, k, i2_1])
+            for i0, i1, i2, i3 in T.grid(T.int64(2), T.int64(3), T.int64(32), T.int64(16)):
+                with T.block("compute"):
+                    i0_2, i1_2, i2_2, k = T.axis.remap("SSSR", [i0, i1, i2, i3])
+                    T.reads(rxplaceholder[i0_2, i1_2, k, i2_2], T_softmax_maxelem[i0_2, i1_2, i2_2])
+                    T.writes(compute_1[i0_2, i1_2, i2_2])
+                    with T.init():
+                        compute_1[i0_2, i1_2, i2_2] = T.float32(0)
+                    compute_1[i0_2, i1_2, i2_2] = compute_1[i0_2, i1_2, i2_2] + T.exp(rxplaceholder[i0_2, i1_2, k, i2_2] - T_softmax_maxelem[i0_2, i1_2, i2_2], dtype="float32")
+            for i0_3, i1_3, i2_3, i3 in T.grid(T.int64(2), T.int64(3), T.int64(16), T.int64(32)):
+                with T.block("compute_1"):
+                    i0_4, i1_4, i2_4, i3_1 = T.axis.remap("SSSS", [i0_3, i1_3, i2_3, i3])
+                    T.reads(rxplaceholder[i0_4, i1_4, i2_4, i3_1], T_softmax_maxelem[i0_4, i1_4, i3_1], compute_1[i0_4, i1_4, i3_1])
+                    T.writes(compute[i0_4, i1_4, i2_4, i3_1])
+                    T.block_attr({"axis": 2})
+                    compute[i0_4, i1_4, i2_4, i3_1] = (rxplaceholder[i0_4, i1_4, i2_4, i3_1] - T_softmax_maxelem[i0_4, i1_4, i3_1] - T.log(compute_1[i0_4, i1_4, i3_1], dtype="float32"))
+    # fmt: on
+
+    mod = LegalizeOps()(LogSoftmax)
+    tvm.ir.assert_structural_equal(mod, Expected)
+
+
+def test_log_softmax_symbolic():
+    # fmt: off
+    @tvm.script.ir_module
+    class LogSoftmax:
+        @R.function
+        def main(x: R.Tensor(("a", "b", "c"), "float32")) -> R.Tensor(("a", "b", "c"), "float32"):
+            a = T.var("int64")
+            b = T.var("int64")
+            c = T.var("int64")
+            gv: R.Tensor((a, b, c), "float32") = R.nn.log_softmax(x)
+            return gv
+
+    @tvm.script.ir_module
+    class Expected:
+        @R.function
+        def main(x: R.Tensor(("a", "b", "c"), dtype="float32")) -> R.Tensor(("a", "b", "c"), dtype="float32"):
+            a = T.var("int64")
+            b = T.var("int64")
+            c = T.var("int64")
+            # block 0
+            gv = R.call_tir(log_softmax, (x,), R.Tensor((a, b, c), dtype="float32"))
+            return gv
+
+        @T.prim_func
+        def log_softmax(var_rxplaceholder: T.handle, var_compute: T.handle):
+            T.func_attr({"tir.noalias": True})
+            a = T.var("int64")
+            b = T.var("int64")
+            c = T.var("int64")
+            rxplaceholder = T.match_buffer(var_rxplaceholder, [a, b, c], dtype="float32")
+            compute = T.match_buffer(var_compute, [a, b, c], dtype="float32")
+            T_softmax_maxelem = T.alloc_buffer([a, b], dtype="float32")
+            compute_1 = T.alloc_buffer([a, b], dtype="float32")
+            for i0, i1, k in T.grid(a, b, c):
+                with T.block("T_softmax_maxelem"):
+                    v_i0, v_i1, v_k = T.axis.remap("SSR", [i0, i1, k])
+                    T.reads(rxplaceholder[v_i0, v_i1, v_k])
+                    T.writes(T_softmax_maxelem[v_i0, v_i1])
+                    with T.init():
+                        T_softmax_maxelem[v_i0, v_i1] = T.float32(-3.4028234663852886e38)
+                    T_softmax_maxelem[v_i0, v_i1] = T.max(T_softmax_maxelem[v_i0, v_i1], rxplaceholder[v_i0, v_i1, v_k])
+            for i0, i1, k in T.grid(a, b, c):
+                with T.block("compute"):
+                    v_i0, v_i1, v_k = T.axis.remap("SSR", [i0, i1, k])
+                    T.reads(rxplaceholder[v_i0, v_i1, v_k], T_softmax_maxelem[v_i0, v_i1])
+                    T.writes(compute_1[v_i0, v_i1])
+                    with T.init():
+                        compute_1[v_i0, v_i1] = T.float32(0)
+                    compute_1[v_i0, v_i1] = compute_1[v_i0, v_i1] + T.exp(rxplaceholder[v_i0, v_i1, v_k] - T_softmax_maxelem[v_i0, v_i1], dtype="float32")
+            for i0, i1, i2 in T.grid(a, b, c):
+                with T.block("compute_1"):
+                    v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
+                    T.reads(rxplaceholder[v_i0, v_i1, v_i2], T_softmax_maxelem[v_i0, v_i1], compute_1[v_i0, v_i1],)
+                    T.writes(compute[v_i0, v_i1, v_i2])
+                    T.block_attr({"axis": 2})
+                    compute[v_i0, v_i1, v_i2] = (rxplaceholder[v_i0, v_i1, v_i2] - T_softmax_maxelem[v_i0, v_i1] - T.log(compute_1[v_i0, v_i1], dtype="float32"))
+    # fmt: on
+
+    mod = LegalizeOps()(LogSoftmax)
+    tvm.ir.assert_structural_equal(mod, Expected)
+
+
+def test_cross_entropy_with_logits():
+    # fmt: off
+    @tvm.script.ir_module
+    class CrossEntropyWithLogits:
+        @R.function
+        def main(x: R.Tensor((3,), "float32"), y: R.Tensor((3,), "float32")) -> R.Tensor(None, "float32", ndim=2):
+            gv: R.Tensor((), "float32") = R.nn.cross_entropy_with_logits(x, y)
+            return gv
+
+    @tvm.script.ir_module
+    class Expected:
+        @R.function
+        def main(x: R.Tensor((3,), dtype="float32"), y: R.Tensor((3,), dtype="float32")) -> R.Tensor(dtype="float32", ndim=2):
+            gv = R.call_tir(cross_entropy_with_logits, (x, y), R.Tensor((), dtype="float32"))
+            return gv
+
+        @T.prim_func
+        def cross_entropy_with_logits(rxplaceholder: T.Buffer[T.int64(3), "float32"], rxplaceholder_1: T.Buffer[T.int64(3), "float32"], T_multiply: T.Buffer[(), "float32"]):
+            T.func_attr({"tir.noalias": True})
+            T_multiply_1 = T.alloc_buffer([T.int64(3)], dtype="float32")
+            T_multiply_red = T.alloc_buffer([], dtype="float32")
+            for i0 in T.serial(T.int64(3)):
+                with T.block("T_multiply"):
+                    ax0 = T.axis.spatial(T.int64(3), i0)
+                    T.reads(rxplaceholder[ax0], rxplaceholder_1[ax0])
+                    T.writes(T_multiply_1[ax0])
+                    T_multiply_1[ax0] = rxplaceholder[ax0] * rxplaceholder_1[ax0]
+            for i0 in T.serial(T.int64(3)):
+                with T.block("T_multiply_red"):
+                    k0 = T.axis.reduce(T.int64(3), i0)
+                    T.reads(T_multiply_1[k0])
+                    T.writes(T_multiply_red[()])
+                    with T.init():
+                        T_multiply_red[()] = T.float32(0)
+                    T_multiply_red[()] = T_multiply_red[()] + T_multiply_1[k0]
+            with T.block("T_multiply_1"):
+                vi = T.axis.spatial(1, T.int64(0))
+                T.reads(T_multiply_red[()])
+                T.writes(T_multiply[()])
+                T_multiply[()] = T_multiply_red[()] * T.float32(-1)
+    # fmt: on
+
+    mod = LegalizeOps()(CrossEntropyWithLogits)
+    tvm.ir.assert_structural_equal(mod, Expected)
+
+
+def test_cross_entropy_with_logits_batch():
+    # fmt: off
+    @tvm.script.ir_module
+    class CrossEntropyWithLogits:
+        @R.function
+        def main(x: R.Tensor((2, 3), "float32"), y: R.Tensor((2, 3), "float32")) -> R.Tensor(None, "float32", ndim=2):
+            gv: R.Tensor((), "float32") = R.nn.cross_entropy_with_logits(x, y)
+            return gv
+
+    @tvm.script.ir_module
+    class Expected:
+        @R.function
+        def main(x: R.Tensor((2, 3), dtype="float32"), y: R.Tensor((2, 3), dtype="float32")) -> R.Tensor(dtype="float32", ndim=2):
+            gv = R.call_tir(cross_entropy_with_logits, (x, y), R.Tensor((), dtype="float32"))
+            return gv
+
+        @T.prim_func
+        def cross_entropy_with_logits(rxplaceholder: T.Buffer[(T.int64(2), T.int64(3)), "float32"], rxplaceholder_1: T.Buffer[(T.int64(2), T.int64(3)), "float32"], T_divide: T.Buffer[(), "float32"]):
+            T.func_attr({"tir.noalias": True})
+            T_multiply = T.alloc_buffer([T.int64(2), T.int64(3)], dtype="float32")
+            T_multiply_red = T.alloc_buffer([], dtype="float32")
+            T_multiply_1 = T.alloc_buffer([], dtype="float32")
+            for i0, i1 in T.grid(T.int64(2), T.int64(3)):
+                with T.block("T_multiply"):
+                    ax0, ax1 = T.axis.remap("SS", [i0, i1])
+                    T.reads(rxplaceholder[ax0, ax1], rxplaceholder_1[ax0, ax1])
+                    T.writes(T_multiply[ax0, ax1])
+                    T_multiply[ax0, ax1] = rxplaceholder[ax0, ax1] * rxplaceholder_1[ax0, ax1]
+            for i0, i1 in T.grid(T.int64(2), T.int64(3)):
+                with T.block("T_multiply_red"):
+                    k0, k1 = T.axis.remap("RR", [i0, i1])
+                    T.reads(T_multiply[k0, k1])
+                    T.writes(T_multiply_red[()])
+                    with T.init():
+                        T_multiply_red[()] = T.float32(0)
+                    T_multiply_red[()] = T_multiply_red[()] + T_multiply[k0, k1]
+            with T.block("T_multiply_1"):
+                vi = T.axis.spatial(1, T.int64(0))
+                T.reads(T_multiply_red[()])
+                T.writes(T_multiply_1[()])
+                T_multiply_1[()] = T_multiply_red[()] * T.float32(-1)
+            with T.block("T_divide"):
+                vi = T.axis.spatial(1, T.int64(0))
+                T.reads(T_multiply_1[()])
+                T.writes(T_divide[()])
+                T_divide[()] = T_multiply_1[()] * T.float32(0.5)
+    # fmt: on
+
+    mod = LegalizeOps()(CrossEntropyWithLogits)
+    tvm.ir.assert_structural_equal(mod, Expected)
+
+
+def test_cross_entropy_with_logits_batch_symbolic():
+    # fmt: off
+    @tvm.script.ir_module
+    class CrossEntropyWithLogits:
+        @R.function
+        def main(x: R.Tensor(("n", "m"), "float32"), y: R.Tensor(("n", "m"), "float32")) -> R.Tensor(None, "float32", ndim=2):
+            n = T.var("int64")
+            m = T.var("int64")
+            gv: R.Tensor((), "float32") = R.nn.cross_entropy_with_logits(x, y)
+            return gv
+
+    @tvm.script.ir_module
+    class Expected:
+        @R.function
+        def main(x: R.Tensor(("n", "m"), dtype="float32"), y: R.Tensor(("n", "m"), dtype="float32")) -> R.Tensor(dtype="float32", ndim=2):
+            gv = R.call_tir(cross_entropy_with_logits, (x, y), R.Tensor((), dtype="float32"))
+            return gv
+
+        @T.prim_func
+        def cross_entropy_with_logits(var_rxplaceholder: T.handle, var_rxplaceholder_1: T.handle, T_divide: T.Buffer[(), "float32"]):
+            T.func_attr({"tir.noalias": True})
+            m = T.var("int64")
+            n = T.var("int64")
+            rxplaceholder = T.match_buffer(var_rxplaceholder, [n, m], dtype="float32")
+            rxplaceholder_1 = T.match_buffer(var_rxplaceholder_1, [n, m], dtype="float32")
+            T_multiply = T.alloc_buffer([n, m], dtype="float32")
+            T_multiply_red = T.alloc_buffer([], dtype="float32")
+            T_multiply_1 = T.alloc_buffer([], dtype="float32")
+            for ax0, ax1 in T.grid(n, m):
+                with T.block("T_multiply"):
+                    v_ax0, v_ax1 = T.axis.remap("SS", [ax0, ax1])
+                    T.reads(rxplaceholder[v_ax0, v_ax1], rxplaceholder_1[v_ax0, v_ax1])
+                    T.writes(T_multiply[v_ax0, v_ax1])
+                    T_multiply[v_ax0, v_ax1] = rxplaceholder[v_ax0, v_ax1] * rxplaceholder_1[v_ax0, v_ax1]
+            for k0, k1 in T.grid(n, m):
+                with T.block("T_multiply_red"):
+                    v_k0, v_k1 = T.axis.remap("RR", [k0, k1])
+                    T.reads(T_multiply[v_k0, v_k1])
+                    T.writes(T_multiply_red[()])
+                    with T.init():
+                        T_multiply_red[()] = T.float32(0)
+                    T_multiply_red[()] = T_multiply_red[()] + T_multiply[v_k0, v_k1]
+            with T.block("T_multiply_1"):
+                vi = T.axis.spatial(1, T.int64(0))
+                T.reads(T_multiply_red[()])
+                T.writes(T_multiply_1[()])
+                T_multiply_1[()] = T_multiply_red[()] * T.float32(-1)
+            with T.block("T_divide"):
+                vi = T.axis.spatial(1, T.int64(0))
+                T.reads(T_multiply_1[()])
+                T.writes(T_divide[()])
+                T_divide[()] = T_multiply_1[()] / T.Cast("float32", n)
+    # fmt: on
+
+    mod = LegalizeOps()(CrossEntropyWithLogits)
+    tvm.ir.assert_structural_equal(mod, Expected)
+
+
 def test_batch_norm():
     # fmt: off
     @tvm.script.ir_module
diff --git a/tests/python/relax/test_tvmscript_parser_op_nn.py b/tests/python/relax/test_tvmscript_parser_op_nn.py
index 4e52bccb86..781700af7b 100644
--- a/tests/python/relax/test_tvmscript_parser_op_nn.py
+++ b/tests/python/relax/test_tvmscript_parser_op_nn.py
@@ -115,6 +115,21 @@ def test_softmax():
     _check(foo, bb.get()["foo"])
 
 
+def test_log_softmax():
+    @R.function
+    def foo(x: R.Tensor((2, 3), "float32")) -> R.Tensor((2, 3), "float32"):
+        gv: R.Tensor((2, 3), "float32") = R.nn.log_softmax(x)
+        return gv
+
+    x = relax.Var("x", R.Tensor((2, 3), "float32"))
+    bb = relax.BlockBuilder()
+    with bb.function("foo", [x]):
+        gv = bb.emit(relax.op.nn.log_softmax(x))
+        bb.emit_func_output(gv)
+
+    _check(foo, bb.get()["foo"])
+
+
 def test_batch_norm():
     @R.function
     def foo(
@@ -189,5 +204,23 @@ def test_dropout():
     _check(foo, bb.get()["foo"])
 
 
+def test_cross_entropy_with_logits():
+    @R.function
+    def foo(
+        predictions: R.Tensor((2, 3), "float32"), labels: R.Tensor((2, 3), "float32")
+    ) -> R.Tensor((), "float32"):
+        gv: R.Tensor((), "float32") = R.nn.cross_entropy_with_logits(predictions, labels)
+        return gv
+
+    predictions = relax.Var("predictions", R.Tensor((2, 3), "float32"))
+    labels = relax.Var("labels", R.Tensor((2, 3), "float32"))
+    bb = relax.BlockBuilder()
+    with bb.function("foo", [predictions, labels]):
+        gv = bb.emit(relax.op.nn.cross_entropy_with_logits(predictions, labels))
+        bb.emit_func_output(gv)
+
+    _check(foo, bb.get()["foo"])
+
+
 if __name__ == "__main__":
     tvm.testing.main()