You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2026/09/15 13:49:09 UTC
(tvm) branch main updated: [REFACTOR] Default mutators to exceptions and migrate arithmetic passes (#20339)
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new 17a95edb6d [REFACTOR] Default mutators to exceptions and migrate arithmetic passes (#20339)
17a95edb6d is described below
commit 17a95edb6d6e2b1691c93dedbb11026756ccf4a0
Author: Tianqi Chen <tq...@users.noreply.github.com>
AuthorDate: Tue Sep 15 09:49:02 2026 -0400
[REFACTOR] Default mutators to exceptions and migrate arithmetic passes (#20339)
Make exceptions the default native visitor and mutator error path, with
Expected adapters at interoperability boundaries. Preserve virtual entry
interception, visit interruption and qualified-parent mutation. Provide
unchecked typed expression entries and category-specific native hook
results, and return inline inputs as value replacements at the generic
entry.
Migrate arithmetic visitors, mutators and simplification passes to
direct results while preserving mutation permissions, ownership, rewrite
ordering and scope restoration. Resolve unchanged values at callers and
adopt the latest upstream FFI visit and consuming-result helpers. Keep
arithmetic traversal focused on semantic operands while preserving types
and metadata, and share analyzer constraint derivation internally.
---
3rdparty/tvm-ffi | 2 +-
include/tvm/arith/analyzer.h | 11 +-
include/tvm/ir/expr_functor.h | 213 ++++++-----
include/tvm/ir/object_functor.h | 195 +++++-----
src/arith/bound_deducer.cc | 80 ++--
src/arith/canonical_simplify.cc | 212 ++++++-----
src/arith/const_int_bound.cc | 125 ++++---
src/arith/constraint_helpers.h | 155 ++++++++
src/arith/detect_linear_equation.cc | 28 +-
src/arith/int_constraints.cc | 5 +-
src/arith/int_set.cc | 64 ++--
src/arith/iter_affine_map.cc | 286 ++++++++------
src/arith/modular_set.cc | 94 ++---
src/arith/presburger_set.cc | 1 -
src/arith/rewrite_simplify.cc | 294 +++++++++------
src/arith/rewrite_simplify.h | 83 +++--
src/arith/simplify_base.cc | 166 +++++++++
src/arith/simplify_base.h | 75 ++++
src/arith/solve_linear_equation.cc | 2 +-
src/arith/solve_linear_inequality.cc | 44 ++-
src/arith/unwrap_vector_expr.cc | 127 ++++++-
src/arith/z3_prover.cc | 78 ++--
src/backend/trn/transform/lower_trainium_layout.cc | 6 +-
src/ir/expr_functor.cc | 414 +++++++++------------
src/s_tir/analysis/domain_touched.cc | 6 +-
src/s_tir/analysis/identify_memcpy.cc | 4 +-
src/s_tir/analysis/oob_checker.cc | 4 +-
src/s_tir/backend/adreno/inject_texture_alloc.cc | 4 +-
src/s_tir/backend/adreno/texture_flatten.cc | 3 +-
.../schedule/primitive/layout_transformation.cc | 6 +-
src/s_tir/schedule/transform.cc | 6 +-
src/s_tir/schedule/transform.h | 4 +-
src/s_tir/transform/hoist_expression.cc | 6 +-
src/s_tir/transform/inject_permuted_layout.cc | 2 +-
src/s_tir/transform/inject_virtual_thread.cc | 6 +-
src/s_tir/transform/lower_async_dma.cc | 17 +-
src/s_tir/transform/lower_vtcm_alloc.cc | 3 +-
src/s_tir/transform/renormalize_split_pattern.cc | 2 +-
.../transform/using_assume_to_reduce_branches.cc | 2 +-
src/{arith => tirx}/ir_mutator_with_analyzer.cc | 127 +------
src/{arith => tirx}/ir_mutator_with_analyzer.h | 44 +--
src/{arith => tirx}/ir_visitor_with_analyzer.cc | 8 +-
src/{arith => tirx}/ir_visitor_with_analyzer.h | 30 +-
src/tirx/transform/flatten_buffer.cc | 4 +-
src/tirx/transform/lower_intrin.cc | 4 +-
src/tirx/transform/lower_tirx_cleanup.cc | 6 +-
src/tirx/transform/narrow_datatype.cc | 5 +-
src/tirx/transform/remove_no_op.cc | 4 +-
src/tirx/transform/stmt_simplify.cc | 2 +-
tests/cpp/expr_functor_test.cc | 30 +-
50 files changed, 1823 insertions(+), 1276 deletions(-)
diff --git a/3rdparty/tvm-ffi b/3rdparty/tvm-ffi
index 4ffc1f35f3..daf594da8e 160000
--- a/3rdparty/tvm-ffi
+++ b/3rdparty/tvm-ffi
@@ -1 +1 @@
-Subproject commit 4ffc1f35f3c9853b53f2e854768a9acc6cbf3b85
+Subproject commit daf594da8e6950fdc02ae35fcf4e0ab2e59a4979
diff --git a/include/tvm/arith/analyzer.h b/include/tvm/arith/analyzer.h
index 4d9c8b1365..889c02193c 100644
--- a/include/tvm/arith/analyzer.h
+++ b/include/tvm/arith/analyzer.h
@@ -26,6 +26,7 @@
#include <tvm/arith/int_set.h>
#include <tvm/ffi/cast.h>
+#include <tvm/ffi/object.h>
#include <tvm/ffi/reflection/registry.h>
#include <tvm/ffi/string.h>
#include <tvm/ir/expr.h>
@@ -192,7 +193,7 @@ class ConstIntBoundAnalyzer {
struct Entry;
class Impl;
/*! \brief Internal impl */
- Impl* impl_;
+ ffi::ObjectPtr<Impl> impl_;
};
/*!
@@ -272,7 +273,7 @@ class ModularSetAnalyzer {
struct Entry;
class Impl;
/*! \brief Internal impl */
- Impl* impl_;
+ ffi::ObjectPtr<Impl> impl_;
};
/*!
@@ -420,7 +421,7 @@ class RewriteSimplifier {
void CopyFrom(const RewriteSimplifier& other);
class Impl;
/*! \brief Internal impl */
- Impl* impl_;
+ ffi::ObjectPtr<Impl> impl_;
};
/*!
@@ -452,7 +453,7 @@ class CanonicalSimplifier {
void CopyFrom(const CanonicalSimplifier& other);
class Impl;
/*! \brief Internal impl */
- Impl* impl_;
+ ffi::ObjectPtr<Impl> impl_;
};
/*! \brief Structure for representing result of known
@@ -714,7 +715,7 @@ class Z3Prover {
TVM_DLL ~Z3Prover();
void CopyFrom(const Z3Prover& other);
class Impl;
- Impl* impl_;
+ ffi::ObjectPtr<Impl> impl_;
};
/*!
diff --git a/include/tvm/ir/expr_functor.h b/include/tvm/ir/expr_functor.h
index f90a5f0d7d..8364299ae2 100644
--- a/include/tvm/ir/expr_functor.h
+++ b/include/tvm/ir/expr_functor.h
@@ -282,7 +282,7 @@ class ExprFunctor<R(const Expr&, Args...)> {
* public:
* TVM_DEFINE_OBJECT_FUNCTOR_DEFAULT_CONSTRUCTOR(MyExprVisitor, ExprVisitor)
* using ExprVisitor::Visit_;
- * virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const MyExprNode* node);
+ * virtual ffi::Optional<VisitInterrupt> Visit_(const MyExprNode* node);
*
* protected:
* static void InitVTable(VTable* vtable) {
@@ -298,45 +298,45 @@ class TVM_DLL ExprVisitor : public ObjectVisitor {
/*! \brief Construct a visitor with the core expression hooks. */
TVM_DEFINE_OBJECT_FUNCTOR_DEFAULT_CONSTRUCTOR(ExprVisitor, ObjectVisitor)
- using ObjectVisitor::VisitExpected;
+ using ObjectVisitor::Visit;
// Override existing hooks directly. Extra types need a fresh inherited table.
- // Hooks borrow the node and return None, an owning interrupt, or an Error.
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const OpaqueExprNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const TupleNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const TupleGetItemNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const TensorLoadNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const VarNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const GlobalVarNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const CallNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const IntImmNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const FloatImmNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const OpNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::StringImmNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::CastNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::AddNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::SubNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::MulNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::DivNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::ModNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::FloorDivNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::FloorModNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::MinNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::MaxNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::EQNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::NENode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::LTNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::LENode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::GTNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::GENode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::AndNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::OrNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::NotNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::SelectNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::LetNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::RampNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::BroadcastNode* node);
- virtual Expected<ffi::Optional<VisitInterrupt>> Visit_(const prim::ShuffleNode* node);
+ // Hooks borrow the node and return None or an owning interrupt, and throw on failure.
+ virtual ffi::Optional<VisitInterrupt> Visit_(const OpaqueExprNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const TupleNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const TupleGetItemNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const TensorLoadNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const VarNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const GlobalVarNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const CallNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const IntImmNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const FloatImmNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const OpNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::StringImmNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::CastNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::AddNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::SubNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::MulNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::DivNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::ModNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::FloorDivNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::FloorModNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::MinNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::MaxNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::EQNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::NENode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::LTNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::LENode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::GTNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::GENode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::AndNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::OrNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::NotNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::SelectNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::LetNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::RampNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::BroadcastNode* node);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const prim::ShuffleNode* node);
protected:
/*!
@@ -367,7 +367,7 @@ class TVM_DLL ExprVisitor : public ObjectVisitor {
* public:
* TVM_DEFINE_OBJECT_FUNCTOR_DEFAULT_CONSTRUCTOR(MyExprMutator, ExprMutator)
* using ExprMutator::Mutate_;
- * virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const MyExprNode* node, InplaceMode
+ * virtual UnchangedOr<Expr> Mutate_(const MyExprNode* node, InplaceMode
* inplace_mode);
*
* protected:
@@ -384,77 +384,76 @@ class TVM_DLL ExprMutator : public ObjectMutator {
/*! \brief Construct a mutator with the core expression hooks. */
TVM_DEFINE_OBJECT_FUNCTOR_DEFAULT_CONSTRUCTOR(ExprMutator, ObjectMutator)
- using ObjectMutator::MutateExpected;
+ using ObjectMutator::Mutate;
+
+ /*!
+ * \brief Mutate a primitive expression, preserving its expression category.
+ * \param expr The borrowed primitive expression.
+ * \param inplace_mode Inherited mutation permission.
+ * \return Unchanged or an owning primitive expression replacement.
+ * \note Calls the virtual AnyView entry. Overrides must preserve PrimExpr;
+ * the result storage is transferred without a runtime type check.
+ */
+ TVM_FFI_INLINE UnchangedOr<PrimExpr> Mutate(const PrimExpr& expr,
+ InplaceMode inplace_mode = InplaceMode::kDisallow) {
+ return ffi::details::UnchangedOrUnsafe::MoveFromTVMFFIAny<PrimExpr>(
+ ffi::details::UnchangedOrUnsafe::MoveToTVMFFIAny(Mutate(ffi::AnyView(expr), inplace_mode)));
+ }
+
+ /*!
+ * \brief Mutate an expression, preserving its expression category.
+ * \param expr The borrowed expression.
+ * \param inplace_mode Inherited mutation permission.
+ * \return Unchanged or an owning expression replacement.
+ * \note Calls the virtual AnyView entry. Overrides must preserve Expr;
+ * the result storage is transferred without a runtime type check.
+ */
+ TVM_FFI_INLINE UnchangedOr<Expr> Mutate(const Expr& expr,
+ InplaceMode inplace_mode = InplaceMode::kDisallow) {
+ return ffi::details::UnchangedOrUnsafe::MoveFromTVMFFIAny<Expr>(
+ ffi::details::UnchangedOrUnsafe::MoveToTVMFFIAny(Mutate(ffi::AnyView(expr), inplace_mode)));
+ }
// A downstream class overrides any existing hook without rebuilding the table.
// Extra node types use a fresh inherited table and SetDispatch<Self, ExtraNode>.
- // Hooks borrow the node and return an owning Expr replacement in Any, Unchanged, or Error.
- // Narrower field types are checked separately. Forward inplace_mode on every child edge.
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const OpaqueExprNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const TupleNode* node, InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const TupleGetItemNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const TensorLoadNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const VarNode* node, InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const GlobalVarNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const CallNode* node, InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const IntImmNode* node, InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const FloatImmNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const OpNode* node, InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::StringImmNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::CastNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::AddNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::SubNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::MulNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::DivNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::ModNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::FloorDivNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::FloorModNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::MinNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::MaxNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::EQNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::NENode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::LTNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::LENode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::GTNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::GENode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::AndNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::OrNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::NotNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::SelectNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::LetNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::RampNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::BroadcastNode* node,
- InplaceMode inplace_mode);
- virtual Expected<UnchangedOr<ffi::Any>> Mutate_(const prim::ShuffleNode* node,
- InplaceMode inplace_mode);
+ // Hooks borrow the node and return Unchanged or an owning replacement in its expression
+ // category, throwing on failure. Narrower field types are checked separately. Forward
+ // inplace_mode on every child edge.
+ virtual UnchangedOr<Expr> Mutate_(const OpaqueExprNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<Expr> Mutate_(const TupleNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<Expr> Mutate_(const TupleGetItemNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const TensorLoadNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<Expr> Mutate_(const VarNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<Expr> Mutate_(const GlobalVarNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<Expr> Mutate_(const CallNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const IntImmNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const FloatImmNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<Expr> Mutate_(const OpNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::StringImmNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::CastNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::AddNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::SubNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::MulNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::DivNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::ModNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::FloorDivNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::FloorModNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::MinNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::MaxNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::EQNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::NENode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::LTNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::LENode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::GTNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::GENode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::AndNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::OrNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::NotNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::SelectNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::LetNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::RampNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::BroadcastNode* node, InplaceMode inplace_mode);
+ virtual UnchangedOr<PrimExpr> Mutate_(const prim::ShuffleNode* node, InplaceMode inplace_mode);
protected:
/*!
diff --git a/include/tvm/ir/object_functor.h b/include/tvm/ir/object_functor.h
index a45251b42e..f3bfe2e7e0 100644
--- a/include/tvm/ir/object_functor.h
+++ b/include/tvm/ir/object_functor.h
@@ -235,32 +235,46 @@ class TVM_DLL ObjectVisitor : public ffi::StructuralVisitorObj {
ObjectVisitor& operator=(const ObjectVisitor& other) = delete;
/*!
- * \brief Visit a borrowed object or inline value.
+ * \brief Visit a borrowed object or inline value, throwing on failure.
* \param value The borrowed value to visit.
- * \return None on completion, an owning VisitInterrupt, or an Error.
- * \note Overrides must convert thrown errors and propagate errors and interrupts from their
- * own work. A qualified Parent::VisitExpected(value) bypasses the current entry override;
- * descendant calls still use virtual dispatch.
+ * \return None on completion or an owning interrupt that halts traversal.
+ * \note A qualified Parent::Visit bypasses this entry override only;
+ * recursive child visits still use virtual dispatch.
*/
- virtual Expected<ffi::Optional<VisitInterrupt>> VisitExpected(ffi::AnyView value) noexcept {
+ virtual ffi::Optional<VisitInterrupt> Visit(ffi::AnyView value) {
if (native_vtable_->CanDispatch(value)) {
- // Exact registrations contain only object node types, so this extraction is borrowed
- // and needs no additional object-type check or owning reference.
const auto* object =
ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const ffi::Object>(value);
try {
- return DispatchNative(object);
+ return (*native_vtable_)(object, this);
} catch (ffi::Error& error) {
- return AttachVisitErrorContext(error, object);
+ ffi::details::UpdateVisitErrorContext(error, ffi::GetRef<ffi::ObjectRef>(object));
+ throw;
}
}
return StructuralVisitDefault(value);
}
+ /*!
+ * \brief Adapt the virtual throwing entrypoint to the structural ABI.
+ * \param value The borrowed value to visit.
+ * \return Completion, interrupt, or the caught error.
+ * \note Follows the FFI safe-call exception contract: Error retains its identity,
+ * and other standard exceptions become InternalError.
+ */
+ Expected<ffi::Optional<VisitInterrupt>> VisitExpected(ffi::AnyView value) noexcept {
+ try {
+ return Visit(value);
+ } catch (const ffi::Error& error) {
+ return ffi::Unexpected(error);
+ } catch (const std::exception& error) {
+ return ffi::Unexpected(ffi::Error("InternalError", error.what(), ""));
+ }
+ }
+
protected:
- /*! \brief Exact native dispatch table with owning interrupt and error results. */
- using VTable =
- ObjectFunctor<Expected<ffi::Optional<VisitInterrupt>>(const ffi::Object*, ObjectVisitor*)>;
+ /*! \brief Exact native dispatch table with owning interrupt results. */
+ using VTable = ObjectFunctor<ffi::Optional<VisitInterrupt>(const ffi::Object*, ObjectVisitor*)>;
/*!
* \brief Construct a visitor with a finalized native dispatch table.
@@ -289,36 +303,18 @@ class TVM_DLL ObjectVisitor : public ffi::StructuralVisitorObj {
private:
// Native table callback; core instantiations may be shared by the library.
template <typename Self, typename Node>
- static Expected<ffi::Optional<VisitInterrupt>> DispatchNode(const ffi::Object* node,
- ObjectVisitor* self) {
+ static ffi::Optional<VisitInterrupt> DispatchNode(const ffi::Object* node, ObjectVisitor* self) {
return static_cast<Self*>(self)->Visit_(static_cast<const Node*>(node));
}
- // Keep one named return value in this scope so native results can be constructed in place.
- TVM_FFI_INLINE Expected<ffi::Optional<VisitInterrupt>> DispatchNative(const ffi::Object* value) {
- Expected<ffi::Optional<VisitInterrupt>> result = (*native_vtable_)(value, this);
- if (TVM_FFI_PREDICT_FALSE(result.is_err())) {
- UpdateVisitErrorContext(result, value);
- }
- return result;
- }
- Expected<ffi::Optional<VisitInterrupt>> StructuralVisitDefault(ffi::AnyView value) noexcept {
- TVMFFIAny result = ffi::details::ExpectedUnsafe::MoveToTVMFFIAny(
- ffi::StructuralVisitorObj::DefaultVisitExpected(value));
- if (TVM_FFI_PREDICT_FALSE(result.type_index == ffi::TypeIndex::kTVMFFIError)) {
- result = ffi::details::AttachStructuralVisitErrorContextRaw(result, value);
+ ffi::Optional<VisitInterrupt> StructuralVisitDefault(ffi::AnyView value) {
+ try {
+ // DefaultVisitExpected does not name the current node; DefaultMutateExpected does.
+ return ffi::StructuralVisitorObj::DefaultVisitExpected(value).value();
+ } catch (ffi::Error& error) {
+ ffi::details::UpdateVisitErrorContext(error, value);
+ throw;
}
- return ffi::details::ExpectedUnsafe::MoveFromTVMFFIAny<ffi::Optional<VisitInterrupt>>(result);
- }
- TVM_FFI_COLD_CODE static Expected<ffi::Optional<VisitInterrupt>> AttachVisitErrorContext(
- ffi::Error& error, const ffi::Object* value) {
- if (value) ffi::details::UpdateVisitErrorContext(error, ffi::GetRef<ffi::ObjectRef>(value));
- return ffi::Unexpected(std::move(error));
- }
- TVM_FFI_COLD_CODE static void UpdateVisitErrorContext(
- const Expected<ffi::Optional<VisitInterrupt>>& result, const ffi::Object* value) {
- ffi::Error error = result.error();
- if (value) ffi::details::UpdateVisitErrorContext(error, ffi::GetRef<ffi::ObjectRef>(value));
}
static const VTable* GlobalVTable() {
static const VTable table = [] {
@@ -359,38 +355,73 @@ class TVM_DLL ObjectMutator : public ffi::StructuralMapEngineBase {
ObjectMutator& operator=(const ObjectMutator& other) = delete;
/*!
- * \brief Mutate a borrowed value, checking current uniqueness before in-place dispatch.
+ * \brief Mutate a borrowed value, throwing on failure.
* \param value The borrowed object or inline value to mutate.
* \param inplace_mode Inherited permission along the path to this value.
- * \return A replacement, Unchanged, or an Error if mutation fails.
- * \note The base implementation establishes current uniqueness before invoking typed hooks.
- * Entry overrides must check uniqueness before writing or forwarding permission,
- * never promote kDisallow, and convert thrown errors from their own work into Expected.
- * Expr inputs require Expr replacements. A qualified Parent::MutateExpected(value, mode)
- * bypasses the current entry override while descendants remain virtual.
- * DefaultMutateExpected trusts its caller's established mode and does not recheck it.
+ * \return An owning replacement or Unchanged. Non-object inputs return their inline value.
+ * \note Establishes uniqueness before invoking typed hooks. Entry overrides
+ * must preserve inherited denial and check uniqueness before writing or
+ * forwarding permission. Qualified Parent::Mutate bypasses the current
+ * override while descendants remain virtual. Completed in-place changes
+ * are not rolled back on error.
+ *
+ * In-place mutation requires every node and owning container on the root-to-current
+ * path to be unique. Inherited inplace_mode records ancestor permission; Mutate
+ * checks the current target. Never promote inherited denial to permission.
+ * Within a Mutate_ hook, forward mode to a parent hook or an immediate child:
+ * \code
+ * Parent_::Mutate_(node, mode); // The current node was already checked.
+ * Mutate(node->elem, mode); // Mutate checks elem's uniqueness.
+ * \endcode
+ * Direct descent that skips an owning ancestor must check that ancestor too:
+ * \code
+ * auto child_mode = node->elem.unique() ? mode : InplaceMode::kDisallow;
+ * Mutate(node->elem->a, child_mode);
+ * \endcode
+ * When in doubt, omit inplace_mode. The default kDisallow is always safe; pass
+ * permission explicitly only when the full ancestor path is accounted for.
*/
- virtual Expected<UnchangedOr<ffi::Any>> MutateExpected(
- ffi::AnyView value, InplaceMode inplace_mode = InplaceMode::kDisallow) noexcept {
- if (native_vtable_->CanDispatch(value)) {
- // Only object node types can be registered. Exact dispatch proves this borrowed extraction
- // is valid without an additional object-type check or owning reference.
- const auto* object =
- ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const ffi::Object>(value);
- // Modes encode disallow/allow as 0/1: preserve inherited permission only for a unique object.
- inplace_mode =
- static_cast<InplaceMode>(inplace_mode == InplaceMode::kAllow && object->unique());
+ virtual UnchangedOr<ffi::Any> Mutate(ffi::AnyView value,
+ InplaceMode inplace_mode = InplaceMode::kDisallow) {
+ const auto* object = value.as<ffi::Object>();
+ if (object == nullptr) return ffi::Any(value);
+ if (!object->unique()) inplace_mode = InplaceMode::kDisallow;
+ if (native_vtable_->CanDispatch(object)) {
try {
- return DispatchNative(object, inplace_mode);
+ return (*native_vtable_)(object, this, inplace_mode);
} catch (ffi::Error& error) {
- return AttachVisitErrorContext(error, object);
+ ffi::details::UpdateVisitErrorContext(error, ffi::GetRef<ffi::ObjectRef>(object));
+ throw;
}
}
- const auto* object = value.as<ffi::Object>();
- // The same 0/1 permission rule also excludes inline values and None.
- inplace_mode = static_cast<InplaceMode>(inplace_mode == InplaceMode::kAllow &&
- object != nullptr && object->unique());
- return ffi::StructuralMutatorObj::DefaultMutateExpected(value, inplace_mode);
+ // Default descent already supplies the current node's error context.
+ return ffi::StructuralMutatorObj::DefaultMutateExpected(value, inplace_mode).value();
+ }
+
+ /*!
+ * \brief Adapt the virtual throwing entrypoint to the structural ABI.
+ * \param value The borrowed value to mutate.
+ * \param inplace_mode Inherited mutation permission.
+ * \return The replacement, Unchanged, or the caught error.
+ * \note Matches the FFI safe-call exception contract. Error identity is preserved;
+ * other standard exceptions become InternalError.
+ */
+ Expected<UnchangedOr<ffi::Any>> MutateExpected(
+ ffi::AnyView value, InplaceMode inplace_mode = InplaceMode::kDisallow) noexcept {
+ try {
+ return Mutate(value, inplace_mode);
+ } catch (const ffi::Error& error) {
+ return ffi::Unexpected(error);
+ } catch (const std::exception& error) {
+ return ffi::Unexpected(ffi::Error("InternalError", error.what(), ""));
+ }
+ }
+
+ /*! \brief Look up a variable replacement, throwing on failure. */
+ ffi::Any VarRemapGet(ffi::AnyView var) { return VarRemapGetExpected(var).value(); }
+ /*! \brief Record a variable replacement, throwing on failure. */
+ void VarRemapSet(ffi::AnyView var, ffi::AnyView value) {
+ VarRemapSetExpected(var, value).value();
}
/*!
@@ -413,9 +444,9 @@ class TVM_DLL ObjectMutator : public ffi::StructuralMapEngineBase {
}
protected:
- /*! \brief Exact native dispatch table with owning replacement and error results. */
- using VTable = ObjectFunctor<Expected<UnchangedOr<ffi::Any>>(const ffi::Object*, ObjectMutator*,
- InplaceMode)>;
+ /*! \brief Exact native dispatch table with owning replacement results. */
+ using VTable =
+ ObjectFunctor<UnchangedOr<ffi::Any>(const ffi::Object*, ObjectMutator*, InplaceMode)>;
/*!
* \brief Construct a mutator with a finalized native dispatch table.
@@ -444,33 +475,15 @@ class TVM_DLL ObjectMutator : public ffi::StructuralMapEngineBase {
private:
// Native table callback; core instantiations may be shared by the library.
template <typename Self, typename Node>
- static Expected<UnchangedOr<ffi::Any>> DispatchNode(const ffi::Object* node, ObjectMutator* self,
- InplaceMode inplace_mode) {
- return static_cast<Self*>(self)->Mutate_(static_cast<const Node*>(node), inplace_mode);
+ static UnchangedOr<ffi::Any> DispatchNode(const ffi::Object* node, ObjectMutator* self,
+ InplaceMode inplace_mode) {
+ return ffi::details::UnchangedOrUnsafe::MoveFromTVMFFIAny<ffi::Any>(
+ ffi::details::UnchangedOrUnsafe::MoveToTVMFFIAny(
+ static_cast<Self*>(self)->Mutate_(static_cast<const Node*>(node), inplace_mode)));
}
using ffi::StructuralMutatorObj::DefaultMutateExpected;
- using ffi::StructuralMutatorObj::Mutate;
-
- // Keep one named return value in this scope so native results can be constructed in place.
- TVM_FFI_INLINE Expected<UnchangedOr<ffi::Any>> DispatchNative(const ffi::Object* value,
- InplaceMode inplace_mode) {
- Expected<UnchangedOr<ffi::Any>> result = (*native_vtable_)(value, this, inplace_mode);
- if (TVM_FFI_PREDICT_FALSE(result.is_err())) {
- UpdateVisitErrorContext(result, value);
- }
- return result;
- }
- TVM_FFI_COLD_CODE static Expected<UnchangedOr<ffi::Any>> AttachVisitErrorContext(
- ffi::Error& error, const ffi::Object* value) {
- if (value) ffi::details::UpdateVisitErrorContext(error, ffi::GetRef<ffi::ObjectRef>(value));
- return ffi::Unexpected(std::move(error));
- }
- TVM_FFI_COLD_CODE static void UpdateVisitErrorContext(
- const Expected<UnchangedOr<ffi::Any>>& result, const ffi::Object* value) {
- ffi::Error error = result.error();
- if (value) ffi::details::UpdateVisitErrorContext(error, ffi::GetRef<ffi::ObjectRef>(value));
- }
+
static const VTable* GlobalVTable() {
static const VTable table = [] {
VTable table;
diff --git a/src/arith/bound_deducer.cc b/src/arith/bound_deducer.cc
index 2be1cfc1e7..55ddbfe73b 100644
--- a/src/arith/bound_deducer.cc
+++ b/src/arith/bound_deducer.cc
@@ -22,13 +22,15 @@
* \brief Utility to deduce bound of expression
*/
#include <tvm/arith/analyzer.h>
+#include <tvm/ffi/extra/structural_visit.h>
#include <tvm/ffi/function.h>
#include <tvm/ffi/reflection/registry.h>
+#include <tvm/ir/expr_functor.h>
#include <tvm/ir/prim/expr.h>
-#include <tvm/tirx/expr_functor.h>
#include <unordered_map>
#include <unordered_set>
+#include <vector>
#include "interval_set.h"
@@ -37,26 +39,31 @@ namespace arith {
using namespace tirx;
-// a visitor to find the path to the target variable
-// from a expression.
-class VariablePathFinder : public ExprVisitor {
+// Find a target path through structural expression fields, including dynamic types.
+// BoundDeduceInputChecker counts occurrences over the same broader domain and can
+// conservatively decline deduction when a target also appears in type metadata.
+class VariablePathFinder {
public:
explicit VariablePathFinder(PrimExpr target) : target_(target) {}
- void VisitExpr(const Expr& node) final {
- if (visited_.count(node.get()) != 0) return;
- visited_.insert(node.get());
-
- if (!found_) path_.push_back(node.get());
- if (node.same_as(target_)) found_ = true;
- ExprVisitor::VisitExpr(node);
- if (!found_) path_.pop_back();
+ void operator()(const Expr& node) {
+ ffi::StructuralVisit(
+ node,
+ [this](const Expr& child,
+ ffi::StructuralVisitorObj* visitor) -> Expected<ffi::Optional<ffi::VisitInterrupt>> {
+ if (!visited_.insert(child.get()).second) return std::nullopt;
+ path_.push_back(child.get());
+ if (child.same_as(target_)) return ffi::VisitInterrupt();
+
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(visitor->DefaultVisitExpected(child));
+ path_.pop_back();
+ return std::nullopt;
+ });
}
std::vector<const ffi::Object*> path_;
private:
- bool found_{false};
PrimExpr target_;
std::unordered_set<const ffi::Object*> visited_;
};
@@ -72,7 +79,7 @@ std::vector<const ffi::Object*> GetPath(PrimExpr target, PrimExpr expr) {
enum CompareOp { kGreater, kLess, kEqual };
// a visitor to deduce the bound of a variable from a expression
-class BoundDeducer : public ExprFunctor<void(const Expr&)> {
+class BoundDeducer : public tvm::ExprFunctor<void(const Expr&)> {
public:
friend class BoundDeduceInputChecker;
friend class Converter;
@@ -83,17 +90,17 @@ class BoundDeducer : public ExprFunctor<void(const Expr&)> {
void Deduce();
- void VisitExpr(const Expr& e) final {
+ void Dispatch(const Expr& e) final {
if (!success_) return;
if (iter_ < path_.size() && e.get() == path_[iter_++]) {
- ExprFunctor::VisitExpr(e);
+ ExprFunctor::Dispatch(e);
} else {
success_ = false;
return;
}
}
- void VisitExprDefault_(const ffi::Object* op) final { success_ = false; }
+ void DispatchDefault_(const ffi::Object* op) final { success_ = false; }
SignType GetSignType(const PrimExpr& e) {
PrimType e_ty = e.ty();
@@ -103,15 +110,15 @@ class BoundDeducer : public ExprFunctor<void(const Expr&)> {
return expr_map_[e].GetSignType();
}
- void VisitExpr_(const VarNode* op) final {}
+ void Dispatch_(const VarNode* op) final {}
- void VisitExpr_(const prim::AddNode* op) final {
+ void Dispatch_(const prim::AddNode* op) final {
bool left = op->a.get() == path_[iter_];
result_ -= left ? op->b : op->a;
- this->VisitExpr(left ? op->a : op->b);
+ this->Dispatch(left ? op->a : op->b);
}
- void VisitExpr_(const prim::SubNode* op) final {
+ void Dispatch_(const prim::SubNode* op) final {
bool left = op->a.get() == path_[iter_];
if (left) {
result_ += op->b;
@@ -120,10 +127,10 @@ class BoundDeducer : public ExprFunctor<void(const Expr&)> {
result_ = -result_;
comp_op = ReverseOp(comp_op);
}
- this->VisitExpr(left ? op->a : op->b);
+ this->Dispatch(left ? op->a : op->b);
}
- void VisitExpr_(const prim::MulNode* op) final {
+ void Dispatch_(const prim::MulNode* op) final {
bool left = op->a.get() == path_[iter_];
PrimExpr operand = left ? op->b : op->a;
PrimExpr target_var = left ? op->a : op->b;
@@ -162,10 +169,10 @@ class BoundDeducer : public ExprFunctor<void(const Expr&)> {
// ( x <= -3/-2 --> x <= 1)
}
}
- this->VisitExpr(left ? op->a : op->b);
+ this->Dispatch(left ? op->a : op->b);
}
- void VisitExpr_(const prim::FloorDivNode* op) final {
+ void Dispatch_(const prim::FloorDivNode* op) final {
if (op->b.get() == path_[iter_]) {
// Skip cases where the var is divisor.
success_ = false;
@@ -208,7 +215,7 @@ class BoundDeducer : public ExprFunctor<void(const Expr&)> {
result_ = result_ - divisor + 1;
}
- this->VisitExpr(op->a);
+ this->Dispatch(op->a);
}
PrimExpr result_;
@@ -231,22 +238,17 @@ class BoundDeducer : public ExprFunctor<void(const Expr&)> {
Analyzer analyzer_;
};
-class BoundDeduceInputChecker : public ExprVisitor {
+class BoundDeduceInputChecker {
public:
bool Check(BoundDeducer* deducer) {
- deducer_ = deducer;
- this->VisitExpr(deducer_->expr_);
+ size_t target_count = 0;
+ ffi::StructuralWalk<ffi::WalkOrder::kPreOrder>(
+ deducer->expr_, [&](const Expr& expr) -> ffi::Expected<ffi::WalkResult> {
+ if (expr.same_as(deducer->target_)) ++target_count;
+ return ffi::WalkResult::Advance();
+ });
return target_count == 1;
}
-
- void VisitExpr(const Expr& e) final {
- if (e.same_as(deducer_->target_)) ++target_count;
- ExprVisitor::VisitExpr(e);
- }
-
- private:
- BoundDeducer* deducer_;
- size_t target_count{0};
};
void BoundDeducer::Init() {
@@ -345,7 +347,7 @@ void BoundDeducer::Deduce() {
}
expr_map_ = EvalSetForEachSubExpr(expr_, hint_map_);
- this->VisitExpr(expr_);
+ this->Dispatch(expr_);
if (success_) {
result_ = analyzer_->Simplify(result_);
diff --git a/src/arith/canonical_simplify.cc b/src/arith/canonical_simplify.cc
index 1a390b81a3..2f48809233 100644
--- a/src/arith/canonical_simplify.cc
+++ b/src/arith/canonical_simplify.cc
@@ -23,6 +23,7 @@
*/
#include <tvm/arith/analyzer.h>
#include <tvm/ffi/cast.h>
+#include <tvm/ffi/expected.h>
#include <tvm/ir/cow.h>
#include <tvm/tirx/analysis.h>
#include <tvm/tirx/op.h>
@@ -567,38 +568,48 @@ void SumExprNode::AddToSelf(const SumExpr& other, int64_t scale) {
class CanonicalSimplifier::Impl : public RewriteSimplifier::Impl {
public:
using Rewriter = RewriteSimplifier::Impl;
+ using Rewriter::Mutate;
explicit Impl(AnalyzerObj* parent) : Rewriter(parent) {}
- PrimExpr CanonicalSimplify(PrimExpr expr) {
- expr = VisitPrimExpr(expr);
- return expr;
+ // Bypass only this entry override; the parent checks the target and descendants stay virtual.
+ TVM_FFI_INLINE UnchangedOr<PrimExpr> DirectMutate(
+ const PrimExpr& value, InplaceMode inplace_mode = InplaceMode::kDisallow) {
+ return ffi::details::UnchangedOrUnsafe::MoveFromTVMFFIAny<PrimExpr>(
+ ffi::details::UnchangedOrUnsafe::MoveToTVMFFIAny(
+ Rewriter::Mutate(ffi::AnyView(value), inplace_mode)));
}
- // override the original mutate function.
- Expr VisitExpr(const Expr& input_expr) final {
- Expr expr = Rewriter::VisitExpr(input_expr);
- if (auto prim_expr = expr.as<PrimExpr>()) {
- return Normalize(prim_expr.value());
+ // Normalize replacements; qualified parent calls bypass this root only.
+ UnchangedOr<ffi::Any> Mutate(ffi::AnyView value,
+ InplaceMode inplace_mode = InplaceMode::kDisallow) final {
+ UnchangedOr<ffi::Any> expr_u = Rewriter::Mutate(value, inplace_mode);
+ try {
+ // Only replacements need normalization; an unchanged result preserves the input form.
+ if (expr_u.IsUnchanged()) return ffi::Unchanged();
+ ffi::Any expr = std::move(expr_u).ValueUnchecked();
+ if (auto prim_expr = expr.as<PrimExpr>()) {
+ return Normalize(prim_expr.value());
+ }
+ return expr;
+ } catch (ffi::Error& error) {
+ if (const auto* node = value.as<ffi::Object>()) {
+ ffi::details::UpdateVisitErrorContext(error, ffi::GetRef<ffi::ObjectRef>(node));
+ }
+ throw;
}
- return expr;
- }
-
- // Normal mutation without normalization.
- PrimExpr CanonicalMutate(PrimExpr expr) {
- return Rewriter::VisitExpr(expr).as_or_throw<PrimExpr>();
}
- using Rewriter::VisitExpr_;
- Expr VisitExpr_(const prim::AddNode* op) final;
- Expr VisitExpr_(const prim::SubNode* op) final;
- Expr VisitExpr_(const prim::MulNode* op) final;
- Expr VisitExpr_(const prim::DivNode* op) final;
- Expr VisitExpr_(const prim::ModNode* op) final;
- Expr VisitExpr_(const prim::FloorDivNode* op) final;
- Expr VisitExpr_(const prim::FloorModNode* op) final;
- Expr VisitExpr_(const prim::CastNode* op) final;
- Expr VisitExpr_(const prim::LTNode* op) final;
+ using Rewriter::Mutate_;
+ UnchangedOr<PrimExpr> Mutate_(const prim::AddNode* op, InplaceMode inplace_mode) final;
+ UnchangedOr<PrimExpr> Mutate_(const prim::SubNode* op, InplaceMode inplace_mode) final;
+ UnchangedOr<PrimExpr> Mutate_(const prim::MulNode* op, InplaceMode inplace_mode) final;
+ UnchangedOr<PrimExpr> Mutate_(const prim::DivNode* op, InplaceMode inplace_mode) final;
+ UnchangedOr<PrimExpr> Mutate_(const prim::ModNode* op, InplaceMode inplace_mode) final;
+ UnchangedOr<PrimExpr> Mutate_(const prim::FloorDivNode* op, InplaceMode inplace_mode) final;
+ UnchangedOr<PrimExpr> Mutate_(const prim::FloorModNode* op, InplaceMode inplace_mode) final;
+ UnchangedOr<PrimExpr> Mutate_(const prim::CastNode* op, InplaceMode inplace_mode) final;
+ UnchangedOr<PrimExpr> Mutate_(const prim::LTNode* op, InplaceMode inplace_mode) final;
private:
/*!
@@ -616,7 +627,7 @@ class CanonicalSimplifier::Impl : public RewriteSimplifier::Impl {
* \param div_mode The division mode.
* \return The result expression;
*/
- SplitExpr SplitModConst(SplitExpr lhs, int64_t cval, DivMode div_mode);
+ SplitExpr SplitModConst(SplitExpr lhs, int64_t cval, DivMode div_mode, InplaceMode inplace_mode);
/*!
* \brief Separate psum into divisible and non-divisible parts.
* \param psum The sum expression.
@@ -723,16 +734,17 @@ class CanonicalSimplifier::Impl : public RewriteSimplifier::Impl {
}
};
-Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::AddNode* op) {
+UnchangedOr<PrimExpr> CanonicalSimplifier::Impl::Mutate_(const prim::AddNode* op,
+ InplaceMode inplace_mode) {
if (!IsIndexTypedExpr(op)) {
- return Rewriter::VisitExpr_(op);
+ return Rewriter::Mutate_(op, inplace_mode);
}
// normalize
- PrimExpr a = this->CanonicalMutate(op->a);
- PrimExpr b = this->CanonicalMutate(op->b);
+ PrimExpr a = DirectMutate(op->a, inplace_mode).ValueOrUnchanged(op->a);
+ PrimExpr b = DirectMutate(op->b, inplace_mode).ValueOrUnchanged(op->b);
// const folding
- if (auto const_res = TryConstFold<prim::Add>(a, b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Add>(a, b)) return *std::move(const_res);
// canonical form simplification.
SumExpr ret = ToSumExpr(std::move(a));
@@ -747,16 +759,17 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::AddNode* op) {
return ret;
}
-Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::SubNode* op) {
+UnchangedOr<PrimExpr> CanonicalSimplifier::Impl::Mutate_(const prim::SubNode* op,
+ InplaceMode inplace_mode) {
if (!IsIndexTypedExpr(op)) {
- return Rewriter::VisitExpr_(op);
+ return Rewriter::Mutate_(op, inplace_mode);
}
// normalize
- PrimExpr a = this->CanonicalMutate(op->a);
- PrimExpr b = this->CanonicalMutate(op->b);
+ PrimExpr a = DirectMutate(op->a, inplace_mode).ValueOrUnchanged(op->a);
+ PrimExpr b = DirectMutate(op->b, inplace_mode).ValueOrUnchanged(op->b);
// const folding
- if (auto const_res = TryConstFold<prim::Sub>(a, b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Sub>(a, b)) return *std::move(const_res);
// canonical form simplification.
SumExpr ret = ToSumExpr(std::move(a));
@@ -771,16 +784,17 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::SubNode* op) {
return ret;
}
-Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::MulNode* op) {
+UnchangedOr<PrimExpr> CanonicalSimplifier::Impl::Mutate_(const prim::MulNode* op,
+ InplaceMode inplace_mode) {
if (!IsIndexTypedExpr(op)) {
- return Rewriter::VisitExpr_(op);
+ return Rewriter::Mutate_(op, inplace_mode);
}
// normalize
- PrimExpr a = this->CanonicalMutate(op->a);
- PrimExpr b = this->CanonicalMutate(op->b);
+ PrimExpr a = DirectMutate(op->a, inplace_mode).ValueOrUnchanged(op->a);
+ PrimExpr b = DirectMutate(op->b, inplace_mode).ValueOrUnchanged(op->b);
// const folding
- if (auto const_res = TryConstFold<prim::Mul>(a, b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Mul>(a, b)) return *std::move(const_res);
// x * c
if (a.as<IntImmNode>()) {
@@ -808,7 +822,8 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::MulNode* op) {
const prim::MulNode* mul = ret.as<prim::MulNode>();
if (mul && mul->a.same_as(op->a) && mul->b.same_as(op->b)) {
- return ffi::GetRef<PrimExpr>(op);
+ // Keep the original expression; it needs no replacement normalization.
+ return ffi::Unchanged();
} else {
return ret;
}
@@ -941,16 +956,17 @@ bool CanonicalSimplifier::Impl::ProdDivSimplify(PrimExpr* plhs, PrimExpr* prhs,
return true;
}
-Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::DivNode* op) {
+UnchangedOr<PrimExpr> CanonicalSimplifier::Impl::Mutate_(const prim::DivNode* op,
+ InplaceMode inplace_mode) {
if (!IsIndexTypedExpr(op)) {
- return Rewriter::VisitExpr_(op);
+ return Rewriter::Mutate_(op, inplace_mode);
}
- PrimExpr a = this->CanonicalMutate(op->a);
- PrimExpr b = this->CanonicalMutate(op->b);
+ PrimExpr a = DirectMutate(op->a, inplace_mode).ValueOrUnchanged(op->a);
+ PrimExpr b = DirectMutate(op->b, inplace_mode).ValueOrUnchanged(op->b);
// const folding
- if (auto const_res = TryConstFold<prim::Div>(a, b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Div>(a, b)) return *std::move(const_res);
PVar<IntImm> c1;
// x / c1
if (c1.Match(b) && c1.Eval()->value > 0) {
@@ -999,21 +1015,23 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::DivNode* op) {
return truncdiv(a, b);
}
if (op->a.same_as(a) && op->b.same_as(b)) {
- return ffi::GetRef<PrimExpr>(op);
+ // Keep the original expression; it needs no replacement normalization.
+ return ffi::Unchanged();
} else {
return prim::Div(a, b);
}
}
-Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::FloorDivNode* op) {
+UnchangedOr<PrimExpr> CanonicalSimplifier::Impl::Mutate_(const prim::FloorDivNode* op,
+ InplaceMode inplace_mode) {
if (!IsIndexTypedExpr(op)) {
- return Rewriter::VisitExpr_(op);
+ return Rewriter::Mutate_(op, inplace_mode);
}
- PrimExpr a = this->CanonicalMutate(op->a);
- PrimExpr b = this->CanonicalMutate(op->b);
+ PrimExpr a = DirectMutate(op->a, inplace_mode).ValueOrUnchanged(op->a);
+ PrimExpr b = DirectMutate(op->b, inplace_mode).ValueOrUnchanged(op->b);
// const folding
- if (auto const_res = TryConstFold<prim::FloorDiv>(a, b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::FloorDiv>(a, b)) return *std::move(const_res);
PVar<IntImm> c1;
// x / c1
if (c1.Match(b) && c1.Eval()->value > 0) {
@@ -1055,7 +1073,11 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::FloorDivNode* op) {
if (split_a->lower_factor == 1 && split_a->scale == 1 &&
split_a->upper_factor != SplitExprNode::kPosInf && split_a->upper_factor % cval == 0 &&
split_a->DivModeCompatibleTo(kFloorDiv)) {
- PrimExpr raw_index = this->CanonicalMutate(split_a->index);
+ InplaceMode index_mode = inplace_mode;
+ // This descent skips the SplitExpr that owns the index.
+ if (!a.unique()) index_mode = InplaceMode::kDisallow;
+ PrimExpr raw_index =
+ DirectMutate(split_a->index, index_mode).ValueOrUnchanged(split_a->index);
if (const auto* psum = raw_index.as<SumExprNode>()) {
SumExpr lhs, extra;
SeparateDivisibleParts(psum, cval, &lhs, &extra);
@@ -1075,7 +1097,8 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::FloorDivNode* op) {
}
// Apply floormod(floordiv_result, m) to complete the identity
PrimExpr div_result = Normalize(lhs);
- return this->VisitPrimExpr(floormod(div_result, IntImm(a.ty(), new_mod)));
+ PrimExpr expr = floormod(div_result, IntImm(a.ty(), new_mod));
+ return Mutate(expr, inplace_mode).ValueOrUnchanged(expr);
}
}
}
@@ -1091,13 +1114,15 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::FloorDivNode* op) {
return floordiv(a, b);
}
if (op->a.same_as(a) && op->b.same_as(b)) {
- return ffi::GetRef<PrimExpr>(op);
+ // Keep the original expression; it needs no replacement normalization.
+ return ffi::Unchanged();
} else {
return prim::FloorDiv(a, b);
}
}
-SplitExpr CanonicalSimplifier::Impl::SplitModConst(SplitExpr lhs, int64_t cval, DivMode div_mode) {
+SplitExpr CanonicalSimplifier::Impl::SplitModConst(SplitExpr lhs, int64_t cval, DivMode div_mode,
+ InplaceMode inplace_mode) {
TVM_FFI_ICHECK_GT(cval, 0);
lhs = ConvertDivMode(lhs, div_mode);
@@ -1121,8 +1146,10 @@ SplitExpr CanonicalSimplifier::Impl::SplitModConst(SplitExpr lhs, int64_t cval,
// Perhaps there are more chances in simplifying the index
// Do a recursive call to simplify the mod with the new factor.
if (new_upper_factor < lhs->upper_factor && lhs->upper_factor != SplitExprNode::kPosInf) {
- auto updated = ToSplitExpr(
- this->VisitPrimExpr(ModImpl(lhs->index, IntImm(lhs.ty(), new_upper_factor), div_mode)));
+ PrimExpr reduced = ModImpl(lhs->index, IntImm(lhs.ty(), new_upper_factor), div_mode);
+ // Unchanged belongs to the reduced expression, not the caller's original modulo.
+ auto updated =
+ ToSplitExpr(Mutate(reduced, inplace_mode).ValueOrUnchanged(std::move(reduced)));
// re-apply the lower_factor
if (lhs->lower_factor != 1) {
auto ret = SplitDivConst(updated, lhs->lower_factor, div_mode);
@@ -1151,16 +1178,17 @@ SplitExpr CanonicalSimplifier::Impl::SplitModConst(SplitExpr lhs, int64_t cval,
return lhs;
}
-Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::ModNode* op) {
+UnchangedOr<PrimExpr> CanonicalSimplifier::Impl::Mutate_(const prim::ModNode* op,
+ InplaceMode inplace_mode) {
if (!IsIndexTypedExpr(op)) {
- return Rewriter::VisitExpr_(op);
+ return Rewriter::Mutate_(op, inplace_mode);
}
// normalize
- PrimExpr a = this->CanonicalMutate(op->a);
- PrimExpr b = this->CanonicalMutate(op->b);
+ PrimExpr a = DirectMutate(op->a, inplace_mode).ValueOrUnchanged(op->a);
+ PrimExpr b = DirectMutate(op->b, inplace_mode).ValueOrUnchanged(op->b);
// const folding
- if (auto const_res = TryConstFold<prim::Mod>(a, b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Mod>(a, b)) return *std::move(const_res);
PVar<IntImm> c1;
// x % c1
@@ -1197,7 +1225,7 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::ModNode* op) {
if (cbound->min_value >= 0 && cbound->min_value - psum->base + new_base >= 0) {
SumExpr sum_expr = a.as_or_throw<SumExpr>();
sum_expr.CopyOnWrite()->base = new_base;
- return SplitModConst(ToSplitExpr(std::move(sum_expr)), cval, kTruncDiv);
+ return SplitModConst(ToSplitExpr(std::move(sum_expr)), cval, kTruncDiv, inplace_mode);
}
} else {
// if a >= 0 && a < cval, then result == 0
@@ -1206,7 +1234,7 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::ModNode* op) {
return a;
}
}
- return SplitModConst(ToSplitExpr(std::move(a)), cval, kTruncDiv);
+ return SplitModConst(ToSplitExpr(std::move(a)), cval, kTruncDiv, inplace_mode);
}
// normal path
a = Normalize(a);
@@ -1219,22 +1247,24 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::ModNode* op) {
}
if (op->a.same_as(a) && op->b.same_as(b)) {
- return ffi::GetRef<PrimExpr>(op);
+ // Keep the original expression; it needs no replacement normalization.
+ return ffi::Unchanged();
} else {
return prim::Mod(a, b);
}
}
-Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::FloorModNode* op) {
+UnchangedOr<PrimExpr> CanonicalSimplifier::Impl::Mutate_(const prim::FloorModNode* op,
+ InplaceMode inplace_mode) {
if (!IsIndexTypedExpr(op)) {
- return Rewriter::VisitExpr_(op);
+ return Rewriter::Mutate_(op, inplace_mode);
}
// normalize
- PrimExpr a = this->CanonicalMutate(op->a);
- PrimExpr b = this->CanonicalMutate(op->b);
+ PrimExpr a = DirectMutate(op->a, inplace_mode).ValueOrUnchanged(op->a);
+ PrimExpr b = DirectMutate(op->b, inplace_mode).ValueOrUnchanged(op->b);
// const folding
- if (auto const_res = TryConstFold<prim::FloorMod>(a, b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::FloorMod>(a, b)) return *std::move(const_res);
PVar<IntImm> c1;
// x % c1
@@ -1263,7 +1293,7 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::FloorModNode* op) {
int64_t new_base = floormod(psum->base, cval);
SumExpr sum_expr = std::move(a).as_or_throw<SumExpr>();
sum_expr.CopyOnWrite()->base = new_base;
- return SplitModConst(ToSplitExpr(std::move(sum_expr)), cval, kFloorDiv);
+ return SplitModConst(ToSplitExpr(std::move(sum_expr)), cval, kFloorDiv, inplace_mode);
} else {
// if a >= 0 && a < cval, then result == a
auto cbound = analyzer_->const_int_bound(Normalize(a));
@@ -1271,7 +1301,7 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::FloorModNode* op) {
return a;
}
}
- return SplitModConst(ToSplitExpr(std::move(a)), cval, kFloorDiv);
+ return SplitModConst(ToSplitExpr(std::move(a)), cval, kFloorDiv, inplace_mode);
}
// normal path
a = Normalize(a);
@@ -1284,22 +1314,24 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::FloorModNode* op) {
}
if (op->a.same_as(a) && op->b.same_as(b)) {
- return ffi::GetRef<PrimExpr>(op);
+ // Keep the original expression; it needs no replacement normalization.
+ return ffi::Unchanged();
} else {
return prim::FloorMod(a, b);
}
}
-Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::CastNode* op) {
+UnchangedOr<PrimExpr> CanonicalSimplifier::Impl::Mutate_(const prim::CastNode* op,
+ InplaceMode inplace_mode) {
// The cast reasoning below is index-centric; for unsigned operands it runs
// only under the caller's no-overflow assertion (uint_as_index).
if (!IsIndexTypedExpr(op) &&
!(uint_as_index::Enabled() && op->ExprNode::ty.as<PrimTypeNode>()->dtype.code ==
static_cast<uint8_t>(DLDataTypeCode::kDLUInt))) {
- return Rewriter::VisitExpr_(op);
+ return Rewriter::Mutate_(op, inplace_mode);
}
// normalize
- PrimExpr value = this->CanonicalMutate(op->value);
+ PrimExpr value = DirectMutate(op->value, inplace_mode).ValueOrUnchanged(op->value);
// PushCastToChildren
if (value.as<SumExprNode>()) {
SumExpr se = value.as_or_throw<SumExpr>();
@@ -1315,12 +1347,14 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::CastNode* op) {
return se;
}
}
- return Rewriter::VisitExpr_(op);
+ return Rewriter::Mutate_(op, inplace_mode);
}
-Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::LTNode* op) {
+UnchangedOr<PrimExpr> CanonicalSimplifier::Impl::Mutate_(const prim::LTNode* op,
+ InplaceMode inplace_mode) {
// First convert a < b into a - b < 0
- PrimExpr expr = this->CanonicalMutate(op->a - op->b);
+ PrimExpr difference = op->a - op->b;
+ PrimExpr expr = DirectMutate(difference, inplace_mode).ValueOrUnchanged(std::move(difference));
// Case: x0 * s0 + x1 * s1 + ... + xn + c < 0, let d = gcd(s0, s1, ..., s{n-1}, c)
// 1. if can prove 0 <= xn < d, then we can simplify
// the expression to x0 * (s0/d) + x1 * (s1/d) + ... + x{n-1} * (s{n-1}/d) < c/d,
@@ -1340,7 +1374,7 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::LTNode* op) {
}
// Skip if gcd == 1 or all s_n are 1
if (!has_non_one_scale || gcd <= 1) {
- return Rewriter::VisitExpr_(op);
+ return Rewriter::Mutate_(op, inplace_mode);
}
SumExpr divisible, extra;
SeparateDivisibleParts(lhs, gcd, &divisible, &extra);
@@ -1351,7 +1385,8 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::LTNode* op) {
this->analyzer_->CanProve(normal_extra >= IntImm(dtype, 0))) {
// Case 1. 0 <= xn < d
divisible.CopyOnWrite()->DivideBy(gcd);
- return Rewriter::VisitExpr(divisible->Normalize() < IntImm(dtype, 0));
+ PrimExpr normalized = divisible->Normalize() < IntImm(dtype, 0);
+ return DirectMutate(normalized, inplace_mode).ValueOrUnchanged(std::move(normalized));
} else if (extra->args.size() == 1 && extra->args[0]->scale == 1 &&
extra->args[0]->upper_factor != ConstIntBoundNode::kPosInf &&
extra->args[0]->upper_factor % (gcd * extra->args[0]->lower_factor) == 0) {
@@ -1368,24 +1403,27 @@ Expr CanonicalSimplifier::Impl::VisitExpr_(const prim::LTNode* op) {
int64_t lower_factor = gcd * extra->args[0]->lower_factor;
PrimExpr extra_expr = floormod(floordiv(split_expr->index, lower_factor),
floordiv(split_expr->upper_factor, lower_factor));
- return Rewriter::VisitExpr(divisible->Normalize() + extra_expr < IntImm(dtype, 0));
+ PrimExpr normalized = divisible->Normalize() + extra_expr < IntImm(dtype, 0);
+ // Unchanged belongs to the new comparison, not the original input.
+ return DirectMutate(normalized, inplace_mode).ValueOrUnchanged(std::move(normalized));
}
}
- return Rewriter::VisitExpr_(op);
+ return Rewriter::Mutate_(op, inplace_mode);
}
PrimExpr CanonicalSimplifier::operator()(const PrimExpr& expr) {
- return impl_->CanonicalSimplify(expr);
+ return impl_->Mutate(expr).ValueOrUnchanged(expr);
}
void CanonicalSimplifier::Update(const Var& var, const PrimExpr& info, bool override) {
impl_->Update(var, info, override);
}
-CanonicalSimplifier::CanonicalSimplifier(AnalyzerObj* parent) : impl_(new Impl(parent)) {}
+CanonicalSimplifier::CanonicalSimplifier(AnalyzerObj* parent)
+ : impl_(ffi::make_object<Impl>(parent)) {}
-CanonicalSimplifier::~CanonicalSimplifier() { delete impl_; }
+CanonicalSimplifier::~CanonicalSimplifier() = default;
void CanonicalSimplifier::CopyFrom(const CanonicalSimplifier& other) {
impl_->CopyFrom(*other.impl_);
diff --git a/src/arith/const_int_bound.cc b/src/arith/const_int_bound.cc
index f10317a4c4..1e4670feae 100644
--- a/src/arith/const_int_bound.cc
+++ b/src/arith/const_int_bound.cc
@@ -24,10 +24,10 @@
#include <tvm/ffi/cast.h>
#include <tvm/ffi/function.h>
#include <tvm/ffi/reflection/registry.h>
+#include <tvm/ir/expr_functor.h>
#include <tvm/ir/op.h>
#include <tvm/ir/prim/builtin.h>
#include <tvm/tirx/builtin.h>
-#include <tvm/tirx/expr_functor.h>
#include <algorithm>
#include <optional>
@@ -93,7 +93,9 @@ struct ConstIntBoundAnalyzer::Entry {
}
};
-class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::Entry(const Expr&)> {
+class ConstIntBoundAnalyzer::Impl
+ : public ffi::Object,
+ public tvm::ExprFunctor<ConstIntBoundAnalyzer::Entry(const Expr&)> {
public:
explicit Impl(AnalyzerObj* parent) : parent_(parent) {}
/*! \brief additional bound info about expr in bound */
@@ -110,8 +112,8 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
bool IsBound(const Var& var) const { return var_map_.find(var) != var_map_.end(); }
void Bind(const Var& var, const Range& range, bool allow_override) {
- Entry a = VisitExpr(range->min);
- Entry b = VisitExpr(range->extent);
+ Entry a = Dispatch(range->min);
+ Entry b = Dispatch(range->extent);
Entry ret;
ret.min_value = a.min_value;
ret.max_value = InfAwareAdd(a.max_value, InfAwareAdd(b.max_value, -1));
@@ -132,16 +134,16 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
var_map_[var] = info;
}
- Entry VisitExpr_(const prim::LetNode* op) final {
+ Entry Dispatch_(const prim::LetNode* op) final {
auto it = var_map_.find(op->var);
// if the var has not been binded, update the info.
if (it == var_map_.end()) {
- var_map_[op->var] = this->VisitExpr(op->value);
- Entry ret = VisitExpr(op->body);
+ var_map_[op->var] = this->Dispatch(op->value);
+ Entry ret = Dispatch(op->body);
var_map_.erase(op->var);
return ret;
} else {
- return VisitExpr(op->body);
+ return Dispatch(op->body);
}
}
@@ -150,13 +152,13 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
}
// Override visitor behaviors
- Entry VisitExprDefault_(const ffi::Object* op) final {
+ Entry DispatchDefault_(const ffi::Object* op) final {
return Everything(static_cast<const ExprNode*>(op)->ty.as_or_throw<PrimType>());
}
- Entry VisitExpr(const Expr& expr) final {
+ Entry Dispatch(const Expr& expr) final {
PrimExpr prim_expr = expr.as_or_throw<PrimExpr>();
- Entry res = ExprFunctor::VisitExpr(expr);
+ Entry res = ExprFunctor::Dispatch(expr);
tirx::ExprDeepEqual equal;
// a linear search over additional info
// assume we won't have a lot of conditions
@@ -180,19 +182,19 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
return res;
}
- Entry VisitExpr_(const prim::RampNode* op) final {
+ Entry Dispatch_(const prim::RampNode* op) final {
// op = {base + i * stride | 0 <= i < lanes}
// Entry(op) = Union(Entry(base + i * stride) | 0 <= i < lanes)
// Note that `base + i * stride` is linear w.r.t. `i`
// Entry(op) = Union(Entry(base + i * stride) | i = 0, i = lanes-1)
- Entry a = VisitExpr(op->base);
- Entry b = VisitExpr(op->base + (op->lanes - 1) * op->stride);
+ Entry a = Dispatch(op->base);
+ Entry b = Dispatch(op->base + (op->lanes - 1) * op->stride);
return Union(a, b);
}
- Entry VisitExpr_(const prim::BroadcastNode* op) final { return VisitExpr(op->value); }
+ Entry Dispatch_(const prim::BroadcastNode* op) final { return Dispatch(op->value); }
- Entry VisitExpr_(const prim::CastNode* op) final {
+ Entry Dispatch_(const prim::CastNode* op) final {
Entry a;
// int(ceil(log2(cast(n,"float64")))) is used as the
@@ -201,7 +203,7 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
if (auto opt = FindCeilLog2Arg(op)) {
a = CeilLog2Bounds(opt.value());
} else {
- a = VisitExpr(op->value);
+ a = Dispatch(op->value);
}
Entry b = Everything(op->ty.as_or_throw<PrimType>());
@@ -233,11 +235,11 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
return divisor;
}
- Entry VisitExpr_(const IntImmNode* op) final { return MakeBound(op->value, op->value); }
+ Entry Dispatch_(const IntImmNode* op) final { return MakeBound(op->value, op->value); }
- Entry VisitExpr_(const prim::AddNode* op) final {
- Entry a = VisitExpr(op->a);
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::AddNode* op) final {
+ Entry a = Dispatch(op->a);
+ Entry b = Dispatch(op->b);
Entry ret;
ret.min_value = InfAwareAdd(a.min_value, b.min_value);
ret.max_value = InfAwareAdd(a.max_value, b.max_value);
@@ -245,9 +247,9 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
return ret;
}
- Entry VisitExpr_(const prim::SubNode* op) final {
- Entry a = VisitExpr(op->a);
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::SubNode* op) final {
+ Entry a = Dispatch(op->a);
+ Entry b = Dispatch(op->b);
Entry ret;
ret.min_value = InfAwareAdd(a.min_value, -b.max_value);
ret.max_value = InfAwareAdd(a.max_value, -b.min_value);
@@ -255,21 +257,21 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
return ret;
}
- Entry VisitExpr_(const prim::MulNode* op) final {
- Entry a = VisitExpr(op->a);
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::MulNode* op) final {
+ Entry a = Dispatch(op->a);
+ Entry b = Dispatch(op->b);
return BinaryOpBoundary(a, b, InfAwareMul);
}
- Entry VisitExpr_(const prim::DivNode* op) final {
- Entry a = VisitExpr(op->a);
- Entry b = AssumeNoZeroDivisor(VisitExpr(op->b));
+ Entry Dispatch_(const prim::DivNode* op) final {
+ Entry a = Dispatch(op->a);
+ Entry b = AssumeNoZeroDivisor(Dispatch(op->b));
return HandleDivision(a, b, op->ty.as_or_throw<PrimType>(), InfAwareDiv);
}
- Entry VisitExpr_(const prim::ModNode* op) final {
- Entry a = VisitExpr(op->a);
- Entry b = AssumeNoZeroDivisor(VisitExpr(op->b));
+ Entry Dispatch_(const prim::ModNode* op) final {
+ Entry a = Dispatch(op->a);
+ Entry b = AssumeNoZeroDivisor(Dispatch(op->b));
if (b.min_value > 0) {
int64_t b_max_cap = InfAwareAdd(b.max_value, -1);
@@ -357,13 +359,13 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
}
}
- Entry VisitExpr_(const prim::FloorDivNode* op) final {
- Entry a = VisitExpr(op->a);
- Entry b = AssumeNoZeroDivisor(VisitExpr(op->b));
+ Entry Dispatch_(const prim::FloorDivNode* op) final {
+ Entry a = Dispatch(op->a);
+ Entry b = AssumeNoZeroDivisor(Dispatch(op->b));
return HandleDivision(a, b, op->ty.as_or_throw<PrimType>(), InfAwareFloorDiv);
}
- Entry VisitExpr_(const prim::FloorModNode* op) final {
+ Entry Dispatch_(const prim::FloorModNode* op) final {
/* let a / b = x + y, where x is integer, y \in [0, 1)
* floormod(a, b) = a - floordiv(a, b) * b
* floordiv(a, b) = x
@@ -381,8 +383,8 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
* min(0, b_min + 1) <= b * y <= max(0, b_max - 1)
* That is, min(0, b_min + 1) <= floormod(a, b) <= max(0, b_max - 1)
*/
- Entry a = VisitExpr(op->a);
- Entry b = AssumeNoZeroDivisor(VisitExpr(op->b));
+ Entry a = Dispatch(op->a);
+ Entry b = AssumeNoZeroDivisor(Dispatch(op->b));
if (b.min_value > 0) {
int64_t b_max_cap = InfAwareAdd(b.max_value, -1);
@@ -445,31 +447,31 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
}
}
- Entry VisitExpr_(const prim::MinNode* op) final {
- Entry a = VisitExpr(op->a);
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::MinNode* op) final {
+ Entry a = Dispatch(op->a);
+ Entry b = Dispatch(op->b);
Entry ret;
ret.min_value = std::min(a.min_value, b.min_value);
ret.max_value = std::min(a.max_value, b.max_value);
return ret;
}
- Entry VisitExpr_(const prim::MaxNode* op) final {
- Entry a = VisitExpr(op->a);
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::MaxNode* op) final {
+ Entry a = Dispatch(op->a);
+ Entry b = Dispatch(op->b);
Entry ret;
ret.min_value = std::max(a.min_value, b.min_value);
ret.max_value = std::max(a.max_value, b.max_value);
return ret;
}
- Entry VisitExpr_(const prim::SelectNode* op) final {
- Entry a = VisitExpr(op->true_value);
- Entry b = VisitExpr(op->false_value);
+ Entry Dispatch_(const prim::SelectNode* op) final {
+ Entry a = Dispatch(op->true_value);
+ Entry b = Dispatch(op->false_value);
return Union(a, b);
}
- Entry VisitExpr_(const CallNode* op) final {
+ Entry Dispatch_(const CallNode* op) final {
// only special handle >> and & which can be
// used for index calculation.
@@ -484,7 +486,7 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
}
}
- Entry VisitExpr_(const VarNode* op) final {
+ Entry Dispatch_(const VarNode* op) final {
Var v = ffi::GetRef<Var>(op);
auto it = var_map_.find(v);
if (it != var_map_.end()) {
@@ -495,8 +497,8 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
}
Entry VisitLeftShift(const CallNode* op) {
- Entry a = VisitExpr(op->args[0].as_or_throw<PrimExpr>());
- Entry b = VisitExpr(op->args[1].as_or_throw<PrimExpr>());
+ Entry a = Dispatch(op->args[0].as_or_throw<PrimExpr>());
+ Entry b = Dispatch(op->args[1].as_or_throw<PrimExpr>());
if (a.min_value < 0 || b.min_value < 0) {
// If either operand can negative, we may run into undefined
@@ -509,14 +511,14 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
}
Entry VisitRightShift(const CallNode* op) {
- Entry a = VisitExpr(op->args[0].as_or_throw<PrimExpr>());
- Entry b = VisitExpr(op->args[1].as_or_throw<PrimExpr>());
+ Entry a = Dispatch(op->args[0].as_or_throw<PrimExpr>());
+ Entry b = Dispatch(op->args[1].as_or_throw<PrimExpr>());
return BinaryOpBoundary(a, b, InfAwareRightShift);
}
Entry VisitBitwiseAnd(const CallNode* op) {
- Entry a = VisitExpr(op->args[0].as_or_throw<PrimExpr>());
- Entry b = VisitExpr(op->args[1].as_or_throw<PrimExpr>());
+ Entry a = Dispatch(op->args[0].as_or_throw<PrimExpr>());
+ Entry b = Dispatch(op->args[1].as_or_throw<PrimExpr>());
// handle positive index case.
if (a.min_value >= 0 && b.min_value >= 0) {
return MakeBound(0, std::min(a.max_value, b.max_value));
@@ -874,7 +876,7 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
int64_t val = std::ceil(std::log2(as_float->value));
return MakeBound(val, val);
} else {
- Entry arg_bounds = VisitExpr(arg);
+ Entry arg_bounds = Dispatch(arg);
return MakeBound(std::ceil(std::log2(arg_bounds.min_value)),
std::ceil(std::log2(arg_bounds.max_value)));
}
@@ -882,13 +884,13 @@ class ConstIntBoundAnalyzer::Impl : public ExprFunctor<ConstIntBoundAnalyzer::En
};
ConstIntBound ConstIntBoundAnalyzer::operator()(const PrimExpr& expr) const {
- Entry ret = impl_->VisitExpr(expr);
+ Entry ret = impl_->Dispatch(expr);
return ConstIntBound(ret.min_value, ret.max_value);
}
ConstIntBound ConstIntBoundAnalyzer::operator()(const PrimExpr& expr, BoundMapType* bound) {
impl_->bound_ = bound;
- Entry ret = impl_->VisitExpr(expr);
+ Entry ret = impl_->Dispatch(expr);
impl_->bound_ = nullptr;
return ConstIntBound(ret.min_value, ret.max_value);
}
@@ -907,9 +909,10 @@ std::function<void()> ConstIntBoundAnalyzer::EnterConstraint(const PrimExpr& con
return impl_->EnterConstraint(constraint);
}
-ConstIntBoundAnalyzer::ConstIntBoundAnalyzer(AnalyzerObj* parent) : impl_(new Impl(parent)) {}
+ConstIntBoundAnalyzer::ConstIntBoundAnalyzer(AnalyzerObj* parent)
+ : impl_(ffi::make_object<Impl>(parent)) {}
-ConstIntBoundAnalyzer::~ConstIntBoundAnalyzer() { delete impl_; }
+ConstIntBoundAnalyzer::~ConstIntBoundAnalyzer() = default;
void ConstIntBoundAnalyzer::CopyFrom(const ConstIntBoundAnalyzer& other) {
impl_->CopyFrom(*other.impl_);
diff --git a/src/arith/constraint_helpers.h b/src/arith/constraint_helpers.h
new file mode 100644
index 0000000000..0af659c548
--- /dev/null
+++ b/src/arith/constraint_helpers.h
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file arith/constraint_helpers.h
+ * \brief Shared constraint derivation for arithmetic and statement simplifiers.
+ */
+#ifndef TVM_ARITH_CONSTRAINT_HELPERS_H_
+#define TVM_ARITH_CONSTRAINT_HELPERS_H_
+
+#include <tvm/arith/analyzer.h>
+#include <tvm/ir/prim/builtin.h>
+#include <tvm/ir/prim/expr.h>
+#include <tvm/ir/with_context.h>
+
+#include <cstdint>
+#include <vector>
+
+namespace tvm {
+namespace arith {
+namespace detail {
+
+enum class CompareKind { kEQ, kLT, kLE, kGT, kGE };
+
+inline bool TryGetIntImm(const PrimExpr& expr, int64_t* value) {
+ if (const auto* imm = expr.as<IntImmNode>()) {
+ *value = imm->value;
+ return true;
+ }
+ return false;
+}
+
+inline void AppendFloorDivConstraints(const prim::FloorDivNode* div, int64_t value,
+ CompareKind kind, std::vector<PrimExpr>* out) {
+ int64_t divisor_value = 0;
+ if (!TryGetIntImm(div->b, &divisor_value) || divisor_value <= 0) return;
+
+ PrimType dtype = div->a.ty();
+ PrimExpr divisor = IntImm(dtype, divisor_value);
+ PrimExpr k = IntImm(dtype, value);
+ PrimExpr lo = k * divisor;
+ PrimExpr hi = (k + IntImm(dtype, 1)) * divisor;
+
+ switch (kind) {
+ case CompareKind::kEQ:
+ out->push_back(div->a >= lo);
+ out->push_back(div->a < hi);
+ break;
+ case CompareKind::kLT:
+ out->push_back(div->a < lo);
+ break;
+ case CompareKind::kLE:
+ out->push_back(div->a < hi);
+ break;
+ case CompareKind::kGT:
+ out->push_back(div->a >= hi);
+ break;
+ case CompareKind::kGE:
+ out->push_back(div->a >= lo);
+ break;
+ }
+}
+
+inline CompareKind InvertCompare(CompareKind kind) {
+ switch (kind) {
+ case CompareKind::kEQ:
+ return CompareKind::kEQ;
+ case CompareKind::kLT:
+ return CompareKind::kGT;
+ case CompareKind::kLE:
+ return CompareKind::kGE;
+ case CompareKind::kGT:
+ return CompareKind::kLT;
+ case CompareKind::kGE:
+ return CompareKind::kLE;
+ }
+ return CompareKind::kEQ;
+}
+
+inline void CollectFloorDivConstraintsFromCompare(const PrimExpr& lhs, const PrimExpr& rhs,
+ CompareKind kind, std::vector<PrimExpr>* out) {
+ int64_t value = 0;
+ if (const auto* div = lhs.as<prim::FloorDivNode>()) {
+ if (TryGetIntImm(rhs, &value)) AppendFloorDivConstraints(div, value, kind, out);
+ }
+ if (const auto* div = rhs.as<prim::FloorDivNode>()) {
+ if (TryGetIntImm(lhs, &value)) {
+ AppendFloorDivConstraints(div, value, InvertCompare(kind), out);
+ }
+ }
+}
+
+inline void CollectDerivedConstraintFacts(const PrimExpr& condition, std::vector<PrimExpr>* out) {
+ if (const auto* and_node = condition.as<prim::AndNode>()) {
+ CollectDerivedConstraintFacts(and_node->a, out);
+ CollectDerivedConstraintFacts(and_node->b, out);
+ return;
+ }
+ if (const auto* call = condition.as<CallNode>()) {
+ if (call->op.same_as(prim::builtin::bitwise_and()) && call->args.size() == 2) {
+ PrimExpr lhs = call->args[0].as_or_throw<PrimExpr>();
+ PrimExpr rhs = call->args[1].as_or_throw<PrimExpr>();
+ if (lhs.ty().MatchesElementType(DLDataTypeCode::kDLBool, 8) &&
+ rhs.ty().MatchesElementType(DLDataTypeCode::kDLBool, 8)) {
+ CollectDerivedConstraintFacts(lhs, out);
+ CollectDerivedConstraintFacts(rhs, out);
+ return;
+ }
+ }
+ }
+ if (const auto* eq = condition.as<prim::EQNode>()) {
+ CollectFloorDivConstraintsFromCompare(eq->a, eq->b, CompareKind::kEQ, out);
+ } else if (const auto* lt = condition.as<prim::LTNode>()) {
+ CollectFloorDivConstraintsFromCompare(lt->a, lt->b, CompareKind::kLT, out);
+ } else if (const auto* le = condition.as<prim::LENode>()) {
+ CollectFloorDivConstraintsFromCompare(le->a, le->b, CompareKind::kLE, out);
+ } else if (const auto* gt = condition.as<prim::GTNode>()) {
+ CollectFloorDivConstraintsFromCompare(gt->a, gt->b, CompareKind::kGT, out);
+ } else if (const auto* ge = condition.as<prim::GENode>()) {
+ CollectFloorDivConstraintsFromCompare(ge->a, ge->b, CompareKind::kGE, out);
+ }
+}
+
+inline void EnterConstraintFacts(WithGroup<ConstraintContext>* constraints, AnalyzerObj* analyzer,
+ const PrimExpr& condition) {
+ Analyzer analyzer_ref = ffi::GetRef<Analyzer>(analyzer);
+ constraints->Emplace(analyzer_ref, condition);
+ std::vector<PrimExpr> derived;
+ CollectDerivedConstraintFacts(condition, &derived);
+ for (const PrimExpr& fact : derived) {
+ constraints->Emplace(analyzer_ref, fact);
+ }
+}
+
+} // namespace detail
+} // namespace arith
+} // namespace tvm
+
+#endif // TVM_ARITH_CONSTRAINT_HELPERS_H_
diff --git a/src/arith/detect_linear_equation.cc b/src/arith/detect_linear_equation.cc
index a2a8ffdd5d..fb12f54674 100644
--- a/src/arith/detect_linear_equation.cc
+++ b/src/arith/detect_linear_equation.cc
@@ -25,9 +25,9 @@
#include <tvm/ffi/extra/structural_visit.h>
#include <tvm/ffi/function.h>
#include <tvm/ffi/reflection/registry.h>
+#include <tvm/ir/expr_functor.h>
#include <tvm/ir/prim/expr.h>
#include <tvm/tirx/analysis.h>
-#include <tvm/tirx/expr_functor.h>
#include <tvm/tirx/op.h>
namespace tvm {
@@ -46,12 +46,12 @@ struct IntervalEntry {
PrimExpr max_value;
};
-class LinearEqDetector : public ExprFunctor<LinearEqEntry(const Expr&, const PrimExpr&)> {
+class LinearEqDetector : public tvm::ExprFunctor<LinearEqEntry(const Expr&, const PrimExpr&)> {
public:
explicit LinearEqDetector(PrimVar var) : var_(var) {}
bool Detect(const PrimExpr& e, LinearEqEntry* ret) {
- *ret = VisitExpr(e, e);
+ *ret = Dispatch(e, e);
if (fail_) return false;
if (!ret->base.defined()) {
ret->base = IntImm(var_->ty.as_or_throw<PrimType>(), 0);
@@ -62,30 +62,30 @@ class LinearEqDetector : public ExprFunctor<LinearEqEntry(const Expr&, const Pri
return true;
}
- LinearEqEntry VisitExpr_(const prim::AddNode* op, const PrimExpr& e) final {
+ LinearEqEntry Dispatch_(const prim::AddNode* op, const PrimExpr& e) final {
if (fail_) return LinearEqEntry();
- LinearEqEntry a = VisitExpr(op->a, op->a);
- LinearEqEntry b = VisitExpr(op->b, op->b);
+ LinearEqEntry a = Dispatch(op->a, op->a);
+ LinearEqEntry b = Dispatch(op->b, op->b);
LinearEqEntry ret;
ret.base = AddCombine(a.base, b.base);
ret.coeff = AddCombine(a.coeff, b.coeff);
return ret;
}
- LinearEqEntry VisitExpr_(const prim::SubNode* op, const PrimExpr& e) final {
+ LinearEqEntry Dispatch_(const prim::SubNode* op, const PrimExpr& e) final {
if (fail_) return LinearEqEntry();
- LinearEqEntry a = VisitExpr(op->a, op->a);
- LinearEqEntry b = VisitExpr(op->b, op->b);
+ LinearEqEntry a = Dispatch(op->a, op->a);
+ LinearEqEntry b = Dispatch(op->b, op->b);
LinearEqEntry ret;
ret.base = SubCombine(a.base, b.base);
ret.coeff = SubCombine(a.coeff, b.coeff);
return ret;
}
- LinearEqEntry VisitExpr_(const prim::MulNode* op, const PrimExpr& e) final {
+ LinearEqEntry Dispatch_(const prim::MulNode* op, const PrimExpr& e) final {
if (fail_) return LinearEqEntry();
- LinearEqEntry a = VisitExpr(op->a, op->a);
- LinearEqEntry b = VisitExpr(op->b, op->b);
+ LinearEqEntry a = Dispatch(op->a, op->a);
+ LinearEqEntry b = Dispatch(op->b, op->b);
if (a.coeff.defined()) {
std::swap(a, b);
}
@@ -98,7 +98,7 @@ class LinearEqDetector : public ExprFunctor<LinearEqEntry(const Expr&, const Pri
ret.coeff = MulCombine(a.base, b.coeff);
return ret;
}
- LinearEqEntry VisitExpr_(const VarNode* op, const PrimExpr& e) final {
+ LinearEqEntry Dispatch_(const VarNode* op, const PrimExpr& e) final {
LinearEqEntry ret;
if (op == var_.get()) {
PrimType dtype = op->ty.as_or_throw<PrimType>();
@@ -108,7 +108,7 @@ class LinearEqDetector : public ExprFunctor<LinearEqEntry(const Expr&, const Pri
}
return ret;
}
- LinearEqEntry VisitExprDefault_(const ffi::Object* op, const PrimExpr& e) final {
+ LinearEqEntry DispatchDefault_(const ffi::Object* op, const PrimExpr& e) final {
if (fail_) return LinearEqEntry();
auto walkfn = [this](const Var& var) -> ffi::Expected<ffi::WalkResult> {
return var.get() == var_.get() ? ffi::WalkResult::Interrupt(ffi::VisitInterrupt(var))
diff --git a/src/arith/int_constraints.cc b/src/arith/int_constraints.cc
index a16553e8b2..1d7bdf90e0 100644
--- a/src/arith/int_constraints.cc
+++ b/src/arith/int_constraints.cc
@@ -27,7 +27,6 @@
#include <tvm/ffi/function.h>
#include <tvm/ffi/reflection/registry.h>
#include <tvm/ir/prim/expr.h>
-#include <tvm/tirx/expr_functor.h>
#include <tvm/tirx/op.h>
#include <algorithm>
@@ -263,7 +262,7 @@ IntConstraintsTransform IntConstraintsTransform::operator+(
Analyzer ana_first;
ana_first->Bind(operator->()->src->ranges);
auto f_dst_to_src = [this](const Var& var) -> ffi::Expected<ffi::UnchangedOr<ffi::Any>> {
- if (auto repl = operator->()->dst_to_src.Get(var)) return ffi::Any(*std::move(repl));
+ if (auto repl = operator->()->dst_to_src.Get(var)) return *std::move(repl);
return ffi::Unchanged();
};
for (auto p : other->dst_to_src) {
@@ -275,7 +274,7 @@ IntConstraintsTransform IntConstraintsTransform::operator+(
Analyzer ana_second;
ana_second->Bind(other->dst->ranges);
auto f_src_to_dst = [&other](const Var& var) -> ffi::Expected<ffi::UnchangedOr<ffi::Any>> {
- if (auto repl = other->src_to_dst.Get(var)) return ffi::Any(*std::move(repl));
+ if (auto repl = other->src_to_dst.Get(var)) return *std::move(repl);
return ffi::Unchanged();
};
for (auto p : operator->()->src_to_dst) {
diff --git a/src/arith/int_set.cc b/src/arith/int_set.cc
index 9063c0b620..58f289540d 100644
--- a/src/arith/int_set.cc
+++ b/src/arith/int_set.cc
@@ -27,9 +27,9 @@
#include <tvm/ffi/extra/structural_visit.h>
#include <tvm/ffi/function.h>
#include <tvm/ffi/reflection/registry.h>
+#include <tvm/ir/expr_functor.h>
#include <tvm/ir/prim/expr.h>
#include <tvm/runtime/logging.h>
-#include <tvm/tirx/expr_functor.h>
#include <tvm/tirx/op.h>
#include <algorithm>
@@ -401,7 +401,7 @@ using namespace tirx;
// Simplified version of int set evaluator that operates on IntervalSet
// We might use better set analysis in the future to replace the intervalset.
-class IntervalSetEvaluator : public ExprFunctor<IntervalSet(const Expr&)> {
+class IntervalSetEvaluator : public tvm::ExprFunctor<IntervalSet(const Expr&)> {
public:
IntervalSetEvaluator(AnalyzerObj* analyzer, const ffi::Map<Var, IntSet>& dom_map,
const std::vector<std::pair<Var, IntSet>>* dom_constraints = nullptr,
@@ -411,7 +411,7 @@ class IntervalSetEvaluator : public ExprFunctor<IntervalSet(const Expr&)> {
dom_constraints_(dom_constraints),
eval_vec_(eval_vec) {}
- IntervalSet Eval(const PrimExpr& val) { return this->VisitExpr(val); }
+ IntervalSet Eval(const PrimExpr& val) { return this->Dispatch(val); }
// evaluate and relax the set
IntervalSet Eval(IntervalSet val) {
// avoid recursive indefinite recursive expansion.
@@ -424,11 +424,11 @@ class IntervalSetEvaluator : public ExprFunctor<IntervalSet(const Expr&)> {
return IntervalSet(min_set->min_value, max_set->max_value);
}
- IntervalSet VisitExpr_(const IntImmNode* op) final {
+ IntervalSet Dispatch_(const IntImmNode* op) final {
return IntervalSet::SinglePoint(ffi::GetRef<PrimExpr>(op));
}
- IntervalSet VisitExpr_(const VarNode* op) final {
+ IntervalSet Dispatch_(const VarNode* op) final {
Var var = ffi::GetRef<Var>(op);
auto it = dom_map_.find(var);
@@ -486,45 +486,45 @@ class IntervalSetEvaluator : public ExprFunctor<IntervalSet(const Expr&)> {
return relaxed;
}
- IntervalSet VisitExpr_(const prim::AddNode* op) final { return VisitBinaryExpr_<prim::Add>(op); }
+ IntervalSet Dispatch_(const prim::AddNode* op) final { return VisitBinaryExpr_<prim::Add>(op); }
- IntervalSet VisitExpr_(const prim::SubNode* op) final { return VisitBinaryExpr_<prim::Sub>(op); }
+ IntervalSet Dispatch_(const prim::SubNode* op) final { return VisitBinaryExpr_<prim::Sub>(op); }
- IntervalSet VisitExpr_(const prim::MulNode* op) final { return VisitBinaryExpr_<prim::Mul>(op); }
+ IntervalSet Dispatch_(const prim::MulNode* op) final { return VisitBinaryExpr_<prim::Mul>(op); }
- IntervalSet VisitExpr_(const prim::DivNode* op) final { return VisitBinaryExpr_<prim::Div>(op); }
+ IntervalSet Dispatch_(const prim::DivNode* op) final { return VisitBinaryExpr_<prim::Div>(op); }
- IntervalSet VisitExpr_(const prim::ModNode* op) final { return VisitBinaryExpr_<prim::Mod>(op); }
+ IntervalSet Dispatch_(const prim::ModNode* op) final { return VisitBinaryExpr_<prim::Mod>(op); }
- IntervalSet VisitExpr_(const prim::FloorDivNode* op) final {
+ IntervalSet Dispatch_(const prim::FloorDivNode* op) final {
return VisitBinaryExpr_<prim::FloorDiv>(op);
}
- IntervalSet VisitExpr_(const prim::FloorModNode* op) final {
+ IntervalSet Dispatch_(const prim::FloorModNode* op) final {
return VisitBinaryExpr_<prim::FloorMod>(op);
}
- IntervalSet VisitExpr_(const prim::MinNode* op) final { return VisitBinaryExpr_<prim::Min>(op); }
+ IntervalSet Dispatch_(const prim::MinNode* op) final { return VisitBinaryExpr_<prim::Min>(op); }
- IntervalSet VisitExpr_(const prim::MaxNode* op) final { return VisitBinaryExpr_<prim::Max>(op); }
+ IntervalSet Dispatch_(const prim::MaxNode* op) final { return VisitBinaryExpr_<prim::Max>(op); }
- IntervalSet VisitExpr_(const prim::EQNode* op) final { return VisitBinaryExpr_<prim::EQ>(op); }
+ IntervalSet Dispatch_(const prim::EQNode* op) final { return VisitBinaryExpr_<prim::EQ>(op); }
- IntervalSet VisitExpr_(const prim::NENode* op) final { return VisitBinaryExpr_<prim::NE>(op); }
+ IntervalSet Dispatch_(const prim::NENode* op) final { return VisitBinaryExpr_<prim::NE>(op); }
- IntervalSet VisitExpr_(const prim::LTNode* op) final { return VisitBinaryExpr_<prim::LT>(op); }
+ IntervalSet Dispatch_(const prim::LTNode* op) final { return VisitBinaryExpr_<prim::LT>(op); }
- IntervalSet VisitExpr_(const prim::LENode* op) final { return VisitBinaryExpr_<prim::LE>(op); }
+ IntervalSet Dispatch_(const prim::LENode* op) final { return VisitBinaryExpr_<prim::LE>(op); }
- IntervalSet VisitExpr_(const prim::GTNode* op) final { return VisitBinaryExpr_<prim::GT>(op); }
+ IntervalSet Dispatch_(const prim::GTNode* op) final { return VisitBinaryExpr_<prim::GT>(op); }
- IntervalSet VisitExpr_(const prim::GENode* op) final { return VisitBinaryExpr_<prim::GE>(op); }
+ IntervalSet Dispatch_(const prim::GENode* op) final { return VisitBinaryExpr_<prim::GE>(op); }
- IntervalSet VisitExpr_(const prim::AndNode* op) final { return VisitBinaryExpr_<prim::And>(op); }
+ IntervalSet Dispatch_(const prim::AndNode* op) final { return VisitBinaryExpr_<prim::And>(op); }
- IntervalSet VisitExpr_(const prim::OrNode* op) final { return VisitBinaryExpr_<prim::Or>(op); }
+ IntervalSet Dispatch_(const prim::OrNode* op) final { return VisitBinaryExpr_<prim::Or>(op); }
- IntervalSet VisitExpr_(const prim::RampNode* op) final {
+ IntervalSet Dispatch_(const prim::RampNode* op) final {
TVM_FFI_ICHECK(eval_vec_);
IntervalSet base = Eval(op->base);
PVar<IntImm> stride;
@@ -564,18 +564,18 @@ class IntervalSetEvaluator : public ExprFunctor<IntervalSet(const Expr&)> {
return IntervalSet::Everything();
}
- IntervalSet VisitExpr_(const prim::BroadcastNode* op) final {
+ IntervalSet Dispatch_(const prim::BroadcastNode* op) final {
TVM_FFI_ICHECK(eval_vec_);
- return VisitExpr(op->value);
+ return Dispatch(op->value);
}
- IntervalSet VisitExpr_(const prim::SelectNode* op) final {
+ IntervalSet Dispatch_(const prim::SelectNode* op) final {
IntervalSet true_set = this->Eval(op->true_value);
IntervalSet false_set = this->Eval(op->false_value);
return Union(analyzer_, false_set, true_set);
}
- IntervalSet VisitExpr_(const prim::CastNode* op) final {
+ IntervalSet Dispatch_(const prim::CastNode* op) final {
IntervalSet value_set = this->Eval(op->value);
// short cut for the int set.
if (value_set->min_value.same_as(value_set->max_value)) {
@@ -591,7 +591,7 @@ class IntervalSetEvaluator : public ExprFunctor<IntervalSet(const Expr&)> {
return IntervalSet(min_value, max_value);
}
- IntervalSet VisitExpr_(const TensorLoadNode* op) final {
+ IntervalSet Dispatch_(const TensorLoadNode* op) final {
PrimType op_ty = op->ty.as_or_throw<PrimType>();
if (!op_ty.MatchesCode(DLDataTypeCode::kDLInt, DLDataTypeCode::kDLUInt)) {
DLOG(WARNING) << "cannot evaluate set TensorLoad which loads from a " << op_ty->dtype
@@ -613,7 +613,7 @@ class IntervalSetEvaluator : public ExprFunctor<IntervalSet(const Expr&)> {
return IntervalSet::SinglePoint(ffi::GetRef<PrimExpr>(op));
}
- IntervalSet VisitExpr_(const CallNode* op) final {
+ IntervalSet Dispatch_(const CallNode* op) final {
if (op->op.same_as(prim::builtin::vscale())) {
PrimExpr call = ffi::GetRef<Call>(op).as_or_throw<PrimExpr>();
return IntervalSet(call, call);
@@ -621,7 +621,7 @@ class IntervalSetEvaluator : public ExprFunctor<IntervalSet(const Expr&)> {
return IntervalSet::Everything();
}
- IntervalSet VisitExprDefault_(const ffi::Object* op) final {
+ IntervalSet DispatchDefault_(const ffi::Object* op) final {
DLOG(WARNING) << "cannot evaluate set type " << op->GetTypeKey();
return IntervalSet::Everything();
}
@@ -1130,8 +1130,8 @@ class SubExprIntervalSetEvaluator : public IntervalSetEvaluator {
explicit SubExprIntervalSetEvaluator(AnalyzerObj* analyzer, const ffi::Map<Var, IntSet>& dom_map)
: IntervalSetEvaluator(analyzer, dom_map) {}
- IntervalSet VisitExpr(const Expr& n) final {
- IntervalSet ret = IntervalSetEvaluator::VisitExpr(n);
+ IntervalSet Dispatch(const Expr& n) final {
+ IntervalSet ret = IntervalSetEvaluator::Dispatch(n);
expr_map[n.as_or_throw<PrimExpr>()] = ret;
return ret;
}
diff --git a/src/arith/iter_affine_map.cc b/src/arith/iter_affine_map.cc
index 534b826d61..af52fb78d0 100644
--- a/src/arith/iter_affine_map.cc
+++ b/src/arith/iter_affine_map.cc
@@ -23,11 +23,12 @@
#include <tvm/arith/analyzer.h>
#include <tvm/arith/iter_affine_map.h>
#include <tvm/ffi/cast.h>
+#include <tvm/ffi/expected.h>
#include <tvm/ffi/extra/structural_visit.h>
#include <tvm/ffi/reflection/registry.h>
+#include <tvm/ir/expr_functor.h>
#include <tvm/ir/prim/expr.h>
#include <tvm/tirx/analysis.h>
-#include <tvm/tirx/expr_functor.h>
#include <tvm/tirx/op.h>
#include <utility>
@@ -170,9 +171,11 @@ struct IterMarkWithOffset {
};
/*! \brief Rewriter to rewrite PrimExpr to IterMapExpr when possible */
-class IterMapRewriter : public ExprMutator {
+class IterMapRewriter : public tvm::ExprMutator {
public:
- using Parent = ExprMutator;
+ using Parent = tvm::ExprMutator;
+ using Parent::Mutate;
+ using Parent::Mutate_;
explicit IterMapRewriter(AnalyzerObj* analyzer, const ffi::Map<PrimVar, Range>& input_iters,
IterMapLevel check_level, bool simplify_trivial_iterators,
@@ -204,20 +207,25 @@ class IterMapRewriter : public ExprMutator {
bool requires_padding() const { return requires_padding_; }
IterSumExpr Rewrite(const PrimExpr& expr) {
- return NormalizeToIterWithOffset(ToIterSumExpr(DirectMutate(expr)));
+ PrimExpr rewritten = DirectMutate(expr).ValueOrUnchanged(expr);
+ return NormalizeToIterWithOffset(ToIterSumExpr(rewritten));
}
IterSumExpr RewriteAndUpdatePadding(const PrimExpr& expr) {
- update_iterator_padding_ = true;
- auto res = Rewrite(expr);
- update_iterator_padding_ = false;
- return res;
+ struct PaddingGuard {
+ bool& flag;
+ bool saved;
+ explicit PaddingGuard(bool& flag) : flag(flag), saved(flag) { flag = true; }
+ ~PaddingGuard() { flag = saved; }
+ } padding_guard(update_iterator_padding_);
+ return Rewrite(expr);
}
IterSumExpr RewriteIterConstraint(const PrimExpr& expr,
const ffi::Optional<PrimExpr>& predicate_induced_min,
const ffi::Optional<PrimExpr>& predicate_induced_max) {
- return NormalizeToIterOnBoundExpr(ToIterSumExpr(DirectMutate(expr)), predicate_induced_min,
+ PrimExpr rewritten = DirectMutate(expr).ValueOrUnchanged(expr);
+ return NormalizeToIterOnBoundExpr(ToIterSumExpr(rewritten), predicate_induced_min,
predicate_induced_max);
}
@@ -228,7 +236,8 @@ class IterMapRewriter : public ExprMutator {
* \note The result base may contain items that is not
*/
IterSumExpr RewriteToNormalizedIterSum(const PrimExpr& expr) {
- return NormalizeToIterSum(ToIterSumExpr(DirectMutate(expr)));
+ PrimExpr rewritten = DirectMutate(expr).ValueOrUnchanged(expr);
+ return NormalizeToIterSum(ToIterSumExpr(rewritten));
}
/*!
@@ -312,29 +321,43 @@ class IterMapRewriter : public ExprMutator {
return true;
}
- // override the original mutate function.
- Expr VisitExpr(const Expr& input_expr) final {
- Expr expr = ExprMutator::VisitExpr(input_expr);
- if (expr->IsInstance<IterMapExprNode>()) {
- ErrorLogger(this) << "IterMapExpr or subclasses should only result from calls in "
- << "IterMapRewriter using DirectMutate. "
- << "Indirect return occurred in " << input_expr;
- return input_expr;
+ // Bypass only this entry override; the parent checks the target and descendants stay virtual.
+ TVM_FFI_INLINE UnchangedOr<PrimExpr> DirectMutate(
+ const PrimExpr& value, InplaceMode inplace_mode = InplaceMode::kDisallow) {
+ return ffi::details::UnchangedOrUnsafe::MoveFromTVMFFIAny<PrimExpr>(
+ ffi::details::UnchangedOrUnsafe::MoveToTVMFFIAny(
+ Parent::Mutate(ffi::AnyView(value), inplace_mode)));
+ }
+
+ // Qualified parent calls bypass this result guard for their root only.
+ UnchangedOr<ffi::Any> Mutate(ffi::AnyView input,
+ InplaceMode inplace_mode = InplaceMode::kDisallow) final {
+ UnchangedOr<ffi::Any> value_u = Parent::Mutate(input, inplace_mode);
+ try {
+ // Unchanged introduces no IterMapExpr replacement, so the result guard is unnecessary.
+ if (value_u.IsUnchanged()) return ffi::Unchanged();
+ ffi::Any value = std::move(value_u).ValueUnchecked();
+ if (value.as<IterMapExprNode>()) {
+ ErrorLogger(this) << "IterMapExpr or subclasses should only result from qualified "
+ << "parent mutation in IterMapRewriter. "
+ << "Indirect return occurred in " << input;
+ return ffi::Unchanged();
+ }
+ return value;
+ } catch (ffi::Error& error) {
+ if (const auto* node = input.as<ffi::Object>()) {
+ ffi::details::UpdateVisitErrorContext(error, ffi::GetRef<ffi::ObjectRef>(node));
+ }
+ throw;
}
- return expr;
}
- // Normal mutation without normalization.
- PrimExpr DirectMutate(const PrimExpr& expr) {
- return ExprMutator::VisitExpr(expr).as_or_throw<PrimExpr>();
- }
-
- Expr VisitExpr_(const VarNode* op) final;
- Expr VisitExpr_(const prim::AddNode* op) final;
- Expr VisitExpr_(const prim::SubNode* op) final;
- Expr VisitExpr_(const prim::MulNode* op) final;
- Expr VisitExpr_(const prim::FloorDivNode* op) final;
- Expr VisitExpr_(const prim::FloorModNode* op) final;
+ UnchangedOr<Expr> Mutate_(const VarNode* op, InplaceMode inplace_mode) final;
+ UnchangedOr<PrimExpr> Mutate_(const prim::AddNode* op, InplaceMode inplace_mode) final;
+ UnchangedOr<PrimExpr> Mutate_(const prim::SubNode* op, InplaceMode inplace_mode) final;
+ UnchangedOr<PrimExpr> Mutate_(const prim::MulNode* op, InplaceMode inplace_mode) final;
+ UnchangedOr<PrimExpr> Mutate_(const prim::FloorDivNode* op, InplaceMode inplace_mode) final;
+ UnchangedOr<PrimExpr> Mutate_(const prim::FloorModNode* op, InplaceMode inplace_mode) final;
private:
/* \brief Preprocessing common to both FloorDiv and FloorMod
@@ -1483,17 +1506,18 @@ IterMapResult DetectIterMap(const ffi::Array<PrimExpr>& indices,
constraints.begin(), constraints.end(),
[](const IterConstraint& a, const IterConstraint& b) { return a.expr_size < b.expr_size; });
- IterMapRewriter rewriter(analyzer_ptr, constrained_input_iters, check_level,
- simplify_trivial_iterators, &result->errors);
+ auto rewriter =
+ ffi::make_object<IterMapRewriter>(analyzer_ptr, constrained_input_iters, check_level,
+ simplify_trivial_iterators, &result->errors);
// Step0.0: rewrite constraints in the order from size-small ones to size-big ones
for (const IterConstraint& constraint : constraints) {
- auto res = rewriter.RewriteIterConstraint(constraint.iter, constraint.lower_bound,
- constraint.upper_bound);
+ auto res = rewriter->RewriteIterConstraint(constraint.iter, constraint.lower_bound,
+ constraint.upper_bound);
if (result->errors.size() > 0) {
return result;
}
}
- if (!rewriter.CheckConstraints()) {
+ if (!rewriter->CheckConstraints()) {
result->errors.push_back("Invalid constraints.");
return result;
}
@@ -1505,7 +1529,7 @@ IterMapResult DetectIterMap(const ffi::Array<PrimExpr>& indices,
bool allow_padding = check_level != IterMapLevel::Bijective;
if (allow_padding) {
for (PrimExpr value : indices) {
- rewrite_indices.push_back(rewriter.RewriteAndUpdatePadding(value));
+ rewrite_indices.push_back(rewriter->RewriteAndUpdatePadding(value));
if (result->errors.size() > 0) {
return result;
}
@@ -1513,20 +1537,20 @@ IterMapResult DetectIterMap(const ffi::Array<PrimExpr>& indices,
}
// Step0.2: Rewrite indices in the second round.
- if (!allow_padding || rewriter.requires_padding()) {
+ if (!allow_padding || rewriter->requires_padding()) {
rewrite_indices.clear();
for (PrimExpr value : indices) {
- rewrite_indices.push_back(rewriter.Rewrite(value));
+ rewrite_indices.push_back(rewriter->Rewrite(value));
if (result->errors.size() > 0) {
return result;
}
}
}
- result->padding_predicate = rewriter.padding_predicate();
+ result->padding_predicate = rewriter->padding_predicate();
//
// Step1: IterIndependenceChecker checks if the iterator are independent.
- if (!rewriter.CheckMapping(rewrite_indices, check_level)) {
+ if (!rewriter->CheckMapping(rewrite_indices, check_level)) {
if (check_level == IterMapLevel::Bijective) {
result->errors.push_back("Index mapping does not form a bijective transform.");
} else {
@@ -1562,10 +1586,10 @@ IterSumExpr NormalizeToIterSum(PrimExpr index, const ffi::Map<PrimVar, Range>& i
std::vector<IterConstraint> constraints;
IterMapLevel check_level = IterMapLevel::NoCheck;
bool simplify_trivial_iterators = true;
- IterMapRewriter rewriter(analyzer_ptr, input_iters, check_level, simplify_trivial_iterators,
- &result->errors);
+ auto rewriter = ffi::make_object<IterMapRewriter>(analyzer_ptr, input_iters, check_level,
+ simplify_trivial_iterators, &result->errors);
- return rewriter.RewriteToNormalizedIterSum(index);
+ return rewriter->RewriteToNormalizedIterSum(index);
}
TVM_FFI_STATIC_INIT_BLOCK() {
@@ -1578,26 +1602,28 @@ TVM_FFI_STATIC_INIT_BLOCK() {
});
}
-Expr IterMapRewriter::VisitExpr_(const VarNode* op) {
+UnchangedOr<Expr> IterMapRewriter::Mutate_(const VarNode* op, InplaceMode inplace_mode) {
auto var = ffi::GetRef<Var>(op);
auto it = var_map_.find(var);
if (it != var_map_.end()) return it->second;
- return var;
+ // An unmapped variable reuses the input identity without a replacement to validate.
+ return ffi::Unchanged();
}
-Expr IterMapRewriter::VisitExpr_(const prim::AddNode* op) {
+UnchangedOr<PrimExpr> IterMapRewriter::Mutate_(const prim::AddNode* op, InplaceMode inplace_mode) {
if (!IsIndexTypedExpr(op)) {
- return Parent::VisitExpr_(op);
+ return Parent::Mutate_(op, inplace_mode);
}
- PrimExpr a = this->DirectMutate(op->a);
- PrimExpr b = this->DirectMutate(op->b);
+ PrimExpr a = DirectMutate(op->a, inplace_mode).ValueOrUnchanged(op->a);
+ PrimExpr b = DirectMutate(op->b, inplace_mode).ValueOrUnchanged(op->b);
// const folding
- if (auto const_res = TryConstFold<prim::Add>(a, b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Add>(a, b)) return std::move(const_res).value();
// does not contain iter map.
if (!a->IsInstance<IterMapExprNode>() && !b->IsInstance<IterMapExprNode>()) {
if (op->a.same_as(a) && op->b.same_as(b)) {
- return ffi::GetRef<PrimExpr>(op);
+ // Reuse the original expression without passing it through the replacement guard.
+ return ffi::Unchanged();
} else {
return prim::Add(a, b);
}
@@ -1618,21 +1644,22 @@ Expr IterMapRewriter::VisitExpr_(const prim::AddNode* op) {
return ret;
}
-Expr IterMapRewriter::VisitExpr_(const prim::SubNode* op) {
+UnchangedOr<PrimExpr> IterMapRewriter::Mutate_(const prim::SubNode* op, InplaceMode inplace_mode) {
if (!IsIndexTypedExpr(op)) {
- return Parent::VisitExpr_(op);
+ return Parent::Mutate_(op, inplace_mode);
}
- PrimExpr a = this->DirectMutate(op->a);
- PrimExpr b = this->DirectMutate(op->b);
+ PrimExpr a = DirectMutate(op->a, inplace_mode).ValueOrUnchanged(op->a);
+ PrimExpr b = DirectMutate(op->b, inplace_mode).ValueOrUnchanged(op->b);
// const folding
- if (auto const_res = TryConstFold<prim::Sub>(a, b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Sub>(a, b)) return std::move(const_res).value();
// does not contain iter map.
if (!a->IsInstance<IterMapExprNode>() && !b->IsInstance<IterMapExprNode>()) {
if (op->a.same_as(a) && op->b.same_as(b)) {
- return ffi::GetRef<PrimExpr>(op);
+ // Reuse the original expression without passing it through the replacement guard.
+ return ffi::Unchanged();
} else {
return prim::Sub(a, b);
}
@@ -1653,21 +1680,22 @@ Expr IterMapRewriter::VisitExpr_(const prim::SubNode* op) {
return ret;
}
-Expr IterMapRewriter::VisitExpr_(const prim::MulNode* op) {
+UnchangedOr<PrimExpr> IterMapRewriter::Mutate_(const prim::MulNode* op, InplaceMode inplace_mode) {
if (!IsIndexTypedExpr(op)) {
- return Parent::VisitExpr_(op);
+ return Parent::Mutate_(op, inplace_mode);
}
// normalize
- PrimExpr a = this->DirectMutate(op->a);
- PrimExpr b = this->DirectMutate(op->b);
+ PrimExpr a = DirectMutate(op->a, inplace_mode).ValueOrUnchanged(op->a);
+ PrimExpr b = DirectMutate(op->b, inplace_mode).ValueOrUnchanged(op->b);
// const folding
- if (auto const_res = TryConstFold<prim::Mul>(a, b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Mul>(a, b)) return std::move(const_res).value();
// does not contain iter map.
if (!a->IsInstance<IterMapExprNode>() && !b->IsInstance<IterMapExprNode>()) {
if (op->a.same_as(a) && op->b.same_as(b)) {
- return ffi::GetRef<PrimExpr>(op);
+ // Reuse the original expression without passing it through the replacement guard.
+ return ffi::Unchanged();
} else {
return prim::Mul(a, b);
}
@@ -1677,7 +1705,8 @@ Expr IterMapRewriter::VisitExpr_(const prim::MulNode* op) {
// cannot multiply two iterators, mark as unresolved.
ErrorLogger(this) << "Product of two iterators cannot be represented as an IterMap, "
<< "occurs in " << ffi::GetRef<prim::Mul>(op);
- return ffi::GetRef<PrimExpr>(op);
+ // No supported replacement was found; keep the original expression.
+ return ffi::Unchanged();
}
if (!a->IsInstance<IterMapExprNode>()) {
@@ -1966,21 +1995,23 @@ PrimExpr IterMapRewriter::SplitFloorDivConst(IterSplitExpr lhs, PrimExpr base, P
}
}
-Expr IterMapRewriter::VisitExpr_(const prim::FloorDivNode* op) {
+UnchangedOr<PrimExpr> IterMapRewriter::Mutate_(const prim::FloorDivNode* op,
+ InplaceMode inplace_mode) {
if (!IsIndexTypedExpr(op)) {
- return Parent::VisitExpr_(op);
+ return Parent::Mutate_(op, inplace_mode);
}
- PrimExpr a = this->DirectMutate(op->a);
- PrimExpr b = this->DirectMutate(op->b);
+ PrimExpr a = DirectMutate(op->a, inplace_mode).ValueOrUnchanged(op->a);
+ PrimExpr b = DirectMutate(op->b, inplace_mode).ValueOrUnchanged(op->b);
// const folding
- if (auto const_res = TryConstFold<prim::FloorDiv>(a, b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::FloorDiv>(a, b)) return std::move(const_res).value();
// does not contain iter map.
if (!a->IsInstance<IterMapExprNode>() && !b->IsInstance<IterMapExprNode>()) {
if (op->a.same_as(a) && op->b.same_as(b)) {
- return ffi::GetRef<PrimExpr>(op);
+ // Reuse the original expression without passing it through the replacement guard.
+ return ffi::Unchanged();
} else {
return prim::FloorDiv(a, b);
}
@@ -1990,17 +2021,20 @@ Expr IterMapRewriter::VisitExpr_(const prim::FloorDivNode* op) {
// cannot divide an iterator, mark as unresolved.
ErrorLogger(this) << "Cannot represent as an IterMap: the divisor in "
<< ffi::GetRef<PrimExpr>(op) << " may not be an iterator";
- return ffi::GetRef<PrimExpr>(op);
+ // No supported replacement was found; keep the original expression.
+ return ffi::Unchanged();
}
IterSumExpr preprocessed = PreprocessDividend(a.as_or_throw<IterMapExpr>(), op->a);
if (!preprocessed.defined()) {
- return ffi::GetRef<PrimExpr>(op);
+ // No supported replacement was found; keep the original expression.
+ return ffi::Unchanged();
}
TVM_FFI_ICHECK_EQ(preprocessed->args.size(), 1U);
PrimExpr remainder = SplitFloorDivConst(preprocessed->args[0], preprocessed->base, b);
if (!remainder.defined()) {
- return ffi::GetRef<PrimExpr>(op);
+ // No supported replacement was found; keep the original expression.
+ return ffi::Unchanged();
}
return remainder;
}
@@ -2067,21 +2101,23 @@ PrimExpr IterMapRewriter::SplitFloorModConst(IterSplitExpr lhs, PrimExpr base, P
/* scale = */ padded->scale);
}
-Expr IterMapRewriter::VisitExpr_(const prim::FloorModNode* op) {
+UnchangedOr<PrimExpr> IterMapRewriter::Mutate_(const prim::FloorModNode* op,
+ InplaceMode inplace_mode) {
if (!IsIndexTypedExpr(op)) {
- return Parent::VisitExpr_(op);
+ return Parent::Mutate_(op, inplace_mode);
}
- PrimExpr a = this->DirectMutate(op->a);
- PrimExpr b = this->DirectMutate(op->b);
+ PrimExpr a = DirectMutate(op->a, inplace_mode).ValueOrUnchanged(op->a);
+ PrimExpr b = DirectMutate(op->b, inplace_mode).ValueOrUnchanged(op->b);
// const folding
- if (auto const_res = TryConstFold<prim::FloorMod>(a, b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::FloorMod>(a, b)) return std::move(const_res).value();
// does not contain iter map.
if (!a->IsInstance<IterMapExprNode>() && !b->IsInstance<IterMapExprNode>()) {
if (op->a.same_as(a) && op->b.same_as(b)) {
- return ffi::GetRef<PrimExpr>(op);
+ // Reuse the original expression without passing it through the replacement guard.
+ return ffi::Unchanged();
} else {
return prim::FloorMod(a, b);
}
@@ -2091,59 +2127,79 @@ Expr IterMapRewriter::VisitExpr_(const prim::FloorModNode* op) {
// cannot mod an iterator, mark as unresolved.
ErrorLogger(this) << "Cannot represent as an IterMap: the right-hand side of FloorMod in "
<< ffi::GetRef<PrimExpr>(op) << " may not be an iterator";
- return ffi::GetRef<PrimExpr>(op);
+ // No supported replacement was found; keep the original expression.
+ return ffi::Unchanged();
}
IterSumExpr preprocessed = PreprocessDividend(a.as_or_throw<IterMapExpr>(), op->a);
if (!preprocessed.defined()) {
- return ffi::GetRef<PrimExpr>(op);
+ // No supported replacement was found; keep the original expression.
+ return ffi::Unchanged();
}
TVM_FFI_ICHECK_EQ(preprocessed->args.size(), 1U);
PrimExpr remainder = SplitFloorModConst(preprocessed->args[0], preprocessed->base, b);
if (!remainder.defined()) {
- return ffi::GetRef<PrimExpr>(op);
+ // No supported replacement was found; keep the original expression.
+ return ffi::Unchanged();
}
return remainder;
}
/*! * \brief Given an expression that may contain IterVarMapExpr, transform it to normal PrimExpr.
*/
-class IterMapToExprNormalizer : public ExprMutator {
+class IterMapToExprNormalizer : public tvm::ExprMutator {
public:
- explicit IterMapToExprNormalizer(AnalyzerObj* analyzer) : analyzer_(analyzer) {}
+ using Parent = tvm::ExprMutator;
+ using Parent::Mutate_;
- PrimExpr Convert(const PrimExpr& expr) { return VisitExpr(expr).as_or_throw<PrimExpr>(); }
+ explicit IterMapToExprNormalizer(AnalyzerObj* analyzer)
+ : Parent(NativeVTable()), analyzer_(analyzer) {}
- private:
- /*! \brief Override VisitExpr for iter expr type processing */
- Expr VisitExpr(const Expr& expr) override {
- if (auto op = expr.as<IterSplitExpr>()) {
- return ConvertIterSplitExpr(op.value());
- } else if (auto op = expr.as<IterSumExpr>()) {
- return ConvertIterSumExpr(op.value());
- } else {
- return ExprMutator::VisitExpr(expr);
- }
+ UnchangedOr<PrimExpr> Mutate_(const IterSplitExprNode* op, InplaceMode inplace_mode) {
+ return ConvertIterSplitExpr(ffi::GetRef<IterSplitExpr>(op), inplace_mode);
+ }
+
+ UnchangedOr<PrimExpr> Mutate_(const IterSumExprNode* op, InplaceMode inplace_mode) {
+ return ConvertIterSumExpr(ffi::GetRef<IterSumExpr>(op), inplace_mode);
}
- PrimExpr ConvertIterSumExpr(const IterSumExpr& expr) {
+ private:
+ static const VTable* NativeVTable() {
+ static const VTable table = [] {
+ VTable table;
+ Parent::InitVTable(&table);
+ SetDispatch<IterMapToExprNormalizer, IterSplitExprNode>(&table);
+ SetDispatch<IterMapToExprNormalizer, IterSumExprNode>(&table);
+ table.Finalize();
+ return table;
+ }();
+ return &table;
+ }
+
+ PrimExpr ConvertIterSumExpr(const IterSumExpr& expr, InplaceMode inplace_mode) {
PrimExpr res = 0;
- for (const IterSplitExpr& arg : expr->args) {
- res += ConvertIterSplitExpr(arg);
+ InplaceMode args_mode = inplace_mode;
+ // Ensure uniqueness along expr -> args -> arg; Mutate checks each arg.
+ if (!expr->args.unique()) args_mode = InplaceMode::kDisallow;
+ // Borrow stored elements: an owning typed iterator would suppress in-place mutation.
+ for (const ffi::Any& arg : *expr->args.GetArrayObj()) {
+ res += ffi::details::AnyUnsafe::MoveFromAnyAfterCheck<PrimExpr>(
+ Mutate(arg, args_mode).ValueOrUnchanged(arg));
}
res += expr->base;
return res;
}
- PrimExpr ConvertIterSplitExpr(const IterSplitExpr& expr) {
+ PrimExpr ConvertIterSplitExpr(const IterSplitExpr& expr, InplaceMode inplace_mode) {
PrimExpr source;
if (auto opt = expr->source->source.as<Var>()) {
source = opt.value().as_or_throw<PrimExpr>();
- } else if (auto opt = expr->source->source.as<IterSumExpr>()) {
- source = ConvertIterSumExpr(opt.value());
} else {
- source = VisitPrimExpr(expr->source->source);
+ InplaceMode source_mode = inplace_mode;
+ // Ensure uniqueness along expr -> source -> source; the IterMark is skipped.
+ if (!expr->source.unique()) source_mode = InplaceMode::kDisallow;
+ source = Mutate(expr->source->source, source_mode).ValueOrUnchanged(expr->source->source);
}
if (analyzer_->CanProve(expr->extent == expr->source->extent) && is_one(expr->lower_factor)) {
return source * expr->scale;
@@ -2175,9 +2231,9 @@ bool IterMapRewriter::CanProveDivisible(const PrimExpr& lhs, const PrimExpr& rhs
return clhs->value % crhs->value == 0;
}
- IterMapToExprNormalizer normalizer(analyzer_);
- PrimExpr dividend = normalizer.Convert(lhs);
- PrimExpr divisor = normalizer.Convert(rhs);
+ auto normalizer = ffi::make_object<IterMapToExprNormalizer>(analyzer_);
+ PrimExpr dividend = normalizer->Mutate(lhs).ValueOrUnchanged(lhs);
+ PrimExpr divisor = normalizer->Mutate(rhs).ValueOrUnchanged(rhs);
return analyzer_->CanProveEqual(dividend, divisor) ||
analyzer_->CanProve(floormod(dividend, divisor) == 0);
@@ -2185,8 +2241,8 @@ bool IterMapRewriter::CanProveDivisible(const PrimExpr& lhs, const PrimExpr& rhs
PrimExpr NormalizeIterMapToExpr(const PrimExpr& expr) {
arith::Analyzer analyzer;
- IterMapToExprNormalizer normalizer(analyzer.get());
- return normalizer.Convert(expr);
+ auto normalizer = ffi::make_object<IterMapToExprNormalizer>(analyzer.get());
+ return normalizer->Mutate(expr).ValueOrUnchanged(expr);
}
TVM_FFI_STATIC_INIT_BLOCK() {
@@ -2224,8 +2280,10 @@ ffi::Array<PrimExpr> IterMapSimplify(const ffi::Array<PrimExpr>& indices,
}
ffi::Array<PrimExpr> simplified;
simplified.reserve(rewrite.size());
- IterMapToExprNormalizer converter(ana_ptr);
- for (const auto& expr : rewrite) simplified.push_back(converter.Convert(expr));
+ auto converter = ffi::make_object<IterMapToExprNormalizer>(ana_ptr);
+ for (const auto& expr : rewrite) {
+ simplified.push_back(converter->Mutate(expr).ValueOrUnchanged(expr));
+ }
return simplified;
}
@@ -2388,14 +2446,18 @@ class SubspaceDivider {
if (need_predicate) {
// if we have a predicate on this sum expr, then we cannot divide it into Y*E+X
// it should either be Y*1+0 or 0*E(X)+X
- IterMapToExprNormalizer converter(analyzer_);
+ auto converter = ffi::make_object<IterMapToExprNormalizer>(analyzer_);
if (inner_args.empty()) {
// Y*1+0
- outer_preds_ = outer_preds_ && (converter.Convert(outer_source) < mark_extent);
+ PrimExpr converted =
+ converter->Mutate(outer_source).ValueOrUnchanged(std::as_const(outer_source));
+ outer_preds_ = outer_preds_ && (converted < mark_extent);
return DivisionResult::Outer(outer_source, mark_extent);
} else if (outer_args.empty()) {
// 0*E(X)+X
- inner_preds_ = inner_preds_ && (converter.Convert(inner_source) < mark_extent);
+ PrimExpr converted =
+ converter->Mutate(inner_source).ValueOrUnchanged(std::as_const(inner_source));
+ inner_preds_ = inner_preds_ && (converted < mark_extent);
return DivisionResult::Inner(inner_source, mark_extent);
} else {
unresolved_count_++;
diff --git a/src/arith/modular_set.cc b/src/arith/modular_set.cc
index 42ba7f9ad8..97097802c4 100644
--- a/src/arith/modular_set.cc
+++ b/src/arith/modular_set.cc
@@ -25,9 +25,9 @@
#include <tvm/ffi/cast.h>
#include <tvm/ffi/function.h>
#include <tvm/ffi/reflection/registry.h>
+#include <tvm/ir/expr_functor.h>
#include <tvm/ir/prim/builtin.h>
#include <tvm/tirx/builtin.h>
-#include <tvm/tirx/expr_functor.h>
#include <tvm/tirx/op.h>
#include <limits>
@@ -98,7 +98,8 @@ struct ModularSetAnalyzer::Entry {
}
};
-class ModularSetAnalyzer::Impl : public ExprFunctor<ModularSetAnalyzer::Entry(const Expr&)> {
+class ModularSetAnalyzer::Impl : public ffi::Object,
+ public tvm::ExprFunctor<ModularSetAnalyzer::Entry(const Expr&)> {
public:
explicit Impl(AnalyzerObj* parent) : parent_(parent) {}
@@ -133,42 +134,42 @@ class ModularSetAnalyzer::Impl : public ExprFunctor<ModularSetAnalyzer::Entry(co
}
// Override visitor behaviors
- Entry VisitExprDefault_(const ffi::Object* op) final { return Everything(); }
+ Entry DispatchDefault_(const ffi::Object* op) final { return Everything(); }
- Entry VisitExpr_(const prim::LetNode* op) final {
+ Entry Dispatch_(const prim::LetNode* op) final {
auto it = var_map_.find(op->var);
// if the var has not been binded, update the info.
if (it == var_map_.end()) {
- var_map_[op->var] = this->VisitExpr(op->value);
- Entry ret = VisitExpr(op->body);
+ var_map_[op->var] = this->Dispatch(op->value);
+ Entry ret = Dispatch(op->body);
var_map_.erase(op->var);
return ret;
} else {
- return VisitExpr(op->body);
+ return Dispatch(op->body);
}
}
- Entry VisitExpr_(const prim::CastNode* op) final { return VisitExpr(op->value); }
+ Entry Dispatch_(const prim::CastNode* op) final { return Dispatch(op->value); }
- Entry VisitExpr_(const IntImmNode* op) final { return Entry(0, op->value); }
+ Entry Dispatch_(const IntImmNode* op) final { return Entry(0, op->value); }
- Entry VisitExpr_(const prim::AddNode* op) final {
- Entry a = VisitExpr(op->a);
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::AddNode* op) final {
+ Entry a = Dispatch(op->a);
+ Entry b = Dispatch(op->b);
int64_t coeff = ZeroAwareGCD(a.coeff, b.coeff);
return Entry(coeff, a.base + b.base);
}
- Entry VisitExpr_(const prim::SubNode* op) final {
- Entry a = VisitExpr(op->a);
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::SubNode* op) final {
+ Entry a = Dispatch(op->a);
+ Entry b = Dispatch(op->b);
int64_t coeff = ZeroAwareGCD(a.coeff, b.coeff);
return Entry(coeff, a.base - b.base);
}
- Entry VisitExpr_(const prim::MulNode* op) final {
- Entry a = VisitExpr(op->a);
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::MulNode* op) final {
+ Entry a = Dispatch(op->a);
+ Entry b = Dispatch(op->b);
// Simplification rule, x, y, z are in Z
// (p x + n) (q y + m)
// -> pq xy + pm x + qn y + mn
@@ -181,7 +182,7 @@ class ModularSetAnalyzer::Impl : public ExprFunctor<ModularSetAnalyzer::Entry(co
}
Entry DivByConst(const PrimExpr& lhs, int64_t val, bool round_down) {
- Entry a = VisitExpr(lhs);
+ Entry a = Dispatch(lhs);
TVM_FFI_ICHECK_NE(val, 0);
if (a.coeff % val == 0) {
if (a.base == 0) {
@@ -197,42 +198,42 @@ class ModularSetAnalyzer::Impl : public ExprFunctor<ModularSetAnalyzer::Entry(co
return Everything();
}
- Entry VisitExpr_(const prim::DivNode* op) final {
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::DivNode* op) final {
+ Entry b = Dispatch(op->b);
if (b.is_const()) {
return DivByConst(op->a, b.base, false);
}
return Everything();
}
- Entry VisitExpr_(const prim::FloorDivNode* op) final {
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::FloorDivNode* op) final {
+ Entry b = Dispatch(op->b);
if (b.is_const()) {
return DivByConst(op->a, b.base, true);
}
return Everything();
}
- Entry VisitExpr_(const prim::MinNode* op) final {
- Entry a = VisitExpr(op->a);
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::MinNode* op) final {
+ Entry a = Dispatch(op->a);
+ Entry b = Dispatch(op->b);
return Union(a, b);
}
- Entry VisitExpr_(const prim::MaxNode* op) final {
- Entry a = VisitExpr(op->a);
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::MaxNode* op) final {
+ Entry a = Dispatch(op->a);
+ Entry b = Dispatch(op->b);
return Union(a, b);
}
- Entry VisitExpr_(const prim::SelectNode* op) final {
- Entry a = VisitExpr(op->true_value);
- Entry b = VisitExpr(op->false_value);
+ Entry Dispatch_(const prim::SelectNode* op) final {
+ Entry a = Dispatch(op->true_value);
+ Entry b = Dispatch(op->false_value);
return Union(a, b);
}
Entry ModByConst(const PrimExpr& lhs, int64_t val, bool round_down) {
- Entry a = VisitExpr(lhs);
+ Entry a = Dispatch(lhs);
TVM_FFI_ICHECK_NE(val, 0);
int64_t coeff = ZeroAwareGCD(a.coeff, val);
if (a.base % coeff == 0 ||
@@ -242,23 +243,23 @@ class ModularSetAnalyzer::Impl : public ExprFunctor<ModularSetAnalyzer::Entry(co
return Everything();
}
- Entry VisitExpr_(const prim::FloorModNode* op) final {
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::FloorModNode* op) final {
+ Entry b = Dispatch(op->b);
if (b.is_const()) {
return ModByConst(op->a, b.base, true);
}
return Everything();
}
- Entry VisitExpr_(const prim::ModNode* op) final {
- Entry b = VisitExpr(op->b);
+ Entry Dispatch_(const prim::ModNode* op) final {
+ Entry b = Dispatch(op->b);
if (b.is_const()) {
return ModByConst(op->a, b.base, false);
}
return Everything();
}
- Entry VisitExpr_(const CallNode* op) final {
+ Entry Dispatch_(const CallNode* op) final {
// only special handle >> which can be
// used for index calculation.
if (op->op.same_as(prim::builtin::shift_right())) {
@@ -272,7 +273,7 @@ class ModularSetAnalyzer::Impl : public ExprFunctor<ModularSetAnalyzer::Entry(co
}
}
- Entry VisitExpr_(const VarNode* op) final {
+ Entry Dispatch_(const VarNode* op) final {
Var v = ffi::GetRef<Var>(op);
auto it = var_map_.find(v);
if (it != var_map_.end()) {
@@ -283,8 +284,8 @@ class ModularSetAnalyzer::Impl : public ExprFunctor<ModularSetAnalyzer::Entry(co
}
Entry VisitLeftShift(const CallNode* op) {
- Entry a = VisitExpr(op->args[0].as_or_throw<PrimExpr>());
- Entry b = VisitExpr(op->args[1].as_or_throw<PrimExpr>());
+ Entry a = Dispatch(op->args[0].as_or_throw<PrimExpr>());
+ Entry b = Dispatch(op->args[1].as_or_throw<PrimExpr>());
if (b.is_const()) {
return Entry(a.coeff << b.base, a.base << b.base);
}
@@ -292,7 +293,7 @@ class ModularSetAnalyzer::Impl : public ExprFunctor<ModularSetAnalyzer::Entry(co
}
Entry VisitRightShift(const CallNode* op) {
- Entry b = VisitExpr(op->args[1].as_or_throw<PrimExpr>());
+ Entry b = Dispatch(op->args[1].as_or_throw<PrimExpr>());
// a c x / c -> a x
if (b.is_const()) {
return DivByConst(op->args[0].as_or_throw<PrimExpr>(), static_cast<int64_t>(1) << b.base,
@@ -302,7 +303,7 @@ class ModularSetAnalyzer::Impl : public ExprFunctor<ModularSetAnalyzer::Entry(co
}
Entry VisitBitwiseAnd(const CallNode* op) {
- Entry b = VisitExpr(op->args[1].as_or_throw<PrimExpr>());
+ Entry b = Dispatch(op->args[1].as_or_throw<PrimExpr>());
if (b.is_const()) {
int shift;
if (is_const_power_of_two_integer(IntImm::Int32(b.base + 1), &shift)) {
@@ -396,7 +397,7 @@ class ModularSetAnalyzer::Impl : public ExprFunctor<ModularSetAnalyzer::Entry(co
};
ModularSet ModularSetAnalyzer::operator()(const PrimExpr& expr) {
- Entry ret = impl_->VisitExpr(expr);
+ Entry ret = impl_->Dispatch(expr);
return ModularSet(ret.coeff, ret.base);
}
@@ -408,9 +409,10 @@ std::function<void()> ModularSetAnalyzer::EnterConstraint(const PrimExpr& constr
return impl_->EnterConstraint(constraint);
}
-ModularSetAnalyzer::ModularSetAnalyzer(AnalyzerObj* parent) : impl_(new Impl(parent)) {}
+ModularSetAnalyzer::ModularSetAnalyzer(AnalyzerObj* parent)
+ : impl_(ffi::make_object<Impl>(parent)) {}
-ModularSetAnalyzer::~ModularSetAnalyzer() { delete impl_; }
+ModularSetAnalyzer::~ModularSetAnalyzer() = default;
void ModularSetAnalyzer::CopyFrom(const ModularSetAnalyzer& other) {
impl_->CopyFrom(*other.impl_);
diff --git a/src/arith/presburger_set.cc b/src/arith/presburger_set.cc
index ab4cc2048d..2f38200d61 100644
--- a/src/arith/presburger_set.cc
+++ b/src/arith/presburger_set.cc
@@ -31,7 +31,6 @@
#include <tvm/ffi/function.h>
#include <tvm/ffi/reflection/registry.h>
#include <tvm/ir/prim/expr.h>
-#include <tvm/tirx/expr_functor.h>
#include <algorithm>
#include <unordered_map>
diff --git a/src/arith/rewrite_simplify.cc b/src/arith/rewrite_simplify.cc
index f7d823f154..9091310286 100644
--- a/src/arith/rewrite_simplify.cc
+++ b/src/arith/rewrite_simplify.cc
@@ -26,6 +26,7 @@
#include <tvm/arith/analyzer.h>
#include <tvm/ffi/cast.h>
+#include <tvm/ffi/expected.h>
#include <tvm/ir/op.h>
#include <tvm/ir/prim/builtin.h>
#include <tvm/tirx/builtin.h>
@@ -101,11 +102,11 @@ TVM_FFI_STATIC_INIT_BLOCK() { RewriteSimplifierStatsNode::RegisterReflection();
}
// macro for rewrite + recursively rewrite ResExpr
-#define TVM_TRY_RECURSIVE_REWRITE(SrcExpr, ResExpr) \
- RecordAttemptedRewrite(); \
- if ((SrcExpr).Match(ret)) { \
- RecordRewrite(); \
- return RecursiveRewrite((ResExpr).Eval()); \
+#define TVM_TRY_RECURSIVE_REWRITE(SrcExpr, ResExpr) \
+ RecordAttemptedRewrite(); \
+ if ((SrcExpr).Match(ret)) { \
+ RecordRewrite(); \
+ return RecursiveRewrite((ResExpr).Eval(), inplace_mode); \
}
// macro rewrite only if CondExor is true after match.
@@ -121,7 +122,7 @@ TVM_FFI_STATIC_INIT_BLOCK() { RewriteSimplifierStatsNode::RegisterReflection();
RecordAttemptedRewrite(); \
if ((SrcExpr).Match(ret, [&]() { return (CondExpr); })) { \
RecordRewrite(); \
- return RecursiveRewrite((ResExpr).Eval()); \
+ return RecursiveRewrite((ResExpr).Eval(), inplace_mode); \
}
// NOTE for developers:
@@ -226,7 +227,8 @@ CompareResult RewriteSimplifier::Impl::TryComparisonOfProductAndSum(const PrimEx
PVar<PrimExpr> A, B, C, D;
// diff is `(A+B)*C - (A*B)*D`.
- PrimExpr diff = this->VisitPrimExpr(x - y);
+ PrimExpr difference = x - y;
+ PrimExpr diff = Mutate(difference).ValueOrUnchanged(difference);
if (PMatchesOneOf{
(A + B) * C + (A * B) * D,
@@ -363,7 +365,7 @@ CompareResult RewriteSimplifier::Impl::TryCompare(const PrimExpr& x, int64_t val
//
// For stronger comparison proof that is out of the recursive simplifcation
// consider look at analyzer::CanProveStrong
- PrimExpr diff = this->VisitPrimExpr(x);
+ PrimExpr diff = Mutate(x).ValueOrUnchanged(x);
if (const auto* ptr = diff.as<IntImmNode>()) {
if (ptr->value == val) {
return CompareResult::kEQ;
@@ -400,9 +402,10 @@ CompareResult RewriteSimplifier::Impl::TryCompare(const PrimExpr& x, int64_t val
return CompareResult::kUnknown;
}
-Expr RewriteSimplifier::Impl::VisitExpr(const Expr& e) {
- stats_.nodes_visited++;
- return IRMutatorWithAnalyzer::VisitExpr(e);
+UnchangedOr<ffi::Any> RewriteSimplifier::Impl::Mutate(ffi::AnyView value,
+ InplaceMode inplace_mode) {
+ ++stats_.nodes_visited;
+ return SimplifierBase::Mutate(value, inplace_mode);
}
void RewriteSimplifier::Impl::Update(const Var& var, const PrimExpr& info, bool can_override) {
@@ -418,10 +421,12 @@ void RewriteSimplifier::Impl::Update(const Var& var, const PrimExpr& info, bool
var_map_[var] = info;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::AddNode* op) {
- PrimExpr ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::AddNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret =
+ SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
op = ret.as<prim::AddNode>();
- if (auto const_res = TryConstFold<prim::Add>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Add>(op->a, op->b)) return *std::move(const_res);
// Pattern var to match any expression
PVar<PrimExpr> x, y, z, b1, b2, s1, s2;
// Pattern var match IntImm
@@ -537,7 +542,7 @@ std::function<void()> RewriteSimplifier::Impl::EnterConstraint(const PrimExpr& c
size_t old_literal_size = literal_constraints_.size();
// we will compare the already simplified result with the constraint,
// so simplify the constraint as well
- PrimExpr new_constraint = VisitPrimExpr(constraint);
+ PrimExpr new_constraint = Mutate(constraint).ValueOrUnchanged(constraint);
for (const PrimExpr& subconstraint : ExtractConstraints(new_constraint, false)) {
if (is_assume || SideEffect(subconstraint) <= CallEffectKind::kPure) {
literal_constraints_.push_back(subconstraint);
@@ -570,10 +575,12 @@ RewriteSimplifier::Extension RewriteSimplifier::Impl::GetEnabledExtensions() con
return enabled_extensions_;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::SubNode* op) {
- PrimExpr ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::SubNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret =
+ SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
op = ret.as<prim::SubNode>();
- if (auto const_res = TryConstFold<prim::Sub>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Sub>(op->a, op->b)) return *std::move(const_res);
// Pattern var to match any expression
PVar<PrimExpr> x, y, z, b1, b2, s1, s2;
// Pattern var match IntImm
@@ -759,10 +766,12 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::SubNode* op) {
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::MulNode* op) {
- PrimExpr ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::MulNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret =
+ SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
op = ret.as<prim::MulNode>();
- if (auto const_res = TryConstFold<prim::Mul>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Mul>(op->a, op->b)) return *std::move(const_res);
// Pattern var to match any expression
PVar<PrimExpr> x, y, z, b1, b2, s1, s2;
// Pattern var match IntImm
@@ -798,10 +807,12 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::MulNode* op) {
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::DivNode* op) {
- PrimExpr ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::DivNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret =
+ SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
op = ret.as<prim::DivNode>();
- if (auto const_res = TryConstFold<prim::Div>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Div>(op->a, op->b)) return *std::move(const_res);
// Pattern var to match any expression
PVar<PrimExpr> x, y, z, b1;
// Pattern var match IntImm
@@ -951,10 +962,12 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::DivNode* op) {
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::ModNode* op) {
- PrimExpr ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::ModNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret =
+ SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
op = ret.as<prim::ModNode>();
- if (auto const_res = TryConstFold<prim::Mod>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Mod>(op->a, op->b)) return *std::move(const_res);
// Pattern var to match any expression
PVar<PrimExpr> x, y, z, b1;
@@ -1042,10 +1055,12 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::ModNode* op) {
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::FloorDivNode* op) {
- PrimExpr ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::FloorDivNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret =
+ SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
op = ret.as<prim::FloorDivNode>();
- if (auto const_res = TryConstFold<prim::FloorDiv>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::FloorDiv>(op->a, op->b)) return *std::move(const_res);
// Pattern var to match any expression
PVar<PrimExpr> x, y, z, b1;
// Pattern var match IntImm
@@ -1244,10 +1259,12 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::FloorDivNode* op) {
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::FloorModNode* op) {
- PrimExpr ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::FloorModNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret =
+ SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
op = ret.as<prim::FloorModNode>();
- if (auto const_res = TryConstFold<prim::FloorMod>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::FloorMod>(op->a, op->b)) return *std::move(const_res);
// Pattern var to match any expression
PVar<PrimExpr> x, y, z, b1;
@@ -1416,10 +1433,12 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::FloorModNode* op) {
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::MinNode* op) {
- PrimExpr ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::MinNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret =
+ SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
op = ret.as<prim::MinNode>();
- if (auto const_res = TryConstFold<prim::Min>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Min>(op->a, op->b)) return *std::move(const_res);
// Pattern var to match any expression
PVar<PrimExpr> x, y, z, s1, s2;
@@ -1600,10 +1619,12 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::MinNode* op) {
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::MaxNode* op) {
- PrimExpr ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::MaxNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret =
+ SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
op = ret.as<prim::MaxNode>();
- if (auto const_res = TryConstFold<prim::Max>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Max>(op->a, op->b)) return *std::move(const_res);
// Pattern var to match any expression
PVar<PrimExpr> x, y, z, s1, s2;
@@ -1809,21 +1830,26 @@ ffi::Optional<PrimExpr> RewriteSimplifier::Impl::TryMatchLiteralConstraint(
return std::nullopt;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::EQNode* op) {
- prim::EQ ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<prim::EQ>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::EQNode* op,
+ InplaceMode inplace_mode) {
+ // The qualified native hook preserves EQ; primitive children use unchecked category-preserving
+ // results.
+ prim::EQ ret = SimplifierBase::Mutate_(op, inplace_mode)
+ .as_or_throw<UnchangedOr<prim::EQ>>()
+ .ValueOrUnchanged(ffi::GetRef<prim::EQ>(op));
op = ret.get();
if (auto const_res = TryConstFold<prim::EQ>(op->a, op->b)) {
- return const_res.value();
+ return *std::move(const_res);
}
if (auto match = TryMatchLiteralConstraint(ret)) {
return match.value();
}
- return ApplyRewriteRules(ret);
+ return ApplyRewriteRules(ret, inplace_mode);
}
-PrimExpr RewriteSimplifier::Impl::ApplyRewriteRules(prim::EQ ret) {
+PrimExpr RewriteSimplifier::Impl::ApplyRewriteRules(prim::EQ ret, InplaceMode inplace_mode) {
// Pattern var to match any expression
PVar<PrimExpr> x, y;
// Pattern var match IntImm
@@ -1865,11 +1891,13 @@ PrimExpr RewriteSimplifier::Impl::ApplyRewriteRules(prim::EQ ret) {
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::NENode* op) {
- PrimExpr ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::NENode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret =
+ SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
op = ret.as<prim::NENode>();
- if (auto const_res = TryConstFold<prim::NE>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::NE>(op->a, op->b)) return *std::move(const_res);
if (auto match = TryMatchLiteralConstraint(ret)) return match.value();
if (IsIndexTypedExpr(op->a)) {
@@ -1886,7 +1914,7 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::NENode* op) {
// (a < b) or (b < a)
// False or (b < a)
// b < a
- return ApplyRewriteRules(prim::LT(op->b, op->a));
+ return ApplyRewriteRules(prim::LT(op->b, op->a), inplace_mode);
} else if (result == CompareResult::kLE) {
// Known: a <= b
//
@@ -1894,19 +1922,22 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::NENode* op) {
// (a < b) or (b < a)
// (a < b) or False
// a < b
- return ApplyRewriteRules(prim::LT(op->a, op->b));
+ return ApplyRewriteRules(prim::LT(op->a, op->b), inplace_mode);
}
}
- return ApplyRewriteRules(prim::Not(ApplyRewriteRules(prim::EQ(op->a, op->b))));
+ return ApplyRewriteRules(prim::Not(ApplyRewriteRules(prim::EQ(op->a, op->b), inplace_mode)),
+ inplace_mode);
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::LENode* op) {
- PrimExpr ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::LENode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret =
+ SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
op = ret.as<prim::LENode>();
TVM_FFI_ICHECK(op);
- if (auto const_res = TryConstFold<prim::LE>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::LE>(op->a, op->b)) return *std::move(const_res);
if (auto match = TryMatchLiteralConstraint(ret)) return match.value();
// Check for applicable rewrites before attempting to prove/disprove
@@ -1914,7 +1945,8 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::LENode* op) {
// simplifies to (ceildiv(A,B)<=x) when (A%B!=0). Performing the
// TryCompare first would simplify to the equivalent
// (floordiv(A,B)<x) in these cases instead.
- ret = ApplyRewriteRules(prim::Not(ApplyRewriteRules(prim::LT(op->b, op->a))));
+ ret = ApplyRewriteRules(prim::Not(ApplyRewriteRules(prim::LT(op->b, op->a), inplace_mode)),
+ inplace_mode);
if (auto op = ret.as<prim::LENode>(); op && IsIndexTypedExpr(op->a)) {
CompareResult result = TryCompare(op->a, op->b);
@@ -1930,7 +1962,7 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::LENode* op) {
// (a < b) or (a == b)
// (a < b) or False
// a < b
- return ApplyRewriteRules(prim::LT(op->a, op->b));
+ return ApplyRewriteRules(prim::LT(op->a, op->b), inplace_mode);
} else if (result == CompareResult::kGE) {
// Known: a >= b
//
@@ -1938,32 +1970,41 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::LENode* op) {
// (a < b) or (a == b)
// False or (a == b)
// a == b
- return ApplyRewriteRules(prim::EQ(op->a, op->b));
+ return ApplyRewriteRules(prim::EQ(op->a, op->b), inplace_mode);
}
}
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::GTNode* op) {
- return this->VisitPrimExpr(op->b < op->a);
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::GTNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr expr = op->b < op->a;
+ return Mutate(expr, inplace_mode).ValueOrUnchanged(expr);
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::GENode* op) {
- return this->VisitPrimExpr(op->b <= op->a);
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::GENode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr expr = op->b <= op->a;
+ return Mutate(expr, inplace_mode).ValueOrUnchanged(expr);
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::LTNode* op) {
- prim::LT node = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<prim::LT>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::LTNode* op,
+ InplaceMode inplace_mode) {
+ // The qualified native hook preserves LT; primitive children use unchecked category-preserving
+ // results.
+ prim::LT node = SimplifierBase::Mutate_(op, inplace_mode)
+ .as_or_throw<UnchangedOr<prim::LT>>()
+ .ValueOrUnchanged(ffi::GetRef<prim::LT>(op));
op = node.get();
- if (auto const_res = TryConstFold<prim::LT>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::LT>(op->a, op->b)) return *std::move(const_res);
if (auto match = TryMatchLiteralConstraint(node)) return match.value();
- return ApplyRewriteRules(node);
+ return ApplyRewriteRules(node, inplace_mode);
}
-PrimExpr RewriteSimplifier::Impl::ApplyRewriteRules(prim::LT ret) {
+PrimExpr RewriteSimplifier::Impl::ApplyRewriteRules(prim::LT ret, InplaceMode inplace_mode) {
// Pattern var to match any expression
PVar<PrimExpr> x, y, z, s1, s2;
// Pattern var match IntImm
@@ -2109,7 +2150,7 @@ PrimExpr RewriteSimplifier::Impl::ApplyRewriteRules(prim::LT ret) {
return std::nullopt;
}();
if (merge_constants) {
- return RecursiveRewrite(merge_constants.value());
+ return RecursiveRewrite(merge_constants.value(), inplace_mode);
}
auto common_factor = [&]() -> int64_t {
@@ -2120,21 +2161,27 @@ PrimExpr RewriteSimplifier::Impl::ApplyRewriteRules(prim::LT ret) {
return ZeroAwareGCD(gcd_lhs, gcd_rhs);
}();
if (common_factor > 1) {
- return RecursiveRewrite(floordiv(ret->a, common_factor) < floordiv(ret->b, common_factor));
+ return RecursiveRewrite(floordiv(ret->a, common_factor) < floordiv(ret->b, common_factor),
+ inplace_mode);
}
}
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::NotNode* op) {
- prim::Not ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<prim::Not>();
- if (auto const_res = TryConstFold<prim::Not>(ret->a)) return const_res.value();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::NotNode* op,
+ InplaceMode inplace_mode) {
+ // The qualified native hook preserves Not; primitive children use unchecked category-preserving
+ // results.
+ prim::Not ret = SimplifierBase::Mutate_(op, inplace_mode)
+ .as_or_throw<UnchangedOr<prim::Not>>()
+ .ValueOrUnchanged(ffi::GetRef<prim::Not>(op));
+ if (auto const_res = TryConstFold<prim::Not>(ret->a)) return *std::move(const_res);
if (auto match = TryMatchLiteralConstraint(ret)) return match.value();
- return ApplyRewriteRules(ret);
+ return ApplyRewriteRules(ret, inplace_mode);
}
-PrimExpr RewriteSimplifier::Impl::ApplyRewriteRules(prim::Not ret) {
+PrimExpr RewriteSimplifier::Impl::ApplyRewriteRules(prim::Not ret, InplaceMode inplace_mode) {
// Pattern var to match any expression
PVar<PrimExpr> x, y;
PVar<PrimExpr> lanes;
@@ -2154,13 +2201,13 @@ PrimExpr RewriteSimplifier::Impl::ApplyRewriteRules(prim::Not ret) {
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::AndNode* op) {
- PrimExpr ret = [&]() -> PrimExpr {
- // If this extension isn't enabled, just delegate out.
- if (!(enabled_extensions_ & kApplyConstraintsToBooleanBranches)) {
- return IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
- }
-
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::AndNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret;
+ // If this extension isn't enabled, just delegate out.
+ if (!(enabled_extensions_ & kApplyConstraintsToBooleanBranches)) {
+ ret = SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
+ } else {
PrimExpr a = op->a;
PrimExpr b = op->b;
@@ -2176,7 +2223,10 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::AndNode* op) {
const PrimExpr& constraint = (i % 2 == 0) ? b : a;
With<ConstraintContext> context(analyzer_, constraint);
- PrimExpr updated = VisitPrimExpr(to_update);
+ // Keep this iteration out of place: convergence uses same_as() to detect updates.
+ // An in-place change would preserve identity and could stop the loop early.
+ PrimExpr updated =
+ Mutate(to_update, InplaceMode::kDisallow).ValueOrUnchanged(std::as_const(to_update));
if (!to_update.same_as(updated)) {
to_update = updated;
@@ -2193,15 +2243,15 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::AndNode* op) {
// Otherwise, follow ExprMutator's convention of returning the
// original object.
if (a.same_as(op->a) && b.same_as(op->b)) {
- return ffi::GetRef<PrimExpr>(op);
+ ret = ffi::GetRef<PrimExpr>(op);
} else {
- return prim::And(a, b);
+ ret = prim::And(a, b);
}
- }();
+ }
op = ret.as<prim::AndNode>();
- if (auto const_res = TryConstFold<prim::And>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::And>(op->a, op->b)) return *std::move(const_res);
if (auto match = TryMatchLiteralConstraint(ret)) return match.value();
if ((enabled_extensions_ & RewriteSimplifier::kConvertBooleanToAndOfOrs) &&
!recursively_visiting_boolean_) {
@@ -2301,15 +2351,15 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::AndNode* op) {
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::OrNode* op) {
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::OrNode* op,
+ InplaceMode inplace_mode) {
PrimExpr orig = ffi::GetRef<PrimExpr>(op);
- PrimExpr ret = [&]() -> PrimExpr {
- // If this extension isn't enabled, just delegate out.
- if (!(enabled_extensions_ & kApplyConstraintsToBooleanBranches)) {
- return IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
- }
-
+ PrimExpr ret;
+ // If this extension isn't enabled, just delegate out.
+ if (!(enabled_extensions_ & kApplyConstraintsToBooleanBranches)) {
+ ret = SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
+ } else {
PrimExpr a = op->a;
PrimExpr b = op->b;
@@ -2325,7 +2375,10 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::OrNode* op) {
const PrimExpr& constraint = (i % 2 == 0) ? b : a;
With<ConstraintContext> context(analyzer_, NormalizeBooleanOperators(prim::Not(constraint)));
- PrimExpr updated = VisitPrimExpr(to_update);
+ // Keep this iteration out of place: convergence uses same_as() to detect updates.
+ // An in-place change would preserve identity and could stop the loop early.
+ PrimExpr updated =
+ Mutate(to_update, InplaceMode::kDisallow).ValueOrUnchanged(std::as_const(to_update));
if (!to_update.same_as(updated)) {
to_update = updated;
@@ -2342,14 +2395,14 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::OrNode* op) {
// Otherwise, follow ExprMutator's convention of returning the
// original object.
if (a.same_as(op->a) && b.same_as(op->b)) {
- return ffi::GetRef<PrimExpr>(op);
+ ret = ffi::GetRef<PrimExpr>(op);
} else {
- return prim::Or(a, b);
+ ret = prim::Or(a, b);
}
- }();
+ }
op = ret.as<prim::OrNode>();
- if (auto const_res = TryConstFold<prim::Or>(op->a, op->b)) return const_res.value();
+ if (auto const_res = TryConstFold<prim::Or>(op->a, op->b)) return *std::move(const_res);
if (auto match = TryMatchLiteralConstraint(ret)) return match.value();
if ((enabled_extensions_ & RewriteSimplifier::kConvertBooleanToAndOfOrs) &&
!recursively_visiting_boolean_) {
@@ -2401,8 +2454,10 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::OrNode* op) {
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::SelectNode* op) {
- PrimExpr ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::SelectNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret =
+ SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
op = ret.as<prim::SelectNode>();
if (op == nullptr) return ret;
// Pattern var to match any expression
@@ -2411,9 +2466,9 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const prim::SelectNode* op) {
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const CallNode* op) {
+UnchangedOr<Expr> RewriteSimplifier::Impl::Mutate_(const CallNode* op, InplaceMode inplace_mode) {
// add condition context to if_then_else
- Expr expr = IRMutatorWithAnalyzer::VisitExpr_(op);
+ Expr expr = SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<Expr>(op));
auto opt_ret = expr.as<PrimExpr>();
if (!opt_ret) {
return expr;
@@ -2495,8 +2550,8 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const CallNode* op) {
// Only check constant cases to avoid recursion
if (is_const_number(inner_else_expr) && is_const_number(else_expr) &&
analyzer_->CanProve(inner_else_expr == else_expr)) {
- return Call(ret_ty, op->op, {cond && inner_cond, inner_then_expr, else_expr}, op->attrs, {},
- op->span)
+ return Call(ret_ty, op->op, {cond && inner_cond, inner_then_expr, else_expr}, op->attrs,
+ op->ty_args, op->span)
.as_or_throw<PrimExpr>();
}
}
@@ -2505,10 +2560,13 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const CallNode* op) {
return ret;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const VarNode* op) {
+UnchangedOr<Expr> RewriteSimplifier::Impl::Mutate_(const VarNode* op, InplaceMode inplace_mode) {
Var var = ffi::GetRef<Var>(op);
auto prim_var = var.as<PrimVar>();
- if (!prim_var) return var;
+ if (!prim_var) {
+ // Non-primitive variables retain the original identity without a replacement.
+ return ffi::Unchanged();
+ }
PrimType op_ty = prim_var.value().ty();
if (op_ty.MatchesElementType(DLDataTypeCode::kDLBool, 8) && !op_ty.IsScalableVector() &&
!op_ty.IsFixedLengthVector()) {
@@ -2521,11 +2579,14 @@ Expr RewriteSimplifier::Impl::VisitExpr_(const VarNode* op) {
if (it != var_map_.end()) {
return it->second;
}
- return ffi::GetRef<Var>(op);
+ // Reuse the original node identity; there is no replacement to process.
+ return ffi::Unchanged();
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::CastNode* op) {
- PrimExpr ret = IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<PrimExpr>();
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::CastNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr ret =
+ SimplifierBase::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<PrimExpr>(op));
op = ret.as<prim::CastNode>();
return cast(ret.ty(), op->value);
}
@@ -2538,17 +2599,19 @@ bool RewriteSimplifier::Impl::CanInlineLet(const prim::LetNode* op) {
return false;
}
-Expr RewriteSimplifier::Impl::VisitExpr_(const prim::LetNode* op) {
- PrimExpr value = this->VisitPrimExpr(op->value);
+UnchangedOr<PrimExpr> RewriteSimplifier::Impl::Mutate_(const prim::LetNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr value = Mutate(op->value, inplace_mode).ValueOrUnchanged(op->value);
if (CanInlineLet(op)) {
// it is fine to discard the let binding
// because the value will always be inlined in the simplifier.
analyzer_->Bind(op->var, value);
- return this->VisitPrimExpr(op->body);
+ return Mutate(op->body, inplace_mode).ValueOrUnchanged(op->body);
}
- PrimExpr body = this->VisitPrimExpr(op->body);
+ PrimExpr body = Mutate(op->body, inplace_mode).ValueOrUnchanged(op->body);
if (value.same_as(op->value) && body.same_as(op->body)) {
- return ffi::GetRef<PrimExpr>(op);
+ // Reuse the original node identity; there is no replacement to process.
+ return ffi::Unchanged();
} else {
return prim::Let(op->var, value, body);
}
@@ -2559,7 +2622,8 @@ PrimExpr RewriteSimplifier::operator()(const PrimExpr& expr) {
PrimExpr res = expr;
int max_iter = 2;
for (int i = 0; i < max_iter; ++i) {
- PrimExpr new_expr = impl_->VisitPrimExpr(res);
+ // Keep the default kDisallow: convergence requires same_as to detect updates.
+ PrimExpr new_expr = impl_->Mutate(res).ValueOrUnchanged(std::as_const(res));
if (new_expr.same_as(res)) return res;
res = new_expr;
}
@@ -2590,9 +2654,9 @@ void RewriteSimplifier::SetMaximumRewriteSteps(int64_t maximum) {
impl_->SetMaximumRewriteSteps(maximum);
}
-RewriteSimplifier::RewriteSimplifier(AnalyzerObj* parent) : impl_(new Impl(parent)) {}
+RewriteSimplifier::RewriteSimplifier(AnalyzerObj* parent) : impl_(ffi::make_object<Impl>(parent)) {}
-RewriteSimplifier::~RewriteSimplifier() { delete impl_; }
+RewriteSimplifier::~RewriteSimplifier() = default;
void RewriteSimplifier::CopyFrom(const RewriteSimplifier& other) { impl_->CopyFrom(*other.impl_); }
diff --git a/src/arith/rewrite_simplify.h b/src/arith/rewrite_simplify.h
index 55014a87cf..78cf06e403 100644
--- a/src/arith/rewrite_simplify.h
+++ b/src/arith/rewrite_simplify.h
@@ -34,8 +34,8 @@
#include <vector>
#include "const_fold.h"
-#include "ir_mutator_with_analyzer.h"
#include "pattern_match.h"
+#include "simplify_base.h"
namespace tvm {
namespace arith {
@@ -84,39 +84,40 @@ struct RewriteSimplifierStats : ffi::ObjectRef {
*
* This class can be inheritated for other simplifiers.
*/
-class RewriteSimplifier::Impl : public IRMutatorWithAnalyzer {
+class RewriteSimplifier::Impl : public SimplifierBase {
public:
- using IRMutatorWithAnalyzer::VisitExpr_;
- using IRMutatorWithAnalyzer::VisitPrimExpr;
+ using SimplifierBase::Mutate;
+ using SimplifierBase::Mutate_;
- explicit Impl(AnalyzerObj* parent) : IRMutatorWithAnalyzer(parent) {}
+ explicit Impl(AnalyzerObj* parent) : SimplifierBase(parent) {}
- Expr VisitExpr(const Expr& e) override;
+ UnchangedOr<ffi::Any> Mutate(ffi::AnyView value,
+ InplaceMode inplace_mode = InplaceMode::kDisallow) override;
void Update(const Var& var, const PrimExpr& info, bool override_info);
- Expr VisitExpr_(const prim::AddNode* op) override;
- Expr VisitExpr_(const prim::SubNode* op) override;
- Expr VisitExpr_(const prim::MulNode* op) override;
- Expr VisitExpr_(const prim::DivNode* op) override;
- Expr VisitExpr_(const prim::ModNode* op) override;
- Expr VisitExpr_(const prim::FloorDivNode* op) override;
- Expr VisitExpr_(const prim::FloorModNode* op) override;
- Expr VisitExpr_(const prim::MinNode* op) override;
- Expr VisitExpr_(const prim::MaxNode* op) override;
- Expr VisitExpr_(const prim::EQNode* op) override;
- Expr VisitExpr_(const prim::NENode* op) override;
- Expr VisitExpr_(const prim::LTNode* op) override;
- Expr VisitExpr_(const prim::LENode* op) override;
- Expr VisitExpr_(const prim::GTNode* op) override;
- Expr VisitExpr_(const prim::GENode* op) override;
- Expr VisitExpr_(const prim::AndNode* op) override;
- Expr VisitExpr_(const prim::OrNode* op) override;
- Expr VisitExpr_(const prim::NotNode* op) override;
- Expr VisitExpr_(const prim::SelectNode* op) override;
- Expr VisitExpr_(const CallNode* op) override;
- Expr VisitExpr_(const VarNode* op) override;
- Expr VisitExpr_(const prim::CastNode* op) override;
- Expr VisitExpr_(const prim::LetNode* op) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::AddNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::SubNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::MulNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::DivNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::ModNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::FloorDivNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::FloorModNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::MinNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::MaxNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::EQNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::NENode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::LTNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::LENode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::GTNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::GENode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::AndNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::OrNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::NotNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::SelectNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<Expr> Mutate_(const CallNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<Expr> Mutate_(const VarNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::CastNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::LetNode* op, InplaceMode inplace_mode) override;
std::function<void()> EnterConstraint(const PrimExpr& constraint, bool is_assume);
@@ -207,24 +208,24 @@ class RewriteSimplifier::Impl : public IRMutatorWithAnalyzer {
/*! \brief Rewrite rules for Less Than comparisons
*
- * These are separate from the VisitExpr_(const LTNode*) method, as
+ * These are separate from the Mutate_(const LTNode*) method, as
* they may required from rewrites of LT or LE.
*/
- PrimExpr ApplyRewriteRules(prim::LT node);
+ PrimExpr ApplyRewriteRules(prim::LT node, InplaceMode inplace_mode);
/*! \brief Rewrite rules for Equal comparisons
*
- * These are separate from the VisitExpr_(const EQNode*) method, as
+ * These are separate from the Mutate_(const EQNode*) method, as
* they may required from rewrites of LE or NE.
*/
- PrimExpr ApplyRewriteRules(prim::EQ node);
+ PrimExpr ApplyRewriteRules(prim::EQ node, InplaceMode inplace_mode);
/*! \brief Rewrite rules for Equal comparisons
*
- * These are separate from the VisitExpr_(const EQNode*) method, as
+ * These are separate from the Mutate_(const EQNode*) method, as
* they may required from rewrites of LT, LE, or NE.
*/
- PrimExpr ApplyRewriteRules(prim::Not node);
+ PrimExpr ApplyRewriteRules(prim::Not node, InplaceMode inplace_mode);
private:
CompareResult TryCompareUsingKnownInequalities(const PrimExpr& x, const PrimExpr& y);
@@ -248,14 +249,16 @@ class RewriteSimplifier::Impl : public IRMutatorWithAnalyzer {
// Recursive rewrite x
// we limit maximum depth of recursive rewrite allowed to
// avoid infinite loop
- PrimExpr RecursiveRewrite(const PrimExpr& x) {
+ PrimExpr RecursiveRewrite(const PrimExpr& x, InplaceMode inplace_mode) {
stats_.num_recursive_rewrites++;
if (recur_depth_ >= kMaxRecurDepth) return x;
- ++recur_depth_;
+ struct DepthGuard {
+ int64_t& depth;
+ explicit DepthGuard(int64_t& depth) : depth(depth) { ++depth; }
+ ~DepthGuard() { --depth; }
+ } depth_guard(recur_depth_);
stats_.max_recursive_depth = std::max(recur_depth_, stats_.max_recursive_depth);
- PrimExpr res = this->VisitPrimExpr(x);
- --recur_depth_;
- return res;
+ return Mutate(x, inplace_mode).ValueOrUnchanged(x);
}
template <typename TA>
diff --git a/src/arith/simplify_base.cc b/src/arith/simplify_base.cc
new file mode 100644
index 0000000000..8facc06743
--- /dev/null
+++ b/src/arith/simplify_base.cc
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "simplify_base.h"
+
+#include <tvm/ir/prim/builtin.h>
+#include <tvm/tirx/analysis.h>
+#include <tvm/tirx/op.h>
+
+#include "constraint_helpers.h"
+
+namespace tvm {
+namespace arith {
+
+using detail::EnterConstraintFacts;
+
+UnchangedOr<Expr> SimplifierBase::Mutate_(const TupleNode* op, InplaceMode inplace_mode) {
+ auto fields = Mutate(op->fields, inplace_mode).as_or_throw<UnchangedOr<ffi::Array<Expr>>>();
+ if (fields.UnchangedOrSameAs(op->fields)) return ffi::Unchanged();
+ if (inplace_mode == InplaceMode::kAllow) {
+ const_cast<TupleNode*>(op)->fields = std::move(fields).ValueUnchecked();
+ return ffi::Unchanged();
+ }
+ auto copy = ffi::make_object<TupleNode>(*op);
+ copy->fields = std::move(fields).ValueUnchecked();
+ return Tuple(std::move(copy));
+}
+
+UnchangedOr<Expr> SimplifierBase::Mutate_(const TupleGetItemNode* op, InplaceMode inplace_mode) {
+ auto tuple = Mutate(op->tuple, inplace_mode);
+ if (tuple.UnchangedOrSameAs(op->tuple)) return ffi::Unchanged();
+ if (inplace_mode == InplaceMode::kAllow) {
+ const_cast<TupleGetItemNode*>(op)->tuple = std::move(tuple).ValueUnchecked();
+ return ffi::Unchanged();
+ }
+ auto copy = ffi::make_object<TupleGetItemNode>(*op);
+ copy->tuple = std::move(tuple).ValueUnchecked();
+ return TupleGetItem(std::move(copy));
+}
+
+UnchangedOr<PrimExpr> SimplifierBase::Mutate_(const TensorLoadNode* op, InplaceMode inplace_mode) {
+ auto source = Mutate(op->source, inplace_mode);
+ auto indices = Mutate(op->indices, inplace_mode).as_or_throw<UnchangedOr<ffi::Array<PrimExpr>>>();
+ if (source.UnchangedOrSameAs(op->source) && indices.UnchangedOrSameAs(op->indices)) {
+ return ffi::Unchanged();
+ }
+ if (inplace_mode == InplaceMode::kAllow) {
+ auto* writable = const_cast<TensorLoadNode*>(op);
+ if (!source.IsUnchanged()) writable->source = std::move(source).ValueUnchecked();
+ if (!indices.IsUnchanged()) writable->indices = std::move(indices).ValueUnchecked();
+ return ffi::Unchanged();
+ }
+ auto copy = ffi::make_object<TensorLoadNode>(*op);
+ copy->source = std::move(source).ValueOrUnchanged(std::move(copy->source));
+ copy->indices = std::move(indices).ValueOrUnchanged(std::move(copy->indices));
+ return TensorLoad(std::move(copy));
+}
+
+UnchangedOr<Expr> SimplifierBase::Mutate_(const CallNode* op, InplaceMode inplace_mode) {
+ if (op->op.same_as(prim::builtin::if_then_else())) {
+ InplaceMode inplace_mode_args = inplace_mode;
+ // Ensure uniqueness along op -> args -> args[i].
+ // op was already checked; check args here, and Mutate checks args[i].
+ if (!op->args.unique()) inplace_mode_args = InplaceMode::kDisallow;
+ // Borrow stored elements: owning typed handles would suppress in-place mutation.
+ const auto* args = op->args.GetArrayObj();
+ PrimExpr cond =
+ Mutate((*args)[0], inplace_mode_args).ValueOrUnchanged((*args)[0]).as_or_throw<PrimExpr>();
+ Expr true_value = ffi::details::AnyUnsafe::MoveFromAnyAfterCheck<Expr>(
+ constraint_scope_
+ .WithNewScope([&]() {
+ EnterConstraintFacts(&constraint_scope_.Current(), analyzer_, cond);
+ return Mutate((*args)[1], inplace_mode_args);
+ })
+ .ValueOrUnchanged((*args)[1]));
+ Expr false_value;
+ {
+ PrimExpr not_cond = prim::Not(cond);
+ false_value = ffi::details::AnyUnsafe::MoveFromAnyAfterCheck<Expr>(
+ constraint_scope_
+ .WithNewScope([&]() {
+ constraint_scope_.Current().Emplace(analyzer_, not_cond);
+ return Mutate((*args)[2], inplace_mode_args);
+ })
+ .ValueOrUnchanged((*args)[2]));
+ }
+ if (tirx::is_zero(cond)) return false_value;
+ if (tirx::is_one(cond)) return true_value;
+ if (cond.same_as(op->args[0]) && true_value.same_as(op->args[1]) &&
+ false_value.same_as(op->args[2])) {
+ // Reuse the original node identity; there is no replacement to process.
+ return ffi::Unchanged();
+ }
+ return Call(op->ty, op->op, {cond, true_value, false_value}, op->attrs, op->ty_args, op->span);
+ }
+ auto args = Mutate(op->args, inplace_mode).as_or_throw<UnchangedOr<ffi::Array<Expr>>>();
+ if (args.UnchangedOrSameAs(op->args)) return ffi::Unchanged();
+ if (inplace_mode == InplaceMode::kAllow) {
+ const_cast<CallNode*>(op)->args = std::move(args).ValueUnchecked();
+ return ffi::Unchanged();
+ }
+ auto copy = ffi::make_object<CallNode>(*op);
+ copy->args = std::move(args).ValueUnchecked();
+ return Call(std::move(copy));
+}
+
+UnchangedOr<PrimExpr> SimplifierBase::Mutate_(const prim::LetNode* op, InplaceMode inplace_mode) {
+ PrimExpr value = Mutate(op->value, inplace_mode).ValueOrUnchanged(op->value);
+ if (tirx::SideEffect(value) <= tirx::CallEffectKind::kPure) {
+ analyzer_->Bind(op->var, value);
+ }
+ PrimExpr body = Mutate(op->body, inplace_mode).ValueOrUnchanged(op->body);
+ if (value.same_as(op->value) && body.same_as(op->body)) {
+ // Reuse the original node identity; there is no replacement to process.
+ return ffi::Unchanged();
+ }
+ return prim::Let(op->var, value, body, op->span);
+}
+
+UnchangedOr<PrimExpr> SimplifierBase::Mutate_(const prim::SelectNode* op,
+ InplaceMode inplace_mode) {
+ PrimExpr cond = Mutate(op->condition, inplace_mode).ValueOrUnchanged(op->condition);
+ PrimExpr true_value = constraint_scope_
+ .WithNewScope([&]() {
+ EnterConstraintFacts(&constraint_scope_.Current(), analyzer_, cond);
+ return Mutate(op->true_value, inplace_mode);
+ })
+ .ValueOrUnchanged(op->true_value);
+ PrimExpr false_value;
+ {
+ PrimExpr neg_cond = analyzer_->rewrite_simplify(prim::Not(cond));
+ false_value = constraint_scope_
+ .WithNewScope([&]() {
+ constraint_scope_.Current().Emplace(analyzer_, neg_cond);
+ return Mutate(op->false_value, inplace_mode);
+ })
+ .ValueOrUnchanged(op->false_value);
+ }
+ if (tirx::is_zero(cond)) return false_value;
+ if (tirx::is_one(cond)) return true_value;
+ if (cond.same_as(op->condition) && true_value.same_as(op->true_value) &&
+ false_value.same_as(op->false_value)) {
+ // Reuse the original node identity; there is no replacement to process.
+ return ffi::Unchanged();
+ }
+ return prim::Select(cond, true_value, false_value, op->span);
+}
+
+} // namespace arith
+} // namespace tvm
diff --git a/src/arith/simplify_base.h b/src/arith/simplify_base.h
new file mode 100644
index 0000000000..56b3930e36
--- /dev/null
+++ b/src/arith/simplify_base.h
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file simplify_base.h
+ * \brief Expression mutator base class for arith simplifiers.
+ */
+#ifndef TVM_ARITH_SIMPLIFY_BASE_H_
+#define TVM_ARITH_SIMPLIFY_BASE_H_
+
+#include <tvm/arith/analyzer.h>
+#include <tvm/ir/expr_functor.h>
+#include <tvm/ir/scope_stack.h>
+#include <tvm/ir/with_context.h>
+
+#include <utility>
+
+namespace tvm {
+namespace arith {
+
+/*!
+ * \brief Arithmetic semantic-operand traversal and analyzer constraint handling.
+ *
+ * The native arithmetic expression hooks leave types and metadata unchanged.
+ * This policy and branch-constraint handling are kept in a dedicated base so
+ * ordinary expression mutation retains full traversal. Extension nodes retain
+ * the inherited structural fallback.
+ */
+class SimplifierBase : public tvm::ExprMutator {
+ public:
+ using Parent = tvm::ExprMutator;
+ explicit SimplifierBase(AnalyzerObj* analyzer) : analyzer_(analyzer) {}
+
+ virtual ~SimplifierBase() = default;
+
+ using Parent::Mutate_;
+
+ UnchangedOr<Expr> Mutate_(const OpaqueExprNode* op, InplaceMode inplace_mode) override {
+ return ffi::Unchanged();
+ }
+
+ UnchangedOr<Expr> Mutate_(const VarNode* op, InplaceMode inplace_mode) override {
+ return ffi::Unchanged();
+ }
+ UnchangedOr<Expr> Mutate_(const TupleNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<Expr> Mutate_(const TupleGetItemNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const TensorLoadNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::LetNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<PrimExpr> Mutate_(const prim::SelectNode* op, InplaceMode inplace_mode) override;
+ UnchangedOr<Expr> Mutate_(const CallNode* op, InplaceMode inplace_mode) override;
+
+ protected:
+ AnalyzerObj* analyzer_;
+ ScopeStack<WithGroup<ConstraintContext>> constraint_scope_;
+};
+
+} // namespace arith
+} // namespace tvm
+#endif // TVM_ARITH_SIMPLIFY_BASE_H_
diff --git a/src/arith/solve_linear_equation.cc b/src/arith/solve_linear_equation.cc
index c00af8a58b..6f1d2625d9 100644
--- a/src/arith/solve_linear_equation.cc
+++ b/src/arith/solve_linear_equation.cc
@@ -451,7 +451,7 @@ IntConstraintsTransform SolveLinearEquations(const IntConstraints& system_to_sol
// Add the rest conditions
auto f_subst = [&old_to_new_map](const Var& var) -> ffi::Expected<ffi::UnchangedOr<ffi::Any>> {
- if (auto repl = old_to_new_map.Get(var)) return ffi::Any(*std::move(repl));
+ if (auto repl = old_to_new_map.Get(var)) return *std::move(repl);
return ffi::Unchanged();
};
for (const PrimExpr& cond : rest) {
diff --git a/src/arith/solve_linear_inequality.cc b/src/arith/solve_linear_inequality.cc
index 54f1fbc80b..839616090e 100644
--- a/src/arith/solve_linear_inequality.cc
+++ b/src/arith/solve_linear_inequality.cc
@@ -28,9 +28,9 @@
#include <tvm/ffi/extra/structural_mutate.h>
#include <tvm/ffi/function.h>
#include <tvm/ffi/reflection/registry.h>
+#include <tvm/ir/expr_functor.h>
#include <tvm/ir/prim/expr.h>
#include <tvm/tirx/analysis.h>
-#include <tvm/tirx/expr_functor.h>
#include <tvm/tirx/op.h>
#include <utility>
@@ -81,14 +81,26 @@ void DebugPrint(const std::vector<PrimExpr>& current_ineq_set,
/*!
* \brief normalize to the form `expr <= 0`
*/
-class NormalizeComparisons : public ExprMutator {
+class NormalizeComparisons : public tvm::ExprMutator {
public:
- Expr VisitExpr_(const prim::EQNode* op) override { return Make<prim::EQ>(op->a, op->b); }
- Expr VisitExpr_(const prim::NENode* op) override { return Make<prim::NE>(op->a, op->b); }
- Expr VisitExpr_(const prim::LTNode* op) override { return Make<prim::LT>(op->a, op->b); }
- Expr VisitExpr_(const prim::LENode* op) override { return Make<prim::LE>(op->a, op->b); }
- Expr VisitExpr_(const prim::GTNode* op) override { return Make<prim::LT>(op->b, op->a); }
- Expr VisitExpr_(const prim::GENode* op) override { return Make<prim::LE>(op->b, op->a); }
+ UnchangedOr<PrimExpr> Mutate_(const prim::EQNode* op, InplaceMode inplace_mode) override {
+ return Make<prim::EQ>(op->a, op->b);
+ }
+ UnchangedOr<PrimExpr> Mutate_(const prim::NENode* op, InplaceMode inplace_mode) override {
+ return Make<prim::NE>(op->a, op->b);
+ }
+ UnchangedOr<PrimExpr> Mutate_(const prim::LTNode* op, InplaceMode inplace_mode) override {
+ return Make<prim::LT>(op->a, op->b);
+ }
+ UnchangedOr<PrimExpr> Mutate_(const prim::LENode* op, InplaceMode inplace_mode) override {
+ return Make<prim::LE>(op->a, op->b);
+ }
+ UnchangedOr<PrimExpr> Mutate_(const prim::GTNode* op, InplaceMode inplace_mode) override {
+ return Make<prim::LT>(op->b, op->a);
+ }
+ UnchangedOr<PrimExpr> Mutate_(const prim::GENode* op, InplaceMode inplace_mode) override {
+ return Make<prim::LE>(op->b, op->a);
+ }
private:
template <class T>
@@ -219,10 +231,10 @@ PartialSolvedInequalities SolveLinearInequalities(const IntConstraints& system_t
// Simplify each inequality into the form `expr <= 0` and add to current formulas
for (const PrimExpr& ineq : system_to_solve->relations) {
- AddInequality(¤t_ineq_set_to_solve,
- NormalizeComparisons()(analyzer->Simplify(ineq, kSimplifyRewriteCanonicalRewrite))
- .as_or_throw<PrimExpr>(),
- analyzer.get());
+ PrimExpr simplified = analyzer->Simplify(ineq, kSimplifyRewriteCanonicalRewrite);
+ PrimExpr normalized =
+ ffi::make_object<NormalizeComparisons>()->Mutate(simplified).ValueOrUnchanged(simplified);
+ AddInequality(¤t_ineq_set_to_solve, normalized, analyzer.get());
}
ffi::Map<Var, IntGroupBounds> res_bounds;
@@ -262,9 +274,9 @@ PartialSolvedInequalities SolveLinearInequalities(const IntConstraints& system_t
// we need rewrite_simplify -> canonical_simplify -> rewrite_simplify
// to help simplify things like (((y + 10) - (-1*(y - 20))) <= 0) => y - 5 <= 0
// with steps = 2 it's (y*2) - 10 <= 0
+ new_ineq = analyzer->Simplify(new_ineq, kSimplifyRewriteCanonicalRewrite);
new_ineq =
- NormalizeComparisons()(analyzer->Simplify(new_ineq, kSimplifyRewriteCanonicalRewrite))
- .as_or_throw<PrimExpr>();
+ ffi::make_object<NormalizeComparisons>()->Mutate(new_ineq).ValueOrUnchanged(new_ineq);
AddInequality(&next_ineq_set_to_solve, new_ineq, analyzer.get());
}
}
@@ -471,12 +483,12 @@ IntConstraintsTransform SolveInequalitiesDeskewRange(const IntConstraints& inequ
analyzer->Bind(vranges);
auto subst = [&res_src_to_dst](const Var& var) -> ffi::Expected<ffi::UnchangedOr<ffi::Any>> {
- if (auto repl = res_src_to_dst.Get(var)) return ffi::Any(*std::move(repl));
+ if (auto repl = res_src_to_dst.Get(var)) return *std::move(repl);
return ffi::Unchanged();
};
auto f_dst_to_src =
[&res_dst_to_src](const Var& var) -> ffi::Expected<ffi::UnchangedOr<ffi::Any>> {
- if (auto repl = res_dst_to_src.Get(var)) return ffi::Any(*std::move(repl));
+ if (auto repl = res_dst_to_src.Get(var)) return *std::move(repl);
return ffi::Unchanged();
};
diff --git a/src/arith/unwrap_vector_expr.cc b/src/arith/unwrap_vector_expr.cc
index caa5dd6259..7c583fefb0 100644
--- a/src/arith/unwrap_vector_expr.cc
+++ b/src/arith/unwrap_vector_expr.cc
@@ -26,42 +26,127 @@
#include <tvm/arith/analyzer.h>
#include <tvm/ffi/cast.h>
+#include <tvm/ir/expr_functor.h>
#include <tvm/ir/prim/builtin.h>
#include <tvm/ir/prim/expr.h>
#include <tvm/tirx/analysis.h>
+#include <tvm/tirx/buffer.h>
#include <tvm/tirx/builtin.h>
-#include <tvm/tirx/expr_functor.h>
#include <tvm/tirx/op.h>
#include <unordered_map>
+#include <utility>
namespace tvm {
namespace arith {
using namespace tirx;
-class Scalarizer : public ExprMutator {
+class Scalarizer : public tvm::ExprMutator {
public:
explicit Scalarizer(PrimExpr lane) : lane_(lane) {}
- Expr VisitExpr_(const prim::RampNode* op) final { return op->base + lane_ * op->stride; }
+#define TVM_SCALARIZER_BINARY_MUTATE_(Name) \
+ UnchangedOr<PrimExpr> Mutate_(const prim::Name##Node* op, InplaceMode inplace_mode) final { \
+ return Rebuild(op, [](const prim::Name##Node* node) { return prim::Name(node->a, node->b); }); \
+ }
+
+ TVM_SCALARIZER_BINARY_MUTATE_(Add);
+ TVM_SCALARIZER_BINARY_MUTATE_(Sub);
+ TVM_SCALARIZER_BINARY_MUTATE_(Mul);
+ TVM_SCALARIZER_BINARY_MUTATE_(Div);
+ TVM_SCALARIZER_BINARY_MUTATE_(Mod);
+ TVM_SCALARIZER_BINARY_MUTATE_(FloorDiv);
+ TVM_SCALARIZER_BINARY_MUTATE_(FloorMod);
+ TVM_SCALARIZER_BINARY_MUTATE_(Min);
+ TVM_SCALARIZER_BINARY_MUTATE_(Max);
+ TVM_SCALARIZER_BINARY_MUTATE_(EQ);
+ TVM_SCALARIZER_BINARY_MUTATE_(NE);
+ TVM_SCALARIZER_BINARY_MUTATE_(LT);
+ TVM_SCALARIZER_BINARY_MUTATE_(LE);
+ TVM_SCALARIZER_BINARY_MUTATE_(GT);
+ TVM_SCALARIZER_BINARY_MUTATE_(GE);
+ TVM_SCALARIZER_BINARY_MUTATE_(And);
+ TVM_SCALARIZER_BINARY_MUTATE_(Or);
+
+#undef TVM_SCALARIZER_BINARY_MUTATE_
+
+ UnchangedOr<PrimExpr> Mutate_(const prim::CastNode* op, InplaceMode inplace_mode) final {
+ return Rebuild(op, [](const prim::CastNode* node) {
+ return prim::Cast(node->ExprNode::ty.as_or_throw<PrimType>(), node->value);
+ });
+ }
- Expr VisitExpr_(const prim::BroadcastNode* op) final { return op->value; }
+ UnchangedOr<PrimExpr> Mutate_(const prim::NotNode* op, InplaceMode inplace_mode) final {
+ return Rebuild(op, [](const prim::NotNode* node) { return prim::Not(node->a); });
+ }
+
+ UnchangedOr<PrimExpr> Mutate_(const prim::SelectNode* op, InplaceMode inplace_mode) final {
+ return Rebuild(op, [](const prim::SelectNode* node) {
+ return prim::Select(node->condition, node->true_value, node->false_value);
+ });
+ }
- Expr VisitExpr_(const VarNode* op) final {
- Var var = ffi::GetRef<Var>(op);
+ UnchangedOr<PrimExpr> Mutate_(const prim::ShuffleNode* op, InplaceMode inplace_mode) final {
+ return Rebuild(op, [](const prim::ShuffleNode* node) {
+ return prim::Shuffle(node->vectors, node->indices);
+ });
+ }
+
+ UnchangedOr<Expr> Mutate_(const TupleNode* op, InplaceMode inplace_mode) final {
+ return Rebuild(op, [](const TupleNode* node) { return tvm::Tuple(node->fields, node->span); });
+ }
+
+ UnchangedOr<Expr> Mutate_(const TupleGetItemNode* op, InplaceMode inplace_mode) final {
+ return Rebuild(op, [](const TupleGetItemNode* node) {
+ return TupleGetItem(node->tuple, node->index, node->span);
+ });
+ }
+ UnchangedOr<PrimExpr> Mutate_(const TensorLoadNode* op, InplaceMode inplace_mode) final {
+ return Rebuild(op, [](const TensorLoadNode* node) {
+ return tirx::BufferLoad(node->source.as_or_throw<tirx::BufferVar>(), node->indices,
+ node->span);
+ });
+ }
+
+ UnchangedOr<Expr> Mutate_(const CallNode* op, InplaceMode inplace_mode) final {
+ return Rebuild(op, [op](const CallNode* node) -> Expr {
+ if (!op->op.same_as(tirx::builtin::buffer_data()) || node->args.same_as(op->args)) {
+ return ffi::GetRef<Call>(node);
+ }
+ TVM_FFI_ICHECK_EQ(node->args.size(), 1);
+ const auto* buffer_var = node->args[0].as<VarNode>();
+ TVM_FFI_ICHECK(buffer_var);
+ const auto* buffer_type = buffer_var->ty.as<tirx::BufferTypeNode>();
+ TVM_FFI_ICHECK(buffer_type);
+ return Call(buffer_type->DataPointerType(), node->op, node->args, node->attrs, node->ty_args,
+ node->span);
+ });
+ }
+
+ UnchangedOr<PrimExpr> Mutate_(const prim::RampNode* op, InplaceMode inplace_mode) final {
+ return op->base + lane_ * op->stride;
+ }
+
+ UnchangedOr<PrimExpr> Mutate_(const prim::BroadcastNode* op, InplaceMode inplace_mode) final {
+ return op->value;
+ }
+
+ UnchangedOr<Expr> Mutate_(const VarNode* op, InplaceMode inplace_mode) final {
auto it = let_var_remap_.find(op);
if (it != let_var_remap_.end()) {
return it->second;
} else {
- return ExprMutator::VisitExpr_(op);
+ return tvm::ExprMutator::Mutate_(op, inplace_mode);
}
}
- Expr VisitExpr_(const prim::LetNode* op) final {
+ UnchangedOr<PrimExpr> Mutate_(const prim::LetNode* op, InplaceMode inplace_mode) final {
PrimType value_ty = op->value.ty();
if (value_ty.lanes() == 1) {
- return ExprMutator::VisitExpr_(op);
+ return Rebuild(op, [](const prim::LetNode* node) {
+ return prim::Let(node->var, node->value, node->body);
+ });
}
auto it = let_var_remap_.find(op->var.get());
@@ -70,15 +155,31 @@ class Scalarizer : public ExprMutator {
PrimType var_ty = op->var.as_or_throw<PrimVar>().ty();
PrimVar new_var(op->var->name + "_scalar", var_ty.WithLanes(1));
let_var_remap_[op->var.get()] = new_var;
+ struct RemapGuard {
+ std::unordered_map<const VarNode*, PrimVar>& remap;
+ const VarNode* var;
+ ~RemapGuard() { remap.erase(var); }
+ } remap_guard{let_var_remap_, op->var.get()};
- PrimExpr value = this->VisitPrimExpr(op->value);
- PrimExpr body = this->VisitPrimExpr(op->body);
+ PrimExpr value = Mutate(op->value, inplace_mode).ValueOrUnchanged(op->value);
+ PrimExpr body = Mutate(op->body, inplace_mode).ValueOrUnchanged(op->body);
- let_var_remap_.erase(op->var.get());
return prim::Let(op->var, value, body);
}
private:
+ // Scalarized children can change a node's result type. Reuse native child mutation,
+ // then the existing constructors to derive the type and check their invariants.
+ // Disable in-place mutation so changed children cannot be reported as Unchanged.
+ template <typename Node, typename F>
+ auto Rebuild(const Node* op, F rebuild)
+ -> decltype(tvm::ExprMutator::Mutate_(op, InplaceMode::kDisallow)) {
+ auto rewritten_u = tvm::ExprMutator::Mutate_(op, InplaceMode::kDisallow);
+ if (rewritten_u.IsUnchanged()) return ffi::Unchanged();
+ auto rewritten = std::move(rewritten_u).ValueUnchecked();
+ return rebuild(static_cast<const Node*>(rewritten.get()));
+ }
+
// The lane to extract
PrimExpr lane_;
@@ -87,7 +188,7 @@ class Scalarizer : public ExprMutator {
};
PrimExpr UnwrapVectorExpr(const PrimExpr& vector_expr, const PrimExpr& lane) {
- return Scalarizer(lane)(vector_expr).as_or_throw<PrimExpr>();
+ return ffi::make_object<Scalarizer>(lane)->Mutate(vector_expr).ValueOrUnchanged(vector_expr);
}
} // namespace arith
diff --git a/src/arith/z3_prover.cc b/src/arith/z3_prover.cc
index 1229e82db0..f6bf51cc12 100644
--- a/src/arith/z3_prover.cc
+++ b/src/arith/z3_prover.cc
@@ -30,12 +30,12 @@
#include <tvm/arith/analyzer.h>
#include <tvm/ffi/extra/structural_equal.h>
#include <tvm/ffi/extra/structural_hash.h>
+#include <tvm/ir/expr_functor.h>
#include <tvm/ir/prim/builtin.h>
#include <tvm/ir/prim/expr.h>
#include <tvm/runtime/logging.h>
#include <tvm/tirx/analysis.h>
#include <tvm/tirx/builtin.h>
-#include <tvm/tirx/expr_functor.h>
#include <tvm/tirx/op.h>
#include <tvm/tirx/op_attr_types.h>
@@ -108,9 +108,9 @@ void EnterZ3ContextScope() {}
void ExitZ3ContextScope() {}
-class Z3Prover::Impl : ExprFunctor<z3::expr(const Expr&)> {
+class Z3Prover::Impl : public ffi::Object, tvm::ExprFunctor<z3::expr(const Expr&)> {
public:
- using Base = ExprFunctor<z3::expr(const Expr&)>;
+ using Base = tvm::ExprFunctor<z3::expr(const Expr&)>;
using Self = Z3Prover::Impl;
AnalyzerObj* analyzer;
@@ -405,7 +405,7 @@ class Z3Prover::Impl : ExprFunctor<z3::expr(const Expr&)> {
void ApplyBindValue(const Var& var, const PrimExpr& value) {
// we add the binding whenever the value is pure,
- // because non-pure parts are handling by creating free variables in VisitExpr
+ // because non-pure parts are handling by creating free variables in Dispatch
MemoPut(var.as_or_throw<PrimExpr>(), ConvertInt(value));
}
@@ -710,12 +710,12 @@ class Z3Prover::Impl : ExprFunctor<z3::expr(const Expr&)> {
}
/// @brief Visit expression with memoization
- z3::expr VisitExpr(const Expr& expr) override {
+ z3::expr Dispatch(const Expr& expr) override {
PrimExpr e = expr.as_or_throw<PrimExpr>();
if (const z3::expr* hit = MemoGet(e)) {
return *hit;
}
- auto res = Base::VisitExpr(e);
+ auto res = Base::Dispatch(e);
auto side_effect = SideEffect(e);
if (side_effect <= CallEffectKind::kPure) {
MemoPut(e, res);
@@ -754,7 +754,7 @@ class Z3Prover::Impl : ExprFunctor<z3::expr(const Expr&)> {
/// @brief Visit the expression and convert it into z3 integer expression
z3::expr VisitInt(const PrimExpr& expr) {
- auto e = VisitExpr(expr);
+ auto e = Dispatch(expr);
if (e.is_bool()) {
return z3::ite(e, ctx->int_val(1), ctx->int_val(0));
} else {
@@ -764,7 +764,7 @@ class Z3Prover::Impl : ExprFunctor<z3::expr(const Expr&)> {
/// @brief Visit the expression and convert it into z3 boolean expression
z3::expr VisitBool(const PrimExpr& e) {
- auto expr = VisitExpr(e);
+ auto expr = Dispatch(e);
if (expr.is_bool()) {
return expr;
} else {
@@ -781,13 +781,13 @@ class Z3Prover::Impl : ExprFunctor<z3::expr(const Expr&)> {
}
}
- z3::expr VisitExpr_(const prim::LetNode* op) override {
+ z3::expr Dispatch_(const prim::LetNode* op) override {
if (IsZ3SupportedExpr(op->var.get())) {
MemoPut(op->var.as_or_throw<PrimExpr>(), VisitInt(op->value));
}
- return VisitExpr(op->body);
+ return Dispatch(op->body);
}
- z3::expr VisitExpr_(const prim::CastNode* op) override {
+ z3::expr Dispatch_(const prim::CastNode* op) override {
// if the inner dtype is valid, we just visit it
if (IsZ3SupportedExpr(op->value.get()) && IsZ3SupportedExpr(op)) {
return VisitInt(op->value);
@@ -796,14 +796,14 @@ class Z3Prover::Impl : ExprFunctor<z3::expr(const Expr&)> {
return Create(op);
}
}
- z3::expr VisitExpr_(const VarNode* op) override { return Create(op); }
- z3::expr VisitExpr_(const TensorLoadNode* op) override { return Create(op); }
- z3::expr VisitExpr_(const prim::MinNode* op) override {
+ z3::expr Dispatch_(const VarNode* op) override { return Create(op); }
+ z3::expr Dispatch_(const TensorLoadNode* op) override { return Create(op); }
+ z3::expr Dispatch_(const prim::MinNode* op) override {
auto a = VisitInt(op->a);
auto b = VisitInt(op->b);
return z3::ite(a < b, a, b);
}
- z3::expr VisitExpr_(const prim::MaxNode* op) override {
+ z3::expr Dispatch_(const prim::MaxNode* op) override {
auto a = VisitInt(op->a);
auto b = VisitInt(op->b);
return z3::ite(a > b, a, b);
@@ -828,59 +828,59 @@ class Z3Prover::Impl : ExprFunctor<z3::expr(const Expr&)> {
static z3::expr floormod(const z3::expr& a, const z3::expr& b) {
return z3::ite(b > 0, a % b, -((-a) % b));
}
- z3::expr VisitExpr_(const prim::AddNode* op) override {
+ z3::expr Dispatch_(const prim::AddNode* op) override {
return VisitArith(z3::operator+, op, op->a, op->b);
}
- z3::expr VisitExpr_(const prim::SubNode* op) override {
+ z3::expr Dispatch_(const prim::SubNode* op) override {
return VisitArith(z3::operator-, op, op->a, op->b);
}
- z3::expr VisitExpr_(const prim::MulNode* op) override {
+ z3::expr Dispatch_(const prim::MulNode* op) override {
return VisitArith(z3::operator*, op, op->a, op->b);
}
- z3::expr VisitExpr_(const prim::DivNode* op) override {
+ z3::expr Dispatch_(const prim::DivNode* op) override {
return VisitArith(truncdiv, op, op->a, op->b);
}
- z3::expr VisitExpr_(const prim::ModNode* op) override {
+ z3::expr Dispatch_(const prim::ModNode* op) override {
return VisitArith(truncmod, op, op->a, op->b);
}
- z3::expr VisitExpr_(const prim::FloorDivNode* op) override {
+ z3::expr Dispatch_(const prim::FloorDivNode* op) override {
return VisitArith(floordiv, op, op->a, op->b);
}
- z3::expr VisitExpr_(const prim::FloorModNode* op) override {
+ z3::expr Dispatch_(const prim::FloorModNode* op) override {
return VisitArith(floormod, op, op->a, op->b);
}
- z3::expr VisitExpr_(const prim::EQNode* op) override {
+ z3::expr Dispatch_(const prim::EQNode* op) override {
return VisitArith(z3::operator==, op, op->a, op->b);
}
- z3::expr VisitExpr_(const prim::NENode* op) override {
+ z3::expr Dispatch_(const prim::NENode* op) override {
return VisitArith(z3::operator!=, op, op->a, op->b);
}
- z3::expr VisitExpr_(const prim::LTNode* op) override {
+ z3::expr Dispatch_(const prim::LTNode* op) override {
return VisitArith(z3::operator<, op, op->a, op->b);
}
- z3::expr VisitExpr_(const prim::LENode* op) override {
+ z3::expr Dispatch_(const prim::LENode* op) override {
return VisitArith(z3::operator<=, op, op->a, op->b);
}
- z3::expr VisitExpr_(const prim::GTNode* op) override {
+ z3::expr Dispatch_(const prim::GTNode* op) override {
return VisitArith(z3::operator>, op, op->a, op->b);
}
- z3::expr VisitExpr_(const prim::GENode* op) override {
+ z3::expr Dispatch_(const prim::GENode* op) override {
return VisitArith(z3::operator>=, op, op->a, op->b);
}
- z3::expr VisitExpr_(const prim::AndNode* op) override {
+ z3::expr Dispatch_(const prim::AndNode* op) override {
return VisitBool(op->a) && VisitBool(op->b);
}
- z3::expr VisitExpr_(const prim::OrNode* op) override {
+ z3::expr Dispatch_(const prim::OrNode* op) override {
return VisitBool(op->a) || VisitBool(op->b);
}
- z3::expr VisitExpr_(const prim::NotNode* op) override { return !VisitBool(op->a); }
- z3::expr VisitExpr_(const prim::SelectNode* op) override {
+ z3::expr Dispatch_(const prim::NotNode* op) override { return !VisitBool(op->a); }
+ z3::expr Dispatch_(const prim::SelectNode* op) override {
return z3::ite(VisitBool(op->condition), VisitInt(op->true_value), VisitInt(op->false_value));
}
- z3::expr VisitExpr_(const IntImmNode* op) override { return ctx->int_val(op->value); }
+ z3::expr Dispatch_(const IntImmNode* op) override { return ctx->int_val(op->value); }
// Bitwise operations
- z3::expr VisitExpr_(const CallNode* op) override {
+ z3::expr Dispatch_(const CallNode* op) override {
// Check if this is a bitwise operation
if (op->op.same_as(prim::builtin::bitwise_and())) {
return VisitBitwiseOp(z3::operator&, op);
@@ -978,7 +978,7 @@ class Z3Prover::Impl : ExprFunctor<z3::expr(const Expr&)> {
}
}
- z3::expr VisitExprDefault_(const Object* op) override {
+ z3::expr DispatchDefault_(const Object* op) override {
// Z3 is a best-effort fallback that runs only after the native analyzers
// have already failed. An unsupported node must not crash the build, so we
// model it as a fresh unconstrained free variable, which keeps the proof
@@ -1014,8 +1014,8 @@ TVM_DLL int64_t Z3Prover::CountSatisfyingValues(const Var& var, int64_t max_coun
int64_t min_consecutive) {
return impl_->CountSatisfyingValues(var, max_count, min_consecutive);
}
-Z3Prover::Z3Prover(AnalyzerObj* parent) : impl_(new Impl{parent}) {}
-TVM_DLL Z3Prover::~Z3Prover() { delete impl_; }
+Z3Prover::Z3Prover(AnalyzerObj* parent) : impl_(ffi::make_object<Impl>(parent)) {}
+TVM_DLL Z3Prover::~Z3Prover() = default;
} // namespace tvm::arith
@@ -1038,7 +1038,7 @@ void ExitZ3ContextScope() {}
// Stub implementation used when Z3 support is not built. All proving queries
// conservatively report "cannot prove" while keeping the public API available.
-class Z3Prover::Impl {};
+class Z3Prover::Impl : public ffi::Object {};
TVM_DLL bool Z3Prover::IsEnabled() const { return false; }
TVM_DLL bool Z3Prover::CanProve(const PrimExpr& expr) { return false; }
@@ -1061,7 +1061,7 @@ TVM_DLL int64_t Z3Prover::CountSatisfyingValues(const Var& var, int64_t max_coun
void Z3Prover::CopyFrom(const Z3Prover& other) {}
ffi::String Z3Prover::GetStats() { return "; Z3 Prover is disabled."; }
Z3Prover::Z3Prover(AnalyzerObj*) : impl_(nullptr) {}
-TVM_DLL Z3Prover::~Z3Prover() {}
+TVM_DLL Z3Prover::~Z3Prover() = default;
} // namespace tvm::arith
diff --git a/src/backend/trn/transform/lower_trainium_layout.cc b/src/backend/trn/transform/lower_trainium_layout.cc
index 297cdf35c1..8d52070847 100644
--- a/src/backend/trn/transform/lower_trainium_layout.cc
+++ b/src/backend/trn/transform/lower_trainium_layout.cc
@@ -38,7 +38,7 @@
#include <utility>
#include <vector>
-#include "../../../arith/ir_mutator_with_analyzer.h"
+#include "../../../tirx/ir_mutator_with_analyzer.h"
namespace tvm {
namespace tirx {
@@ -53,7 +53,7 @@ static bool IsTrainiumLayout(const TileLayoutNode* layout) {
});
}
-class TrainiumLayoutApplier : public arith::IRMutatorWithAnalyzer {
+class TrainiumLayoutApplier : public tirx::IRMutatorWithAnalyzer {
public:
static std::pair<Stmt, ffi::Array<Var>> Lower(const Stmt& stmt, const ffi::Array<Var>& params) {
arith::Analyzer ana;
@@ -90,7 +90,7 @@ class TrainiumLayoutApplier : public arith::IRMutatorWithAnalyzer {
using IRMutatorWithAnalyzer::VisitStmt_;
explicit TrainiumLayoutApplier(const arith::Analyzer& analyzer)
- : arith::IRMutatorWithAnalyzer(analyzer) {}
+ : tirx::IRMutatorWithAnalyzer(analyzer) {}
ffi::Any VisitAny(const ffi::Any& any) {
if (any == nullptr) {
diff --git a/src/ir/expr_functor.cc b/src/ir/expr_functor.cc
index c4f86f4bd2..637a7ccdb0 100644
--- a/src/ir/expr_functor.cc
+++ b/src/ir/expr_functor.cc
@@ -59,225 +59,219 @@ void ExprVisitor::InitVTable(VTable* vtable) {
SetDispatch<ExprVisitor, prim::ShuffleNode>(vtable);
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const OpaqueExprNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->ty));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const OpaqueExprNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->ty));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const TupleNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->ty));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->fields));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const TupleNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->ty));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->fields));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const TupleGetItemNode* node) {
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const TupleGetItemNode* node) {
// skips: index
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->ty));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->tuple));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->ty));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->tuple));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const TensorLoadNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->ty));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->source));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->indices));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const TensorLoadNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->ty));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->source));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->indices));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const VarNode* node) {
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const VarNode* node) {
// Primitive types have no children; dynamic type fields are visited.
if (!node->ty.as<PrimTypeNode>()) {
// Clamp Simple for dynamic type fields; Pattern continues through them.
if (this->def_region_kind() == kTVMFFIDefRegionKindSimple) {
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->WithDefRegionKind(
- kTVMFFIDefRegionKindNone, [&]() { return this->VisitExpected(node->ty); }));
+ kTVMFFIDefRegionKindNone, [&]() { return this->Visit(node->ty); }));
} else {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->ty));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->ty));
}
}
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const GlobalVarNode* node) {
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const GlobalVarNode* node) {
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const CallNode* node) {
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const CallNode* node) {
// Skip constant attrs and primitive result types.
if (!node->ty.as<PrimTypeNode>()) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->ty));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->ty));
}
// Interned operators have no children; function-valued operators are visited.
if (!node->op.as<OpNode>()) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->op));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->op));
}
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->args));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->args));
// Empty type arguments are skipped, including the container callback.
if (!node->ty_args.empty()) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->ty_args));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->ty_args));
}
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const IntImmNode* node) {
- return std::nullopt;
-}
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const IntImmNode* node) { return std::nullopt; }
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const FloatImmNode* node) {
- return std::nullopt;
-}
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const FloatImmNode* node) { return std::nullopt; }
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const OpNode* node) {
- return std::nullopt;
-}
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const OpNode* node) { return std::nullopt; }
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::StringImmNode* node) {
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::StringImmNode* node) {
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::CastNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->value));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::CastNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->value));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::AddNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::AddNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::SubNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::SubNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::MulNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::MulNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::DivNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::DivNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::ModNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::ModNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::FloorDivNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::FloorDivNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::FloorModNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::FloorModNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::MinNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::MinNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::MaxNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::MaxNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::EQNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::EQNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::NENode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::NENode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::LTNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::LTNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::LENode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::LENode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::GTNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::GTNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::GENode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::GENode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::AndNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::AndNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::OrNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->b));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::OrNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->b));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::NotNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->a));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::NotNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->a));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::SelectNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->condition));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->true_value));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->false_value));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::SelectNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->condition));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->true_value));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->false_value));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::LetNode* node) {
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::LetNode* node) {
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->WithDefRegionKind(
- kTVMFFIDefRegionKindSimple, [&]() { return this->VisitExpected(node->var); }));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->value));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->body));
+ kTVMFFIDefRegionKindSimple, [&]() { return this->Visit(node->var); }));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->value));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->body));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::RampNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->base));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->stride));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->lanes));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::RampNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->base));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->stride));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->lanes));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::BroadcastNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->value));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->lanes));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::BroadcastNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->value));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->lanes));
return std::nullopt;
}
-Expected<ffi::Optional<VisitInterrupt>> ExprVisitor::Visit_(const prim::ShuffleNode* node) {
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->vectors));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->VisitExpected(node->indices));
+ffi::Optional<VisitInterrupt> ExprVisitor::Visit_(const prim::ShuffleNode* node) {
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->vectors));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(this->Visit(node->indices));
return std::nullopt;
}
@@ -320,10 +314,8 @@ void ExprMutator::InitVTable(VTable* vtable) {
SetDispatch<ExprMutator, prim::ShuffleNode>(vtable);
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const OpaqueExprNode* node,
- InplaceMode inplace_mode) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<Type>, ty,
- this->MutateExpected(node->ty, inplace_mode));
+UnchangedOr<Expr> ExprMutator::Mutate_(const OpaqueExprNode* node, InplaceMode inplace_mode) {
+ auto ty = this->Mutate(node->ty, inplace_mode).as_or_throw<UnchangedOr<Type>>();
if (ty.UnchangedOrSameAs(node->ty)) return ffi::Unchanged();
if (inplace_mode == InplaceMode::kAllow) {
auto* writable = const_cast<OpaqueExprNode*>(node);
@@ -332,16 +324,14 @@ Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const OpaqueExprNode* node,
} else {
auto copy = ffi::make_object<OpaqueExprNode>(*node);
copy->ty = std::move(ty).ValueOrUnchanged(std::move(copy->ty));
- return ffi::Any(Expr(std::move(copy)));
+ return Expr(std::move(copy));
}
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const TupleNode* node,
- InplaceMode inplace_mode) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<Type>, ty,
- this->MutateExpected(node->ty, inplace_mode));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<ffi::Array<Expr>>, fields,
- this->MutateExpected(node->fields, inplace_mode));
+UnchangedOr<Expr> ExprMutator::Mutate_(const TupleNode* node, InplaceMode inplace_mode) {
+ auto ty = this->Mutate(node->ty, inplace_mode).as_or_throw<UnchangedOr<Type>>();
+ auto fields =
+ this->Mutate(node->fields, inplace_mode).as_or_throw<UnchangedOr<ffi::Array<Expr>>>();
if (ty.UnchangedOrSameAs(node->ty) && fields.UnchangedOrSameAs(node->fields))
return ffi::Unchanged();
if (inplace_mode == InplaceMode::kAllow) {
@@ -353,16 +343,13 @@ Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const TupleNode* node,
auto copy = ffi::make_object<TupleNode>(*node);
copy->ty = std::move(ty).ValueOrUnchanged(std::move(copy->ty));
copy->fields = std::move(fields).ValueOrUnchanged(std::move(copy->fields));
- return ffi::Any(Expr(std::move(copy)));
+ return Expr(std::move(copy));
}
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const TupleGetItemNode* node,
- InplaceMode inplace_mode) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<Type>, ty,
- this->MutateExpected(node->ty, inplace_mode));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<Expr>, tuple,
- MutateExpected(node->tuple, inplace_mode));
+UnchangedOr<Expr> ExprMutator::Mutate_(const TupleGetItemNode* node, InplaceMode inplace_mode) {
+ auto ty = this->Mutate(node->ty, inplace_mode).as_or_throw<UnchangedOr<Type>>();
+ auto tuple = Mutate(node->tuple, inplace_mode);
if (ty.UnchangedOrSameAs(node->ty) && tuple.UnchangedOrSameAs(node->tuple))
return ffi::Unchanged();
if (inplace_mode == InplaceMode::kAllow) {
@@ -374,18 +361,15 @@ Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const TupleGetItemNode* nod
auto copy = ffi::make_object<TupleGetItemNode>(*node);
copy->ty = std::move(ty).ValueOrUnchanged(std::move(copy->ty));
copy->tuple = std::move(tuple).ValueOrUnchanged(std::move(copy->tuple));
- return ffi::Any(Expr(std::move(copy)));
+ return Expr(std::move(copy));
}
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const TensorLoadNode* node,
- InplaceMode inplace_mode) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<Type>, ty,
- this->MutateExpected(node->ty, inplace_mode));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<Expr>, source,
- MutateExpected(node->source, inplace_mode));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<ffi::Array<PrimExpr>>, indices,
- this->MutateExpected(node->indices, inplace_mode));
+UnchangedOr<PrimExpr> ExprMutator::Mutate_(const TensorLoadNode* node, InplaceMode inplace_mode) {
+ auto ty = this->Mutate(node->ty, inplace_mode).as_or_throw<UnchangedOr<Type>>();
+ auto source = Mutate(node->source, inplace_mode);
+ auto indices =
+ this->Mutate(node->indices, inplace_mode).as_or_throw<UnchangedOr<ffi::Array<PrimExpr>>>();
if (ty.UnchangedOrSameAs(node->ty) && source.UnchangedOrSameAs(node->source) &&
indices.UnchangedOrSameAs(node->indices))
return ffi::Unchanged();
@@ -400,36 +384,31 @@ Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const TensorLoadNode* node,
copy->ty = std::move(ty).ValueOrUnchanged(std::move(copy->ty));
copy->source = std::move(source).ValueOrUnchanged(std::move(copy->source));
copy->indices = std::move(indices).ValueOrUnchanged(std::move(copy->indices));
- return ffi::Any(Expr(std::move(copy)));
+ return PrimExpr(std::move(copy));
}
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const GlobalVarNode* node,
- InplaceMode inplace_mode) {
+UnchangedOr<Expr> ExprMutator::Mutate_(const GlobalVarNode* node, InplaceMode inplace_mode) {
// Registry atoms and constant leaves do not descend into metadata or types.
return ffi::Unchanged();
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const CallNode* node,
- InplaceMode inplace_mode) {
+UnchangedOr<Expr> ExprMutator::Mutate_(const CallNode* node, InplaceMode inplace_mode) {
UnchangedOr<Type> ty = ffi::Unchanged();
if (!node->ty.as<PrimTypeNode>()) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<Type>, mapped_ty,
- this->MutateExpected(node->ty, inplace_mode));
+ auto mapped_ty = this->Mutate(node->ty, inplace_mode).as_or_throw<UnchangedOr<Type>>();
ty = std::move(mapped_ty);
}
UnchangedOr<Expr> op = ffi::Unchanged();
if (!node->op.as<OpNode>()) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<Expr>, mapped_op,
- MutateExpected(node->op, inplace_mode));
+ auto mapped_op = Mutate(node->op, inplace_mode);
op = std::move(mapped_op);
}
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<ffi::Array<Expr>>, args,
- this->MutateExpected(node->args, inplace_mode));
+ auto args = this->Mutate(node->args, inplace_mode).as_or_throw<UnchangedOr<ffi::Array<Expr>>>();
UnchangedOr<ffi::Array<Type>> ty_args = ffi::Unchanged();
if (!node->ty_args.empty()) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<ffi::Array<Type>>, mapped_ty_args,
- this->MutateExpected(node->ty_args, inplace_mode));
+ auto mapped_ty_args =
+ this->Mutate(node->ty_args, inplace_mode).as_or_throw<UnchangedOr<ffi::Array<Type>>>();
ty_args = std::move(mapped_ty_args);
}
if (ty.UnchangedOrSameAs(node->ty) && op.UnchangedOrSameAs(node->op) &&
@@ -448,37 +427,33 @@ Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const CallNode* node,
copy->op = std::move(op).ValueOrUnchanged(std::move(copy->op));
copy->args = std::move(args).ValueOrUnchanged(std::move(copy->args));
copy->ty_args = std::move(ty_args).ValueOrUnchanged(std::move(copy->ty_args));
- return ffi::Any(Expr(std::move(copy)));
+ return Expr(std::move(copy));
}
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const IntImmNode* node,
- InplaceMode inplace_mode) {
+UnchangedOr<PrimExpr> ExprMutator::Mutate_(const IntImmNode* node, InplaceMode inplace_mode) {
// Registry atoms and constant leaves do not descend into metadata or types.
return ffi::Unchanged();
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const FloatImmNode* node,
- InplaceMode inplace_mode) {
+UnchangedOr<PrimExpr> ExprMutator::Mutate_(const FloatImmNode* node, InplaceMode inplace_mode) {
// Registry atoms and constant leaves do not descend into metadata or types.
return ffi::Unchanged();
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const OpNode* node, InplaceMode inplace_mode) {
+UnchangedOr<Expr> ExprMutator::Mutate_(const OpNode* node, InplaceMode inplace_mode) {
// Registry atoms and constant leaves do not descend into metadata or types.
return ffi::Unchanged();
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::StringImmNode* node,
- InplaceMode inplace_mode) {
+UnchangedOr<PrimExpr> ExprMutator::Mutate_(const prim::StringImmNode* node,
+ InplaceMode inplace_mode) {
// Registry atoms and constant leaves do not descend into metadata or types.
return ffi::Unchanged();
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::CastNode* node,
- InplaceMode inplace_mode) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, value,
- MutateExpected(node->value, inplace_mode));
+UnchangedOr<PrimExpr> ExprMutator::Mutate_(const prim::CastNode* node, InplaceMode inplace_mode) {
+ auto value = Mutate(node->value, inplace_mode);
if (value.UnchangedOrSameAs(node->value)) return ffi::Unchanged();
if (inplace_mode == InplaceMode::kAllow) {
auto* writable = const_cast<prim::CastNode*>(node);
@@ -487,17 +462,15 @@ Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::CastNode* node,
} else {
auto copy = ffi::make_object<prim::CastNode>(*node);
copy->value = std::move(value).ValueOrUnchanged(std::move(copy->value));
- return ffi::Any(Expr(std::move(copy)));
+ return PrimExpr(std::move(copy));
}
}
#define TVM_IR_BINARY_MUTATE_IMPL(Name) \
- Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::Name##Node* node, \
- InplaceMode inplace_mode) { \
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, a, \
- MutateExpected(node->a, inplace_mode)); \
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, b, \
- MutateExpected(node->b, inplace_mode)); \
+ UnchangedOr<PrimExpr> ExprMutator::Mutate_(const prim::Name##Node* node, \
+ InplaceMode inplace_mode) { \
+ auto a = Mutate(node->a, inplace_mode); \
+ auto b = Mutate(node->b, inplace_mode); \
if (a.UnchangedOrSameAs(node->a) && b.UnchangedOrSameAs(node->b)) return ffi::Unchanged(); \
if (inplace_mode == InplaceMode::kAllow) { \
auto* writable = const_cast<prim::Name##Node*>(node); \
@@ -508,7 +481,7 @@ Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::CastNode* node,
auto copy = ffi::make_object<prim::Name##Node>(*node); \
copy->a = std::move(a).ValueOrUnchanged(std::move(copy->a)); \
copy->b = std::move(b).ValueOrUnchanged(std::move(copy->b)); \
- return ffi::Any(Expr(std::move(copy))); \
+ return PrimExpr(std::move(copy)); \
} \
}
TVM_IR_BINARY_MUTATE_IMPL(Add)
@@ -530,10 +503,8 @@ TVM_IR_BINARY_MUTATE_IMPL(And)
TVM_IR_BINARY_MUTATE_IMPL(Or)
#undef TVM_IR_BINARY_MUTATE_IMPL
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::NotNode* node,
- InplaceMode inplace_mode) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, a,
- MutateExpected(node->a, inplace_mode));
+UnchangedOr<PrimExpr> ExprMutator::Mutate_(const prim::NotNode* node, InplaceMode inplace_mode) {
+ auto a = Mutate(node->a, inplace_mode);
if (a.UnchangedOrSameAs(node->a)) return ffi::Unchanged();
if (inplace_mode == InplaceMode::kAllow) {
auto* writable = const_cast<prim::NotNode*>(node);
@@ -542,18 +513,14 @@ Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::NotNode* node,
} else {
auto copy = ffi::make_object<prim::NotNode>(*node);
copy->a = std::move(a).ValueOrUnchanged(std::move(copy->a));
- return ffi::Any(Expr(std::move(copy)));
+ return PrimExpr(std::move(copy));
}
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::SelectNode* node,
- InplaceMode inplace_mode) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, condition,
- MutateExpected(node->condition, inplace_mode));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, true_value,
- MutateExpected(node->true_value, inplace_mode));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, false_value,
- MutateExpected(node->false_value, inplace_mode));
+UnchangedOr<PrimExpr> ExprMutator::Mutate_(const prim::SelectNode* node, InplaceMode inplace_mode) {
+ auto condition = Mutate(node->condition, inplace_mode);
+ auto true_value = Mutate(node->true_value, inplace_mode);
+ auto false_value = Mutate(node->false_value, inplace_mode);
if (condition.UnchangedOrSameAs(node->condition) &&
true_value.UnchangedOrSameAs(node->true_value) &&
false_value.UnchangedOrSameAs(node->false_value))
@@ -569,20 +536,16 @@ Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::SelectNode* nod
copy->condition = std::move(condition).ValueOrUnchanged(std::move(copy->condition));
copy->true_value = std::move(true_value).ValueOrUnchanged(std::move(copy->true_value));
copy->false_value = std::move(false_value).ValueOrUnchanged(std::move(copy->false_value));
- return ffi::Any(Expr(std::move(copy)));
+ return PrimExpr(std::move(copy));
}
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::LetNode* node,
- InplaceMode inplace_mode) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<Var>, var,
- WithDefRegionKind(kTVMFFIDefRegionKindSimple, [&] {
- return MutateExpected(node->var, inplace_mode);
- }));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, value,
- MutateExpected(node->value, inplace_mode));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, body,
- MutateExpected(node->body, inplace_mode));
+UnchangedOr<PrimExpr> ExprMutator::Mutate_(const prim::LetNode* node, InplaceMode inplace_mode) {
+ auto var = WithDefRegionKind(kTVMFFIDefRegionKindSimple, [&] {
+ return Mutate(node->var, inplace_mode);
+ }).as_or_throw<UnchangedOr<Var>>();
+ auto value = Mutate(node->value, inplace_mode);
+ auto body = Mutate(node->body, inplace_mode);
if (var.UnchangedOrSameAs(node->var) && value.UnchangedOrSameAs(node->value) &&
body.UnchangedOrSameAs(node->body))
return ffi::Unchanged();
@@ -597,18 +560,14 @@ Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::LetNode* node,
copy->var = std::move(var).ValueOrUnchanged(std::move(copy->var));
copy->value = std::move(value).ValueOrUnchanged(std::move(copy->value));
copy->body = std::move(body).ValueOrUnchanged(std::move(copy->body));
- return ffi::Any(Expr(std::move(copy)));
+ return PrimExpr(std::move(copy));
}
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::RampNode* node,
- InplaceMode inplace_mode) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, base,
- MutateExpected(node->base, inplace_mode));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, stride,
- MutateExpected(node->stride, inplace_mode));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, lanes,
- MutateExpected(node->lanes, inplace_mode));
+UnchangedOr<PrimExpr> ExprMutator::Mutate_(const prim::RampNode* node, InplaceMode inplace_mode) {
+ auto base = Mutate(node->base, inplace_mode);
+ auto stride = Mutate(node->stride, inplace_mode);
+ auto lanes = Mutate(node->lanes, inplace_mode);
if (base.UnchangedOrSameAs(node->base) && stride.UnchangedOrSameAs(node->stride) &&
lanes.UnchangedOrSameAs(node->lanes))
return ffi::Unchanged();
@@ -623,16 +582,14 @@ Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::RampNode* node,
copy->base = std::move(base).ValueOrUnchanged(std::move(copy->base));
copy->stride = std::move(stride).ValueOrUnchanged(std::move(copy->stride));
copy->lanes = std::move(lanes).ValueOrUnchanged(std::move(copy->lanes));
- return ffi::Any(Expr(std::move(copy)));
+ return PrimExpr(std::move(copy));
}
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::BroadcastNode* node,
- InplaceMode inplace_mode) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, value,
- MutateExpected(node->value, inplace_mode));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<PrimExpr>, lanes,
- MutateExpected(node->lanes, inplace_mode));
+UnchangedOr<PrimExpr> ExprMutator::Mutate_(const prim::BroadcastNode* node,
+ InplaceMode inplace_mode) {
+ auto value = Mutate(node->value, inplace_mode);
+ auto lanes = Mutate(node->lanes, inplace_mode);
if (value.UnchangedOrSameAs(node->value) && lanes.UnchangedOrSameAs(node->lanes))
return ffi::Unchanged();
if (inplace_mode == InplaceMode::kAllow) {
@@ -644,16 +601,16 @@ Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::BroadcastNode*
auto copy = ffi::make_object<prim::BroadcastNode>(*node);
copy->value = std::move(value).ValueOrUnchanged(std::move(copy->value));
copy->lanes = std::move(lanes).ValueOrUnchanged(std::move(copy->lanes));
- return ffi::Any(Expr(std::move(copy)));
+ return PrimExpr(std::move(copy));
}
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::ShuffleNode* node,
- InplaceMode inplace_mode) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<ffi::Array<PrimExpr>>, vectors,
- this->MutateExpected(node->vectors, inplace_mode));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<ffi::Array<PrimExpr>>, indices,
- this->MutateExpected(node->indices, inplace_mode));
+UnchangedOr<PrimExpr> ExprMutator::Mutate_(const prim::ShuffleNode* node,
+ InplaceMode inplace_mode) {
+ auto vectors =
+ this->Mutate(node->vectors, inplace_mode).as_or_throw<UnchangedOr<ffi::Array<PrimExpr>>>();
+ auto indices =
+ this->Mutate(node->indices, inplace_mode).as_or_throw<UnchangedOr<ffi::Array<PrimExpr>>>();
if (vectors.UnchangedOrSameAs(node->vectors) && indices.UnchangedOrSameAs(node->indices))
return ffi::Unchanged();
if (inplace_mode == InplaceMode::kAllow) {
@@ -665,49 +622,46 @@ Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const prim::ShuffleNode* no
auto copy = ffi::make_object<prim::ShuffleNode>(*node);
copy->vectors = std::move(vectors).ValueOrUnchanged(std::move(copy->vectors));
copy->indices = std::move(indices).ValueOrUnchanged(std::move(copy->indices));
- return ffi::Any(Expr(std::move(copy)));
+ return PrimExpr(std::move(copy));
}
}
-Expected<UnchangedOr<ffi::Any>> ExprMutator::Mutate_(const VarNode* node,
- InplaceMode inplace_mode) {
+UnchangedOr<Expr> ExprMutator::Mutate_(const VarNode* node, InplaceMode inplace_mode) {
if (node->ty.as<PrimTypeNode>()) return ffi::Unchanged();
if (TVM_FFI_PREDICT_TRUE(var_remap_.empty() && def_region_kind() == kTVMFFIDefRegionKindNone)) {
return ffi::Unchanged();
}
- Expected<ffi::Any> remap_result = VarRemapGetExpected(ffi::AnyView(node));
- TVM_FFI_S_MUTATE_MAYBE_EARLY_RETURN(remap_result);
- if (ffi::details::ExpectedUnsafe::GetData(remap_result).type_index() !=
- ffi::TypeIndex::kTVMFFINone) {
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<Expr>, mapped, std::move(remap_result));
- return mapped;
+ ffi::Any remap_result = VarRemapGet(ffi::AnyView(node));
+ if (remap_result.type_index() != ffi::TypeIndex::kTVMFFINone) {
+ return std::move(remap_result).as_or_throw<UnchangedOr<Expr>>();
}
if (def_region_kind() == kTVMFFIDefRegionKindNone) return ffi::Unchanged();
- UnchangedOr<ffi::Any> result = ffi::Unchanged();
+ UnchangedOr<Expr> result = ffi::Unchanged();
ffi::Any mapped_value = ffi::Unchanged();
// PrimType has no children; dynamic type fields inherit Pattern but are visited outside Simple.
if (!node->ty.as<PrimTypeNode>()) {
- Expected<UnchangedOr<ffi::Any>> mapped_ty_result =
+ UnchangedOr<ffi::Any> mapped_ty_result =
def_region_kind() == kTVMFFIDefRegionKindSimple
? WithDefRegionKind(kTVMFFIDefRegionKindNone,
- [&] { return this->MutateExpected(node->ty, inplace_mode); })
- : this->MutateExpected(node->ty, inplace_mode);
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(UnchangedOr<Type>, mapped_ty, std::move(mapped_ty_result));
+ [&] { return this->Mutate(node->ty, inplace_mode); })
+ : this->Mutate(node->ty, inplace_mode);
+ auto mapped_ty = std::move(mapped_ty_result).as_or_throw<UnchangedOr<Type>>();
if (!mapped_ty.UnchangedOrSameAs(node->ty)) {
+ Expr mapped_expr;
if (inplace_mode == InplaceMode::kAllow) {
const_cast<VarNode*>(node)->ty = std::move(mapped_ty).ValueUnchecked();
- mapped_value = ffi::Any(node);
+ mapped_expr = ffi::GetRef<Expr>(node);
} else {
auto copy = ffi::make_object<VarNode>(*node);
copy->ty = std::move(mapped_ty).ValueUnchecked();
- mapped_value = ffi::Any(std::move(copy));
+ mapped_expr = Expr(std::move(copy));
}
- result = mapped_value;
+ result = mapped_expr;
+ mapped_value = std::move(mapped_expr);
}
}
if (!result.IsUnchanged() || def_region_kind() == kTVMFFIDefRegionKindPattern) {
- auto set_result = VarRemapSetExpected(ffi::AnyView(node), mapped_value);
- TVM_FFI_S_MUTATE_MAYBE_EARLY_RETURN(set_result);
+ VarRemapSet(ffi::AnyView(node), mapped_value);
}
return result;
}
diff --git a/src/s_tir/analysis/domain_touched.cc b/src/s_tir/analysis/domain_touched.cc
index f3d22c1ecc..8852f505dc 100644
--- a/src/s_tir/analysis/domain_touched.cc
+++ b/src/s_tir/analysis/domain_touched.cc
@@ -34,7 +34,7 @@
#include <unordered_map>
#include <unordered_set>
-#include "../../arith/ir_visitor_with_analyzer.h"
+#include "../../tirx/ir_visitor_with_analyzer.h"
namespace tvm {
namespace s_tir {
@@ -63,7 +63,7 @@ using BufferDomainAccess = std::tuple<LoadAccess, StoreAccess, CombinedAccess>;
} // namespace
// Find Read region of the tensor in the stmt.
-class BufferTouchedDomain final : public arith::IRVisitorWithAnalyzer {
+class BufferTouchedDomain final : public tirx::IRVisitorWithAnalyzer {
public:
BufferTouchedDomain(const Stmt& stmt) { operator()(stmt); }
@@ -99,7 +99,7 @@ class BufferTouchedDomain final : public arith::IRVisitorWithAnalyzer {
}
private:
- using Parent = arith::IRVisitorWithAnalyzer;
+ using Parent = tirx::IRVisitorWithAnalyzer;
using Parent::VisitExpr_;
using Parent::VisitStmt_;
diff --git a/src/s_tir/analysis/identify_memcpy.cc b/src/s_tir/analysis/identify_memcpy.cc
index 7b94596c65..12f0025015 100644
--- a/src/s_tir/analysis/identify_memcpy.cc
+++ b/src/s_tir/analysis/identify_memcpy.cc
@@ -38,7 +38,7 @@
#include <string>
#include <variant>
-#include "../../arith/ir_visitor_with_analyzer.h"
+#include "../../tirx/ir_visitor_with_analyzer.h"
namespace tvm {
namespace s_tir {
@@ -297,7 +297,7 @@ TVM_FFI_STATIC_INIT_BLOCK() {
refl::GlobalDef().def("s_tir.analysis._identify_memcpy", [](const Stmt& stmt) {
ffi::Array<ffi::ObjectRef> output;
- struct Visitor : arith::IRVisitorWithAnalyzer {
+ struct Visitor : tirx::IRVisitorWithAnalyzer {
explicit Visitor(ffi::Array<ffi::ObjectRef>* output) : output(output) {}
ffi::Array<ffi::ObjectRef>* output;
diff --git a/src/s_tir/analysis/oob_checker.cc b/src/s_tir/analysis/oob_checker.cc
index 04c44ff5a3..b193d0b1ba 100644
--- a/src/s_tir/analysis/oob_checker.cc
+++ b/src/s_tir/analysis/oob_checker.cc
@@ -24,7 +24,7 @@
#include <tvm/ffi/reflection/registry.h>
#include <tvm/tirx/transform.h>
-#include "../../arith/ir_visitor_with_analyzer.h"
+#include "../../tirx/ir_visitor_with_analyzer.h"
#include "../schedule/error.h"
namespace tvm {
@@ -70,7 +70,7 @@ class OOBError : public s_tir::ScheduleError {
IRModule mod_;
std::vector<OOBLocation> locations_;
};
-class OOBCheckerVisitor final : public arith::IRVisitorWithAnalyzer {
+class OOBCheckerVisitor final : public tirx::IRVisitorWithAnalyzer {
using IRVisitorWithAnalyzer::VisitExpr_;
using IRVisitorWithAnalyzer::VisitStmt_;
diff --git a/src/s_tir/backend/adreno/inject_texture_alloc.cc b/src/s_tir/backend/adreno/inject_texture_alloc.cc
index a55e17d200..d7be10b4da 100644
--- a/src/s_tir/backend/adreno/inject_texture_alloc.cc
+++ b/src/s_tir/backend/adreno/inject_texture_alloc.cc
@@ -26,8 +26,8 @@
#include <tvm/tirx/analysis.h>
#include <tvm/tirx/stmt_functor.h>
-#include "../../../arith/ir_mutator_with_analyzer.h"
#include "../../../backend/opencl/runtime/texture.h"
+#include "../../../tirx/ir_mutator_with_analyzer.h"
#include "../../../tirx/transform/ir_utils.h"
namespace tvm {
@@ -43,7 +43,7 @@ using runtime::IsTextureStorage;
/*!
* \brief Inject Texture Alloc Intrinsic right after AllocBufferNode are realized.
*/
-class TextureAllocInjector : public arith::IRMutatorWithAnalyzer {
+class TextureAllocInjector : public tirx::IRMutatorWithAnalyzer {
public:
static PrimFunc Inject(PrimFunc func) {
arith::Analyzer ana;
diff --git a/src/s_tir/backend/adreno/texture_flatten.cc b/src/s_tir/backend/adreno/texture_flatten.cc
index 21a8d5b172..ac3cb6cd1f 100644
--- a/src/s_tir/backend/adreno/texture_flatten.cc
+++ b/src/s_tir/backend/adreno/texture_flatten.cc
@@ -33,9 +33,9 @@
#include <unordered_map>
-#include "../../../arith/ir_visitor_with_analyzer.h"
#include "../../../backend/opencl/runtime/texture.h"
#include "../../../runtime/thread_storage_scope.h"
+#include "../../../tirx/ir_visitor_with_analyzer.h"
namespace tvm {
namespace s_tir {
@@ -43,7 +43,6 @@ using namespace tvm::prim;
namespace backend {
namespace adreno {
using namespace tvm::tirx;
-using arith::IRVisitorWithAnalyzer;
using runtime::ApplyTexture2DFlattening;
using runtime::DefaultTextureLayoutSeparator;
using runtime::IsTextureStorage;
diff --git a/src/s_tir/schedule/primitive/layout_transformation.cc b/src/s_tir/schedule/primitive/layout_transformation.cc
index 9be86ca6e1..6e09125efe 100644
--- a/src/s_tir/schedule/primitive/layout_transformation.cc
+++ b/src/s_tir/schedule/primitive/layout_transformation.cc
@@ -26,7 +26,7 @@
#include <optional>
#include <variant>
-#include "../../../arith/ir_mutator_with_analyzer.h"
+#include "../../../tirx/ir_mutator_with_analyzer.h"
#include "../utils.h"
namespace tvm {
@@ -783,7 +783,7 @@ class ReuseBlocksCollector : public tirx::StmtVisitor {
ffi::Map<SBlock, SBlock> new_block_to_old_;
};
-class TransformLayoutRewriter : private arith::IRMutatorWithAnalyzer {
+class TransformLayoutRewriter : private tirx::IRMutatorWithAnalyzer {
public:
/*!
* \brief Rewrite the access to the buffer after the transformation
@@ -840,7 +840,7 @@ class TransformLayoutRewriter : private arith::IRMutatorWithAnalyzer {
*indices = this->IterMapSimplifyWithContext(*indices, true);
}
- using Parent = arith::IRMutatorWithAnalyzer;
+ using Parent = tirx::IRMutatorWithAnalyzer;
using Parent::VisitExpr_;
using Parent::VisitStmt_;
diff --git a/src/s_tir/schedule/transform.cc b/src/s_tir/schedule/transform.cc
index 0777a4d322..424a0128bd 100644
--- a/src/s_tir/schedule/transform.cc
+++ b/src/s_tir/schedule/transform.cc
@@ -458,7 +458,7 @@ void BlockBufferAccessSimplifier::SimplifyBufferIndices(ffi::Array<PrimExpr>* in
}
Stmt BlockBufferAccessSimplifier::VisitStmt_(const SBlockNode* op) {
- SBlock block = arith::IRMutatorWithAnalyzer::VisitStmt_(op).as_or_throw<SBlock>();
+ SBlock block = tirx::IRMutatorWithAnalyzer::VisitStmt_(op).as_or_throw<SBlock>();
auto* n = block.CopyOnWrite();
SimplifyAccessRegion(&n->reads);
SimplifyAccessRegion(&n->writes);
@@ -466,13 +466,13 @@ Stmt BlockBufferAccessSimplifier::VisitStmt_(const SBlockNode* op) {
}
Stmt BlockBufferAccessSimplifier::VisitStmt_(const BufferStoreNode* op) {
- BufferStore node = arith::IRMutatorWithAnalyzer::VisitStmt_(op).as_or_throw<BufferStore>();
+ BufferStore node = tirx::IRMutatorWithAnalyzer::VisitStmt_(op).as_or_throw<BufferStore>();
SimplifyBufferIndices(&node.CopyOnWrite()->indices);
return node;
}
Expr BlockBufferAccessSimplifier::VisitExpr_(const TensorLoadNode* op) {
- TensorLoad node = arith::IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<TensorLoad>();
+ TensorLoad node = tirx::IRMutatorWithAnalyzer::VisitExpr_(op).as_or_throw<TensorLoad>();
SimplifyBufferIndices(&node.CopyOnWrite()->indices);
return node;
}
diff --git a/src/s_tir/schedule/transform.h b/src/s_tir/schedule/transform.h
index 8c67362ed1..2e108f6514 100644
--- a/src/s_tir/schedule/transform.h
+++ b/src/s_tir/schedule/transform.h
@@ -27,8 +27,8 @@
#include <unordered_map>
#include <utility>
-#include "../../arith/ir_mutator_with_analyzer.h"
#include "../../tirx/ir/functor_common.h"
+#include "../../tirx/ir_mutator_with_analyzer.h"
namespace tvm {
namespace s_tir {
@@ -236,7 +236,7 @@ ffi::Optional<s_tir::LoopRV> TileWithTensorIntrin(const s_tir::Schedule& sch,
/*!
* \brief Simplifier for indices of buffer access and block buffer access regions.
*/
-class BlockBufferAccessSimplifier : public arith::IRMutatorWithAnalyzer {
+class BlockBufferAccessSimplifier : public tirx::IRMutatorWithAnalyzer {
public:
/*!
* \brief Simplify indices of buffer access and block buffer access regions in the statement
diff --git a/src/s_tir/transform/hoist_expression.cc b/src/s_tir/transform/hoist_expression.cc
index 40a4b47daf..280c588932 100644
--- a/src/s_tir/transform/hoist_expression.cc
+++ b/src/s_tir/transform/hoist_expression.cc
@@ -36,8 +36,8 @@
#include <utility>
#include "../../arith/interval_set.h"
-#include "../../arith/ir_mutator_with_analyzer.h"
#include "../../runtime/thread_storage_scope.h"
+#include "../../tirx/ir_mutator_with_analyzer.h"
#include "../../tirx/transform/ir_utils.h"
namespace tvm {
@@ -451,7 +451,7 @@ class HoistInfoCollector : public StmtExprVisitor {
std::unordered_set<const VarNode*> active_loop_vars;
};
-class ExpressionHoister : public arith::IRMutatorWithAnalyzer {
+class ExpressionHoister : public tirx::IRMutatorWithAnalyzer {
public:
static Stmt Hoist(Stmt stmt, HoistExpressionConfig config) {
auto loop_info = HoistInfoCollector::Collect(stmt, config);
@@ -464,7 +464,7 @@ class ExpressionHoister : public arith::IRMutatorWithAnalyzer {
}
private:
- using Parent = arith::IRMutatorWithAnalyzer;
+ using Parent = tirx::IRMutatorWithAnalyzer;
using Parent::VisitExpr_;
using Parent::VisitStmt_;
diff --git a/src/s_tir/transform/inject_permuted_layout.cc b/src/s_tir/transform/inject_permuted_layout.cc
index 46cc35f37d..2a396c0002 100644
--- a/src/s_tir/transform/inject_permuted_layout.cc
+++ b/src/s_tir/transform/inject_permuted_layout.cc
@@ -29,9 +29,9 @@
#include <tvm/tirx/op.h>
#include <tvm/tirx/stmt_functor.h>
-#include "../../arith/ir_mutator_with_analyzer.h"
#include "../../runtime/thread_storage_scope.h"
#include "../../support/utils.h"
+#include "../../tirx/ir_mutator_with_analyzer.h"
#include "../../tirx/transform/ir_utils.h"
namespace tvm {
diff --git a/src/s_tir/transform/inject_virtual_thread.cc b/src/s_tir/transform/inject_virtual_thread.cc
index c3dced0308..a188f66ade 100644
--- a/src/s_tir/transform/inject_virtual_thread.cc
+++ b/src/s_tir/transform/inject_virtual_thread.cc
@@ -33,7 +33,7 @@
#include <unordered_set>
-#include "../../arith/ir_mutator_with_analyzer.h"
+#include "../../tirx/ir_mutator_with_analyzer.h"
#include "../../tirx/transform/ir_utils.h"
namespace tvm {
@@ -215,7 +215,7 @@ class VarTouchedAnalysis : public StmtVisitor {
// Inject virtual thread loop
// rewrite the buffer access pattern when necessary.
-class VTInjector : public arith::IRMutatorWithAnalyzer {
+class VTInjector : public tirx::IRMutatorWithAnalyzer {
public:
using IRMutatorWithAnalyzer::VisitExpr_;
using IRMutatorWithAnalyzer::VisitStmt_;
@@ -616,7 +616,7 @@ class VTInjector : public arith::IRMutatorWithAnalyzer {
std::unordered_map<BufferVar, BufferVar, ffi::ObjectPtrHash, ffi::ObjectPtrEqual> buf_remap_;
};
-class VirtualThreadInjector : public arith::IRMutatorWithAnalyzer {
+class VirtualThreadInjector : public tirx::IRMutatorWithAnalyzer {
public:
using IRMutatorWithAnalyzer::IRMutatorWithAnalyzer;
using IRMutatorWithAnalyzer::VisitStmt_;
diff --git a/src/s_tir/transform/lower_async_dma.cc b/src/s_tir/transform/lower_async_dma.cc
index ae6c61e889..06d84f6a1f 100644
--- a/src/s_tir/transform/lower_async_dma.cc
+++ b/src/s_tir/transform/lower_async_dma.cc
@@ -22,7 +22,6 @@
*/
#include <tvm/arith/analyzer.h>
-#include <tvm/arith/bound.h>
#include <tvm/arith/iter_affine_map.h>
#include <tvm/ffi/cast.h>
#include <tvm/ffi/reflection/registry.h>
@@ -37,7 +36,7 @@
#include <optional>
#include <set>
-#include "../../arith/ir_mutator_with_analyzer.h"
+#include "../../tirx/ir_mutator_with_analyzer.h"
#include "../../tirx/transform/ir_utils.h"
namespace tvm {
@@ -45,7 +44,7 @@ namespace s_tir {
using namespace tvm::prim;
using namespace tvm::tirx;
-class AsyncDMALowerer : public arith::IRMutatorWithAnalyzer {
+class AsyncDMALowerer : public tirx::IRMutatorWithAnalyzer {
public:
explicit AsyncDMALowerer(bool dma_bypass_cache, const arith::Analyzer& analyzer)
: IRMutatorWithAnalyzer(analyzer), dma_bypass_cache_(dma_bypass_cache) {}
@@ -54,7 +53,7 @@ class AsyncDMALowerer : public arith::IRMutatorWithAnalyzer {
Stmt VisitStmt_(const ForNode* loop) final {
// if for loop is not within async_commit_queue_scope
if (!async_queue_id_.has_value()) {
- return arith::IRMutatorWithAnalyzer::VisitStmt_(loop);
+ return tirx::IRMutatorWithAnalyzer::VisitStmt_(loop);
}
// if for loop is not a memcpy of a contiguous region, it might be a cuda cp.async behavior
@@ -62,7 +61,7 @@ class AsyncDMALowerer : public arith::IRMutatorWithAnalyzer {
s_tir::IdentifyMemCpy(ffi::GetRef<For>(loop), ffi::GetRef<arith::Analyzer>(analyzer_));
if (!mem_copy.has_value() || mem_copy->dest->region.size() != 1 ||
mem_copy->source->region.size() != 1) {
- return arith::IRMutatorWithAnalyzer::VisitStmt_(loop);
+ return tirx::IRMutatorWithAnalyzer::VisitStmt_(loop);
}
// now that we are about to perform the `copy` transform
@@ -93,7 +92,7 @@ class AsyncDMALowerer : public arith::IRMutatorWithAnalyzer {
Stmt VisitStmt_(const AttrStmtNode* op) final {
// populate analyzer knowledge of loop iterators
- auto previsit = arith::IRMutatorWithAnalyzer::VisitStmt_(op);
+ auto previsit = tirx::IRMutatorWithAnalyzer::VisitStmt_(op);
// Convert this, for example:
// attr [0] "async_wait_queue_scope" = 0;
@@ -131,7 +130,7 @@ class AsyncDMALowerer : public arith::IRMutatorWithAnalyzer {
.as_or_throw<PrimExpr>());
// concatenate the call with the body and return
- return SeqStmt({call_dma_wait, arith::IRMutatorWithAnalyzer::VisitStmt(async_wait->body)});
+ return SeqStmt({call_dma_wait, tirx::IRMutatorWithAnalyzer::VisitStmt(async_wait->body)});
// Convert this, for example:
// attr [0] "async_commit_queue_scope" = 0;
@@ -153,7 +152,7 @@ class AsyncDMALowerer : public arith::IRMutatorWithAnalyzer {
auto queue_id_node = op->value.as<IntImmNode>();
TVM_FFI_ICHECK(queue_id_node);
async_queue_id_ = queue_id_node->value;
- auto result = arith::IRMutatorWithAnalyzer::VisitStmt_(op);
+ auto result = tirx::IRMutatorWithAnalyzer::VisitStmt_(op);
if (dmas_in_group_ > 1) {
auto call_dma_start_group =
Evaluate(Call(PrimType::Int(32), tirx::builtin::dma_start_group(),
@@ -169,7 +168,7 @@ class AsyncDMALowerer : public arith::IRMutatorWithAnalyzer {
dmas_in_group_ = 0;
return result;
}
- return arith::IRMutatorWithAnalyzer::VisitStmt_(op);
+ return tirx::IRMutatorWithAnalyzer::VisitStmt_(op);
}
private:
diff --git a/src/s_tir/transform/lower_vtcm_alloc.cc b/src/s_tir/transform/lower_vtcm_alloc.cc
index a7e20cd893..8ad6190846 100644
--- a/src/s_tir/transform/lower_vtcm_alloc.cc
+++ b/src/s_tir/transform/lower_vtcm_alloc.cc
@@ -22,8 +22,7 @@
#include <tvm/s_tir/transform.h>
#include <tvm/tirx/builtin.h>
#include <tvm/tirx/stmt.h>
-
-#include "../../arith/ir_visitor_with_analyzer.h"
+#include <tvm/tirx/stmt_functor.h>
namespace tvm {
namespace s_tir {
diff --git a/src/s_tir/transform/renormalize_split_pattern.cc b/src/s_tir/transform/renormalize_split_pattern.cc
index 4f1c52fb37..cf40dc794f 100644
--- a/src/s_tir/transform/renormalize_split_pattern.cc
+++ b/src/s_tir/transform/renormalize_split_pattern.cc
@@ -29,8 +29,8 @@
#include <tvm/tirx/stmt.h>
#include <tvm/tirx/stmt_functor.h>
-#include "../../arith/ir_mutator_with_analyzer.h"
#include "../../arith/pattern_match.h"
+#include "../../tirx/ir_mutator_with_analyzer.h"
namespace tvm {
namespace s_tir {
diff --git a/src/s_tir/transform/using_assume_to_reduce_branches.cc b/src/s_tir/transform/using_assume_to_reduce_branches.cc
index bab4f053ee..bb193a2f45 100644
--- a/src/s_tir/transform/using_assume_to_reduce_branches.cc
+++ b/src/s_tir/transform/using_assume_to_reduce_branches.cc
@@ -49,7 +49,7 @@
#include <unordered_map>
#include "../../arith/constraint_extract.h"
-#include "../../arith/ir_mutator_with_analyzer.h"
+#include "../../tirx/ir_mutator_with_analyzer.h"
#include "tvm/ir/expr.h"
namespace tvm {
namespace s_tir {
diff --git a/src/arith/ir_mutator_with_analyzer.cc b/src/tirx/ir_mutator_with_analyzer.cc
similarity index 69%
rename from src/arith/ir_mutator_with_analyzer.cc
rename to src/tirx/ir_mutator_with_analyzer.cc
index 867826d7d3..5eb27da46e 100644
--- a/src/arith/ir_mutator_with_analyzer.cc
+++ b/src/tirx/ir_mutator_with_analyzer.cc
@@ -18,141 +18,24 @@
*/
/*!
- * \file tvm/arith/ir_mutator_with_analyzer.cc
+ * \file tirx/ir_mutator_with_analyzer.cc
*/
#include "ir_mutator_with_analyzer.h"
#include <tvm/arith/iter_affine_map.h>
#include <tvm/ffi/cast.h>
#include <tvm/ir/op.h>
-#include <tvm/ir/prim/builtin.h>
#include <tvm/s_tir/stmt.h>
#include <tvm/tirx/analysis.h>
#include <tvm/tirx/builtin.h>
#include <tvm/tirx/op.h>
-#include <vector>
+#include "../arith/constraint_helpers.h"
namespace tvm {
-namespace arith {
+namespace tirx {
-using namespace tirx;
-
-namespace {
-
-enum class CompareKind { kEQ, kLT, kLE, kGT, kGE };
-
-bool TryGetIntImm(const PrimExpr& expr, int64_t* value) {
- if (const auto* imm = expr.as<IntImmNode>()) {
- *value = imm->value;
- return true;
- }
- return false;
-}
-
-void AppendFloorDivConstraints(const prim::FloorDivNode* div, int64_t value, CompareKind kind,
- std::vector<PrimExpr>* out) {
- int64_t divisor_value = 0;
- if (!TryGetIntImm(div->b, &divisor_value) || divisor_value <= 0) return;
-
- PrimType dtype = div->a.ty();
- PrimExpr divisor = IntImm(dtype, divisor_value);
- PrimExpr k = IntImm(dtype, value);
- PrimExpr lo = k * divisor;
- PrimExpr hi = (k + IntImm(dtype, 1)) * divisor;
-
- switch (kind) {
- case CompareKind::kEQ:
- out->push_back(div->a >= lo);
- out->push_back(div->a < hi);
- break;
- case CompareKind::kLT:
- out->push_back(div->a < lo);
- break;
- case CompareKind::kLE:
- out->push_back(div->a < hi);
- break;
- case CompareKind::kGT:
- out->push_back(div->a >= hi);
- break;
- case CompareKind::kGE:
- out->push_back(div->a >= lo);
- break;
- }
-}
-
-CompareKind InvertCompare(CompareKind kind) {
- switch (kind) {
- case CompareKind::kEQ:
- return CompareKind::kEQ;
- case CompareKind::kLT:
- return CompareKind::kGT;
- case CompareKind::kLE:
- return CompareKind::kGE;
- case CompareKind::kGT:
- return CompareKind::kLT;
- case CompareKind::kGE:
- return CompareKind::kLE;
- }
- return CompareKind::kEQ;
-}
-
-void CollectFloorDivConstraintsFromCompare(const PrimExpr& lhs, const PrimExpr& rhs,
- CompareKind kind, std::vector<PrimExpr>* out) {
- int64_t value = 0;
- if (const auto* div = lhs.as<prim::FloorDivNode>()) {
- if (TryGetIntImm(rhs, &value)) AppendFloorDivConstraints(div, value, kind, out);
- }
- if (const auto* div = rhs.as<prim::FloorDivNode>()) {
- if (TryGetIntImm(lhs, &value)) {
- AppendFloorDivConstraints(div, value, InvertCompare(kind), out);
- }
- }
-}
-
-void CollectDerivedConstraintFacts(const PrimExpr& condition, std::vector<PrimExpr>* out) {
- if (const auto* and_node = condition.as<prim::AndNode>()) {
- CollectDerivedConstraintFacts(and_node->a, out);
- CollectDerivedConstraintFacts(and_node->b, out);
- return;
- }
- if (const auto* call = condition.as<CallNode>()) {
- if (call->op.same_as(prim::builtin::bitwise_and()) && call->args.size() == 2) {
- PrimExpr lhs = call->args[0].as_or_throw<PrimExpr>();
- PrimExpr rhs = call->args[1].as_or_throw<PrimExpr>();
- if (lhs.ty().MatchesElementType(DLDataTypeCode::kDLBool, 8) &&
- rhs.ty().MatchesElementType(DLDataTypeCode::kDLBool, 8)) {
- CollectDerivedConstraintFacts(lhs, out);
- CollectDerivedConstraintFacts(rhs, out);
- return;
- }
- }
- }
- if (const auto* eq = condition.as<prim::EQNode>()) {
- CollectFloorDivConstraintsFromCompare(eq->a, eq->b, CompareKind::kEQ, out);
- } else if (const auto* lt = condition.as<prim::LTNode>()) {
- CollectFloorDivConstraintsFromCompare(lt->a, lt->b, CompareKind::kLT, out);
- } else if (const auto* le = condition.as<prim::LENode>()) {
- CollectFloorDivConstraintsFromCompare(le->a, le->b, CompareKind::kLE, out);
- } else if (const auto* gt = condition.as<prim::GTNode>()) {
- CollectFloorDivConstraintsFromCompare(gt->a, gt->b, CompareKind::kGT, out);
- } else if (const auto* ge = condition.as<prim::GENode>()) {
- CollectFloorDivConstraintsFromCompare(ge->a, ge->b, CompareKind::kGE, out);
- }
-}
-
-void EnterConstraintFacts(WithGroup<ConstraintContext>* constraints, AnalyzerObj* analyzer,
- const PrimExpr& condition) {
- arith::Analyzer analyzer_ref = ffi::GetRef<arith::Analyzer>(analyzer);
- constraints->Emplace(analyzer_ref, condition);
- std::vector<PrimExpr> derived;
- CollectDerivedConstraintFacts(condition, &derived);
- for (const PrimExpr& fact : derived) {
- constraints->Emplace(analyzer_ref, fact);
- }
-}
-
-} // namespace
+using arith::detail::EnterConstraintFacts;
void IRMutatorWithAnalyzer::MarkBufferParamShapes(const tirx::PrimFunc& func) {
// Mark all symbolic buffer-parameter shape values as positive.
@@ -393,5 +276,5 @@ Expr IRMutatorWithAnalyzer::VisitExpr_(const prim::SelectNode* op) {
}
}
-} // namespace arith
+} // namespace tirx
} // namespace tvm
diff --git a/src/arith/ir_mutator_with_analyzer.h b/src/tirx/ir_mutator_with_analyzer.h
similarity index 75%
rename from src/arith/ir_mutator_with_analyzer.h
rename to src/tirx/ir_mutator_with_analyzer.h
index d05e8e9a5f..c96ce8b17e 100644
--- a/src/arith/ir_mutator_with_analyzer.h
+++ b/src/tirx/ir_mutator_with_analyzer.h
@@ -18,11 +18,11 @@
*/
/*!
- * \file tvm/arithmetic/ir_mutator_with_analyzer.h
+ * \file tirx/ir_mutator_with_analyzer.h
* \brief IR mutator base-class with an analyzer context.
*/
-#ifndef TVM_ARITH_IR_MUTATOR_WITH_ANALYZER_H_
-#define TVM_ARITH_IR_MUTATOR_WITH_ANALYZER_H_
+#ifndef TVM_TIRX_IR_MUTATOR_WITH_ANALYZER_H_
+#define TVM_TIRX_IR_MUTATOR_WITH_ANALYZER_H_
#include <tvm/arith/analyzer.h>
#include <tvm/ffi/cast.h>
@@ -36,7 +36,7 @@
#include <utility>
namespace tvm {
-namespace arith {
+namespace tirx {
/*!
* \brief IRMutator with an analyzer context.
@@ -45,24 +45,24 @@ namespace arith {
* It will populates scope-related info such as bounds of loop-variables and constraints
* for the analyzer, so that the child class can do accurate context-dependent analysis.
*
- * \sa src/arithmetic/ir_mutator_with_analyzer.cc
+ * \sa src/tirx/ir_mutator_with_analyzer.cc
*/
-class IRMutatorWithAnalyzer : public tirx::StmtExprMutator {
+class IRMutatorWithAnalyzer : public StmtExprMutator {
public:
- explicit IRMutatorWithAnalyzer(const Analyzer& analyzer) : analyzer_(analyzer.get()) {}
- explicit IRMutatorWithAnalyzer(AnalyzerObj* analyzer) : analyzer_(analyzer) {}
+ explicit IRMutatorWithAnalyzer(const arith::Analyzer& analyzer) : analyzer_(analyzer.get()) {}
+ explicit IRMutatorWithAnalyzer(arith::AnalyzerObj* analyzer) : analyzer_(analyzer) {}
using StmtExprMutator::VisitExpr_;
using StmtExprMutator::VisitStmt_;
// override functions that need to populate the context information.
- tirx::Stmt VisitStmt_(const tirx::ForNode* op) override;
- tirx::Stmt VisitStmt_(const tirx::SBlockNode* op) override;
- tirx::Stmt VisitStmt_(const tirx::BindNode* op) override;
- tirx::Stmt VisitStmt_(const tirx::IfThenElseNode* op) override;
- tirx::Stmt VisitStmt_(const tirx::AttrStmtNode* op) override;
- tirx::Stmt VisitStmt_(const tirx::AssertStmtNode* op) override;
- tirx::Stmt VisitStmt_(const tirx::SeqStmtNode* op) override;
+ Stmt VisitStmt_(const ForNode* op) override;
+ Stmt VisitStmt_(const SBlockNode* op) override;
+ Stmt VisitStmt_(const BindNode* op) override;
+ Stmt VisitStmt_(const IfThenElseNode* op) override;
+ Stmt VisitStmt_(const AttrStmtNode* op) override;
+ Stmt VisitStmt_(const AssertStmtNode* op) override;
+ Stmt VisitStmt_(const SeqStmtNode* op) override;
Expr VisitExpr_(const prim::LetNode* op) override;
Expr VisitExpr_(const prim::SelectNode* op) override;
Expr VisitExpr_(const CallNode* op) override;
@@ -74,7 +74,7 @@ class IRMutatorWithAnalyzer : public tirx::StmtExprMutator {
* \note call this function before Visit function's body to maximize
* simplification efficiency
*/
- void MarkBufferParamShapes(const tirx::PrimFunc& func);
+ void MarkBufferParamShapes(const PrimFunc& func);
/*!
* \brief Use internal bound information to perform inter map simplification of indices.
@@ -84,15 +84,15 @@ class IRMutatorWithAnalyzer : public tirx::StmtExprMutator {
bool non_trivial_only);
/*! \brief internal analyzer field. */
- AnalyzerObj* analyzer_;
+ arith::AnalyzerObj* analyzer_;
/*! \brief Scope stack for accumulated assert constraints. */
- ScopeStack<WithGroup<ConstraintContext>> constraint_scope_;
+ ScopeStack<WithGroup<arith::ConstraintContext>> constraint_scope_;
// the following two fields are useful in case we want
// note however that iter map analysis are usually more
// expensive and we only encourage doing them during
// necessary cases like layout remapping
/*! \brief Recorded loop iterators */
- ffi::Map<tirx::PrimVar, Range> iter_vars_;
+ ffi::Map<PrimVar, Range> iter_vars_;
/*! \brief iterator predicates */
ffi::Array<PrimExpr> iter_predicates_;
/*!
@@ -102,7 +102,7 @@ class IRMutatorWithAnalyzer : public tirx::StmtExprMutator {
*/
template <typename FLambda>
void WithRecordIterPredicate(PrimExpr condition, FLambda callback) {
- std::unordered_set<const tirx::VarNode*> iter_var_nodes;
+ std::unordered_set<const VarNode*> iter_var_nodes;
for (const auto& [var, _] : iter_vars_) {
iter_var_nodes.insert(var.get());
}
@@ -123,6 +123,6 @@ class IRMutatorWithAnalyzer : public tirx::StmtExprMutator {
}
}
};
-} // namespace arith
+} // namespace tirx
} // namespace tvm
-#endif // TVM_ARITH_IR_MUTATOR_WITH_ANALYZER_H_
+#endif // TVM_TIRX_IR_MUTATOR_WITH_ANALYZER_H_
diff --git a/src/arith/ir_visitor_with_analyzer.cc b/src/tirx/ir_visitor_with_analyzer.cc
similarity index 97%
rename from src/arith/ir_visitor_with_analyzer.cc
rename to src/tirx/ir_visitor_with_analyzer.cc
index 0179cf6cd4..a5949b01dd 100644
--- a/src/arith/ir_visitor_with_analyzer.cc
+++ b/src/tirx/ir_visitor_with_analyzer.cc
@@ -18,7 +18,7 @@
*/
/*!
- * \file tvm/arith/ir_visitor_with_analyzer.cc
+ * \file tirx/ir_visitor_with_analyzer.cc
*/
#include "ir_visitor_with_analyzer.h"
@@ -30,9 +30,7 @@
#include <tvm/tirx/op.h>
namespace tvm {
-namespace arith {
-
-using namespace tirx;
+namespace tirx {
void IRVisitorWithAnalyzer::VisitStmt_(const ForNode* op) {
constraint_scope_.WithNewScope([&]() {
@@ -141,5 +139,5 @@ PrimExpr IRVisitorWithAnalyzer::ExtractRealCondition(PrimExpr condition) const {
return condition;
}
-} // namespace arith
+} // namespace tirx
} // namespace tvm
diff --git a/src/arith/ir_visitor_with_analyzer.h b/src/tirx/ir_visitor_with_analyzer.h
similarity index 72%
rename from src/arith/ir_visitor_with_analyzer.h
rename to src/tirx/ir_visitor_with_analyzer.h
index 3a363f13b2..54a0ea2140 100644
--- a/src/arith/ir_visitor_with_analyzer.h
+++ b/src/tirx/ir_visitor_with_analyzer.h
@@ -18,12 +18,12 @@
*/
/*!
- * \file tvm/arithmetic/ir_visitor_with_analyzer.h
+ * \file tirx/ir_visitor_with_analyzer.h
* \brief IR visitor class with an analyzer context.
*/
-#ifndef TVM_ARITH_IR_VISITOR_WITH_ANALYZER_H_
-#define TVM_ARITH_IR_VISITOR_WITH_ANALYZER_H_
+#ifndef TVM_TIRX_IR_VISITOR_WITH_ANALYZER_H_
+#define TVM_TIRX_IR_VISITOR_WITH_ANALYZER_H_
#include <tvm/arith/analyzer.h>
#include <tvm/ir/prim/expr.h>
@@ -32,22 +32,22 @@
#include <tvm/tirx/stmt_functor.h>
namespace tvm {
-namespace arith {
+namespace tirx {
-class IRVisitorWithAnalyzer : public tirx::StmtExprVisitor {
+class IRVisitorWithAnalyzer : public StmtExprVisitor {
public:
PrimExpr Simplify(const PrimExpr& expr) { return analyzer_->Simplify(expr); }
using StmtExprVisitor::VisitExpr_;
using StmtExprVisitor::VisitStmt_;
- void VisitStmt_(const tirx::ForNode* op);
- void VisitStmt_(const tirx::SBlockNode* op);
- void VisitStmt_(const tirx::BindNode* op);
- void VisitStmt_(const tirx::IfThenElseNode* op);
- void VisitStmt_(const tirx::AttrStmtNode* op);
- void VisitStmt_(const tirx::AssertStmtNode* op);
- void VisitStmt_(const tirx::SeqStmtNode* op);
+ void VisitStmt_(const ForNode* op);
+ void VisitStmt_(const SBlockNode* op);
+ void VisitStmt_(const BindNode* op);
+ void VisitStmt_(const IfThenElseNode* op);
+ void VisitStmt_(const AttrStmtNode* op);
+ void VisitStmt_(const AssertStmtNode* op);
+ void VisitStmt_(const SeqStmtNode* op);
void VisitExpr_(const CallNode* op);
void VisitExpr_(const prim::LetNode* op);
@@ -60,7 +60,7 @@ class IRVisitorWithAnalyzer : public tirx::StmtExprVisitor {
arith::Analyzer analyzer_;
/*! \brief Scope stack for accumulated assert constraints. */
- ScopeStack<WithGroup<ConstraintContext>> constraint_scope_;
+ ScopeStack<WithGroup<arith::ConstraintContext>> constraint_scope_;
/*! \brief Extract a constraint from a conditional statement
*
@@ -70,6 +70,6 @@ class IRVisitorWithAnalyzer : public tirx::StmtExprVisitor {
PrimExpr ExtractRealCondition(PrimExpr condition) const;
};
-} // namespace arith
+} // namespace tirx
} // namespace tvm
-#endif // TVM_ARITH_IR_VISITOR_WITH_ANALYZER_H_
+#endif // TVM_TIRX_IR_VISITOR_WITH_ANALYZER_H_
diff --git a/src/tirx/transform/flatten_buffer.cc b/src/tirx/transform/flatten_buffer.cc
index ca5e192c21..3a413f215d 100644
--- a/src/tirx/transform/flatten_buffer.cc
+++ b/src/tirx/transform/flatten_buffer.cc
@@ -32,7 +32,7 @@
#include <unordered_set>
-#include "../../arith/ir_mutator_with_analyzer.h"
+#include "../ir_mutator_with_analyzer.h"
#include "ir_utils.h"
namespace tvm {
@@ -57,7 +57,7 @@ namespace tirx {
* Every use site then only looks the pair up; a use before its definition is
* a hard error instead of a silently stale reference.
*/
-class BufferFlattener : public arith::IRMutatorWithAnalyzer {
+class BufferFlattener : public IRMutatorWithAnalyzer {
public:
static PrimFunc Flatten(PrimFunc func) {
arith::Analyzer ana;
diff --git a/src/tirx/transform/lower_intrin.cc b/src/tirx/transform/lower_intrin.cc
index 0545aa3be9..9e6b187200 100644
--- a/src/tirx/transform/lower_intrin.cc
+++ b/src/tirx/transform/lower_intrin.cc
@@ -37,8 +37,8 @@
#include <limits>
#include <unordered_set>
-#include "../../arith/ir_mutator_with_analyzer.h"
#include "../../arith/pattern_match.h"
+#include "../ir_mutator_with_analyzer.h"
namespace tvm {
namespace tirx {
@@ -124,7 +124,7 @@ static Expr LowerAccessPtr(const CallNode* call,
return Call(call->ty, builtin::address_of(), {buf_load});
}
-class IntrinInjecter : public tvm::arith::IRMutatorWithAnalyzer {
+class IntrinInjecter : public IRMutatorWithAnalyzer {
public:
using IRMutatorWithAnalyzer::VisitExpr_;
using IRMutatorWithAnalyzer::VisitStmt_;
diff --git a/src/tirx/transform/lower_tirx_cleanup.cc b/src/tirx/transform/lower_tirx_cleanup.cc
index e154a6b96e..9d0aaa1d98 100644
--- a/src/tirx/transform/lower_tirx_cleanup.cc
+++ b/src/tirx/transform/lower_tirx_cleanup.cc
@@ -38,13 +38,13 @@
#include <utility>
#include <vector>
-#include "../../arith/ir_mutator_with_analyzer.h"
+#include "../ir_mutator_with_analyzer.h"
#include "ir_utils.h"
namespace tvm {
namespace tirx {
-class LayoutApplier : public arith::IRMutatorWithAnalyzer {
+class LayoutApplier : public IRMutatorWithAnalyzer {
public:
static std::pair<Stmt, ffi::Array<Var>> Flatten(const Stmt& stmt, const ffi::Array<Var>& params,
const Target& target) {
@@ -83,7 +83,7 @@ class LayoutApplier : public arith::IRMutatorWithAnalyzer {
using IRMutatorWithAnalyzer::VisitStmt_;
explicit LayoutApplier(const arith::Analyzer& analyzer, const Target& target)
- : arith::IRMutatorWithAnalyzer(analyzer), target_(target) {}
+ : IRMutatorWithAnalyzer(analyzer), target_(target) {}
ffi::Any VisitAny(const ffi::Any& any) {
if (any == nullptr) {
diff --git a/src/tirx/transform/narrow_datatype.cc b/src/tirx/transform/narrow_datatype.cc
index 47b6d805b2..3a1980fc7e 100644
--- a/src/tirx/transform/narrow_datatype.cc
+++ b/src/tirx/transform/narrow_datatype.cc
@@ -22,6 +22,7 @@
* \brief narrow the datatype of indexing vars
*/
+#include <tvm/arith/analyzer.h>
#include <tvm/ffi/cast.h>
#include <tvm/ffi/function.h>
#include <tvm/ffi/reflection/registry.h>
@@ -29,10 +30,9 @@
#include <tvm/s_tir/stmt.h>
#include <tvm/tirx/builtin.h>
#include <tvm/tirx/op.h>
+#include <tvm/tirx/stmt_functor.h>
#include <tvm/tirx/transform.h>
-#include "../../arith/ir_mutator_with_analyzer.h"
-#include "../../arith/ir_visitor_with_analyzer.h"
#include "../ir/data_type_rewriter.h"
namespace tvm {
@@ -63,7 +63,6 @@ namespace tirx {
using arith::Analyzer;
using arith::ConstIntBound;
-using arith::IRMutatorWithAnalyzer;
// Determine the result dtype for Var, IntImm and Cast,
// which will be stored in `vmap` eventually.
diff --git a/src/tirx/transform/remove_no_op.cc b/src/tirx/transform/remove_no_op.cc
index 8582d5efe3..6b81d382c7 100644
--- a/src/tirx/transform/remove_no_op.cc
+++ b/src/tirx/transform/remove_no_op.cc
@@ -36,8 +36,8 @@
#include <unordered_map>
#include "../../arith/const_fold.h"
-#include "../../arith/ir_mutator_with_analyzer.h"
#include "../analysis/var_use_def_analysis.h"
+#include "../ir_mutator_with_analyzer.h"
#include "ir_utils.h"
namespace tvm {
@@ -73,7 +73,7 @@ TVM_FFI_STATIC_INIT_BLOCK() { RemoveNoOpConfigNode::RegisterReflection(); }
TVM_REGISTER_PASS_CONFIG_OPTION("tirx.RemoveNoOp", RemoveNoOpConfig);
// Mark the statement of each stage.
-class NoOpRemover : public arith::IRMutatorWithAnalyzer {
+class NoOpRemover : public IRMutatorWithAnalyzer {
public:
static Stmt Apply(Stmt stmt, const arith::Analyzer& analyzer, bool ignore_profiler_call = false) {
NoOpRemover visitor(analyzer, ignore_profiler_call);
diff --git a/src/tirx/transform/stmt_simplify.cc b/src/tirx/transform/stmt_simplify.cc
index f07bd3342f..1c393e0e3e 100644
--- a/src/tirx/transform/stmt_simplify.cc
+++ b/src/tirx/transform/stmt_simplify.cc
@@ -36,7 +36,7 @@
#include <tvm/tirx/op.h>
#include <tvm/tirx/transform.h>
-#include "../../arith/ir_mutator_with_analyzer.h"
+#include "../ir_mutator_with_analyzer.h"
namespace tvm {
namespace arith {
diff --git a/tests/cpp/expr_functor_test.cc b/tests/cpp/expr_functor_test.cc
index b62cc9dd93..53de7bd14c 100644
--- a/tests/cpp/expr_functor_test.cc
+++ b/tests/cpp/expr_functor_test.cc
@@ -52,7 +52,7 @@ class Collect : public ExprVisitor {
public:
using ExprVisitor::Visit_;
std::vector<int64_t> values;
- Expected<ffi::Optional<VisitInterrupt>> Visit_(const IntImmNode* node) override {
+ ffi::Optional<VisitInterrupt> Visit_(const IntImmNode* node) override {
values.push_back(node->value);
return std::nullopt;
}
@@ -72,9 +72,8 @@ TEST(ExprVisitor, StructuralFallback) {
class Rewrite : public ExprMutator {
public:
using ExprMutator::Mutate_;
- Expected<UnchangedOr<ffi::Any>> Mutate_(const IntImmNode* node,
- InplaceMode inplace_mode) override {
- return ffi::Any(IntImm::Int32(node->value + 1));
+ UnchangedOr<PrimExpr> Mutate_(const IntImmNode* node, InplaceMode inplace_mode) override {
+ return IntImm::Int32(node->value + 1);
}
};
@@ -92,5 +91,28 @@ TEST(ExprMutator, StructuralFallback) {
EXPECT_EQ(pair->right.as<IntImmNode>()->value, 2);
}
+class ThrowNativeError : public ExprMutator {
+ public:
+ using ExprMutator::Mutate_;
+ ffi::Error error{"ValueError", "native mutation error", ""};
+ UnchangedOr<PrimExpr> Mutate_(const IntImmNode*, InplaceMode) override { throw error; }
+};
+
+TEST(ExprMutator, NativeErrorContextThroughStructuralFallback) {
+ auto mutator = ffi::make_object<ThrowNativeError>();
+ PrimExpr sum = prim::Add(IntImm::Int32(1), IntImm::Int32(2));
+ PairExpr pair(sum, IntImm::Int32(3));
+ auto result = mutator->MutateExpected(pair);
+ ASSERT_TRUE(result.is_err());
+ EXPECT_TRUE(result.error().same_as(mutator->error));
+ auto context = ffi::VisitErrorContext::TryGetFromError(result.error());
+ ASSERT_TRUE(context.has_value());
+ const auto& pattern = context.value()->reverse_visit_pattern;
+ ASSERT_EQ(pattern.size(), 3);
+ EXPECT_TRUE(pattern[0].same_as(sum.as<prim::AddNode>()->a));
+ EXPECT_TRUE(pattern[1].same_as(sum));
+ EXPECT_TRUE(pattern[2].same_as(pair));
+}
+
} // namespace
} // namespace tvm