You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2026/09/27 01:48:57 UTC
(tvm) branch main updated: [REFACTOR][IR] Introduce OpDef and a dedicated Op registry (#20473)
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new 135dea7856 [REFACTOR][IR] Introduce OpDef and a dedicated Op registry (#20473)
135dea7856 is described below
commit 135dea785626aa7c413c54675dddb28edda4dcea
Author: Tianqi Chen <tq...@users.noreply.github.com>
AuthorDate: Sat Sep 26 21:48:51 2026 -0400
[REFACTOR][IR] Introduce OpDef and a dedicated Op registry (#20473)
---
include/tvm/ir/attr_registry_map.h | 144 ----
include/tvm/ir/op.h | 475 ++++-------
include/tvm/tirx/op.h | 7 -
include/tvm/tirx/op_attr_types.h | 2 +-
python/tvm/backend/cuda/ptx/engine.py | 12 +-
python/tvm/ir/op.py | 133 +--
python/tvm/relax/op/base.py | 14 +-
python/tvm/target/intrin.py | 4 +-
python/tvm/tirx/op.py | 13 +-
python/tvm/tirx/script/ir_builder/op.py | 30 +-
src/backend/cuda/codegen/codegen_cuda.cc | 30 +-
src/backend/cuda/codegen/intrin_rule_cuda.cc | 333 ++++----
src/backend/cuda/codegen/llvm/intrin_rule_nvptx.cc | 114 +--
src/backend/cuda/op/iket.cc | 78 +-
src/backend/cuda/op/target_builtin.cc | 498 +++++++----
src/backend/cuda/transforms/lower_iket.cc | 16 +-
.../hexagon/codegen/llvm/intrin_rule_hexagon.cc | 308 +++----
src/backend/metal/codegen/codegen_metal.cc | 8 +-
src/backend/metal/codegen/intrin_rule_metal.cc | 234 +++---
src/backend/metal/op/target_builtin.cc | 69 +-
src/backend/opencl/codegen/intrin_rule_opencl.cc | 114 +--
src/backend/rocm/codegen/llvm/intrin_rule_rocm.cc | 154 ++--
src/backend/trn/codegen/codegen_trn.cc | 32 +-
src/backend/trn/op/target_builtin.cc | 91 +-
src/backend/vulkan/codegen/codegen_spirv.cc | 8 +-
src/backend/vulkan/codegen/intrin_rule_spirv.cc | 156 ++--
src/backend/webgpu/codegen/intrin_rule_webgpu.cc | 241 +++---
src/ir/attr_registry.h | 172 ----
src/ir/op.cc | 324 ++++---
src/ir/prim/builtin.cc | 78 +-
.../backend/adreno/annotate_custom_storage.cc | 8 +-
.../backend/adreno/fold_vdevice_scope_change.cc | 2 +-
src/relax/backend/contrib/tensorrt/codegen.cc | 2 +-
src/relax/backend/task_extraction.cc | 2 +-
src/relax/backend/vm/codegen_vm.cc | 10 +-
src/relax/backend/vm/lower_runtime_builtin.cc | 34 +-
src/relax/backend/vm/vm_shape_lower.cc | 4 +-
.../transform/lower_global_view_to_local_view.cc | 2 +-
.../distributed/transform/propagate_sharding.cc | 22 +-
src/relax/ir/expr.cc | 2 +-
src/relax/op/ccl/ccl.cc | 64 +-
src/relax/op/distributed/binary.cc | 119 ++-
src/relax/op/distributed/binary.h | 8 -
src/relax/op/distributed/ccl.cc | 8 +-
src/relax/op/distributed/distributed.cc | 61 +-
src/relax/op/distributed/linear_algebra.cc | 7 +-
src/relax/op/distributed/manipulate.cc | 15 +-
src/relax/op/distributed/nn.cc | 19 +-
src/relax/op/distributed/op.cc | 13 +-
src/relax/op/distributed/statistical.cc | 16 +-
src/relax/op/distributed/statistical.h | 4 +-
src/relax/op/distributed/unary.cc | 63 +-
src/relax/op/distributed/unary.h | 7 +-
src/relax/op/image/resize.cc | 98 ++-
src/relax/op/memory/view.cc | 46 +-
src/relax/op/nn/attention.cc | 77 +-
src/relax/op/nn/convolution.cc | 138 +--
src/relax/op/nn/convolution.h | 2 +-
src/relax/op/nn/nn.cc | 397 +++++----
src/relax/op/nn/nn.h | 13 -
src/relax/op/nn/pooling.cc | 165 ++--
src/relax/op/op.cc | 603 +++++++-------
src/relax/op/op_common.cc | 21 +-
src/relax/op/op_common.h | 48 +-
src/relax/op/tensor/binary.cc | 7 +-
src/relax/op/tensor/binary.h | 56 +-
src/relax/op/tensor/create.cc | 222 ++---
src/relax/op/tensor/datatype.cc | 34 +-
src/relax/op/tensor/grad.cc | 113 +--
src/relax/op/tensor/index.cc | 65 +-
src/relax/op/tensor/inspect.cc | 150 ++--
src/relax/op/tensor/linear_algebra.cc | 51 +-
src/relax/op/tensor/manipulate.cc | 450 +++++-----
src/relax/op/tensor/qdq.cc | 38 +-
src/relax/op/tensor/sampling.cc | 19 +-
src/relax/op/tensor/search.cc | 85 +-
src/relax/op/tensor/set.cc | 60 +-
src/relax/op/tensor/sorting.cc | 45 +-
src/relax/op/tensor/statistical.cc | 163 +++-
src/relax/op/tensor/statistical.h | 26 -
src/relax/op/tensor/ternary.cc | 21 +-
src/relax/op/tensor/unary.cc | 523 +++++++++++-
src/relax/op/tensor/unary.h | 20 -
src/relax/op/vision/multibox_transform_loc.cc | 21 +-
src/relax/op/vision/nms.cc | 63 +-
src/relax/op/vision/roi_align.cc | 21 +-
src/relax/op/vision/roi_pool.cc | 21 +-
src/relax/script/ir_builder/distributed.cc | 2 +-
src/relax/script/printer/call.cc | 18 +-
src/relax/script/printer/utils.h | 4 +-
src/relax/transform/alter_op_impl.cc | 4 +-
.../transform/attach_attr_layout_free_buffers.cc | 2 +-
src/relax/transform/call_tir_rewrite.cc | 8 +-
src/relax/transform/convert_layout.cc | 2 +-
src/relax/transform/dataflow_inplace.cc | 2 +-
src/relax/transform/decompose_ops.cc | 10 +-
src/relax/transform/fold_constant.cc | 2 +-
src/relax/transform/fuse_ops.cc | 4 +-
src/relax/transform/fuse_tir.cc | 16 +-
src/relax/transform/kill_after_last_use.cc | 18 +-
src/relax/transform/lambda_lift.cc | 6 +-
src/relax/transform/legalize_ops.cc | 8 +-
src/relax/transform/lift_transform_params.cc | 6 +-
src/relax/transform/lower_alloc_tensor.cc | 6 +-
src/relax/transform/realize_vdevice.cc | 10 +-
src/relax/transform/remove_purity_checking.cc | 8 +-
src/relax/transform/rewrite_cuda_graph.cc | 14 +-
src/relax/transform/rewrite_dataflow_reshape.cc | 2 +-
src/relax/transform/run_codegen.cc | 2 +-
.../specialize_primfunc_based_on_callsite.cc | 2 +-
src/relax/transform/split_call_tir_by_pattern.cc | 6 +-
.../transform/split_layout_rewrite_preproc.cc | 2 +-
src/relax/transform/static_plan_block_memory.cc | 18 +-
src/relax/transform/to_mixed_precision.cc | 2 +-
.../postproc/rewrite_cooperative_fetch.cc | 4 +-
src/s_tir/schedule/analysis/analysis.cc | 2 +-
src/s_tir/schedule/utils.h | 1 -
src/s_tir/transform/inject_permuted_layout.cc | 4 +-
src/s_tir/transform/inject_ptx_async_copy.cc | 6 +-
src/s_tir/transform/inject_ptx_ldg32.cc | 2 +-
src/s_tir/transform/inject_software_pipeline.cc | 10 +-
src/s_tir/transform/memhammer_lower_auto_copy.cc | 4 +-
.../transform/memhammer_tensorcore_rewrite.cc | 4 +-
.../transform/merge_shared_memory_allocations.cc | 2 +-
src/s_tir/transform/tensorcore_infer_fragment.cc | 10 +-
src/target/intrin_rule.cc | 344 ++++----
src/target/llvm/intrin_rule_llvm.cc | 280 ++++---
src/te/tensor.cc | 18 +-
src/tirx/analysis/filter_canonical.cc | 2 +-
src/tirx/ir/data_type_rewriter.cc | 2 +-
src/tirx/ir/exec_scope.cc | 2 +-
src/tirx/ir/ir_mutator_with_analyzer.cc | 4 +-
src/tirx/ir/ir_visitor_with_analyzer.cc | 2 +-
src/tirx/ir/layout/utils.h | 2 -
src/tirx/ir/stmt.cc | 18 +-
src/tirx/op/builtin.cc | 927 ++++++++++++++-------
src/tirx/op/op.cc | 373 ++++++---
src/tirx/op/tirx.cc | 308 ++++++-
src/tirx/transform/lower_intrin.cc | 2 +-
src/tirx/transform/lower_tvm_builtin.cc | 4 +-
src/tirx/transform/lower_warp_memory.cc | 18 +-
src/tirx/transform/remove_no_op.cc | 8 +-
src/tirx/transform/tile_primitive_dispatch.cc | 2 +-
tests/cpp/tir_analysis_side_effect.cc | 3 +-
.../s_tir/script/test_s_tir_script_printer.py | 10 +-
145 files changed, 6076 insertions(+), 4752 deletions(-)
diff --git a/include/tvm/ir/attr_registry_map.h b/include/tvm/ir/attr_registry_map.h
deleted file mode 100644
index fd4c53bc4f..0000000000
--- a/include/tvm/ir/attr_registry_map.h
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*!
- * \file tvm/ir/attr_registry_map.h
- * \brief Attribute map used in registry.
- */
-#ifndef TVM_IR_ATTR_REGISTRY_MAP_H_
-#define TVM_IR_ATTR_REGISTRY_MAP_H_
-
-#include <tvm/ffi/string.h>
-
-#include <utility>
-#include <vector>
-
-namespace tvm {
-
-/*!
- * \brief Generic attribute map.
- * \tparam KeyType the type of the key.
- */
-template <typename KeyType>
-class AttrRegistryMapContainerMap {
- public:
- /*!
- * \brief Check if the map has key.
- * \param key The key to the map
- * \return 1 if key is contained in map, 0 otherwise.
- */
- int count(const KeyType& key) const {
- if (key.defined()) {
- const uint32_t idx = key->AttrRegistryIndex();
- return idx < data_.size() ? (data_[idx].second != 0) : 0;
- } else {
- return 0;
- }
- }
- /*!
- * \brief get the corresponding value element at key.
- * \param key The key to the map
- * \return the const reference to the content value.
- */
- const ffi::Any& operator[](const KeyType& key) const {
- TVM_FFI_ICHECK(key.defined());
- const uint32_t idx = key->AttrRegistryIndex();
- TVM_FFI_ICHECK(idx < data_.size() && data_[idx].second != 0)
- << "Attribute " << attr_name_ << " has not been registered for " << key->name;
- return data_[idx].first;
- }
- /*!
- * \brief get the corresponding value element at key with default value.
- * \param key The key to the map
- * \param def_value The default value when the key does not exist.
- * \return the const reference to the content value.
- * \tparam ValueType The content value type.
- */
- template <typename ValueType>
- ValueType get(const KeyType& key, ValueType def_value) const {
- TVM_FFI_ICHECK(key.defined());
- const uint32_t idx = key->AttrRegistryIndex();
- if (idx < data_.size() && data_[idx].second != 0) {
- if constexpr (std::is_same_v<ValueType, ffi::Any>) {
- return data_[idx].first;
- } else {
- return data_[idx].first.template cast<ValueType>();
- }
- } else {
- return def_value;
- }
- }
-
- private:
- /*! \brief The name of the attr field */
- ffi::String attr_name_;
- /*! \brief The internal data. */
- std::vector<std::pair<ffi::Any, int>> data_;
- /*! \brief The constructor */
- AttrRegistryMapContainerMap() = default;
- template <typename, typename>
- friend class AttrRegistry;
- friend class OpRegEntry;
-};
-
-/*!
- * \brief ffi::Map<Key, ValueType> used to store meta-data.
- * \tparam KeyType The type of the key
- * \tparam ValueType The type of the value stored in map.
- */
-template <typename KeyType, typename ValueType>
-class AttrRegistryMap {
- public:
- /*!
- * \brief constructor
- * \param map The internal map.
- */
- explicit AttrRegistryMap(const AttrRegistryMapContainerMap<KeyType>& map) : map_(map) {}
- /*!
- * \brief Check if the map has op as key.
- * \param key The key to the map
- * \return 1 if op is contained in map, 0 otherwise.
- */
- int count(const KeyType& key) const { return map_.count(key); }
- /*!
- * \brief get the corresponding value element at key.
- * \param key The key to the map
- * \return the const reference to the content value.
- */
- ValueType operator[](const KeyType& key) const {
- if constexpr (std::is_same_v<ValueType, ffi::Any>) {
- return map_[key];
- } else {
- return map_[key].template cast<ValueType>();
- }
- }
- /*!
- * \brief get the corresponding value element at key with default value.
- * \param key The key to the map
- * \param def_value The default value when the key does not exist.
- * \return the const reference to the content value.
- */
- ValueType get(const KeyType& key, ValueType def_value) const { return map_.get(key, def_value); }
-
- protected:
- /*! \brief The internal map field */
- const AttrRegistryMapContainerMap<KeyType>& map_;
-};
-
-} // namespace tvm
-#endif // TVM_IR_ATTR_REGISTRY_MAP_H_
diff --git a/include/tvm/ir/op.h b/include/tvm/ir/op.h
index 02568266df..9a6a530ef9 100644
--- a/include/tvm/ir/op.h
+++ b/include/tvm/ir/op.h
@@ -19,404 +19,297 @@
/*!
* \file tvm/ir/op.h
- * \brief Primitive operators(builtin intrinsics)
- * and registry for them.
+ * \brief Canonical primitive operators and their registration.
*/
#ifndef TVM_IR_OP_H_
#define TVM_IR_OP_H_
-#include <tvm/ffi/error.h>
-#include <tvm/ffi/function.h>
+#include <tvm/ffi/container/list.h>
#include <tvm/ffi/reflection/registry.h>
-#include <tvm/ir/attr_registry_map.h>
#include <tvm/ir/attrs.h>
#include <tvm/ir/env_func.h>
#include <tvm/ir/expr.h>
#include <tvm/ir/type.h>
-#include <string>
+#include <type_traits>
#include <utility>
-#include <vector>
namespace tvm {
-// forward declare name.
template <typename>
class OpAttrMap;
-/*!
- * \brief Information about an input field of an Op (name, type, description).
- *
- * Populated via OpRegEntry::add_argument and consumed both by
- * internal sanity checks / error messages and by external tooling
- * that wants to introspect an Op's argument schema.
- */
+/*! \brief An operator argument's name and documentation. */
class ArgumentInfoNode : public ffi::Object {
public:
- /*! \brief name of the field */
+ /*! \brief Argument name. */
ffi::String name;
- /*! \brief type docstring information in str. */
- ffi::String type_info;
- /*! \brief detailed description of the type */
- ffi::String description;
+ /*! \brief Argument documentation. */
+ ffi::String doc;
static void RegisterReflection() {
- namespace rfl = ffi::reflection;
- rfl::ObjectDef<ArgumentInfoNode>()
+ namespace refl = ffi::reflection;
+ refl::ObjectDef<ArgumentInfoNode>()
.def_ro("name", &ArgumentInfoNode::name)
- .def_ro("type_info", &ArgumentInfoNode::type_info)
- .def_ro("description", &ArgumentInfoNode::description);
+ .def_ro("doc", &ArgumentInfoNode::doc);
}
static constexpr TVMFFISEqHashKind _type_s_eq_hash_kind = kTVMFFISEqHashKindTreeNode;
-
TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.ArgumentInfo", ArgumentInfoNode, ffi::Object);
};
-/*! \brief Managed reference to ArgumentInfoNode. */
+/*! \brief Managed reference to an argument descriptor. */
class ArgumentInfo : public ffi::ObjectRef {
public:
TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(ArgumentInfo, ffi::ObjectRef, ArgumentInfoNode);
};
-// TODO(tvm-team): migrate low-level intrinsics to use Op
-/*!
- * \brief Primitive Op(builtin intrinsics)
- *
- * This data structure stores the meta-data
- * about primitive operators that can be invoked via Call.
- *
- * Low-level IR intrinsics(such as libc.expf) are also
- * implemented via Op.
- *
- * \sa Op
- */
+/*! \brief Metadata for a canonical primitive operator invoked through Call. */
class OpNode : public ExprNode {
+ private:
+ // Dense process-local index into attribute columns; never serialized.
+ uint32_t index_{0};
+
public:
- /*! \brief name of the operator */
+ /*! \brief Canonical operator name. */
ffi::String name;
- /*!
- * \brief detailed description of the operator
- * This can be used to generate docstring automatically for the operator.
- */
- ffi::String description;
- /* \brief Information of input arguments to the operator */
- ffi::Array<ArgumentInfo> arguments;
- /*!
- * \brief The type key of the attribute field
- * This can be empty, in which case it defaults to anything.
- */
+ /*! \brief Operator documentation. */
+ ffi::String doc;
+ /*! \brief Descriptors for the ordered required value-argument prefix. */
+ ffi::Array<ArgumentInfo> args_info;
+ /*! \brief Accept at least args_info.size() value arguments when true, exactly that count
+ * otherwise. */
+ bool allow_extra_args{false};
+ /*! \brief Descriptors for Call.ty_args; inference owns optional/variable type-argument checks. */
+ ffi::Array<ArgumentInfo> ty_args_info;
+ /*! \brief Attribute object type key, or empty when unrestricted. */
ffi::String attrs_type_key;
- /*!
- * \brief attribute type index,
- * this field varies in each run and is not exposed to frontend.
- */
+ /*! \brief Runtime index corresponding to attrs_type_key; not serialized. */
uint32_t attrs_type_index{0};
- /*!
- * \brief number of input arguments to the operator,
- * -1 means it is variable length
- */
- int32_t num_inputs = -1;
- /*!
- * \brief support level of the operator,
- * The lower the more priority it contains.
- * This is in analogies to BLAS levels.
- */
- int32_t support_level = 10;
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<OpNode>()
- .def_ro("name", &OpNode::name)
- .def_ro("description", &OpNode::description, refl::AttachFieldFlag::SEqHashIgnore())
- .def_ro("arguments", &OpNode::arguments, refl::AttachFieldFlag::SEqHashIgnore())
- .def_ro("attrs_type_key", &OpNode::attrs_type_key, refl::AttachFieldFlag::SEqHashIgnore())
- .def_ro("num_inputs", &OpNode::num_inputs, refl::AttachFieldFlag::SEqHashIgnore())
- .def_ro("support_level", &OpNode::support_level, refl::AttachFieldFlag::SEqHashIgnore());
- }
+ TVM_DLL static void RegisterReflection();
static constexpr TVMFFISEqHashKind _type_s_eq_hash_kind = kTVMFFISEqHashKindUniqueInstance;
TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.Op", OpNode, ExprNode);
private:
- /*! \return the internal attr registry index. */
- uint32_t AttrRegistryIndex() const { return index_; }
- /*! \brief repr to be printed in registry*/
- std::string AttrRegistryName() const { return name; }
-
- // friend class
+ friend class OpRegistry;
+ friend class OpDef;
+ friend class Op;
template <typename>
- friend class AttrRegistryMapContainerMap;
- template <typename, typename>
- friend class AttrRegistry;
- friend class OpRegEntry;
-
- // Program internal unique index of operator.
- // Used to help index the program.
- uint32_t index_{0};
+ friend class OpAttrMap;
};
/*!
- * \brief Managed reference class to OpNode.
- * \sa OpNode
+ * \brief Reference-counted handle to a canonical named operator.
+ *
+ * OpDef temporarily builds metadata on the same Op returned by Get. Independent
+ * registrations may attach different attributes; replacing an existing attribute
+ * requires explicit override. Cached attribute maps observe subsequent changes.
+ * args_info describes required value operands; allow_extra_args permits a suffix.
+ * ty_args_info describes type arguments without imposing a runtime count.
+ *
+ * \code
+ * TVM_FFI_STATIC_INIT_BLOCK() {
+ * OpDef("example.identity", "Return the input expression.")
+ * .add_arg("value", "The input expression.")
+ * .set_attr<bool>("FPurity", true);
+ * }
+ * // Copies the handle; the canonical node is shared.
+ * Op op = Op::Get("example.identity");
+ * bool pure = Op::GetAttrMap<bool>("FPurity")[op];
+ * \endcode
*/
class Op : public Expr {
public:
+ /*! \brief Construct a handle from a defined node. \param n The operator node. */
explicit Op(ffi::ObjectPtr<OpNode> n) : Expr(std::move(n)) {
TVM_FFI_CHECK(defined(), ValueError) << "Op expects a defined OpNode";
}
/*!
- * \brief Get additional registered attribute about operators.
- * If nothing has been registered, an empty OpAttrMap will be returned.
- * \param attr_name The name of the attribute.
- * \return An OpAttrMap of specified attr_name.
- * \tparam ValueType The type of the attribute.
+ * \brief Get a typed live view of a registered attribute column.
+ * \tparam ValueType The attribute value type.
+ * \param attr_name The column name.
+ * \return A shared view that observes registration, replacement, reset, and growth.
+ * \throws InternalError if the column has never been registered.
*/
template <typename ValueType>
- inline static OpAttrMap<ValueType> GetAttrMap(const ffi::String& attr_name);
+ static OpAttrMap<ValueType> GetAttrMap(const ffi::String& attr_name);
/*!
- * \brief Checks if an attr map is present in the registry.
- * \param attr_name The name of the attribute.
- * \return bool True if the attr is present.
+ * \brief Check whether an attribute column exists, even if some Ops lack a value.
+ * \param attr_name The column name.
+ * \return Whether the column has been registered.
*/
TVM_DLL static bool HasAttrMap(const ffi::String& attr_name);
/*!
- * \brief Get an Op for a given operator name.
- * Will raise an error if the op has not been registered.
- * \param op_name Name of the operator.
- * \return Pointer to a Op, valid throughout program lifetime.
+ * \brief Look up a registered operator by name.
+ * \param op_name The canonical name.
+ * \return A copy of the reference-counted handle to the canonical node.
+ * \throws AttributeError if the operator is not registered.
*/
- TVM_DLL static const Op& Get(const ffi::String& op_name);
-
+ TVM_DLL static Op Get(const ffi::String& op_name);
+ /*! \brief List registered operator names. \return Names in unspecified order. */
+ TVM_DLL static ffi::Array<ffi::String> ListNames();
TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(Op, Expr, OpNode);
private:
- /*!
- * \brief Get generic attrmap given attr name
- * \param key The attribute key
- * \return The attr map.
- */
- TVM_DLL static const AttrRegistryMapContainerMap<Op>& GetAttrMapContainer(const ffi::String& key);
+ TVM_DLL static ffi::List<ffi::Any> GetAttrColumn(const ffi::String& attr_name);
};
-/*!
- * \brief Helper structure to register operators
- * \sa TVM_REGISTER_OP
- */
-class OpRegEntry {
+/*! \brief Noncopyable temporary builder for a canonical Op; see Op for the example. */
+class OpDef {
public:
- /*! \return the operator */
- const Op& op() const { return op_; }
/*!
- * \brief setter function during registration
- * Set the description of operator
- * \param descr the description string.
- * \return reference to self.
+ * \brief Get or create the named Op without changing its documentation.
+ * \param name The canonical operator name.
*/
- inline OpRegEntry& describe(const std::string& descr); // NOLINT(*)
+ TVM_DLL explicit OpDef(const ffi::String& name);
/*!
- * \brief Add argument information to the function.
- * \param name Name of the argument.
- * \param type Type of the argument.
- * \param description Description of the argument.
- * \return reference to self.
+ * \brief Get or create the named Op and set its documentation.
+ * \param name The canonical operator name.
+ * \param doc Operator documentation, replacing any existing documentation.
*/
- inline OpRegEntry& add_argument(const std::string& name, const std::string& type,
- const std::string& description);
+ TVM_DLL OpDef(const ffi::String& name, const ffi::String& doc);
+ OpDef(const OpDef&) = delete;
+ OpDef& operator=(const OpDef&) = delete;
+ OpDef(OpDef&&) = delete;
+ OpDef& operator=(OpDef&&) = delete;
+ /*! \brief Get the canonical operator. \return A copied handle, safe after this builder dies. */
+ Op op() const { return op_; }
/*!
- * \brief Set the attrs type key and index to be AttrsType.
- * \tparam AttrsType the attribute type to b set.
- * \return reference to self.
+ * \brief Append a required value-argument descriptor.
+ * \param name Argument name.
+ * \param doc Argument documentation.
+ * \return This builder. Without allow_extra_args(), this is part of the complete list.
*/
- template <typename AttrsType>
- inline OpRegEntry& set_attrs_type();
+ TVM_DLL OpDef& add_arg(const ffi::String& name, const ffi::String& doc);
/*!
- * \brief Set the attrs type key and index to be AttrsType.
- * \param key The attribute type key to be set.
- * \return reference to self.
+ * \brief Set the attribute object type key and runtime index together.
+ * \tparam AttrsType The attribute object node type.
+ * \return This builder.
*/
- inline OpRegEntry& set_attrs_type_key(const ffi::String& key);
+ template <typename AttrsType>
+ OpDef& attrs_type() {
+ uint32_t index = AttrsType::RuntimeTypeIndex();
+ get()->attrs_type_key = AttrsType::_type_key;
+ get()->attrs_type_index = index;
+ return *this;
+ }
/*!
- * \brief Set the num_inputs
- * \param n The number of inputs to be set.
- * \return reference to self.
+ * \brief Resolve and set the attribute object type key and runtime index together.
+ * \param key A registered attribute object type key.
+ * \return This builder. An unknown key raises an error before modifying metadata.
*/
- inline OpRegEntry& set_num_inputs(int32_t n); // NOLINT(*)
+ TVM_DLL OpDef& set_attrs_type_key(const ffi::String& key);
/*!
- * \brief Set the support level of op.
- * \param level The support level.
- * \return reference to self.
+ * \brief Allow value arguments after the required prefix described by args_info.
+ * \return This builder. An empty prefix accepts any number of value arguments.
*/
- inline OpRegEntry& set_support_level(int32_t level); // NOLINT(*)
+ OpDef& allow_extra_args() {
+ get()->allow_extra_args = true;
+ return *this;
+ }
/*!
- * \brief Register additional attributes to operator.
- * \param attr_name The name of the attribute.
- * \param value The value to be set.
- * \param plevel The priority level of this set,
- * an higher priority level attribute
- * will replace lower priority level attribute.
- * Must be bigger than 0.
- *
- * Cannot set with same plevel twice in the code.
- *
- * \tparam ValueType The type of the value to be set.
+ * \brief Append a descriptor for a type argument in Call.ty_args.
+ * \param name Type-argument name.
+ * \param doc Type-argument documentation.
+ * \return This builder. Inference retains responsibility for type-argument counts.
*/
- template <typename ValueType>
- inline OpRegEntry& set_attr(const std::string& attr_name, // NOLINT(*)
- const ValueType& value, int plevel = 10);
-
+ TVM_DLL OpDef& add_ty_arg(const ffi::String& name, const ffi::String& doc);
/*!
- * \brief Resets an attr of the registry.
- * \param attr_name The name of the attribute.
+ * \brief Register an extensible attribute, rejecting duplicates unless overridden.
+ * \tparam ValueType The attribute value type.
+ * \param attr_name Attribute column name.
+ * \param value Non-null attribute value.
+ * \param override Whether to replace an existing value; no prior value is retained.
+ * \return This builder. Existing cached maps observe the new value.
*/
- inline void reset_attr(const std::string& attr_name);
-
- // set the name of the op to be the same as registry
- inline OpRegEntry& set_name() { // NOLINT(*)
- if (get()->name.length() == 0) {
- get()->name = name;
- }
+ template <typename ValueType>
+ OpDef& set_attr(const ffi::String& attr_name, const ValueType& value, bool override = false) {
+ UpdateAttr(attr_name, ffi::Any(value), override);
return *this;
}
/*!
- * \brief Register or get a new entry.
- * \param name The name of the operator.
- * \return the corresponding entry.
+ * \brief Remove the current attribute value, with no priority fallback.
+ * \param attr_name Attribute column name; missing columns and values are ignored.
+ * \return This builder. Existing cached maps observe the removal.
*/
- TVM_DLL static OpRegEntry& RegisterOrGet(const ffi::String& name);
+ TVM_DLL OpDef& reset_attr(const ffi::String& attr_name);
private:
- template <typename, typename>
- friend class AttrRegistry;
- // the name
- std::string name;
- /*! \brief The operator */
+ OpNode* get() { return const_cast<OpNode*>(op_.operator->()); }
+ TVM_DLL void UpdateAttr(const ffi::String& attr_name, ffi::Any value, bool override);
Op op_;
- /*! \brief Construct the non-null Op for this registry entry. */
- static Op MakeOp(uint32_t reg_index);
- // private constructor
- TVM_DLL OpRegEntry(uint32_t reg_index);
- // return internal pointer to op.
- inline OpNode* get();
- // update the attribute OpAttrMap
- TVM_DLL void UpdateAttr(const ffi::String& key, ffi::Any value, int plevel);
};
/*!
- * \brief ffi::Map<Op,ValueType> used to store meta-information about Op.
- * \tparam ValueType The type of the value stored in map.
+ * \brief Typed live view of an Op attribute column.
+ *
+ * Copies share the retained mutable column and observe later registrations,
+ * replacements, resets, and growth. Values are returned by value, never as
+ * references into a backing array. Registration is not synchronized with readers.
+ * \tparam ValueType The attribute value type.
*/
template <typename ValueType>
-class OpAttrMap : public AttrRegistryMap<Op, ValueType> {
+class OpAttrMap {
public:
/*!
- * \brief get the corresponding value element at op with default value.
- * \param expr The key to the map
- * \param def_value The default value when the key does not exist
- * or if expr is not an Op.
- * \return the const reference to the content value.
+ * \brief Check whether an operator has a value in this column.
+ * \param op The operator.
+ * \return 1 if present, 0 otherwise.
*/
- inline ValueType get(const Expr& expr, ValueType def_value) const;
-
- using TParent = AttrRegistryMap<Op, ValueType>;
- using TParent::count;
- using TParent::get;
- using TParent::operator[];
+ int count(const Op& op) const {
+ return op->index_ < column_.size() && column_[op->index_] != nullptr;
+ }
+ /*!
+ * \brief Look up an operator's attribute, raising InternalError if absent.
+ * \param op The operator.
+ * \return The value converted to ValueType.
+ */
+ ValueType operator[](const Op& op) const {
+ TVM_FFI_ICHECK(count(op)) << "Attribute " << attr_name_ << " has not been registered for "
+ << op->name;
+ if constexpr (std::is_same_v<ValueType, ffi::Any>) {
+ return column_[op->index_];
+ } else {
+ return column_[op->index_].template cast<ValueType>();
+ }
+ }
+ /*!
+ * \brief Look up an operator's attribute with a fallback.
+ * \param op The operator.
+ * \param def_value Fallback when the attribute is absent.
+ * \return The registered value or def_value.
+ */
+ ValueType get(const Op& op, ValueType def_value) const {
+ return count(op) ? (*this)[op] : def_value;
+ }
+ /*!
+ * \brief Look up an expression's attribute with a fallback.
+ * \param expr A defined expression.
+ * \param def_value Fallback when expr is not an Op or its attribute is absent.
+ * \return The registered value or def_value.
+ */
+ ValueType get(const Expr& expr, ValueType def_value) const {
+ TVM_FFI_ICHECK(expr.defined());
+ if (const auto* op = expr.as<OpNode>()) {
+ return get(ffi::GetRef<Op>(op), def_value);
+ }
+ return def_value;
+ }
private:
friend class Op;
- // constructor
- explicit OpAttrMap(const AttrRegistryMapContainerMap<Op>& map) : TParent(map) {}
+ OpAttrMap(ffi::List<ffi::Any> column, ffi::String attr_name)
+ : column_(std::move(column)), attr_name_(std::move(attr_name)) {}
+ ffi::List<ffi::Any> column_;
+ ffi::String attr_name_;
};
-// internal macros to make
-#define TVM_OP_REGISTER_VAR_DEF [[maybe_unused]] static ::tvm::OpRegEntry& __make_##Op
-
-/*!
- * \def TVM_REGISTER_OP
- * \brief Register a new operator, or set attribute of the corresponding op.
- *
- * \param OpName The name of registry
- *
- * \code
- *
- * TVM_REGISTER_OP("add")
- * .describe("add two inputs together")
- * .set_num_inputs(2)
- * .set_attr<OpKernel>("gpu_kernel", AddKernel);
- *
- * \endcode
- */
-#define TVM_REGISTER_OP(OpName) \
- TVM_FFI_STR_CONCAT(TVM_OP_REGISTER_VAR_DEF, __COUNTER__) = \
- ::tvm::OpRegEntry::RegisterOrGet(OpName).set_name()
-
-// implementations
-
-template <typename ValueType>
-inline OpAttrMap<ValueType> Op::GetAttrMap(const ffi::String& key) {
- return OpAttrMap<ValueType>(Op::GetAttrMapContainer(key));
-}
-
-inline OpNode* OpRegEntry::get() { return const_cast<OpNode*>(op_.operator->()); }
-
-inline OpRegEntry& OpRegEntry::describe(const std::string& descr) { // NOLINT(*)
- get()->description = descr;
- return *this;
-}
-
-inline OpRegEntry& OpRegEntry::add_argument(const std::string& name, const std::string& type,
- const std::string& description) {
- auto n = ffi::make_object<ArgumentInfoNode>();
- n->name = name;
- n->type_info = type;
- n->description = description;
- get()->arguments.push_back(ArgumentInfo(n));
- return *this;
-}
-
-inline OpRegEntry& OpRegEntry::set_num_inputs(int32_t n) { // NOLINT(*)
- get()->num_inputs = n;
- return *this;
-}
-
-template <typename AttrsType>
-inline OpRegEntry& OpRegEntry::set_attrs_type() { // NOLINT(*)
- get()->attrs_type_key = AttrsType::_type_key;
- get()->attrs_type_index = AttrsType::RuntimeTypeIndex();
- return *this;
-}
-
-inline OpRegEntry& OpRegEntry::set_attrs_type_key(const ffi::String& key) { // NOLINT(*)
- get()->attrs_type_key = key;
- get()->attrs_type_index = tvm::ffi::TypeKeyToIndex(key.c_str());
- return *this;
-}
-
-inline OpRegEntry& OpRegEntry::set_support_level(int32_t n) { // NOLINT(*)
- get()->support_level = n;
- return *this;
-}
-
template <typename ValueType>
-inline OpRegEntry& OpRegEntry::set_attr( // NOLINT(*)
- const std::string& attr_name, const ValueType& value, int plevel) {
- TVM_FFI_ICHECK_GT(plevel, 0) << "plevel in set_attr must be greater than 0";
- UpdateAttr(attr_name, Any(value), plevel);
- return *this;
-}
-
-// member functions of OpAttrMap
-
-template <typename ValueType>
-inline ValueType OpAttrMap<ValueType>::get(const Expr& expr, ValueType def_value) const {
- TVM_FFI_ICHECK(expr.defined());
- if (const OpNode* op = expr.as<OpNode>()) {
- return this->map_.get(ffi::GetRef<Op>(op), def_value);
- } else {
- return def_value;
- }
+OpAttrMap<ValueType> Op::GetAttrMap(const ffi::String& attr_name) {
+ return OpAttrMap<ValueType>(GetAttrColumn(attr_name), attr_name);
}
} // namespace tvm
diff --git a/include/tvm/tirx/op.h b/include/tvm/tirx/op.h
index 7089522496..32b417ba2e 100644
--- a/include/tvm/tirx/op.h
+++ b/include/tvm/tirx/op.h
@@ -45,13 +45,6 @@
namespace tvm::prim {
-#define TVM_TIR_REGISTER_OP(OpName) \
- TVM_REGISTER_OP("tirx." OpName) \
- .set_attr<TScriptPrinterName>("TScriptPrinterName", OpName) \
- .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"), /*plevel=*/1)
-
-#define TVM_TIRX_REGISTER_OP(OpName) TVM_TIR_REGISTER_OP(OpName)
-
// Shared primitive construction and constants are declared in ir/prim/op.h.
/*!
diff --git a/include/tvm/tirx/op_attr_types.h b/include/tvm/tirx/op_attr_types.h
index f333d03887..db185cdf47 100644
--- a/include/tvm/tirx/op_attr_types.h
+++ b/include/tvm/tirx/op_attr_types.h
@@ -21,7 +21,7 @@
* \file tvm/tirx/op_attr_types.h
* \brief Attribute types in the Op registry for TIR ops.
*
- * These attributes can be set via OpRegEntry::set_attr
+ * These attributes can be set via OpDef::set_attr
*
* \sa tvm/ir/op.h
*/
diff --git a/python/tvm/backend/cuda/ptx/engine.py b/python/tvm/backend/cuda/ptx/engine.py
index 0b7de8a736..242ed6b759 100644
--- a/python/tvm/backend/cuda/ptx/engine.py
+++ b/python/tvm/backend/cuda/ptx/engine.py
@@ -40,7 +40,7 @@ per-instruction generated or hand-written code:
from tvm.backend.cuda.codegen.registry import register_codegen
from tvm.backend.cuda.codegen.utils import parse_str
from tvm.backend.cuda.op import cuda_cvta_generic_to_shared, cuda_func_call
-from tvm.ir import Call, TensorLoad
+from tvm.ir import Call, Op, TensorLoad
from tvm.ir.op import register_op_attr
from tvm.ir.type import PointerType, PrimType
from tvm.runtime import const
@@ -92,6 +92,9 @@ def register_table(table: dict[str, InstructionEntry]) -> None:
# First attr call implicitly creates the Op registry entry. Effect
# kind must exist before any side-effect analysis sees the op.
register_op_attr(entry.op_name, "TCallEffectKind", _EFFECT_OPAQUE)
+ # Encoded variants have no shared named operand prefix; codegen checks
+ # each variant's operands, predicate, modifier tokens, and marker.
+ Op.get(entry.op_name).set_allow_extra_args()
# The printer name is the *surface* path a user can type, which is the
# mnemonic (several `mov_*` entries all answer to `T.ptx.mov`), not the
# table key. Reparsing re-dispatches on the operand shape.
@@ -102,7 +105,7 @@ def register_table(table: dict[str, InstructionEntry]) -> None:
# `PTXNamespace.__getattr__`. The escape is the identity for every
# other family, and `gen_stubs` already spells the attribute this way.
family = escape_token(entry.family)
- register_op_attr(entry.op_name, "TScriptPrinterName", f"ptx.{family}", level=20)
+ register_op_attr(entry.op_name, "TScriptPrinterName", f"ptx.{family}")
register_op_attr(entry.op_name, "TIRxOpCategory", "device_intrin")
register_op_attr(entry.op_name, "TDeviceIntrinsicNamespace", "ptx")
register_codegen(f"ptx.{entry.name}")(_make_codegen(entry))
@@ -111,7 +114,10 @@ def register_table(table: dict[str, InstructionEntry]) -> None:
def register_addr() -> None:
"""Register the pure address-expression op consumed by PTX instructions."""
register_op_attr(_ADDR_OP_NAME, "TCallEffectKind", _EFFECT_PURE)
- register_op_attr(_ADDR_OP_NAME, "TScriptPrinterName", "ptx.addr", level=20)
+ op = Op.get(_ADDR_OP_NAME)
+ op.add_arg("base", "The base pointer.")
+ op.add_arg("byte_offset", "The offset in bytes.")
+ register_op_attr(_ADDR_OP_NAME, "TScriptPrinterName", "ptx.addr")
register_op_attr(_ADDR_OP_NAME, "TIRxOpCategory", "device_intrin")
register_op_attr(_ADDR_OP_NAME, "TDeviceIntrinsicNamespace", "ptx")
register_codegen("ptx.addr")(_unconsumed_addr_codegen)
diff --git a/python/tvm/ir/op.py b/python/tvm/ir/op.py
index e1de79d9e1..ae5d500e13 100644
--- a/python/tvm/ir/op.py
+++ b/python/tvm/ir/op.py
@@ -32,133 +32,52 @@ class Op(Expr):
@staticmethod
def get(op_name):
- """Get the Op for a given name
+ """Get a registered operator by name.
Parameters
----------
op_name : str
- The operator name
+ The canonical operator name.
Returns
-------
- op : Op
- The op of the corresponding name
+ Op
+ A handle to the registered operator.
"""
return _ffi_api.GetOp(op_name)
- def get_attr(self, attr_name):
- """Get additional attribute about the operator.
-
- Parameters
- ----------
- attr_name : str
- The attribute name.
-
- Returns
- -------
- value : object
- The attribute value
- """
- return _ffi_api.OpGetAttr(self, attr_name)
-
- def has_attr(self, attr_name):
- """Check whether the operator has additional attribute.
-
- Parameters
- ----------
- attr_name : str
- The attribute name.
+ @staticmethod
+ def list_op_names():
+ """List registered operator names in unspecified order.
Returns
-------
- value : bool
- Whether the operator has additional attribute
+ list[str]
+ The registered operator names.
"""
- return _ffi_api.OpHasAttr(self, attr_name)
+ return _ffi_api.ListOpNames()
- def set_attr(self, attr_name, value, plevel=10):
- """Set attribute about the operator.
+ def set_attr(self, attr_name, value, override=False):
+ """Set an operator attribute.
Parameters
----------
attr_name : str
- The attribute name
-
+ Attribute column name.
value : object
- The attribute value
-
- plevel : int
- The priority level
- """
- _ffi_api.OpSetAttr(self, attr_name, value, plevel)
-
- def reset_attr(self, attr_name):
- """Reset attribute about the operator.
-
- Parameters
- ----------
- attr_name : str
- The attribute name
- """
- _ffi_api.OpResetAttr(self, attr_name)
-
- def add_argument(self, name, type, description): # pylint: disable=redefined-builtin
- """Add arguments information to the function.
-
- Parameters
- ----------
- name : str
- The argument name.
- type : str
- The argument type.
- description : str
- The argument description.
- """
- _ffi_api.OpAddArgument(self, name, type, description)
-
- def set_support_level(self, level):
- """Set the support level of op.
-
- Parameters
- ----------
- level : int
- The support level.
- """
- _ffi_api.OpSetSupportLevel(self, level)
-
- def set_num_inputs(self, n):
- """Set the support level of op.
-
- Parameters
- ----------
- n : int
- The input number.
- """
- _ffi_api.OpSetNumInputs(self, n)
-
- def set_attrs_type_key(self, key):
- """Set the attribute type key of op.
-
- Parameters
- ----------
- key : str
- The type key.
- """
- _ffi_api.OpSetAttrsTypeKey(self, key)
-
- @staticmethod
- def list_op_names():
- """List all the op names in the op registry.
+ Non-None attribute value.
+ override : bool, optional
+ Replace an existing value if True. Duplicate registration otherwise
+ raises ValueError. Cached views observe replacements; no history is kept.
Returns
-------
- value : List[str]
- The registered op names
+ None
"""
- return _ffi_api.ListOpNames()
+ self._set_attr(attr_name, value, override)
-def register_op_attr(op_name, attr_key, value=None, level=10):
+def register_op_attr(op_name, attr_key, value=None, override=False):
"""Register an operator property of an operator by name.
Parameters
@@ -172,18 +91,20 @@ def register_op_attr(op_name, attr_key, value=None, level=10):
value : object, optional
The value to set
- level : int, optional
- The priority level
+ override : bool, optional
+ Replace an existing value if True; otherwise duplicate registration raises
+ ValueError. Cached views observe replacements; no priority history is kept.
Returns
-------
- fregister : function
- Register function if value is not specified.
+ result : object or function
+ The registered value when supplied, or a decorator that registers and
+ returns its argument. The named Op is created if it does not exist.
"""
def _register(v):
"""internal register function"""
- _ffi_api.RegisterOpAttr(op_name, attr_key, v, level)
+ _ffi_api.RegisterOpAttr(op_name, attr_key, v, override)
return v
return _register(value) if value is not None else _register
diff --git a/python/tvm/relax/op/base.py b/python/tvm/relax/op/base.py
index 77f577fe64..60914ed803 100644
--- a/python/tvm/relax/op/base.py
+++ b/python/tvm/relax/op/base.py
@@ -37,7 +37,7 @@ py_print = print # pylint: disable=invalid-name
def register_gradient(
op_name: str,
fgradient: Callable[[Var, Call, Var, "BlockBuilder"], list[Expr]] | None = None,
- level: int = 10,
+ override: bool = False,
):
"""Register operator gradient function for a relax operator.
@@ -50,10 +50,16 @@ def register_gradient(
-> partials: List[Expr]
The gradient function being used.
- level: int
- The priority level
+ override: bool, optional
+ Replace an existing gradient if True; duplicate registration otherwise
+ raises ValueError.
+
+ Returns
+ -------
+ result : callable
+ The registered gradient, or a decorator if fgradient is not supplied.
"""
- return tvm.ir.register_op_attr(op_name, "FPrimalGradient", fgradient, level)
+ return tvm.ir.register_op_attr(op_name, "FPrimalGradient", fgradient, override)
def null_value() -> Call:
diff --git a/python/tvm/target/intrin.py b/python/tvm/target/intrin.py
index 11ec8c33cc..eee85d15fb 100644
--- a/python/tvm/target/intrin.py
+++ b/python/tvm/target/intrin.py
@@ -76,6 +76,6 @@ def _rule_float_direct(op):
# opencl pattern for exp
-register_intrin_lowering("tirx.exp", target="opencl", f=_rule_float_direct, level=99)
+register_intrin_lowering("tirx.exp", target="opencl", f=_rule_float_direct, override=True)
# default pattern for exp
-register_intrin_lowering("tirx.exp", target="default", f=_rule_float_suffix, level=99)
+register_intrin_lowering("tirx.exp", target="default", f=_rule_float_suffix, override=True)
diff --git a/python/tvm/tirx/op.py b/python/tvm/tirx/op.py
index 49fc608891..f54910acc3 100644
--- a/python/tvm/tirx/op.py
+++ b/python/tvm/tirx/op.py
@@ -55,7 +55,7 @@ def register_intrin_lowering(
target,
*,
f=None,
- level=10,
+ override=False,
):
"""Register Op lowering function
@@ -70,18 +70,19 @@ def register_intrin_lowering(
f : function, optional
The function to be registered.
- level : int
- The priority level
+ override : bool, optional
+ Replace an existing lowering if True; duplicate registration otherwise
+ raises ValueError.
Returns
-------
- fregister : function
- Register op lowering function if f is not specified.
+ result : function
+ The registered lowering, or a decorator if f is not supplied.
"""
def _register(f):
"""internal register function"""
- _ffi_api.RegisterOpLowerIntrinsic(op_name, f, target, level)
+ tvm.ir.register_op_attr(op_name, target + ".FLowerIntrinsic", f, override)
return f
return _register(f) if f is not None else _register
diff --git a/python/tvm/tirx/script/ir_builder/op.py b/python/tvm/tirx/script/ir_builder/op.py
index 946a8fcd3e..a8e794dfef 100644
--- a/python/tvm/tirx/script/ir_builder/op.py
+++ b/python/tvm/tirx/script/ir_builder/op.py
@@ -247,16 +247,15 @@ class WebGPUNamespace:
webgpu = WebGPUNamespace()
-def _register_script_namespace_printer_names(ns_obj, dotted_prefix):
+def _register_script_namespace_printer_names(ns_obj, dotted_prefix, override=False):
def register_printer_name(op_name, script_name):
try:
- ir.Op.get(op_name)
- except Exception:
+ op = ir.Op.get(op_name)
+ except AttributeError:
return
- try:
- _register_op_attr(op_name, "TScriptPrinterName", script_name, level=20)
- except Exception:
- pass
+ if op.has_attr("TScriptPrinterName") and op.get_attr("TScriptPrinterName") == script_name:
+ return
+ _register_op_attr(op_name, "TScriptPrinterName", script_name, override=override)
def visit(ns_obj, dotted_prefix):
# If the namespace object itself maps to an op via __call__
@@ -298,8 +297,19 @@ def _get_script_namespace(name: str) -> object:
raise AttributeError(f"No script namespace {name!r}")
-def register_script_namespace(name: str, namespace: object) -> object:
- """Register a TVMScript namespace on the TIRx builder facade."""
+def register_script_namespace(name: str, namespace: object, override: bool = False) -> object:
+ """Register a construction namespace and return it.
+
+ Parameters
+ ----------
+ name : str
+ Namespace name on the TIRx builder facade.
+ namespace : object
+ Construction namespace object.
+ override : bool, optional
+ Replace differing operator printer names if True. Existing equal names
+ are reused; other duplicates raise ValueError.
+ """
_SCRIPT_NAMESPACES[name] = namespace
globals()[name] = namespace
if "__all__" in globals() and name not in __all__:
@@ -320,7 +330,7 @@ def register_script_namespace(name: str, namespace: object) -> object:
if isinstance(module_all, list) and name not in module_all:
module_all.append(name)
- _register_script_namespace_printer_names(namespace, name)
+ _register_script_namespace_printer_names(namespace, name, override)
return namespace
diff --git a/src/backend/cuda/codegen/codegen_cuda.cc b/src/backend/cuda/codegen/codegen_cuda.cc
index ce06c4f8a4..7cba4d4c1d 100644
--- a/src/backend/cuda/codegen/codegen_cuda.cc
+++ b/src/backend/cuda/codegen/codegen_cuda.cc
@@ -1035,19 +1035,19 @@ void CodeGenCUDA::Dispatch_(const CallNode* op, std::ostream& os) {
}
}
- static const Op& tvm_fill_fragment_op = Op::Get("tirx.tvm_fill_fragment");
- static const Op& tvm_load_matrix_sync_op = Op::Get("tirx.tvm_load_matrix_sync");
- static const Op& tvm_store_matrix_sync_op = Op::Get("tirx.tvm_store_matrix_sync");
- static const Op& tvm_mma_sync_op = Op::Get("tirx.tvm_mma_sync");
- static const Op& tvm_bmma_sync_op = Op::Get("tirx.tvm_bmma_sync");
- static const Op& mma_store_op = Op::Get("tirx.mma_store");
- static const Op& mma_fill_op = Op::Get("tirx.mma_fill");
- static const Op& ptx_mma_legacy_op = Op::Get("tirx.ptx_legacy.mma");
- static const Op& ptx_ldmatrix_legacy_op = Op::Get("tirx.ptx_legacy.ldmatrix");
- static const Op& mma_store_legacy_op = Op::Get("tirx.mma_store_legacy");
- static const Op& mma_fill_legacy_op = Op::Get("tirx.mma_fill_legacy");
- static const Op& ptx_ldg32_op = Op::Get("tirx.s_tir.ldg32");
- static const Op& cuda_func_call_op = Op::Get("tirx.cuda.func_call");
+ static const Op tvm_fill_fragment_op = Op::Get("tirx.tvm_fill_fragment");
+ static const Op tvm_load_matrix_sync_op = Op::Get("tirx.tvm_load_matrix_sync");
+ static const Op tvm_store_matrix_sync_op = Op::Get("tirx.tvm_store_matrix_sync");
+ static const Op tvm_mma_sync_op = Op::Get("tirx.tvm_mma_sync");
+ static const Op tvm_bmma_sync_op = Op::Get("tirx.tvm_bmma_sync");
+ static const Op mma_store_op = Op::Get("tirx.mma_store");
+ static const Op mma_fill_op = Op::Get("tirx.mma_fill");
+ static const Op ptx_mma_legacy_op = Op::Get("tirx.ptx_legacy.mma");
+ static const Op ptx_ldmatrix_legacy_op = Op::Get("tirx.ptx_legacy.ldmatrix");
+ static const Op mma_store_legacy_op = Op::Get("tirx.mma_store_legacy");
+ static const Op mma_fill_legacy_op = Op::Get("tirx.mma_fill_legacy");
+ static const Op ptx_ldg32_op = Op::Get("tirx.s_tir.ldg32");
+ static const Op cuda_func_call_op = Op::Get("tirx.cuda.func_call");
if (op->op.same_as(tvm_fill_fragment_op)) {
codegen_tags_.insert("mma");
@@ -1590,7 +1590,7 @@ void CodeGenCUDA::Dispatch_(const AttrStmtNode* op) {
TVM_FFI_ICHECK(queue_id && queue_id->value == 0)
<< "For CUDA, the index of an async queue must be 0.";
this->Dispatch(op->body);
- static const Op& ptx_cp_async_commit_group_op = Op::Get("tirx.ptx.cp_async_commit_group");
+ static const Op ptx_cp_async_commit_group_op = Op::Get("tirx.ptx.cp_async_commit_group");
// ptx Call layout: [operands...] [slot tokens] [pred marker ""].
auto commit_group = Call(PrimType::Void(), ptx_cp_async_commit_group_op,
{StringImm("async"), StringImm("commit_group"), StringImm("")})
@@ -1605,7 +1605,7 @@ void CodeGenCUDA::Dispatch_(const AttrStmtNode* op) {
TVM_FFI_ICHECK(queue_id && queue_id->value == 0)
<< "For CUDA, the index of an async queue must be 0.";
auto wait_cnt = wait_attrs.second;
- static const Op& ptx_cp_async_wait_group_op = Op::Get("tirx.ptx.cp_async_wait_group");
+ static const Op ptx_cp_async_wait_group_op = Op::Get("tirx.ptx.cp_async_wait_group");
// ptx Call layout: [operands...] [slot tokens] [pred marker ""]. The group
// count is a role="imm" operand baked into the instruction text, so it must
// already be a compile-time constant here (the pipeline pass guarantees it).
diff --git a/src/backend/cuda/codegen/intrin_rule_cuda.cc b/src/backend/cuda/codegen/intrin_rule_cuda.cc
index de2599ebe7..f9a3f2b5ff 100644
--- a/src/backend/cuda/codegen/intrin_rule_cuda.cc
+++ b/src/backend/cuda/codegen/intrin_rule_cuda.cc
@@ -129,17 +129,17 @@ struct CUDAPopcount {
struct CUDAWarpIntrinsic {
const Op operator()(const PrimType& ty, const Op& orig_op) const {
if (orig_op.same_as(builtin::tvm_warp_shuffle())) {
- static const Op& cuda_shfl_sync_op = Op::Get("tirx.cuda.__shfl_sync");
+ static const Op cuda_shfl_sync_op = Op::Get("tirx.cuda.__shfl_sync");
return cuda_shfl_sync_op;
} else if (orig_op.same_as(builtin::tvm_warp_shuffle_up())) {
- static const Op& cuda_shfl_up_sync_op = Op::Get("tirx.cuda.__shfl_up_sync");
+ static const Op cuda_shfl_up_sync_op = Op::Get("tirx.cuda.__shfl_up_sync");
return cuda_shfl_up_sync_op;
} else if (orig_op.same_as(builtin::tvm_warp_shuffle_down())) {
- static const Op& cuda_shfl_down_sync_op = Op::Get("tirx.cuda.__shfl_down_sync");
+ static const Op cuda_shfl_down_sync_op = Op::Get("tirx.cuda.__shfl_down_sync");
return cuda_shfl_down_sync_op;
} else {
TVM_FFI_ICHECK(orig_op.same_as(builtin::tvm_warp_shuffle_xor()));
- static const Op& cuda_shfl_xor_sync_op = Op::Get("tirx.cuda.__shfl_xor_sync");
+ static const Op cuda_shfl_xor_sync_op = Op::Get("tirx.cuda.__shfl_xor_sync");
return cuda_shfl_xor_sync_op;
}
}
@@ -147,7 +147,7 @@ struct CUDAWarpIntrinsic {
static PrimExpr DispatchCUDAWarpActiveMask(const PrimExpr& e) {
const CallNode* call = e.as<CallNode>();
- static const Op& cuda_active_mask_op = Op::Get("tirx.cuda.__activemask");
+ static const Op cuda_active_mask_op = Op::Get("tirx.cuda.__activemask");
ffi::Array<PrimExpr> args = call->args.as_or_throw<ffi::Array<PrimExpr>>();
return Call(e.ty(), cuda_active_mask_op, args).as_or_throw<PrimExpr>();
}
@@ -165,183 +165,170 @@ static PrimExpr DispatchCUDAShuffle(const PrimExpr& e) {
void RegisterCudaIntrinRules() {
// clang-format off
-TVM_REGISTER_OP("prim.clz")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic",
- DispatchPureExtern<CUDAMath, /*dtype_from_arg=*/true>);
+ OpDef("prim.clz")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic",
+ DispatchPureExtern<CUDAMath, /*dtype_from_arg=*/true>);
-TVM_REGISTER_OP("tirx.floor")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+ OpDef("tirx.floor")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-TVM_REGISTER_OP("prim.ceil")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+ OpDef("prim.ceil")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-TVM_REGISTER_OP("tirx.trunc")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+ OpDef("tirx.trunc")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-TVM_REGISTER_OP("tirx.fabs")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+ OpDef("tirx.fabs")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-TVM_REGISTER_OP("tirx.round")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+ OpDef("tirx.round")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-TVM_REGISTER_OP("tirx.nearbyint")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+ OpDef("tirx.nearbyint")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-TVM_REGISTER_OP("tirx.exp")
- .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+ OpDef("tirx.exp")
+ .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-TVM_REGISTER_OP("tirx.exp2")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+ OpDef("tirx.exp2")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-TVM_REGISTER_OP("tirx.exp10")
- .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.erf")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.log")
- .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("prim.log2")
- .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.log10")
- .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.tan")
- // Now the fast math version of tan and the default version of tan are same.
- .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic",
- DispatchPureExtern<CUDAFastMathTan>)
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.cos")
- .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.cosh")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.sin")
- .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.sinh")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.atan")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.tanh")
- .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.sqrt")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.rsqrt")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.pow")
- .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.popcount")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAPopcount>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_up")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_down")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_xor")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_activemask")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAWarpActiveMask);
-
-TVM_REGISTER_OP("tirx.fmod")
- .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-// Register low-level CUDA device intrinsics.
-// TODO(tvm-team): consider make CUDA its own subfolder and create a file for low-level builtins.
-TVM_REGISTER_OP("tirx.cuda.__shfl_sync")
- .set_num_inputs(4)
- .add_argument("mask", "Expr", "The thread mask.")
- .add_argument("var", "Expr", "The variable to sync.")
- .add_argument("lane", "Expr", "The source thread id.")
- .add_argument("width", "Expr", "The warp thread width, must be a power of 2.")
- .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
- .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"),
- 10)
- .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__shfl_sync"), 10)
- .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_sync")
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
- .set_attr<bool>("cuda.need_warp_shuffle", true);
-
-TVM_REGISTER_OP("tirx.cuda.__shfl_up_sync")
- .set_num_inputs(4)
- .add_argument("mask", "Expr", "The thread mask.")
- .add_argument("var", "Expr", "The variable to sync.")
- .add_argument("delta", "Expr", "The source lane id offset to be added.")
- .add_argument("width", "Expr", "The warp thread width, must be a power of 2.")
- .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
- .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"),
- 10)
- .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__shfl_up_sync"),
- 10)
- .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_up_sync")
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
- .set_attr<bool>("cuda.need_warp_shuffle", true);
-
-TVM_REGISTER_OP("tirx.cuda.__shfl_down_sync")
- .set_num_inputs(4)
- .add_argument("mask", "Expr", "The thread mask.")
- .add_argument("var", "Expr", "The variable to sync.")
- .add_argument("delta", "Expr", "The source lane id offset to be subtracted.")
- .add_argument("width", "Expr", "The warp thread width, must be a power of 2.")
- .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
- .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"),
- 10)
- .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
- ffi::String("cuda.__shfl_down_sync"), 10)
- .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_down_sync")
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
- .set_attr<bool>("cuda.need_warp_shuffle", true);
-
-TVM_REGISTER_OP("tirx.cuda.__shfl_xor_sync")
- .set_num_inputs(4)
- .add_argument("mask", "Expr", "The thread mask.")
- .add_argument("var", "Expr", "The variable to sync.")
- .add_argument("lane_mask", "Expr", "The lane mask.")
- .add_argument("width", "Expr", "The warp thread width, must be a power of 2.")
- .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
- .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"),
- 10)
- .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__shfl_xor_sync"),
- 10)
- .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_xor_sync")
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
- .set_attr<bool>("cuda.need_warp_shuffle", true);
-
-TVM_REGISTER_OP("tirx.cuda.__activemask")
- .set_num_inputs(0)
- .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
- .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"),
- 10)
- .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__activemask"),
- 10)
- .set_attr<TGlobalSymbol>("TGlobalSymbol", "__activemask")
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
- .set_attr<bool>("cuda.need_warp_shuffle", true);
+ OpDef("tirx.exp10")
+ .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.erf")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.log")
+ .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("prim.log2")
+ .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.log10")
+ .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.tan")
+ // Now the fast math version of tan and the default version of tan are same.
+ .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic",
+ DispatchPureExtern<CUDAFastMathTan>)
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.cos")
+ .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.cosh")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.sin")
+ .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.sinh")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.atan")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.tanh")
+ .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.sqrt")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.rsqrt")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.pow")
+ .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ OpDef("tirx.popcount")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAPopcount>);
+
+ OpDef("tirx.tvm_warp_shuffle")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
+
+ OpDef("tirx.tvm_warp_shuffle_up")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
+
+ OpDef("tirx.tvm_warp_shuffle_down")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
+
+ OpDef("tirx.tvm_warp_shuffle_xor")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
+
+ OpDef("tirx.tvm_warp_activemask")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAWarpActiveMask);
+
+ OpDef("tirx.fmod")
+ .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+ // Register low-level CUDA device intrinsics.
+ // TODO(tvm-team): consider make CUDA its own subfolder and create a file for low-level builtins.
+ OpDef("tirx.cuda.__shfl_sync")
+ .add_arg("mask", "The thread mask.")
+ .add_arg("var", "The variable to sync.")
+ .add_arg("lane", "The source thread id.")
+ .add_arg("width", "The warp thread width, must be a power of 2.")
+ .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+ .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__shfl_sync"))
+ .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_sync")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+ .set_attr<bool>("cuda.need_warp_shuffle", true);
+
+ OpDef("tirx.cuda.__shfl_up_sync")
+ .add_arg("mask", "The thread mask.")
+ .add_arg("var", "The variable to sync.")
+ .add_arg("delta", "The source lane id offset to be added.")
+ .add_arg("width", "The warp thread width, must be a power of 2.")
+ .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+ .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__shfl_up_sync"))
+ .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_up_sync")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+ .set_attr<bool>("cuda.need_warp_shuffle", true);
+
+ OpDef("tirx.cuda.__shfl_down_sync")
+ .add_arg("mask", "The thread mask.")
+ .add_arg("var", "The variable to sync.")
+ .add_arg("delta", "The source lane id offset to be subtracted.")
+ .add_arg("width", "The warp thread width, must be a power of 2.")
+ .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+ .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
+ ffi::String("cuda.__shfl_down_sync"))
+ .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_down_sync")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+ .set_attr<bool>("cuda.need_warp_shuffle", true);
+
+ OpDef("tirx.cuda.__shfl_xor_sync")
+ .add_arg("mask", "The thread mask.")
+ .add_arg("var", "The variable to sync.")
+ .add_arg("lane_mask", "The lane mask.")
+ .add_arg("width", "The warp thread width, must be a power of 2.")
+ .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+ .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__shfl_xor_sync"))
+ .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_xor_sync")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+ .set_attr<bool>("cuda.need_warp_shuffle", true);
+
+ OpDef("tirx.cuda.__activemask")
+ .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+ .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__activemask"))
+ .set_attr<TGlobalSymbol>("TGlobalSymbol", "__activemask")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
+ .set_attr<bool>("cuda.need_warp_shuffle", true);
// clang-format on
}
diff --git a/src/backend/cuda/codegen/llvm/intrin_rule_nvptx.cc b/src/backend/cuda/codegen/llvm/intrin_rule_nvptx.cc
index 3a057e5b44..bf456d1980 100644
--- a/src/backend/cuda/codegen/llvm/intrin_rule_nvptx.cc
+++ b/src/backend/cuda/codegen/llvm/intrin_rule_nvptx.cc
@@ -61,83 +61,87 @@ inline PrimExpr DispatchPureExternLibDevice(const PrimExpr& e) {
namespace llvm {
using tirx::FLowerIntrinsic;
-TVM_REGISTER_OP("tirx.floor")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ // clang-format off
+ OpDef("tirx.floor")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("prim.ceil")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("prim.ceil")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.round")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
- // Redirect to nearbyint (ties-to-even) to match constant-folding semantics.
- using namespace tirx;
- const CallNode* call = e.as<CallNode>();
- TVM_FFI_ICHECK(call != nullptr);
- static const Op& nearbyint_op = Op::Get("tirx.nearbyint");
- auto new_call =
- Call(call->ty.as_or_throw<PrimType>(), nearbyint_op, call->args).as_or_throw<PrimExpr>();
- return DispatchPureExternLibDevice(new_call);
- });
+ OpDef("tirx.round")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
+ // Redirect to nearbyint (ties-to-even) to match constant-folding semantics.
+ using namespace tirx;
+ const CallNode* call = e.as<CallNode>();
+ TVM_FFI_ICHECK(call != nullptr);
+ static const Op nearbyint_op = Op::Get("tirx.nearbyint");
+ auto new_call = Call(call->ty.as_or_throw<PrimType>(), nearbyint_op, call->args)
+ .as_or_throw<PrimExpr>();
+ return DispatchPureExternLibDevice(new_call);
+ });
-TVM_REGISTER_OP("tirx.nearbyint")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.nearbyint")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.trunc")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.trunc")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.fabs")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.fabs")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.exp")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.exp")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.exp2")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.exp2")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.exp10")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.exp10")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.erf")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.erf")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.fma")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.fma")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.log")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.log")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("prim.log2")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("prim.log2")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.log10")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.log10")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.sqrt")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.sqrt")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.pow")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.pow")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.tanh")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.tanh")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.tan")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.tan")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.cos")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.cos")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.cosh")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.cosh")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.sin")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.sin")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.sinh")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.sinh")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
-TVM_REGISTER_OP("tirx.atan")
- .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ OpDef("tirx.atan")
+ .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+ // clang-format on
+}
} // namespace llvm
} // namespace codegen
diff --git a/src/backend/cuda/op/iket.cc b/src/backend/cuda/op/iket.cc
index faac989fea..2ffb0fab6c 100644
--- a/src/backend/cuda/op/iket.cc
+++ b/src/backend/cuda/op/iket.cc
@@ -27,22 +27,68 @@
namespace tvm {
namespace tirx {
-#define TVM_REGISTER_CUDA_IKET_OP(OpName) \
- TVM_REGISTER_OP("tirx.cuda.iket_" #OpName) \
- .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin")) \
- .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda")) \
- .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket." #OpName)) \
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
-
-TVM_REGISTER_CUDA_IKET_OP(mark);
-TVM_REGISTER_CUDA_IKET_OP(range_start);
-TVM_REGISTER_CUDA_IKET_OP(range_end);
-TVM_REGISTER_CUDA_IKET_OP(range_push);
-TVM_REGISTER_CUDA_IKET_OP(range_pop);
-TVM_REGISTER_CUDA_IKET_OP(sentinel_token);
-TVM_REGISTER_CUDA_IKET_OP(official_event);
-
-#undef TVM_REGISTER_CUDA_IKET_OP
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("tirx.cuda.iket_mark")
+ .add_arg("name", "The name.")
+ .allow_extra_args()
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+ "mark"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.cuda.iket_range_start")
+ .add_arg("name", "The name.")
+ .allow_extra_args()
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+ "range_start"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.cuda.iket_range_end")
+ .add_arg("token", "The token.")
+ .allow_extra_args()
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+ "range_end"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.cuda.iket_range_push")
+ .add_arg("name", "The name.")
+ .allow_extra_args()
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+ "range_push"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.cuda.iket_range_pop")
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+ "range_pop"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.cuda.iket_sentinel_token")
+ .add_arg("name", "The name.")
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+ "sentinel_token"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.cuda.iket_official_event")
+ .add_arg("event_id", "The event identifier.")
+ .add_arg("source_code", "The source code.")
+ .allow_extra_args()
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+ "official_event"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+}
} // namespace tirx
} // namespace tvm
diff --git a/src/backend/cuda/op/target_builtin.cc b/src/backend/cuda/op/target_builtin.cc
index 5619bcc9aa..f30749370c 100644
--- a/src/backend/cuda/op/target_builtin.cc
+++ b/src/backend/cuda/op/target_builtin.cc
@@ -29,99 +29,203 @@
#include <tvm/tirx/op.h>
#include <tvm/tirx/op_attr_types.h>
+#include <initializer_list>
#include <string>
namespace tvm {
namespace tirx {
namespace builtin {
-#define TIRX_DEFINE_BUILTIN_FUNC(OpName) \
- OpRegEntry::RegisterOrGet("tirx." #OpName) \
- .set_name() \
- .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String(#OpName), 1) \
- .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"), /*plevel=*/1)
-
namespace {
void RegisterDeviceIntrinsicAliases();
}
void RegisterCudaTargetBuiltins() {
- // clang-format off
-static bool registered = false;
-if (registered) return;
-registered = true;
-
-TIRX_DEFINE_BUILTIN_FUNC(tvm_load_matrix_sync)
- .set_attr<TCallEffectKind>("TCallEffectKind",
- static_cast<int64_t>(CallEffectKind::kReadState));
-
-TIRX_DEFINE_BUILTIN_FUNC(tvm_mma_sync)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(tvm_bmma_sync)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(tvm_fill_fragment)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(tvm_store_matrix_sync)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-// Siblings of mma_store / mma_fill that accept
-// (ptr_var, offset) pairs. Codegen emits `ptr + offset` C-pointer
-// arithmetic and lower_warp_memory rewrites the offset's group component
-// to its thread-local index. Used by the s_tir tensor_intrin tensorize
-// path so per-thread fragment offsets stay element-accurate.
-TIRX_DEFINE_BUILTIN_FUNC(mma_store_legacy)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(mma_fill_legacy)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-OpRegEntry::RegisterOrGet("tirx.s_tir.ldg32")
- .set_name()
- .set_num_inputs(4)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
- .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("s_tir.ldg32"), 20)
- .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
- .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("s_tir"), 10);
-
-// Raw legacy cp.async form emitted by InjectPTXAsyncCopy (and round-tripped by
-// the T.s_tir.cp_async_raw.legacy 6-arg surface). It carries the element dtype in Call.dtype
-// and prints it dtype-first; user-issued copies go through T.ptx instead.
-OpRegEntry::RegisterOrGet("tirx.s_tir.cp_async_raw")
- .set_name()
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
- .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
- .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("s_tir"))
- .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("s_tir.cp_async_raw"))
- .set_attr<TScriptDtypePrintLocation>("TScriptDtypePrintLocation",
- static_cast<int64_t>(ScriptDtypePrintLocation::kFirst));
-
-TIRX_DEFINE_BUILTIN_FUNC(mma_store)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
- .set_attr<TScriptDtypePrintLocation>("TScriptDtypePrintLocation",
- static_cast<int64_t>(ScriptDtypePrintLocation::kFirst));
-
-TIRX_DEFINE_BUILTIN_FUNC(mma_fill)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
- .set_attr<TScriptDtypePrintLocation>("TScriptDtypePrintLocation",
- static_cast<int64_t>(ScriptDtypePrintLocation::kFirst));
-
-TIRX_DEFINE_BUILTIN_FUNC(timer_init_cuda)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(timer_start_cuda)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(timer_end_cuda)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(timer_finalize_cuda)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-RegisterDeviceIntrinsicAliases();
- // clang-format on
+ static bool registered = false;
+ if (registered) return;
+ registered = true;
+
+ OpDef("tirx.tvm_load_matrix_sync")
+ .add_arg("fragment", "The matrix fragment.")
+ .add_arg("m", "The M dimension.")
+ .add_arg("n", "The N dimension.")
+ .add_arg("k", "The K dimension.")
+ .add_arg("index", "The index.")
+ .add_arg("buffer_ptr", "The buffer pointer.")
+ .add_arg("stride", "The stride.")
+ .add_arg("layout", "The layout.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("tvm_load_matrix_sync"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind",
+ static_cast<int64_t>(CallEffectKind::kReadState));
+
+ OpDef("tirx.tvm_mma_sync")
+ .add_arg("fragment_d", "The D fragment.")
+ .add_arg("index_d", "The D fragment index.")
+ .add_arg("fragment_a", "The A fragment.")
+ .add_arg("index_a", "The A fragment index.")
+ .add_arg("fragment_b", "The B fragment.")
+ .add_arg("index_b", "The B fragment index.")
+ .add_arg("fragment_c", "The C fragment.")
+ .add_arg("index_c", "The C fragment index.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("tvm_mma_sync"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.tvm_bmma_sync")
+ .add_arg("fragment_d", "The D fragment.")
+ .add_arg("index_d", "The D fragment index.")
+ .add_arg("fragment_a", "The A fragment.")
+ .add_arg("index_a", "The A fragment index.")
+ .add_arg("fragment_b", "The B fragment.")
+ .add_arg("index_b", "The B fragment index.")
+ .add_arg("fragment_c", "The C fragment.")
+ .add_arg("index_c", "The C fragment index.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("tvm_bmma_sync"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.tvm_fill_fragment")
+ .add_arg("fragment", "The matrix fragment.")
+ .add_arg("m", "The M dimension.")
+ .add_arg("n", "The N dimension.")
+ .add_arg("k", "The K dimension.")
+ .add_arg("index", "The index.")
+ .add_arg("value", "The value to use.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("tvm_fill_fragment"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.tvm_store_matrix_sync")
+ .add_arg("fragment", "The matrix fragment.")
+ .add_arg("m", "The M dimension.")
+ .add_arg("n", "The N dimension.")
+ .add_arg("k", "The K dimension.")
+ .add_arg("index", "The index.")
+ .add_arg("buffer_ptr", "The buffer pointer.")
+ .add_arg("stride", "The stride.")
+ .add_arg("layout", "The layout.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("tvm_store_matrix_sync"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ // Siblings of mma_store / mma_fill that accept
+ // (ptr_var, offset) pairs. Codegen emits `ptr + offset` C-pointer
+ // arithmetic and lower_warp_memory rewrites the offset's group component
+ // to its thread-local index. Used by the s_tir tensor_intrin tensorize
+ // path so per-thread fragment offsets stay element-accurate.
+ OpDef("tirx.mma_store_legacy")
+ .add_arg("m", "The M dimension.")
+ .add_arg("n", "The N dimension.")
+ .add_arg("dst_ptr", "The destination pointer.")
+ .add_arg("src_ptr", "The source pointer.")
+ .add_arg("src_offset", "The source offset.")
+ .add_arg("dst_stride", "The destination stride.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.mma_store_legacy"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.mma_fill_legacy")
+ .add_arg("local_size", "The local allocation size.")
+ .add_arg("local_ptr", "The local pointer.")
+ .add_arg("offset", "The offset.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.mma_fill_legacy"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.s_tir.ldg32")
+ .add_arg("reg", "The register.")
+ .add_arg("guard", "The guard predicate.")
+ .add_arg("addr", "The address.")
+ .add_arg("local_addr", "The local address.")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("s_tir.ldg32"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("s_tir"));
+
+ // Raw legacy cp.async form emitted by InjectPTXAsyncCopy (and round-tripped by
+ // the T.s_tir.cp_async_raw.legacy 6-arg surface). It carries the element dtype in Call.dtype
+ // and prints it dtype-first; user-issued copies go through T.ptx instead.
+ OpDef("tirx.s_tir.cp_async_raw")
+ .add_arg("dst_ptr", "The destination pointer.")
+ .add_arg("dst_offset", "The destination offset.")
+ .add_arg("src_ptr", "The source pointer.")
+ .add_arg("src_offset", "The source offset.")
+ .add_arg("cp_size", "The copy size.")
+ .allow_extra_args()
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("s_tir"))
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("s_tir.cp_async_raw"))
+ .set_attr<TScriptDtypePrintLocation>("TScriptDtypePrintLocation",
+ static_cast<int64_t>(ScriptDtypePrintLocation::kFirst));
+
+ OpDef("tirx.mma_store")
+ .add_arg("m", "The M dimension.")
+ .add_arg("n", "The N dimension.")
+ .add_arg("dst_ptr", "The destination pointer.")
+ .add_arg("src_ptr", "The source pointer.")
+ .add_arg("src_offset", "The source offset.")
+ .add_arg("dst_stride", "The destination stride.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.mma_store"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+ .set_attr<TScriptDtypePrintLocation>("TScriptDtypePrintLocation",
+ static_cast<int64_t>(ScriptDtypePrintLocation::kFirst));
+
+ OpDef("tirx.mma_fill")
+ .add_arg("local_size", "The local allocation size.")
+ .add_arg("local_ptr", "The local pointer.")
+ .add_arg("offset", "The offset.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.mma_fill"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+ .set_attr<TScriptDtypePrintLocation>("TScriptDtypePrintLocation",
+ static_cast<int64_t>(ScriptDtypePrintLocation::kFirst));
+
+ OpDef("tirx.timer_init_cuda")
+ .add_arg("profiler_buffer", "The profiler buffer.")
+ .add_arg("profiler_tag", "The profiler tag.")
+ .add_arg("profiler_write_offset", "The profiler write offset.")
+ .add_arg("num_groups", "The number of groups.")
+ .add_arg("group_id", "The group identifier.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.timer_init"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.timer_start_cuda")
+ .add_arg("event_type", "The event type.")
+ .add_arg("profiler_buffer", "The profiler buffer.")
+ .add_arg("profiler_tag", "The profiler tag.")
+ .add_arg("profiler_write_offset", "The profiler write offset.")
+ .add_arg("profiler_write_stride", "The profiler write stride.")
+ .add_arg("leader_cond", "The leader condition.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.timer_start"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.timer_end_cuda")
+ .add_arg("event_type", "The event type.")
+ .add_arg("profiler_buffer", "The profiler buffer.")
+ .add_arg("profiler_tag", "The profiler tag.")
+ .add_arg("profiler_write_offset", "The profiler write offset.")
+ .add_arg("profiler_write_stride", "The profiler write stride.")
+ .add_arg("leader_cond", "The leader condition.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.timer_end"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.timer_finalize_cuda")
+ .add_arg("profiler_buffer", "The profiler buffer.")
+ .add_arg("profiler_tag", "The profiler tag.")
+ .add_arg("profiler_write_offset", "The profiler write offset.")
+ .add_arg("profiler_write_stride", "The profiler write stride.")
+ .add_arg("leader_cond", "The leader condition.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.timer_finalize"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ RegisterDeviceIntrinsicAliases();
}
namespace {
@@ -130,6 +234,8 @@ struct DeviceIntrinsicRegistration {
const char* name;
const char* namespace_name;
CallEffectKind effect_kind;
+ std::initializer_list<const char*> args;
+ bool allow_extra_args;
};
void RegisterDeviceIntrinsic(const DeviceIntrinsicRegistration& reg) {
@@ -143,117 +249,157 @@ void RegisterDeviceIntrinsic(const DeviceIntrinsicRegistration& reg) {
std::string canonical_op_name = "tirx." + namespace_name + "." + suffix;
ffi::String namespace_attr(namespace_name);
+ // Match the nested construction namespaces at the canonical registration site.
+ if (namespace_name == "cuda" &&
+ (suffix.rfind("tcgen05_", 0) == 0 || suffix.rfind("wgmma_", 0) == 0)) {
+ suffix[suffix.find('_')] = '.';
+ } else if (namespace_name == "nvshmem" &&
+ ((suffix.size() >= 6 && suffix.compare(suffix.size() - 6, 6, "_block") == 0) ||
+ (suffix.size() >= 5 && suffix.compare(suffix.size() - 5, 5, "_warp") == 0))) {
+ suffix[suffix.rfind('_')] = '.';
+ }
ffi::String printer_name(namespace_name + "." + suffix);
int64_t effect = static_cast<int64_t>(reg.effect_kind);
- auto register_one = [&](const std::string& op_name) {
- OpRegEntry::RegisterOrGet(op_name)
- .set_name()
- .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"),
- /*plevel=*/15)
- .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", namespace_attr,
- /*plevel=*/15)
- .set_attr<TCallEffectKind>("TCallEffectKind", effect, /*plevel=*/15)
- .set_attr<TScriptPrinterName>("TScriptPrinterName", printer_name, /*plevel=*/15);
- };
-
- register_one(canonical_op_name);
+ OpDef def(canonical_op_name);
+ for (const char* name : reg.args) {
+ def.add_arg(name, "");
+ }
+ if (reg.allow_extra_args) {
+ def.allow_extra_args();
+ }
+ def.set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", namespace_attr)
+ .set_attr<TCallEffectKind>("TCallEffectKind", effect)
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", printer_name);
}
-#define TIRX_DEVICE_INTRIN_ALIAS(OpName, Namespace, EffectKind) \
- {#OpName, #Namespace, CallEffectKind::EffectKind}
+#define TIRX_DEVICE_INTRIN_ALIAS(OpName, Namespace, EffectKind, ...) \
+ {#OpName, #Namespace, CallEffectKind::EffectKind, __VA_ARGS__}
const DeviceIntrinsicRegistration kDeviceIntrinsics[] = {
- TIRX_DEVICE_INTRIN_ALIAS(cuda_any_sync, cuda, kPure),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_atomic_add, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_atomic_cas, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_wait_until, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_ballot_sync, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_bfloat1622float2, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_bfloat162float, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_clock64, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_cluster_sync, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_cta_reduce, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_cta_sync, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_cvta_generic_to_shared, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_elect_sync, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_fadd2_rn, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_fdividef, cuda, kPure),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_ffs_u32, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_float22bfloat162_rn, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_float22bfloat162_rn_from_float2, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_float22half2, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_float2_x, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_float2_y, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_float8tohalf8, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_float_as_uint, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_fmul2_rn, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_fp8x4_e4m3_from_float4, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_func_call, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_get_tmem_addr, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_grid_sync, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_half2float, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_half8tofloat8, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_hmax2, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_hmin2, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_ldg, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_make_float2, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_mbarrier_wait, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_mbarrier_wait_acquire_cluster, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_mov_sreg, cuda, kPure),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_nano_sleep, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_printf, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_reduce_add_sync_u32, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_reduce_min_sync_u32, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_runtime_instr_desc, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_sm100_2sm_leader_smem_addr, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_smem_addr_from_uint64, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_syncthreads_and, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_syncthreads_or, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_tcgen05_encode_instr_descriptor, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_tcgen05_encode_instr_descriptor_block_scaled, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_tcgen05_encode_matrix_descriptor, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_thread_fence, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_thread_rank, cuda, kPure),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_trap_when_assert_failed, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_uint_as_float, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_warp_reduce, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_warp_sync, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_warpgroup_sync, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_wgmma_encode_matrix_descriptor, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(cuda_wgmma_noop_barrier, cuda, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_barrier_all, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_fence, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_getmem_nbi, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_getmem_nbi_block, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_getmem_nbi_warp, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_my_pe, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_n_pes, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_nbi, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_nbi_block, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_nbi_warp, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_signal_nbi, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_signal_nbi_block, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_signal_nbi_warp, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_quiet, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_signal_op, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(nvshmem_wait_until, nvshmem, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(ptx_legacy_ldmatrix, ptx_legacy, kOpaque),
- TIRX_DEVICE_INTRIN_ALIAS(ptx_legacy_mma, ptx_legacy, kOpaque),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_any_sync, cuda, kPure, {"mask", "pred"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_atomic_add, cuda, kOpaque, {"res_addr", "value"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_atomic_cas, cuda, kOpaque, {"ptr", "old_val", "new_val"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(
+ cuda_wait_until, cuda, kOpaque,
+ {"dst", "ptr", "condition", "scope", "space", "ptx_type", "backoff_ns"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_ballot_sync, cuda, kOpaque, {"mask", "pred"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_bfloat1622float2, cuda, kOpaque, {"packed"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_bfloat162float, cuda, kOpaque, {"src"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_clock64, cuda, kOpaque, {}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_cluster_sync, cuda, kOpaque, {}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_cta_reduce, cuda, kOpaque,
+ {"value", "op", "num_warps", "scratch"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_cta_sync, cuda, kOpaque, {}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_cvta_generic_to_shared, cuda, kOpaque, {"ptr"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_elect_sync, cuda, kOpaque, {}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_fadd2_rn, cuda, kOpaque, {"a", "b"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_fdividef, cuda, kPure, {"x", "y"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_ffs_u32, cuda, kOpaque, {"value"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_float22bfloat162_rn, cuda, kOpaque, {"v0", "v1"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_float22bfloat162_rn_from_float2, cuda, kOpaque, {"packed"},
+ false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_float22half2, cuda, kOpaque, {"dst", "src"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_float2_x, cuda, kOpaque, {"packed"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_float2_y, cuda, kOpaque, {"packed"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_float8tohalf8, cuda, kOpaque, {"src_addr", "dst_addr"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_float_as_uint, cuda, kOpaque, {"x"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_fmul2_rn, cuda, kOpaque, {"a", "b"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_fp8x4_e4m3_from_float4, cuda, kOpaque, {"x", "y", "z", "w"},
+ false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_func_call, cuda, kOpaque, {"func_name"}, true),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_get_tmem_addr, cuda, kOpaque,
+ {"addr", "row_offset", "col_offset"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_grid_sync, cuda, kOpaque, {}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_half2float, cuda, kOpaque, {"src"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_half8tofloat8, cuda, kOpaque, {"src_addr", "dst_addr"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_hmax2, cuda, kOpaque, {"a", "b"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_hmin2, cuda, kOpaque, {"a", "b"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_ldg, cuda, kOpaque, {}, true),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_make_float2, cuda, kOpaque, {"x", "y"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_mbarrier_wait, cuda, kOpaque, {"bar", "phase"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_mbarrier_wait_acquire_cluster, cuda, kOpaque, {"bar", "phase"},
+ false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_mov_sreg, cuda, kPure, {"bits", "reg_name"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_nano_sleep, cuda, kOpaque, {"time"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_printf, cuda, kOpaque, {"fmt"}, true),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_reduce_add_sync_u32, cuda, kOpaque, {"mask", "value"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_reduce_min_sync_u32, cuda, kOpaque, {"mask", "value"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_runtime_instr_desc, cuda, kOpaque, {"desc", "sf_id"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_sm100_2sm_leader_smem_addr, cuda, kOpaque, {"ptr"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_smem_addr_from_uint64, cuda, kOpaque, {"cluster_addr"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_syncthreads_and, cuda, kOpaque, {"cond"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_syncthreads_or, cuda, kOpaque, {"cond"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_tcgen05_encode_instr_descriptor, cuda, kOpaque,
+ {"desc", "d_dtype", "a_dtype", "b_dtype", "M", "N", "K", "trans_a",
+ "trans_b", "n_cta_groups", "neg_a", "neg_b", "sat_d", "is_sparse"},
+ false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_tcgen05_encode_instr_descriptor_block_scaled, cuda, kOpaque,
+ {"desc", "d_dtype", "a_dtype", "b_dtype", "sfa_dtype", "sfb_dtype",
+ "sfa_tmem_addr", "sfb_tmem_addr", "M", "N", "K", "trans_a", "trans_b",
+ "n_cta_groups", "neg_a", "neg_b", "is_sparse"},
+ false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_tcgen05_encode_matrix_descriptor, cuda, kOpaque,
+ {"desc", "addr", "ldo", "sdo", "swizzle"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_thread_fence, cuda, kOpaque, {}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_thread_rank, cuda, kPure, {}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_trap_when_assert_failed, cuda, kOpaque, {"cond"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_uint_as_float, cuda, kOpaque, {"bits"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_warp_reduce, cuda, kOpaque, {"value", "op", "width"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_warp_sync, cuda, kOpaque, {}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_warpgroup_sync, cuda, kOpaque, {"bar_no"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_wgmma_encode_matrix_descriptor, cuda, kOpaque,
+ {"desc", "addr", "ldo", "sdo", "swizzle"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(cuda_wgmma_noop_barrier, cuda, kOpaque, {"reg"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_barrier_all, nvshmem, kOpaque, {}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_fence, nvshmem, kOpaque, {}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_getmem_nbi, nvshmem, kOpaque, {"dst", "src", "nelems", "pe"},
+ false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_getmem_nbi_block, nvshmem, kOpaque,
+ {"dst", "src", "nelems", "pe"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_getmem_nbi_warp, nvshmem, kOpaque,
+ {"dst", "src", "nelems", "pe"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_my_pe, nvshmem, kOpaque, {}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_n_pes, nvshmem, kOpaque, {}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_nbi, nvshmem, kOpaque, {"dst", "src", "nelems", "pe"},
+ false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_nbi_block, nvshmem, kOpaque,
+ {"dst", "src", "nelems", "pe"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_nbi_warp, nvshmem, kOpaque,
+ {"dst", "src", "nelems", "pe"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_signal_nbi, nvshmem, kOpaque,
+ {"dst", "src", "nelems", "sig_addr", "signal", "sig_op", "pe"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_signal_nbi_block, nvshmem, kOpaque,
+ {"dst", "src", "nelems", "sig_addr", "signal", "sig_op", "pe"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_signal_nbi_warp, nvshmem, kOpaque,
+ {"dst", "src", "nelems", "sig_addr", "signal", "sig_op", "pe"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_quiet, nvshmem, kOpaque, {}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_signal_op, nvshmem, kOpaque,
+ {"sig_addr", "signal", "sig_op", "pe"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(nvshmem_wait_until, nvshmem, kOpaque,
+ {"ivar", "cmp", "cmp_value", "type"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(
+ ptx_legacy_ldmatrix,
+ ptx_legacy, kOpaque,
+ {"trans", "num", "dtype", "local_ptr", "local_offset", "smem_ptr", "smem_offset"}, false),
+ TIRX_DEVICE_INTRIN_ALIAS(
+ ptx_legacy_mma, ptx_legacy, kOpaque,
+ {"shape", "a_layout", "b_layout", "a_dtype", "b_dtype", "c_dtype", "a_ptr", "a_offset",
+ "b_ptr", "b_offset", "acc_ptr", "c_offset", "saturate"},
+ true),
};
+#undef TIRX_DEVICE_INTRIN_ALIAS
+
void RegisterDeviceIntrinsicAliases() {
for (const auto& reg : kDeviceIntrinsics) {
RegisterDeviceIntrinsic(reg);
}
}
-#undef TIRX_DEVICE_INTRIN_ALIAS
-
} // namespace
-#undef TIRX_DEFINE_BUILTIN_FUNC
-
TVM_FFI_STATIC_INIT_BLOCK() { RegisterCudaTargetBuiltins(); }
} // namespace builtin
diff --git a/src/backend/cuda/transforms/lower_iket.cc b/src/backend/cuda/transforms/lower_iket.cc
index ce153e2cfe..79d7c32b94 100644
--- a/src/backend/cuda/transforms/lower_iket.cc
+++ b/src/backend/cuda/transforms/lower_iket.cc
@@ -87,32 +87,32 @@ enum class PayloadType : uint32_t {
};
const Op& IketMarkOp() {
- static const Op& op = Op::Get("tirx.cuda.iket_mark");
+ static const Op op = Op::Get("tirx.cuda.iket_mark");
return op;
}
const Op& IketRangeStartOp() {
- static const Op& op = Op::Get("tirx.cuda.iket_range_start");
+ static const Op op = Op::Get("tirx.cuda.iket_range_start");
return op;
}
const Op& IketRangeEndOp() {
- static const Op& op = Op::Get("tirx.cuda.iket_range_end");
+ static const Op op = Op::Get("tirx.cuda.iket_range_end");
return op;
}
const Op& IketRangePushOp() {
- static const Op& op = Op::Get("tirx.cuda.iket_range_push");
+ static const Op op = Op::Get("tirx.cuda.iket_range_push");
return op;
}
const Op& IketRangePopOp() {
- static const Op& op = Op::Get("tirx.cuda.iket_range_pop");
+ static const Op op = Op::Get("tirx.cuda.iket_range_pop");
return op;
}
const Op& IketSentinelOp() {
- static const Op& op = Op::Get("tirx.cuda.iket_sentinel_token");
+ static const Op op = Op::Get("tirx.cuda.iket_sentinel_token");
return op;
}
@@ -1117,13 +1117,13 @@ class InstrumentOfficialKernel : public StmtExprMutator {
}
PrimExpr Event(PrimExpr event_id) const {
- static const Op& event_op = Op::Get("tirx.cuda.iket_official_event");
+ static const Op event_op = Op::Get("tirx.cuda.iket_official_event");
return Call(PrimType::UInt(32), event_op,
{prim::cast(PrimType::UInt(32), event_id), StringImm(device_source_)});
}
PrimExpr Event(PrimExpr event_id, PrimExpr payload) const {
- static const Op& event_op = Op::Get("tirx.cuda.iket_official_event");
+ static const Op event_op = Op::Get("tirx.cuda.iket_official_event");
return Call(
PrimType::UInt(32), event_op,
{prim::cast(PrimType::UInt(32), event_id), StringImm(device_source_), std::move(payload)});
diff --git a/src/backend/hexagon/codegen/llvm/intrin_rule_hexagon.cc b/src/backend/hexagon/codegen/llvm/intrin_rule_hexagon.cc
index efa81d905c..cacbd924d6 100644
--- a/src/backend/hexagon/codegen/llvm/intrin_rule_hexagon.cc
+++ b/src/backend/hexagon/codegen/llvm/intrin_rule_hexagon.cc
@@ -25,13 +25,6 @@
#include "../../../../target/llvm/intrin_rule_llvm.h"
-#define TVM_REGISTER_QHL_OP_FP16(INTRIN_FUNC, NUM_SIGN) \
- TVM_REGISTER_OP("tirx." #INTRIN_FUNC) \
- .set_attr<FLowerIntrinsic>( \
- "hexagon.FLowerIntrinsic", \
- DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_##INTRIN_FUNC, ::llvm::Intrinsic::INTRIN_FUNC, \
- NUM_SIGN>);
-
namespace tvm {
namespace codegen {
using namespace tvm::prim;
@@ -95,154 +88,173 @@ void RegisterHexagonIntrinRules() {
registered = true;
// clang-format off
-TVM_REGISTER_OP("tirx.fma")
- .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::fmuladd, 3>);
-
-TVM_REGISTER_OP("tirx.log")
- .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::log, 1>);
-
-TVM_REGISTER_OP("tirx.trunc")
- .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::trunc, 1>);
-
-TVM_REGISTER_OP("tirx.fabs")
- .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::fabs, 1>);
-
-TVM_REGISTER_OP("tirx.round")
- .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
-
-TVM_REGISTER_OP("tirx.ctpop")
- .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"), 1)
- .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::ctpop, 1>);
-TVM_REGISTER_OP("tirx.tanh")
- .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic", [](const PrimExpr& e) {
- const CallNode* call = e.as<CallNode>();
- TVM_FFI_ICHECK(call != nullptr);
- PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
- PrimType x_ty = x.ty();
+ OpDef("tirx.fma")
+ .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::fmuladd, 3>);
+
+ OpDef("tirx.log")
+ .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::log, 1>);
+
+ OpDef("tirx.trunc")
+ .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::trunc, 1>);
+
+ OpDef("tirx.fabs")
+ .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::fabs, 1>);
+
+ OpDef("tirx.round")
+ .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
+
+ OpDef("tirx.ctpop")
+ .add_arg("x", "The input value.")
+ .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::ctpop, 1>);
+
+ OpDef("tirx.tanh")
+ .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic", [](const PrimExpr& e) {
+ const CallNode* call = e.as<CallNode>();
+ TVM_FFI_ICHECK(call != nullptr);
+ PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
+ PrimType x_ty = x.ty();
#if ENABLE_QHL
- // Check target for qfloat enablement
- const auto f = tvm::ffi::Function::GetGlobal("target.TargetCurrent");
- TVM_FFI_ICHECK(f.has_value()) << "target.TargetCurrent is not registered";
- const auto ret = (*f)(true);
- bool useqhl = true;
- if (auto opt_target = ret.as<Target>()) {
- const std::string tstring = opt_target.value()->str();
- useqhl = tstring.find("+hvx-qfloat") != std::string::npos;
- }
-
- // Enable QHL library for FP16 data type
- if (x_ty.MatchesElementType(DLDataTypeCode::kDLFloat, 16) &&
- (x_ty.IsFixedLengthVector() || x_ty.IsScalableVector()) && useqhl) {
- std::string tvm_wrapper("tvm_vect_qhmath_hvx_tanh_ahf");
- return TVMExternCall(call, tvm_wrapper);
- }
+ // Check target for qfloat enablement
+ const auto f = tvm::ffi::Function::GetGlobal("target.TargetCurrent");
+ TVM_FFI_ICHECK(f.has_value()) << "target.TargetCurrent is not registered";
+ const auto ret = (*f)(true);
+ bool useqhl = true;
+ if (auto opt_target = ret.as<Target>()) {
+ const std::string tstring = opt_target.value()->str();
+ useqhl = tstring.find("+hvx-qfloat") != std::string::npos;
+ }
+
+ // Enable QHL library for FP16 data type
+ if (x_ty.MatchesElementType(DLDataTypeCode::kDLFloat, 16) &&
+ (x_ty.IsFixedLengthVector() || x_ty.IsScalableVector()) && useqhl) {
+ std::string tvm_wrapper("tvm_vect_qhmath_hvx_tanh_ahf");
+ return TVMExternCall(call, tvm_wrapper);
+ }
#endif
- PrimExpr one = tvm::prim::MakeConst(x_ty, 1);
- PrimExpr two = tvm::prim::MakeConst(x_ty, 2);
- PrimExpr neg_two = tvm::prim::MakeConst(x_ty, -2);
-
- PrimExpr exp_neg2x = exp(neg_two * x);
- PrimExpr exp_pos2x = exp(two * x);
-
- PrimExpr tanh_pos = (one - exp_neg2x) / (one + exp_neg2x);
- PrimExpr tanh_neg = (exp_pos2x - one) / (exp_pos2x + one);
- // MakeConst can handle both vector and scalar types.
- PrimExpr tanh_x = prim::Select(x >= tvm::prim::MakeConst(x_ty, 0), tanh_pos, tanh_neg);
- return tanh_x;
- });
-
-TVM_REGISTER_OP("tirx.tan")
- .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic", [](const PrimExpr& e) {
- const CallNode* call = e.as<CallNode>();
- TVM_FFI_ICHECK(call != nullptr);
- PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
- PrimType x_ty = x.ty();
+ PrimExpr one = tvm::prim::MakeConst(x_ty, 1);
+ PrimExpr two = tvm::prim::MakeConst(x_ty, 2);
+ PrimExpr neg_two = tvm::prim::MakeConst(x_ty, -2);
+
+ PrimExpr exp_neg2x = exp(neg_two * x);
+ PrimExpr exp_pos2x = exp(two * x);
+
+ PrimExpr tanh_pos = (one - exp_neg2x) / (one + exp_neg2x);
+ PrimExpr tanh_neg = (exp_pos2x - one) / (exp_pos2x + one);
+ // MakeConst can handle both vector and scalar types.
+ PrimExpr tanh_x = prim::Select(x >= tvm::prim::MakeConst(x_ty, 0), tanh_pos, tanh_neg);
+ return tanh_x;
+ });
+
+ OpDef("tirx.tan")
+ .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic", [](const PrimExpr& e) {
+ const CallNode* call = e.as<CallNode>();
+ TVM_FFI_ICHECK(call != nullptr);
+ PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
+ PrimType x_ty = x.ty();
#if ENABLE_QHL
- // Check target for qfloat enablement
- const auto f = tvm::ffi::Function::GetGlobal("target.TargetCurrent");
- TVM_FFI_ICHECK(f.has_value()) << "target.TargetCurrent is not registered";
- const auto ret = (*f)(true);
- bool useqhl = true;
- if (auto opt_target = ret.as<Target>()) {
- const std::string tstring = opt_target.value()->str();
- useqhl = tstring.find("+hvx-qfloat") != std::string::npos;
- }
-
- // Enable QHL library for FP16 data type
- if (x_ty.MatchesElementType(DLDataTypeCode::kDLFloat, 16) &&
- (x_ty.IsFixedLengthVector() || x_ty.IsScalableVector()) && useqhl) {
- std::string tvm_wrapper("tvm_vect_qhmath_hvx_tan_ahf");
- return TVMExternCall(call, tvm_wrapper);
- }
+ // Check target for qfloat enablement
+ const auto f = tvm::ffi::Function::GetGlobal("target.TargetCurrent");
+ TVM_FFI_ICHECK(f.has_value()) << "target.TargetCurrent is not registered";
+ const auto ret = (*f)(true);
+ bool useqhl = true;
+ if (auto opt_target = ret.as<Target>()) {
+ const std::string tstring = opt_target.value()->str();
+ useqhl = tstring.find("+hvx-qfloat") != std::string::npos;
+ }
+
+ // Enable QHL library for FP16 data type
+ if (x_ty.MatchesElementType(DLDataTypeCode::kDLFloat, 16) &&
+ (x_ty.IsFixedLengthVector() || x_ty.IsScalableVector()) && useqhl) {
+ std::string tvm_wrapper("tvm_vect_qhmath_hvx_tan_ahf");
+ return TVMExternCall(call, tvm_wrapper);
+ }
#endif
- PrimExpr tan_x = sin(x) / cos(x);
- return tan_x;
- });
-
-TVM_REGISTER_OP("tirx.nearbyint")
- .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
-
-TVM_REGISTER_OP("tirx.sigmoid")
- .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic", [](const PrimExpr& e) {
- const CallNode* call = e.as<CallNode>();
- TVM_FFI_ICHECK(call != nullptr);
- PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
- PrimType x_ty = x.ty();
+ PrimExpr tan_x = sin(x) / cos(x);
+ return tan_x;
+ });
+
+ OpDef("tirx.nearbyint")
+ .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
+
+ OpDef("tirx.sigmoid")
+ .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic", [](const PrimExpr& e) {
+ const CallNode* call = e.as<CallNode>();
+ TVM_FFI_ICHECK(call != nullptr);
+ PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
+ PrimType x_ty = x.ty();
#if ENABLE_QHL
- // Check target for qfloat enablement
- const auto f = tvm::ffi::Function::GetGlobal("target.TargetCurrent");
- TVM_FFI_ICHECK(f.has_value()) << "target.TargetCurrent is not registered";
- const auto ret = (*f)(true);
- bool useqhl = true;
- if (auto opt_target = ret.as<Target>()) {
- const std::string tstring = opt_target.value()->str();
- useqhl = tstring.find("+hvx-qfloat") != std::string::npos;
- }
-
- PrimExpr MinBound = tvm::prim::MakeConst(x_ty, -8);
- PrimExpr MaxBound = tvm::prim::MakeConst(x_ty, 8);
- const PrimExpr v1 = prim::Max(x, MinBound);
- const PrimExpr v2 = prim::Min(v1, MaxBound);
-
- ffi::Array<tvm::PrimExpr> new_args = {v2};
- const Call new_call =
- Call(call->ty.as_or_throw<PrimType>(), call->op, new_args);
-
- // Enable QHL library for FP16 data type
- if (x_ty.MatchesElementType(DLDataTypeCode::kDLFloat, 16) &&
- (x_ty.IsFixedLengthVector() || x_ty.IsScalableVector()) && useqhl) {
- std::string tvm_wrapper("tvm_vect_qhmath_hvx_sigmoid_ahf");
- return TVMExternCall(new_call.get(), tvm_wrapper);
- }
+ // Check target for qfloat enablement
+ const auto f = tvm::ffi::Function::GetGlobal("target.TargetCurrent");
+ TVM_FFI_ICHECK(f.has_value()) << "target.TargetCurrent is not registered";
+ const auto ret = (*f)(true);
+ bool useqhl = true;
+ if (auto opt_target = ret.as<Target>()) {
+ const std::string tstring = opt_target.value()->str();
+ useqhl = tstring.find("+hvx-qfloat") != std::string::npos;
+ }
+
+ PrimExpr MinBound = tvm::prim::MakeConst(x_ty, -8);
+ PrimExpr MaxBound = tvm::prim::MakeConst(x_ty, 8);
+ const PrimExpr v1 = prim::Max(x, MinBound);
+ const PrimExpr v2 = prim::Min(v1, MaxBound);
+
+ ffi::Array<tvm::PrimExpr> new_args = {v2};
+ const Call new_call = Call(call->ty.as_or_throw<PrimType>(), call->op, new_args);
+
+ // Enable QHL library for FP16 data type
+ if (x_ty.MatchesElementType(DLDataTypeCode::kDLFloat, 16) &&
+ (x_ty.IsFixedLengthVector() || x_ty.IsScalableVector()) && useqhl) {
+ std::string tvm_wrapper("tvm_vect_qhmath_hvx_sigmoid_ahf");
+ return TVMExternCall(new_call.get(), tvm_wrapper);
+ }
#endif
- PrimExpr one = tvm::prim::MakeConst(x_ty, 1);
- return one / (one + exp(-x));
- });
-
-TVM_REGISTER_OP("prim.ceil")
- .set_attr<FLowerIntrinsic>(
- "hexagon.FLowerIntrinsic",
- DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_ceil, ::llvm::Intrinsic::ceil, 1>);
-
-TVM_REGISTER_QHL_OP_FP16(cos, 1)
-
-TVM_REGISTER_QHL_OP_FP16(exp, 1)
-
-TVM_REGISTER_QHL_OP_FP16(floor, 1)
-
-TVM_REGISTER_QHL_OP_FP16(sin, 1)
-
-TVM_REGISTER_QHL_OP_FP16(pow, 2)
-
-TVM_REGISTER_QHL_OP_FP16(sqrt, 1)
+ PrimExpr one = tvm::prim::MakeConst(x_ty, 1);
+ return one / (one + exp(-x));
+ });
+
+ OpDef("prim.ceil")
+ .set_attr<FLowerIntrinsic>(
+ "hexagon.FLowerIntrinsic",
+ DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_ceil, ::llvm::Intrinsic::ceil, 1>);
+
+ OpDef("tirx.cos")
+ .set_attr<FLowerIntrinsic>(
+ "hexagon.FLowerIntrinsic",
+ DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_cos, ::llvm::Intrinsic::cos, 1>);
+
+ OpDef("tirx.exp")
+ .set_attr<FLowerIntrinsic>(
+ "hexagon.FLowerIntrinsic",
+ DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_exp, ::llvm::Intrinsic::exp, 1>);
+
+ OpDef("tirx.floor")
+ .set_attr<FLowerIntrinsic>(
+ "hexagon.FLowerIntrinsic",
+ DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_floor, ::llvm::Intrinsic::floor, 1>);
+
+ OpDef("tirx.sin")
+ .set_attr<FLowerIntrinsic>(
+ "hexagon.FLowerIntrinsic",
+ DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_sin, ::llvm::Intrinsic::sin, 1>);
+
+ OpDef("tirx.pow")
+ .set_attr<FLowerIntrinsic>(
+ "hexagon.FLowerIntrinsic",
+ DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_pow, ::llvm::Intrinsic::pow, 2>);
+
+ OpDef("tirx.sqrt")
+ .set_attr<FLowerIntrinsic>(
+ "hexagon.FLowerIntrinsic",
+ DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_sqrt, ::llvm::Intrinsic::sqrt, 1>);
// clang-format on
}
diff --git a/src/backend/metal/codegen/codegen_metal.cc b/src/backend/metal/codegen/codegen_metal.cc
index 5325cefe6d..74809fafdd 100644
--- a/src/backend/metal/codegen/codegen_metal.cc
+++ b/src/backend/metal/codegen/codegen_metal.cc
@@ -449,10 +449,10 @@ void CodeGenMetal::Dispatch_(const CallNode* op, std::ostream& os) { // NOLINT(
<< "Only 8x8 matrix is supported, but got " << col_val << "x" << row_val;
};
- static const Op& make_filled_simdgroup_matrix_op = Op::Get("tirx.make_filled_simdgroup_matrix");
- static const Op& simdgroup_load_op = Op::Get("tirx.simdgroup_load");
- static const Op& simdgroup_store_op = Op::Get("tirx.simdgroup_store");
- static const Op& simdgroup_multiply_accumulate_op = Op::Get("tirx.simdgroup_multiply_accumulate");
+ static const Op make_filled_simdgroup_matrix_op = Op::Get("tirx.make_filled_simdgroup_matrix");
+ static const Op simdgroup_load_op = Op::Get("tirx.simdgroup_load");
+ static const Op simdgroup_store_op = Op::Get("tirx.simdgroup_store");
+ static const Op simdgroup_multiply_accumulate_op = Op::Get("tirx.simdgroup_multiply_accumulate");
if (op->op.same_as(make_filled_simdgroup_matrix_op)) {
TVM_FFI_ICHECK_EQ(op->args.size(), 5);
diff --git a/src/backend/metal/codegen/intrin_rule_metal.cc b/src/backend/metal/codegen/intrin_rule_metal.cc
index cff357ab19..b5eb1f86ae 100644
--- a/src/backend/metal/codegen/intrin_rule_metal.cc
+++ b/src/backend/metal/codegen/intrin_rule_metal.cc
@@ -33,14 +33,14 @@ using tirx::FLowerIntrinsic;
struct MetalWarpIntrinsic {
const Op operator()(PrimType t, const Op& orig_op) const {
if (orig_op.same_as(builtin::tvm_warp_shuffle())) {
- static const Op& metal_simd_shuffle_op = Op::Get("tirx.metal.simd_shuffle");
+ static const Op metal_simd_shuffle_op = Op::Get("tirx.metal.simd_shuffle");
return metal_simd_shuffle_op;
} else if (orig_op.same_as(builtin::tvm_warp_shuffle_up())) {
- static const Op& metal_simd_shuffle_up_op = Op::Get("tirx.metal.simd_shuffle_up");
+ static const Op metal_simd_shuffle_up_op = Op::Get("tirx.metal.simd_shuffle_up");
return metal_simd_shuffle_up_op;
} else {
TVM_FFI_ICHECK(orig_op.same_as(builtin::tvm_warp_shuffle_down()));
- static const Op& metal_simd_shuffle_down_op = Op::Get("tirx.metal.simd_shuffle_down");
+ static const Op metal_simd_shuffle_down_op = Op::Get("tirx.metal.simd_shuffle_down");
return metal_simd_shuffle_down_op;
}
}
@@ -58,128 +58,122 @@ static PrimExpr DispatchMetalShuffle(const PrimExpr& e) {
void RegisterMetalIntrinRules() {
// clang-format off
-TVM_REGISTER_OP("prim.clz")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("prim.clz")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.floor")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.floor")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("prim.ceil")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("prim.ceil")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.trunc")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.trunc")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.fabs")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.fabs")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.round")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
- // Metal's rint() uses ties-to-even, matching constant-folding semantics.
- const CallNode* call = e.as<CallNode>();
- TVM_FFI_ICHECK(call != nullptr);
- ffi::Array<Expr> new_args = {StringImm("rint")};
- for (const PrimExpr& arg : call->args.as_or_throw<ffi::Array<PrimExpr>>()) {
- new_args.push_back(arg);
- }
- return Call(e.ty(), tirx::builtin::call_pure_extern(), new_args).as_or_throw<PrimExpr>();
- });
-
-TVM_REGISTER_OP("tirx.nearbyint")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.exp")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.exp2")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.exp10")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.log")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("prim.log2")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.log10")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.tanh")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchNumericalStableTanh);
-
-TVM_REGISTER_OP("tirx.sqrt")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.pow")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.popcount")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.fmod")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.sin")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.sinh")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.cos")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.cosh")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.erf").set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchFastErf);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchMetalShuffle<MetalWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_up")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchMetalShuffle<MetalWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_down")
- .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchMetalShuffle<MetalWarpIntrinsic>);
-
-// Register low-level Metal device intrinsics.
-TVM_REGISTER_OP("tirx.metal.simd_shuffle")
- .set_num_inputs(2)
- .add_argument("var", "Expr", "The variable to sync.")
- .add_argument("lane", "Expr", "The source thread id.")
- .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
- .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("metal"),
- 10)
- .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("metal.simd_shuffle"),
- 10)
- .set_attr<TGlobalSymbol>("TGlobalSymbol", "simd_shuffle")
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TVM_REGISTER_OP("tirx.metal.simd_shuffle_up")
- .set_num_inputs(2)
- .add_argument("var", "Expr", "The variable to sync.")
- .add_argument("delta", "Expr", "The source lane id offset to be added.")
- .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
- .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("metal"),
- 10)
- .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
- ffi::String("metal.simd_shuffle_up"), 10)
- .set_attr<TGlobalSymbol>("TGlobalSymbol", "simd_shuffle_up")
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TVM_REGISTER_OP("tirx.metal.simd_shuffle_down")
- .set_num_inputs(2)
- .add_argument("var", "Expr", "The variable to sync.")
- .add_argument("delta", "Expr", "The source lane id offset to be subtracted.")
- .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
- .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("metal"),
- 10)
- .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
- ffi::String("metal.simd_shuffle_down"), 10)
- .set_attr<TGlobalSymbol>("TGlobalSymbol", "simd_shuffle_down")
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+ OpDef("tirx.round")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
+ // Metal's rint() uses ties-to-even, matching constant-folding semantics.
+ const CallNode* call = e.as<CallNode>();
+ TVM_FFI_ICHECK(call != nullptr);
+ ffi::Array<Expr> new_args = {StringImm("rint")};
+ for (const PrimExpr& arg : call->args.as_or_throw<ffi::Array<PrimExpr>>()) {
+ new_args.push_back(arg);
+ }
+ return Call(e.ty(), tirx::builtin::call_pure_extern(), new_args).as_or_throw<PrimExpr>();
+ });
+
+ OpDef("tirx.nearbyint")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.exp")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.exp2")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.exp10")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.log")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("prim.log2")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.log10")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.tanh")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchNumericalStableTanh);
+
+ OpDef("tirx.sqrt")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.pow")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.popcount")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.fmod")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.sin")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.sinh")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.cos")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.cosh")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.erf")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchFastErf);
+
+ OpDef("tirx.tvm_warp_shuffle")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchMetalShuffle<MetalWarpIntrinsic>);
+
+ OpDef("tirx.tvm_warp_shuffle_up")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchMetalShuffle<MetalWarpIntrinsic>);
+
+ OpDef("tirx.tvm_warp_shuffle_down")
+ .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchMetalShuffle<MetalWarpIntrinsic>);
+
+ // Register low-level Metal device intrinsics.
+ OpDef("tirx.metal.simd_shuffle")
+ .add_arg("var", "The variable to sync.")
+ .add_arg("lane", "The source thread id.")
+ .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("metal"))
+ .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("metal.simd_shuffle"))
+ .set_attr<TGlobalSymbol>("TGlobalSymbol", "simd_shuffle")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.metal.simd_shuffle_up")
+ .add_arg("var", "The variable to sync.")
+ .add_arg("delta", "The source lane id offset to be added.")
+ .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("metal"))
+ .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
+ ffi::String("metal.simd_shuffle_up"))
+ .set_attr<TGlobalSymbol>("TGlobalSymbol", "simd_shuffle_up")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.metal.simd_shuffle_down")
+ .add_arg("var", "The variable to sync.")
+ .add_arg("delta", "The source lane id offset to be subtracted.")
+ .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("metal"))
+ .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
+ ffi::String("metal.simd_shuffle_down"))
+ .set_attr<TGlobalSymbol>("TGlobalSymbol", "simd_shuffle_down")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
// clang-format on
}
diff --git a/src/backend/metal/op/target_builtin.cc b/src/backend/metal/op/target_builtin.cc
index 08b3bfd95a..e6fdcdbcd4 100644
--- a/src/backend/metal/op/target_builtin.cc
+++ b/src/backend/metal/op/target_builtin.cc
@@ -31,34 +31,61 @@ namespace tvm {
namespace tirx {
namespace builtin {
-#define TIRX_DEFINE_BUILTIN_FUNC(OpName) \
- OpRegEntry::RegisterOrGet("tirx." #OpName) \
- .set_name() \
- .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String(#OpName), 1) \
- .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"), /*plevel=*/1)
-
void RegisterMetalTargetBuiltins() {
- // clang-format off
-static bool registered = false;
-if (registered) return;
-registered = true;
+ static bool registered = false;
+ if (registered) return;
+ registered = true;
-TIRX_DEFINE_BUILTIN_FUNC(make_filled_simdgroup_matrix)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+ OpDef("tirx.make_filled_simdgroup_matrix")
+ .add_arg("d", "The D operand.")
+ .add_arg("index", "The index.")
+ .add_arg("value", "The value to use.")
+ .add_arg("col", "The column index.")
+ .add_arg("row", "The row index.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName",
+ ffi::String("metal.make_filled_simdgroup_matrix"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-TIRX_DEFINE_BUILTIN_FUNC(simdgroup_load)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+ OpDef("tirx.simdgroup_load")
+ .add_arg("d", "The D operand.")
+ .add_arg("index", "The index.")
+ .add_arg("ptr", "The pointer.")
+ .add_arg("stride", "The stride.")
+ .add_arg("col", "The column index.")
+ .add_arg("row", "The row index.")
+ .add_arg("transpose_matrix", "Whether to transpose the matrix.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("metal.simdgroup_load"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-TIRX_DEFINE_BUILTIN_FUNC(simdgroup_store)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+ OpDef("tirx.simdgroup_store")
+ .add_arg("d", "The D operand.")
+ .add_arg("index", "The index.")
+ .add_arg("ptr", "The pointer.")
+ .add_arg("stride", "The stride.")
+ .add_arg("col", "The column index.")
+ .add_arg("row", "The row index.")
+ .add_arg("transpose_matrix", "Whether to transpose the matrix.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("metal.simdgroup_store"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-TIRX_DEFINE_BUILTIN_FUNC(simdgroup_multiply_accumulate)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
- // clang-format on
+ OpDef("tirx.simdgroup_multiply_accumulate")
+ .add_arg("d", "The D operand.")
+ .add_arg("index_d", "The D fragment index.")
+ .add_arg("a", "The A operand.")
+ .add_arg("index_a", "The A fragment index.")
+ .add_arg("b", "The B operand.")
+ .add_arg("index_b", "The B fragment index.")
+ .add_arg("c", "The C operand.")
+ .add_arg("index_c", "The C fragment index.")
+ .set_attr<TScriptPrinterName>("TScriptPrinterName",
+ ffi::String("metal.simdgroup_multiply_accumulate"))
+ .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
}
-#undef TIRX_DEFINE_BUILTIN_FUNC
-
TVM_FFI_STATIC_INIT_BLOCK() { RegisterMetalTargetBuiltins(); }
} // namespace builtin
diff --git a/src/backend/opencl/codegen/intrin_rule_opencl.cc b/src/backend/opencl/codegen/intrin_rule_opencl.cc
index a911e07873..4a80c424a3 100644
--- a/src/backend/opencl/codegen/intrin_rule_opencl.cc
+++ b/src/backend/opencl/codegen/intrin_rule_opencl.cc
@@ -53,86 +53,86 @@ void RegisterOpenCLIntrinRules() {
registered = true;
// clang-format off
-TVM_REGISTER_OP("prim.clz")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("prim.clz")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.floor")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.floor")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("prim.ceil")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("prim.ceil")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.trunc")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.trunc")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.fabs")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.fabs")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.round")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
- // OpenCL's rint() uses ties-to-even, matching constant-folding semantics.
- const CallNode* call = e.as<CallNode>();
- TVM_FFI_ICHECK(call != nullptr);
- ffi::Array<Expr> new_args = {StringImm("rint")};
- for (const PrimExpr& arg : call->args.as_or_throw<ffi::Array<PrimExpr>>()) {
- new_args.push_back(arg);
- }
- return Call(e.ty(), tirx::builtin::call_pure_extern(), new_args).as_or_throw<PrimExpr>();
- });
+ OpDef("tirx.round")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
+ // OpenCL's rint() uses ties-to-even, matching constant-folding semantics.
+ const CallNode* call = e.as<CallNode>();
+ TVM_FFI_ICHECK(call != nullptr);
+ ffi::Array<Expr> new_args = {StringImm("rint")};
+ for (const PrimExpr& arg : call->args.as_or_throw<ffi::Array<PrimExpr>>()) {
+ new_args.push_back(arg);
+ }
+ return Call(e.ty(), tirx::builtin::call_pure_extern(), new_args).as_or_throw<PrimExpr>();
+ });
-TVM_REGISTER_OP("tirx.nearbyint")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.nearbyint")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.exp")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.exp")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.erf")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.erf")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.exp2")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.exp2")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.exp10")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.exp10")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.log")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.log")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("prim.log2")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("prim.log2")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.log10")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.log10")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.tanh")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.tanh")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.sqrt")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.sqrt")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.pow")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.pow")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.popcount")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.popcount")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.fmod")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.fmod")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.sin")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.sin")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.sinh")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.sinh")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.cos")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.cos")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.cosh")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.cosh")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle")
- .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchIntelShuffle);
+ OpDef("tirx.tvm_warp_shuffle")
+ .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchIntelShuffle);
// clang-format on
}
diff --git a/src/backend/rocm/codegen/llvm/intrin_rule_rocm.cc b/src/backend/rocm/codegen/llvm/intrin_rule_rocm.cc
index 8f7d80ef0a..67b606e5d3 100644
--- a/src/backend/rocm/codegen/llvm/intrin_rule_rocm.cc
+++ b/src/backend/rocm/codegen/llvm/intrin_rule_rocm.cc
@@ -117,110 +117,110 @@ namespace llvm {
using tirx::FLowerIntrinsic;
void RegisterROCMIntrinRules() {
+ // dummy because we don't have the activemask
// clang-format off
-// dummy because we don't have the activemask
-TVM_REGISTER_OP("tirx.tvm_warp_activemask")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
- PrimExpr zero = IntImm::Int32(0);
- return zero;
- });
+ OpDef("tirx.tvm_warp_activemask")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
+ PrimExpr zero = IntImm::Int32(0);
+ return zero;
+ });
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchShuffle);
+ OpDef("tirx.tvm_warp_shuffle")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchShuffle);
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_up")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchShuffle);
+ OpDef("tirx.tvm_warp_shuffle_up")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchShuffle);
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_down")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchShuffle);
+ OpDef("tirx.tvm_warp_shuffle_down")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchShuffle);
-TVM_REGISTER_OP("tirx.floor")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::floor, 1>);
+ OpDef("tirx.floor")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::floor, 1>);
-TVM_REGISTER_OP("prim.ceil")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::ceil, 1>);
+ OpDef("prim.ceil")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::ceil, 1>);
-TVM_REGISTER_OP("tirx.round")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
+ OpDef("tirx.round")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
-TVM_REGISTER_OP("tirx.nearbyint")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
+ OpDef("tirx.nearbyint")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
-TVM_REGISTER_OP("tirx.trunc")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::trunc, 1>);
+ OpDef("tirx.trunc")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::trunc, 1>);
-TVM_REGISTER_OP("tirx.fabs")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::fabs, 1>);
+ OpDef("tirx.fabs")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::fabs, 1>);
-TVM_REGISTER_OP("tirx.exp")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::exp, 1>);
+ OpDef("tirx.exp")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::exp, 1>);
-TVM_REGISTER_OP("tirx.exp2")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::exp2, 1>);
+ OpDef("tirx.exp2")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::exp2, 1>);
-TVM_REGISTER_OP("tirx.fma")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::fmuladd, 3>);
+ OpDef("tirx.fma")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::fmuladd, 3>);
-TVM_REGISTER_OP("tirx.log")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::log, 1>);
+ OpDef("tirx.log")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::log, 1>);
-TVM_REGISTER_OP("prim.log2")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::log2, 1>);
+ OpDef("prim.log2")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::log2, 1>);
-TVM_REGISTER_OP("tirx.log10")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::log10, 1>);
+ OpDef("tirx.log10")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::log10, 1>);
-TVM_REGISTER_OP("tirx.sqrt")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::sqrt, 1>);
+ OpDef("tirx.sqrt")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::sqrt, 1>);
-TVM_REGISTER_OP("tirx.pow")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::pow, 2>);
+ OpDef("tirx.pow")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::pow, 2>);
-TVM_REGISTER_OP("tirx.cos")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::cos, 1>);
+ OpDef("tirx.cos")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::cos, 1>);
-TVM_REGISTER_OP("tirx.sin")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- DispatchLLVMPureIntrin<::llvm::Intrinsic::sin, 1>);
+ OpDef("tirx.sin")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ DispatchLLVMPureIntrin<::llvm::Intrinsic::sin, 1>);
-TVM_REGISTER_OP("tirx.tanh")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
- ::tvm::codegen::intrin::DispatchNumericalStableTanh);
+ OpDef("tirx.tanh")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ ::tvm::codegen::intrin::DispatchNumericalStableTanh);
-TVM_REGISTER_OP("tirx.erf")
- .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", ::tvm::codegen::intrin::DispatchFastErf);
+ OpDef("tirx.erf")
+ .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", ::tvm::codegen::intrin::DispatchFastErf);
+ // clang-format on
-// TVM_REGISTER_OP("tirx.tan").set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-// DispatchPureExternOCML);
+ // OpDef("tirx.tan").set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ // DispatchPureExternOCML);
-// TVM_REGISTER_OP("tirx.cosh")
-// .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchPureExternOCML);
+ // OpDef("tirx.cosh")
+ // .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchPureExternOCML);
-// TVM_REGISTER_OP("tirx.sinh")
-// .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchPureExternOCML);
+ // OpDef("tirx.sinh")
+ // .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchPureExternOCML);
-// TVM_REGISTER_OP("tirx.atan")
-// .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchPureExternOCML);
+ // OpDef("tirx.atan")
+ // .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchPureExternOCML);
-// TVM_REGISTER_OP("tirx.exp10")
-// .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-// DispatchLLVMPureIntrin<::llvm::Intrinsic::exp10, 1>);
- // clang-format on
+ // OpDef("tirx.exp10")
+ // .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+ // DispatchLLVMPureIntrin<::llvm::Intrinsic::exp10, 1>);
}
} // namespace llvm
diff --git a/src/backend/trn/codegen/codegen_trn.cc b/src/backend/trn/codegen/codegen_trn.cc
index dccb4455a8..9889040c46 100644
--- a/src/backend/trn/codegen/codegen_trn.cc
+++ b/src/backend/trn/codegen/codegen_trn.cc
@@ -370,22 +370,22 @@ void CodeGenTrainium::Dispatch_(const CallNode* op, std::ostream& os) { // NOLI
auto is_op = [&](const Op& compat, const char* canonical_name) {
return op->op.same_as(compat) || (op_node != nullptr && op_node->name == canonical_name);
};
- static const Op& nki_matmul_op = Op::Get("tirx.nki.matmul");
- static const Op& nki_load_op = Op::Get("tirx.nki.load");
- static const Op& nki_store_op = Op::Get("tirx.nki.store");
- static const Op& nki_tensor_copy_op = Op::Get("tirx.nki.tensor_copy");
- static const Op& nki_activation_op = Op::Get("tirx.nki.activation");
- static const Op& nki_reciprocal_op = Op::Get("tirx.nki.reciprocal");
- static const Op& nki_tensortensor_op = Op::Get("tirx.nki.tensortensor");
- static const Op& nki_tensorscalar_op = Op::Get("tirx.nki.tensorscalar");
- static const Op& nki_memset_op = Op::Get("tirx.nki.memset");
- static const Op& nki_tensorreduce_op = Op::Get("tirx.nki.tensorreduce");
- static const Op& nki_activation_reduce_op = Op::Get("tirx.nki.activation_reduce");
- static const Op& nki_tensorscalar_reduce_op = Op::Get("tirx.nki.tensorscalar_reduce");
- static const Op& nki_identity_op = Op::Get("tirx.nki.identity");
- static const Op& nki_scalar_tensor_tensor_op = Op::Get("tirx.nki.scalar_tensor_tensor");
- static const Op& nki_scalar_tensor_scalar_op = Op::Get("tirx.nki.scalar_tensor_scalar");
- static const Op& nki_affine_select_op = Op::Get("tirx.nki.affine_select");
+ static const Op nki_matmul_op = Op::Get("tirx.nki.matmul");
+ static const Op nki_load_op = Op::Get("tirx.nki.load");
+ static const Op nki_store_op = Op::Get("tirx.nki.store");
+ static const Op nki_tensor_copy_op = Op::Get("tirx.nki.tensor_copy");
+ static const Op nki_activation_op = Op::Get("tirx.nki.activation");
+ static const Op nki_reciprocal_op = Op::Get("tirx.nki.reciprocal");
+ static const Op nki_tensortensor_op = Op::Get("tirx.nki.tensortensor");
+ static const Op nki_tensorscalar_op = Op::Get("tirx.nki.tensorscalar");
+ static const Op nki_memset_op = Op::Get("tirx.nki.memset");
+ static const Op nki_tensorreduce_op = Op::Get("tirx.nki.tensorreduce");
+ static const Op nki_activation_reduce_op = Op::Get("tirx.nki.activation_reduce");
+ static const Op nki_tensorscalar_reduce_op = Op::Get("tirx.nki.tensorscalar_reduce");
+ static const Op nki_identity_op = Op::Get("tirx.nki.identity");
+ static const Op nki_scalar_tensor_tensor_op = Op::Get("tirx.nki.scalar_tensor_tensor");
+ static const Op nki_scalar_tensor_scalar_op = Op::Get("tirx.nki.scalar_tensor_scalar");
+ static const Op nki_affine_select_op = Op::Get("tirx.nki.affine_select");
if (is_op(nki_matmul_op, "tirx.nki.matmul")) {
TVM_FFI_ICHECK_EQ(op->args.size(), 4);
diff --git a/src/backend/trn/op/target_builtin.cc b/src/backend/trn/op/target_builtin.cc
index 2341a9683d..64d06b00e0 100644
--- a/src/backend/trn/op/target_builtin.cc
+++ b/src/backend/trn/op/target_builtin.cc
@@ -30,6 +30,7 @@
#include <tvm/tirx/op.h>
#include <tvm/tirx/op_attr_types.h>
+#include <initializer_list>
#include <string>
namespace tvm {
@@ -41,20 +42,24 @@ void RegisterNKIIntrinsicAliases();
}
void RegisterTRNTargetBuiltins() {
- // clang-format off
-static bool registered = false;
-if (registered) return;
-registered = true;
+ static bool registered = false;
+ if (registered) return;
+ registered = true;
-RegisterNKIIntrinsicAliases();
- // clang-format on
+ RegisterNKIIntrinsicAliases();
}
namespace {
-void RegisterNKIIntrinsic(const char* name) {
+struct NKIIntrinsicRegistration {
+ const char* name;
+ std::initializer_list<const char*> args;
+ bool allow_extra_args;
+};
+
+void RegisterNKIIntrinsic(const NKIIntrinsicRegistration& reg) {
std::string prefix = "nki_";
- std::string suffix(name);
+ std::string suffix(reg.name);
if (suffix.rfind(prefix, 0) == 0) {
suffix = suffix.substr(prefix.size());
}
@@ -64,42 +69,50 @@ void RegisterNKIIntrinsic(const char* name) {
ffi::String printer_name("nki." + suffix);
int64_t effect = static_cast<int64_t>(CallEffectKind::kOpaque);
- auto register_one = [&](const std::string& op_name) {
- OpRegEntry::RegisterOrGet(op_name)
- .set_name()
- .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"),
- /*plevel=*/15)
- .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", namespace_attr,
- /*plevel=*/15)
- .set_attr<TCallEffectKind>("TCallEffectKind", effect, /*plevel=*/15)
- .set_attr<TScriptPrinterName>("TScriptPrinterName", printer_name, /*plevel=*/15);
- };
-
- register_one(canonical_op_name);
+ OpDef def(canonical_op_name);
+ for (const char* name : reg.args) {
+ def.add_arg(name, "");
+ }
+ if (reg.allow_extra_args) {
+ def.allow_extra_args();
+ }
+ def.set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", namespace_attr)
+ .set_attr<TCallEffectKind>("TCallEffectKind", effect)
+ .set_attr<TScriptPrinterName>("TScriptPrinterName", printer_name);
}
-const char* kNKIIntrinsics[] = {
- "nki_activation",
- "nki_activation_reduce",
- "nki_affine_select",
- "nki_identity",
- "nki_load",
- "nki_matmul",
- "nki_memset",
- "nki_reciprocal",
- "nki_scalar_tensor_scalar",
- "nki_scalar_tensor_tensor",
- "nki_store",
- "nki_tensor_copy",
- "nki_tensorreduce",
- "nki_tensorscalar",
- "nki_tensorscalar_reduce",
- "nki_tensortensor",
+const NKIIntrinsicRegistration kNKIIntrinsics[] = {
+ {"nki_activation", {"result", "data", "opcode", "bias", "scale"}, false},
+ {"nki_activation_reduce",
+ {"reduce_res", "act_res", "data", "opcode", "reduce_opcode", "bias", "scale"},
+ false},
+ {"nki_affine_select", {"result", "pred", "true_value", "false_value"}, false},
+ {"nki_identity", {"result", "size"}, false},
+ {"nki_load", {"res", "data"}, false},
+ {"nki_matmul", {"res", "lhs", "rhs", "accum"}, false},
+ {"nki_memset", {"result", "value"}, false},
+ {"nki_reciprocal", {"result", "data"}, false},
+ {"nki_scalar_tensor_scalar",
+ {"result", "data", "operand0", "operand1", "opcode0", "opcode1", "reverse0", "reverse1"},
+ false},
+ {"nki_scalar_tensor_tensor",
+ {"result", "data", "operand0", "operand1", "opcode0", "opcode1", "reverse0", "reverse1"},
+ false},
+ {"nki_store", {"res", "data"}, false},
+ {"nki_tensor_copy", {"res", "data"}, false},
+ {"nki_tensorreduce", {"result", "data", "opcode", "negate"}, true},
+ {"nki_tensorscalar", {"result", "operand0", "operand1", "opcode", "reverse"}, false},
+ {"nki_tensorscalar_reduce",
+ {"reduce_res", "tensorscalar_res", "operand0", "operand1", "opcode", "reduce_opcode",
+ "reverse"},
+ false},
+ {"nki_tensortensor", {"result", "operand0", "operand1", "opcode"}, false},
};
void RegisterNKIIntrinsicAliases() {
- for (const char* op_name : kNKIIntrinsics) {
- RegisterNKIIntrinsic(op_name);
+ for (const auto& reg : kNKIIntrinsics) {
+ RegisterNKIIntrinsic(reg);
}
}
diff --git a/src/backend/vulkan/codegen/codegen_spirv.cc b/src/backend/vulkan/codegen/codegen_spirv.cc
index 0c648ac929..99e9a5899a 100644
--- a/src/backend/vulkan/codegen/codegen_spirv.cc
+++ b/src/backend/vulkan/codegen/codegen_spirv.cc
@@ -467,10 +467,10 @@ spirv::Value CodeGenSPIRV::Dispatch_(const CallNode* op) {
return spirv::Value();
}
- static const Op& tvm_fill_fragment_op = Op::Get("tirx.tvm_fill_fragment");
- static const Op& tvm_load_matrix_sync_op = Op::Get("tirx.tvm_load_matrix_sync");
- static const Op& tvm_mma_sync_op = Op::Get("tirx.tvm_mma_sync");
- static const Op& tvm_store_matrix_sync_op = Op::Get("tirx.tvm_store_matrix_sync");
+ static const Op tvm_fill_fragment_op = Op::Get("tirx.tvm_fill_fragment");
+ static const Op tvm_load_matrix_sync_op = Op::Get("tirx.tvm_load_matrix_sync");
+ static const Op tvm_mma_sync_op = Op::Get("tirx.tvm_mma_sync");
+ static const Op tvm_store_matrix_sync_op = Op::Get("tirx.tvm_store_matrix_sync");
if (op->op.same_as(tvm_fill_fragment_op)) {
TVM_FFI_ICHECK_EQ(op->args.size(), 6U);
diff --git a/src/backend/vulkan/codegen/intrin_rule_spirv.cc b/src/backend/vulkan/codegen/intrin_rule_spirv.cc
index bca215bbdb..6086a1d88c 100644
--- a/src/backend/vulkan/codegen/intrin_rule_spirv.cc
+++ b/src/backend/vulkan/codegen/intrin_rule_spirv.cc
@@ -70,87 +70,85 @@ void RegisterVulkanLowerIntrinRules() {
if (registered) return;
registered = true;
- // clang-format off
-TVM_REGISTER_OP("tirx.floor")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Floor>);
+ OpDef("tirx.floor")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Floor>);
-TVM_REGISTER_OP("prim.ceil")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Ceil>);
+ OpDef("prim.ceil")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Ceil>);
-TVM_REGISTER_OP("tirx.round")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic",
- DispatchGLSLPureIntrin<GLSLstd450RoundEven>);
+ OpDef("tirx.round")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic",
+ DispatchGLSLPureIntrin<GLSLstd450RoundEven>);
-TVM_REGISTER_OP("tirx.nearbyint")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic",
- DispatchGLSLPureIntrin<GLSLstd450RoundEven>);
+ OpDef("tirx.nearbyint")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic",
+ DispatchGLSLPureIntrin<GLSLstd450RoundEven>);
-TVM_REGISTER_OP("tirx.trunc")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Trunc>);
+ OpDef("tirx.trunc")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Trunc>);
-TVM_REGISTER_OP("tirx.fabs")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450FAbs>);
+ OpDef("tirx.fabs")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450FAbs>);
-TVM_REGISTER_OP("tirx.exp")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Exp>);
+ OpDef("tirx.exp")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Exp>);
-TVM_REGISTER_OP("tirx.exp2")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Exp2>);
+ OpDef("tirx.exp2")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Exp2>);
-TVM_REGISTER_OP("tirx.sin")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Sin>);
+ OpDef("tirx.sin")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Sin>);
-TVM_REGISTER_OP("tirx.cos")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Cos>);
+ OpDef("tirx.cos")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Cos>);
-TVM_REGISTER_OP("tirx.tan")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Tan>);
+ OpDef("tirx.tan")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Tan>);
-TVM_REGISTER_OP("tirx.asin")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Asin>);
+ OpDef("tirx.asin")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Asin>);
-TVM_REGISTER_OP("tirx.acos")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Acos>);
+ OpDef("tirx.acos")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Acos>);
-TVM_REGISTER_OP("tirx.atan")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Atan>);
+ OpDef("tirx.atan")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Atan>);
-TVM_REGISTER_OP("tirx.sinh")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Sinh>);
+ OpDef("tirx.sinh")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Sinh>);
-TVM_REGISTER_OP("tirx.cosh")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Cosh>);
+ OpDef("tirx.cosh")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Cosh>);
-TVM_REGISTER_OP("tirx.tanh")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Tanh>);
+ OpDef("tirx.tanh")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Tanh>);
-TVM_REGISTER_OP("tirx.asinh")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Asinh>);
+ OpDef("tirx.asinh")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Asinh>);
-TVM_REGISTER_OP("tirx.acosh")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Acosh>);
+ OpDef("tirx.acosh")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Acosh>);
-TVM_REGISTER_OP("tirx.atanh")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Atanh>);
+ OpDef("tirx.atanh")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Atanh>);
-TVM_REGISTER_OP("tirx.atan2")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Atan2>);
+ OpDef("tirx.atan2")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Atan2>);
-TVM_REGISTER_OP("tirx.log")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Log>);
+ OpDef("tirx.log")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Log>);
-TVM_REGISTER_OP("prim.log2")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Log2>);
+ OpDef("prim.log2")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Log2>);
-TVM_REGISTER_OP("tirx.sqrt")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Sqrt>);
+ OpDef("tirx.sqrt")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Sqrt>);
-TVM_REGISTER_OP("tirx.pow")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Pow>);
+ OpDef("tirx.pow")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Pow>);
-TVM_REGISTER_OP("tirx.erf")
- .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", codegen::intrin ::DispatchFastErf);
- // clang-format on
+ OpDef("tirx.erf")
+ .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", codegen::intrin ::DispatchFastErf);
}
} // namespace intrin
@@ -163,29 +161,29 @@ void RegisterVulkanLegalizeRules() {
registered = true;
// clang-format off
-TVM_REGISTER_OP("prim.clz")
- .set_attr<FLegalize>("vulkan.FLegalize", [](const PrimExpr& e) -> PrimExpr {
- const CallNode* call = e.as<CallNode>();
- TVM_FFI_ICHECK(call != nullptr);
- TVM_FFI_ICHECK_EQ(call->args.size(), 1);
- PrimExpr arg = call->args[0].as_or_throw<PrimExpr>();
- PrimType arg_ty = arg.ty();
- PrimExpr msb;
- if (arg_ty.bits() == 64) {
- // SPIR-V FindUMsb intrinsic only supports 32 bit input
- auto int32 = PrimType::Int(32);
- PrimExpr arg_hi32 = tvm::prim::Cast(int32, arg >> 32);
- PrimExpr arg_lo32 = tvm::prim::Cast(int32, arg);
- PrimExpr msb_hi = CallGLSLIntrin<GLSLstd450FindUMsb>(e, {arg_hi32});
- PrimExpr msb_lo = CallGLSLIntrin<GLSLstd450FindUMsb>(e, {arg_lo32});
- msb = tvm::if_then_else(arg_hi32 == 0, msb_lo, msb_hi + 32);
- } else if (arg_ty.bits() == 32) {
- msb = CallGLSLIntrin<GLSLstd450FindUMsb>(e);
- } else {
- TVM_FFI_THROW(InternalError) << "SPIR-V clz only supports a 32 bit or 64 bit integer.";
- }
- return PrimExpr(arg_ty.bits() - 1) - msb;
- });
+ OpDef("prim.clz")
+ .set_attr<FLegalize>("vulkan.FLegalize", [](const PrimExpr& e) -> PrimExpr {
+ const CallNode* call = e.as<CallNode>();
+ TVM_FFI_ICHECK(call != nullptr);
+ TVM_FFI_ICHECK_EQ(call->args.size(), 1);
+ PrimExpr arg = call->args[0].as_or_throw<PrimExpr>();
+ PrimType arg_ty = arg.ty();
+ PrimExpr msb;
+ if (arg_ty.bits() == 64) {
+ // SPIR-V FindUMsb intrinsic only supports 32 bit input
+ auto int32 = PrimType::Int(32);
+ PrimExpr arg_hi32 = tvm::prim::Cast(int32, arg >> 32);
+ PrimExpr arg_lo32 = tvm::prim::Cast(int32, arg);
+ PrimExpr msb_hi = CallGLSLIntrin<GLSLstd450FindUMsb>(e, {arg_hi32});
+ PrimExpr msb_lo = CallGLSLIntrin<GLSLstd450FindUMsb>(e, {arg_lo32});
+ msb = tvm::if_then_else(arg_hi32 == 0, msb_lo, msb_hi + 32);
+ } else if (arg_ty.bits() == 32) {
+ msb = CallGLSLIntrin<GLSLstd450FindUMsb>(e);
+ } else {
+ TVM_FFI_THROW(InternalError) << "SPIR-V clz only supports a 32 bit or 64 bit integer.";
+ }
+ return PrimExpr(arg_ty.bits() - 1) - msb;
+ });
// clang-format on
}
} // namespace legalize
diff --git a/src/backend/webgpu/codegen/intrin_rule_webgpu.cc b/src/backend/webgpu/codegen/intrin_rule_webgpu.cc
index 75ce8c6367..d030a78417 100644
--- a/src/backend/webgpu/codegen/intrin_rule_webgpu.cc
+++ b/src/backend/webgpu/codegen/intrin_rule_webgpu.cc
@@ -36,14 +36,14 @@ using tirx::FLowerIntrinsic;
struct WebGPUWarpIntrinsic {
const Op operator()(PrimType t, const Op& orig_op) const {
if (orig_op.same_as(builtin::tvm_warp_shuffle())) {
- static const Op& webgpu_subgroup_shuffle_op = Op::Get("tirx.webgpu.subgroup_shuffle");
+ static const Op webgpu_subgroup_shuffle_op = Op::Get("tirx.webgpu.subgroup_shuffle");
return webgpu_subgroup_shuffle_op;
} else if (orig_op.same_as(builtin::tvm_warp_shuffle_up())) {
- static const Op& webgpu_subgroup_shuffle_up_op = Op::Get("tirx.webgpu.subgroup_shuffle_up");
+ static const Op webgpu_subgroup_shuffle_up_op = Op::Get("tirx.webgpu.subgroup_shuffle_up");
return webgpu_subgroup_shuffle_up_op;
} else {
TVM_FFI_ICHECK(orig_op.same_as(builtin::tvm_warp_shuffle_down()));
- static const Op& webgpu_subgroup_shuffle_down_op =
+ static const Op webgpu_subgroup_shuffle_down_op =
Op::Get("tirx.webgpu.subgroup_shuffle_down");
return webgpu_subgroup_shuffle_down_op;
}
@@ -66,145 +66,142 @@ void RegisterWebGPUIntrinRules() {
if (registered) return;
registered = true;
- // clang-format off
-// See full list of builtin: https://www.w3.org/TR/WGSL/#builtin-functions
-
-struct ReturnAbs {
- std::string operator()(PrimType t, std::string name) const { return "abs"; }
-};
-
-TVM_REGISTER_OP("tirx.fabs")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<ReturnAbs>);
-
-TVM_REGISTER_OP("tirx.acos")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.acosh")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.asin")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.asinh")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ // See full list of builtin: https://www.w3.org/TR/WGSL/#builtin-functions
-TVM_REGISTER_OP("tirx.atan")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ struct ReturnAbs {
+ std::string operator()(PrimType t, std::string name) const { return "abs"; }
+ };
-TVM_REGISTER_OP("tirx.atan2")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("prim.ceil")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.cos")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.cosh")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ // clang-format off
+ OpDef("tirx.fabs")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<ReturnAbs>);
-TVM_REGISTER_OP("tirx.exp")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.acos")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.exp2")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.acosh")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.floor")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.asin")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.fma")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.asinh")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.log")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.atan")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("prim.log2")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.atan2")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.pow")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("prim.ceil")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-struct ReturnRound {
- std::string operator()(PrimType t, std::string name) const { return "round"; }
-};
+ OpDef("tirx.cos")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-// WGSL round() uses ties-to-even (banker's rounding), matching IEEE 754 and ONNX Round spec.
-TVM_REGISTER_OP("tirx.nearbyint")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<ReturnRound>);
+ OpDef("tirx.cosh")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.round")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.exp")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.sin")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.exp2")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.sinh")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.floor")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.sqrt")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.fma")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.tan")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.log")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.tanh")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchNumericalStableTanh);
+ OpDef("prim.log2")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-TVM_REGISTER_OP("tirx.trunc")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ OpDef("tirx.pow")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+ // clang-format on
-// extra dispatch
-TVM_REGISTER_OP("tirx.erf").set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchFastErf);
+ struct ReturnRound {
+ std::string operator()(PrimType t, std::string name) const { return "round"; }
+ };
-// warp-level primitives. Follows implementation in intrin_rule_metal.cc
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic",
- DispatchWebGPUShuffle<WebGPUWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_up")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic",
- DispatchWebGPUShuffle<WebGPUWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_down")
- .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic",
- DispatchWebGPUShuffle<WebGPUWarpIntrinsic>);
-
-// Register low-level WebGPU device intrinsics.
-TVM_REGISTER_OP("tirx.webgpu.subgroup_shuffle")
- .set_num_inputs(2)
- .add_argument("var", "Expr", "The variable to sync.")
- .add_argument("lane", "Expr", "The source thread id.")
- .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
- .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("webgpu"),
- 10)
- .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
- ffi::String("webgpu.subgroup_shuffle"), 10)
- .set_attr<TGlobalSymbol>("TGlobalSymbol", "subgroupShuffle")
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TVM_REGISTER_OP("tirx.webgpu.subgroup_shuffle_up")
- .set_num_inputs(2)
- .add_argument("var", "Expr", "The variable to sync.")
- .add_argument("delta", "Expr", "The source lane id offset to be added.")
- .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
- .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("webgpu"),
- 10)
- .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
- ffi::String("webgpu.subgroup_shuffle_up"), 10)
- .set_attr<TGlobalSymbol>("TGlobalSymbol", "subgroupShuffleUp")
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TVM_REGISTER_OP("tirx.webgpu.subgroup_shuffle_down")
- .set_num_inputs(2)
- .add_argument("var", "Expr", "The variable to sync.")
- .add_argument("delta", "Expr", "The source lane id offset to be subtracted.")
- .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
- .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("webgpu"),
- 10)
- .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
- ffi::String("webgpu.subgroup_shuffle_down"), 10)
- .set_attr<TGlobalSymbol>("TGlobalSymbol", "subgroupShuffleDown")
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+ // WGSL round() uses ties-to-even (banker's rounding), matching IEEE 754 and ONNX Round spec.
+ // clang-format off
+ OpDef("tirx.nearbyint")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<ReturnRound>);
+
+ OpDef("tirx.round")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.sin")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.sinh")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.sqrt")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.tan")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ OpDef("tirx.tanh")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchNumericalStableTanh);
+
+ OpDef("tirx.trunc")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+ // extra dispatch
+ OpDef("tirx.erf")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchFastErf);
+
+ // warp-level primitives. Follows implementation in intrin_rule_metal.cc
+ OpDef("tirx.tvm_warp_shuffle")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic",
+ DispatchWebGPUShuffle<WebGPUWarpIntrinsic>);
+
+ OpDef("tirx.tvm_warp_shuffle_up")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic",
+ DispatchWebGPUShuffle<WebGPUWarpIntrinsic>);
+
+ OpDef("tirx.tvm_warp_shuffle_down")
+ .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic",
+ DispatchWebGPUShuffle<WebGPUWarpIntrinsic>);
+
+ // Register low-level WebGPU device intrinsics.
+ OpDef("tirx.webgpu.subgroup_shuffle")
+ .add_arg("var", "The variable to sync.")
+ .add_arg("lane", "The source thread id.")
+ .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("webgpu"))
+ .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
+ ffi::String("webgpu.subgroup_shuffle"))
+ .set_attr<TGlobalSymbol>("TGlobalSymbol", "subgroupShuffle")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.webgpu.subgroup_shuffle_up")
+ .add_arg("var", "The variable to sync.")
+ .add_arg("delta", "The source lane id offset to be added.")
+ .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("webgpu"))
+ .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
+ ffi::String("webgpu.subgroup_shuffle_up"))
+ .set_attr<TGlobalSymbol>("TGlobalSymbol", "subgroupShuffleUp")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+ OpDef("tirx.webgpu.subgroup_shuffle_down")
+ .add_arg("var", "The variable to sync.")
+ .add_arg("delta", "The source lane id offset to be subtracted.")
+ .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+ .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("webgpu"))
+ .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
+ ffi::String("webgpu.subgroup_shuffle_down"))
+ .set_attr<TGlobalSymbol>("TGlobalSymbol", "subgroupShuffleDown")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
// clang-format on
}
diff --git a/src/ir/attr_registry.h b/src/ir/attr_registry.h
deleted file mode 100644
index de80e5054e..0000000000
--- a/src/ir/attr_registry.h
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/*!
- * \file tvm/ir/attr_registry.h
- * \brief Common global registry for objects that also have additional attrs.
- */
-#ifndef TVM_IR_ATTR_REGISTRY_H_
-#define TVM_IR_ATTR_REGISTRY_H_
-
-#include <tvm/ffi/function.h>
-#include <tvm/ir/attr_registry_map.h>
-
-#include <memory>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-
-namespace tvm {
-
-/*!
- * \brief Implementation of registry with attributes.
- *
- * \tparam EntryType The type of the registry entry.
- * \tparam KeyType The actual key that is used to lookup the attributes.
- * each entry has a corresponding key by default.
- */
-template <typename EntryType, typename KeyType>
-class AttrRegistry {
- public:
- using TSelf = AttrRegistry<EntryType, KeyType>;
- /*!
- * \brief Get an entry from the registry.
- * \param name The name of the item.
- * \return The corresponding entry.
- */
- const EntryType* Get(const ffi::String& name) const {
- auto it = entry_map_.find(name);
- if (it != entry_map_.end()) return it->second;
- return nullptr;
- }
-
- /*!
- * \brief Get an entry or register a new one.
- * \param name The name of the item.
- * \return The corresponding entry.
- */
- EntryType& RegisterOrGet(const ffi::String& name) {
- auto it = entry_map_.find(name);
- if (it != entry_map_.end()) return *it->second;
- uint32_t registry_index = static_cast<uint32_t>(entries_.size());
- auto entry = std::unique_ptr<EntryType>(new EntryType(registry_index));
- auto* eptr = entry.get();
- eptr->name = name;
- entry_map_[name] = eptr;
- entries_.emplace_back(std::move(entry));
- return *eptr;
- }
-
- /*!
- * \brief List all the entry names in the registry.
- * \return The entry names.
- */
- ffi::Array<ffi::String> ListAllNames() const {
- ffi::Array<ffi::String> names;
- for (const auto& kv : entry_map_) {
- names.push_back(kv.first);
- }
- return names;
- }
-
- /*!
- * \brief Update the attribute stable.
- * \param attr_name The name of the attribute.
- * \param key The key to the attribute table.
- * \param value The value to be set.
- * \param plevel The support level.
- */
- void UpdateAttr(const ffi::String& attr_name, const KeyType& key, Any value, int plevel) {
- using ffi::Any;
- auto& op_map = attrs_[attr_name];
- if (op_map == nullptr) {
- op_map.reset(new AttrRegistryMapContainerMap<KeyType>());
- op_map->attr_name_ = attr_name;
- }
-
- uint32_t index = key->AttrRegistryIndex();
- if (op_map->data_.size() <= index) {
- op_map->data_.resize(index + 1, std::make_pair(Any(), 0));
- }
- std::pair<ffi::Any, int>& p = op_map->data_[index];
- TVM_FFI_ICHECK(p.second != plevel)
- << "Attribute " << attr_name << " of " << key->AttrRegistryName()
- << " is already registered with same plevel=" << plevel;
- TVM_FFI_ICHECK(value != nullptr) << "Registered packed_func is Null for " << attr_name
- << " of operator " << key->AttrRegistryName();
- if (p.second < plevel && value != nullptr) {
- op_map->data_[index] = std::make_pair(value, plevel);
- }
- }
-
- /*!
- * \brief Reset an attribute table entry.
- * \param attr_name The name of the attribute.
- * \param key The key to the attribute table.
- */
- void ResetAttr(const ffi::String& attr_name, const KeyType& key) {
- auto& op_map = attrs_[attr_name];
- if (op_map == nullptr) {
- return;
- }
- uint32_t index = key->AttrRegistryIndex();
- if (op_map->data_.size() > index) {
- op_map->data_[index] = std::make_pair(ffi::Any(), 0);
- }
- }
-
- /*!
- * \brief Get an internal attribute map.
- * \param attr_name The name of the attribute.
- * \return The result attribute map.
- */
- const AttrRegistryMapContainerMap<KeyType>& GetAttrMap(const ffi::String& attr_name) {
- auto it = attrs_.find(attr_name);
- if (it == attrs_.end()) {
- TVM_FFI_THROW(InternalError) << "Attribute \'" << attr_name << "\' is not registered";
- }
- return *it->second.get();
- }
-
- /*!
- * \brief Check of attribute has been registered.
- * \param attr_name The name of the attribute.
- * \return The check result.
- */
- bool HasAttrMap(const ffi::String& attr_name) { return attrs_.count(attr_name); }
-
- /*!
- * \return a global singleton of the registry.
- */
- static TSelf* Global() {
- static TSelf* inst = new TSelf();
- return inst;
- }
-
- private:
- // entries in the registry
- std::vector<std::unique_ptr<EntryType>> entries_;
- // map from name to entries.
- std::unordered_map<ffi::String, EntryType*> entry_map_;
- // storage of additional attribute table.
- std::unordered_map<ffi::String, std::unique_ptr<AttrRegistryMapContainerMap<KeyType>>> attrs_;
-};
-
-} // namespace tvm
-#endif // TVM_IR_ATTR_REGISTRY_H_
diff --git a/src/ir/op.cc b/src/ir/op.cc
index 8f7897e502..7f63eb7ca2 100644
--- a/src/ir/op.cc
+++ b/src/ir/op.cc
@@ -21,169 +21,241 @@
* \file src/ir/op.cc
* \brief Primitive operators and intrinsics.
*/
+#include <tvm/ffi/container/dict.h>
#include <tvm/ffi/extra/structural_mutate.h>
#include <tvm/ffi/extra/structural_visit.h>
-#include <tvm/ffi/function.h>
#include <tvm/ffi/reflection/registry.h>
#include <tvm/ir/op.h>
-#include <tvm/ir/type.h>
-#include <memory>
+namespace tvm {
-#include "attr_registry.h"
+// Owns canonical Ops and mutable attribute columns for the lifetime of the process.
+class OpRegistry {
+ public:
+ static OpRegistry* Global() {
+ static auto* registry = new OpRegistry();
+ return registry;
+ }
-namespace tvm {
+ Op Get(const ffi::String& name) const {
+ auto op = ops_.Get(name);
+ TVM_FFI_CHECK(op.has_value(), AttributeError) << "Operator " << name << " is not registered";
+ return *op;
+ }
-namespace {
+ Op GetOrCreate(const ffi::String& name) {
+ if (auto op = ops_.Get(name)) return *op;
+ auto node = ffi::make_object<OpNode>();
+ node->name = name;
+ node->index_ = static_cast<uint32_t>(ops_.size());
+ Op op(std::move(node));
+ ops_.Set(name, op);
+ return op;
+ }
-TVMFFIAny OpVisit(ffi::StructuralVisitorObj*, ffi::AnyView) noexcept {
- // Ops are unique registry atoms. Avoid reflecting through their registry metadata.
- return ffi::AnyView(nullptr).CopyToTVMFFIAny();
-}
+ bool Contains(const ffi::String& name) const { return ops_.count(name); }
-TVMFFIAny OpMutate(ffi::StructuralMutatorObj*, ffi::AnyView) noexcept {
- return ffi::Unchanged().CopyToTVMFFIAny();
-}
+ ffi::Array<ffi::String> ListNames() const {
+ ffi::Array<ffi::String> names;
+ for (const auto& [name, op] : ops_) names.push_back(name);
+ return names;
+ }
-TVMFFIAny OpMaybeInplaceMutate(ffi::StructuralMutatorObj*, ffi::AnyView) noexcept {
- return ffi::Unchanged().CopyToTVMFFIAny();
-}
+ bool HasAttrMap(const ffi::String& name) const { return attrs_.count(name); }
-} // namespace
+ ffi::List<ffi::Any> GetAttrColumn(const ffi::String& name) const {
+ auto column = attrs_.Get(name);
+ TVM_FFI_CHECK(column.has_value(), InternalError)
+ << "Attribute '" << name << "' is not registered";
+ return *column;
+ }
-TVM_FFI_STATIC_INIT_BLOCK() { ArgumentInfoNode::RegisterReflection(); }
+ void SetAttr(const Op& op, const ffi::String& name, ffi::Any value, bool override) {
+ TVM_FFI_CHECK(value != nullptr, ValueError)
+ << "Registered attribute is null for " << name << " of operator " << op->name;
+ auto column = attrs_.Get(name).value_or(ffi::List<ffi::Any>());
+ uint32_t index = op->index_;
+ TVM_FFI_CHECK(override || index >= column.size() || column[index] == nullptr, ValueError)
+ << "Attribute " << name << " of " << op->name << " is already registered";
+ if (index >= column.size()) column.resize(index + 1);
+ column.Set(index, std::move(value));
+ // Store the same List object: cached views must survive both updates and growth.
+ attrs_.Set(name, column);
+ }
-using ffi::Any;
-using ffi::Function;
-using ffi::PackedArgs;
+ void ResetAttr(const Op& op, const ffi::String& name) {
+ if (auto column = attrs_.Get(name)) {
+ if (op->index_ < column->size()) column->Set(op->index_, ffi::Any());
+ }
+ }
-using OpRegistry = AttrRegistry<OpRegEntry, Op>;
+ private:
+ ffi::Dict<ffi::String, Op> ops_;
+ ffi::Dict<ffi::String, ffi::List<ffi::Any>> attrs_;
+};
-// find operator by name
-const Op& Op::Get(const ffi::String& name) {
- const OpRegEntry* reg = OpRegistry::Global()->Get(name);
- TVM_FFI_CHECK(reg != nullptr, AttributeError) << "Operator " << name << " is not registered";
- return reg->op();
+Op Op::Get(const ffi::String& name) { return OpRegistry::Global()->Get(name); }
+
+ffi::Array<ffi::String> Op::ListNames() { return OpRegistry::Global()->ListNames(); }
+
+bool Op::HasAttrMap(const ffi::String& name) { return OpRegistry::Global()->HasAttrMap(name); }
+
+ffi::List<ffi::Any> Op::GetAttrColumn(const ffi::String& name) {
+ return OpRegistry::Global()->GetAttrColumn(name);
}
-Op OpRegEntry::MakeOp(uint32_t reg_index) {
- ffi::ObjectPtr<OpNode> n = ffi::make_object<OpNode>();
- n->index_ = reg_index;
- return Op(n);
+OpDef::OpDef(const ffi::String& name) : op_(OpRegistry::Global()->GetOrCreate(name)) {}
+
+OpDef::OpDef(const ffi::String& name, const ffi::String& doc) : OpDef(name) { get()->doc = doc; }
+
+OpDef& OpDef::add_arg(const ffi::String& name, const ffi::String& doc) {
+ auto node = ffi::make_object<ArgumentInfoNode>();
+ node->name = name;
+ node->doc = doc;
+ get()->args_info.push_back(ArgumentInfo(std::move(node)));
+ return *this;
}
-TVM_FFI_STATIC_INIT_BLOCK() {
- namespace refl = tvm::ffi::reflection;
- OpNode::RegisterReflection();
- refl::TypeAttrDef<OpNode>()
- .attr(refl::type_attr::kStructuralVisit, reinterpret_cast<void*>(&OpVisit))
- .attr(refl::type_attr::kStructuralMutate, reinterpret_cast<void*>(&OpMutate))
- .attr(refl::type_attr::kStructuralMaybeInplaceMutate,
- reinterpret_cast<void*>(&OpMaybeInplaceMutate));
- refl::TypeAttrDef<OpNode>()
- .def("__data_to_json__",
- [](const OpNode* node) -> ffi::String {
- // simply save as the string
- return node->name;
- })
- .def("__data_from_json__", [](const ffi::String& name) -> Op { return Op::Get(name); });
+OpDef& OpDef::add_ty_arg(const ffi::String& name, const ffi::String& doc) {
+ auto node = ffi::make_object<ArgumentInfoNode>();
+ node->name = name;
+ node->doc = doc;
+ get()->ty_args_info.push_back(ArgumentInfo(std::move(node)));
+ return *this;
}
-OpRegEntry::OpRegEntry(uint32_t reg_index) : op_(MakeOp(reg_index)) {}
+OpDef& OpDef::set_attrs_type_key(const ffi::String& key) {
+ uint32_t index = ffi::TypeKeyToIndex(key.c_str());
+ get()->attrs_type_key = key;
+ get()->attrs_type_index = index;
+ return *this;
+}
+
+void OpDef::UpdateAttr(const ffi::String& name, ffi::Any value, bool override) {
+ OpRegistry::Global()->SetAttr(op_, name, std::move(value), override);
+}
-OpRegEntry& OpRegEntry::RegisterOrGet(const ffi::String& name) {
- return OpRegistry::Global()->RegisterOrGet(name);
+OpDef& OpDef::reset_attr(const ffi::String& name) {
+ OpRegistry::Global()->ResetAttr(op_, name);
+ return *this;
}
-// Get attribute map by key
-const AttrRegistryMapContainerMap<Op>& Op::GetAttrMapContainer(const ffi::String& attr_name) {
- return OpRegistry::Global()->GetAttrMap(attr_name);
+void OpNode::RegisterReflection() {
+ namespace refl = ffi::reflection;
+ // clang-format off
+ refl::ObjectDef<OpNode>()
+ .def_ro("index", &OpNode::index_, refl::AttachFieldFlag::SEqHashIgnore())
+ .def_ro("name", &OpNode::name)
+ .def_ro("doc", &OpNode::doc, refl::AttachFieldFlag::SEqHashIgnore())
+ .def_ro("args_info", &OpNode::args_info, refl::AttachFieldFlag::SEqHashIgnore())
+ .def_ro("attrs_type_key", &OpNode::attrs_type_key, refl::AttachFieldFlag::SEqHashIgnore())
+ .def_ro("allow_extra_args", &OpNode::allow_extra_args, refl::AttachFieldFlag::SEqHashIgnore())
+ .def_ro("ty_args_info", &OpNode::ty_args_info, refl::AttachFieldFlag::SEqHashIgnore())
+ .def_static("get", &Op::Get,
+ "get(op_name: str) -> Op\n\nReturn the canonical named Op; raise AttributeError "
+ "if unregistered.")
+ .def_static(
+ "list_op_names", &Op::ListNames,
+ "list_op_names() -> list[str]\n\nReturn registered operator names in unspecified order.")
+ .def(
+ "get_attr",
+ [](Op op, ffi::String name) {
+ return Op::GetAttrMap<ffi::Any>(name).get(op, ffi::Any());
+ },
+ "get_attr(attr_name: str) -> object\n\nReturn this Op's attribute or None; an "
+ "unregistered column raises InternalError.")
+ .def(
+ "has_attr", [](Op, ffi::String name) { return Op::HasAttrMap(name); },
+ "has_attr(attr_name: str) -> bool\n\nReturn whether the attribute column exists in the "
+ "registry.")
+ .def("_set_attr", [](Op op, ffi::String name, ffi::Any value, bool override) {
+ OpDef(op->name)
+ .set_attr(name, value, override);
+ })
+ .def(
+ "reset_attr", [](Op op, ffi::String name) {
+ OpDef(op->name)
+ .reset_attr(name);
+ },
+ "reset_attr(attr_name: str) -> None\n\nRemove this Op's current value; missing values "
+ "are ignored and cached views observe removal.")
+ .def(
+ "add_arg",
+ [](Op op, ffi::String name, ffi::String doc) {
+ OpDef(op->name)
+ .add_arg(name, doc);
+ },
+ "add_arg(name: str, doc: str) -> None\n\nAppend an argument's "
+ "name and documentation.")
+ .def(
+ "set_allow_extra_args", [](Op op) {
+ OpDef(op->name)
+ .allow_extra_args();
+ },
+ "set_allow_extra_args() -> None\n\nAllow value arguments after the required prefix.")
+ .def(
+ "add_ty_arg",
+ [](Op op, ffi::String name, ffi::String doc) {
+ OpDef(op->name)
+ .add_ty_arg(name, doc);
+ },
+ "add_ty_arg(name: str, doc: str) -> None\n\nAppend a "
+ "type-argument "
+ "name and documentation without imposing a count rule.")
+ .def(
+ "set_attrs_type_key",
+ [](Op op, ffi::String key) {
+ OpDef(op->name)
+ .set_attrs_type_key(key);
+ },
+ "set_attrs_type_key(key: str) -> None\n\nResolve and set the attribute object type key "
+ "and runtime index together; an unknown key raises before updating.");
+ // clang-format on
}
-// Check if a key is present in the registry.
-bool Op::HasAttrMap(const ffi::String& attr_name) {
- return OpRegistry::Global()->HasAttrMap(attr_name);
+namespace {
+
+TVMFFIAny OpVisit(ffi::StructuralVisitorObj*, ffi::AnyView) noexcept {
+ // Ops are unique registry atoms. Avoid reflecting through their registry metadata.
+ return ffi::AnyView(nullptr).CopyToTVMFFIAny();
}
-// Resets attr of the OpAttrMap.
-void OpRegEntry::reset_attr(const std::string& attr_name) {
- OpRegistry::Global()->ResetAttr(attr_name, op_);
+TVMFFIAny OpMutate(ffi::StructuralMutatorObj*, ffi::AnyView) noexcept {
+ return ffi::Unchanged().CopyToTVMFFIAny();
}
-void OpRegEntry::UpdateAttr(const ffi::String& key, ffi::Any value, int plevel) {
- OpRegistry::Global()->UpdateAttr(key, op_, value, plevel);
+TVMFFIAny OpMaybeInplaceMutate(ffi::StructuralMutatorObj*, ffi::AnyView) noexcept {
+ return ffi::Unchanged().CopyToTVMFFIAny();
}
-// Frontend APIs
+} // namespace
+
TVM_FFI_STATIC_INIT_BLOCK() {
- namespace refl = tvm::ffi::reflection;
+ namespace refl = ffi::reflection;
+ ArgumentInfoNode::RegisterReflection();
+ OpNode::RegisterReflection();
+ refl::TypeAttrDef<OpNode>()
+ .attr(refl::type_attr::kStructuralVisit, reinterpret_cast<void*>(&OpVisit))
+ .attr(refl::type_attr::kStructuralMutate, reinterpret_cast<void*>(&OpMutate))
+ .attr(refl::type_attr::kStructuralMaybeInplaceMutate,
+ reinterpret_cast<void*>(&OpMaybeInplaceMutate))
+ .def("__data_to_json__", [](const OpNode* node) { return node->name; })
+ .def("__data_from_json__", &Op::Get);
+ // clang-format off
refl::GlobalDef()
- .def("ir.ListOpNames", []() { return OpRegistry::Global()->ListAllNames(); })
- .def("ir.GetOp", [](ffi::String name) -> Op { return Op::Get(name); })
- .def("ir.OpGetAttr",
- [](Op op, ffi::String attr_name) -> ffi::Any {
- auto op_map = Op::GetAttrMap<ffi::Any>(attr_name);
- ffi::Any rv;
- if (op_map.count(op)) {
- rv = op_map[op];
- }
- return rv;
- })
- .def("ir.OpHasAttr",
- [](Op op, ffi::String attr_name) -> bool { return Op::HasAttrMap(attr_name); })
- .def("ir.OpSetAttr",
- [](Op op, ffi::String attr_name, ffi::AnyView value, int plevel) {
- auto& reg = OpRegistry::Global()->RegisterOrGet(op->name).set_name();
- reg.set_attr(attr_name, value, plevel);
- })
- .def("ir.OpResetAttr",
- [](Op op, ffi::String attr_name) {
- auto& reg = OpRegistry::Global()->RegisterOrGet(op->name);
- reg.reset_attr(attr_name);
- })
+ .def("ir.GetOp", &Op::Get)
+ .def("ir.ListOpNames", &Op::ListNames)
.def("ir.RegisterOp",
- [](ffi::String op_name, ffi::String descr) {
- const OpRegEntry* reg = OpRegistry::Global()->Get(op_name);
- TVM_FFI_CHECK(reg == nullptr, AttributeError)
- << "Operator " << op_name << " is registered before";
- auto& op = OpRegistry::Global()->RegisterOrGet(op_name).set_name();
- op.describe(descr);
- })
- .def("ir.OpAddArgument",
- [](Op op, ffi::String name, ffi::String type, ffi::String description) {
- auto& reg = OpRegistry::Global()->RegisterOrGet(op->name).set_name();
- reg.add_argument(name, type, description);
- })
- .def("ir.OpSetSupportLevel",
- [](Op op, int level) {
- auto& reg = OpRegistry::Global()->RegisterOrGet(op->name).set_name();
- reg.set_support_level(level);
+ [](ffi::String name, ffi::String doc) {
+ TVM_FFI_CHECK(!OpRegistry::Global()->Contains(name), AttributeError)
+ << "Operator " << name << " is registered before";
+ OpDef(name, doc);
})
- .def("ir.OpSetNumInputs",
- [](Op op, int n) {
- auto& reg = OpRegistry::Global()->RegisterOrGet(op->name).set_name();
- reg.set_num_inputs(n);
- })
- .def("ir.OpSetAttrsTypeKey",
- [](Op op, ffi::String key) {
- auto& reg = OpRegistry::Global()->RegisterOrGet(op->name).set_name();
- reg.set_attrs_type_key(key);
- })
- .def("ir.RegisterOpAttr",
- [](ffi::String op_name, ffi::String attr_key, ffi::AnyView value, int plevel) {
- auto& reg = OpRegistry::Global()->RegisterOrGet(op_name).set_name();
- // enable resgiteration and override of certain properties
- if (attr_key == "num_inputs" && plevel > 128) {
- reg.set_num_inputs(value.cast<int>());
- } else if (attr_key == "attrs_type_key" && plevel > 128) {
- TVM_FFI_THROW(InternalError) << "attrs type key no longer supported";
- } else {
- reg.set_attr(attr_key, value, plevel);
- }
- });
- // override OpNode to use name as the repr
+ .def("ir.RegisterOpAttr", [](ffi::String name, ffi::String key, ffi::Any value, bool override) {
+ OpDef(name)
+ .set_attr(key, value, override);
+ });
+ // clang-format on
}
-// Pattern A (RM): auto-default repr from reflection.
-
} // namespace tvm
diff --git a/src/ir/prim/builtin.cc b/src/ir/prim/builtin.cc
index 5d1def4789..bb0f6d7d99 100644
--- a/src/ir/prim/builtin.cc
+++ b/src/ir/prim/builtin.cc
@@ -22,37 +22,53 @@
namespace tvm {
namespace prim {
namespace builtin {
-#define PRIM_DEFINE_BUILTIN_FUNC(OpName) \
- const Op& OpName() { \
- static const Op& op = Op::Get("prim." #OpName); \
- return op; \
- } \
- TVM_REGISTER_OP("prim." #OpName)
-
-PRIM_DEFINE_BUILTIN_FUNC(likely)
- .set_num_inputs(1)
- .set_attr<TCallEffectKind>("TCallEffectKind",
- static_cast<int64_t>(CallEffectKind::kExprAnnotation))
- .set_attr<bool>("TVectorizable", true);
-PRIM_DEFINE_BUILTIN_FUNC(if_then_else)
- .set_num_inputs(3)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure));
-PRIM_DEFINE_BUILTIN_FUNC(vscale).set_attr<TCallEffectKind>(
- "TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure));
-
-PRIM_DEFINE_BUILTIN_FUNC(ceil)
- .set_num_inputs(1)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
- .set_attr<bool>("TVectorizable", true);
-PRIM_DEFINE_BUILTIN_FUNC(log2)
- .set_num_inputs(1)
- .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
- .set_attr<bool>("TVectorizable", true);
-
-PRIM_DEFINE_BUILTIN_FUNC(clz).set_num_inputs(1).set_attr<TCallEffectKind>(
- "TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure));
-
-#undef PRIM_DEFINE_BUILTIN_FUNC
+
+#define TVM_DEFINE_CACHED_OP_GETTER(Name, RegisteredName) \
+ const Op& Name() { \
+ static const Op op = Op::Get(RegisteredName); \
+ return op; \
+ }
+
+TVM_DEFINE_CACHED_OP_GETTER(likely, "prim.likely")
+TVM_DEFINE_CACHED_OP_GETTER(if_then_else, "prim.if_then_else")
+TVM_DEFINE_CACHED_OP_GETTER(vscale, "prim.vscale")
+TVM_DEFINE_CACHED_OP_GETTER(ceil, "prim.ceil")
+TVM_DEFINE_CACHED_OP_GETTER(log2, "prim.log2")
+TVM_DEFINE_CACHED_OP_GETTER(clz, "prim.clz")
+
+#undef TVM_DEFINE_CACHED_OP_GETTER
+
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("prim.likely")
+ .add_arg("x", "The input value.")
+ .set_attr<TCallEffectKind>("TCallEffectKind",
+ static_cast<int64_t>(CallEffectKind::kExprAnnotation))
+ .set_attr<bool>("TVectorizable", true);
+
+ OpDef("prim.if_then_else")
+ .add_arg("condition", "The condition.")
+ .add_arg("true_value", "The value when the condition is true.")
+ .add_arg("false_value", "The value when the condition is false.")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure));
+
+ OpDef("prim.vscale")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure));
+
+ OpDef("prim.ceil")
+ .add_arg("x", "The input value.")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
+ .set_attr<bool>("TVectorizable", true);
+
+ OpDef("prim.log2")
+ .add_arg("x", "The input value.")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
+ .set_attr<bool>("TVectorizable", true);
+
+ OpDef("prim.clz")
+ .add_arg("x", "The input value.")
+ .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure));
+}
+
} // namespace builtin
} // namespace prim
} // namespace tvm
diff --git a/src/relax/backend/adreno/annotate_custom_storage.cc b/src/relax/backend/adreno/annotate_custom_storage.cc
index d6735510b4..1748149407 100644
--- a/src/relax/backend/adreno/annotate_custom_storage.cc
+++ b/src/relax/backend/adreno/annotate_custom_storage.cc
@@ -337,7 +337,7 @@ class CollectConsumerScopeInfo : public ExprVisitor {
}
void VisitExpr_(const CallNode* call) final {
- static const Op& call_tir_op = Op::Get("relax.call_tir");
+ static const Op call_tir_op = Op::Get("relax.call_tir");
GlobalVar gv;
ffi::Array<Attrs> op_attrs;
ffi::Optional<int64_t> op_pattern = static_cast<int64_t>(OpPatternKind::kOpaque);
@@ -495,7 +495,7 @@ class CollectProducerScopeInfo : public ExprVisitor {
void VisitBinding_(const VarBindingNode* binding, const CallNode* call) final {
ExprVisitor::VisitBinding_(binding, call);
- static const Op& call_tir_op = Op::Get("relax.call_tir");
+ static const Op call_tir_op = Op::Get("relax.call_tir");
Type out_ty = Type::Missing();
if (call->op.same_as(call_tir_op)) {
@@ -621,7 +621,7 @@ class DefineVDevice : ExprMutator {
Expr VisitExpr_(const CallNode* call_node) override {
auto call = ExprMutator::VisitExpr_(call_node).as_or_throw<Call>();
- static const Op& call_tir_op = Op::Get("relax.call_tir");
+ static const Op call_tir_op = Op::Get("relax.call_tir");
GlobalVar gv;
Tuple func_args;
@@ -749,7 +749,7 @@ class DefineVDevice : ExprMutator {
return std::nullopt;
}
- const Op& hint_on_device_op_ = Op::Get("relax.hint_on_device");
+ const Op hint_on_device_op_ = Op::Get("relax.hint_on_device");
IRModule mod_;
IRModule updates_;
Target target_;
diff --git a/src/relax/backend/adreno/fold_vdevice_scope_change.cc b/src/relax/backend/adreno/fold_vdevice_scope_change.cc
index 0a0341f6a7..de39763fab 100644
--- a/src/relax/backend/adreno/fold_vdevice_scope_change.cc
+++ b/src/relax/backend/adreno/fold_vdevice_scope_change.cc
@@ -133,7 +133,7 @@ class CollectConsumerDetails : public ExprVisitor {
}
void VisitExpr_(const CallNode* call) final {
- static const Op& call_tir_op = Op::Get("relax.call_tir");
+ static const Op call_tir_op = Op::Get("relax.call_tir");
Tuple func_args;
if (call->op.same_as(call_tir_op)) {
diff --git a/src/relax/backend/contrib/tensorrt/codegen.cc b/src/relax/backend/contrib/tensorrt/codegen.cc
index 7a4df792db..c3fe8e15cc 100644
--- a/src/relax/backend/contrib/tensorrt/codegen.cc
+++ b/src/relax/backend/contrib/tensorrt/codegen.cc
@@ -125,7 +125,7 @@ class CollectFromCompositeFunctionBody : public ExprVisitor {
void SetArgumentAttributes(const CallNode* call_node) {
const auto* op_node = call_node->op.as<OpNode>();
if (op_node == nullptr) return;
- const ffi::Array<ArgumentInfo>& arg_infos = op_node->arguments;
+ const ffi::Array<ArgumentInfo>& arg_infos = op_node->args_info;
for (size_t i = 0; i < call_node->args.size() && i < arg_infos.size(); ++i) {
const Expr& arg = call_node->args[i];
const std::string key = "arg_" + std::string(arg_infos[i]->name);
diff --git a/src/relax/backend/task_extraction.cc b/src/relax/backend/task_extraction.cc
index 108aa6ffb1..1ed4333de5 100644
--- a/src/relax/backend/task_extraction.cc
+++ b/src/relax/backend/task_extraction.cc
@@ -101,7 +101,7 @@ class TaskExtractor : public ExprVisitor {
}
void VisitExpr_(const CallNode* call) final {
- static const Op& call_tir_op = Op::Get("relax.call_tir");
+ static const Op call_tir_op = Op::Get("relax.call_tir");
// TODO(@tvm-team): When we differentiate the call for tirx function and packed function,
// this logic should be changed accordingly.
diff --git a/src/relax/backend/vm/codegen_vm.cc b/src/relax/backend/vm/codegen_vm.cc
index 53d01f3f31..5423fa7936 100644
--- a/src/relax/backend/vm/codegen_vm.cc
+++ b/src/relax/backend/vm/codegen_vm.cc
@@ -431,11 +431,11 @@ class CodeGenVM : public ExprFunctor<Instruction::Arg(const Expr&)> {
/*! \brief the context module. */
IRModule ctx_mod_;
/*! \brief Cache ops that need to be frequently used later to reduce lookup overhead. */
- const Op& alloc_storage_op_ = Op::Get("relax.vm.alloc_storage");
- const Op& alloc_tensor_op_ = Op::Get("relax.vm.alloc_tensor");
- const Op& kill_object_op_ = Op::Get("relax.vm.kill_object");
- const Op& call_builtin_with_ctx_op_ = Op::Get("relax.call_builtin_with_ctx");
- const Op& null_value_op_ = Op::Get("relax.null_value");
+ const Op alloc_storage_op_ = Op::Get("relax.vm.alloc_storage");
+ const Op alloc_tensor_op_ = Op::Get("relax.vm.alloc_tensor");
+ const Op kill_object_op_ = Op::Get("relax.vm.kill_object");
+ const Op call_builtin_with_ctx_op_ = Op::Get("relax.call_builtin_with_ctx");
+ const Op null_value_op_ = Op::Get("relax.null_value");
};
/*!
diff --git a/src/relax/backend/vm/lower_runtime_builtin.cc b/src/relax/backend/vm/lower_runtime_builtin.cc
index f96cd1a8a2..7cde67fc5e 100644
--- a/src/relax/backend/vm/lower_runtime_builtin.cc
+++ b/src/relax/backend/vm/lower_runtime_builtin.cc
@@ -218,27 +218,27 @@ class LowerRuntimeBuiltinMutator : public ExprMutator {
Attrs(), {object_ty_});
}
- const Op& call_builtin_with_ctx_op_ = Op::Get("relax.call_builtin_with_ctx");
+ const Op call_builtin_with_ctx_op_ = Op::Get("relax.call_builtin_with_ctx");
const Type object_ty_ = AnyType();
const Type void_ty_ = TupleType(ffi::Array<Type>({}));
// object to pattern match.
- const Op& call_tir_dyn_op_ = Op::Get("relax.vm.call_tir_dyn");
- const Op& reshape_op_ = Op::Get("relax.reshape");
- const Op& shape_of_op_ = Op::Get("relax.shape_of");
- const Op& tensor_to_shape_op_ = Op::Get("relax.tensor_to_shape");
- const Op& call_py_func_op_ = Op::Get("relax.call_py_func");
- const Op& to_vdevice_op_ = Op::Get("relax.to_vdevice");
- const Op& make_closure_op_ = Op::Get("relax.make_closure");
- const Op& invoke_closure_op_ = Op::Get("relax.invoke_closure");
- const Op& alloc_tensor_op_ = Op::Get("relax.builtin.alloc_tensor");
- const Op& mem_alloc_storage_op_ = Op::Get("relax.memory.alloc_storage");
- const Op& mem_alloc_tensor_op_ = Op::Get("relax.memory.alloc_tensor");
- const Op& mem_kill_storage_op_ = Op::Get("relax.memory.kill_storage");
- const Op& mem_kill_tensor_op_ = Op::Get("relax.memory.kill_tensor");
+ const Op call_tir_dyn_op_ = Op::Get("relax.vm.call_tir_dyn");
+ const Op reshape_op_ = Op::Get("relax.reshape");
+ const Op shape_of_op_ = Op::Get("relax.shape_of");
+ const Op tensor_to_shape_op_ = Op::Get("relax.tensor_to_shape");
+ const Op call_py_func_op_ = Op::Get("relax.call_py_func");
+ const Op to_vdevice_op_ = Op::Get("relax.to_vdevice");
+ const Op make_closure_op_ = Op::Get("relax.make_closure");
+ const Op invoke_closure_op_ = Op::Get("relax.invoke_closure");
+ const Op alloc_tensor_op_ = Op::Get("relax.builtin.alloc_tensor");
+ const Op mem_alloc_storage_op_ = Op::Get("relax.memory.alloc_storage");
+ const Op mem_alloc_tensor_op_ = Op::Get("relax.memory.alloc_tensor");
+ const Op mem_kill_storage_op_ = Op::Get("relax.memory.kill_storage");
+ const Op mem_kill_tensor_op_ = Op::Get("relax.memory.kill_tensor");
// functions to lower to
- const Op& vm_alloc_storage_op_ = Op::Get("relax.vm.alloc_storage");
- const Op& vm_alloc_tensor_op_ = Op::Get("relax.vm.alloc_tensor");
- const Op& vm_kill_object_op_ = Op::Get("relax.vm.kill_object");
+ const Op vm_alloc_storage_op_ = Op::Get("relax.vm.alloc_storage");
+ const Op vm_alloc_tensor_op_ = Op::Get("relax.vm.alloc_tensor");
+ const Op vm_kill_object_op_ = Op::Get("relax.vm.kill_object");
// Function to compute allocated shape.
const ExternFunc builtin_compute_alloc_shape_{"vm.builtin.compute_alloc_shape"};
const ExternFunc builtin_call_tir_dyn_{"vm.builtin.call_tir_dyn"};
diff --git a/src/relax/backend/vm/vm_shape_lower.cc b/src/relax/backend/vm/vm_shape_lower.cc
index a8807e2728..8017dc4f49 100644
--- a/src/relax/backend/vm/vm_shape_lower.cc
+++ b/src/relax/backend/vm/vm_shape_lower.cc
@@ -933,8 +933,8 @@ class VMShapeLowerMutator
*/
std::vector<PrimExprSlot*> ready_vars_;
// call builtin cop
- const Op& call_builtin_with_ctx_op_ = Op::Get("relax.call_builtin_with_ctx");
- const Op& null_value_op_ = Op::Get("relax.null_value");
+ const Op call_builtin_with_ctx_op_ = Op::Get("relax.call_builtin_with_ctx");
+ const Op null_value_op_ = Op::Get("relax.null_value");
// common type
const Type object_ty_ = AnyType();
const Type void_ty_ = TupleType(ffi::Array<Type>({}));
diff --git a/src/relax/distributed/transform/lower_global_view_to_local_view.cc b/src/relax/distributed/transform/lower_global_view_to_local_view.cc
index 02ec388acf..55a5f7f211 100644
--- a/src/relax/distributed/transform/lower_global_view_to_local_view.cc
+++ b/src/relax/distributed/transform/lower_global_view_to_local_view.cc
@@ -383,7 +383,7 @@ class LowerTIRToLocalView : public ExprMutator {
}
void VisitBinding_(const VarBindingNode* binding, const CallNode* val) final {
- static const Op& call_tir_op = Op::Get("relax.call_tir");
+ static const Op call_tir_op = Op::Get("relax.call_tir");
if (!val->op.same_as(call_tir_op)) {
ExprMutator::VisitBinding_(binding, val);
return;
diff --git a/src/relax/distributed/transform/propagate_sharding.cc b/src/relax/distributed/transform/propagate_sharding.cc
index a7a1d0c2c6..be1e46d59e 100644
--- a/src/relax/distributed/transform/propagate_sharding.cc
+++ b/src/relax/distributed/transform/propagate_sharding.cc
@@ -47,7 +47,7 @@ void CollectAxisGraphBinary(const VarBindingNode* binding, const CallNode* call,
"add", "subtract", "multiply", "divide", "power", "floor_divide", "equal",
"greater", "greater_equal", "less", "less_equal", "not_equal", "minimum", "maximum"};
for (const auto& op_name : binary_op_names) {
- const Op& binary_op = Op::Get("relax." + op_name);
+ const Op binary_op = Op::Get("relax." + op_name);
if (call->op.same_as(binary_op)) {
BuildAxisGraphBinary(binding->var, ffi::GetRef<Call>(call), axis_group_graph);
break;
@@ -70,7 +70,7 @@ void CollectAxisGraphUnary(const VarBindingNode* binding, const CallNode* call,
"isinf", "isnan", "dist.annotate_sharding",
"erf", "nn.gelu", "builtin.stop_lift_params"};
for (const auto& op_name : unary_op_names) {
- const Op& unary_op = Op::Get("relax." + op_name);
+ const Op unary_op = Op::Get("relax." + op_name);
if (call->op.same_as(unary_op)) {
BuildAxisGraphUnary(binding->var, ffi::GetRef<Call>(call), axis_group_graph);
}
@@ -82,7 +82,7 @@ void CollectAxisGraphReduce(const VarBindingNode* binding, const CallNode* call,
const std::vector<std::string> reduction_op_names = {"sum", "max", "min", "prod",
"mean", "std", "variance", "nn.softmax"};
for (const auto& op_name : reduction_op_names) {
- const Op& reduction_op = Op::Get("relax." + op_name);
+ const Op reduction_op = Op::Get("relax." + op_name);
if (call->op.same_as(reduction_op)) {
BuildAxisGraphReduce(binding->var, ffi::GetRef<Call>(call), axis_group_graph);
break;
@@ -92,7 +92,7 @@ void CollectAxisGraphReduce(const VarBindingNode* binding, const CallNode* call,
void CollectAxisGraphMatmul(const VarBindingNode* binding, const CallNode* call,
AxisGroupGraph* axis_group_graph) {
- static const Op& matmul_op = Op::Get("relax.matmul");
+ static const Op matmul_op = Op::Get("relax.matmul");
if (call->op.same_as(matmul_op)) {
BuildAxisGraphMatmul(binding->var, ffi::GetRef<Call>(call), axis_group_graph);
}
@@ -100,7 +100,7 @@ void CollectAxisGraphMatmul(const VarBindingNode* binding, const CallNode* call,
void CollectAxisGraphPermuteDims(const VarBindingNode* binding, const CallNode* call,
AxisGroupGraph* axis_group_graph) {
- static const Op& permute_dims_op = Op::Get("relax.permute_dims");
+ static const Op permute_dims_op = Op::Get("relax.permute_dims");
if (call->op.same_as(permute_dims_op)) {
BuildAxisGraphPermuteDims(binding->var, ffi::GetRef<Call>(call), axis_group_graph);
}
@@ -108,7 +108,7 @@ void CollectAxisGraphPermuteDims(const VarBindingNode* binding, const CallNode*
void CollectAxisGraphReshape(const VarBindingNode* binding, const CallNode* call,
AxisGroupGraph* axis_group_graph) {
- static const Op& reshape_op = Op::Get("relax.reshape");
+ static const Op reshape_op = Op::Get("relax.reshape");
if (call->op.same_as(reshape_op)) {
BuildAxisGraphReshape(binding->var, ffi::GetRef<Call>(call), axis_group_graph);
}
@@ -117,7 +117,7 @@ void CollectAxisGraphReshape(const VarBindingNode* binding, const CallNode* call
void CollectAxisGraphForDeviceMesh(const VarBindingNode* binding, const CallNode* call,
AxisGroupGraph* axis_group_graph) {
ffi::Array<Expr> tensor_list;
- static const Op& call_tir_op = Op::Get("relax.call_tir");
+ static const Op call_tir_op = Op::Get("relax.call_tir");
ffi::Array<Expr> args;
if (call->op.same_as(call_tir_op)) {
args = call->args[1].as_or_throw<Tuple>()->fields;
@@ -157,7 +157,7 @@ class AxisGroupGraphBuilder : public ExprVisitor {
CollectAxisGraphMatmul(binding, val, axis_group_graph_);
CollectAxisGraphPermuteDims(binding, val, axis_group_graph_);
CollectAxisGraphReshape(binding, val, axis_group_graph_);
- static const Op& call_tir_op = Op::Get("relax.call_tir");
+ static const Op call_tir_op = Op::Get("relax.call_tir");
if (val->op.same_as(call_tir_op)) {
if (ffi::Optional<tirx::PrimFunc> func = MatchPrimFunc(mod_, val->args[0])) {
BuildAxisGraphCallTIR(binding->var, ffi::GetRef<Call>(val), func.value(),
@@ -225,7 +225,7 @@ class ShardingAnnotationCollector : public ExprVisitor {
explicit ShardingAnnotationCollector(AxisGroupGraph* axis_group_graph)
: axis_group_graph_(axis_group_graph) {}
void VisitBinding_(const VarBindingNode* binding, const CallNode* val) {
- static const Op& annotate_sharding_op = Op::Get("relax.dist.annotate_sharding");
+ static const Op annotate_sharding_op = Op::Get("relax.dist.annotate_sharding");
if (val->op.same_as(annotate_sharding_op)) {
const auto* attrs = val->attrs.as<DistributionAttrs>();
TVM_FFI_ICHECK(attrs);
@@ -435,7 +435,7 @@ class DistributedIRBuilder : public ExprMutator {
}
Expr VisitExpr_(const CallNode* call) final {
- static const Op& call_tir_op = Op::Get("relax.call_tir");
+ static const Op call_tir_op = Op::Get("relax.call_tir");
FBuildAxisGraph f = [&](const Var& var, const Call& call, AxisGroupGraph* axis_group_graph) {
ffi::Optional<tirx::PrimFunc> prim_func =
MatchPrimFunc(this->builder_->GetContextIRModule(), call->args[0]);
@@ -472,7 +472,7 @@ class DistributedIRBuilder : public ExprMutator {
}
Expr RemoveAnnotateSharding(Call call) {
- static const Op& annotate_sharding_op = Op::Get("relax.dist.annotate_sharding");
+ static const Op annotate_sharding_op = Op::Get("relax.dist.annotate_sharding");
if (call->op.same_as(annotate_sharding_op)) {
return call->args[0];
} else {
diff --git a/src/relax/ir/expr.cc b/src/relax/ir/expr.cc
index 5c89eec025..472dee24a3 100644
--- a/src/relax/ir/expr.cc
+++ b/src/relax/ir/expr.cc
@@ -809,7 +809,7 @@ Expr GetShapeOf(const Expr& expr) {
TVM_FFI_ICHECK(tinfo != nullptr) << "ShapeOf can only be applied to expr with TensorType";
if (tinfo->shape.has_value()) return tinfo->shape.value();
- static const Op& op = Op::Get("relax.shape_of");
+ static const Op op = Op::Get("relax.shape_of");
// default case, call shape of, eagerly normalize the expr.
Call call_shape_of(Type::Missing(), op, {expr}, {}, {});
UpdateType(call_shape_of, ShapeType(tinfo->ndim));
diff --git a/src/relax/op/ccl/ccl.cc b/src/relax/op/ccl/ccl.cc
index f5b660d1bc..f26578afc1 100644
--- a/src/relax/op/ccl/ccl.cc
+++ b/src/relax/op/ccl/ccl.cc
@@ -42,7 +42,7 @@ Expr allreduce(Expr x, ffi::String op_type, bool in_group) {
attrs->op_type = std::move(op_type);
attrs->in_group = std::move(in_group);
- static const Op& op = Op::Get("relax.ccl.allreduce");
+ static const Op op = Op::Get("relax.ccl.allreduce");
return Call(Type::Missing(), op, {std::move(x)}, Attrs{attrs}, {});
}
@@ -56,13 +56,14 @@ Type InferTypeAllReduce(const Call& call, const BlockBuilder& ctx) {
return input_ty;
}
-TVM_REGISTER_OP("relax.ccl.allreduce")
- .set_attrs_type<AllReduceAttrs>()
- .set_num_inputs(1)
- .add_argument("x", "Tensor", "Input to which allreduce will be applied.")
- .set_attr<FInferType>("FInferType", InferTypeAllReduce)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.ccl.allreduce")
+ .add_arg("x", "Input to which allreduce will be applied.")
+ .attrs_type<AllReduceAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeAllReduce)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.ccl.allgather */
@@ -71,7 +72,7 @@ Expr allgather(Expr x, int num_workers, bool in_group) {
attrs->num_workers = std::move(num_workers);
attrs->in_group = std::move(in_group);
- static const Op& op = Op::Get("relax.ccl.allgather");
+ static const Op op = Op::Get("relax.ccl.allgather");
return Call(Type::Missing(), op, {std::move(x)}, Attrs{attrs}, {});
}
@@ -96,16 +97,17 @@ Type InferTypeAllGather(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr(output_shape), output_dtype, input_ty->vdevice);
}
-TVM_REGISTER_OP("relax.ccl.allgather")
- .set_num_inputs(1)
- .add_argument("x", "Tensor", "Input to which allgather will be applied.")
- .set_attr<FInferType>("FInferType", InferTypeAllGather)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.ccl.allgather")
+ .add_arg("x", "Input to which allgather will be applied.")
+ .set_attr<FInferType>("FInferType", InferTypeAllGather)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.ccl.broadcast_from_worker0 */
Expr broadcast_from_worker0(Expr x) {
- static const Op& op = Op::Get("relax.ccl.broadcast_from_worker0");
+ static const Op op = Op::Get("relax.ccl.broadcast_from_worker0");
return Call(Type::Missing(), op, {std::move(x)}, {}, {});
}
@@ -119,12 +121,13 @@ Type InferTypeBroadcastFromZero(const Call& call, const BlockBuilder& ctx) {
return input_ty;
}
-TVM_REGISTER_OP("relax.ccl.broadcast_from_worker0")
- .set_num_inputs(1)
- .add_argument("x", "Tensor", "Input to be broadcast.")
- .set_attr<FInferType>("FInferType", InferTypeBroadcastFromZero)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.ccl.broadcast_from_worker0")
+ .add_arg("x", "Input to be broadcast.")
+ .set_attr<FInferType>("FInferType", InferTypeBroadcastFromZero)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.ccl.scatter_from_worker0 */
@@ -132,7 +135,7 @@ Expr scatter_from_worker0(Expr data, int num_workers, int axis) {
ffi::ObjectPtr<ScatterCollectiveAttrs> attrs = ffi::make_object<ScatterCollectiveAttrs>();
attrs->num_workers = std::move(num_workers);
attrs->axis = std::move(axis);
- static const Op& op = Op::Get("relax.ccl.scatter_from_worker0");
+ static const Op op = Op::Get("relax.ccl.scatter_from_worker0");
return Call(Type::Missing(), op, {std::move(data)}, Attrs{attrs}, {});
}
@@ -166,13 +169,14 @@ Type InferTypeScatter(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr(output_shape), output_dtype, input_ty->vdevice);
}
-TVM_REGISTER_OP("relax.ccl.scatter_from_worker0")
- .set_num_inputs(1)
- .add_argument("x", "Tensor",
- "The buffer to be divided into equal parts and sent to each worker accordingly.")
- .set_attrs_type<ScatterCollectiveAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeScatter)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.ccl.scatter_from_worker0")
+ .add_arg("x",
+ "The buffer to be divided into equal parts and sent to each worker accordingly.")
+ .attrs_type<ScatterCollectiveAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeScatter)
+ .set_attr<bool>("FPurity", true);
+}
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/distributed/binary.cc b/src/relax/op/distributed/binary.cc
index 5dca865f78..2ec0550ed4 100644
--- a/src/relax/op/distributed/binary.cc
+++ b/src/relax/op/distributed/binary.cc
@@ -36,42 +36,89 @@ Type InferDistTypeBroadcastCMP(const Call& call, const BlockBuilder& ctx) {
/***************** Arithmetic operators *****************/
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(add);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(divide);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(floor_divide);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(multiply);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(power);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(subtract);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(mod);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(floor_mod);
-
-/***************** Comparison operators *****************/
-
-RELAX_REGISTER_CMP_DIST_INFER_TYPE(equal);
-RELAX_REGISTER_CMP_DIST_INFER_TYPE(greater);
-RELAX_REGISTER_CMP_DIST_INFER_TYPE(greater_equal);
-RELAX_REGISTER_CMP_DIST_INFER_TYPE(less);
-RELAX_REGISTER_CMP_DIST_INFER_TYPE(less_equal);
-RELAX_REGISTER_CMP_DIST_INFER_TYPE(not_equal);
-
-/***************** Min/Max operators *****************/
-
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(minimum);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(maximum);
-
-/***************** Logical operators *****************/
-
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(logical_and);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(logical_or);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(logical_xor);
-
-/***************** Bitwise operators *****************/
-
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(bitwise_and);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(bitwise_or);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(bitwise_xor);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(left_shift);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(right_shift);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ // clang-format off
+ OpDef("relax.add")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.divide")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.floor_divide")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.multiply")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.power")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.subtract")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.mod")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.floor_mod")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ /***************** Comparison operators *****************/
+
+ OpDef("relax.equal")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP);
+
+ OpDef("relax.greater")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP);
+
+ OpDef("relax.greater_equal")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP);
+
+ OpDef("relax.less")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP);
+
+ OpDef("relax.less_equal")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP);
+
+ OpDef("relax.not_equal")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP);
+
+ /***************** Min/Max operators *****************/
+
+ OpDef("relax.minimum")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.maximum")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ /***************** Logical operators *****************/
+
+ OpDef("relax.logical_and")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.logical_or")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.logical_xor")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ /***************** Bitwise operators *****************/
+
+ OpDef("relax.bitwise_and")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.bitwise_or")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.bitwise_xor")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.left_shift")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+ OpDef("relax.right_shift")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+ // clang-format on
+}
} // namespace distributed
} // namespace relax
diff --git a/src/relax/op/distributed/binary.h b/src/relax/op/distributed/binary.h
index df07bc29e7..f68f2bf153 100644
--- a/src/relax/op/distributed/binary.h
+++ b/src/relax/op/distributed/binary.h
@@ -73,14 +73,6 @@ Type InferDistTypeBroadcastArith(const Call& call, const BlockBuilder& ctx);
Type InferDistTypeBroadcastCMP(const Call& call, const BlockBuilder& ctx);
-#define RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(OpName) \
- TVM_REGISTER_OP("relax." #OpName) \
- .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith)
-
-#define RELAX_REGISTER_CMP_DIST_INFER_TYPE(OpName) \
- TVM_REGISTER_OP("relax." #OpName) \
- .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP)
-
} // namespace distributed
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/distributed/ccl.cc b/src/relax/op/distributed/ccl.cc
index 4f899cdec4..8ec459d25d 100644
--- a/src/relax/op/distributed/ccl.cc
+++ b/src/relax/op/distributed/ccl.cc
@@ -37,8 +37,12 @@ Type InferDistTypeAllReduce(const Call& call, const BlockBuilder& ctx) {
Placement::FromText(std::string(device_mesh->shape.size(), 'R')));
}
-TVM_REGISTER_OP("relax.ccl.allreduce")
- .set_attr<FInferType>("dist.FInferType", InferDistTypeAllReduce);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ // clang-format off
+ OpDef("relax.ccl.allreduce")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeAllReduce);
+ // clang-format on
+}
} // namespace distributed
} // namespace relax
diff --git a/src/relax/op/distributed/distributed.cc b/src/relax/op/distributed/distributed.cc
index 0ae767aaf0..5075f8d354 100644
--- a/src/relax/op/distributed/distributed.cc
+++ b/src/relax/op/distributed/distributed.cc
@@ -49,7 +49,7 @@ Expr annotate_sharding(Expr input, distributed::DeviceMesh device_mesh,
attrs->device_mesh = device_mesh;
attrs->placement = placement;
- static const Op& op = Op::Get("relax.dist.annotate_sharding");
+ static const Op op = Op::Get("relax.dist.annotate_sharding");
return Call(Type::Missing(), op, {std::move(input)}, Attrs(attrs), {});
}
@@ -62,12 +62,13 @@ Type InferTypeAnnotateSharding(const Call& call, const BlockBuilder& ctx) {
return GetType(call->args[0]);
}
-TVM_REGISTER_OP("relax.dist.annotate_sharding")
- .set_num_inputs(1)
- .add_argument("input", "Tensor", "The input tensor.")
- .set_attr<FInferType>("FInferType", InferTypeAnnotateSharding)
- .set_attr<FInferType>("dist.FInferType", InferTypeAnnotateSharding)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.dist.annotate_sharding")
+ .add_arg("input", "The input tensor.")
+ .set_attr<FInferType>("FInferType", InferTypeAnnotateSharding)
+ .set_attr<FInferType>("dist.FInferType", InferTypeAnnotateSharding)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.dist.redistribute */
@@ -77,7 +78,7 @@ Expr redistribute(Expr input, distributed::DeviceMesh device_mesh,
attrs->device_mesh = device_mesh;
attrs->placement = placement;
- static const Op& op = Op::Get("relax.dist.redistribute");
+ static const Op op = Op::Get("relax.dist.redistribute");
return Call(Type::Missing(), op, {std::move(input)}, Attrs(attrs), {});
}
@@ -93,11 +94,12 @@ Type InferDistTypeRedistribute(const Call& call, const BlockBuilder& ctx) {
return distributed::DTensorType(ty->tensor_ty, attrs->device_mesh, attrs->placement);
}
-TVM_REGISTER_OP("relax.dist.redistribute")
- .set_num_inputs(1)
- .add_argument("input", "Tensor", "The input tensor.")
- .set_attr<FInferType>("dist.FInferType", InferDistTypeRedistribute)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.dist.redistribute")
+ .add_arg("input", "The input tensor.")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeRedistribute)
+ .set_attr<bool>("FPurity", true);
+}
Type InferTypeCallTIRLocalView(const Call& call, const BlockBuilder& ctx) {
if (call->ty_args.size() != 1) {
@@ -110,12 +112,14 @@ Type InferTypeCallTIRLocalView(const Call& call, const BlockBuilder& ctx) {
return call->ty_args[0];
}
-TVM_REGISTER_OP("relax.dist.call_tir_local_view")
- .set_num_inputs(2)
- .add_argument("func", "Expr", "The destination-passing-style function.")
- .add_argument("args", "Tuple", "The input arguments.")
- .set_attr<FInferType>("FInferType", InferTypeCallTIRLocalView)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.dist.call_tir_local_view")
+ .add_arg("func", "The destination-passing-style function.")
+ .add_arg("args", "The input arguments.")
+ .add_ty_arg("out_type", "The output type.")
+ .set_attr<FInferType>("FInferType", InferTypeCallTIRLocalView)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeCallTIRLocalView(Expr func, Tuple args, ffi::Array<distributed::DTensorType> out_ty_list) {
for (const distributed::DTensorType& ty : out_ty_list) {
@@ -133,7 +137,7 @@ Expr MakeCallTIRLocalView(Expr func, Tuple args, ffi::Array<distributed::DTensor
out_ty = TupleType({out_ty_list.begin(), out_ty_list.end()});
}
- static const Op& op = Op::Get("relax.dist.call_tir_local_view");
+ static const Op op = Op::Get("relax.dist.call_tir_local_view");
return Call(Type::Missing(), op, {func, args}, {}, {out_ty});
}
@@ -204,7 +208,7 @@ Expr redistribute_replica_to_shard(Expr input, int num_workers, int axis) {
ffi::ObjectPtr<ScatterCollectiveAttrs> attrs = ffi::make_object<ScatterCollectiveAttrs>();
attrs->num_workers = std::move(num_workers);
attrs->axis = std::move(axis);
- static const Op& op = Op::Get("relax.dist.redistribute_replica_to_shard");
+ static const Op op = Op::Get("relax.dist.redistribute_replica_to_shard");
return Call(Type::Missing(), op, {std::move(input)}, Attrs{attrs}, {});
}
@@ -213,15 +217,14 @@ TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.dist.redistribute_replica_to_shard",
redistribute_replica_to_shard);
-}
-TVM_REGISTER_OP("relax.dist.redistribute_replica_to_shard")
- .set_num_inputs(1)
- .add_argument("input", "Tensor", "The buffer to be sliced.")
- .set_attrs_type<ScatterCollectiveAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeRtoS)
- .set_attr<FInferType>("dist.FInferType", InferDistTypeRtoS)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.dist.redistribute_replica_to_shard")
+ .add_arg("input", "The buffer to be sliced.")
+ .attrs_type<ScatterCollectiveAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeRtoS)
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeRtoS)
+ .set_attr<bool>("FPurity", true);
+}
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/distributed/linear_algebra.cc b/src/relax/op/distributed/linear_algebra.cc
index 9fc5d29583..e7240c1bc0 100644
--- a/src/relax/op/distributed/linear_algebra.cc
+++ b/src/relax/op/distributed/linear_algebra.cc
@@ -96,7 +96,12 @@ Type InferDistTypeMatmul(const Call& call, const BlockBuilder& ctx) {
TensorType output_tensor_ty(ShapeExpr(output_shape), out_dtype);
return InferShardingSpec(call, ctx, output_tensor_ty, distributed::BuildAxisGraphMatmul);
}
-TVM_REGISTER_OP("relax.matmul").set_attr<FInferType>("dist.FInferType", InferDistTypeMatmul);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ // clang-format off
+ OpDef("relax.matmul")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeMatmul);
+ // clang-format on
+}
} // namespace distributed
} // namespace relax
diff --git a/src/relax/op/distributed/manipulate.cc b/src/relax/op/distributed/manipulate.cc
index 6d30cc879e..2febd5dd6d 100644
--- a/src/relax/op/distributed/manipulate.cc
+++ b/src/relax/op/distributed/manipulate.cc
@@ -77,8 +77,12 @@ Type InferDistTypePermuteDims(const Call& call, const BlockBuilder& ctx) {
return InferShardingSpec(call, ctx, output_tensor_ty, distributed::BuildAxisGraphPermuteDims);
}
-TVM_REGISTER_OP("relax.permute_dims")
- .set_attr<FInferType>("dist.FInferType", InferDistTypePermuteDims);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ // clang-format off
+ OpDef("relax.permute_dims")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypePermuteDims);
+ // clang-format on
+}
Type InferDistTypeReshape(const Call& call, const BlockBuilder& ctx) {
if (call->args.size() != 2) {
@@ -128,7 +132,12 @@ Type InferDistTypeReshape(const Call& call, const BlockBuilder& ctx) {
return InferShardingSpec(call, ctx, output_tensor_ty, distributed::BuildAxisGraphReshape);
}
-TVM_REGISTER_OP("relax.reshape").set_attr<FInferType>("dist.FInferType", InferDistTypeReshape);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ // clang-format off
+ OpDef("relax.reshape")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeReshape);
+ // clang-format on
+}
} // namespace distributed
} // namespace relax
diff --git a/src/relax/op/distributed/nn.cc b/src/relax/op/distributed/nn.cc
index d87a2c1bfa..48fac7c458 100644
--- a/src/relax/op/distributed/nn.cc
+++ b/src/relax/op/distributed/nn.cc
@@ -46,19 +46,20 @@ Type InferDistTypeSoftmax(const Call& call, const BlockBuilder& ctx) {
return InferShardingSpec(call, ctx, input_tensor_ty, distributed::BuildAxisGraphReduce);
}
-TVM_REGISTER_OP("relax.nn.softmax").set_attr<FInferType>("dist.FInferType", InferDistTypeSoftmax);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ // clang-format off
+ OpDef("relax.nn.softmax")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeSoftmax);
-/* relax.nn.relu */
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.relu, /*require_float_dtype=*/false);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.relu, /*require_float_dtype=*/false);
-/* relax.nn.gelu */
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.gelu, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.gelu, /*require_float_dtype=*/true);
-/* relax.nn.gelu_tanh */
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.gelu_tanh, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.gelu_tanh, /*require_float_dtype=*/true);
-/* relax.nn.silu */
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.silu, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.silu, /*require_float_dtype=*/true);
+ // clang-format on
+}
} // namespace distributed
} // namespace relax
diff --git a/src/relax/op/distributed/op.cc b/src/relax/op/distributed/op.cc
index 4b10b33ed4..697b7fee5d 100644
--- a/src/relax/op/distributed/op.cc
+++ b/src/relax/op/distributed/op.cc
@@ -35,7 +35,12 @@ Type InferDistTypeCallTIR(const Call& call, const BlockBuilder& ctx) {
return call->ty_args[0];
}
-TVM_REGISTER_OP("relax.call_tir").set_attr<FInferType>("dist.FInferType", InferDistTypeCallTIR);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ // clang-format off
+ OpDef("relax.call_tir")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeCallTIR);
+ // clang-format on
+}
Type InferDistTypeStopLiftParams(const Call& call, const BlockBuilder& ctx) {
if (call->args.size() != 1) {
@@ -44,8 +49,10 @@ Type InferDistTypeStopLiftParams(const Call& call, const BlockBuilder& ctx) {
return call->args[0]->ty.as_or_throw<Type>();
}
-TVM_REGISTER_OP("relax.builtin.stop_lift_params")
- .set_attr<FInferType>("dist.FInferType", InferDistTypeStopLiftParams);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.builtin.stop_lift_params")
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeStopLiftParams);
+}
} // namespace distributed
} // namespace relax
diff --git a/src/relax/op/distributed/statistical.cc b/src/relax/op/distributed/statistical.cc
index d24cc537da..3bdd322d35 100644
--- a/src/relax/op/distributed/statistical.cc
+++ b/src/relax/op/distributed/statistical.cc
@@ -75,13 +75,15 @@ Type InferDistTypeStatistical(const Call& call, const BlockBuilder& ctx) {
return InferShardingSpec(call, ctx, output_tensor_ty, distributed::BuildAxisGraphReduce);
}
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(max);
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(mean);
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(min);
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(prod);
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(std);
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(sum);
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(variance);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(max);
+ RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(mean);
+ RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(min);
+ RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(prod);
+ RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(std);
+ RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(sum);
+ RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(variance);
+}
} // namespace distributed
} // namespace relax
diff --git a/src/relax/op/distributed/statistical.h b/src/relax/op/distributed/statistical.h
index 6aa63ae32a..7d44e1532e 100644
--- a/src/relax/op/distributed/statistical.h
+++ b/src/relax/op/distributed/statistical.h
@@ -33,9 +33,11 @@ namespace distributed {
Type InferDistTypeStatistical(const Call& call, const BlockBuilder& ctx);
+// clang-format off
#define RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(OpName) \
- TVM_REGISTER_OP("relax." #OpName) \
+ OpDef("relax." #OpName) \
.set_attr<FInferType>("dist.FInferType", InferDistTypeStatistical)
+// clang-format on
} // namespace distributed
} // namespace relax
diff --git a/src/relax/op/distributed/unary.cc b/src/relax/op/distributed/unary.cc
index 8e4ccce23a..9b59b0ad4e 100644
--- a/src/relax/op/distributed/unary.cc
+++ b/src/relax/op/distributed/unary.cc
@@ -28,37 +28,38 @@ Type InferDistTypeUnaryCheck(const Call& call, const BlockBuilder& ctx) {
[](const TensorType& input_ty) { return PrimType::Bool(); });
}
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(abs, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(acos, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(acosh, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(asin, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(asinh, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(atan, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(atanh, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(bitwise_not, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(ceil, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(cos, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(cosh, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(exp, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(floor, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(log, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(logical_not, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(negative, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(round, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(rsqrt, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sigmoid, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sign, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sin, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sinh, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(square, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sqrt, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(tan, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(tanh, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(erf, /*require_float_dtype=*/true);
-
-RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(isfinite);
-RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(isinf);
-RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(isnan);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(abs, /*require_float_dtype=*/false);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(acos, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(acosh, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(asin, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(asinh, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(atan, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(atanh, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(bitwise_not, /*require_float_dtype=*/false);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(ceil, /*require_float_dtype=*/false);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(cos, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(cosh, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(exp, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(floor, /*require_float_dtype=*/false);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(log, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(logical_not, /*require_float_dtype=*/false);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(negative, /*require_float_dtype=*/false);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(round, /*require_float_dtype=*/false);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(rsqrt, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sigmoid, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sign, /*require_float_dtype=*/false);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sin, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sinh, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(square, /*require_float_dtype=*/false);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sqrt, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(tan, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(tanh, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(erf, /*require_float_dtype=*/true);
+ RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(isfinite);
+ RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(isinf);
+ RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(isnan);
+}
} // namespace distributed
} // namespace relax
diff --git a/src/relax/op/distributed/unary.h b/src/relax/op/distributed/unary.h
index 6988a8adae..4e2bb8f64e 100644
--- a/src/relax/op/distributed/unary.h
+++ b/src/relax/op/distributed/unary.h
@@ -64,12 +64,15 @@ Type InferDistTypeUnaryArith(const Call& call, const BlockBuilder& ctx) {
Type InferDistTypeUnaryCheck(const Call& call, const BlockBuilder& ctx);
+// clang-format off
#define RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(OpName, RequireFloatDtype) \
- TVM_REGISTER_OP("relax." #OpName) \
+ OpDef("relax." #OpName) \
.set_attr<FInferType>("dist.FInferType", InferDistTypeUnaryArith<RequireFloatDtype>)
#define RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(OpName) \
- TVM_REGISTER_OP("relax." #OpName).set_attr<FInferType>("dist.FInferType", InferDistTypeUnaryCheck)
+ OpDef("relax." #OpName) \
+ .set_attr<FInferType>("dist.FInferType", InferDistTypeUnaryCheck)
+// clang-format on
} // namespace distributed
} // namespace relax
diff --git a/src/relax/op/image/resize.cc b/src/relax/op/image/resize.cc
index 76d1f868ed..1476ddd74b 100644
--- a/src/relax/op/image/resize.cc
+++ b/src/relax/op/image/resize.cc
@@ -33,8 +33,10 @@
namespace tvm {
namespace relax {
-TVM_FFI_STATIC_INIT_BLOCK() { Resize2DAttrs::RegisterReflection(); }
-TVM_FFI_STATIC_INIT_BLOCK() { Resize3DAttrs::RegisterReflection(); }
+TVM_FFI_STATIC_INIT_BLOCK() {
+ Resize2DAttrs::RegisterReflection();
+ Resize3DAttrs::RegisterReflection();
+}
/* relax.resize2d */
@@ -53,7 +55,7 @@ Expr resize2d(Expr data, Expr size, ffi::Array<FloatImm> roi, ffi::String layout
attrs->extrapolation_value = extrapolation_value;
attrs->out_dtype = out_dtype;
- static const Op& op = Op::Get("relax.image.resize2d");
+ static const Op op = Op::Get("relax.image.resize2d");
return Call(Type::Missing(), op, {std::move(data), std::move(size)}, Attrs(attrs), {});
}
@@ -142,15 +144,16 @@ InferLayoutOutput InferLayoutResize2d(
Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.image.resize2d")
- .set_attrs_type<Resize2DAttrs>()
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "The input tensor.")
- .add_argument("size", "Shape", "The output image shape.")
- .set_attr<FInferType>("FInferType", InferTypeResize2D)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutResize2d)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.image.resize2d")
+ .add_arg("data", "The input tensor.")
+ .add_arg("size", "The output image shape.")
+ .attrs_type<Resize2DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeResize2D)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutResize2d)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.resize3d */
@@ -169,7 +172,7 @@ Expr resize3d(Expr data, Expr size, ffi::Array<FloatImm> roi, ffi::String layout
attrs->extrapolation_value = extrapolation_value;
attrs->out_dtype = out_dtype;
- static const Op& op = Op::Get("relax.image.resize3d");
+ static const Op op = Op::Get("relax.image.resize3d");
return Call(Type::Missing(), op, {std::move(data), std::move(size)}, Attrs(attrs), {});
}
@@ -256,19 +259,20 @@ InferLayoutOutput InferLayoutResize3d(
Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.image.resize3d")
- .set_attrs_type<Resize3DAttrs>()
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "The input tensor.")
- .add_argument("size", "Shape", "The output image shape.")
- .set_attr<FInferType>("FInferType", InferTypeResize3D)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutResize3d)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
-
-/* relax.grid_sample */
-
-TVM_FFI_STATIC_INIT_BLOCK() { GridSampleAttrs::RegisterReflection(); }
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.image.resize3d")
+ .add_arg("data", "The input tensor.")
+ .add_arg("size", "The output image shape.")
+ .attrs_type<Resize3DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeResize3D)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutResize3d)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+
+ /* relax.grid_sample */
+
+ GridSampleAttrs::RegisterReflection();
+}
Expr grid_sample(Expr data, Expr grid, ffi::String method, ffi::String layout,
ffi::String padding_mode, bool align_corners) {
@@ -278,7 +282,7 @@ Expr grid_sample(Expr data, Expr grid, ffi::String method, ffi::String layout,
attrs->padding_mode = std::move(padding_mode);
attrs->align_corners = align_corners;
- static const Op& op = Op::Get("relax.image.grid_sample");
+ static const Op op = Op::Get("relax.image.grid_sample");
return Call(Type::Missing(), op, {std::move(data), std::move(grid)}, Attrs(attrs), {});
}
@@ -347,27 +351,28 @@ Type InferTypeGridSample(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr(out_shape), out_dtype, data_ty->vdevice);
}
-TVM_REGISTER_OP("relax.image.grid_sample")
- .set_attrs_type<GridSampleAttrs>()
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "The input tensor.")
- .add_argument("grid", "Tensor", "The grid tensor for sampling.")
- .set_attr<FInferType>("FInferType", InferTypeGridSample)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.image.grid_sample")
+ .add_arg("data", "The input tensor.")
+ .add_arg("grid", "The grid tensor for sampling.")
+ .attrs_type<GridSampleAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeGridSample)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.image.affine_grid */
Expr affine_grid(Expr data, Expr size, bool align_corners) {
ffi::ObjectPtr<AffineGridAttrs> attrs = ffi::make_object<AffineGridAttrs>();
attrs->align_corners = align_corners;
- static const Op& op = Op::Get("relax.image.affine_grid");
+ static const Op op = Op::Get("relax.image.affine_grid");
return Call(Type::Missing(), op, {std::move(data), std::move(size)}, Attrs(attrs), {});
}
-TVM_FFI_STATIC_INIT_BLOCK() { AffineGridAttrs::RegisterReflection(); }
-
TVM_FFI_STATIC_INIT_BLOCK() {
+ AffineGridAttrs::RegisterReflection();
+
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.image.affine_grid", affine_grid);
}
@@ -446,14 +451,15 @@ Type InferTypeAffineGrid(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr(out_shape), out_dtype, data_ty->vdevice);
}
-TVM_REGISTER_OP("relax.image.affine_grid")
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "The input affine matrix tensor.")
- .add_argument("size", "Shape", "The target output shape (H, W).")
- .set_attrs_type<AffineGridAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeAffineGrid)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.image.affine_grid")
+ .add_arg("data", "The input affine matrix tensor.")
+ .add_arg("size", "The target output shape (H, W).")
+ .attrs_type<AffineGridAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeAffineGrid)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/memory/view.cc b/src/relax/op/memory/view.cc
index 39353d2a35..5d25cf3219 100644
--- a/src/relax/op/memory/view.cc
+++ b/src/relax/op/memory/view.cc
@@ -36,7 +36,7 @@ Expr view(Expr x, ffi::Optional<Expr> shape, ffi::Optional<Expr> dtype,
ffi::Optional<Expr> relative_byte_offset) {
Tuple void_expr(ffi::Array<Expr>{});
- static const Op& op = Op::Get("relax.memory.view");
+ static const Op op = Op::Get("relax.memory.view");
return Call(Type::Missing(), op,
{
x,
@@ -108,7 +108,7 @@ Type InferTypeView(const Call& call, const BlockBuilder& ctx) {
}
}
- static const Op& null_value_op = Op::Get("relax.null_value");
+ static const Op null_value_op = Op::Get("relax.null_value");
if (const CallNode* call_node = arg_value.as<CallNode>()) {
if (call_node->op.same_as(null_value_op)) {
// No datatype change is applied. This is the non-void
@@ -338,7 +338,7 @@ Expr LowerBuiltinView(const BlockBuilder& bb, const Call& call) {
}
}
- static const Op& null_value_op = Op::Get("relax.null_value");
+ static const Op null_value_op = Op::Get("relax.null_value");
if (const CallNode* call_node = arg.as<CallNode>()) {
return call_node->op.same_as(null_value_op);
}
@@ -392,20 +392,21 @@ Expr LowerBuiltinView(const BlockBuilder& bb, const Call& call) {
return Call(Type::Missing(), runtime_view_func, {data, shape, dtype, relative_byte_offset});
}
-TVM_REGISTER_OP("relax.memory.view")
- .set_num_inputs(4)
- .add_argument("x", "Tensor", "The input tensor.")
- .add_argument("shape", "Shape", "The view's shape.")
- .add_argument("dtype", "DataType", "The view's data type.")
- .add_argument("relative_byte_offset", "Prim(\"int64\")",
- "The view's byte offset, relative to the input tensor's byte offset.")
- .set_attr<bool>("RequiresArgumentShapes", false)
- .set_attr<FInferType>("FInferType", InferTypeView)
- .set_attr<bool>("FPurity", true)
- .set_attr<FLowerBuiltin>("FLowerBuiltin", LowerBuiltinView);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.memory.view")
+ .add_arg("x", "The input tensor.")
+ .add_arg("shape", "The view's shape.")
+ .add_arg("dtype", "The view's data type.")
+ .add_arg("relative_byte_offset",
+ "The view's byte offset, relative to the input tensor's byte offset.")
+ .set_attr<bool>("RequiresArgumentShapes", false)
+ .set_attr<FInferType>("FInferType", InferTypeView)
+ .set_attr<bool>("FPurity", true)
+ .set_attr<FLowerBuiltin>("FLowerBuiltin", LowerBuiltinView);
+}
Expr ensure_zero_offset(const Expr& x) {
- static const Op& op = Op::Get("relax.memory.ensure_zero_offset");
+ static const Op op = Op::Get("relax.memory.ensure_zero_offset");
return Call(Type::Missing(), op, {x});
}
@@ -428,13 +429,14 @@ Expr LowerBuiltinEnsureZeroOffset(const BlockBuilder& bb, const Call& call) {
return Call(Type::Missing(), builtin_ensure_zero_offset_, call->args, Attrs(), {GetType(call)});
}
-TVM_REGISTER_OP("relax.memory.ensure_zero_offset")
- .set_num_inputs(1)
- .add_argument("x", "Tensor", "The input tensor.")
- .set_attr<bool>("RequiresArgumentShapes", false)
- .set_attr<FInferType>("FInferType", InferTypeEnsureZeroOffset)
- .set_attr<bool>("FPurity", true)
- .set_attr<FLowerBuiltin>("FLowerBuiltin", LowerBuiltinEnsureZeroOffset);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.memory.ensure_zero_offset")
+ .add_arg("x", "The input tensor.")
+ .set_attr<bool>("RequiresArgumentShapes", false)
+ .set_attr<FInferType>("FInferType", InferTypeEnsureZeroOffset)
+ .set_attr<bool>("FPurity", true)
+ .set_attr<FLowerBuiltin>("FLowerBuiltin", LowerBuiltinEnsureZeroOffset);
+}
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/nn/attention.cc b/src/relax/op/nn/attention.cc
index f38a0d7a2d..b271db5659 100644
--- a/src/relax/op/nn/attention.cc
+++ b/src/relax/op/nn/attention.cc
@@ -149,45 +149,44 @@ Call InferMixedPrecisionAttention(const Call& call, DLDataType out_dtype) {
.as_or_throw<Call>();
}
-TVM_REGISTER_OP("relax.nn.attention")
- .set_attrs_type<AttentionAttrs>()
- .set_num_inputs(3)
- .add_argument("query", "Tensor", "The input queries tensor.")
- .add_argument("key", "Tensor", "The input keys tensor.")
- .add_argument("value", "Tensor", "The input values tensor.")
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
- .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionAttention)
- .set_attr<FInferType>("FInferType", InferTypeAttention)
- .set_attr<bool>("FPurity", true);
-
-TVM_REGISTER_OP("relax.nn.attention_bias")
- .set_attrs_type<AttentionAttrs>()
- .set_num_inputs(4)
- .add_argument("query", "Tensor", "The input queries tensor.")
- .add_argument("key", "Tensor", "The input keys tensor.")
- .add_argument("value", "Tensor", "The input values tensor.")
- .add_argument("bias", "Tensor", "The input bias tensor.")
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
- .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionAttention)
- .set_attr<FInferType>("FInferType", InferTypeAttention)
- .set_attr<bool>("FPurity", true);
-
-TVM_REGISTER_OP("relax.nn.attention_var_len")
- .set_attrs_type<AttentionAttrs>()
- .set_num_inputs(7)
- .add_argument("query", "Tensor", "The input queries tensor.")
- .add_argument("key", "Tensor", "The input keys tensor.")
- .add_argument("value", "Tensor", "The input values tensor.")
- .add_argument("seqstart_q", "Tensor", "The cumsum of query sequence lengths, prepended with 0.")
- .add_argument("seqstart_k", "Tensor", "The cumsum of key sequence lengths, prepended with 0.")
- .add_argument("max_seqlen_q", "Tensor", "The maximum query sequence length in the batch.")
- .add_argument("max_seqlen_k", "Tensor", "The maximum key sequence length in the batch.")
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
- .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionAttention)
- .set_attr<FInferType>("FInferType", InferTypeAttention)
- .set_attr<bool>("FPurity", true);
-
-TVM_FFI_STATIC_INIT_BLOCK() { AttentionAttrs::RegisterReflection(); }
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.attention")
+ .add_arg("query", "The input queries tensor.")
+ .add_arg("key", "The input keys tensor.")
+ .add_arg("value", "The input values tensor.")
+ .attrs_type<AttentionAttrs>()
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+ .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionAttention)
+ .set_attr<FInferType>("FInferType", InferTypeAttention)
+ .set_attr<bool>("FPurity", true);
+
+ OpDef("relax.nn.attention_bias")
+ .add_arg("query", "The input queries tensor.")
+ .add_arg("key", "The input keys tensor.")
+ .add_arg("value", "The input values tensor.")
+ .add_arg("bias", "The input bias tensor.")
+ .attrs_type<AttentionAttrs>()
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+ .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionAttention)
+ .set_attr<FInferType>("FInferType", InferTypeAttention)
+ .set_attr<bool>("FPurity", true);
+
+ OpDef("relax.nn.attention_var_len")
+ .add_arg("query", "The input queries tensor.")
+ .add_arg("key", "The input keys tensor.")
+ .add_arg("value", "The input values tensor.")
+ .add_arg("seqstart_q", "The cumsum of query sequence lengths, prepended with 0.")
+ .add_arg("seqstart_k", "The cumsum of key sequence lengths, prepended with 0.")
+ .add_arg("max_seqlen_q", "The maximum query sequence length in the batch.")
+ .add_arg("max_seqlen_k", "The maximum key sequence length in the batch.")
+ .attrs_type<AttentionAttrs>()
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+ .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionAttention)
+ .set_attr<FInferType>("FInferType", InferTypeAttention)
+ .set_attr<bool>("FPurity", true);
+
+ AttentionAttrs::RegisterReflection();
+}
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/nn/convolution.cc b/src/relax/op/nn/convolution.cc
index 1927a7d5bc..e13541dc10 100644
--- a/src/relax/op/nn/convolution.cc
+++ b/src/relax/op/nn/convolution.cc
@@ -196,16 +196,18 @@ Call InferMixedPrecisionConv1d(const Call& call, DLDataType out_dtype) {
.as_or_throw<Call>();
}
-TVM_REGISTER_OP("relax.nn.conv1d")
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "The input tensor.")
- .add_argument("weight", "Tensor", "The weight tensor.")
- .set_attrs_type<Conv1DAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeConv1d)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv1d)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
- .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv1d)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.conv1d")
+ .add_arg("data", "The input tensor.")
+ .add_arg("weight", "The weight tensor.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .attrs_type<Conv1DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeConv1d)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv1d)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+ .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv1d)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.conv2d */
@@ -408,16 +410,18 @@ Call InferMixedPrecisionConv2d(const Call& call, DLDataType out_dtype) {
.as_or_throw<Call>();
}
-TVM_REGISTER_OP("relax.nn.conv2d")
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "The input tensor.")
- .add_argument("weight", "Tensor", "The weight tensor.")
- .set_attrs_type<Conv2DAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeConv2d)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv2d)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
- .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv2d)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.conv2d")
+ .add_arg("data", "The input tensor.")
+ .add_arg("weight", "The weight tensor.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .attrs_type<Conv2DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeConv2d)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv2d)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+ .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv2d)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.conv3d */
@@ -594,16 +598,18 @@ Call InferMixedPrecisionConv3d(const Call& call, DLDataType out_dtype) {
.as_or_throw<Call>();
}
-TVM_REGISTER_OP("relax.nn.conv3d")
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "The input tensor.")
- .add_argument("weight", "Tensor", "The weight tensor.")
- .set_attrs_type<Conv3DAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeConv3d)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv3d)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
- .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv3d)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.conv3d")
+ .add_arg("data", "The input tensor.")
+ .add_arg("weight", "The weight tensor.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .attrs_type<Conv3DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeConv3d)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv3d)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+ .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv3d)
+ .set_attr<bool>("FPurity", true);
+}
Expr conv1d_transpose(Expr data, Expr weight, ffi::Array<int64_t> strides,
ffi::Array<int64_t> padding, ffi::Array<int64_t> output_padding,
@@ -636,7 +642,7 @@ Expr conv1d_transpose(Expr data, Expr weight, ffi::Array<int64_t> strides,
attrs->kernel_layout = std::move(kernel_layout);
attrs->out_layout = out_layout.value_or(data_layout);
attrs->out_dtype = out_dtype;
- const Op& op = Op::Get("relax.nn.conv1d_transpose");
+ const Op op = Op::Get("relax.nn.conv1d_transpose");
return Call(Type::Missing(), op, {data, weight}, Attrs(attrs), {});
}
@@ -774,16 +780,18 @@ Call InferMixedPrecisionConv1dTranspose(const Call& call, DLDataType out_dtype)
.as_or_throw<Call>();
}
-TVM_REGISTER_OP("relax.nn.conv1d_transpose")
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "The input tensor.")
- .add_argument("weight", "Tensor", "The weight tensor.")
- .set_attrs_type<Conv1DTransposeAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeConv1dTranspose)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv1dTranspose)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
- .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv1dTranspose)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.conv1d_transpose")
+ .add_arg("data", "The input tensor.")
+ .add_arg("weight", "The weight tensor.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .attrs_type<Conv1DTransposeAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeConv1dTranspose)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv1dTranspose)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+ .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv1dTranspose)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.conv2d_transpose */
@@ -827,7 +835,7 @@ Expr conv2d_transpose(Expr data, Expr weight, ffi::Array<int64_t> strides,
attrs->kernel_layout = std::move(kernel_layout);
attrs->out_layout = out_layout.value_or(data_layout);
attrs->out_dtype = out_dtype;
- const Op& op = Op::Get("relax.nn.conv2d_transpose");
+ const Op op = Op::Get("relax.nn.conv2d_transpose");
return Call(Type::Missing(), op, {data, weight}, Attrs(attrs), {});
}
@@ -1003,16 +1011,18 @@ Call InferMixedPrecisionConv2dTranspose(const Call& call, DLDataType out_dtype)
.as_or_throw<Call>();
}
-TVM_REGISTER_OP("relax.nn.conv2d_transpose")
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "The input tensor.")
- .add_argument("weight", "Tensor", "The weight tensor.")
- .set_attrs_type<Conv2DTransposeAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeConv2dTranspose)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv2dTranspose)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
- .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv2dTranspose)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.conv2d_transpose")
+ .add_arg("data", "The input tensor.")
+ .add_arg("weight", "The weight tensor.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .attrs_type<Conv2DTransposeAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeConv2dTranspose)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv2dTranspose)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+ .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv2dTranspose)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.conv3d_transpose */
@@ -1059,7 +1069,7 @@ Expr conv3d_transpose(Expr data, Expr weight, ffi::Array<int64_t> strides,
attrs->kernel_layout = std::move(kernel_layout);
attrs->out_layout = out_layout.value_or(data_layout);
attrs->out_dtype = out_dtype;
- const Op& op = Op::Get("relax.nn.conv3d_transpose");
+ const Op op = Op::Get("relax.nn.conv3d_transpose");
return Call(Type::Missing(), op, {data, weight}, Attrs(attrs), {});
}
@@ -1243,16 +1253,18 @@ Call InferMixedPrecisionConv3dTranspose(const Call& call, DLDataType out_dtype)
.as_or_throw<Call>();
}
-TVM_REGISTER_OP("relax.nn.conv3d_transpose")
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "The input tensor.")
- .add_argument("weight", "Tensor", "The weight tensor.")
- .set_attrs_type<Conv3DTransposeAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeConv3dTranspose)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv3dTranspose)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
- .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv3dTranspose)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.conv3d_transpose")
+ .add_arg("data", "The input tensor.")
+ .add_arg("weight", "The weight tensor.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .attrs_type<Conv3DTransposeAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeConv3dTranspose)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv3dTranspose)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+ .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv3dTranspose)
+ .set_attr<bool>("FPurity", true);
+}
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/nn/convolution.h b/src/relax/op/nn/convolution.h
index bf3afe0f41..5b655b797a 100644
--- a/src/relax/op/nn/convolution.h
+++ b/src/relax/op/nn/convolution.h
@@ -49,7 +49,7 @@ inline Expr MakeConv(Expr data, Expr weight, ffi::Array<int64_t> strides,
attrs->kernel_layout = std::move(kernel_layout);
attrs->out_layout = std::move(out_layout);
attrs->out_dtype = out_dtype;
- const Op& op = Op::Get(op_name);
+ const Op op = Op::Get(op_name);
return Call(Type::Missing(), op, {data, weight}, Attrs(attrs), {});
}
diff --git a/src/relax/op/nn/nn.cc b/src/relax/op/nn/nn.cc
index 1da6baedd6..38d93b4200 100644
--- a/src/relax/op/nn/nn.cc
+++ b/src/relax/op/nn/nn.cc
@@ -46,40 +46,111 @@ TVM_FFI_STATIC_INIT_BLOCK() {
}
/* relax.nn.relu */
-RELAX_REGISTER_UNARY_NN_OP_AND_IMPL(relu, "nn.relu", /*require_float_dtype=*/false);
+Expr relu(Expr x) {
+ static const Op op = Op::Get("relax.nn.relu");
+ return Call(Type::Missing(), op, {std::move(x)}, Attrs(), {});
+}
-/* relax.nn.gelu */
-RELAX_REGISTER_UNARY_NN_OP_AND_IMPL(gelu, "nn.gelu", /*require_float_dtype=*/true);
+Expr gelu(Expr x) {
+ static const Op op = Op::Get("relax.nn.gelu");
+ return Call(Type::Missing(), op, {std::move(x)}, Attrs(), {});
+}
-/* relax.nn.gelu_tanh */
-RELAX_REGISTER_UNARY_NN_OP_AND_IMPL(gelu_tanh, "nn.gelu_tanh", /*require_float_dtype=*/true);
+Expr gelu_tanh(Expr x) {
+ static const Op op = Op::Get("relax.nn.gelu_tanh");
+ return Call(Type::Missing(), op, {std::move(x)}, Attrs(), {});
+}
+
+Expr selu(Expr x) {
+ static const Op op = Op::Get("relax.nn.selu");
+ return Call(Type::Missing(), op, {std::move(x)}, Attrs(), {});
+}
-/* relax.nn.selu */
-RELAX_REGISTER_UNARY_NN_OP_AND_IMPL(selu, "nn.selu", /*require_float_dtype=*/true);
+Expr silu(Expr x) {
+ static const Op op = Op::Get("relax.nn.silu");
+ return Call(Type::Missing(), op, {std::move(x)}, Attrs(), {});
+}
-/* relax.nn.silu */
-RELAX_REGISTER_UNARY_NN_OP_AND_IMPL(silu, "nn.silu", /*require_float_dtype=*/true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ tvm::ffi::reflection::GlobalDef().def("relax.op.nn.relu", relu);
+
+ OpDef("relax.nn.relu")
+ .add_arg("x", "The input tensor.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true)
+ .set_attr<FInferType>("FInferType", InferTypeUnaryArith<false>);
+
+ /* relax.nn.gelu */
+
+ tvm::ffi::reflection::GlobalDef().def("relax.op.nn.gelu", gelu);
+
+ OpDef("relax.nn.gelu")
+ .add_arg("x", "The input tensor.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true)
+ .set_attr<FInferType>("FInferType", InferTypeUnaryArith<true>);
+
+ /* relax.nn.gelu_tanh */
+
+ tvm::ffi::reflection::GlobalDef().def("relax.op.nn.gelu_tanh", gelu_tanh);
+
+ OpDef("relax.nn.gelu_tanh")
+ .add_arg("x", "The input tensor.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true)
+ .set_attr<FInferType>("FInferType", InferTypeUnaryArith<true>);
+
+ /* relax.nn.selu */
+
+ tvm::ffi::reflection::GlobalDef().def("relax.op.nn.selu", selu);
+
+ OpDef("relax.nn.selu")
+ .add_arg("x", "The input tensor.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true)
+ .set_attr<FInferType>("FInferType", InferTypeUnaryArith<true>);
+
+ /* relax.nn.silu */
+
+ tvm::ffi::reflection::GlobalDef().def("relax.op.nn.silu", silu);
+
+ OpDef("relax.nn.silu")
+ .add_arg("x", "The input tensor.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true)
+ .set_attr<FInferType>("FInferType", InferTypeUnaryArith<true>);
+}
/* relax.nn.leakyrelu */
Expr leakyrelu(Expr data, double alpha) {
auto attrs = ffi::make_object<LeakyReluAttrs>();
attrs->alpha = alpha;
- static const Op& op = Op::Get("relax.nn.leakyrelu");
+ static const Op op = Op::Get("relax.nn.leakyrelu");
return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.nn.leakyrelu", leakyrelu);
-}
-TVM_REGISTER_OP("relax.nn.leakyrelu")
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor.")
- .set_attrs_type<LeakyReluAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeUnaryArith</*require_float_dtype=*/true>)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.nn.leakyrelu")
+ .add_arg("data", "The input tensor.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .attrs_type<LeakyReluAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeUnaryArith</*require_float_dtype=*/true>)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.softplus */
@@ -87,28 +158,28 @@ Expr softplus(Expr data, double beta, double threshold) {
auto attrs = ffi::make_object<SoftplusAttrs>();
attrs->beta = beta;
attrs->threshold = threshold;
- static const Op& op = Op::Get("relax.nn.softplus");
+ static const Op op = Op::Get("relax.nn.softplus");
return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.nn.softplus", softplus);
-}
-TVM_REGISTER_OP("relax.nn.softplus")
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor.")
- .set_attrs_type<SoftplusAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeUnaryArith</*require_float_dtype=*/true>)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.nn.softplus")
+ .add_arg("data", "The input tensor.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .attrs_type<SoftplusAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeUnaryArith</*require_float_dtype=*/true>)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.prelu */
Expr prelu(Expr data, Expr alpha, int axis = 1) {
auto attrs = ffi::make_object<PReluAttrs>();
attrs->axis = axis;
- static const Op& op = Op::Get("relax.nn.prelu");
+ static const Op op = Op::Get("relax.nn.prelu");
return Call(Type::Missing(), op, {data, alpha}, Attrs(attrs), {});
}
@@ -159,21 +230,22 @@ InferLayoutOutput InferLayoutPRelu(
return InferLayoutOutput({layout, alpha_layout}, {layout}, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.nn.prelu")
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "The input tensor.")
- .add_argument("alpha", "Tensor", "The channel-wise learnable slope.")
- .set_attrs_type<PReluAttrs>()
- .set_attr<FInferType>("FInferType", InferTypePRelu)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPRelu)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.prelu")
+ .add_arg("data", "The input tensor.")
+ .add_arg("alpha", "The channel-wise learnable slope.")
+ .attrs_type<PReluAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypePRelu)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPRelu)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.softmax */
Expr softmax(Expr data, int axis) {
auto attrs = ffi::make_object<SoftmaxAttrs>();
attrs->axis = axis;
- static const Op& op = Op::Get("relax.nn.softmax");
+ static const Op op = Op::Get("relax.nn.softmax");
return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
}
@@ -225,33 +297,33 @@ InferLayoutOutput InferLayoutSoftmax(
return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.nn.softmax")
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor.")
- .set_attrs_type<SoftmaxAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeSoftmax)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutSoftmax)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.softmax")
+ .add_arg("data", "The input tensor.")
+ .attrs_type<SoftmaxAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeSoftmax)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutSoftmax)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.log_softmax */
Expr log_softmax(Expr data, int axis) {
auto attrs = ffi::make_object<SoftmaxAttrs>();
attrs->axis = axis;
- static const Op& op = Op::Get("relax.nn.log_softmax");
+ static const Op op = Op::Get("relax.nn.log_softmax");
return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.nn.log_softmax", log_softmax);
-}
-TVM_REGISTER_OP("relax.nn.log_softmax")
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor.")
- .set_attrs_type<SoftmaxAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeSoftmax)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.nn.log_softmax")
+ .add_arg("data", "The input tensor.")
+ .attrs_type<SoftmaxAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeSoftmax)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.pad */
@@ -260,7 +332,7 @@ Expr pad(Expr data, ffi::Array<int64_t> pad_width, ffi::String pad_mode, double
attrs->pad_width = std::move(pad_width);
attrs->pad_mode = std::move(pad_mode);
attrs->pad_value = pad_value;
- static const Op& op = Op::Get("relax.nn.pad");
+ static const Op op = Op::Get("relax.nn.pad");
return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
}
@@ -293,19 +365,20 @@ Type InferTypePad(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr(out_shape), input_ty[0]->dtype);
}
-TVM_REGISTER_OP("relax.nn.pad")
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor.")
- .set_attrs_type<PadAttrs>()
- .set_attr<FInferType>("FInferType", InferTypePad)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.pad")
+ .add_arg("data", "The input tensor.")
+ .attrs_type<PadAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypePad)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.pixel_shuffle */
Expr pixel_shuffle(Expr data, int upscale_factor) {
auto attrs = ffi::make_object<PixelShuffleAttrs>();
attrs->upscale_factor = upscale_factor;
- static const Op& op = Op::Get("relax.nn.pixel_shuffle");
+ static const Op op = Op::Get("relax.nn.pixel_shuffle");
return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
}
@@ -365,18 +438,19 @@ Type InferTypePixelShuffle(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr(out_shape), input->dtype);
}
-TVM_REGISTER_OP("relax.nn.pixel_shuffle")
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor.")
- .set_attrs_type<PixelShuffleAttrs>()
- .set_attr<FInferType>("FInferType", InferTypePixelShuffle)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.pixel_shuffle")
+ .add_arg("data", "The input tensor.")
+ .attrs_type<PixelShuffleAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypePixelShuffle)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.batchnorm */
bool NormCheckDtypeAndShape(const Call& call, const BlockBuilder& ctx,
const ffi::Array<TensorType>& input_ty, ffi::Array<int64_t> axes) {
Op op = call->op.as_or_throw<Op>();
- int n_input = op->arguments.size();
+ int n_input = op->args_info.size();
TensorType data_ty = input_ty[0];
@@ -398,11 +472,11 @@ bool NormCheckDtypeAndShape(const Call& call, const BlockBuilder& ctx,
if (input_ty[i]->dtype != data_ty->dtype) {
TVM_FFI_VISIT_THROW(TypeError, call)
<< op << " requires all the input tensors to have the same dtype. However, the "
- << op->arguments[i]->name << " has dtype " << input_ty[i]->dtype
+ << op->args_info[i]->name << " has dtype " << input_ty[i]->dtype
<< " which is other than the input data's dtype " << data_ty->dtype;
} else if (input_ty[i]->ndim != n_axis) {
TVM_FFI_VISIT_THROW(ValueError, call)
- << op << " requires the input " << op->arguments[i]->name
+ << op << " requires the input " << op->args_info[i]->name
<< " to have as many dimensions as the length of input axes. However, the "
"given one has ndim "
<< input_ty[i]->ndim << ", which is other than the length of axes " << n_axis;
@@ -454,7 +528,7 @@ Expr batch_norm(Expr data, Expr gamma, Expr beta, Expr moving_mean, Expr moving_
attrs->momentum = momentum;
attrs->training = training;
- static const Op& op = Op::Get("relax.nn.batch_norm");
+ static const Op op = Op::Get("relax.nn.batch_norm");
return Call(Type::Missing(), op,
{std::move(data), std::move(gamma), std::move(beta), std::move(moving_mean),
std::move(moving_var)},
@@ -512,17 +586,18 @@ InferLayoutOutput InferLayoutBatchNorm(
{{layout, initial_layouts[3], initial_layouts[4]}}, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.nn.batch_norm")
- .set_attrs_type<BatchNormAttrs>()
- .set_num_inputs(5)
- .add_argument("data", "Tensor", "Input to which batch_norm will be applied.")
- .add_argument("gamma", "Tensor", "The gamma scale factor.")
- .add_argument("beta", "Tensor", "The beta offset factor.")
- .add_argument("moving_mean", "Tensor", "Running mean of input.")
- .add_argument("moving_var", "Tensor", "Running variance of input.")
- .set_attr<FInferType>("FInferType", InferTypeBatchNorm)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutBatchNorm)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.batch_norm")
+ .add_arg("data", "Input to which batch_norm will be applied.")
+ .add_arg("gamma", "The gamma scale factor.")
+ .add_arg("beta", "The beta offset factor.")
+ .add_arg("moving_mean", "Running mean of input.")
+ .add_arg("moving_var", "Running variance of input.")
+ .attrs_type<BatchNormAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeBatchNorm)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutBatchNorm)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.layer_norm */
@@ -534,7 +609,7 @@ Expr layer_norm(Expr data, Expr gamma, Expr beta, ffi::Array<int64_t> axes, doub
attrs->center = center;
attrs->scale = scale;
- static const Op& op = Op::Get("relax.nn.layer_norm");
+ static const Op op = Op::Get("relax.nn.layer_norm");
return Call(Type::Missing(), op, {std::move(data), std::move(gamma), std::move(beta)},
Attrs{attrs}, {});
}
@@ -581,16 +656,17 @@ InferLayoutOutput InferLayoutLayerNorm(
Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.nn.layer_norm")
- .set_attrs_type<LayerNormAttrs>()
- .set_num_inputs(3)
- .add_argument("data", "Tensor", "Input to which layer_norm will be applied.")
- .add_argument("gamma", "Tensor", "The gamma scale factor.")
- .add_argument("beta", "Tensor", "The beta offset factor.")
- .set_attr<FInferType>("FInferType", InferTypeLayerNorm)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutLayerNorm)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.layer_norm")
+ .add_arg("data", "Input to which layer_norm will be applied.")
+ .add_arg("gamma", "The gamma scale factor.")
+ .add_arg("beta", "The beta offset factor.")
+ .attrs_type<LayerNormAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeLayerNorm)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutLayerNorm)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.group_norm */
@@ -604,7 +680,7 @@ Expr group_norm(Expr data, Expr gamma, Expr beta, int num_groups, int channel_ax
attrs->center = center;
attrs->scale = scale;
- static const Op& op = Op::Get("relax.nn.group_norm");
+ static const Op op = Op::Get("relax.nn.group_norm");
return Call(Type::Missing(), op, {std::move(data), std::move(gamma), std::move(beta)},
Attrs{attrs}, {});
}
@@ -644,7 +720,7 @@ Type InferTypeGroupNorm(const Call& call, const BlockBuilder& ctx) {
<< op << " expects that the size of channel_axis must be divisible by " << attrs->num_groups
<< ", but got " << data_shape->values[channel_axis];
}
- for (int i = 1; i < static_cast<int>(op->arguments.size()); ++i) {
+ for (int i = 1; i < static_cast<int>(op->args_info.size()); ++i) {
if (input_ty[i]->dtype != data_ty->dtype) {
TVM_FFI_VISIT_THROW(TypeError, call)
<< op << " expects that all inputs must have the same dtype, but got "
@@ -695,16 +771,17 @@ InferLayoutOutput InferLayoutGroupNorm(
Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.nn.group_norm")
- .set_attrs_type<GroupNormAttrs>()
- .set_num_inputs(3)
- .add_argument("data", "Tensor", "Input to which group_norm will be applied.")
- .add_argument("gamma", "Tensor", "The gamma scale factor.")
- .add_argument("beta", "Tensor", "The beta offset factor.")
- .set_attr<FInferType>("FInferType", InferTypeGroupNorm)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutGroupNorm)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.group_norm")
+ .add_arg("data", "Input to which group_norm will be applied.")
+ .add_arg("gamma", "The gamma scale factor.")
+ .add_arg("beta", "The beta offset factor.")
+ .attrs_type<GroupNormAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeGroupNorm)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutGroupNorm)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.instance_norm */
@@ -717,7 +794,7 @@ Expr instance_norm(Expr data, Expr gamma, Expr beta, int channel_axis, ffi::Arra
attrs->center = center;
attrs->scale = scale;
- static const Op& op = Op::Get("relax.nn.instance_norm");
+ static const Op op = Op::Get("relax.nn.instance_norm");
return Call(Type::Missing(), op, {std::move(data), std::move(gamma), std::move(beta)},
Attrs{attrs}, {});
}
@@ -747,7 +824,7 @@ Type InferTypeInstanceNorm(const Call& call, const BlockBuilder& ctx) {
}
const auto* data_shape = data_ty->shape.as<ShapeExprNode>();
sym::Analyzer analyzer = ctx->GetAnalyzer();
- for (int i = 1; i < static_cast<int>(op->arguments.size()); ++i) {
+ for (int i = 1; i < static_cast<int>(op->args_info.size()); ++i) {
if (input_ty[i]->dtype != data_ty->dtype) {
TVM_FFI_VISIT_THROW(TypeError, call)
<< op << " expects that all inputs must have the same dtype, but got "
@@ -797,16 +874,17 @@ InferLayoutOutput InferLayoutInstanceNorm(
Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.nn.instance_norm")
- .set_attrs_type<InstanceNormAttrs>()
- .set_num_inputs(3)
- .add_argument("data", "Tensor", "Input to which instance_norm will be applied.")
- .add_argument("gamma", "Tensor", "The gamma scale factor.")
- .add_argument("beta", "Tensor", "The beta offset factor.")
- .set_attr<FInferType>("FInferType", InferTypeInstanceNorm)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutInstanceNorm)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.instance_norm")
+ .add_arg("data", "Input to which instance_norm will be applied.")
+ .add_arg("gamma", "The gamma scale factor.")
+ .add_arg("beta", "The beta offset factor.")
+ .attrs_type<InstanceNormAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeInstanceNorm)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutInstanceNorm)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.rms_norm */
Expr rms_norm(Expr data, Expr weight, ffi::Array<int64_t> axes, double epsilon) {
@@ -814,7 +892,7 @@ Expr rms_norm(Expr data, Expr weight, ffi::Array<int64_t> axes, double epsilon)
attrs->axes = std::move(axes);
attrs->epsilon = epsilon;
- static const Op& op = Op::Get("relax.nn.rms_norm");
+ static const Op op = Op::Get("relax.nn.rms_norm");
return Call(Type::Missing(), op, {std::move(data), std::move(weight)}, Attrs{attrs}, {});
}
@@ -857,15 +935,16 @@ InferLayoutOutput InferLayoutRMSNorm(
return InferLayoutOutput({layout, initial_layouts[1]}, {layout}, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.nn.rms_norm")
- .set_attrs_type<RMSNormAttrs>()
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "Input to which rms_norm will be applied.")
- .add_argument("weight", "Tensor", "The scale factor.")
- .set_attr<FInferType>("FInferType", InferTypeRMSNorm)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutRMSNorm)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.rms_norm")
+ .add_arg("data", "Input to which rms_norm will be applied.")
+ .add_arg("weight", "The scale factor.")
+ .attrs_type<RMSNormAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeRMSNorm)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutRMSNorm)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.dropout */
@@ -873,7 +952,7 @@ Expr dropout(Expr data, double rate) {
ffi::ObjectPtr<DropoutAttrs> attrs = ffi::make_object<DropoutAttrs>();
attrs->rate = rate;
- static const Op& op = Op::Get("relax.nn.dropout");
+ static const Op op = Op::Get("relax.nn.dropout");
return Call(Type::Missing(), op, {std::move(data)}, Attrs{attrs}, {});
}
@@ -887,14 +966,15 @@ Type InferTypeDropout(const Call& call, const BlockBuilder& ctx) {
return TupleType({data_ty, data_ty});
}
-TVM_REGISTER_OP("relax.nn.dropout")
- .set_attrs_type<DropoutAttrs>()
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "Input to which dropout will be applied.")
- .set_attr<FInferType>("FInferType", InferTypeDropout)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.dropout")
+ .add_arg("data", "Input to which dropout will be applied.")
+ .attrs_type<DropoutAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeDropout)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.cross_entropy_with_logits */
Type InferTypeCrossEntropy(const Call& call, const BlockBuilder& ctx) {
@@ -942,21 +1022,21 @@ Type InferTypeCrossEntropy(const Call& call, const BlockBuilder& ctx) {
}
Expr cross_entropy_with_logits(Expr predictions, Expr labels) {
- static const Op& op = Op::Get("relax.nn.cross_entropy_with_logits");
+ static const Op op = Op::Get("relax.nn.cross_entropy_with_logits");
return Call(Type::Missing(), op, {std::move(predictions), std::move(labels)}, {}, {});
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.nn.cross_entropy_with_logits", cross_entropy_with_logits);
-}
-TVM_REGISTER_OP("relax.nn.cross_entropy_with_logits")
- .set_num_inputs(2)
- .add_argument("predictions", "Tensor", "The predictions.")
- .add_argument("labels", "Tensor", "The labels.")
- .set_attr<FInferType>("FInferType", InferTypeCrossEntropy)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.nn.cross_entropy_with_logits")
+ .add_arg("predictions", "The predictions.")
+ .add_arg("labels", "The labels.")
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .set_attr<FInferType>("FInferType", InferTypeCrossEntropy)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.nll_loss */
@@ -972,7 +1052,7 @@ Expr nll_loss(Expr predictions, Expr targets, ffi::Optional<Expr> weights, ffi::
attrs->reduction = std::move(reduction);
attrs->ignore_index = ignore_index;
- static const Op& op = Op::Get("relax.nn.nll_loss");
+ static const Op op = Op::Get("relax.nn.nll_loss");
if (weights.has_value()) {
return Call(Type::Missing(), op, {std::move(predictions), std::move(targets), weights.value()},
Attrs{attrs}, {});
@@ -1177,19 +1257,21 @@ Type InferTypeNLLLoss(const Call& call, const BlockBuilder& ctx) {
}
}
-TVM_REGISTER_OP("relax.nn.nll_loss")
- .set_attrs_type<NLLLossAttrs>()
- .set_num_inputs(3)
- .add_argument("predictions", "Tensor", "The prediction tensor.")
- .add_argument("targets", "Tensor", "The target tensor.")
- .add_argument("weights", "ffi::Optional<Tensor>", "The weight of each target values.")
- .set_attr<FInferType>("FInferType", InferTypeNLLLoss)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.nll_loss", "Optional weights: The weight of each target values.")
+ .add_arg("predictions", "The prediction tensor.")
+ .add_arg("targets", "The target tensor.")
+ .allow_extra_args()
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+ .attrs_type<NLLLossAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeNLLLoss)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.batch_flatten */
Expr batch_flatten(Expr data) {
- static const Op& op = Op::Get("relax.nn.batch_flatten");
+ static const Op op = Op::Get("relax.nn.batch_flatten");
return Call(Type::Missing(), op, {std::move(data)}, {}, {});
}
@@ -1229,12 +1311,13 @@ Type InferTypeBatchFlatten(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr({batch_dim, flat_dim}), data_ty->dtype, data_ty->vdevice);
}
-TVM_REGISTER_OP("relax.nn.batch_flatten")
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor.")
- .set_attr<FInferType>("FInferType", InferTypeBatchFlatten)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.batch_flatten")
+ .add_arg("data", "The input tensor.")
+ .set_attr<FInferType>("FInferType", InferTypeBatchFlatten)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/nn/nn.h b/src/relax/op/nn/nn.h
index 068a1462e9..fd36277512 100644
--- a/src/relax/op/nn/nn.h
+++ b/src/relax/op/nn/nn.h
@@ -32,19 +32,6 @@
namespace tvm {
namespace relax {
-/*!
- * \brief Quick helper macro to
- * - expose a make-function interface which construct the call node.
- * - register op to the registry.
- * \param OpName The name of operator to register.
- * \param OpRegName The identifier of the operator in the registry.
- * \param RequireFloatDtype A boolean indicating if the input is required to have float dtype.
- */
-#define RELAX_REGISTER_UNARY_NN_OP_AND_IMPL(OpName, OpRegName, RequireFloatDtype) \
- RELAX_UNARY_OP_INTERFACE(OpName, OpRegName) \
- RELAX_REGISTER_UNARY_OP(OpRegName).set_attr<FInferType>("FInferType", \
- InferTypeUnaryArith<RequireFloatDtype>)
-
/*! \brief Rectified linear unit. */
Expr relu(Expr data);
diff --git a/src/relax/op/nn/pooling.cc b/src/relax/op/nn/pooling.cc
index 50a45d53d0..45f5be6b06 100644
--- a/src/relax/op/nn/pooling.cc
+++ b/src/relax/op/nn/pooling.cc
@@ -64,7 +64,7 @@ Expr MakePool1d(ffi::String op_name, Expr data, ffi::Array<int64_t> pool_size,
attrs->count_include_pad = count_include_pad;
attrs->layout = layout;
attrs->out_layout = out_layout.value_or(layout);
- const Op& op = Op::Get(op_name);
+ const Op op = Op::Get(op_name);
return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
}
@@ -144,14 +144,15 @@ InferLayoutOutput InferLayoutPool1d(
return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.nn.max_pool1d")
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor")
- .set_attrs_type<Pool1DAttrs>()
- .set_attr<FInferType>("FInferType", InferTypePool1D)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool1d)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.max_pool1d")
+ .add_arg("data", "The input tensor")
+ .attrs_type<Pool1DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypePool1D)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool1d)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.max_pool2d */
@@ -188,7 +189,7 @@ Expr MakePool2d(ffi::String op_name, Expr data, ffi::Array<int64_t> pool_size,
attrs->count_include_pad = count_include_pad;
attrs->layout = layout;
attrs->out_layout = out_layout.value_or(layout);
- const Op& op = Op::Get(op_name);
+ const Op op = Op::Get(op_name);
return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
}
@@ -298,14 +299,15 @@ InferLayoutOutput InferLayoutPool2d(
return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.nn.max_pool2d")
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor")
- .set_attrs_type<Pool2DAttrs>()
- .set_attr<FInferType>("FInferType", InferTypePool2D)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool2d)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.max_pool2d")
+ .add_arg("data", "The input tensor")
+ .attrs_type<Pool2DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypePool2D)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool2d)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.max_pool3d */
@@ -345,7 +347,7 @@ Expr MakePool3d(ffi::String op_name, Expr data, ffi::Array<int64_t> pool_size,
attrs->count_include_pad = count_include_pad;
attrs->layout = layout;
attrs->out_layout = out_layout.value_or(layout);
- const Op& op = Op::Get(op_name);
+ const Op op = Op::Get(op_name);
return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
}
@@ -447,14 +449,15 @@ InferLayoutOutput InferLayoutPool3d(
return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.nn.max_pool3d")
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor")
- .set_attrs_type<Pool3DAttrs>()
- .set_attr<FInferType>("FInferType", InferTypePool3D)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool3d)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.max_pool3d")
+ .add_arg("data", "The input tensor")
+ .attrs_type<Pool3DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypePool3D)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool3d)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.avg_pool1d */
Expr avg_pool1d(Expr data, ffi::Array<int64_t> pool_size, ffi::Array<int64_t> strides,
@@ -467,16 +470,15 @@ Expr avg_pool1d(Expr data, ffi::Array<int64_t> pool_size, ffi::Array<int64_t> st
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.nn.avg_pool1d", avg_pool1d);
-}
-TVM_REGISTER_OP("relax.nn.avg_pool1d")
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor")
- .set_attrs_type<Pool1DAttrs>()
- .set_attr<FInferType>("FInferType", InferTypePool1D)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool1d)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.nn.avg_pool1d")
+ .add_arg("data", "The input tensor")
+ .attrs_type<Pool1DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypePool1D)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool1d)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.avg_pool2d */
Expr avg_pool2d(Expr data, ffi::Array<int64_t> pool_size, ffi::Array<int64_t> strides,
@@ -489,16 +491,15 @@ Expr avg_pool2d(Expr data, ffi::Array<int64_t> pool_size, ffi::Array<int64_t> st
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.nn.avg_pool2d", avg_pool2d);
-}
-TVM_REGISTER_OP("relax.nn.avg_pool2d")
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor")
- .set_attrs_type<Pool2DAttrs>()
- .set_attr<FInferType>("FInferType", InferTypePool2D)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool2d)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.nn.avg_pool2d")
+ .add_arg("data", "The input tensor")
+ .attrs_type<Pool2DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypePool2D)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool2d)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.avg_pool3d */
Expr avg_pool3d(Expr data, ffi::Array<int64_t> pool_size, ffi::Array<int64_t> strides,
@@ -511,16 +512,15 @@ Expr avg_pool3d(Expr data, ffi::Array<int64_t> pool_size, ffi::Array<int64_t> st
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.nn.avg_pool3d", avg_pool3d);
-}
-TVM_REGISTER_OP("relax.nn.avg_pool3d")
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor")
- .set_attrs_type<Pool3DAttrs>()
- .set_attr<FInferType>("FInferType", InferTypePool3D)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool3d)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.nn.avg_pool3d")
+ .add_arg("data", "The input tensor")
+ .attrs_type<Pool3DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypePool3D)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool3d)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.adaptive_avg_pool1d */
@@ -537,7 +537,7 @@ Expr adaptive_avg_pool1d(Expr data, ffi::Optional<ffi::Array<int64_t>> output_si
attrs->output_size = std::move(_output_size);
}
- static const Op& op = Op::Get("relax.nn.adaptive_avg_pool1d");
+ static const Op op = Op::Get("relax.nn.adaptive_avg_pool1d");
return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
}
@@ -595,14 +595,15 @@ InferLayoutOutput InferLayoutAdaptiveAvgPool1D(
return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.nn.adaptive_avg_pool1d")
- .set_attrs_type<AdaptivePool1DAttrs>()
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor")
- .set_attr<FInferType>("FInferType", InferTypeAdaptiveAvgPool1D)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutAdaptiveAvgPool1D)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.adaptive_avg_pool1d")
+ .add_arg("data", "The input tensor")
+ .attrs_type<AdaptivePool1DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeAdaptiveAvgPool1D)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutAdaptiveAvgPool1D)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.adaptive_avg_pool2d */
@@ -622,7 +623,7 @@ Expr adaptive_avg_pool2d(Expr data, ffi::Optional<ffi::Array<int64_t>> output_si
attrs->output_size = std::move(_output_size);
}
- static const Op& op = Op::Get("relax.nn.adaptive_avg_pool2d");
+ static const Op op = Op::Get("relax.nn.adaptive_avg_pool2d");
return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
}
@@ -698,14 +699,15 @@ InferLayoutOutput InferLayoutAdaptiveAvgPool2D(
return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.nn.adaptive_avg_pool2d")
- .set_attrs_type<AdaptivePool2DAttrs>()
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor")
- .set_attr<FInferType>("FInferType", InferTypeAdaptiveAvgPool2D)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutAdaptiveAvgPool2D)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.adaptive_avg_pool2d")
+ .add_arg("data", "The input tensor")
+ .attrs_type<AdaptivePool2DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeAdaptiveAvgPool2D)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutAdaptiveAvgPool2D)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.nn.adaptive_avg_pool3d */
@@ -725,7 +727,7 @@ Expr adaptive_avg_pool3d(Expr data, ffi::Optional<ffi::Array<int64_t>> output_si
attrs->output_size = std::move(_output_size);
}
- static const Op& op = Op::Get("relax.nn.adaptive_avg_pool3d");
+ static const Op op = Op::Get("relax.nn.adaptive_avg_pool3d");
return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
}
@@ -785,14 +787,15 @@ InferLayoutOutput InferLayoutAdaptiveAvgPool3D(
return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.nn.adaptive_avg_pool3d")
- .set_attrs_type<AdaptivePool3DAttrs>()
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor")
- .set_attr<FInferType>("FInferType", InferTypeAdaptiveAvgPool3D)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutAdaptiveAvgPool3D)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.nn.adaptive_avg_pool3d")
+ .add_arg("data", "The input tensor")
+ .attrs_type<AdaptivePool3DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeAdaptiveAvgPool3D)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutAdaptiveAvgPool3D)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/op.cc b/src/relax/op/op.cc
index 4c344c18ca..ed913eee58 100644
--- a/src/relax/op/op.cc
+++ b/src/relax/op/op.cc
@@ -113,17 +113,20 @@ Type InferTypeCallPurePacked(const Call& call, const BlockBuilder& ctx) {
}
}
-TVM_REGISTER_OP("relax.call_pure_packed")
- .set_num_inputs(-1)
- .add_argument("args", "ffi::Array<Expr>",
- "The first argument is the function being called. The rest are the "
- "arguments to that function.")
- .set_attr<FInferType>("FInferType", InferTypeCallPurePacked)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.call_pure_packed")
+ .add_arg("func",
+ "The first argument is the function being called. The rest are the "
+ "arguments to that function.")
+ .allow_extra_args()
+ .add_ty_arg("type_args", "Optional type arguments forwarded to the callee.")
+ .set_attr<FInferType>("FInferType", InferTypeCallPurePacked)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeCallPurePacked(const Expr& callee, ffi::Array<Expr> args, const Attrs& attrs,
ffi::Array<Type> ty_args) {
- static const Op& op = Op::Get("relax.call_pure_packed");
+ static const Op op = Op::Get("relax.call_pure_packed");
ffi::Array<Expr> call_args = {callee};
for (auto arg : args) {
call_args.push_back(arg);
@@ -224,25 +227,28 @@ Type InferTypeCallInplacePacked(const Call& call, const BlockBuilder& ctx) {
return ret;
}
-TVM_REGISTER_OP("relax.call_inplace_packed")
- .set_num_inputs(-1)
- .set_attrs_type<CallInplacePackedAttrs>()
- .add_argument("args", "ffi::Array<Expr>",
- "The first argument is the function being called. The rest are the "
- "arguments to that function.")
- .set_attr<FInferType>("FInferType", InferTypeCallInplacePacked)
- // Warning: considered pure, but it has the potential to create visible effects!
- // This should only be used if it has been *checked* that it is safe (no aliases, in-place
- // arguments will no longer be live) and the user believes the packed func to have no
- // side effects other than modifying the arguments specified as "inplace"
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.call_inplace_packed")
+ .add_arg("func",
+ "The first argument is the function being called. The rest are the "
+ "arguments to that function.")
+ .allow_extra_args()
+ .add_ty_arg("type_args", "Optional type arguments forwarded to the callee.")
+ .attrs_type<CallInplacePackedAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeCallInplacePacked)
+ // Warning: considered pure, but it has the potential to create visible effects!
+ // This should only be used if it has been *checked* that it is safe (no aliases, in-place
+ // arguments will no longer be live) and the user believes the packed func to have no
+ // side effects other than modifying the arguments specified as "inplace"
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeCallInplacePacked(Expr func, ffi::Array<Expr> args, ffi::Array<int64_t> inplace_indices,
ffi::Array<Type> ty_args) {
ffi::ObjectPtr<CallInplacePackedAttrs> attrs = ffi::make_object<CallInplacePackedAttrs>();
attrs->inplace_indices = ffi::Array<int64_t>(inplace_indices.begin(), inplace_indices.end());
- static const Op& op = Op::Get("relax.call_inplace_packed");
+ static const Op op = Op::Get("relax.call_inplace_packed");
ffi::Array<Expr> call_args = {func};
call_args.insert(call_args.end(), args.begin(), args.end());
return Call(Type::Missing(), op, call_args, Attrs(attrs), ty_args);
@@ -511,14 +517,16 @@ void ValidateCallTIR(Call call) {
}
}
-TVM_REGISTER_OP("relax.call_tir")
- .set_num_inputs(2)
- .add_argument("func", "Expr", "The destination-passing-style function.")
- .add_argument("args", "Tuple", "The input arguments.")
- .set_attr<FInferType>("FInferType", InferTypeCallTIR)
- .set_attr<FNormalize>("FNormalize", NormalizeCallTIR)
- .set_attr<FValidate>("FValidate", ValidateCallTIR)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.call_tir")
+ .add_arg("func", "The destination-passing-style function.")
+ .add_arg("args", "The input arguments.")
+ .add_ty_arg("out_type", "The output type.")
+ .set_attr<FInferType>("FInferType", InferTypeCallTIR)
+ .set_attr<FNormalize>("FNormalize", NormalizeCallTIR)
+ .set_attr<FValidate>("FValidate", ValidateCallTIR)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeCallTIR(Expr func, Tuple args, ffi::Array<TensorType> out_ty_list) {
for (const TensorType& ty : out_ty_list) {
@@ -536,26 +544,26 @@ Expr MakeCallTIR(Expr func, Tuple args, ffi::Array<TensorType> out_ty_list) {
out_ty = TupleType({out_ty_list.begin(), out_ty_list.end()});
}
- static const Op& op = Op::Get("relax.call_tir");
+ static const Op op = Op::Get("relax.call_tir");
return Call(Type::Missing(), op, {func, args}, {}, {out_ty});
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.call_tir", MakeCallTIR);
-}
-// call_tir_with_grad
+ // call_tir_with_grad
-TVM_REGISTER_OP("relax.call_tir_with_grad")
- .set_num_inputs(2)
- .set_attrs_type<CallTIRWithGradAttrs>()
- .add_argument("func", "Expr", "The destination-passing-style function.")
- .add_argument("args", "Tuple", "The input arguments.")
- .set_attr<FInferType>("FInferType", InferTypeCallTIR)
- .set_attr<FNormalize>("FNormalize", NormalizeCallTIR)
- .set_attr<FValidate>("FValidate", ValidateCallTIR)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.call_tir_with_grad")
+ .add_arg("func", "The destination-passing-style function.")
+ .add_arg("args", "The input arguments.")
+ .add_ty_arg("out_type", "The output type.")
+ .attrs_type<CallTIRWithGradAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeCallTIR)
+ .set_attr<FNormalize>("FNormalize", NormalizeCallTIR)
+ .set_attr<FValidate>("FValidate", ValidateCallTIR)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeCallTIRWithGrad(Expr func, Tuple args, ffi::Array<TensorType> out_ty_list,
ffi::String te_grad_name, ffi::Map<ffi::String, ffi::Any> te_grad_kwargs) {
@@ -578,7 +586,7 @@ Expr MakeCallTIRWithGrad(Expr func, Tuple args, ffi::Array<TensorType> out_ty_li
attrs->te_grad_name = te_grad_name;
attrs->te_grad_kwargs = te_grad_kwargs;
- static const Op& op = Op::Get("relax.call_tir_with_grad");
+ static const Op op = Op::Get("relax.call_tir_with_grad");
return Call(Type::Missing(), op, {func, args}, Attrs(attrs), {out_ty});
}
@@ -676,18 +684,20 @@ Expr NormalizeCallTIRInPlace(const BlockBuilder& ctx, Call call) {
return call;
}
-TVM_REGISTER_OP("relax.call_tir_inplace")
- .set_num_inputs(2)
- .set_attrs_type<CallTIRInplaceAttrs>()
- .add_argument("func", "Expr", "The destination-passing-style function.")
- .add_argument("args", "Tuple", "The input arguments.")
- .set_attr<FInferType>("FInferType", InferTypeCallTIR)
- .set_attr<FNormalize>("FNormalize", NormalizeCallTIRInPlace)
- .set_attr<FValidate>("FValidate", ValidateCallTIR)
- // Warning: considered pure, but it has the potential to create visible effects!
- // This should only be used if it has been *checked* that it is safe (no aliases, in-place
- // arguments will no longer be live)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.call_tir_inplace")
+ .add_arg("func", "The destination-passing-style function.")
+ .add_arg("args", "The input arguments.")
+ .add_ty_arg("out_type", "The output type.")
+ .attrs_type<CallTIRInplaceAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeCallTIR)
+ .set_attr<FNormalize>("FNormalize", NormalizeCallTIRInPlace)
+ .set_attr<FValidate>("FValidate", ValidateCallTIR)
+ // Warning: considered pure, but it has the potential to create visible effects!
+ // This should only be used if it has been *checked* that it is safe (no aliases, in-place
+ // arguments will no longer be live)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeCallTIRInplace(Expr func, Tuple args, ffi::Array<int64_t> inplace_indices,
ffi::Array<TensorType> out_ty_list) {
@@ -709,7 +719,7 @@ Expr MakeCallTIRInplace(Expr func, Tuple args, ffi::Array<int64_t> inplace_indic
out_ty = TupleType({out_ty_list.begin(), out_ty_list.end()});
}
- static const Op& op = Op::Get("relax.call_tir_inplace");
+ static const Op op = Op::Get("relax.call_tir_inplace");
return Call(Type::Missing(), op, {func, args}, Attrs(attrs), {out_ty});
}
@@ -727,14 +737,16 @@ Type InferTypeCallDPSPacked(const Call& call, const BlockBuilder& ctx) {
return call->ty_args[0];
}
-TVM_REGISTER_OP("relax.call_dps_packed")
- .set_num_inputs(2)
- .add_argument("func", "Expr", "The destination-passing-style function.")
- .add_argument("args", "Tuple", "The input arguments.")
- .set_attr<FInferType>("FInferType", InferTypeCallDPSPacked)
- // technically, an impure op could be used with this, but there is
- // little reason to use DPS with an impure op
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.call_dps_packed")
+ .add_arg("func", "The destination-passing-style function.")
+ .add_arg("args", "The input arguments.")
+ .add_ty_arg("out_type", "The output type.")
+ .set_attr<FInferType>("FInferType", InferTypeCallDPSPacked)
+ // technically, an impure op could be used with this, but there is
+ // little reason to use DPS with an impure op
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeCallDPSPacked(Expr func, Tuple args, ffi::Array<TensorType> out_ty_list) {
for (const TensorType& ty : out_ty_list) {
@@ -752,7 +764,7 @@ Expr MakeCallDPSPacked(Expr func, Tuple args, ffi::Array<TensorType> out_ty_list
out_ty = TupleType({out_ty_list.begin(), out_ty_list.end()});
}
- static const Op& op = Op::Get("relax.call_dps_packed");
+ static const Op op = Op::Get("relax.call_dps_packed");
return Call(Type::Missing(), op, {func, args}, {}, {out_ty});
}
@@ -791,13 +803,15 @@ void ValidateCallPyFunc(Call call) {
<< "nor a variable binding that may be normalized to an in-line tuple.";
}
-TVM_REGISTER_OP("relax.call_py_func")
- .set_num_inputs(2)
- .add_argument("func_name", "StringImm", "The name of the Python function to call.")
- .add_argument("args", "Tuple", "The input arguments.")
- .set_attr<FInferType>("FInferType", InferTypeCallPyFunc)
- .set_attr<FValidate>("FValidate", ValidateCallPyFunc)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.call_py_func")
+ .add_arg("func_name", "The name of the Python function to call.")
+ .add_arg("args", "The input arguments.")
+ .add_ty_arg("out_type", "The output type.")
+ .set_attr<FInferType>("FInferType", InferTypeCallPyFunc)
+ .set_attr<FValidate>("FValidate", ValidateCallPyFunc)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeCallPyFunc(StringImm func_name, Tuple args, ffi::Array<TensorType> out_ty_list) {
for (const TensorType& ty : out_ty_list) {
@@ -815,7 +829,7 @@ Expr MakeCallPyFunc(StringImm func_name, Tuple args, ffi::Array<TensorType> out_
out_ty = TupleType({out_ty_list.begin(), out_ty_list.end()});
}
- static const Op& op = Op::Get("relax.call_py_func");
+ static const Op op = Op::Get("relax.call_py_func");
return Call(Type::Missing(), op, {func_name, args}, {}, {out_ty});
}
@@ -835,49 +849,50 @@ Type InferTypeCallBuiltinWithCtx(const Call& call, const BlockBuilder& ctx) {
}
}
-TVM_REGISTER_OP("relax.call_builtin_with_ctx")
- .set_num_inputs(4)
- .add_argument("func", "Expr", "The builtin packed func.")
- .add_argument("args", "Tuple", "The input arguments.")
- .set_attr<FInferType>("FInferType", InferTypeCallBuiltinWithCtx)
- // Most builtins are pure, but some are not, like `vm.builtin.attention_kv_cache_append`
- .set_attr<bool>("FPurity", false);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.call_builtin_with_ctx")
+ .add_arg("func", "The builtin packed func.")
+ .add_arg("args", "The input arguments.")
+ .add_ty_arg("out_type", "Optional output type; omitted for void.")
+ .set_attr<FInferType>("FInferType", InferTypeCallBuiltinWithCtx)
+ // Most builtins are pure, but some are not, like `vm.builtin.attention_kv_cache_append`
+ .set_attr<bool>("FPurity", false);
+}
Expr MakeCallBuiltinWithCtx(Expr func, Tuple args, ffi::Array<Type> ty_args) {
- static const Op& op = Op::Get("relax.call_builtin_with_ctx");
+ static const Op op = Op::Get("relax.call_builtin_with_ctx");
return Call(Type::Missing(), op, {func, args}, Attrs(), ty_args);
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.call_builtin_with_ctx", MakeCallBuiltinWithCtx);
-}
-TVM_REGISTER_OP("relax.null_value")
- .set_num_inputs(0)
- .set_attr<FInferType>("FInferType", ReturnAnyType)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.null_value")
+ .set_attr<FInferType>("FInferType", ReturnAnyType)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeCallNullValue() {
- static const Op& op = Op::Get("relax.null_value");
+ static const Op op = Op::Get("relax.null_value");
return Call(Type::Missing(), op, {}, {}, {});
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.null_value", MakeCallNullValue);
-}
-// print
+ // print
-TVM_REGISTER_OP("relax.print")
- .set_num_inputs(-1)
- .add_argument("vals", "ffi::Array<Expr>",
- "The first value is Python-style format string to use to print. The others "
- "are values to print")
- .set_attr<FInferType>("FInferType", ReturnVoidType)
- .set_attr<FCallPacked>("FCallPacked", "relax.run.print")
- .set_attr<bool>("FPurity", false);
+ OpDef("relax.print")
+ .add_arg("format",
+ "The first value is Python-style format string to use to print. The others "
+ "are values to print")
+ .allow_extra_args()
+ .set_attr<FInferType>("FInferType", ReturnVoidType)
+ .set_attr<FCallPacked>("FCallPacked", "relax.run.print")
+ .set_attr<bool>("FPurity", false);
+}
Expr MakePrint(ffi::Array<Expr> vals, StringImm format) {
ffi::Array<Expr> params;
@@ -885,7 +900,7 @@ Expr MakePrint(ffi::Array<Expr> vals, StringImm format) {
for (const auto val : vals) {
params.push_back(val);
}
- static const Op& op = Op::Get("relax.print");
+ static const Op op = Op::Get("relax.print");
return Call(Type::Missing(), op, params);
}
@@ -914,18 +929,20 @@ Type InferAssertType(const Call& call, const BlockBuilder& ctx) {
return ReturnVoidType(call, ctx);
}
-TVM_REGISTER_OP("relax.assert_op")
- .set_num_inputs(-1)
- .add_argument("vals", "ffi::Array<Expr>",
- "The first value is used as the assertion condition. The second value is "
- "Python-style format string to use for displaying an error message, if the "
- "assert fails. The others are used as format arguments if there is an error.")
- .set_attr<FInferType>("FInferType", InferAssertType)
- .set_attr<FCallPacked>("FCallPacked", "relax.run.assert_op")
- .set_attr<bool>("FPurity", false);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.assert_op")
+ .add_arg("condition",
+ "The first value is used as the assertion condition. The second value is "
+ "Python-style format string to use for displaying an error message, if the "
+ "assert fails. The others are used as format arguments if there is an error.")
+ .allow_extra_args()
+ .set_attr<FInferType>("FInferType", InferAssertType)
+ .set_attr<FCallPacked>("FCallPacked", "relax.run.assert_op")
+ .set_attr<bool>("FPurity", false);
+}
Expr MakeAssertOp(Expr condition, ffi::Array<Expr> vals, StringImm format) {
- static const Op& op = Op::Get("relax.assert_op");
+ static const Op op = Op::Get("relax.assert_op");
ffi::Array<Expr> args = {condition};
args.push_back(format);
for (auto val : vals) {
@@ -937,19 +954,18 @@ Expr MakeAssertOp(Expr condition, ffi::Array<Expr> vals, StringImm format) {
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.assert_op", MakeAssertOp);
-}
-// make_closure
+ // make_closure
-TVM_REGISTER_OP("relax.make_closure")
- .set_num_inputs(2)
- .add_argument("func", "Expr", "The closure.")
- .add_argument("args", "Tuple", "The captured variables.")
- .set_attr<FInferType>("FInferType", ReturnAnyType)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.make_closure")
+ .add_arg("func", "The closure.")
+ .add_arg("args", "The captured variables.")
+ .set_attr<FInferType>("FInferType", ReturnAnyType)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeClosure(Expr func, Tuple args) {
- static const Op& op = Op::Get("relax.make_closure");
+ static const Op op = Op::Get("relax.make_closure");
return Call(Type::Missing(), op, {func, args}, {}, {});
}
@@ -970,53 +986,54 @@ Type InferTypeInvokeClosure(const Call& call, const BlockBuilder& ctx) {
}
}
-TVM_REGISTER_OP("relax.invoke_closure")
- .set_num_inputs(2)
- .add_argument("closure", "Expr", "The VMClosure.")
- .add_argument("args", "Tuple", "The captured variables.")
- .set_attr<FInferType>("FInferType", InferTypeInvokeClosure)
- // Not all closures are pure. Use invoke_pure_closure for specifying purity
- .set_attr<bool>("FPurity", false);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.invoke_closure")
+ .add_arg("closure", "The VMClosure.")
+ .add_arg("args", "The captured variables.")
+ .add_ty_arg("out_types", "Zero or more output types; multiple entries form a tuple.")
+ .set_attr<FInferType>("FInferType", InferTypeInvokeClosure)
+ // Not all closures are pure. Use invoke_pure_closure for specifying purity
+ .set_attr<bool>("FPurity", false);
+}
Expr InvokeClosure(Expr closure, Tuple args, ffi::Array<Type> ty_args) {
- static const Op& op = Op::Get("relax.invoke_closure");
+ static const Op op = Op::Get("relax.invoke_closure");
return Call(Type::Missing(), op, {closure, args}, {}, ty_args);
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.invoke_closure", InvokeClosure);
-}
-// invoke_pure_closure
+ // invoke_pure_closure
-TVM_REGISTER_OP("relax.invoke_pure_closure")
- .set_num_inputs(2)
- .add_argument("closure", "Expr", "The VMClosure.")
- .add_argument("args", "Tuple", "The captured variables.")
- .set_attr<FInferType>("FInferType", InferTypeInvokeClosure)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.invoke_pure_closure")
+ .add_arg("closure", "The VMClosure.")
+ .add_arg("args", "The captured variables.")
+ .add_ty_arg("out_types", "Zero or more output types; multiple entries form a tuple.")
+ .set_attr<FInferType>("FInferType", InferTypeInvokeClosure)
+ .set_attr<bool>("FPurity", true);
+}
Expr InvokePureClosure(Expr closure, Tuple args, ffi::Array<Type> ty_args) {
- static const Op& op = Op::Get("relax.invoke_pure_closure");
+ static const Op op = Op::Get("relax.invoke_pure_closure");
return Call(Type::Missing(), op, {closure, args}, {}, ty_args);
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.invoke_pure_closure", InvokePureClosure);
-}
-// shape_of
+ // shape_of
-TVM_REGISTER_OP("relax.shape_of")
- .set_num_inputs(1)
- .add_argument("input", "Expr", "The input expression")
- .set_attr<FInferType>("FInferType", InferTypeShapeOf)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.shape_of")
+ .add_arg("input", "The input expression")
+ .set_attr<FInferType>("FInferType", InferTypeShapeOf)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeShapeOf(Expr expr) {
- static const Op& op = Op::Get("relax.shape_of");
+ static const Op op = Op::Get("relax.shape_of");
return Call(Type::Missing(), op, {expr}, {}, {});
}
@@ -1035,14 +1052,15 @@ Type InferTypeSize(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr(ffi::Array<PrimExpr>{}), PrimType::Int(64));
}
-TVM_REGISTER_OP("relax.size")
- .set_num_inputs(1)
- .add_argument("input", "Expr", "The input tensor")
- .set_attr<FInferType>("FInferType", InferTypeSize)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.size")
+ .add_arg("input", "The input tensor")
+ .set_attr<FInferType>("FInferType", InferTypeSize)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeSize(Expr expr) {
- static const Op& op = Op::Get("relax.size");
+ static const Op op = Op::Get("relax.size");
return Call(Type::Missing(), op, {expr}, {}, {});
}
@@ -1072,14 +1090,15 @@ Type ReturnTensorToShapeType(const Call& call, const BlockBuilder& ctx) {
return ShapeType(kUnknownNDim);
}
-TVM_REGISTER_OP("relax.tensor_to_shape")
- .set_num_inputs(1)
- .add_argument("input", "Expr", "The input expression")
- .set_attr<FInferType>("FInferType", ReturnTensorToShapeType)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.tensor_to_shape")
+ .add_arg("input", "The input expression")
+ .set_attr<FInferType>("FInferType", ReturnTensorToShapeType)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeTensorToShape(Expr expr) {
- static const Op& op = Op::Get("relax.tensor_to_shape");
+ static const Op op = Op::Get("relax.tensor_to_shape");
return Call(Type::Missing(), op, {expr}, {}, {});
}
@@ -1098,15 +1117,16 @@ Type ReturnShapeToTensorType(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr({PrimExpr(ndim)}), PrimType::Int(64));
}
-TVM_REGISTER_OP("relax.shape_to_tensor")
- .set_num_inputs(1)
- .add_argument("input", "Expr", "The input expression")
- .set_attr<FInferType>("FInferType", ReturnShapeToTensorType)
- .set_attr<FCallPacked>("FCallPacked", "relax.run.shape_to_tensor")
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.shape_to_tensor")
+ .add_arg("input", "The input expression")
+ .set_attr<FInferType>("FInferType", ReturnShapeToTensorType)
+ .set_attr<FCallPacked>("FCallPacked", "relax.run.shape_to_tensor")
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeShapeToTensor(Expr expr) {
- static const Op& op = Op::Get("relax.shape_to_tensor");
+ static const Op op = Op::Get("relax.shape_to_tensor");
return Call(Type::Missing(), op, {expr}, {}, {});
}
@@ -1139,23 +1159,23 @@ Type InferTypeAllocateTensor(const Call& call, const BlockBuilder& ctx) {
return TensorType(call->args[0], out_dtype);
}
-TVM_REGISTER_OP("relax.builtin.alloc_tensor")
- .set_num_inputs(4)
- .add_argument("shape", "Expr", "The shape of the tensor to allocate.")
- .add_argument("dtype", "DataTypeImm", "The dtype of the tensor to allocate.")
- .add_argument("runtime_device_index", "PrimExpr",
- "The device index indicating on which device the tensor is to be "
- "allocated at runtime. Index -1 is reserved for the host device.")
- .add_argument("storage_scope", "StringImm",
- "The storage scope of the storage to allocate. Default is global.")
- .set_attr<FInferType>("FInferType", InferTypeAllocateTensor)
- // memory allocation isn't considered a "visible effect" as far as purity is concerned
- .set_attr<bool>("FPurity", true)
- .set_attr<bool>("TAllocator", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.builtin.alloc_tensor")
+ .add_arg("shape", "The shape of the tensor to allocate.")
+ .add_arg("dtype", "The dtype of the tensor to allocate.")
+ .add_arg("runtime_device_index",
+ "The device index indicating on which device the tensor is to be "
+ "allocated at runtime. Index -1 is reserved for the host device.")
+ .add_arg("storage_scope", "The storage scope of the storage to allocate. Default is global.")
+ .set_attr<FInferType>("FInferType", InferTypeAllocateTensor)
+ // memory allocation isn't considered a "visible effect" as far as purity is concerned
+ .set_attr<bool>("FPurity", true)
+ .set_attr<bool>("TAllocator", true);
+}
Expr MakeAllocTensor(Expr shape, DataTypeImm dtype, PrimExpr runtime_device_index,
StringImm storage_scope) {
- static const Op& op = Op::Get("relax.builtin.alloc_tensor");
+ static const Op op = Op::Get("relax.builtin.alloc_tensor");
return Call(Type::Missing(), op, {shape, dtype, runtime_device_index, storage_scope}, Attrs(),
{});
}
@@ -1163,28 +1183,26 @@ Expr MakeAllocTensor(Expr shape, DataTypeImm dtype, PrimExpr runtime_device_inde
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.builtin.alloc_tensor", MakeAllocTensor);
-}
-// memory planning alloc_storage
+ // memory planning alloc_storage
-TVM_REGISTER_OP("relax.memory.alloc_storage")
- .set_num_inputs(4)
- .add_argument("total_space", "Expr", "The total space of the storage to allocate.")
- .add_argument(
- "virtual_device_index", "PrimExpr",
- "The virtual device index indicating on which device the storage is to be allocated, "
- "Index -1 is reserved for the host device.")
- .add_argument("storage_scope", "StringImm",
- "The storage scope of the storage to allocate. Default is global.")
- .add_argument("dtype", "DataTypeImm", "The dtype of the tensor to allocate.")
- .set_attr<FInferType>("FInferType", ReturnAnyType)
- // memory allocation isn't considered a "visible effect" as far as purity is concerned
- .set_attr<bool>("FPurity", true)
- .set_attr<bool>("TAllocator", true);
+ OpDef("relax.memory.alloc_storage")
+ .add_arg("total_space", "The total space of the storage to allocate.")
+ .add_arg(
+ "virtual_device_index",
+ "The virtual device index indicating on which device the storage is to be allocated, "
+ "Index -1 is reserved for the host device.")
+ .add_arg("storage_scope", "The storage scope of the storage to allocate. Default is global.")
+ .add_arg("dtype", "The dtype of the tensor to allocate.")
+ .set_attr<FInferType>("FInferType", ReturnAnyType)
+ // memory allocation isn't considered a "visible effect" as far as purity is concerned
+ .set_attr<bool>("FPurity", true)
+ .set_attr<bool>("TAllocator", true);
+}
Expr MakeAllocStorage(Expr size, PrimExpr virtual_device_index, StringImm storage_scope,
DataTypeImm dtype) {
- static const Op& op = Op::Get("relax.memory.alloc_storage");
+ static const Op op = Op::Get("relax.memory.alloc_storage");
return Call(Type::Missing(), op, {size, virtual_device_index, storage_scope, dtype}, Attrs(), {});
}
@@ -1218,23 +1236,24 @@ Type InferTypeMemAllocTensor(const Call& call, const BlockBuilder& ctx) {
return TensorType(call->args[2], out_dtype);
}
-TVM_REGISTER_OP("relax.memory.alloc_tensor")
- .set_num_inputs(5)
- .add_argument("storage", "Expr", "The storage to allocate the tensor to.")
- .add_argument("offset", "PrimExpr", "Storage offset to allocate the tensor.")
- .add_argument("shape", "Expr", "The shape of the tensor to allocate.")
- .add_argument("dtype", "DataTypeImm", "The dtype of the tensor to allocate.")
- .add_argument("runtime_device_index", "PrimExpr",
- "The device index indicating on which device the tensor is to be "
- "allocated at runtime. Index -1 is reserved for the host device.")
- .set_attr<FInferType>("FInferType", InferTypeMemAllocTensor)
- // memory allocation isn't considered a "visible effect" as far as purity is concerned
- .set_attr<bool>("FPurity", true)
- .set_attr<bool>("TAllocator", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.memory.alloc_tensor")
+ .add_arg("storage", "The storage to allocate the tensor to.")
+ .add_arg("offset", "Storage offset to allocate the tensor.")
+ .add_arg("shape", "The shape of the tensor to allocate.")
+ .add_arg("dtype", "The dtype of the tensor to allocate.")
+ .add_arg("runtime_device_index",
+ "The device index indicating on which device the tensor is to be "
+ "allocated at runtime. Index -1 is reserved for the host device.")
+ .set_attr<FInferType>("FInferType", InferTypeMemAllocTensor)
+ // memory allocation isn't considered a "visible effect" as far as purity is concerned
+ .set_attr<bool>("FPurity", true)
+ .set_attr<bool>("TAllocator", true);
+}
Expr MakeMemAllocTensor(Expr storage, PrimExpr offset, Expr shape, DataTypeImm dtype,
PrimExpr virtual_device_index) {
- static const Op& op = Op::Get("relax.memory.alloc_tensor");
+ static const Op op = Op::Get("relax.memory.alloc_tensor");
return Call(Type::Missing(), op, {storage, offset, shape, dtype, virtual_device_index}, Attrs(),
{});
}
@@ -1253,65 +1272,61 @@ TVM_FFI_STATIC_INIT_BLOCK() {
IntImm::Int64(0));
}
});
-}
-// memory planning kill_storage
+ // memory planning kill_storage
-TVM_REGISTER_OP("relax.memory.kill_storage")
- .set_num_inputs(1)
- .add_argument("storage", "Expr", "The storage to be killed.")
- .set_attr<FInferType>("FInferType", ReturnVoidType)
- // We mark this as impure so it wouldn't be removed by "remove_all_unused"
- .set_attr<bool>("FPurity", false);
+ OpDef("relax.memory.kill_storage")
+ .add_arg("storage", "The storage to be killed.")
+ .set_attr<FInferType>("FInferType", ReturnVoidType)
+ // We mark this as impure so it wouldn't be removed by "remove_all_unused"
+ .set_attr<bool>("FPurity", false);
+}
Expr MakeMemKillStorage(Expr storage) {
- static const Op& op = Op::Get("relax.memory.kill_storage");
+ static const Op op = Op::Get("relax.memory.kill_storage");
return Call(Type::Missing(), op, {storage}, {}, {});
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.memory.kill_storage", MakeMemKillStorage);
-}
-// memory planning kill_tensor
+ // memory planning kill_tensor
-TVM_REGISTER_OP("relax.memory.kill_tensor")
- .set_num_inputs(1)
- .add_argument("tensor", "Expr", "The tensor to be killed.")
- .set_attr<FInferType>("FInferType", ReturnVoidType)
- // We mark this as impure so it wouldn't be removed by "remove_all_unused"
- .set_attr<bool>("FPurity", false);
+ OpDef("relax.memory.kill_tensor")
+ .add_arg("tensor", "The tensor to be killed.")
+ .set_attr<FInferType>("FInferType", ReturnVoidType)
+ // We mark this as impure so it wouldn't be removed by "remove_all_unused"
+ .set_attr<bool>("FPurity", false);
+}
Expr MakeMemKillTensor(Expr tensor) {
- static const Op& op = Op::Get("relax.memory.kill_tensor");
+ static const Op op = Op::Get("relax.memory.kill_tensor");
return Call(Type::Missing(), op, {tensor}, {}, {});
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.memory.kill_tensor", MakeMemKillTensor);
-}
-// vm alloc_storage
+ // vm alloc_storage
-TVM_REGISTER_OP("relax.vm.alloc_storage")
- .set_num_inputs(4)
- .add_argument("size", "Expr", "The size of the storage to allocate.")
- .add_argument("dtype", "DataTypeImm", "The dtype of the tensor to allocate.")
- .add_argument("runtime_device_index", "PrimExpr",
- "The device index indicating on which device the tensor is "
- "to be allocated at runtime.")
- .add_argument("storage_scope", "StringImm",
- "The storage scope of the storage to allocate. Default is global.")
- .set_attr<FInferType>("FInferType", ReturnAnyType)
- // memory allocation isn't considered a "visible effect" as far as purity is concerned
- .set_attr<bool>("FPurity", true)
- .set_attr<bool>("TAllocator", true);
+ OpDef("relax.vm.alloc_storage")
+ .add_arg("size", "The size of the storage to allocate.")
+ .add_arg("runtime_device_index",
+ "The device index indicating on which device the tensor is "
+ "to be allocated at runtime.")
+ .add_arg("dtype", "The dtype of the tensor to allocate.")
+ .add_arg("storage_scope", "The storage scope of the storage to allocate. Default is global.")
+ .set_attr<FInferType>("FInferType", ReturnAnyType)
+ // memory allocation isn't considered a "visible effect" as far as purity is concerned
+ .set_attr<bool>("FPurity", true)
+ .set_attr<bool>("TAllocator", true);
+}
Expr MakeVMAllocStorage(Expr size, PrimExpr runtime_device_index, DataTypeImm dtype,
StringImm storage_scope) {
- static const Op& op = Op::Get("relax.vm.alloc_storage");
+ static const Op op = Op::Get("relax.vm.alloc_storage");
return Call(Type::Missing(), op, {size, runtime_device_index, dtype, storage_scope}, Attrs(), {});
}
@@ -1346,23 +1361,24 @@ Type InferTypeVMAllocTensor(const Call& call, const BlockBuilder& ctx) {
return TensorType(out_dtype, kUnknownNDim, vdevice);
}
-TVM_REGISTER_OP("relax.vm.alloc_tensor")
- .set_num_inputs(5)
- .add_argument("storage", "Expr", "The storage to allocate the tensor to.")
- .add_argument("offset", "PrimExpr", "Storage offset to allocate the tensor.")
- .add_argument("shape", "Expr", "The shape of the tensor to allocate.")
- .add_argument("dtype", "DataTypeImm", "The dtype of the tensor to allocate.")
- .add_argument("runtime_device_index", "PrimExpr",
- "The device index indicating on which device the tensor is "
- "to be allocated at runtime.")
- .set_attr<FInferType>("FInferType", InferTypeVMAllocTensor)
- // memory allocation isn't considered a "visible effect" as far as purity is concerned
- .set_attr<bool>("FPurity", true)
- .set_attr<bool>("TAllocator", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.vm.alloc_tensor")
+ .add_arg("storage", "The storage to allocate the tensor to.")
+ .add_arg("offset", "Storage offset to allocate the tensor.")
+ .add_arg("shape", "The shape of the tensor to allocate.")
+ .add_arg("dtype", "The dtype of the tensor to allocate.")
+ .add_arg("runtime_device_index",
+ "The device index indicating on which device the tensor is "
+ "to be allocated at runtime.")
+ .set_attr<FInferType>("FInferType", InferTypeVMAllocTensor)
+ // memory allocation isn't considered a "visible effect" as far as purity is concerned
+ .set_attr<bool>("FPurity", true)
+ .set_attr<bool>("TAllocator", true);
+}
Expr MakeVMAllocTensor(Expr storage, PrimExpr offset, Expr shape, DataTypeImm dtype,
PrimExpr runtime_device_index) {
- static const Op& op = Op::Get("relax.vm.alloc_tensor");
+ static const Op op = Op::Get("relax.vm.alloc_tensor");
return Call(Type::Missing(), op, {storage, offset, shape, dtype, runtime_device_index}, Attrs(),
{});
}
@@ -1378,39 +1394,37 @@ TVM_FFI_STATIC_INIT_BLOCK() {
args[3].cast<DataTypeImm>(), IntImm::Int64(0));
}
});
-}
-// vm kill_object
-TVM_REGISTER_OP("relax.vm.kill_object")
- .set_num_inputs(1)
- .add_argument("obj", "Expr", "The object to be killed.")
- .set_attr<FInferType>("FInferType", ReturnVoidType)
- // We mark this as impure so it wouldn't be removed by "remove_all_unused"
- .set_attr<bool>("FPurity", false);
+ // vm kill_object
+
+ OpDef("relax.vm.kill_object")
+ .add_arg("obj", "The object to be killed.")
+ .set_attr<FInferType>("FInferType", ReturnVoidType)
+ // We mark this as impure so it wouldn't be removed by "remove_all_unused"
+ .set_attr<bool>("FPurity", false);
+}
Expr MakeVMKillObject(Expr obj) {
- static const Op& op = Op::Get("relax.vm.kill_object");
+ static const Op op = Op::Get("relax.vm.kill_object");
return Call(Type::Missing(), op, {std::move(obj)}, Attrs(), {});
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.vm.kill_object", MakeVMKillObject);
-}
-// vm call_tir_dyn
+ // vm call_tir_dyn
-TVM_REGISTER_OP("relax.vm.call_tir_dyn")
- .set_num_inputs(2)
- .add_argument("func", "Expr", "The destination-passing-style function.")
- .add_argument("args", "Tuple",
- "The input arguments (list of tensors and last argument is ShapeExpr)")
- .set_attr<FInferType>("FInferType", ReturnVoidType)
- // "relax.vm.call_tir_dyn" works in an in-place way, which is impure.
- .set_attr<bool>("FPurity", false);
+ OpDef("relax.vm.call_tir_dyn")
+ .add_arg("func", "The destination-passing-style function.")
+ .add_arg("args", "The input arguments (list of tensors and last argument is ShapeExpr)")
+ .set_attr<FInferType>("FInferType", ReturnVoidType)
+ // "relax.vm.call_tir_dyn" works in an in-place way, which is impure.
+ .set_attr<bool>("FPurity", false);
+}
Expr MakeCallTIRDyn(Expr func, Tuple args) {
- static const Op& op = Op::Get("relax.vm.call_tir_dyn");
+ static const Op op = Op::Get("relax.vm.call_tir_dyn");
return Call(Type::Missing(), op, {func, args}, Attrs(), {});
}
@@ -1424,14 +1438,15 @@ Type InferTypeStopLiftParams(const Call& call, const BlockBuilder& ctx) {
return InferTypeUnaryArith<false>(call, ctx);
}
-TVM_REGISTER_OP("relax.builtin.stop_lift_params")
- .set_num_inputs(1)
- .add_argument("x", "Expr", "The input data")
- .set_attr<FInferType>("FInferType", InferTypeStopLiftParams)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.builtin.stop_lift_params")
+ .add_arg("x", "The input data")
+ .set_attr<FInferType>("FInferType", InferTypeStopLiftParams)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeStopLiftParams(Expr x) {
- static const Op& op = Op::Get("relax.builtin.stop_lift_params");
+ static const Op op = Op::Get("relax.builtin.stop_lift_params");
return Call(Type::Missing(), op, {x}, Attrs(), {});
}
@@ -1454,15 +1469,16 @@ Type InferToVDeviceType(const Call& call, const BlockBuilder& ctx) {
return TensorType(data_ty->dtype, data_ty->ndim, vdev, data_ty->span);
}
-TVM_REGISTER_OP("relax.to_vdevice")
- .set_num_inputs(1)
- .set_attrs_type<ToVDeviceAttrs>()
- .add_argument("data", "Expr", "The input expression to be copied")
- .set_attr<FInferType>("FInferType", InferToVDeviceType)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.to_vdevice")
+ .add_arg("data", "The input expression to be copied")
+ .attrs_type<ToVDeviceAttrs>()
+ .set_attr<FInferType>("FInferType", InferToVDeviceType)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeToVDevice(Expr data, VDevice dst_vdev) {
- static const Op& op = Op::Get("relax.to_vdevice");
+ static const Op op = Op::Get("relax.to_vdevice");
ffi::ObjectPtr<ToVDeviceAttrs> attrs = ffi::make_object<ToVDeviceAttrs>();
attrs->dst_vdevice = dst_vdev;
return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
@@ -1482,15 +1498,16 @@ Type InferHintOnDeviceType(const Call& call, const BlockBuilder& ctx) {
return data_ty;
}
-TVM_REGISTER_OP("relax.hint_on_device")
- .set_num_inputs(1)
- .set_attrs_type<HintOnDeviceAttrs>()
- .add_argument("data", "Expr", "The input expression")
- .set_attr<FInferType>("FInferType", InferHintOnDeviceType)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.hint_on_device")
+ .add_arg("data", "The input expression")
+ .attrs_type<HintOnDeviceAttrs>()
+ .set_attr<FInferType>("FInferType", InferHintOnDeviceType)
+ .set_attr<bool>("FPurity", true);
+}
Expr MakeHintOnDevice(Expr data, Device device, ffi::String memory_scope = "global") {
- static const Op& op = Op::Get("relax.hint_on_device");
+ static const Op op = Op::Get("relax.hint_on_device");
ffi::ObjectPtr<HintOnDeviceAttrs> attrs = ffi::make_object<HintOnDeviceAttrs>();
attrs->device_type = static_cast<int32_t>(device.device_type);
attrs->index = device.device_id;
diff --git a/src/relax/op/op_common.cc b/src/relax/op/op_common.cc
index 0c656f9f42..67e23768f7 100644
--- a/src/relax/op/op_common.cc
+++ b/src/relax/op/op_common.cc
@@ -30,7 +30,7 @@ namespace relax {
using namespace tvm::prim;
ffi::Array<Expr> GetCallArgs(const Call& call) {
- static const Op& call_tir_op = Op::Get("relax.call_tir");
+ static const Op call_tir_op = Op::Get("relax.call_tir");
ffi::Array<Expr> args;
if (call->op.same_as(call_tir_op)) {
args = call->args[1].as_or_throw<Tuple>()->fields;
@@ -42,10 +42,12 @@ ffi::Array<Expr> GetCallArgs(const Call& call) {
void CheckNumArguments(const Call& call, const BlockBuilder& ctx) {
Op op = call->op.as_or_throw<Op>();
- int expected_input = op->arguments.size();
- if (static_cast<int>(call->args.size()) != expected_input) {
+ int expected_input = op->args_info.size();
+ if (op->allow_extra_args ? call->args.size() < op->args_info.size()
+ : call->args.size() != op->args_info.size()) {
TVM_FFI_VISIT_THROW(ValueError, call)
- << "Operator " << op << " expects " << expected_input << " arguments"
+ << "Operator " << op << " expects " << (op->allow_extra_args ? "at least " : "")
+ << expected_input << " arguments"
<< ", but was called with " << call->args.size() << " arguments";
}
}
@@ -53,10 +55,11 @@ void CheckNumArguments(const Call& call, const BlockBuilder& ctx) {
TensorType GetInputTensorType(const Call& call, size_t i_arg, const BlockBuilder& ctx) {
Op op = call->op.as_or_throw<Op>();
- TVM_FFI_ICHECK_EQ(op->arguments.size(), call->args.size())
+ TVM_FFI_ICHECK(op->allow_extra_args ? call->args.size() >= op->args_info.size()
+ : call->args.size() == op->args_info.size())
<< "Failure caught by this check "
<< "should have previously been caught by `CheckNumArguments`";
- TVM_FFI_ICHECK_LT(i_arg, op->arguments.size());
+ TVM_FFI_ICHECK_LT(i_arg, call->args.size());
auto arg = call->args[i_arg];
auto ty = GetType(arg);
@@ -65,8 +68,10 @@ TensorType GetInputTensorType(const Call& call, size_t i_arg, const BlockBuilder
return tensor_ty.value();
} else {
TVM_FFI_VISIT_THROW(TypeError, call)
- << "Operator " << op << " requires argument " << i_arg << " (" << op->arguments[i_arg]->name
- << ") to be a tensor. "
+ << "Operator " << op << " requires argument " << i_arg
+ << (i_arg < op->args_info.size() ? " (" + std::string(op->args_info[i_arg]->name) + ")"
+ : "")
+ << " to be a tensor. "
<< "However, the argument " << arg << " is instead of type " << ty;
TVM_FFI_UNREACHABLE();
}
diff --git a/src/relax/op/op_common.h b/src/relax/op/op_common.h
index 953892c1f1..b4a9ac2817 100644
--- a/src/relax/op/op_common.h
+++ b/src/relax/op/op_common.h
@@ -142,50 +142,24 @@ std::tuple<ArgTypes...> GetArgTypeHelper(const Call& call, const Op& op, const B
template <typename... ArgTypes>
std::tuple<ArgTypes...> GetArgType(const Call& call, const BlockBuilder& ctx) {
Op op = call->op.as_or_throw<Op>();
- size_t n_input = op->arguments.size();
+ size_t n_input = op->args_info.size();
- // Unfortunately, because the `.add_argument()` calls in
- // TVM_REGISTER_OP occur during initialization of globals and are
+ // Unfortunately, because the `.add_arg()` calls in
+ // OpDef occur during initialization of globals and are
// not available at compile-time, this cannot be a static_assert.
- TVM_FFI_ICHECK_EQ(n_input, sizeof...(ArgTypes))
- << "Internal error: " << op << " op defines " << n_input
- << " arguments in its TVM_REGISTER_OP() call, "
+ TVM_FFI_ICHECK(op->allow_extra_args ? sizeof...(ArgTypes) >= n_input
+ : sizeof...(ArgTypes) == n_input)
+ << "Internal error: " << op << " op defines " << n_input << " arguments in its OpDef() call, "
<< "but GetArgType was given " << sizeof...(ArgTypes) << " template arguments.";
+ CheckNumArguments(call, ctx);
+ TVM_FFI_ICHECK_GE(call->args.size(), sizeof...(ArgTypes));
return detail::GetArgTypeHelper<ArgTypes...>(call, op, ctx,
std::make_index_sequence<sizeof...(ArgTypes)>());
}
/************ Op registration macro ************/
-/*!
- * \brief Quick helper macro to register the operator to registry
- * \param OpRegName The name of operator to register. The name passed in will
- * be prepended with a prefix "relax." as the identifier string in the operator registry.
- */
-#define RELAX_REGISTER_UNARY_OP(OpRegName) \
- TVM_REGISTER_OP("relax." OpRegName) \
- .set_num_inputs(1) \
- .add_argument("x", "Tensor", "The input tensor.") \
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise) \
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow) \
- .set_attr<bool>("FPurity", true)
-
-/*!
- * \brief Quick helper macro to expose a make-function to construct the operator.
- * \param OpName The name of the operator as well as the make-function name, which will
- * be prepended with a prefix "relax.op." as the FFI identifier string for the make function,
- * \param OpRegName The identifier of the operator in the registry.
- */
-#define RELAX_UNARY_OP_INTERFACE(OpName, OpRegName) \
- Expr OpName(Expr x) { \
- static const Op& op = Op::Get("relax." OpRegName); \
- return Call(Type::Missing(), op, {std::move(x)}, Attrs(), {}); \
- } \
- TVM_FFI_STATIC_INIT_BLOCK() { \
- tvm::ffi::reflection::GlobalDef().def("relax.op." OpRegName, OpName); \
- }
-
/*!
* \brief Infer the type for unary elementwise ops.
* \param call The context Call to the operator.
@@ -231,10 +205,8 @@ inline Type InferTypeUnary(const Call& call, const BlockBuilder& ctx, FType f_co
template <int arg_index>
Type ReturnTypeFromArg(const Call& call, const BlockBuilder& ctx) {
Op op = call->op.as_or_throw<Op>();
- int n_input = op->arguments.size();
- if (static_cast<int>(call->args.size()) != n_input) {
- TVM_FFI_VISIT_THROW(ValueError, call) << op << " op should have " << n_input << " arguments";
- }
+ CheckNumArguments(call, ctx);
+ int n_input = call->args.size();
if (arg_index >= n_input) {
TVM_FFI_VISIT_THROW(IndexError, call)
<< op << " op has only " << n_input << "arguments, but try to get the arg with index "
diff --git a/src/relax/op/tensor/binary.cc b/src/relax/op/tensor/binary.cc
index 7bc15b5184..1337db324e 100644
--- a/src/relax/op/tensor/binary.cc
+++ b/src/relax/op/tensor/binary.cc
@@ -35,12 +35,7 @@ using namespace tvm::prim;
template <typename FType>
Type InferTypeBroadcast(const Call& call, const BlockBuilder& ctx, FType f_compute_out_dtype) {
- Op op = call->op.as_or_throw<Op>();
- size_t n_input = op->arguments.size();
- if (call->args.size() != n_input) {
- TVM_FFI_VISIT_THROW(ValueError, call)
- << call->op << " op should have " << n_input << " arguments";
- }
+ CheckNumArguments(call, ctx);
auto lhs_ty = GetType(call->args[0]);
auto rhs_ty = GetType(call->args[1]);
diff --git a/src/relax/op/tensor/binary.h b/src/relax/op/tensor/binary.h
index a4a728e076..46ec595b26 100644
--- a/src/relax/op/tensor/binary.h
+++ b/src/relax/op/tensor/binary.h
@@ -29,37 +29,31 @@
namespace tvm {
namespace relax {
-/*!
- * \brief Quick helper macro
- * - Expose a make function to construct the node.
- * - Register op to the registry.
- * \param OpName The name of operator to register. The name passed in will
- * 1. be prepended with a prefix "relax.op." as the FFI identifier string for the make function,
- * 2. be prepended with a prefix "relax." as the identifier string in the operator registry.
- */
-#define RELAX_REGISTER_BINARY_OP_AND_IMPL(OpName) \
- Expr OpName(Expr x1, Expr x2) { \
- static const Op& op = Op::Get("relax." #OpName); \
- return Call(Type::Missing(), op, {x1, x2}, Attrs(), {}); \
- } \
- TVM_FFI_STATIC_INIT_BLOCK() { \
- tvm::ffi::reflection::GlobalDef().def("relax.op." #OpName, OpName); \
- } \
- TVM_REGISTER_OP("relax." #OpName) \
- .set_num_inputs(2) \
- .add_argument("x1", "Tensor", "The first input tensor.") \
- .add_argument("x2", "Tensor", "The second input tensor.") \
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutBinaryEwise) \
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow) \
- .set_attr<bool>("FPurity", true)
-
-#define RELAX_REGISTER_BINARY_BROADCAST_OP_AND_IMPL(OpName) \
- RELAX_REGISTER_BINARY_OP_AND_IMPL(OpName).set_attr<FInferType>("FInferType", \
- InferTypeBroadcastArith)
-
-#define RELAX_REGISTER_CMP_OP_AND_IMPL(OpName) \
- RELAX_REGISTER_BINARY_OP_AND_IMPL(OpName).set_attr<FInferType>("FInferType", \
- InferTypeBroadcastCMP)
+/* Register a binary constructor and its Op metadata together. */
+#define RELAX_REGISTER_BINARY_OP_AND_IMPL(OpName, ...) \
+ Expr OpName(Expr x1, Expr x2) { \
+ static const Op op = Op::Get("relax." #OpName); \
+ return Call(Type::Missing(), op, {x1, x2}, Attrs(), {}); \
+ } \
+ TVM_FFI_STATIC_INIT_BLOCK() { \
+ tvm::ffi::reflection::GlobalDef().def("relax.op." #OpName, OpName); \
+ OpDef("relax." #OpName) \
+ .add_arg("x1", "The first input tensor.") \
+ .add_arg("x2", "The second input tensor.") \
+ .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.") \
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutBinaryEwise) \
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", \
+ MixedPrecisionPolicyKind::kFollow) \
+ .set_attr<bool>("FPurity", true) __VA_ARGS__; \
+ }
+
+#define RELAX_REGISTER_BINARY_BROADCAST_OP_AND_IMPL(OpName) \
+ RELAX_REGISTER_BINARY_OP_AND_IMPL(OpName, \
+ .set_attr<FInferType>("FInferType", InferTypeBroadcastArith))
+
+#define RELAX_REGISTER_CMP_OP_AND_IMPL(OpName) \
+ RELAX_REGISTER_BINARY_OP_AND_IMPL(OpName, \
+ .set_attr<FInferType>("FInferType", InferTypeBroadcastCMP))
/***************** Arithmetic operators *****************/
diff --git a/src/relax/op/tensor/create.cc b/src/relax/op/tensor/create.cc
index 9776c1cdcd..72504a2d0d 100644
--- a/src/relax/op/tensor/create.cc
+++ b/src/relax/op/tensor/create.cc
@@ -61,7 +61,7 @@ Expr full(ffi::Variant<Expr, ffi::Array<PrimExpr>> shape, Expr fill_value,
ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
attrs->dtype = dtype;
- static const Op& op = Op::Get("relax.full");
+ static const Op op = Op::Get("relax.full");
return Call(Type::Missing(), op, {std::move(shape_in_expr), std::move(fill_value)}, Attrs(attrs),
{});
}
@@ -95,22 +95,23 @@ Type InferTypeFull(const Call& call, const BlockBuilder& ctx) {
return TensorType(/*shape=*/call->args[0], out_dtype, fill_value_ty->vdevice);
}
-TVM_REGISTER_OP("relax.full")
- .set_attrs_type<InitAttrs>()
- .set_num_inputs(2)
- .add_argument("shape", "Shape", "The shape of the created tensor.")
- .add_argument("fill_value", "Tensor", "The scalar tensor, denoting the value to fill.")
- .set_attr<FInferType>("FInferType", InferTypeFull)
- .set_attr<bool>("RequiresArgumentShapes", false)
- .set_attr<bool>("FDataDependent", true)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.full")
+ .add_arg("shape", "The shape of the created tensor.")
+ .add_arg("fill_value", "The scalar tensor, denoting the value to fill.")
+ .attrs_type<InitAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeFull)
+ .set_attr<bool>("RequiresArgumentShapes", false)
+ .set_attr<bool>("FDataDependent", true)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.full_like */
Expr full_like(Expr x, Expr fill_value, ffi::Optional<DLDataType> dtype) {
ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
attrs->dtype = dtype;
- static const Op& op = Op::Get("relax.full_like");
+ static const Op op = Op::Get("relax.full_like");
return Call(Type::Missing(), op, {std::move(x), std::move(fill_value)}, Attrs(attrs), {});
}
@@ -139,14 +140,15 @@ Type InferTypeFullLike(const Call& call, const BlockBuilder& ctx) {
}
}
-TVM_REGISTER_OP("relax.full_like")
- .set_attrs_type<InitAttrs>()
- .set_num_inputs(2)
- .add_argument("x", "Tensor", "The input tensor.")
- .add_argument("fill_value", "Tensor", "The scalar value to fill.")
- .set_attr<FInferType>("FInferType", InferTypeFullLike)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.full_like")
+ .add_arg("x", "The input tensor.")
+ .add_arg("fill_value", "The scalar value to fill.")
+ .attrs_type<InitAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeFullLike)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
// Structure info inference for ones and zeros
Type InferTypeOnesZeros(const Call& call, const BlockBuilder& ctx) {
@@ -183,85 +185,81 @@ Expr ones(Expr shape, DLDataType dtype) {
ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
attrs->dtype = dtype;
- static const Op& op = Op::Get("relax.ones");
+ static const Op op = Op::Get("relax.ones");
return Call(Type::Missing(), op, {std::move(shape)}, Attrs(attrs), {});
}
Expr ones_like(Expr x, ffi::Optional<DLDataType> dtype) {
ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
attrs->dtype = dtype;
- static const Op& op = Op::Get("relax.ones_like");
+ static const Op op = Op::Get("relax.ones_like");
return Call(Type::Missing(), op, {std::move(x)}, Attrs(attrs), {});
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.ones", ones).def("relax.op.ones_like", ones_like);
-}
-TVM_REGISTER_OP("relax.ones")
- .set_attrs_type<InitAttrs>()
- .set_num_inputs(1)
- .add_argument("shape", "Shape", "The shape of the created tensor.")
- .set_attr<FInferType>("FInferType", InferTypeOnesZeros)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
-
-TVM_REGISTER_OP("relax.ones_like")
- .set_attrs_type<InitAttrs>()
- .set_num_inputs(1)
- .add_argument("x", "Tensor", "The input tensor.")
- .set_attr<FInferType>("FInferType", InferTypeOnesLikeZerosLike)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.ones")
+ .add_arg("shape", "The shape of the created tensor.")
+ .attrs_type<InitAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeOnesZeros)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+
+ OpDef("relax.ones_like")
+ .add_arg("x", "The input tensor.")
+ .attrs_type<InitAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeOnesLikeZerosLike)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.zeros & relax.zeros_like */
Expr zeros(Expr shape, DLDataType dtype) {
ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
attrs->dtype = dtype;
- static const Op& op = Op::Get("relax.zeros");
+ static const Op op = Op::Get("relax.zeros");
return Call(Type::Missing(), op, {std::move(shape)}, Attrs(attrs), {});
}
Expr zeros_like(Expr x, ffi::Optional<DLDataType> dtype) {
ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
attrs->dtype = dtype;
- static const Op& op = Op::Get("relax.zeros_like");
+ static const Op op = Op::Get("relax.zeros_like");
return Call(Type::Missing(), op, {std::move(x)}, Attrs(attrs), {});
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
refl::GlobalDef().def("relax.op.zeros", zeros).def("relax.op.zeros_like", zeros_like);
-}
-TVM_REGISTER_OP("relax.zeros")
- .set_attrs_type<InitAttrs>()
- .set_num_inputs(1)
- .add_argument("shape", "Shape", "The shape of the created tensor.")
- .set_attr<FInferType>("FInferType", InferTypeOnesZeros)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
-
-TVM_REGISTER_OP("relax.zeros_like")
- .set_attrs_type<InitAttrs>()
- .set_num_inputs(1)
- .add_argument("x", "Tensor", "The input tensor.")
- .set_attr<FInferType>("FInferType", InferTypeOnesLikeZerosLike)
- .set_attr<bool>("FPurity", true);
+ OpDef("relax.zeros")
+ .add_arg("shape", "The shape of the created tensor.")
+ .attrs_type<InitAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeOnesZeros)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+
+ OpDef("relax.zeros_like")
+ .add_arg("x", "The input tensor.")
+ .attrs_type<InitAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeOnesLikeZerosLike)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.eye & relax.eye_like */
Expr eye(PrimExpr n, PrimExpr m, PrimExpr k, DLDataType dtype) {
ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
attrs->dtype = dtype;
- static const Op& op = Op::Get("relax.eye");
+ static const Op op = Op::Get("relax.eye");
return Call(Type::Missing(), op, {std::move(n), std::move(m), std::move(k)}, Attrs(attrs), {});
}
Expr eye_like(Expr x, PrimExpr k, ffi::Optional<DLDataType> dtype) {
ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
attrs->dtype = dtype;
- static const Op& op = Op::Get("relax.eye_like");
+ static const Op op = Op::Get("relax.eye_like");
return Call(Type::Missing(), op, {std::move(x), std::move(k)}, Attrs(attrs), {});
}
@@ -321,29 +319,29 @@ Type InferTypeEyeLike(const Call& call, const BlockBuilder& ctx) {
return TensorType(x_ty->shape.value(), out_dtype, x_ty->vdevice);
}
-TVM_REGISTER_OP("relax.eye")
- .set_attrs_type<InitAttrs>()
- .set_num_inputs(3)
- .add_argument("n", "PrimExpr", "Number of rows in the output.")
- .add_argument("m", "PrimExpr", "Number of columns in the output.")
- .add_argument("k", "PrimExpr", "Index of the diagonal.")
- .set_attr<FInferType>("FInferType", InferTypeEye)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
-
-TVM_REGISTER_OP("relax.eye_like")
- .set_attrs_type<InitAttrs>()
- .set_num_inputs(2)
- .add_argument("x", "Tensor", "The input tensor.")
- .add_argument("k", "PrimExpr", "Index of the diagonal.")
- .set_attr<FInferType>("FInferType", InferTypeEyeLike)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.eye")
+ .add_arg("n", "Number of rows in the output.")
+ .add_arg("m", "Number of columns in the output.")
+ .add_arg("k", "Index of the diagonal.")
+ .attrs_type<InitAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeEye)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+
+ OpDef("relax.eye_like")
+ .add_arg("x", "The input tensor.")
+ .add_arg("k", "Index of the diagonal.")
+ .attrs_type<InitAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeEyeLike)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.arange */
Expr arange(PrimExpr start, PrimExpr stop, PrimExpr step, DLDataType dtype) {
ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
attrs->dtype = dtype;
- static const Op& op = Op::Get("relax.arange");
+ static const Op op = Op::Get("relax.arange");
return Call(Type::Missing(), op, {std::move(start), std::move(stop), std::move(step)},
Attrs(attrs), {});
}
@@ -388,22 +386,23 @@ Type InferTypeArange(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr({num_elem}), PrimType(dtype));
}
-TVM_REGISTER_OP("relax.arange")
- .set_attrs_type<InitAttrs>()
- .set_num_inputs(3)
- .add_argument("start", "PrimExpr", "The starting value for the set of points.")
- .add_argument("end", "PrimExpr", "The ending value for the set of points.")
- .add_argument("step", "PrimExpr", "The gap between each pair of adjacent points.")
- .set_attr<FInferType>("FInferType", InferTypeArange)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.arange")
+ .add_arg("start", "The starting value for the set of points.")
+ .add_arg("end", "The ending value for the set of points.")
+ .add_arg("step", "The gap between each pair of adjacent points.")
+ .attrs_type<InitAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeArange)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.hamming_window */
Expr hamming_window(PrimExpr window_size, PrimExpr periodic, PrimExpr alpha, PrimExpr beta,
DLDataType dtype) {
ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
attrs->dtype = dtype;
- static const Op& op = Op::Get("relax.hamming_window");
+ static const Op op = Op::Get("relax.hamming_window");
return Call(Type::Missing(), op,
{std::move(window_size), std::move(periodic), std::move(alpha), std::move(beta)},
Attrs(attrs), {});
@@ -442,28 +441,29 @@ Type InferTypeHammingWindow(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr({window_size}), PrimType(dtype));
}
-TVM_REGISTER_OP("relax.hamming_window")
- .set_attrs_type<InitAttrs>()
- .set_num_inputs(4)
- .add_argument("window_size", "PrimExpr", "The size of the window")
- .add_argument("periodic", "PrimExpr",
- "If True, returns a window to be used as periodic function. If False, return a "
- "symmetric window")
- .add_argument("alpha", "PrimExpr", "The coefficient alpha")
- .add_argument("beta", "PrimExpr", "The coefficient beta")
- .set_attr<FInferType>("FInferType", InferTypeHammingWindow)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.hamming_window")
+ .add_arg("window_size", "The size of the window")
+ .add_arg("periodic",
+ "If True, returns a window to be used as periodic function. If False, return a "
+ "symmetric window")
+ .add_arg("alpha", "The coefficient alpha")
+ .add_arg("beta", "The coefficient beta")
+ .attrs_type<InitAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeHammingWindow)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.tril & relax.triu */
Expr tril(Expr x, Expr k) {
- static const Op& op = Op::Get("relax.tril");
+ static const Op op = Op::Get("relax.tril");
return Call(Type::Missing(), op, {x, k});
}
Expr triu(Expr x, Expr k) {
- static const Op& op = Op::Get("relax.triu");
+ static const Op op = Op::Get("relax.triu");
return Call(Type::Missing(), op, {x, k});
}
@@ -486,19 +486,19 @@ Type InferTypeTrilTriu(const Call& call, const BlockBuilder& ctx) {
return data_ty;
}
-TVM_REGISTER_OP("relax.tril")
- .set_num_inputs(2)
- .add_argument("x", "Tensor", "The input tensor.")
- .add_argument("k", "PrimExpr", "The offset of the diagonal.")
- .set_attr<FInferType>("FInferType", InferTypeTrilTriu)
- .set_attr<bool>("FPurity", true);
-
-TVM_REGISTER_OP("relax.triu")
- .set_num_inputs(2)
- .add_argument("x", "Tensor", "The input tensor.")
- .add_argument("k", "PrimExpr", "The offset of the diagonal.")
- .set_attr<FInferType>("FInferType", InferTypeTrilTriu)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.tril")
+ .add_arg("x", "The input tensor.")
+ .add_arg("k", "The offset of the diagonal.")
+ .set_attr<FInferType>("FInferType", InferTypeTrilTriu)
+ .set_attr<bool>("FPurity", true);
+
+ OpDef("relax.triu")
+ .add_arg("x", "The input tensor.")
+ .add_arg("k", "The offset of the diagonal.")
+ .set_attr<FInferType>("FInferType", InferTypeTrilTriu)
+ .set_attr<bool>("FPurity", true);
+}
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/tensor/datatype.cc b/src/relax/op/tensor/datatype.cc
index 5bb18bcb9f..87efbb5d91 100644
--- a/src/relax/op/tensor/datatype.cc
+++ b/src/relax/op/tensor/datatype.cc
@@ -42,7 +42,7 @@ Expr astype(Expr x, DLDataType dtype) {
ffi::ObjectPtr<AstypeAttrs> attrs = ffi::make_object<AstypeAttrs>();
attrs->dtype = dtype;
- static const Op& op = Op::Get("relax.astype");
+ static const Op op = Op::Get("relax.astype");
return Call(Type::Missing(), op, {std::move(x)}, Attrs(attrs), {});
}
@@ -59,14 +59,15 @@ Type InferTypeAstype(const Call& call, const BlockBuilder& ctx) {
return TensorType(new_ty);
}
-TVM_REGISTER_OP("relax.astype")
- .set_attrs_type<AstypeAttrs>()
- .set_num_inputs(1)
- .add_argument("x", "Tensor", "The input tensor")
- .set_attr<FInferType>("FInferType", InferTypeAstype)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.astype")
+ .add_arg("x", "The input tensor")
+ .attrs_type<AstypeAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeAstype)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.wrap_param */
@@ -74,7 +75,7 @@ Expr MakeWrapParam(Expr data, DLDataType dtype) {
ffi::ObjectPtr<WrapParamAttrs> attrs = ffi::make_object<WrapParamAttrs>();
attrs->dtype = dtype;
- static const Op& op = Op::Get("relax.wrap_param");
+ static const Op op = Op::Get("relax.wrap_param");
return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
}
@@ -91,12 +92,13 @@ Type InferTypeWrapParam(const Call& call, const BlockBuilder& ctx) {
return TensorType(new_ty);
}
-TVM_REGISTER_OP("relax.wrap_param")
- .set_attrs_type<WrapParamAttrs>()
- .set_num_inputs(1)
- .add_argument("data", "Tensor", "The input tensor")
- .set_attr<FInferType>("FInferType", InferTypeWrapParam)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.wrap_param")
+ .add_arg("data", "The input tensor")
+ .attrs_type<WrapParamAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeWrapParam)
+ .set_attr<bool>("FPurity", true);
+}
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/tensor/grad.cc b/src/relax/op/tensor/grad.cc
index aabede652d..8b3e222a47 100644
--- a/src/relax/op/tensor/grad.cc
+++ b/src/relax/op/tensor/grad.cc
@@ -34,7 +34,7 @@ namespace relax {
/* relax.grad.no_grad */
Expr no_grad(Expr input) {
- static const Op& op = Op::Get("relax.grad.no_grad");
+ static const Op op = Op::Get("relax.grad.no_grad");
return Call(Type::Missing(), op, {std::move(input)}, {}, {});
}
@@ -45,15 +45,16 @@ TVM_FFI_STATIC_INIT_BLOCK() {
Type InferTypeNoGrad(const Call& call, const BlockBuilder& ctx) { return GetType(call->args[0]); }
-TVM_REGISTER_OP("relax.grad.no_grad")
- .set_num_inputs(1)
- .add_argument("x", "Expr", "The corresponding input tensor.")
- .set_attr<FInferType>("FInferType", InferTypeNoGrad)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.grad.no_grad")
+ .add_arg("x", "The corresponding input tensor.")
+ .set_attr<FInferType>("FInferType", InferTypeNoGrad)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.grad.start_checkpoint */
Expr start_checkpoint(Expr input) {
- static const Op& op = Op::Get("relax.grad.start_checkpoint");
+ static const Op op = Op::Get("relax.grad.start_checkpoint");
return Call(Type::Missing(), op, {std::move(input)}, {}, {});
}
@@ -70,15 +71,16 @@ Type InferTypeStartCheckpoint(const Call& call, const BlockBuilder& ctx) {
return GetType(call->args[0]);
}
-TVM_REGISTER_OP("relax.grad.start_checkpoint")
- .set_num_inputs(1)
- .add_argument("x", "Expr", "The tensor marking the input of the checkpoint stage.")
- .set_attr<FInferType>("FInferType", InferTypeStartCheckpoint)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.grad.start_checkpoint")
+ .add_arg("x", "The tensor marking the input of the checkpoint stage.")
+ .set_attr<FInferType>("FInferType", InferTypeStartCheckpoint)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.grad.end_checkpoint */
Expr end_checkpoint(Expr input) {
- static const Op& op = Op::Get("relax.grad.end_checkpoint");
+ static const Op op = Op::Get("relax.grad.end_checkpoint");
return Call(Type::Missing(), op, {std::move(input)}, {}, {});
}
@@ -95,11 +97,12 @@ Type InferTypeEndCheckpoint(const Call& call, const BlockBuilder& ctx) {
return GetType(call->args[0]);
}
-TVM_REGISTER_OP("relax.grad.end_checkpoint")
- .set_num_inputs(1)
- .add_argument("x", "Expr", "The output of the checkpoint stage.")
- .set_attr<FInferType>("FInferType", InferTypeEndCheckpoint)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.grad.end_checkpoint")
+ .add_arg("x", "The output of the checkpoint stage.")
+ .set_attr<FInferType>("FInferType", InferTypeEndCheckpoint)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.grad.nll_loss_backward */
Expr nll_loss_backward(Expr output_grad, Expr predictions, Expr targets,
@@ -109,7 +112,7 @@ Expr nll_loss_backward(Expr output_grad, Expr predictions, Expr targets,
attrs->reduction = reduction;
attrs->ignore_index = ignore_index;
- static const Op& op = Op::Get("relax.grad.nll_loss_backward");
+ static const Op op = Op::Get("relax.grad.nll_loss_backward");
if (weights.has_value()) {
return Call(
Type::Missing(), op,
@@ -131,15 +134,16 @@ Type InferTypeNLLLossBackward(const Call& call, const BlockBuilder& ctx) {
return GetType(call->args[1]);
}
-TVM_REGISTER_OP("relax.grad.nll_loss_backward")
- .set_attrs_type<NLLLossAttrs>()
- .set_num_inputs(4)
- .add_argument("output_grad", "Tensor", "The output gradient.")
- .add_argument("predictions", "Tensor", "The prediction tensor.")
- .add_argument("targets", "Tensor", "The target tensor.")
- .add_argument("weights", "ffi::Optional<Tensor>", "The weight of each target values.")
- .set_attr<FInferType>("FInferType", InferTypeNLLLossBackward)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.grad.nll_loss_backward", "Optional weights: The weight of each target values.")
+ .add_arg("output_grad", "The output gradient.")
+ .add_arg("predictions", "The prediction tensor.")
+ .add_arg("targets", "The target tensor.")
+ .allow_extra_args()
+ .attrs_type<NLLLossAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeNLLLossBackward)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.grad.max_pool2d_backward */
Expr max_pool2d_backward(Expr output_grad, Expr data, ffi::Array<int64_t> pool_size,
@@ -155,7 +159,7 @@ Expr max_pool2d_backward(Expr output_grad, Expr data, ffi::Array<int64_t> pool_s
attrs->count_include_pad = count_include_pad;
attrs->layout = layout;
attrs->out_layout = out_layout.value_or(layout);
- static const Op& op = Op::Get("relax.grad.max_pool2d_backward");
+ static const Op op = Op::Get("relax.grad.max_pool2d_backward");
return Call(Type::Missing(), op, {std::move(output_grad), std::move(data)}, Attrs(attrs), {});
}
@@ -168,13 +172,14 @@ Type InferTypeMaxPool2DBackward(const Call& call, const BlockBuilder& ctx) {
return GetType(call->args[1]);
}
-TVM_REGISTER_OP("relax.grad.max_pool2d_backward")
- .set_num_inputs(2)
- .add_argument("output_grad", "Tensor", "The output gradient.")
- .add_argument("data", "Tensor", "The input tensor")
- .set_attrs_type<Pool2DAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeMaxPool2DBackward)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.grad.max_pool2d_backward")
+ .add_arg("output_grad", "The output gradient.")
+ .add_arg("data", "The input tensor")
+ .attrs_type<Pool2DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeMaxPool2DBackward)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.grad.avg_pool2d_backward */
Expr avg_pool2d_backward(Expr output_grad, Expr data, ffi::Array<int64_t> pool_size,
@@ -190,7 +195,7 @@ Expr avg_pool2d_backward(Expr output_grad, Expr data, ffi::Array<int64_t> pool_s
attrs->count_include_pad = count_include_pad;
attrs->layout = layout;
attrs->out_layout = out_layout.value_or(layout);
- static const Op& op = Op::Get("relax.grad.avg_pool2d_backward");
+ static const Op op = Op::Get("relax.grad.avg_pool2d_backward");
return Call(Type::Missing(), op, {std::move(output_grad), std::move(data)}, Attrs(attrs), {});
}
@@ -203,13 +208,14 @@ Type InferTypeAvgPool2DBackward(const Call& call, const BlockBuilder& ctx) {
return GetType(call->args[1]);
}
-TVM_REGISTER_OP("relax.grad.avg_pool2d_backward")
- .set_num_inputs(2)
- .add_argument("output_grad", "Tensor", "The output gradient.")
- .add_argument("data", "Tensor", "The input tensor")
- .set_attrs_type<Pool2DAttrs>()
- .set_attr<FInferType>("FInferType", InferTypeAvgPool2DBackward)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.grad.avg_pool2d_backward")
+ .add_arg("output_grad", "The output gradient.")
+ .add_arg("data", "The input tensor")
+ .attrs_type<Pool2DAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeAvgPool2DBackward)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.grad.take_backward */
@@ -217,7 +223,7 @@ Expr take_backward(Expr output_grad, Expr x, Expr indices, ffi::Optional<int64_t
ffi::ObjectPtr<TakeAttrs> attrs = ffi::make_object<TakeAttrs>();
attrs->axis = std::move(axis);
- static const Op& op = Op::Get("relax.grad.take_backward");
+ static const Op op = Op::Get("relax.grad.take_backward");
return Call(Type::Missing(), op, {std::move(output_grad), std::move(x), std::move(indices)},
Attrs(attrs), {});
}
@@ -231,14 +237,15 @@ Type InferTypeTakeBackward(const Call& call, const BlockBuilder& ctx) {
return GetType(call->args[1]);
}
-TVM_REGISTER_OP("relax.grad.take_backward")
- .set_attrs_type<TakeAttrs>()
- .set_num_inputs(3)
- .add_argument("output_grad", "Tensor", "The output gradient.")
- .add_argument("x", "Tensor", "The source tensor.")
- .add_argument("indices", "Tensor", "The indices of the values to extract.")
- .set_attr<FInferType>("FInferType", InferTypeTakeBackward)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.grad.take_backward")
+ .add_arg("output_grad", "The output gradient.")
+ .add_arg("x", "The source tensor.")
+ .add_arg("indices", "The indices of the values to extract.")
+ .attrs_type<TakeAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeTakeBackward)
+ .set_attr<bool>("FPurity", true);
+}
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/tensor/index.cc b/src/relax/op/tensor/index.cc
index d5aa64ab86..00ea940555 100644
--- a/src/relax/op/tensor/index.cc
+++ b/src/relax/op/tensor/index.cc
@@ -52,7 +52,7 @@ Expr take(Expr x, Expr indices, ffi::Optional<int64_t> axis, ffi::String mode) {
attrs->axis = std::move(axis);
attrs->mode = std::move(mode);
- static const Op& op = Op::Get("relax.take");
+ static const Op op = Op::Get("relax.take");
return Call(Type::Missing(), op, {std::move(x), std::move(indices)}, Attrs(attrs), {});
}
@@ -127,13 +127,14 @@ Type InferTypeTake(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr(output_shape), data_ty->dtype, data_ty->vdevice);
}
-TVM_REGISTER_OP("relax.take")
- .set_attrs_type<TakeAttrs>()
- .set_num_inputs(2)
- .add_argument("x", "Tensor", "The source tensor.")
- .add_argument("indices", "Tensor", "The indices of the values to extract.")
- .set_attr<FInferType>("FInferType", InferTypeTake)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.take")
+ .add_arg("x", "The source tensor.")
+ .add_arg("indices", "The indices of the values to extract.")
+ .attrs_type<TakeAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeTake)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.strided_slice */
@@ -173,7 +174,7 @@ Expr strided_slice(Expr x, Expr axes, Expr begin, Expr end, ffi::Optional<Expr>
args.push_back(strides.value());
}
- static const Op& op = Op::Get("relax.strided_slice");
+ static const Op op = Op::Get("relax.strided_slice");
auto call = Call(Type::Missing(), op, args, Attrs(attrs));
return call;
@@ -486,21 +487,26 @@ InferLayoutOutput InferLayoutStridedSlice(
{{IntImm::Int32(1), relax::Tuple(new_axes)}});
}
-TVM_REGISTER_OP("relax.strided_slice")
- .set_attrs_type<StridedSliceAttrs>()
- .set_num_inputs(1)
- .add_argument("x", "Tensor", "The source tensor to be sliced.")
- .set_attr<FInferType>("FInferType", InferTypeStridedSlice)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutStridedSlice)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.strided_slice")
+ .add_arg("x", "The source tensor to be sliced.")
+ .add_arg("axes", "The axes.")
+ .add_arg("begin", "The start index.")
+ .add_arg("end", "The end index.")
+ .allow_extra_args()
+ .attrs_type<StridedSliceAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeStridedSlice)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutStridedSlice)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.dynamic_strided_slice */
Expr dynamic_strided_slice(Expr x, //
Expr begin, //
Expr end, //
Expr strides) {
- static const Op& op = Op::Get("relax.dynamic_strided_slice");
+ static const Op op = Op::Get("relax.dynamic_strided_slice");
return Call(Type::Missing(), op,
{std::move(x), std::move(begin), std::move(end), std::move(strides)}, {});
}
@@ -583,17 +589,18 @@ InferLayoutOutput InferLayoutDynStridedSlice(
return InferLayoutOutput({initial}, {initial}, Attrs());
}
-TVM_REGISTER_OP("relax.dynamic_strided_slice")
- .set_num_inputs(4)
- .add_argument("x", "Tensor", "The source tensor to be sliced.")
- .add_argument("begin", "Tensor", "The indices to begin with in the slicing.")
- .add_argument("end", "Tensor", "Indices indicating end of the slice.")
- .add_argument("strides", "Tensor", "The stride values.")
- .set_attr<FInferType>("FInferType", InferTypeDynStridedSlice)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutDynStridedSlice)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true)
- .set_attr<bool>("FDataDependent", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.dynamic_strided_slice")
+ .add_arg("x", "The source tensor to be sliced.")
+ .add_arg("begin", "The indices to begin with in the slicing.")
+ .add_arg("end", "Indices indicating end of the slice.")
+ .add_arg("strides", "The stride values.")
+ .set_attr<FInferType>("FInferType", InferTypeDynStridedSlice)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutDynStridedSlice)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true)
+ .set_attr<bool>("FDataDependent", true);
+}
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/tensor/inspect.cc b/src/relax/op/tensor/inspect.cc
index 73886a51f5..49bf821dec 100644
--- a/src/relax/op/tensor/inspect.cc
+++ b/src/relax/op/tensor/inspect.cc
@@ -117,7 +117,7 @@ Expr NormalizeToKnownPrimExpr(const BlockBuilder&, Call call) { return call; }
//// relax.tensor_dtype_code
Expr tensor_dtype_code(Expr expr) {
- static const Op& op = Op::Get("relax.inspect.tensor_dtype_code");
+ static const Op op = Op::Get("relax.inspect.tensor_dtype_code");
return Call(Type::Missing(), op, {expr});
}
@@ -134,19 +134,20 @@ Expr LegalizeTensorDtypeCode(const BlockBuilder& bb, const Call& call) {
return Call(Type::Missing(), gvar_getter, {arg});
}
-TVM_REGISTER_OP("relax.inspect.tensor_dtype_code")
- .set_num_inputs(1)
- .add_argument("tensor", "Tensor", "The tensor to be inspected")
- .set_attr<FInferType>("FInferType", InferTypeTensorDtypeCode)
- .set_attr<FLegalize>("FLegalize", LegalizeTensorDtypeCode)
- .set_attr<bool>("RequiresArgumentShapes", false)
- .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.inspect.tensor_dtype_code")
+ .add_arg("tensor", "The tensor to be inspected")
+ .set_attr<FInferType>("FInferType", InferTypeTensorDtypeCode)
+ .set_attr<FLegalize>("FLegalize", LegalizeTensorDtypeCode)
+ .set_attr<bool>("RequiresArgumentShapes", false)
+ .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+ .set_attr<bool>("FPurity", true);
+}
//// relax.tensor_dtype_bits
Expr tensor_dtype_bits(Expr expr) {
- static const Op& op = Op::Get("relax.inspect.tensor_dtype_bits");
+ static const Op op = Op::Get("relax.inspect.tensor_dtype_bits");
return Call(Type::Missing(), op, {expr});
}
@@ -163,19 +164,20 @@ Expr LegalizeTensorDtypeBits(const BlockBuilder& bb, const Call& call) {
return Call(Type::Missing(), gvar_getter, {arg});
}
-TVM_REGISTER_OP("relax.inspect.tensor_dtype_bits")
- .set_num_inputs(1)
- .add_argument("tensor", "Tensor", "The tensor to be inspected")
- .set_attr<FInferType>("FInferType", InferTypeTensorDtypeBits)
- .set_attr<FLegalize>("FLegalize", LegalizeTensorDtypeBits)
- .set_attr<bool>("RequiresArgumentShapes", false)
- .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.inspect.tensor_dtype_bits")
+ .add_arg("tensor", "The tensor to be inspected")
+ .set_attr<FInferType>("FInferType", InferTypeTensorDtypeBits)
+ .set_attr<FLegalize>("FLegalize", LegalizeTensorDtypeBits)
+ .set_attr<bool>("RequiresArgumentShapes", false)
+ .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+ .set_attr<bool>("FPurity", true);
+}
//// relax.tensor_dtype_lanes
Expr tensor_dtype_lanes(Expr expr) {
- static const Op& op = Op::Get("relax.inspect.tensor_dtype_lanes");
+ static const Op op = Op::Get("relax.inspect.tensor_dtype_lanes");
return Call(Type::Missing(), op, {expr});
}
@@ -192,19 +194,20 @@ Expr LegalizeTensorDtypeLanes(const BlockBuilder& bb, const Call& call) {
return Call(Type::Missing(), gvar_getter, {arg});
}
-TVM_REGISTER_OP("relax.inspect.tensor_dtype_lanes")
- .set_num_inputs(1)
- .add_argument("tensor", "Tensor", "The tensor to be inspected")
- .set_attr<FInferType>("FInferType", InferTypeTensorDtypeLanes)
- .set_attr<FLegalize>("FLegalize", LegalizeTensorDtypeLanes)
- .set_attr<bool>("RequiresArgumentShapes", false)
- .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.inspect.tensor_dtype_lanes")
+ .add_arg("tensor", "The tensor to be inspected")
+ .set_attr<FInferType>("FInferType", InferTypeTensorDtypeLanes)
+ .set_attr<FLegalize>("FLegalize", LegalizeTensorDtypeLanes)
+ .set_attr<bool>("RequiresArgumentShapes", false)
+ .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+ .set_attr<bool>("FPurity", true);
+}
//// relax.tensor_ndim
Expr tensor_ndim(Expr expr) {
- static const Op& op = Op::Get("relax.inspect.tensor_ndim");
+ static const Op op = Op::Get("relax.inspect.tensor_ndim");
return Call(Type::Missing(), op, {expr});
}
@@ -221,19 +224,20 @@ Expr LegalizeTensorNDim(const BlockBuilder& bb, const Call& call) {
return Call(Type::Missing(), gvar_getter, {arg});
}
-TVM_REGISTER_OP("relax.inspect.tensor_ndim")
- .set_num_inputs(1)
- .add_argument("tensor", "Tensor", "The tensor to be inspected")
- .set_attr<FInferType>("FInferType", InferTypeTensorNDim)
- .set_attr<FLegalize>("FLegalize", LegalizeTensorNDim)
- .set_attr<bool>("RequiresArgumentShapes", false)
- .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.inspect.tensor_ndim")
+ .add_arg("tensor", "The tensor to be inspected")
+ .set_attr<FInferType>("FInferType", InferTypeTensorNDim)
+ .set_attr<FLegalize>("FLegalize", LegalizeTensorNDim)
+ .set_attr<bool>("RequiresArgumentShapes", false)
+ .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+ .set_attr<bool>("FPurity", true);
+}
//// relax.tensor_shape_i
Expr tensor_shape_i(Expr expr) {
- static const Op& op = Op::Get("relax.inspect.tensor_shape_i");
+ static const Op op = Op::Get("relax.inspect.tensor_shape_i");
return Call(Type::Missing(), op, {expr});
}
@@ -300,20 +304,21 @@ Expr LegalizeTensorShape(const BlockBuilder& bb, const Call& call) {
return Call(Type::Missing(), gvar_getter, call->args);
}
-TVM_REGISTER_OP("relax.inspect.tensor_shape_i")
- .set_num_inputs(2)
- .add_argument("tensor", "Tensor", "The tensor to be inspected")
- .add_argument("axis", "Prim(int64)", "The axis whose extent should be returned")
- .set_attr<FInferType>("FInferType", InferTypeTensorShape)
- .set_attr<FLegalize>("FLegalize", LegalizeTensorShape)
- .set_attr<bool>("RequiresArgumentShapes", false)
- .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.inspect.tensor_shape_i")
+ .add_arg("tensor", "The tensor to be inspected")
+ .add_arg("axis", "The axis whose extent should be returned")
+ .set_attr<FInferType>("FInferType", InferTypeTensorShape)
+ .set_attr<FLegalize>("FLegalize", LegalizeTensorShape)
+ .set_attr<bool>("RequiresArgumentShapes", false)
+ .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+ .set_attr<bool>("FPurity", true);
+}
//// relax.tensor_stride_i
Expr tensor_stride_i(Expr expr) {
- static const Op& op = Op::Get("relax.inspect.tensor_stride_i");
+ static const Op op = Op::Get("relax.inspect.tensor_stride_i");
return Call(Type::Missing(), op, {expr});
}
@@ -348,19 +353,20 @@ Type InferTypeTensorStride(const Call& call, const BlockBuilder&) {
}
}
-TVM_REGISTER_OP("relax.inspect.tensor_stride_i")
- .set_num_inputs(2)
- .add_argument("tensor", "Tensor", "The tensor to be inspected")
- .add_argument("axis", "Prim(int64)", "The axis whose extent should be returned")
- .set_attr<FInferType>("FInferType", InferTypeTensorStride)
- .set_attr<bool>("RequiresArgumentShapes", false)
- .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.inspect.tensor_stride_i")
+ .add_arg("tensor", "The tensor to be inspected")
+ .add_arg("axis", "The axis whose extent should be returned")
+ .set_attr<FInferType>("FInferType", InferTypeTensorStride)
+ .set_attr<bool>("RequiresArgumentShapes", false)
+ .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+ .set_attr<bool>("FPurity", true);
+}
//// relax.tensor_byte_offset
Expr tensor_byte_offset(Expr expr) {
- static const Op& op = Op::Get("relax.inspect.tensor_byte_offset");
+ static const Op op = Op::Get("relax.inspect.tensor_byte_offset");
return Call(Type::Missing(), op, {expr});
}
@@ -380,18 +386,19 @@ Type InferTypeTensorByteOffset(const Call& call, const BlockBuilder&) {
}
}
-TVM_REGISTER_OP("relax.inspect.tensor_byte_offset")
- .set_num_inputs(1)
- .add_argument("tensor", "Tensor", "The tensor to be inspected")
- .set_attr<FInferType>("FInferType", InferTypeTensorByteOffset)
- .set_attr<bool>("RequiresArgumentShapes", false)
- .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.inspect.tensor_byte_offset")
+ .add_arg("tensor", "The tensor to be inspected")
+ .set_attr<FInferType>("FInferType", InferTypeTensorByteOffset)
+ .set_attr<bool>("RequiresArgumentShapes", false)
+ .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+ .set_attr<bool>("FPurity", true);
+}
//// relax.tensor_elem_offset
Expr tensor_elem_offset(Expr expr) {
- static const Op& op = Op::Get("relax.inspect.tensor_elem_offset");
+ static const Op op = Op::Get("relax.inspect.tensor_elem_offset");
return Call(Type::Missing(), op, {expr});
}
@@ -411,13 +418,14 @@ Type InferTypeTensorElemOffset(const Call& call, const BlockBuilder&) {
}
}
-TVM_REGISTER_OP("relax.inspect.tensor_elem_offset")
- .set_num_inputs(1)
- .add_argument("tensor", "Tensor", "The tensor to be inspected")
- .set_attr<FInferType>("FInferType", InferTypeTensorElemOffset)
- .set_attr<bool>("RequiresArgumentShapes", false)
- .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.inspect.tensor_elem_offset")
+ .add_arg("tensor", "The tensor to be inspected")
+ .set_attr<FInferType>("FInferType", InferTypeTensorElemOffset)
+ .set_attr<bool>("RequiresArgumentShapes", false)
+ .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+ .set_attr<bool>("FPurity", true);
+}
} // namespace inspect
} // namespace relax
diff --git a/src/relax/op/tensor/linear_algebra.cc b/src/relax/op/tensor/linear_algebra.cc
index 3a544a3cea..e9bf495c1b 100644
--- a/src/relax/op/tensor/linear_algebra.cc
+++ b/src/relax/op/tensor/linear_algebra.cc
@@ -46,7 +46,7 @@ Expr matmul(Expr x1, Expr x2, ffi::Optional<DLDataType> out_dtype) {
ffi::ObjectPtr<MatmulAttrs> attrs = ffi::make_object<MatmulAttrs>();
attrs->out_dtype = out_dtype;
- static const Op& op = Op::Get("relax.matmul");
+ static const Op op = Op::Get("relax.matmul");
return Call(Type::Missing(), op, {std::move(x1), std::move(x2)}, Attrs(attrs), {});
}
@@ -162,14 +162,15 @@ Call InferMixedPrecisionMatmul(const Call& call, DLDataType out_dtype) {
return matmul(call->args[0], call->args[1], out_dtype).as_or_throw<Call>();
}
-TVM_REGISTER_OP("relax.matmul")
- .set_num_inputs(2)
- .add_argument("x1", "Tensor", "The first input tensor.")
- .add_argument("x2", "Tensor", "The second input tensor.")
- .set_attr<FInferType>("FInferType", InferTypeMatmul)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
- .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionMatmul)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.matmul")
+ .add_arg("x1", "The first input tensor.")
+ .add_arg("x2", "The second input tensor.")
+ .set_attr<FInferType>("FInferType", InferTypeMatmul)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+ .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionMatmul)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.einsum */
@@ -177,7 +178,7 @@ Expr einsum(Expr operands, ffi::String subscripts) {
ffi::ObjectPtr<EinsumAttrs> attrs = ffi::make_object<EinsumAttrs>();
attrs->subscripts = std::move(subscripts);
- static const Op& op = Op::Get("relax.einsum");
+ static const Op op = Op::Get("relax.einsum");
return Call(Type::Missing(), op, {std::move(operands)}, Attrs{attrs}, {});
}
@@ -251,17 +252,18 @@ Type InferTypeEinsum(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr(oshape), operand_ty);
}
-TVM_REGISTER_OP("relax.einsum")
- .set_attrs_type<EinsumAttrs>()
- .set_num_inputs(1)
- .add_argument("operands", "Tensor", "The input tensors.")
- .set_attr<FInferType>("FInferType", InferTypeEinsum)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.einsum")
+ .add_arg("operands", "The input tensors.")
+ .attrs_type<EinsumAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeEinsum)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.outer */
Expr outer(Expr x1, Expr x2) {
- static const Op& op = Op::Get("relax.outer");
+ static const Op op = Op::Get("relax.outer");
return Call(Type::Missing(), op, {std::move(x1), std::move(x2)}, {});
}
@@ -290,13 +292,14 @@ Type InferTypeOuter(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr(output_shape), x1_ty->dtype);
}
-TVM_REGISTER_OP("relax.outer")
- .set_num_inputs(2)
- .add_argument("x1", "Tensor", "The first input tensor.")
- .add_argument("x2", "Tensor", "The second input tensor.")
- .set_attr<FInferType>("FInferType", InferTypeOuter)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.outer")
+ .add_arg("x1", "The first input tensor.")
+ .add_arg("x2", "The second input tensor.")
+ .set_attr<FInferType>("FInferType", InferTypeOuter)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+ .set_attr<bool>("FPurity", true);
+}
} // namespace relax
} // namespace tvm
diff --git a/src/relax/op/tensor/manipulate.cc b/src/relax/op/tensor/manipulate.cc
index 847cf1f344..64d8d62c75 100644
--- a/src/relax/op/tensor/manipulate.cc
+++ b/src/relax/op/tensor/manipulate.cc
@@ -64,7 +64,7 @@ TVM_FFI_STATIC_INIT_BLOCK() {
/* relax.broadcast_to */
Expr broadcast_to(Expr x, Expr shape) {
- static const Op& op = Op::Get("relax.broadcast_to");
+ static const Op op = Op::Get("relax.broadcast_to");
return Call(Type::Missing(), op, {std::move(x), std::move(shape)}, Attrs(), {});
}
@@ -131,13 +131,14 @@ Type InferTypeBroadcastTo(const Call& call, const BlockBuilder& ctx) {
return TensorType(/*shape=*/call->args[1], data_ty->dtype, data_ty->vdevice);
}
-TVM_REGISTER_OP("relax.broadcast_to")
- .set_num_inputs(2)
- .add_argument("x", "Tensor", "The input tensor.")
- .add_argument("shape", "Shape", "The target shape.")
- .set_attr<FInferType>("FInferType", InferTypeBroadcastTo)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.broadcast_to")
+ .add_arg("x", "The input tensor.")
+ .add_arg("shape", "The target shape.")
+ .set_attr<FInferType>("FInferType", InferTypeBroadcastTo)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.concat */
@@ -145,7 +146,7 @@ Expr concat(Expr tensors, ffi::Optional<int64_t> axis) {
ffi::ObjectPtr<ConcatAttrs> attrs = ffi::make_object<ConcatAttrs>();
attrs->axis = std::move(axis);
- static const Op& op = Op::Get("relax.concat");
+ static const Op op = Op::Get("relax.concat");
return Call(Type::Missing(), op, {std::move(tensors)}, Attrs(attrs), {});
}
@@ -390,14 +391,15 @@ InferLayoutOutput InferLayoutConcat(
return InferLayoutOutput({NLayout(input_layouts)}, output_layouts, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.concat")
- .set_attrs_type<ConcatAttrs>()
- .set_num_inputs(1)
- .add_argument("tensors", "Tuple of Tensors", "The input list of tensors.")
- .set_attr<FInferType>("FInferType", InferTypeConcat)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConcat)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.concat")
+ .add_arg("tensors", "The input list of tensors.")
+ .attrs_type<ConcatAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeConcat)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConcat)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.expand_dims */
@@ -405,7 +407,7 @@ Expr expand_dims(Expr x, ffi::Array<int64_t> axis) {
ffi::ObjectPtr<ExpandDimsAttrs> attrs = ffi::make_object<ExpandDimsAttrs>();
attrs->axis = std::move(axis);
- static const Op& op = Op::Get("relax.expand_dims");
+ static const Op op = Op::Get("relax.expand_dims");
return Call(Type::Missing(), op, {std::move(x)}, Attrs{attrs}, {});
}
@@ -494,14 +496,15 @@ InferLayoutOutput InferLayoutExpandDims(
Attrs(call->attrs));
}
-TVM_REGISTER_OP("relax.expand_dims")
- .set_num_inputs(1)
- .set_attrs_type<ExpandDimsAttrs>()
- .add_argument("x", "Tensor", "The input tensor.")
- .set_attr<FInferType>("FInferType", InferTypeExpandDims)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutExpandDims)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.expand_dims")
+ .add_arg("x", "The input tensor.")
+ .attrs_type<ExpandDimsAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeExpandDims)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutExpandDims)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
// Helper function for flatten and reshape.
PrimExpr ComputeShapeProduct(const ffi::Array<PrimExpr>& shape_values) {
@@ -514,7 +517,7 @@ PrimExpr ComputeShapeProduct(const ffi::Array<PrimExpr>& shape_values) {
/* relax.flatten */
Expr flatten(Expr x) {
- static const Op& op = Op::Get("relax.flatten");
+ static const Op op = Op::Get("relax.flatten");
return Call(Type::Missing(), op, {std::move(x)}, {}, {});
}
@@ -541,17 +544,18 @@ Type InferTypeFlatten(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr({std::move(shape_prod)}), data_ty->dtype, data_ty->vdevice);
}
-TVM_REGISTER_OP("relax.flatten")
- .set_num_inputs(1)
- .add_argument("x", "Tensor", "The input tensor.")
- .set_attr<FInferType>("FInferType", InferTypeFlatten)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.flatten")
+ .add_arg("x", "The input tensor.")
+ .set_attr<FInferType>("FInferType", InferTypeFlatten)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.index_tensor */
Expr index_tensor(Expr first, Expr tensors) {
- static const Op& op = Op::Get("relax.index_tensor");
+ static const Op op = Op::Get("relax.index_tensor");
return Call(Type::Missing(), op, {std::move(first), std::move(tensors)}, Attrs(), {});
}
@@ -690,12 +694,13 @@ Type InferTypeIndexTensor(const Call& call, const BlockBuilder& ctx) {
return TensorType(output_dtype, out_ndim, vdev);
}
-TVM_REGISTER_OP("relax.index_tensor")
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "The input data.")
- .add_argument("indices", "List of Tensors", "The indices used to index.")
- .set_attr<FInferType>("FInferType", InferTypeIndexTensor)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.index_tensor")
+ .add_arg("data", "The input data.")
+ .add_arg("indices", "The indices used to index.")
+ .set_attr<FInferType>("FInferType", InferTypeIndexTensor)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.layout_transform */
@@ -704,7 +709,7 @@ Expr layout_transform(Expr x, tirx::IndexMap index_map, ffi::Optional<PrimExpr>
attrs->index_map = std::move(index_map);
attrs->pad_value = std::move(pad_value);
- static const Op& op = Op::Get("relax.layout_transform");
+ static const Op op = Op::Get("relax.layout_transform");
return Call(Type::Missing(), op, {std::move(x)}, Attrs{attrs}, {});
}
@@ -757,13 +762,14 @@ Type InferTypeLayoutTransform(const Call& call, const BlockBuilder& ctx) {
return TensorType(ShapeExpr(output_shape), data_ty->dtype, data_ty->vdevice);
}
-TVM_REGISTER_OP("relax.layout_transform")
- .set_num_inputs(1)
- .set_attrs_type<LayoutTransformAttrs>()
- .add_argument("x", "Tensor", "The input tensor.")
- .set_attr<FInferType>("FInferType", InferTypeLayoutTransform)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.layout_transform")
+ .add_arg("x", "The input tensor.")
+ .attrs_type<LayoutTransformAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeLayoutTransform)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.permute_dims */
@@ -771,7 +777,7 @@ Expr permute_dims(Expr x, ffi::Optional<ffi::Array<int64_t>> axes) {
ffi::ObjectPtr<PermuteDimsAttrs> attrs = ffi::make_object<PermuteDimsAttrs>();
attrs->axes = std::move(axes);
- static const Op& op = Op::Get("relax.permute_dims");
+ static const Op op = Op::Get("relax.permute_dims");
return Call(Type::Missing(), op, {std::move(x)}, Attrs{attrs}, {});
}
@@ -877,14 +883,15 @@ InferLayoutOutput InferLayoutPermuteDims(
return InferLayoutOutput({existing_layout}, {InitialLayoutDecision(ndim)}, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.permute_dims")
- .set_attrs_type<PermuteDimsAttrs>()
- .set_num_inputs(1)
- .add_argument("x", "Tensor", "The input tensor.")
- .set_attr<FInferType>("FInferType", InferTypePermuteDims)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPermuteDims)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.permute_dims")
+ .add_arg("x", "The input tensor.")
+ .attrs_type<PermuteDimsAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypePermuteDims)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPermuteDims)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.reshape */
Expr ConvertNewShapeToExpr(const Expr& data,
@@ -990,7 +997,7 @@ Expr ConvertNewShapeToExpr(const Expr& data,
Expr reshape(Expr x, ffi::Variant<Expr, ffi::Array<PrimExpr>> shape) {
Expr shape_in_expr = ConvertNewShapeToExpr(x, shape);
- static const Op& op = Op::Get("relax.reshape");
+ static const Op op = Op::Get("relax.reshape");
return Call(Type::Missing(), op, {std::move(x), std::move(shape_in_expr)}, Attrs(), {});
}
@@ -1042,13 +1049,14 @@ Type InferTypeReshape(const Call& call, const BlockBuilder& ctx) {
return TensorType(target_shape, data_ty->dtype, data_ty->vdevice);
}
-TVM_REGISTER_OP("relax.reshape")
- .set_num_inputs(2)
- .add_argument("x", "Tensor", "The input tensor.")
- .add_argument("shape", "Shape", "The input new shape.")
- .set_attr<FInferType>("FInferType", InferTypeReshape)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.reshape")
+ .add_arg("x", "The input tensor.")
+ .add_arg("shape", "The input new shape.")
+ .set_attr<FInferType>("FInferType", InferTypeReshape)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.split */
@@ -1080,7 +1088,7 @@ Expr split(Expr x, ffi::Variant<IntImm, ffi::Array<IntImm>> indices_or_sections,
attrs->indices_or_sections = indices_or_sections_obj;
attrs->axis = axis;
- static const Op& op = Op::Get("relax.split");
+ static const Op op = Op::Get("relax.split");
return Call(Type::Missing(), op, {std::move(x)}, Attrs(attrs), {});
}
@@ -1215,14 +1223,15 @@ InferLayoutOutput InferLayoutSplit(
return InferLayoutOutput({existing_layout}, {tuple_layouts}, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.split")
- .set_attrs_type<SplitAttrs>()
- .set_num_inputs(1)
- .add_argument("x", "Tensor", "The input tensor.")
- .set_attr<FInferType>("FInferType", InferTypeSplit)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutSplit)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.split")
+ .add_arg("x", "The input tensor.")
+ .attrs_type<SplitAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeSplit)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutSplit)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.squeeze */
@@ -1230,7 +1239,7 @@ Expr squeeze(Expr x, ffi::Optional<ffi::Array<int64_t>> axis) {
ffi::ObjectPtr<SqueezeAttrs> attrs = ffi::make_object<SqueezeAttrs>();
attrs->axis = std::move(axis);
- static const Op& op = Op::Get("relax.squeeze");
+ static const Op op = Op::Get("relax.squeeze");
return Call(Type::Missing(), op, {std::move(x)}, Attrs{attrs}, {});
}
@@ -1374,14 +1383,15 @@ InferLayoutOutput InferLayoutSqueeze(
Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.squeeze")
- .set_num_inputs(1)
- .set_attrs_type<SqueezeAttrs>()
- .add_argument("x", "Tensor", "The input tensor.")
- .set_attr<FInferType>("FInferType", InferTypeSqueeze)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutSqueeze)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.squeeze")
+ .add_arg("x", "The input tensor.")
+ .attrs_type<SqueezeAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeSqueeze)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutSqueeze)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
void CheckCollapseShape(const Call& call, const BlockBuilder& ctx,
const ffi::Array<PrimExpr>& data_shape,
@@ -1428,7 +1438,7 @@ Expr stack(Expr tensors, ffi::Optional<int64_t> axis) {
ffi::ObjectPtr<StackAttrs> attrs = ffi::make_object<StackAttrs>();
attrs->axis = std::move(axis);
- static const Op& op = Op::Get("relax.stack");
+ static const Op op = Op::Get("relax.stack");
return Call(Type::Missing(), op, {std::move(tensors)}, Attrs(attrs), {});
}
@@ -1626,18 +1636,19 @@ InferLayoutOutput InferLayoutStack(
return InferLayoutOutput({NLayout(input_layouts)}, output_layouts, Attrs(new_attrs));
}
-TVM_REGISTER_OP("relax.stack")
- .set_attrs_type<StackAttrs>()
- .set_num_inputs(1)
- .add_argument("tensors", "Tuple of Tensors", "The input list of tensors to stack")
- .set_attr<FInferType>("FInferType", InferTypeStack)
- .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutStack)
- .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.stack")
+ .add_arg("tensors", "The input list of tensors to stack")
+ .attrs_type<StackAttrs>()
+ .set_attr<FInferType>("FInferType", InferTypeStack)
+ .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutStack)
+ .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.collapse_sum_like */
Expr collapse_sum_like(Expr data, Expr collapse_target) {
- static const Op& op = Op::Get("relax.collapse_sum_like");
+ static const Op op = Op::Get("relax.collapse_sum_like");
return Call(Type::Missing(), op, {std::move(data), std::move(collapse_target)}, Attrs(), {});
}
@@ -1674,17 +1685,17 @@ Type InferTypeCollapseSumLike(const Call& call, const BlockBuilder& ctx) {
}
}
-TVM_REGISTER_OP("relax.collapse_sum_like")
- .set_num_inputs(2)
- .add_argument("data", "Tensor", "The input tensor.")
- .add_argument("collapse_target", "Tensor",
- "The tensor whose shape is the shape to collapse to.")
- .set_attr<FInferType>("FInferType", InferTypeCollapseSumLike)
- .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+ OpDef("relax.collapse_sum_like")
+ .add_arg("data", "The input tensor.")
+ .add_arg("collapse_target", "The tensor whose shape is the shape to collapse to.")
+ .set_attr<FInferType>("FInferType", InferTypeCollapseSumLike)
+ .set_attr<bool>("FPurity", true);
+}
/* relax.collapse_sum_to */
Expr collapse_sum_to(Expr data, Expr shape) {
... 5539 lines suppressed ...