You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2026/09/27 01:48:57 UTC

(tvm) branch main updated: [REFACTOR][IR] Introduce OpDef and a dedicated Op registry (#20473)

This is an automated email from the ASF dual-hosted git repository.

tqchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git


The following commit(s) were added to refs/heads/main by this push:
     new 135dea7856 [REFACTOR][IR] Introduce OpDef and a dedicated Op registry (#20473)
135dea7856 is described below

commit 135dea785626aa7c413c54675dddb28edda4dcea
Author: Tianqi Chen <tq...@users.noreply.github.com>
AuthorDate: Sat Sep 26 21:48:51 2026 -0400

    [REFACTOR][IR] Introduce OpDef and a dedicated Op registry (#20473)
---
 include/tvm/ir/attr_registry_map.h                 | 144 ----
 include/tvm/ir/op.h                                | 475 ++++-------
 include/tvm/tirx/op.h                              |   7 -
 include/tvm/tirx/op_attr_types.h                   |   2 +-
 python/tvm/backend/cuda/ptx/engine.py              |  12 +-
 python/tvm/ir/op.py                                | 133 +--
 python/tvm/relax/op/base.py                        |  14 +-
 python/tvm/target/intrin.py                        |   4 +-
 python/tvm/tirx/op.py                              |  13 +-
 python/tvm/tirx/script/ir_builder/op.py            |  30 +-
 src/backend/cuda/codegen/codegen_cuda.cc           |  30 +-
 src/backend/cuda/codegen/intrin_rule_cuda.cc       | 333 ++++----
 src/backend/cuda/codegen/llvm/intrin_rule_nvptx.cc | 114 +--
 src/backend/cuda/op/iket.cc                        |  78 +-
 src/backend/cuda/op/target_builtin.cc              | 498 +++++++----
 src/backend/cuda/transforms/lower_iket.cc          |  16 +-
 .../hexagon/codegen/llvm/intrin_rule_hexagon.cc    | 308 +++----
 src/backend/metal/codegen/codegen_metal.cc         |   8 +-
 src/backend/metal/codegen/intrin_rule_metal.cc     | 234 +++---
 src/backend/metal/op/target_builtin.cc             |  69 +-
 src/backend/opencl/codegen/intrin_rule_opencl.cc   | 114 +--
 src/backend/rocm/codegen/llvm/intrin_rule_rocm.cc  | 154 ++--
 src/backend/trn/codegen/codegen_trn.cc             |  32 +-
 src/backend/trn/op/target_builtin.cc               |  91 +-
 src/backend/vulkan/codegen/codegen_spirv.cc        |   8 +-
 src/backend/vulkan/codegen/intrin_rule_spirv.cc    | 156 ++--
 src/backend/webgpu/codegen/intrin_rule_webgpu.cc   | 241 +++---
 src/ir/attr_registry.h                             | 172 ----
 src/ir/op.cc                                       | 324 ++++---
 src/ir/prim/builtin.cc                             |  78 +-
 .../backend/adreno/annotate_custom_storage.cc      |   8 +-
 .../backend/adreno/fold_vdevice_scope_change.cc    |   2 +-
 src/relax/backend/contrib/tensorrt/codegen.cc      |   2 +-
 src/relax/backend/task_extraction.cc               |   2 +-
 src/relax/backend/vm/codegen_vm.cc                 |  10 +-
 src/relax/backend/vm/lower_runtime_builtin.cc      |  34 +-
 src/relax/backend/vm/vm_shape_lower.cc             |   4 +-
 .../transform/lower_global_view_to_local_view.cc   |   2 +-
 .../distributed/transform/propagate_sharding.cc    |  22 +-
 src/relax/ir/expr.cc                               |   2 +-
 src/relax/op/ccl/ccl.cc                            |  64 +-
 src/relax/op/distributed/binary.cc                 | 119 ++-
 src/relax/op/distributed/binary.h                  |   8 -
 src/relax/op/distributed/ccl.cc                    |   8 +-
 src/relax/op/distributed/distributed.cc            |  61 +-
 src/relax/op/distributed/linear_algebra.cc         |   7 +-
 src/relax/op/distributed/manipulate.cc             |  15 +-
 src/relax/op/distributed/nn.cc                     |  19 +-
 src/relax/op/distributed/op.cc                     |  13 +-
 src/relax/op/distributed/statistical.cc            |  16 +-
 src/relax/op/distributed/statistical.h             |   4 +-
 src/relax/op/distributed/unary.cc                  |  63 +-
 src/relax/op/distributed/unary.h                   |   7 +-
 src/relax/op/image/resize.cc                       |  98 ++-
 src/relax/op/memory/view.cc                        |  46 +-
 src/relax/op/nn/attention.cc                       |  77 +-
 src/relax/op/nn/convolution.cc                     | 138 +--
 src/relax/op/nn/convolution.h                      |   2 +-
 src/relax/op/nn/nn.cc                              | 397 +++++----
 src/relax/op/nn/nn.h                               |  13 -
 src/relax/op/nn/pooling.cc                         | 165 ++--
 src/relax/op/op.cc                                 | 603 +++++++-------
 src/relax/op/op_common.cc                          |  21 +-
 src/relax/op/op_common.h                           |  48 +-
 src/relax/op/tensor/binary.cc                      |   7 +-
 src/relax/op/tensor/binary.h                       |  56 +-
 src/relax/op/tensor/create.cc                      | 222 ++---
 src/relax/op/tensor/datatype.cc                    |  34 +-
 src/relax/op/tensor/grad.cc                        | 113 +--
 src/relax/op/tensor/index.cc                       |  65 +-
 src/relax/op/tensor/inspect.cc                     | 150 ++--
 src/relax/op/tensor/linear_algebra.cc              |  51 +-
 src/relax/op/tensor/manipulate.cc                  | 450 +++++-----
 src/relax/op/tensor/qdq.cc                         |  38 +-
 src/relax/op/tensor/sampling.cc                    |  19 +-
 src/relax/op/tensor/search.cc                      |  85 +-
 src/relax/op/tensor/set.cc                         |  60 +-
 src/relax/op/tensor/sorting.cc                     |  45 +-
 src/relax/op/tensor/statistical.cc                 | 163 +++-
 src/relax/op/tensor/statistical.h                  |  26 -
 src/relax/op/tensor/ternary.cc                     |  21 +-
 src/relax/op/tensor/unary.cc                       | 523 +++++++++++-
 src/relax/op/tensor/unary.h                        |  20 -
 src/relax/op/vision/multibox_transform_loc.cc      |  21 +-
 src/relax/op/vision/nms.cc                         |  63 +-
 src/relax/op/vision/roi_align.cc                   |  21 +-
 src/relax/op/vision/roi_pool.cc                    |  21 +-
 src/relax/script/ir_builder/distributed.cc         |   2 +-
 src/relax/script/printer/call.cc                   |  18 +-
 src/relax/script/printer/utils.h                   |   4 +-
 src/relax/transform/alter_op_impl.cc               |   4 +-
 .../transform/attach_attr_layout_free_buffers.cc   |   2 +-
 src/relax/transform/call_tir_rewrite.cc            |   8 +-
 src/relax/transform/convert_layout.cc              |   2 +-
 src/relax/transform/dataflow_inplace.cc            |   2 +-
 src/relax/transform/decompose_ops.cc               |  10 +-
 src/relax/transform/fold_constant.cc               |   2 +-
 src/relax/transform/fuse_ops.cc                    |   4 +-
 src/relax/transform/fuse_tir.cc                    |  16 +-
 src/relax/transform/kill_after_last_use.cc         |  18 +-
 src/relax/transform/lambda_lift.cc                 |   6 +-
 src/relax/transform/legalize_ops.cc                |   8 +-
 src/relax/transform/lift_transform_params.cc       |   6 +-
 src/relax/transform/lower_alloc_tensor.cc          |   6 +-
 src/relax/transform/realize_vdevice.cc             |  10 +-
 src/relax/transform/remove_purity_checking.cc      |   8 +-
 src/relax/transform/rewrite_cuda_graph.cc          |  14 +-
 src/relax/transform/rewrite_dataflow_reshape.cc    |   2 +-
 src/relax/transform/run_codegen.cc                 |   2 +-
 .../specialize_primfunc_based_on_callsite.cc       |   2 +-
 src/relax/transform/split_call_tir_by_pattern.cc   |   6 +-
 .../transform/split_layout_rewrite_preproc.cc      |   2 +-
 src/relax/transform/static_plan_block_memory.cc    |  18 +-
 src/relax/transform/to_mixed_precision.cc          |   2 +-
 .../postproc/rewrite_cooperative_fetch.cc          |   4 +-
 src/s_tir/schedule/analysis/analysis.cc            |   2 +-
 src/s_tir/schedule/utils.h                         |   1 -
 src/s_tir/transform/inject_permuted_layout.cc      |   4 +-
 src/s_tir/transform/inject_ptx_async_copy.cc       |   6 +-
 src/s_tir/transform/inject_ptx_ldg32.cc            |   2 +-
 src/s_tir/transform/inject_software_pipeline.cc    |  10 +-
 src/s_tir/transform/memhammer_lower_auto_copy.cc   |   4 +-
 .../transform/memhammer_tensorcore_rewrite.cc      |   4 +-
 .../transform/merge_shared_memory_allocations.cc   |   2 +-
 src/s_tir/transform/tensorcore_infer_fragment.cc   |  10 +-
 src/target/intrin_rule.cc                          | 344 ++++----
 src/target/llvm/intrin_rule_llvm.cc                | 280 ++++---
 src/te/tensor.cc                                   |  18 +-
 src/tirx/analysis/filter_canonical.cc              |   2 +-
 src/tirx/ir/data_type_rewriter.cc                  |   2 +-
 src/tirx/ir/exec_scope.cc                          |   2 +-
 src/tirx/ir/ir_mutator_with_analyzer.cc            |   4 +-
 src/tirx/ir/ir_visitor_with_analyzer.cc            |   2 +-
 src/tirx/ir/layout/utils.h                         |   2 -
 src/tirx/ir/stmt.cc                                |  18 +-
 src/tirx/op/builtin.cc                             | 927 ++++++++++++++-------
 src/tirx/op/op.cc                                  | 373 ++++++---
 src/tirx/op/tirx.cc                                | 308 ++++++-
 src/tirx/transform/lower_intrin.cc                 |   2 +-
 src/tirx/transform/lower_tvm_builtin.cc            |   4 +-
 src/tirx/transform/lower_warp_memory.cc            |  18 +-
 src/tirx/transform/remove_no_op.cc                 |   8 +-
 src/tirx/transform/tile_primitive_dispatch.cc      |   2 +-
 tests/cpp/tir_analysis_side_effect.cc              |   3 +-
 .../s_tir/script/test_s_tir_script_printer.py      |  10 +-
 145 files changed, 6076 insertions(+), 4752 deletions(-)

diff --git a/include/tvm/ir/attr_registry_map.h b/include/tvm/ir/attr_registry_map.h
deleted file mode 100644
index fd4c53bc4f..0000000000
--- a/include/tvm/ir/attr_registry_map.h
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*!
- * \file tvm/ir/attr_registry_map.h
- * \brief Attribute map used in registry.
- */
-#ifndef TVM_IR_ATTR_REGISTRY_MAP_H_
-#define TVM_IR_ATTR_REGISTRY_MAP_H_
-
-#include <tvm/ffi/string.h>
-
-#include <utility>
-#include <vector>
-
-namespace tvm {
-
-/*!
- * \brief Generic attribute map.
- * \tparam KeyType the type of the key.
- */
-template <typename KeyType>
-class AttrRegistryMapContainerMap {
- public:
-  /*!
-   * \brief Check if the map has key.
-   * \param key The key to the map
-   * \return 1 if key is contained in map, 0 otherwise.
-   */
-  int count(const KeyType& key) const {
-    if (key.defined()) {
-      const uint32_t idx = key->AttrRegistryIndex();
-      return idx < data_.size() ? (data_[idx].second != 0) : 0;
-    } else {
-      return 0;
-    }
-  }
-  /*!
-   * \brief get the corresponding value element at key.
-   * \param key The key to the map
-   * \return the const reference to the content value.
-   */
-  const ffi::Any& operator[](const KeyType& key) const {
-    TVM_FFI_ICHECK(key.defined());
-    const uint32_t idx = key->AttrRegistryIndex();
-    TVM_FFI_ICHECK(idx < data_.size() && data_[idx].second != 0)
-        << "Attribute " << attr_name_ << " has not been registered for " << key->name;
-    return data_[idx].first;
-  }
-  /*!
-   * \brief get the corresponding value element at key with default value.
-   * \param key The key to the map
-   * \param def_value The default value when the key does not exist.
-   * \return the const reference to the content value.
-   * \tparam ValueType The content value type.
-   */
-  template <typename ValueType>
-  ValueType get(const KeyType& key, ValueType def_value) const {
-    TVM_FFI_ICHECK(key.defined());
-    const uint32_t idx = key->AttrRegistryIndex();
-    if (idx < data_.size() && data_[idx].second != 0) {
-      if constexpr (std::is_same_v<ValueType, ffi::Any>) {
-        return data_[idx].first;
-      } else {
-        return data_[idx].first.template cast<ValueType>();
-      }
-    } else {
-      return def_value;
-    }
-  }
-
- private:
-  /*! \brief The name of the attr field */
-  ffi::String attr_name_;
-  /*! \brief The internal data. */
-  std::vector<std::pair<ffi::Any, int>> data_;
-  /*! \brief The constructor */
-  AttrRegistryMapContainerMap() = default;
-  template <typename, typename>
-  friend class AttrRegistry;
-  friend class OpRegEntry;
-};
-
-/*!
- * \brief ffi::Map<Key, ValueType> used to store meta-data.
- * \tparam KeyType The type of the key
- * \tparam ValueType The type of the value stored in map.
- */
-template <typename KeyType, typename ValueType>
-class AttrRegistryMap {
- public:
-  /*!
-   * \brief constructor
-   * \param map The internal map.
-   */
-  explicit AttrRegistryMap(const AttrRegistryMapContainerMap<KeyType>& map) : map_(map) {}
-  /*!
-   * \brief Check if the map has op as key.
-   * \param key The key to the map
-   * \return 1 if op is contained in map, 0 otherwise.
-   */
-  int count(const KeyType& key) const { return map_.count(key); }
-  /*!
-   * \brief get the corresponding value element at key.
-   * \param key The key to the map
-   * \return the const reference to the content value.
-   */
-  ValueType operator[](const KeyType& key) const {
-    if constexpr (std::is_same_v<ValueType, ffi::Any>) {
-      return map_[key];
-    } else {
-      return map_[key].template cast<ValueType>();
-    }
-  }
-  /*!
-   * \brief get the corresponding value element at key with default value.
-   * \param key The key to the map
-   * \param def_value The default value when the key does not exist.
-   * \return the const reference to the content value.
-   */
-  ValueType get(const KeyType& key, ValueType def_value) const { return map_.get(key, def_value); }
-
- protected:
-  /*! \brief The internal map field */
-  const AttrRegistryMapContainerMap<KeyType>& map_;
-};
-
-}  // namespace tvm
-#endif  // TVM_IR_ATTR_REGISTRY_MAP_H_
diff --git a/include/tvm/ir/op.h b/include/tvm/ir/op.h
index 02568266df..9a6a530ef9 100644
--- a/include/tvm/ir/op.h
+++ b/include/tvm/ir/op.h
@@ -19,404 +19,297 @@
 
 /*!
  * \file tvm/ir/op.h
- * \brief Primitive operators(builtin intrinsics)
- *        and registry for them.
+ * \brief Canonical primitive operators and their registration.
  */
 #ifndef TVM_IR_OP_H_
 #define TVM_IR_OP_H_
 
-#include <tvm/ffi/error.h>
-#include <tvm/ffi/function.h>
+#include <tvm/ffi/container/list.h>
 #include <tvm/ffi/reflection/registry.h>
-#include <tvm/ir/attr_registry_map.h>
 #include <tvm/ir/attrs.h>
 #include <tvm/ir/env_func.h>
 #include <tvm/ir/expr.h>
 #include <tvm/ir/type.h>
 
-#include <string>
+#include <type_traits>
 #include <utility>
-#include <vector>
 
 namespace tvm {
 
-// forward declare name.
 template <typename>
 class OpAttrMap;
 
-/*!
- * \brief Information about an input field of an Op (name, type, description).
- *
- *  Populated via OpRegEntry::add_argument and consumed both by
- *  internal sanity checks / error messages and by external tooling
- *  that wants to introspect an Op's argument schema.
- */
+/*! \brief An operator argument's name and documentation. */
 class ArgumentInfoNode : public ffi::Object {
  public:
-  /*! \brief name of the field */
+  /*! \brief Argument name. */
   ffi::String name;
-  /*! \brief type docstring information in str. */
-  ffi::String type_info;
-  /*! \brief detailed description of the type */
-  ffi::String description;
+  /*! \brief Argument documentation. */
+  ffi::String doc;
 
   static void RegisterReflection() {
-    namespace rfl = ffi::reflection;
-    rfl::ObjectDef<ArgumentInfoNode>()
+    namespace refl = ffi::reflection;
+    refl::ObjectDef<ArgumentInfoNode>()
         .def_ro("name", &ArgumentInfoNode::name)
-        .def_ro("type_info", &ArgumentInfoNode::type_info)
-        .def_ro("description", &ArgumentInfoNode::description);
+        .def_ro("doc", &ArgumentInfoNode::doc);
   }
 
   static constexpr TVMFFISEqHashKind _type_s_eq_hash_kind = kTVMFFISEqHashKindTreeNode;
-
   TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.ArgumentInfo", ArgumentInfoNode, ffi::Object);
 };
 
-/*! \brief Managed reference to ArgumentInfoNode. */
+/*! \brief Managed reference to an argument descriptor. */
 class ArgumentInfo : public ffi::ObjectRef {
  public:
   TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(ArgumentInfo, ffi::ObjectRef, ArgumentInfoNode);
 };
 
-// TODO(tvm-team): migrate low-level intrinsics to use Op
-/*!
- * \brief Primitive Op(builtin intrinsics)
- *
- * This data structure stores the meta-data
- * about primitive operators that can be invoked via Call.
- *
- * Low-level IR intrinsics(such as libc.expf) are also
- * implemented via Op.
- *
- * \sa Op
- */
+/*! \brief Metadata for a canonical primitive operator invoked through Call. */
 class OpNode : public ExprNode {
+ private:
+  // Dense process-local index into attribute columns; never serialized.
+  uint32_t index_{0};
+
  public:
-  /*! \brief name of the operator */
+  /*! \brief Canonical operator name. */
   ffi::String name;
-  /*!
-   * \brief detailed description of the operator
-   *  This can be used to generate docstring automatically for the operator.
-   */
-  ffi::String description;
-  /* \brief Information of input arguments to the operator */
-  ffi::Array<ArgumentInfo> arguments;
-  /*!
-   * \brief The type key of the attribute field
-   *  This can be empty, in which case it defaults to anything.
-   */
+  /*! \brief Operator documentation. */
+  ffi::String doc;
+  /*! \brief Descriptors for the ordered required value-argument prefix. */
+  ffi::Array<ArgumentInfo> args_info;
+  /*! \brief Accept at least args_info.size() value arguments when true, exactly that count
+   * otherwise. */
+  bool allow_extra_args{false};
+  /*! \brief Descriptors for Call.ty_args; inference owns optional/variable type-argument checks. */
+  ffi::Array<ArgumentInfo> ty_args_info;
+  /*! \brief Attribute object type key, or empty when unrestricted. */
   ffi::String attrs_type_key;
-  /*!
-   * \brief attribute type index,
-   * this field varies in each run and is not exposed to frontend.
-   */
+  /*! \brief Runtime index corresponding to attrs_type_key; not serialized. */
   uint32_t attrs_type_index{0};
-  /*!
-   * \brief number of input arguments to the operator,
-   * -1 means it is variable length
-   */
-  int32_t num_inputs = -1;
-  /*!
-   * \brief support level of the operator,
-   *  The lower the more priority it contains.
-   *  This is in analogies to BLAS levels.
-   */
-  int32_t support_level = 10;
 
-  static void RegisterReflection() {
-    namespace refl = tvm::ffi::reflection;
-    refl::ObjectDef<OpNode>()
-        .def_ro("name", &OpNode::name)
-        .def_ro("description", &OpNode::description, refl::AttachFieldFlag::SEqHashIgnore())
-        .def_ro("arguments", &OpNode::arguments, refl::AttachFieldFlag::SEqHashIgnore())
-        .def_ro("attrs_type_key", &OpNode::attrs_type_key, refl::AttachFieldFlag::SEqHashIgnore())
-        .def_ro("num_inputs", &OpNode::num_inputs, refl::AttachFieldFlag::SEqHashIgnore())
-        .def_ro("support_level", &OpNode::support_level, refl::AttachFieldFlag::SEqHashIgnore());
-  }
+  TVM_DLL static void RegisterReflection();
 
   static constexpr TVMFFISEqHashKind _type_s_eq_hash_kind = kTVMFFISEqHashKindUniqueInstance;
   TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.Op", OpNode, ExprNode);
 
  private:
-  /*! \return the internal attr registry index. */
-  uint32_t AttrRegistryIndex() const { return index_; }
-  /*! \brief repr to be printed in registry*/
-  std::string AttrRegistryName() const { return name; }
-
-  // friend class
+  friend class OpRegistry;
+  friend class OpDef;
+  friend class Op;
   template <typename>
-  friend class AttrRegistryMapContainerMap;
-  template <typename, typename>
-  friend class AttrRegistry;
-  friend class OpRegEntry;
-
-  // Program internal unique index of operator.
-  // Used to help index the program.
-  uint32_t index_{0};
+  friend class OpAttrMap;
 };
 
 /*!
- * \brief Managed reference class to OpNode.
- * \sa OpNode
+ * \brief Reference-counted handle to a canonical named operator.
+ *
+ * OpDef temporarily builds metadata on the same Op returned by Get. Independent
+ * registrations may attach different attributes; replacing an existing attribute
+ * requires explicit override. Cached attribute maps observe subsequent changes.
+ * args_info describes required value operands; allow_extra_args permits a suffix.
+ * ty_args_info describes type arguments without imposing a runtime count.
+ *
+ * \code
+ * TVM_FFI_STATIC_INIT_BLOCK() {
+ *   OpDef("example.identity", "Return the input expression.")
+ *       .add_arg("value", "The input expression.")
+ *       .set_attr<bool>("FPurity", true);
+ * }
+ * // Copies the handle; the canonical node is shared.
+ * Op op = Op::Get("example.identity");
+ * bool pure = Op::GetAttrMap<bool>("FPurity")[op];
+ * \endcode
  */
 class Op : public Expr {
  public:
+  /*! \brief Construct a handle from a defined node. \param n The operator node. */
   explicit Op(ffi::ObjectPtr<OpNode> n) : Expr(std::move(n)) {
     TVM_FFI_CHECK(defined(), ValueError) << "Op expects a defined OpNode";
   }
 
   /*!
-   * \brief Get additional registered attribute about operators.
-   *  If nothing has been registered, an empty OpAttrMap will be returned.
-   * \param attr_name The name of the attribute.
-   * \return An OpAttrMap of specified attr_name.
-   * \tparam ValueType The type of the attribute.
+   * \brief Get a typed live view of a registered attribute column.
+   * \tparam ValueType The attribute value type.
+   * \param attr_name The column name.
+   * \return A shared view that observes registration, replacement, reset, and growth.
+   * \throws InternalError if the column has never been registered.
    */
   template <typename ValueType>
-  inline static OpAttrMap<ValueType> GetAttrMap(const ffi::String& attr_name);
+  static OpAttrMap<ValueType> GetAttrMap(const ffi::String& attr_name);
   /*!
-   * \brief Checks if an attr map is present in the registry.
-   * \param attr_name The name of the attribute.
-   * \return bool True if the attr is present.
+   * \brief Check whether an attribute column exists, even if some Ops lack a value.
+   * \param attr_name The column name.
+   * \return Whether the column has been registered.
    */
   TVM_DLL static bool HasAttrMap(const ffi::String& attr_name);
   /*!
-   * \brief Get an Op for a given operator name.
-   *  Will raise an error if the op has not been registered.
-   * \param op_name Name of the operator.
-   * \return Pointer to a Op, valid throughout program lifetime.
+   * \brief Look up a registered operator by name.
+   * \param op_name The canonical name.
+   * \return A copy of the reference-counted handle to the canonical node.
+   * \throws AttributeError if the operator is not registered.
    */
-  TVM_DLL static const Op& Get(const ffi::String& op_name);
-
+  TVM_DLL static Op Get(const ffi::String& op_name);
+  /*! \brief List registered operator names. \return Names in unspecified order. */
+  TVM_DLL static ffi::Array<ffi::String> ListNames();
   TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(Op, Expr, OpNode);
 
  private:
-  /*!
-   * \brief Get generic attrmap given attr name
-   * \param key The attribute key
-   * \return The attr map.
-   */
-  TVM_DLL static const AttrRegistryMapContainerMap<Op>& GetAttrMapContainer(const ffi::String& key);
+  TVM_DLL static ffi::List<ffi::Any> GetAttrColumn(const ffi::String& attr_name);
 };
 
-/*!
- * \brief Helper structure to register operators
- * \sa TVM_REGISTER_OP
- */
-class OpRegEntry {
+/*! \brief Noncopyable temporary builder for a canonical Op; see Op for the example. */
+class OpDef {
  public:
-  /*! \return the operator */
-  const Op& op() const { return op_; }
   /*!
-   * \brief setter function during registration
-   *  Set the description of operator
-   * \param descr the description string.
-   * \return reference to self.
+   * \brief Get or create the named Op without changing its documentation.
+   * \param name The canonical operator name.
    */
-  inline OpRegEntry& describe(const std::string& descr);  // NOLINT(*)
+  TVM_DLL explicit OpDef(const ffi::String& name);
   /*!
-   * \brief Add argument information to the function.
-   * \param name Name of the argument.
-   * \param type Type of the argument.
-   * \param description Description of the argument.
-   * \return reference to self.
+   * \brief Get or create the named Op and set its documentation.
+   * \param name The canonical operator name.
+   * \param doc Operator documentation, replacing any existing documentation.
    */
-  inline OpRegEntry& add_argument(const std::string& name, const std::string& type,
-                                  const std::string& description);
+  TVM_DLL OpDef(const ffi::String& name, const ffi::String& doc);
+  OpDef(const OpDef&) = delete;
+  OpDef& operator=(const OpDef&) = delete;
+  OpDef(OpDef&&) = delete;
+  OpDef& operator=(OpDef&&) = delete;
+  /*! \brief Get the canonical operator. \return A copied handle, safe after this builder dies. */
+  Op op() const { return op_; }
   /*!
-   * \brief Set the attrs type key and index to be AttrsType.
-   * \tparam AttrsType the attribute type to b set.
-   * \return reference to self.
+   * \brief Append a required value-argument descriptor.
+   * \param name Argument name.
+   * \param doc Argument documentation.
+   * \return This builder. Without allow_extra_args(), this is part of the complete list.
    */
-  template <typename AttrsType>
-  inline OpRegEntry& set_attrs_type();
+  TVM_DLL OpDef& add_arg(const ffi::String& name, const ffi::String& doc);
   /*!
-   * \brief Set the attrs type key and index to be AttrsType.
-   * \param key The attribute type key to be set.
-   * \return reference to self.
+   * \brief Set the attribute object type key and runtime index together.
+   * \tparam AttrsType The attribute object node type.
+   * \return This builder.
    */
-  inline OpRegEntry& set_attrs_type_key(const ffi::String& key);
+  template <typename AttrsType>
+  OpDef& attrs_type() {
+    uint32_t index = AttrsType::RuntimeTypeIndex();
+    get()->attrs_type_key = AttrsType::_type_key;
+    get()->attrs_type_index = index;
+    return *this;
+  }
   /*!
-   * \brief Set the num_inputs
-   * \param n The number of inputs to be set.
-   * \return reference to self.
+   * \brief Resolve and set the attribute object type key and runtime index together.
+   * \param key A registered attribute object type key.
+   * \return This builder. An unknown key raises an error before modifying metadata.
    */
-  inline OpRegEntry& set_num_inputs(int32_t n);  // NOLINT(*)
+  TVM_DLL OpDef& set_attrs_type_key(const ffi::String& key);
   /*!
-   * \brief Set the support level of op.
-   * \param level The support level.
-   * \return reference to self.
+   * \brief Allow value arguments after the required prefix described by args_info.
+   * \return This builder. An empty prefix accepts any number of value arguments.
    */
-  inline OpRegEntry& set_support_level(int32_t level);  // NOLINT(*)
+  OpDef& allow_extra_args() {
+    get()->allow_extra_args = true;
+    return *this;
+  }
   /*!
-   * \brief Register additional attributes to operator.
-   * \param attr_name The name of the attribute.
-   * \param value The value to be set.
-   * \param plevel The priority level of this set,
-   *  an higher priority level attribute
-   *  will replace lower priority level attribute.
-   *  Must be bigger than 0.
-   *
-   *  Cannot set with same plevel twice in the code.
-   *
-   * \tparam ValueType The type of the value to be set.
+   * \brief Append a descriptor for a type argument in Call.ty_args.
+   * \param name Type-argument name.
+   * \param doc Type-argument documentation.
+   * \return This builder. Inference retains responsibility for type-argument counts.
    */
-  template <typename ValueType>
-  inline OpRegEntry& set_attr(const std::string& attr_name,  // NOLINT(*)
-                              const ValueType& value, int plevel = 10);
-
+  TVM_DLL OpDef& add_ty_arg(const ffi::String& name, const ffi::String& doc);
   /*!
-   * \brief Resets an attr of the registry.
-   * \param attr_name The name of the attribute.
+   * \brief Register an extensible attribute, rejecting duplicates unless overridden.
+   * \tparam ValueType The attribute value type.
+   * \param attr_name Attribute column name.
+   * \param value Non-null attribute value.
+   * \param override Whether to replace an existing value; no prior value is retained.
+   * \return This builder. Existing cached maps observe the new value.
    */
-  inline void reset_attr(const std::string& attr_name);
-
-  // set the name of the op to be the same as registry
-  inline OpRegEntry& set_name() {  // NOLINT(*)
-    if (get()->name.length() == 0) {
-      get()->name = name;
-    }
+  template <typename ValueType>
+  OpDef& set_attr(const ffi::String& attr_name, const ValueType& value, bool override = false) {
+    UpdateAttr(attr_name, ffi::Any(value), override);
     return *this;
   }
   /*!
-   * \brief Register or get a new entry.
-   * \param name The name of the operator.
-   * \return the corresponding entry.
+   * \brief Remove the current attribute value, with no priority fallback.
+   * \param attr_name Attribute column name; missing columns and values are ignored.
+   * \return This builder. Existing cached maps observe the removal.
    */
-  TVM_DLL static OpRegEntry& RegisterOrGet(const ffi::String& name);
+  TVM_DLL OpDef& reset_attr(const ffi::String& attr_name);
 
  private:
-  template <typename, typename>
-  friend class AttrRegistry;
-  // the name
-  std::string name;
-  /*! \brief The operator */
+  OpNode* get() { return const_cast<OpNode*>(op_.operator->()); }
+  TVM_DLL void UpdateAttr(const ffi::String& attr_name, ffi::Any value, bool override);
   Op op_;
-  /*! \brief Construct the non-null Op for this registry entry. */
-  static Op MakeOp(uint32_t reg_index);
-  // private constructor
-  TVM_DLL OpRegEntry(uint32_t reg_index);
-  // return internal pointer to op.
-  inline OpNode* get();
-  // update the attribute OpAttrMap
-  TVM_DLL void UpdateAttr(const ffi::String& key, ffi::Any value, int plevel);
 };
 
 /*!
- * \brief ffi::Map<Op,ValueType> used to store meta-information about Op.
- * \tparam ValueType The type of the value stored in map.
+ * \brief Typed live view of an Op attribute column.
+ *
+ * Copies share the retained mutable column and observe later registrations,
+ * replacements, resets, and growth. Values are returned by value, never as
+ * references into a backing array. Registration is not synchronized with readers.
+ * \tparam ValueType The attribute value type.
  */
 template <typename ValueType>
-class OpAttrMap : public AttrRegistryMap<Op, ValueType> {
+class OpAttrMap {
  public:
   /*!
-   * \brief get the corresponding value element at op with default value.
-   * \param expr The key to the map
-   * \param def_value The default value when the key does not exist
-   *         or if expr is not an Op.
-   * \return the const reference to the content value.
+   * \brief Check whether an operator has a value in this column.
+   * \param op The operator.
+   * \return 1 if present, 0 otherwise.
    */
-  inline ValueType get(const Expr& expr, ValueType def_value) const;
-
-  using TParent = AttrRegistryMap<Op, ValueType>;
-  using TParent::count;
-  using TParent::get;
-  using TParent::operator[];
+  int count(const Op& op) const {
+    return op->index_ < column_.size() && column_[op->index_] != nullptr;
+  }
+  /*!
+   * \brief Look up an operator's attribute, raising InternalError if absent.
+   * \param op The operator.
+   * \return The value converted to ValueType.
+   */
+  ValueType operator[](const Op& op) const {
+    TVM_FFI_ICHECK(count(op)) << "Attribute " << attr_name_ << " has not been registered for "
+                              << op->name;
+    if constexpr (std::is_same_v<ValueType, ffi::Any>) {
+      return column_[op->index_];
+    } else {
+      return column_[op->index_].template cast<ValueType>();
+    }
+  }
+  /*!
+   * \brief Look up an operator's attribute with a fallback.
+   * \param op The operator.
+   * \param def_value Fallback when the attribute is absent.
+   * \return The registered value or def_value.
+   */
+  ValueType get(const Op& op, ValueType def_value) const {
+    return count(op) ? (*this)[op] : def_value;
+  }
+  /*!
+   * \brief Look up an expression's attribute with a fallback.
+   * \param expr A defined expression.
+   * \param def_value Fallback when expr is not an Op or its attribute is absent.
+   * \return The registered value or def_value.
+   */
+  ValueType get(const Expr& expr, ValueType def_value) const {
+    TVM_FFI_ICHECK(expr.defined());
+    if (const auto* op = expr.as<OpNode>()) {
+      return get(ffi::GetRef<Op>(op), def_value);
+    }
+    return def_value;
+  }
 
  private:
   friend class Op;
-  // constructor
-  explicit OpAttrMap(const AttrRegistryMapContainerMap<Op>& map) : TParent(map) {}
+  OpAttrMap(ffi::List<ffi::Any> column, ffi::String attr_name)
+      : column_(std::move(column)), attr_name_(std::move(attr_name)) {}
+  ffi::List<ffi::Any> column_;
+  ffi::String attr_name_;
 };
 
-// internal macros to make
-#define TVM_OP_REGISTER_VAR_DEF [[maybe_unused]] static ::tvm::OpRegEntry& __make_##Op
-
-/*!
- * \def TVM_REGISTER_OP
- * \brief Register a new operator, or set attribute of the corresponding op.
- *
- * \param OpName The name of registry
- *
- * \code
- *
- *  TVM_REGISTER_OP("add")
- *  .describe("add two inputs together")
- *  .set_num_inputs(2)
- *  .set_attr<OpKernel>("gpu_kernel", AddKernel);
- *
- * \endcode
- */
-#define TVM_REGISTER_OP(OpName)                              \
-  TVM_FFI_STR_CONCAT(TVM_OP_REGISTER_VAR_DEF, __COUNTER__) = \
-      ::tvm::OpRegEntry::RegisterOrGet(OpName).set_name()
-
-// implementations
-
-template <typename ValueType>
-inline OpAttrMap<ValueType> Op::GetAttrMap(const ffi::String& key) {
-  return OpAttrMap<ValueType>(Op::GetAttrMapContainer(key));
-}
-
-inline OpNode* OpRegEntry::get() { return const_cast<OpNode*>(op_.operator->()); }
-
-inline OpRegEntry& OpRegEntry::describe(const std::string& descr) {  // NOLINT(*)
-  get()->description = descr;
-  return *this;
-}
-
-inline OpRegEntry& OpRegEntry::add_argument(const std::string& name, const std::string& type,
-                                            const std::string& description) {
-  auto n = ffi::make_object<ArgumentInfoNode>();
-  n->name = name;
-  n->type_info = type;
-  n->description = description;
-  get()->arguments.push_back(ArgumentInfo(n));
-  return *this;
-}
-
-inline OpRegEntry& OpRegEntry::set_num_inputs(int32_t n) {  // NOLINT(*)
-  get()->num_inputs = n;
-  return *this;
-}
-
-template <typename AttrsType>
-inline OpRegEntry& OpRegEntry::set_attrs_type() {  // NOLINT(*)
-  get()->attrs_type_key = AttrsType::_type_key;
-  get()->attrs_type_index = AttrsType::RuntimeTypeIndex();
-  return *this;
-}
-
-inline OpRegEntry& OpRegEntry::set_attrs_type_key(const ffi::String& key) {  // NOLINT(*)
-  get()->attrs_type_key = key;
-  get()->attrs_type_index = tvm::ffi::TypeKeyToIndex(key.c_str());
-  return *this;
-}
-
-inline OpRegEntry& OpRegEntry::set_support_level(int32_t n) {  // NOLINT(*)
-  get()->support_level = n;
-  return *this;
-}
-
 template <typename ValueType>
-inline OpRegEntry& OpRegEntry::set_attr(  // NOLINT(*)
-    const std::string& attr_name, const ValueType& value, int plevel) {
-  TVM_FFI_ICHECK_GT(plevel, 0) << "plevel in set_attr must be greater than 0";
-  UpdateAttr(attr_name, Any(value), plevel);
-  return *this;
-}
-
-// member functions of OpAttrMap
-
-template <typename ValueType>
-inline ValueType OpAttrMap<ValueType>::get(const Expr& expr, ValueType def_value) const {
-  TVM_FFI_ICHECK(expr.defined());
-  if (const OpNode* op = expr.as<OpNode>()) {
-    return this->map_.get(ffi::GetRef<Op>(op), def_value);
-  } else {
-    return def_value;
-  }
+OpAttrMap<ValueType> Op::GetAttrMap(const ffi::String& attr_name) {
+  return OpAttrMap<ValueType>(GetAttrColumn(attr_name), attr_name);
 }
 
 }  // namespace tvm
diff --git a/include/tvm/tirx/op.h b/include/tvm/tirx/op.h
index 7089522496..32b417ba2e 100644
--- a/include/tvm/tirx/op.h
+++ b/include/tvm/tirx/op.h
@@ -45,13 +45,6 @@
 
 namespace tvm::prim {
 
-#define TVM_TIR_REGISTER_OP(OpName)                               \
-  TVM_REGISTER_OP("tirx." OpName)                                 \
-      .set_attr<TScriptPrinterName>("TScriptPrinterName", OpName) \
-      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"), /*plevel=*/1)
-
-#define TVM_TIRX_REGISTER_OP(OpName) TVM_TIR_REGISTER_OP(OpName)
-
 // Shared primitive construction and constants are declared in ir/prim/op.h.
 
 /*!
diff --git a/include/tvm/tirx/op_attr_types.h b/include/tvm/tirx/op_attr_types.h
index f333d03887..db185cdf47 100644
--- a/include/tvm/tirx/op_attr_types.h
+++ b/include/tvm/tirx/op_attr_types.h
@@ -21,7 +21,7 @@
  * \file tvm/tirx/op_attr_types.h
  * \brief Attribute types in the Op registry for TIR ops.
  *
- * These attributes can be set via OpRegEntry::set_attr
+ * These attributes can be set via OpDef::set_attr
  *
  * \sa tvm/ir/op.h
  */
diff --git a/python/tvm/backend/cuda/ptx/engine.py b/python/tvm/backend/cuda/ptx/engine.py
index 0b7de8a736..242ed6b759 100644
--- a/python/tvm/backend/cuda/ptx/engine.py
+++ b/python/tvm/backend/cuda/ptx/engine.py
@@ -40,7 +40,7 @@ per-instruction generated or hand-written code:
 from tvm.backend.cuda.codegen.registry import register_codegen
 from tvm.backend.cuda.codegen.utils import parse_str
 from tvm.backend.cuda.op import cuda_cvta_generic_to_shared, cuda_func_call
-from tvm.ir import Call, TensorLoad
+from tvm.ir import Call, Op, TensorLoad
 from tvm.ir.op import register_op_attr
 from tvm.ir.type import PointerType, PrimType
 from tvm.runtime import const
@@ -92,6 +92,9 @@ def register_table(table: dict[str, InstructionEntry]) -> None:
         # First attr call implicitly creates the Op registry entry. Effect
         # kind must exist before any side-effect analysis sees the op.
         register_op_attr(entry.op_name, "TCallEffectKind", _EFFECT_OPAQUE)
+        # Encoded variants have no shared named operand prefix; codegen checks
+        # each variant's operands, predicate, modifier tokens, and marker.
+        Op.get(entry.op_name).set_allow_extra_args()
         # The printer name is the *surface* path a user can type, which is the
         # mnemonic (several `mov_*` entries all answer to `T.ptx.mov`), not the
         # table key. Reparsing re-dispatches on the operand shape.
@@ -102,7 +105,7 @@ def register_table(table: dict[str, InstructionEntry]) -> None:
         # `PTXNamespace.__getattr__`. The escape is the identity for every
         # other family, and `gen_stubs` already spells the attribute this way.
         family = escape_token(entry.family)
-        register_op_attr(entry.op_name, "TScriptPrinterName", f"ptx.{family}", level=20)
+        register_op_attr(entry.op_name, "TScriptPrinterName", f"ptx.{family}")
         register_op_attr(entry.op_name, "TIRxOpCategory", "device_intrin")
         register_op_attr(entry.op_name, "TDeviceIntrinsicNamespace", "ptx")
         register_codegen(f"ptx.{entry.name}")(_make_codegen(entry))
@@ -111,7 +114,10 @@ def register_table(table: dict[str, InstructionEntry]) -> None:
 def register_addr() -> None:
     """Register the pure address-expression op consumed by PTX instructions."""
     register_op_attr(_ADDR_OP_NAME, "TCallEffectKind", _EFFECT_PURE)
-    register_op_attr(_ADDR_OP_NAME, "TScriptPrinterName", "ptx.addr", level=20)
+    op = Op.get(_ADDR_OP_NAME)
+    op.add_arg("base", "The base pointer.")
+    op.add_arg("byte_offset", "The offset in bytes.")
+    register_op_attr(_ADDR_OP_NAME, "TScriptPrinterName", "ptx.addr")
     register_op_attr(_ADDR_OP_NAME, "TIRxOpCategory", "device_intrin")
     register_op_attr(_ADDR_OP_NAME, "TDeviceIntrinsicNamespace", "ptx")
     register_codegen("ptx.addr")(_unconsumed_addr_codegen)
diff --git a/python/tvm/ir/op.py b/python/tvm/ir/op.py
index e1de79d9e1..ae5d500e13 100644
--- a/python/tvm/ir/op.py
+++ b/python/tvm/ir/op.py
@@ -32,133 +32,52 @@ class Op(Expr):
 
     @staticmethod
     def get(op_name):
-        """Get the Op for a given name
+        """Get a registered operator by name.
 
         Parameters
         ----------
         op_name : str
-            The operator name
+            The canonical operator name.
 
         Returns
         -------
-        op : Op
-            The op of the corresponding name
+        Op
+            A handle to the registered operator.
         """
         return _ffi_api.GetOp(op_name)
 
-    def get_attr(self, attr_name):
-        """Get additional attribute about the operator.
-
-        Parameters
-        ----------
-        attr_name : str
-            The attribute name.
-
-        Returns
-        -------
-        value : object
-            The attribute value
-        """
-        return _ffi_api.OpGetAttr(self, attr_name)
-
-    def has_attr(self, attr_name):
-        """Check whether the operator has additional attribute.
-
-        Parameters
-        ----------
-        attr_name : str
-            The attribute name.
+    @staticmethod
+    def list_op_names():
+        """List registered operator names in unspecified order.
 
         Returns
         -------
-        value : bool
-            Whether the operator has additional attribute
+        list[str]
+            The registered operator names.
         """
-        return _ffi_api.OpHasAttr(self, attr_name)
+        return _ffi_api.ListOpNames()
 
-    def set_attr(self, attr_name, value, plevel=10):
-        """Set attribute about the operator.
+    def set_attr(self, attr_name, value, override=False):
+        """Set an operator attribute.
 
         Parameters
         ----------
         attr_name : str
-            The attribute name
-
+            Attribute column name.
         value : object
-            The attribute value
-
-        plevel : int
-            The priority level
-        """
-        _ffi_api.OpSetAttr(self, attr_name, value, plevel)
-
-    def reset_attr(self, attr_name):
-        """Reset attribute about the operator.
-
-        Parameters
-        ----------
-        attr_name : str
-            The attribute name
-        """
-        _ffi_api.OpResetAttr(self, attr_name)
-
-    def add_argument(self, name, type, description):  # pylint: disable=redefined-builtin
-        """Add arguments information to the function.
-
-        Parameters
-        ----------
-        name : str
-            The argument name.
-        type : str
-            The argument type.
-        description : str
-            The argument description.
-        """
-        _ffi_api.OpAddArgument(self, name, type, description)
-
-    def set_support_level(self, level):
-        """Set the support level of op.
-
-        Parameters
-        ----------
-        level : int
-            The support level.
-        """
-        _ffi_api.OpSetSupportLevel(self, level)
-
-    def set_num_inputs(self, n):
-        """Set the support level of op.
-
-        Parameters
-        ----------
-        n : int
-            The input number.
-        """
-        _ffi_api.OpSetNumInputs(self, n)
-
-    def set_attrs_type_key(self, key):
-        """Set the attribute type key of op.
-
-        Parameters
-        ----------
-        key : str
-            The type key.
-        """
-        _ffi_api.OpSetAttrsTypeKey(self, key)
-
-    @staticmethod
-    def list_op_names():
-        """List all the op names in the op registry.
+            Non-None attribute value.
+        override : bool, optional
+            Replace an existing value if True. Duplicate registration otherwise
+            raises ValueError. Cached views observe replacements; no history is kept.
 
         Returns
         -------
-        value : List[str]
-            The registered op names
+        None
         """
-        return _ffi_api.ListOpNames()
+        self._set_attr(attr_name, value, override)
 
 
-def register_op_attr(op_name, attr_key, value=None, level=10):
+def register_op_attr(op_name, attr_key, value=None, override=False):
     """Register an operator property of an operator by name.
 
     Parameters
@@ -172,18 +91,20 @@ def register_op_attr(op_name, attr_key, value=None, level=10):
     value : object, optional
         The value to set
 
-    level : int, optional
-        The priority level
+    override : bool, optional
+        Replace an existing value if True; otherwise duplicate registration raises
+        ValueError. Cached views observe replacements; no priority history is kept.
 
     Returns
     -------
-    fregister : function
-        Register function if value is not specified.
+    result : object or function
+        The registered value when supplied, or a decorator that registers and
+        returns its argument. The named Op is created if it does not exist.
     """
 
     def _register(v):
         """internal register function"""
-        _ffi_api.RegisterOpAttr(op_name, attr_key, v, level)
+        _ffi_api.RegisterOpAttr(op_name, attr_key, v, override)
         return v
 
     return _register(value) if value is not None else _register
diff --git a/python/tvm/relax/op/base.py b/python/tvm/relax/op/base.py
index 77f577fe64..60914ed803 100644
--- a/python/tvm/relax/op/base.py
+++ b/python/tvm/relax/op/base.py
@@ -37,7 +37,7 @@ py_print = print  # pylint: disable=invalid-name
 def register_gradient(
     op_name: str,
     fgradient: Callable[[Var, Call, Var, "BlockBuilder"], list[Expr]] | None = None,
-    level: int = 10,
+    override: bool = False,
 ):
     """Register operator gradient function for a relax operator.
 
@@ -50,10 +50,16 @@ def register_gradient(
          -> partials: List[Expr]
         The gradient function being used.
 
-    level: int
-        The priority level
+    override: bool, optional
+        Replace an existing gradient if True; duplicate registration otherwise
+        raises ValueError.
+
+    Returns
+    -------
+    result : callable
+        The registered gradient, or a decorator if fgradient is not supplied.
     """
-    return tvm.ir.register_op_attr(op_name, "FPrimalGradient", fgradient, level)
+    return tvm.ir.register_op_attr(op_name, "FPrimalGradient", fgradient, override)
 
 
 def null_value() -> Call:
diff --git a/python/tvm/target/intrin.py b/python/tvm/target/intrin.py
index 11ec8c33cc..eee85d15fb 100644
--- a/python/tvm/target/intrin.py
+++ b/python/tvm/target/intrin.py
@@ -76,6 +76,6 @@ def _rule_float_direct(op):
 
 
 # opencl pattern for exp
-register_intrin_lowering("tirx.exp", target="opencl", f=_rule_float_direct, level=99)
+register_intrin_lowering("tirx.exp", target="opencl", f=_rule_float_direct, override=True)
 # default pattern for exp
-register_intrin_lowering("tirx.exp", target="default", f=_rule_float_suffix, level=99)
+register_intrin_lowering("tirx.exp", target="default", f=_rule_float_suffix, override=True)
diff --git a/python/tvm/tirx/op.py b/python/tvm/tirx/op.py
index 49fc608891..f54910acc3 100644
--- a/python/tvm/tirx/op.py
+++ b/python/tvm/tirx/op.py
@@ -55,7 +55,7 @@ def register_intrin_lowering(
     target,
     *,
     f=None,
-    level=10,
+    override=False,
 ):
     """Register Op lowering function
 
@@ -70,18 +70,19 @@ def register_intrin_lowering(
     f : function, optional
         The function to be registered.
 
-    level : int
-        The priority level
+    override : bool, optional
+        Replace an existing lowering if True; duplicate registration otherwise
+        raises ValueError.
 
     Returns
     -------
-    fregister : function
-        Register op lowering function if f is not specified.
+    result : function
+        The registered lowering, or a decorator if f is not supplied.
     """
 
     def _register(f):
         """internal register function"""
-        _ffi_api.RegisterOpLowerIntrinsic(op_name, f, target, level)
+        tvm.ir.register_op_attr(op_name, target + ".FLowerIntrinsic", f, override)
         return f
 
     return _register(f) if f is not None else _register
diff --git a/python/tvm/tirx/script/ir_builder/op.py b/python/tvm/tirx/script/ir_builder/op.py
index 946a8fcd3e..a8e794dfef 100644
--- a/python/tvm/tirx/script/ir_builder/op.py
+++ b/python/tvm/tirx/script/ir_builder/op.py
@@ -247,16 +247,15 @@ class WebGPUNamespace:
 webgpu = WebGPUNamespace()
 
 
-def _register_script_namespace_printer_names(ns_obj, dotted_prefix):
+def _register_script_namespace_printer_names(ns_obj, dotted_prefix, override=False):
     def register_printer_name(op_name, script_name):
         try:
-            ir.Op.get(op_name)
-        except Exception:
+            op = ir.Op.get(op_name)
+        except AttributeError:
             return
-        try:
-            _register_op_attr(op_name, "TScriptPrinterName", script_name, level=20)
-        except Exception:
-            pass
+        if op.has_attr("TScriptPrinterName") and op.get_attr("TScriptPrinterName") == script_name:
+            return
+        _register_op_attr(op_name, "TScriptPrinterName", script_name, override=override)
 
     def visit(ns_obj, dotted_prefix):
         # If the namespace object itself maps to an op via __call__
@@ -298,8 +297,19 @@ def _get_script_namespace(name: str) -> object:
     raise AttributeError(f"No script namespace {name!r}")
 
 
-def register_script_namespace(name: str, namespace: object) -> object:
-    """Register a TVMScript namespace on the TIRx builder facade."""
+def register_script_namespace(name: str, namespace: object, override: bool = False) -> object:
+    """Register a construction namespace and return it.
+
+    Parameters
+    ----------
+    name : str
+        Namespace name on the TIRx builder facade.
+    namespace : object
+        Construction namespace object.
+    override : bool, optional
+        Replace differing operator printer names if True. Existing equal names
+        are reused; other duplicates raise ValueError.
+    """
     _SCRIPT_NAMESPACES[name] = namespace
     globals()[name] = namespace
     if "__all__" in globals() and name not in __all__:
@@ -320,7 +330,7 @@ def register_script_namespace(name: str, namespace: object) -> object:
         if isinstance(module_all, list) and name not in module_all:
             module_all.append(name)
 
-    _register_script_namespace_printer_names(namespace, name)
+    _register_script_namespace_printer_names(namespace, name, override)
     return namespace
 
 
diff --git a/src/backend/cuda/codegen/codegen_cuda.cc b/src/backend/cuda/codegen/codegen_cuda.cc
index ce06c4f8a4..7cba4d4c1d 100644
--- a/src/backend/cuda/codegen/codegen_cuda.cc
+++ b/src/backend/cuda/codegen/codegen_cuda.cc
@@ -1035,19 +1035,19 @@ void CodeGenCUDA::Dispatch_(const CallNode* op, std::ostream& os) {
     }
   }
 
-  static const Op& tvm_fill_fragment_op = Op::Get("tirx.tvm_fill_fragment");
-  static const Op& tvm_load_matrix_sync_op = Op::Get("tirx.tvm_load_matrix_sync");
-  static const Op& tvm_store_matrix_sync_op = Op::Get("tirx.tvm_store_matrix_sync");
-  static const Op& tvm_mma_sync_op = Op::Get("tirx.tvm_mma_sync");
-  static const Op& tvm_bmma_sync_op = Op::Get("tirx.tvm_bmma_sync");
-  static const Op& mma_store_op = Op::Get("tirx.mma_store");
-  static const Op& mma_fill_op = Op::Get("tirx.mma_fill");
-  static const Op& ptx_mma_legacy_op = Op::Get("tirx.ptx_legacy.mma");
-  static const Op& ptx_ldmatrix_legacy_op = Op::Get("tirx.ptx_legacy.ldmatrix");
-  static const Op& mma_store_legacy_op = Op::Get("tirx.mma_store_legacy");
-  static const Op& mma_fill_legacy_op = Op::Get("tirx.mma_fill_legacy");
-  static const Op& ptx_ldg32_op = Op::Get("tirx.s_tir.ldg32");
-  static const Op& cuda_func_call_op = Op::Get("tirx.cuda.func_call");
+  static const Op tvm_fill_fragment_op = Op::Get("tirx.tvm_fill_fragment");
+  static const Op tvm_load_matrix_sync_op = Op::Get("tirx.tvm_load_matrix_sync");
+  static const Op tvm_store_matrix_sync_op = Op::Get("tirx.tvm_store_matrix_sync");
+  static const Op tvm_mma_sync_op = Op::Get("tirx.tvm_mma_sync");
+  static const Op tvm_bmma_sync_op = Op::Get("tirx.tvm_bmma_sync");
+  static const Op mma_store_op = Op::Get("tirx.mma_store");
+  static const Op mma_fill_op = Op::Get("tirx.mma_fill");
+  static const Op ptx_mma_legacy_op = Op::Get("tirx.ptx_legacy.mma");
+  static const Op ptx_ldmatrix_legacy_op = Op::Get("tirx.ptx_legacy.ldmatrix");
+  static const Op mma_store_legacy_op = Op::Get("tirx.mma_store_legacy");
+  static const Op mma_fill_legacy_op = Op::Get("tirx.mma_fill_legacy");
+  static const Op ptx_ldg32_op = Op::Get("tirx.s_tir.ldg32");
+  static const Op cuda_func_call_op = Op::Get("tirx.cuda.func_call");
 
   if (op->op.same_as(tvm_fill_fragment_op)) {
     codegen_tags_.insert("mma");
@@ -1590,7 +1590,7 @@ void CodeGenCUDA::Dispatch_(const AttrStmtNode* op) {
     TVM_FFI_ICHECK(queue_id && queue_id->value == 0)
         << "For CUDA, the index of an async queue must be 0.";
     this->Dispatch(op->body);
-    static const Op& ptx_cp_async_commit_group_op = Op::Get("tirx.ptx.cp_async_commit_group");
+    static const Op ptx_cp_async_commit_group_op = Op::Get("tirx.ptx.cp_async_commit_group");
     // ptx Call layout: [operands...] [slot tokens] [pred marker ""].
     auto commit_group = Call(PrimType::Void(), ptx_cp_async_commit_group_op,
                              {StringImm("async"), StringImm("commit_group"), StringImm("")})
@@ -1605,7 +1605,7 @@ void CodeGenCUDA::Dispatch_(const AttrStmtNode* op) {
     TVM_FFI_ICHECK(queue_id && queue_id->value == 0)
         << "For CUDA, the index of an async queue must be 0.";
     auto wait_cnt = wait_attrs.second;
-    static const Op& ptx_cp_async_wait_group_op = Op::Get("tirx.ptx.cp_async_wait_group");
+    static const Op ptx_cp_async_wait_group_op = Op::Get("tirx.ptx.cp_async_wait_group");
     // ptx Call layout: [operands...] [slot tokens] [pred marker ""]. The group
     // count is a role="imm" operand baked into the instruction text, so it must
     // already be a compile-time constant here (the pipeline pass guarantees it).
diff --git a/src/backend/cuda/codegen/intrin_rule_cuda.cc b/src/backend/cuda/codegen/intrin_rule_cuda.cc
index de2599ebe7..f9a3f2b5ff 100644
--- a/src/backend/cuda/codegen/intrin_rule_cuda.cc
+++ b/src/backend/cuda/codegen/intrin_rule_cuda.cc
@@ -129,17 +129,17 @@ struct CUDAPopcount {
 struct CUDAWarpIntrinsic {
   const Op operator()(const PrimType& ty, const Op& orig_op) const {
     if (orig_op.same_as(builtin::tvm_warp_shuffle())) {
-      static const Op& cuda_shfl_sync_op = Op::Get("tirx.cuda.__shfl_sync");
+      static const Op cuda_shfl_sync_op = Op::Get("tirx.cuda.__shfl_sync");
       return cuda_shfl_sync_op;
     } else if (orig_op.same_as(builtin::tvm_warp_shuffle_up())) {
-      static const Op& cuda_shfl_up_sync_op = Op::Get("tirx.cuda.__shfl_up_sync");
+      static const Op cuda_shfl_up_sync_op = Op::Get("tirx.cuda.__shfl_up_sync");
       return cuda_shfl_up_sync_op;
     } else if (orig_op.same_as(builtin::tvm_warp_shuffle_down())) {
-      static const Op& cuda_shfl_down_sync_op = Op::Get("tirx.cuda.__shfl_down_sync");
+      static const Op cuda_shfl_down_sync_op = Op::Get("tirx.cuda.__shfl_down_sync");
       return cuda_shfl_down_sync_op;
     } else {
       TVM_FFI_ICHECK(orig_op.same_as(builtin::tvm_warp_shuffle_xor()));
-      static const Op& cuda_shfl_xor_sync_op = Op::Get("tirx.cuda.__shfl_xor_sync");
+      static const Op cuda_shfl_xor_sync_op = Op::Get("tirx.cuda.__shfl_xor_sync");
       return cuda_shfl_xor_sync_op;
     }
   }
@@ -147,7 +147,7 @@ struct CUDAWarpIntrinsic {
 
 static PrimExpr DispatchCUDAWarpActiveMask(const PrimExpr& e) {
   const CallNode* call = e.as<CallNode>();
-  static const Op& cuda_active_mask_op = Op::Get("tirx.cuda.__activemask");
+  static const Op cuda_active_mask_op = Op::Get("tirx.cuda.__activemask");
   ffi::Array<PrimExpr> args = call->args.as_or_throw<ffi::Array<PrimExpr>>();
   return Call(e.ty(), cuda_active_mask_op, args).as_or_throw<PrimExpr>();
 }
@@ -165,183 +165,170 @@ static PrimExpr DispatchCUDAShuffle(const PrimExpr& e) {
 
 void RegisterCudaIntrinRules() {
   // clang-format off
-TVM_REGISTER_OP("prim.clz")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic",
-                               DispatchPureExtern<CUDAMath, /*dtype_from_arg=*/true>);
+  OpDef("prim.clz")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic",
+                                 DispatchPureExtern<CUDAMath, /*dtype_from_arg=*/true>);
 
-TVM_REGISTER_OP("tirx.floor")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+  OpDef("tirx.floor")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
 
-TVM_REGISTER_OP("prim.ceil")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+  OpDef("prim.ceil")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
 
-TVM_REGISTER_OP("tirx.trunc")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+  OpDef("tirx.trunc")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
 
-TVM_REGISTER_OP("tirx.fabs")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+  OpDef("tirx.fabs")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
 
-TVM_REGISTER_OP("tirx.round")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+  OpDef("tirx.round")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
 
-TVM_REGISTER_OP("tirx.nearbyint")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+  OpDef("tirx.nearbyint")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
 
-TVM_REGISTER_OP("tirx.exp")
-    .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+  OpDef("tirx.exp")
+      .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
 
-TVM_REGISTER_OP("tirx.exp2")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+  OpDef("tirx.exp2")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
 
-TVM_REGISTER_OP("tirx.exp10")
-    .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.erf")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.log")
-    .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("prim.log2")
-    .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.log10")
-    .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.tan")
-    // Now the fast math version of tan and the default version of tan are same.
-    .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic",
-                               DispatchPureExtern<CUDAFastMathTan>)
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.cos")
-    .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.cosh")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.sin")
-    .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.sinh")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.atan")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.tanh")
-    .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.sqrt")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.rsqrt")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.pow")
-    .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-TVM_REGISTER_OP("tirx.popcount")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAPopcount>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_up")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_down")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_xor")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_activemask")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAWarpActiveMask);
-
-TVM_REGISTER_OP("tirx.fmod")
-    .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
-
-// Register low-level CUDA device intrinsics.
-// TODO(tvm-team): consider make CUDA its own subfolder and create a file for low-level builtins.
-TVM_REGISTER_OP("tirx.cuda.__shfl_sync")
-    .set_num_inputs(4)
-    .add_argument("mask", "Expr", "The thread mask.")
-    .add_argument("var", "Expr", "The variable to sync.")
-    .add_argument("lane", "Expr", "The source thread id.")
-    .add_argument("width", "Expr", "The warp thread width, must be a power of 2.")
-    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
-    .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"),
-                                               10)
-    .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__shfl_sync"), 10)
-    .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_sync")
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
-    .set_attr<bool>("cuda.need_warp_shuffle", true);
-
-TVM_REGISTER_OP("tirx.cuda.__shfl_up_sync")
-    .set_num_inputs(4)
-    .add_argument("mask", "Expr", "The thread mask.")
-    .add_argument("var", "Expr", "The variable to sync.")
-    .add_argument("delta", "Expr", "The source lane id offset to be added.")
-    .add_argument("width", "Expr", "The warp thread width, must be a power of 2.")
-    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
-    .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"),
-                                               10)
-    .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__shfl_up_sync"),
-                                        10)
-    .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_up_sync")
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
-    .set_attr<bool>("cuda.need_warp_shuffle", true);
-
-TVM_REGISTER_OP("tirx.cuda.__shfl_down_sync")
-    .set_num_inputs(4)
-    .add_argument("mask", "Expr", "The thread mask.")
-    .add_argument("var", "Expr", "The variable to sync.")
-    .add_argument("delta", "Expr", "The source lane id offset to be subtracted.")
-    .add_argument("width", "Expr", "The warp thread width, must be a power of 2.")
-    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
-    .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"),
-                                               10)
-    .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
-                                        ffi::String("cuda.__shfl_down_sync"), 10)
-    .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_down_sync")
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
-    .set_attr<bool>("cuda.need_warp_shuffle", true);
-
-TVM_REGISTER_OP("tirx.cuda.__shfl_xor_sync")
-    .set_num_inputs(4)
-    .add_argument("mask", "Expr", "The thread mask.")
-    .add_argument("var", "Expr", "The variable to sync.")
-    .add_argument("lane_mask", "Expr", "The lane mask.")
-    .add_argument("width", "Expr", "The warp thread width, must be a power of 2.")
-    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
-    .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"),
-                                               10)
-    .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__shfl_xor_sync"),
-                                        10)
-    .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_xor_sync")
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
-    .set_attr<bool>("cuda.need_warp_shuffle", true);
-
-TVM_REGISTER_OP("tirx.cuda.__activemask")
-    .set_num_inputs(0)
-    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
-    .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"),
-                                               10)
-    .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__activemask"),
-                                        10)
-    .set_attr<TGlobalSymbol>("TGlobalSymbol", "__activemask")
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
-    .set_attr<bool>("cuda.need_warp_shuffle", true);
+  OpDef("tirx.exp10")
+      .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.erf")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.log")
+      .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("prim.log2")
+      .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.log10")
+      .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.tan")
+      // Now the fast math version of tan and the default version of tan are same.
+      .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic",
+                                 DispatchPureExtern<CUDAFastMathTan>)
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.cos")
+      .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.cosh")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.sin")
+      .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.sinh")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.atan")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.tanh")
+      .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.sqrt")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.rsqrt")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.pow")
+      .set_attr<FLowerIntrinsic>("cuda.fastmath.FLowerIntrinsic", DispatchPureExtern<CUDAFastMath>)
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  OpDef("tirx.popcount")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAPopcount>);
+
+  OpDef("tirx.tvm_warp_shuffle")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
+
+  OpDef("tirx.tvm_warp_shuffle_up")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
+
+  OpDef("tirx.tvm_warp_shuffle_down")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
+
+  OpDef("tirx.tvm_warp_shuffle_xor")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAShuffle<CUDAWarpIntrinsic>);
+
+  OpDef("tirx.tvm_warp_activemask")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchCUDAWarpActiveMask);
+
+  OpDef("tirx.fmod")
+      .set_attr<FLowerIntrinsic>("cuda.FLowerIntrinsic", DispatchPureExtern<CUDAMath>);
+
+  // Register low-level CUDA device intrinsics.
+  // TODO(tvm-team): consider make CUDA its own subfolder and create a file for low-level builtins.
+  OpDef("tirx.cuda.__shfl_sync")
+      .add_arg("mask", "The thread mask.")
+      .add_arg("var", "The variable to sync.")
+      .add_arg("lane", "The source thread id.")
+      .add_arg("width", "The warp thread width, must be a power of 2.")
+      .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+      .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__shfl_sync"))
+      .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_sync")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+      .set_attr<bool>("cuda.need_warp_shuffle", true);
+
+  OpDef("tirx.cuda.__shfl_up_sync")
+      .add_arg("mask", "The thread mask.")
+      .add_arg("var", "The variable to sync.")
+      .add_arg("delta", "The source lane id offset to be added.")
+      .add_arg("width", "The warp thread width, must be a power of 2.")
+      .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+      .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__shfl_up_sync"))
+      .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_up_sync")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+      .set_attr<bool>("cuda.need_warp_shuffle", true);
+
+  OpDef("tirx.cuda.__shfl_down_sync")
+      .add_arg("mask", "The thread mask.")
+      .add_arg("var", "The variable to sync.")
+      .add_arg("delta", "The source lane id offset to be subtracted.")
+      .add_arg("width", "The warp thread width, must be a power of 2.")
+      .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+      .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
+                                          ffi::String("cuda.__shfl_down_sync"))
+      .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_down_sync")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+      .set_attr<bool>("cuda.need_warp_shuffle", true);
+
+  OpDef("tirx.cuda.__shfl_xor_sync")
+      .add_arg("mask", "The thread mask.")
+      .add_arg("var", "The variable to sync.")
+      .add_arg("lane_mask", "The lane mask.")
+      .add_arg("width", "The warp thread width, must be a power of 2.")
+      .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+      .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__shfl_xor_sync"))
+      .set_attr<TGlobalSymbol>("TGlobalSymbol", "__shfl_xor_sync")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+      .set_attr<bool>("cuda.need_warp_shuffle", true);
+
+  OpDef("tirx.cuda.__activemask")
+      .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+      .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.__activemask"))
+      .set_attr<TGlobalSymbol>("TGlobalSymbol", "__activemask")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
+      .set_attr<bool>("cuda.need_warp_shuffle", true);
   // clang-format on
 }
 
diff --git a/src/backend/cuda/codegen/llvm/intrin_rule_nvptx.cc b/src/backend/cuda/codegen/llvm/intrin_rule_nvptx.cc
index 3a057e5b44..bf456d1980 100644
--- a/src/backend/cuda/codegen/llvm/intrin_rule_nvptx.cc
+++ b/src/backend/cuda/codegen/llvm/intrin_rule_nvptx.cc
@@ -61,83 +61,87 @@ inline PrimExpr DispatchPureExternLibDevice(const PrimExpr& e) {
 namespace llvm {
 using tirx::FLowerIntrinsic;
 
-TVM_REGISTER_OP("tirx.floor")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  // clang-format off
+  OpDef("tirx.floor")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("prim.ceil")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("prim.ceil")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.round")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
-      // Redirect to nearbyint (ties-to-even) to match constant-folding semantics.
-      using namespace tirx;
-      const CallNode* call = e.as<CallNode>();
-      TVM_FFI_ICHECK(call != nullptr);
-      static const Op& nearbyint_op = Op::Get("tirx.nearbyint");
-      auto new_call =
-          Call(call->ty.as_or_throw<PrimType>(), nearbyint_op, call->args).as_or_throw<PrimExpr>();
-      return DispatchPureExternLibDevice(new_call);
-    });
+  OpDef("tirx.round")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
+        // Redirect to nearbyint (ties-to-even) to match constant-folding semantics.
+        using namespace tirx;
+        const CallNode* call = e.as<CallNode>();
+        TVM_FFI_ICHECK(call != nullptr);
+        static const Op nearbyint_op = Op::Get("tirx.nearbyint");
+        auto new_call = Call(call->ty.as_or_throw<PrimType>(), nearbyint_op, call->args)
+                            .as_or_throw<PrimExpr>();
+        return DispatchPureExternLibDevice(new_call);
+      });
 
-TVM_REGISTER_OP("tirx.nearbyint")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.nearbyint")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.trunc")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.trunc")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.fabs")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.fabs")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.exp")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.exp")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.exp2")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.exp2")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.exp10")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.exp10")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.erf")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.erf")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.fma")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.fma")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.log")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.log")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("prim.log2")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("prim.log2")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.log10")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.log10")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.sqrt")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.sqrt")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.pow")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.pow")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.tanh")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.tanh")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.tan")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.tan")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.cos")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.cos")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.cosh")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.cosh")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.sin")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.sin")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.sinh")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.sinh")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
 
-TVM_REGISTER_OP("tirx.atan")
-    .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  OpDef("tirx.atan")
+      .set_attr<FLowerIntrinsic>("nvptx.FLowerIntrinsic", DispatchPureExternLibDevice);
+  // clang-format on
+}
 
 }  // namespace llvm
 }  // namespace codegen
diff --git a/src/backend/cuda/op/iket.cc b/src/backend/cuda/op/iket.cc
index faac989fea..2ffb0fab6c 100644
--- a/src/backend/cuda/op/iket.cc
+++ b/src/backend/cuda/op/iket.cc
@@ -27,22 +27,68 @@
 namespace tvm {
 namespace tirx {
 
-#define TVM_REGISTER_CUDA_IKET_OP(OpName)                                                    \
-  TVM_REGISTER_OP("tirx.cuda.iket_" #OpName)                                                 \
-      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))              \
-      .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda")) \
-      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket." #OpName)) \
-      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
-
-TVM_REGISTER_CUDA_IKET_OP(mark);
-TVM_REGISTER_CUDA_IKET_OP(range_start);
-TVM_REGISTER_CUDA_IKET_OP(range_end);
-TVM_REGISTER_CUDA_IKET_OP(range_push);
-TVM_REGISTER_CUDA_IKET_OP(range_pop);
-TVM_REGISTER_CUDA_IKET_OP(sentinel_token);
-TVM_REGISTER_CUDA_IKET_OP(official_event);
-
-#undef TVM_REGISTER_CUDA_IKET_OP
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("tirx.cuda.iket_mark")
+      .add_arg("name", "The name.")
+      .allow_extra_args()
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+                                                                      "mark"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.cuda.iket_range_start")
+      .add_arg("name", "The name.")
+      .allow_extra_args()
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+                                                                      "range_start"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.cuda.iket_range_end")
+      .add_arg("token", "The token.")
+      .allow_extra_args()
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+                                                                      "range_end"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.cuda.iket_range_push")
+      .add_arg("name", "The name.")
+      .allow_extra_args()
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+                                                                      "range_push"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.cuda.iket_range_pop")
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+                                                                      "range_pop"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.cuda.iket_sentinel_token")
+      .add_arg("name", "The name.")
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+                                                                      "sentinel_token"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.cuda.iket_official_event")
+      .add_arg("event_id", "The event identifier.")
+      .add_arg("source_code", "The source code.")
+      .allow_extra_args()
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("cuda"))
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.iket."
+                                                                      "official_event"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+}
 
 }  // namespace tirx
 }  // namespace tvm
diff --git a/src/backend/cuda/op/target_builtin.cc b/src/backend/cuda/op/target_builtin.cc
index 5619bcc9aa..f30749370c 100644
--- a/src/backend/cuda/op/target_builtin.cc
+++ b/src/backend/cuda/op/target_builtin.cc
@@ -29,99 +29,203 @@
 #include <tvm/tirx/op.h>
 #include <tvm/tirx/op_attr_types.h>
 
+#include <initializer_list>
 #include <string>
 
 namespace tvm {
 namespace tirx {
 namespace builtin {
 
-#define TIRX_DEFINE_BUILTIN_FUNC(OpName)                                           \
-  OpRegEntry::RegisterOrGet("tirx." #OpName)                                       \
-      .set_name()                                                                  \
-      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String(#OpName), 1) \
-      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"), /*plevel=*/1)
-
 namespace {
 void RegisterDeviceIntrinsicAliases();
 }
 
 void RegisterCudaTargetBuiltins() {
-  // clang-format off
-static bool registered = false;
-if (registered) return;
-registered = true;
-
-TIRX_DEFINE_BUILTIN_FUNC(tvm_load_matrix_sync)
-    .set_attr<TCallEffectKind>("TCallEffectKind",
-                               static_cast<int64_t>(CallEffectKind::kReadState));
-
-TIRX_DEFINE_BUILTIN_FUNC(tvm_mma_sync)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(tvm_bmma_sync)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(tvm_fill_fragment)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(tvm_store_matrix_sync)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-// Siblings of mma_store / mma_fill that accept
-// (ptr_var, offset) pairs. Codegen emits `ptr + offset` C-pointer
-// arithmetic and lower_warp_memory rewrites the offset's group component
-// to its thread-local index. Used by the s_tir tensor_intrin tensorize
-// path so per-thread fragment offsets stay element-accurate.
-TIRX_DEFINE_BUILTIN_FUNC(mma_store_legacy)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(mma_fill_legacy)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-OpRegEntry::RegisterOrGet("tirx.s_tir.ldg32")
-    .set_name()
-    .set_num_inputs(4)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
-    .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("s_tir.ldg32"), 20)
-    .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
-    .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("s_tir"), 10);
-
-// Raw legacy cp.async form emitted by InjectPTXAsyncCopy (and round-tripped by
-// the T.s_tir.cp_async_raw.legacy 6-arg surface). It carries the element dtype in Call.dtype
-// and prints it dtype-first; user-issued copies go through T.ptx instead.
-OpRegEntry::RegisterOrGet("tirx.s_tir.cp_async_raw")
-    .set_name()
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
-    .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
-    .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("s_tir"))
-    .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("s_tir.cp_async_raw"))
-    .set_attr<TScriptDtypePrintLocation>("TScriptDtypePrintLocation",
-                                         static_cast<int64_t>(ScriptDtypePrintLocation::kFirst));
-
-TIRX_DEFINE_BUILTIN_FUNC(mma_store)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
-    .set_attr<TScriptDtypePrintLocation>("TScriptDtypePrintLocation",
-                                         static_cast<int64_t>(ScriptDtypePrintLocation::kFirst));
-
-TIRX_DEFINE_BUILTIN_FUNC(mma_fill)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
-    .set_attr<TScriptDtypePrintLocation>("TScriptDtypePrintLocation",
-                                         static_cast<int64_t>(ScriptDtypePrintLocation::kFirst));
-
-TIRX_DEFINE_BUILTIN_FUNC(timer_init_cuda)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(timer_start_cuda)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(timer_end_cuda)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TIRX_DEFINE_BUILTIN_FUNC(timer_finalize_cuda)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-RegisterDeviceIntrinsicAliases();
-  // clang-format on
+  static bool registered = false;
+  if (registered) return;
+  registered = true;
+
+  OpDef("tirx.tvm_load_matrix_sync")
+      .add_arg("fragment", "The matrix fragment.")
+      .add_arg("m", "The M dimension.")
+      .add_arg("n", "The N dimension.")
+      .add_arg("k", "The K dimension.")
+      .add_arg("index", "The index.")
+      .add_arg("buffer_ptr", "The buffer pointer.")
+      .add_arg("stride", "The stride.")
+      .add_arg("layout", "The layout.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("tvm_load_matrix_sync"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind",
+                                 static_cast<int64_t>(CallEffectKind::kReadState));
+
+  OpDef("tirx.tvm_mma_sync")
+      .add_arg("fragment_d", "The D fragment.")
+      .add_arg("index_d", "The D fragment index.")
+      .add_arg("fragment_a", "The A fragment.")
+      .add_arg("index_a", "The A fragment index.")
+      .add_arg("fragment_b", "The B fragment.")
+      .add_arg("index_b", "The B fragment index.")
+      .add_arg("fragment_c", "The C fragment.")
+      .add_arg("index_c", "The C fragment index.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("tvm_mma_sync"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.tvm_bmma_sync")
+      .add_arg("fragment_d", "The D fragment.")
+      .add_arg("index_d", "The D fragment index.")
+      .add_arg("fragment_a", "The A fragment.")
+      .add_arg("index_a", "The A fragment index.")
+      .add_arg("fragment_b", "The B fragment.")
+      .add_arg("index_b", "The B fragment index.")
+      .add_arg("fragment_c", "The C fragment.")
+      .add_arg("index_c", "The C fragment index.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("tvm_bmma_sync"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.tvm_fill_fragment")
+      .add_arg("fragment", "The matrix fragment.")
+      .add_arg("m", "The M dimension.")
+      .add_arg("n", "The N dimension.")
+      .add_arg("k", "The K dimension.")
+      .add_arg("index", "The index.")
+      .add_arg("value", "The value to use.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("tvm_fill_fragment"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.tvm_store_matrix_sync")
+      .add_arg("fragment", "The matrix fragment.")
+      .add_arg("m", "The M dimension.")
+      .add_arg("n", "The N dimension.")
+      .add_arg("k", "The K dimension.")
+      .add_arg("index", "The index.")
+      .add_arg("buffer_ptr", "The buffer pointer.")
+      .add_arg("stride", "The stride.")
+      .add_arg("layout", "The layout.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("tvm_store_matrix_sync"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  // Siblings of mma_store / mma_fill that accept
+  // (ptr_var, offset) pairs. Codegen emits `ptr + offset` C-pointer
+  // arithmetic and lower_warp_memory rewrites the offset's group component
+  // to its thread-local index. Used by the s_tir tensor_intrin tensorize
+  // path so per-thread fragment offsets stay element-accurate.
+  OpDef("tirx.mma_store_legacy")
+      .add_arg("m", "The M dimension.")
+      .add_arg("n", "The N dimension.")
+      .add_arg("dst_ptr", "The destination pointer.")
+      .add_arg("src_ptr", "The source pointer.")
+      .add_arg("src_offset", "The source offset.")
+      .add_arg("dst_stride", "The destination stride.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.mma_store_legacy"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.mma_fill_legacy")
+      .add_arg("local_size", "The local allocation size.")
+      .add_arg("local_ptr", "The local pointer.")
+      .add_arg("offset", "The offset.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.mma_fill_legacy"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.s_tir.ldg32")
+      .add_arg("reg", "The register.")
+      .add_arg("guard", "The guard predicate.")
+      .add_arg("addr", "The address.")
+      .add_arg("local_addr", "The local address.")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("s_tir.ldg32"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("s_tir"));
+
+  // Raw legacy cp.async form emitted by InjectPTXAsyncCopy (and round-tripped by
+  // the T.s_tir.cp_async_raw.legacy 6-arg surface). It carries the element dtype in Call.dtype
+  // and prints it dtype-first; user-issued copies go through T.ptx instead.
+  OpDef("tirx.s_tir.cp_async_raw")
+      .add_arg("dst_ptr", "The destination pointer.")
+      .add_arg("dst_offset", "The destination offset.")
+      .add_arg("src_ptr", "The source pointer.")
+      .add_arg("src_offset", "The source offset.")
+      .add_arg("cp_size", "The copy size.")
+      .allow_extra_args()
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("s_tir"))
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("s_tir.cp_async_raw"))
+      .set_attr<TScriptDtypePrintLocation>("TScriptDtypePrintLocation",
+                                           static_cast<int64_t>(ScriptDtypePrintLocation::kFirst));
+
+  OpDef("tirx.mma_store")
+      .add_arg("m", "The M dimension.")
+      .add_arg("n", "The N dimension.")
+      .add_arg("dst_ptr", "The destination pointer.")
+      .add_arg("src_ptr", "The source pointer.")
+      .add_arg("src_offset", "The source offset.")
+      .add_arg("dst_stride", "The destination stride.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.mma_store"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+      .set_attr<TScriptDtypePrintLocation>("TScriptDtypePrintLocation",
+                                           static_cast<int64_t>(ScriptDtypePrintLocation::kFirst));
+
+  OpDef("tirx.mma_fill")
+      .add_arg("local_size", "The local allocation size.")
+      .add_arg("local_ptr", "The local pointer.")
+      .add_arg("offset", "The offset.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.mma_fill"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque))
+      .set_attr<TScriptDtypePrintLocation>("TScriptDtypePrintLocation",
+                                           static_cast<int64_t>(ScriptDtypePrintLocation::kFirst));
+
+  OpDef("tirx.timer_init_cuda")
+      .add_arg("profiler_buffer", "The profiler buffer.")
+      .add_arg("profiler_tag", "The profiler tag.")
+      .add_arg("profiler_write_offset", "The profiler write offset.")
+      .add_arg("num_groups", "The number of groups.")
+      .add_arg("group_id", "The group identifier.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.timer_init"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.timer_start_cuda")
+      .add_arg("event_type", "The event type.")
+      .add_arg("profiler_buffer", "The profiler buffer.")
+      .add_arg("profiler_tag", "The profiler tag.")
+      .add_arg("profiler_write_offset", "The profiler write offset.")
+      .add_arg("profiler_write_stride", "The profiler write stride.")
+      .add_arg("leader_cond", "The leader condition.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.timer_start"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.timer_end_cuda")
+      .add_arg("event_type", "The event type.")
+      .add_arg("profiler_buffer", "The profiler buffer.")
+      .add_arg("profiler_tag", "The profiler tag.")
+      .add_arg("profiler_write_offset", "The profiler write offset.")
+      .add_arg("profiler_write_stride", "The profiler write stride.")
+      .add_arg("leader_cond", "The leader condition.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.timer_end"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.timer_finalize_cuda")
+      .add_arg("profiler_buffer", "The profiler buffer.")
+      .add_arg("profiler_tag", "The profiler tag.")
+      .add_arg("profiler_write_offset", "The profiler write offset.")
+      .add_arg("profiler_write_stride", "The profiler write stride.")
+      .add_arg("leader_cond", "The leader condition.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("cuda.timer_finalize"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  RegisterDeviceIntrinsicAliases();
 }
 
 namespace {
@@ -130,6 +234,8 @@ struct DeviceIntrinsicRegistration {
   const char* name;
   const char* namespace_name;
   CallEffectKind effect_kind;
+  std::initializer_list<const char*> args;
+  bool allow_extra_args;
 };
 
 void RegisterDeviceIntrinsic(const DeviceIntrinsicRegistration& reg) {
@@ -143,117 +249,157 @@ void RegisterDeviceIntrinsic(const DeviceIntrinsicRegistration& reg) {
 
   std::string canonical_op_name = "tirx." + namespace_name + "." + suffix;
   ffi::String namespace_attr(namespace_name);
+  // Match the nested construction namespaces at the canonical registration site.
+  if (namespace_name == "cuda" &&
+      (suffix.rfind("tcgen05_", 0) == 0 || suffix.rfind("wgmma_", 0) == 0)) {
+    suffix[suffix.find('_')] = '.';
+  } else if (namespace_name == "nvshmem" &&
+             ((suffix.size() >= 6 && suffix.compare(suffix.size() - 6, 6, "_block") == 0) ||
+              (suffix.size() >= 5 && suffix.compare(suffix.size() - 5, 5, "_warp") == 0))) {
+    suffix[suffix.rfind('_')] = '.';
+  }
   ffi::String printer_name(namespace_name + "." + suffix);
   int64_t effect = static_cast<int64_t>(reg.effect_kind);
 
-  auto register_one = [&](const std::string& op_name) {
-    OpRegEntry::RegisterOrGet(op_name)
-        .set_name()
-        .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"),
-                                  /*plevel=*/15)
-        .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", namespace_attr,
-                                             /*plevel=*/15)
-        .set_attr<TCallEffectKind>("TCallEffectKind", effect, /*plevel=*/15)
-        .set_attr<TScriptPrinterName>("TScriptPrinterName", printer_name, /*plevel=*/15);
-  };
-
-  register_one(canonical_op_name);
+  OpDef def(canonical_op_name);
+  for (const char* name : reg.args) {
+    def.add_arg(name, "");
+  }
+  if (reg.allow_extra_args) {
+    def.allow_extra_args();
+  }
+  def.set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", namespace_attr)
+      .set_attr<TCallEffectKind>("TCallEffectKind", effect)
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", printer_name);
 }
 
-#define TIRX_DEVICE_INTRIN_ALIAS(OpName, Namespace, EffectKind) \
-  {#OpName, #Namespace, CallEffectKind::EffectKind}
+#define TIRX_DEVICE_INTRIN_ALIAS(OpName, Namespace, EffectKind, ...) \
+  {#OpName, #Namespace, CallEffectKind::EffectKind, __VA_ARGS__}
 
 const DeviceIntrinsicRegistration kDeviceIntrinsics[] = {
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_any_sync, cuda, kPure),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_atomic_add, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_atomic_cas, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_wait_until, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_ballot_sync, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_bfloat1622float2, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_bfloat162float, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_clock64, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_cluster_sync, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_cta_reduce, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_cta_sync, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_cvta_generic_to_shared, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_elect_sync, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_fadd2_rn, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_fdividef, cuda, kPure),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_ffs_u32, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_float22bfloat162_rn, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_float22bfloat162_rn_from_float2, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_float22half2, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_float2_x, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_float2_y, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_float8tohalf8, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_float_as_uint, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_fmul2_rn, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_fp8x4_e4m3_from_float4, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_func_call, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_get_tmem_addr, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_grid_sync, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_half2float, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_half8tofloat8, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_hmax2, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_hmin2, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_ldg, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_make_float2, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_mbarrier_wait, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_mbarrier_wait_acquire_cluster, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_mov_sreg, cuda, kPure),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_nano_sleep, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_printf, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_reduce_add_sync_u32, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_reduce_min_sync_u32, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_runtime_instr_desc, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_sm100_2sm_leader_smem_addr, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_smem_addr_from_uint64, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_syncthreads_and, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_syncthreads_or, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_tcgen05_encode_instr_descriptor, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_tcgen05_encode_instr_descriptor_block_scaled, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_tcgen05_encode_matrix_descriptor, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_thread_fence, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_thread_rank, cuda, kPure),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_trap_when_assert_failed, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_uint_as_float, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_warp_reduce, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_warp_sync, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_warpgroup_sync, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_wgmma_encode_matrix_descriptor, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(cuda_wgmma_noop_barrier, cuda, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_barrier_all, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_fence, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_getmem_nbi, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_getmem_nbi_block, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_getmem_nbi_warp, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_my_pe, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_n_pes, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_nbi, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_nbi_block, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_nbi_warp, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_signal_nbi, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_signal_nbi_block, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_signal_nbi_warp, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_quiet, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_signal_op, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_wait_until, nvshmem, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(ptx_legacy_ldmatrix, ptx_legacy, kOpaque),
-    TIRX_DEVICE_INTRIN_ALIAS(ptx_legacy_mma, ptx_legacy, kOpaque),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_any_sync, cuda, kPure, {"mask", "pred"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_atomic_add, cuda, kOpaque, {"res_addr", "value"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_atomic_cas, cuda, kOpaque, {"ptr", "old_val", "new_val"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(
+        cuda_wait_until, cuda, kOpaque,
+        {"dst", "ptr", "condition", "scope", "space", "ptx_type", "backoff_ns"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_ballot_sync, cuda, kOpaque, {"mask", "pred"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_bfloat1622float2, cuda, kOpaque, {"packed"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_bfloat162float, cuda, kOpaque, {"src"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_clock64, cuda, kOpaque, {}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_cluster_sync, cuda, kOpaque, {}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_cta_reduce, cuda, kOpaque,
+                             {"value", "op", "num_warps", "scratch"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_cta_sync, cuda, kOpaque, {}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_cvta_generic_to_shared, cuda, kOpaque, {"ptr"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_elect_sync, cuda, kOpaque, {}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_fadd2_rn, cuda, kOpaque, {"a", "b"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_fdividef, cuda, kPure, {"x", "y"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_ffs_u32, cuda, kOpaque, {"value"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_float22bfloat162_rn, cuda, kOpaque, {"v0", "v1"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_float22bfloat162_rn_from_float2, cuda, kOpaque, {"packed"},
+                             false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_float22half2, cuda, kOpaque, {"dst", "src"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_float2_x, cuda, kOpaque, {"packed"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_float2_y, cuda, kOpaque, {"packed"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_float8tohalf8, cuda, kOpaque, {"src_addr", "dst_addr"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_float_as_uint, cuda, kOpaque, {"x"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_fmul2_rn, cuda, kOpaque, {"a", "b"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_fp8x4_e4m3_from_float4, cuda, kOpaque, {"x", "y", "z", "w"},
+                             false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_func_call, cuda, kOpaque, {"func_name"}, true),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_get_tmem_addr, cuda, kOpaque,
+                             {"addr", "row_offset", "col_offset"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_grid_sync, cuda, kOpaque, {}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_half2float, cuda, kOpaque, {"src"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_half8tofloat8, cuda, kOpaque, {"src_addr", "dst_addr"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_hmax2, cuda, kOpaque, {"a", "b"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_hmin2, cuda, kOpaque, {"a", "b"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_ldg, cuda, kOpaque, {}, true),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_make_float2, cuda, kOpaque, {"x", "y"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_mbarrier_wait, cuda, kOpaque, {"bar", "phase"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_mbarrier_wait_acquire_cluster, cuda, kOpaque, {"bar", "phase"},
+                             false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_mov_sreg, cuda, kPure, {"bits", "reg_name"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_nano_sleep, cuda, kOpaque, {"time"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_printf, cuda, kOpaque, {"fmt"}, true),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_reduce_add_sync_u32, cuda, kOpaque, {"mask", "value"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_reduce_min_sync_u32, cuda, kOpaque, {"mask", "value"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_runtime_instr_desc, cuda, kOpaque, {"desc", "sf_id"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_sm100_2sm_leader_smem_addr, cuda, kOpaque, {"ptr"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_smem_addr_from_uint64, cuda, kOpaque, {"cluster_addr"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_syncthreads_and, cuda, kOpaque, {"cond"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_syncthreads_or, cuda, kOpaque, {"cond"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_tcgen05_encode_instr_descriptor, cuda, kOpaque,
+                             {"desc", "d_dtype", "a_dtype", "b_dtype", "M", "N", "K", "trans_a",
+                              "trans_b", "n_cta_groups", "neg_a", "neg_b", "sat_d", "is_sparse"},
+                             false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_tcgen05_encode_instr_descriptor_block_scaled, cuda, kOpaque,
+                             {"desc", "d_dtype", "a_dtype", "b_dtype", "sfa_dtype", "sfb_dtype",
+                              "sfa_tmem_addr", "sfb_tmem_addr", "M", "N", "K", "trans_a", "trans_b",
+                              "n_cta_groups", "neg_a", "neg_b", "is_sparse"},
+                             false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_tcgen05_encode_matrix_descriptor, cuda, kOpaque,
+                             {"desc", "addr", "ldo", "sdo", "swizzle"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_thread_fence, cuda, kOpaque, {}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_thread_rank, cuda, kPure, {}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_trap_when_assert_failed, cuda, kOpaque, {"cond"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_uint_as_float, cuda, kOpaque, {"bits"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_warp_reduce, cuda, kOpaque, {"value", "op", "width"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_warp_sync, cuda, kOpaque, {}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_warpgroup_sync, cuda, kOpaque, {"bar_no"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_wgmma_encode_matrix_descriptor, cuda, kOpaque,
+                             {"desc", "addr", "ldo", "sdo", "swizzle"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(cuda_wgmma_noop_barrier, cuda, kOpaque, {"reg"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_barrier_all, nvshmem, kOpaque, {}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_fence, nvshmem, kOpaque, {}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_getmem_nbi, nvshmem, kOpaque, {"dst", "src", "nelems", "pe"},
+                             false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_getmem_nbi_block, nvshmem, kOpaque,
+                             {"dst", "src", "nelems", "pe"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_getmem_nbi_warp, nvshmem, kOpaque,
+                             {"dst", "src", "nelems", "pe"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_my_pe, nvshmem, kOpaque, {}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_n_pes, nvshmem, kOpaque, {}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_nbi, nvshmem, kOpaque, {"dst", "src", "nelems", "pe"},
+                             false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_nbi_block, nvshmem, kOpaque,
+                             {"dst", "src", "nelems", "pe"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_nbi_warp, nvshmem, kOpaque,
+                             {"dst", "src", "nelems", "pe"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_signal_nbi, nvshmem, kOpaque,
+                             {"dst", "src", "nelems", "sig_addr", "signal", "sig_op", "pe"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_signal_nbi_block, nvshmem, kOpaque,
+                             {"dst", "src", "nelems", "sig_addr", "signal", "sig_op", "pe"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_putmem_signal_nbi_warp, nvshmem, kOpaque,
+                             {"dst", "src", "nelems", "sig_addr", "signal", "sig_op", "pe"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_quiet, nvshmem, kOpaque, {}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_signal_op, nvshmem, kOpaque,
+                             {"sig_addr", "signal", "sig_op", "pe"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(nvshmem_wait_until, nvshmem, kOpaque,
+                             {"ivar", "cmp", "cmp_value", "type"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(
+        ptx_legacy_ldmatrix,
+        ptx_legacy, kOpaque,
+        {"trans", "num", "dtype", "local_ptr", "local_offset", "smem_ptr", "smem_offset"}, false),
+    TIRX_DEVICE_INTRIN_ALIAS(
+        ptx_legacy_mma, ptx_legacy, kOpaque,
+        {"shape", "a_layout", "b_layout", "a_dtype", "b_dtype", "c_dtype", "a_ptr", "a_offset",
+         "b_ptr", "b_offset", "acc_ptr", "c_offset", "saturate"},
+        true),
 };
 
+#undef TIRX_DEVICE_INTRIN_ALIAS
+
 void RegisterDeviceIntrinsicAliases() {
   for (const auto& reg : kDeviceIntrinsics) {
     RegisterDeviceIntrinsic(reg);
   }
 }
 
-#undef TIRX_DEVICE_INTRIN_ALIAS
-
 }  // namespace
 
-#undef TIRX_DEFINE_BUILTIN_FUNC
-
 TVM_FFI_STATIC_INIT_BLOCK() { RegisterCudaTargetBuiltins(); }
 
 }  // namespace builtin
diff --git a/src/backend/cuda/transforms/lower_iket.cc b/src/backend/cuda/transforms/lower_iket.cc
index ce153e2cfe..79d7c32b94 100644
--- a/src/backend/cuda/transforms/lower_iket.cc
+++ b/src/backend/cuda/transforms/lower_iket.cc
@@ -87,32 +87,32 @@ enum class PayloadType : uint32_t {
 };
 
 const Op& IketMarkOp() {
-  static const Op& op = Op::Get("tirx.cuda.iket_mark");
+  static const Op op = Op::Get("tirx.cuda.iket_mark");
   return op;
 }
 
 const Op& IketRangeStartOp() {
-  static const Op& op = Op::Get("tirx.cuda.iket_range_start");
+  static const Op op = Op::Get("tirx.cuda.iket_range_start");
   return op;
 }
 
 const Op& IketRangeEndOp() {
-  static const Op& op = Op::Get("tirx.cuda.iket_range_end");
+  static const Op op = Op::Get("tirx.cuda.iket_range_end");
   return op;
 }
 
 const Op& IketRangePushOp() {
-  static const Op& op = Op::Get("tirx.cuda.iket_range_push");
+  static const Op op = Op::Get("tirx.cuda.iket_range_push");
   return op;
 }
 
 const Op& IketRangePopOp() {
-  static const Op& op = Op::Get("tirx.cuda.iket_range_pop");
+  static const Op op = Op::Get("tirx.cuda.iket_range_pop");
   return op;
 }
 
 const Op& IketSentinelOp() {
-  static const Op& op = Op::Get("tirx.cuda.iket_sentinel_token");
+  static const Op op = Op::Get("tirx.cuda.iket_sentinel_token");
   return op;
 }
 
@@ -1117,13 +1117,13 @@ class InstrumentOfficialKernel : public StmtExprMutator {
   }
 
   PrimExpr Event(PrimExpr event_id) const {
-    static const Op& event_op = Op::Get("tirx.cuda.iket_official_event");
+    static const Op event_op = Op::Get("tirx.cuda.iket_official_event");
     return Call(PrimType::UInt(32), event_op,
                 {prim::cast(PrimType::UInt(32), event_id), StringImm(device_source_)});
   }
 
   PrimExpr Event(PrimExpr event_id, PrimExpr payload) const {
-    static const Op& event_op = Op::Get("tirx.cuda.iket_official_event");
+    static const Op event_op = Op::Get("tirx.cuda.iket_official_event");
     return Call(
         PrimType::UInt(32), event_op,
         {prim::cast(PrimType::UInt(32), event_id), StringImm(device_source_), std::move(payload)});
diff --git a/src/backend/hexagon/codegen/llvm/intrin_rule_hexagon.cc b/src/backend/hexagon/codegen/llvm/intrin_rule_hexagon.cc
index efa81d905c..cacbd924d6 100644
--- a/src/backend/hexagon/codegen/llvm/intrin_rule_hexagon.cc
+++ b/src/backend/hexagon/codegen/llvm/intrin_rule_hexagon.cc
@@ -25,13 +25,6 @@
 
 #include "../../../../target/llvm/intrin_rule_llvm.h"
 
-#define TVM_REGISTER_QHL_OP_FP16(INTRIN_FUNC, NUM_SIGN)                                        \
-  TVM_REGISTER_OP("tirx." #INTRIN_FUNC)                                                        \
-      .set_attr<FLowerIntrinsic>(                                                              \
-          "hexagon.FLowerIntrinsic",                                                           \
-          DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_##INTRIN_FUNC, ::llvm::Intrinsic::INTRIN_FUNC, \
-                                    NUM_SIGN>);
-
 namespace tvm {
 namespace codegen {
 using namespace tvm::prim;
@@ -95,154 +88,173 @@ void RegisterHexagonIntrinRules() {
   registered = true;
 
   // clang-format off
-TVM_REGISTER_OP("tirx.fma")
-    .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::fmuladd, 3>);
-
-TVM_REGISTER_OP("tirx.log")
-    .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::log, 1>);
-
-TVM_REGISTER_OP("tirx.trunc")
-    .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::trunc, 1>);
-
-TVM_REGISTER_OP("tirx.fabs")
-    .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::fabs, 1>);
-
-TVM_REGISTER_OP("tirx.round")
-    .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
-
-TVM_REGISTER_OP("tirx.ctpop")
-    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"), 1)
-    .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::ctpop, 1>);
-TVM_REGISTER_OP("tirx.tanh")
-    .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic", [](const PrimExpr& e) {
-      const CallNode* call = e.as<CallNode>();
-      TVM_FFI_ICHECK(call != nullptr);
-      PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
-      PrimType x_ty = x.ty();
+  OpDef("tirx.fma")
+      .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::fmuladd, 3>);
+
+  OpDef("tirx.log")
+      .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::log, 1>);
+
+  OpDef("tirx.trunc")
+      .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::trunc, 1>);
+
+  OpDef("tirx.fabs")
+      .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::fabs, 1>);
+
+  OpDef("tirx.round")
+      .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
+
+  OpDef("tirx.ctpop")
+      .add_arg("x", "The input value.")
+      .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::ctpop, 1>);
+
+  OpDef("tirx.tanh")
+      .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic", [](const PrimExpr& e) {
+    const CallNode* call = e.as<CallNode>();
+    TVM_FFI_ICHECK(call != nullptr);
+    PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
+    PrimType x_ty = x.ty();
 
 #if ENABLE_QHL
-      // Check target for qfloat enablement
-      const auto f = tvm::ffi::Function::GetGlobal("target.TargetCurrent");
-      TVM_FFI_ICHECK(f.has_value()) << "target.TargetCurrent is not registered";
-      const auto ret = (*f)(true);
-      bool useqhl = true;
-      if (auto opt_target = ret.as<Target>()) {
-        const std::string tstring = opt_target.value()->str();
-        useqhl = tstring.find("+hvx-qfloat") != std::string::npos;
-      }
-
-      // Enable QHL library for FP16 data type
-      if (x_ty.MatchesElementType(DLDataTypeCode::kDLFloat, 16) &&
-          (x_ty.IsFixedLengthVector() || x_ty.IsScalableVector()) && useqhl) {
-        std::string tvm_wrapper("tvm_vect_qhmath_hvx_tanh_ahf");
-        return TVMExternCall(call, tvm_wrapper);
-      }
+    // Check target for qfloat enablement
+    const auto f = tvm::ffi::Function::GetGlobal("target.TargetCurrent");
+    TVM_FFI_ICHECK(f.has_value()) << "target.TargetCurrent is not registered";
+    const auto ret = (*f)(true);
+    bool useqhl = true;
+    if (auto opt_target = ret.as<Target>()) {
+      const std::string tstring = opt_target.value()->str();
+      useqhl = tstring.find("+hvx-qfloat") != std::string::npos;
+    }
+
+    // Enable QHL library for FP16 data type
+    if (x_ty.MatchesElementType(DLDataTypeCode::kDLFloat, 16) &&
+        (x_ty.IsFixedLengthVector() || x_ty.IsScalableVector()) && useqhl) {
+      std::string tvm_wrapper("tvm_vect_qhmath_hvx_tanh_ahf");
+      return TVMExternCall(call, tvm_wrapper);
+    }
 #endif
-      PrimExpr one = tvm::prim::MakeConst(x_ty, 1);
-      PrimExpr two = tvm::prim::MakeConst(x_ty, 2);
-      PrimExpr neg_two = tvm::prim::MakeConst(x_ty, -2);
-
-      PrimExpr exp_neg2x = exp(neg_two * x);
-      PrimExpr exp_pos2x = exp(two * x);
-
-      PrimExpr tanh_pos = (one - exp_neg2x) / (one + exp_neg2x);
-      PrimExpr tanh_neg = (exp_pos2x - one) / (exp_pos2x + one);
-      // MakeConst can handle both vector and scalar types.
-      PrimExpr tanh_x = prim::Select(x >= tvm::prim::MakeConst(x_ty, 0), tanh_pos, tanh_neg);
-      return tanh_x;
-    });
-
-TVM_REGISTER_OP("tirx.tan")
-    .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic", [](const PrimExpr& e) {
-      const CallNode* call = e.as<CallNode>();
-      TVM_FFI_ICHECK(call != nullptr);
-      PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
-      PrimType x_ty = x.ty();
+    PrimExpr one = tvm::prim::MakeConst(x_ty, 1);
+    PrimExpr two = tvm::prim::MakeConst(x_ty, 2);
+    PrimExpr neg_two = tvm::prim::MakeConst(x_ty, -2);
+
+    PrimExpr exp_neg2x = exp(neg_two * x);
+    PrimExpr exp_pos2x = exp(two * x);
+
+    PrimExpr tanh_pos = (one - exp_neg2x) / (one + exp_neg2x);
+    PrimExpr tanh_neg = (exp_pos2x - one) / (exp_pos2x + one);
+    // MakeConst can handle both vector and scalar types.
+    PrimExpr tanh_x = prim::Select(x >= tvm::prim::MakeConst(x_ty, 0), tanh_pos, tanh_neg);
+    return tanh_x;
+  });
+
+  OpDef("tirx.tan")
+      .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic", [](const PrimExpr& e) {
+    const CallNode* call = e.as<CallNode>();
+    TVM_FFI_ICHECK(call != nullptr);
+    PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
+    PrimType x_ty = x.ty();
 #if ENABLE_QHL
-      // Check target for qfloat enablement
-      const auto f = tvm::ffi::Function::GetGlobal("target.TargetCurrent");
-      TVM_FFI_ICHECK(f.has_value()) << "target.TargetCurrent is not registered";
-      const auto ret = (*f)(true);
-      bool useqhl = true;
-      if (auto opt_target = ret.as<Target>()) {
-        const std::string tstring = opt_target.value()->str();
-        useqhl = tstring.find("+hvx-qfloat") != std::string::npos;
-      }
-
-      // Enable QHL library for FP16 data type
-      if (x_ty.MatchesElementType(DLDataTypeCode::kDLFloat, 16) &&
-          (x_ty.IsFixedLengthVector() || x_ty.IsScalableVector()) && useqhl) {
-        std::string tvm_wrapper("tvm_vect_qhmath_hvx_tan_ahf");
-        return TVMExternCall(call, tvm_wrapper);
-      }
+    // Check target for qfloat enablement
+    const auto f = tvm::ffi::Function::GetGlobal("target.TargetCurrent");
+    TVM_FFI_ICHECK(f.has_value()) << "target.TargetCurrent is not registered";
+    const auto ret = (*f)(true);
+    bool useqhl = true;
+    if (auto opt_target = ret.as<Target>()) {
+      const std::string tstring = opt_target.value()->str();
+      useqhl = tstring.find("+hvx-qfloat") != std::string::npos;
+    }
+
+    // Enable QHL library for FP16 data type
+    if (x_ty.MatchesElementType(DLDataTypeCode::kDLFloat, 16) &&
+        (x_ty.IsFixedLengthVector() || x_ty.IsScalableVector()) && useqhl) {
+      std::string tvm_wrapper("tvm_vect_qhmath_hvx_tan_ahf");
+      return TVMExternCall(call, tvm_wrapper);
+    }
 #endif
-      PrimExpr tan_x = sin(x) / cos(x);
-      return tan_x;
-    });
-
-TVM_REGISTER_OP("tirx.nearbyint")
-    .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
-
-TVM_REGISTER_OP("tirx.sigmoid")
-    .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic", [](const PrimExpr& e) {
-      const CallNode* call = e.as<CallNode>();
-      TVM_FFI_ICHECK(call != nullptr);
-      PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
-      PrimType x_ty = x.ty();
+    PrimExpr tan_x = sin(x) / cos(x);
+    return tan_x;
+  });
+
+  OpDef("tirx.nearbyint")
+      .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
+
+  OpDef("tirx.sigmoid")
+      .set_attr<FLowerIntrinsic>("hexagon.FLowerIntrinsic", [](const PrimExpr& e) {
+    const CallNode* call = e.as<CallNode>();
+    TVM_FFI_ICHECK(call != nullptr);
+    PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
+    PrimType x_ty = x.ty();
 #if ENABLE_QHL
-      // Check target for qfloat enablement
-      const auto f = tvm::ffi::Function::GetGlobal("target.TargetCurrent");
-      TVM_FFI_ICHECK(f.has_value()) << "target.TargetCurrent is not registered";
-      const auto ret = (*f)(true);
-      bool useqhl = true;
-      if (auto opt_target = ret.as<Target>()) {
-        const std::string tstring = opt_target.value()->str();
-        useqhl = tstring.find("+hvx-qfloat") != std::string::npos;
-      }
-
-      PrimExpr MinBound = tvm::prim::MakeConst(x_ty, -8);
-      PrimExpr MaxBound = tvm::prim::MakeConst(x_ty, 8);
-      const PrimExpr v1 = prim::Max(x, MinBound);
-      const PrimExpr v2 = prim::Min(v1, MaxBound);
-
-      ffi::Array<tvm::PrimExpr> new_args = {v2};
-      const Call new_call =
-          Call(call->ty.as_or_throw<PrimType>(), call->op, new_args);
-
-      // Enable QHL library for FP16 data type
-      if (x_ty.MatchesElementType(DLDataTypeCode::kDLFloat, 16) &&
-          (x_ty.IsFixedLengthVector() || x_ty.IsScalableVector()) && useqhl) {
-        std::string tvm_wrapper("tvm_vect_qhmath_hvx_sigmoid_ahf");
-        return TVMExternCall(new_call.get(), tvm_wrapper);
-      }
+    // Check target for qfloat enablement
+    const auto f = tvm::ffi::Function::GetGlobal("target.TargetCurrent");
+    TVM_FFI_ICHECK(f.has_value()) << "target.TargetCurrent is not registered";
+    const auto ret = (*f)(true);
+    bool useqhl = true;
+    if (auto opt_target = ret.as<Target>()) {
+      const std::string tstring = opt_target.value()->str();
+      useqhl = tstring.find("+hvx-qfloat") != std::string::npos;
+    }
+
+    PrimExpr MinBound = tvm::prim::MakeConst(x_ty, -8);
+    PrimExpr MaxBound = tvm::prim::MakeConst(x_ty, 8);
+    const PrimExpr v1 = prim::Max(x, MinBound);
+    const PrimExpr v2 = prim::Min(v1, MaxBound);
+
+    ffi::Array<tvm::PrimExpr> new_args = {v2};
+    const Call new_call = Call(call->ty.as_or_throw<PrimType>(), call->op, new_args);
+
+    // Enable QHL library for FP16 data type
+    if (x_ty.MatchesElementType(DLDataTypeCode::kDLFloat, 16) &&
+        (x_ty.IsFixedLengthVector() || x_ty.IsScalableVector()) && useqhl) {
+      std::string tvm_wrapper("tvm_vect_qhmath_hvx_sigmoid_ahf");
+      return TVMExternCall(new_call.get(), tvm_wrapper);
+    }
 #endif
-      PrimExpr one = tvm::prim::MakeConst(x_ty, 1);
-      return one / (one + exp(-x));
-    });
-
-TVM_REGISTER_OP("prim.ceil")
-    .set_attr<FLowerIntrinsic>(
-        "hexagon.FLowerIntrinsic",
-        DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_ceil, ::llvm::Intrinsic::ceil, 1>);
-
-TVM_REGISTER_QHL_OP_FP16(cos, 1)
-
-TVM_REGISTER_QHL_OP_FP16(exp, 1)
-
-TVM_REGISTER_QHL_OP_FP16(floor, 1)
-
-TVM_REGISTER_QHL_OP_FP16(sin, 1)
-
-TVM_REGISTER_QHL_OP_FP16(pow, 2)
-
-TVM_REGISTER_QHL_OP_FP16(sqrt, 1)
+    PrimExpr one = tvm::prim::MakeConst(x_ty, 1);
+    return one / (one + exp(-x));
+  });
+
+  OpDef("prim.ceil")
+      .set_attr<FLowerIntrinsic>(
+          "hexagon.FLowerIntrinsic",
+          DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_ceil, ::llvm::Intrinsic::ceil, 1>);
+
+  OpDef("tirx.cos")
+      .set_attr<FLowerIntrinsic>(
+          "hexagon.FLowerIntrinsic",
+          DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_cos, ::llvm::Intrinsic::cos, 1>);
+
+  OpDef("tirx.exp")
+      .set_attr<FLowerIntrinsic>(
+          "hexagon.FLowerIntrinsic",
+          DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_exp, ::llvm::Intrinsic::exp, 1>);
+
+  OpDef("tirx.floor")
+      .set_attr<FLowerIntrinsic>(
+          "hexagon.FLowerIntrinsic",
+          DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_floor, ::llvm::Intrinsic::floor, 1>);
+
+  OpDef("tirx.sin")
+      .set_attr<FLowerIntrinsic>(
+          "hexagon.FLowerIntrinsic",
+          DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_sin, ::llvm::Intrinsic::sin, 1>);
+
+  OpDef("tirx.pow")
+      .set_attr<FLowerIntrinsic>(
+          "hexagon.FLowerIntrinsic",
+          DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_pow, ::llvm::Intrinsic::pow, 2>);
+
+  OpDef("tirx.sqrt")
+      .set_attr<FLowerIntrinsic>(
+          "hexagon.FLowerIntrinsic",
+          DispatchTVMQHLWrapperFp16<tvm_qhl_ahf_sqrt, ::llvm::Intrinsic::sqrt, 1>);
   // clang-format on
 }
 
diff --git a/src/backend/metal/codegen/codegen_metal.cc b/src/backend/metal/codegen/codegen_metal.cc
index 5325cefe6d..74809fafdd 100644
--- a/src/backend/metal/codegen/codegen_metal.cc
+++ b/src/backend/metal/codegen/codegen_metal.cc
@@ -449,10 +449,10 @@ void CodeGenMetal::Dispatch_(const CallNode* op, std::ostream& os) {  // NOLINT(
         << "Only 8x8 matrix is supported, but got " << col_val << "x" << row_val;
   };
 
-  static const Op& make_filled_simdgroup_matrix_op = Op::Get("tirx.make_filled_simdgroup_matrix");
-  static const Op& simdgroup_load_op = Op::Get("tirx.simdgroup_load");
-  static const Op& simdgroup_store_op = Op::Get("tirx.simdgroup_store");
-  static const Op& simdgroup_multiply_accumulate_op = Op::Get("tirx.simdgroup_multiply_accumulate");
+  static const Op make_filled_simdgroup_matrix_op = Op::Get("tirx.make_filled_simdgroup_matrix");
+  static const Op simdgroup_load_op = Op::Get("tirx.simdgroup_load");
+  static const Op simdgroup_store_op = Op::Get("tirx.simdgroup_store");
+  static const Op simdgroup_multiply_accumulate_op = Op::Get("tirx.simdgroup_multiply_accumulate");
 
   if (op->op.same_as(make_filled_simdgroup_matrix_op)) {
     TVM_FFI_ICHECK_EQ(op->args.size(), 5);
diff --git a/src/backend/metal/codegen/intrin_rule_metal.cc b/src/backend/metal/codegen/intrin_rule_metal.cc
index cff357ab19..b5eb1f86ae 100644
--- a/src/backend/metal/codegen/intrin_rule_metal.cc
+++ b/src/backend/metal/codegen/intrin_rule_metal.cc
@@ -33,14 +33,14 @@ using tirx::FLowerIntrinsic;
 struct MetalWarpIntrinsic {
   const Op operator()(PrimType t, const Op& orig_op) const {
     if (orig_op.same_as(builtin::tvm_warp_shuffle())) {
-      static const Op& metal_simd_shuffle_op = Op::Get("tirx.metal.simd_shuffle");
+      static const Op metal_simd_shuffle_op = Op::Get("tirx.metal.simd_shuffle");
       return metal_simd_shuffle_op;
     } else if (orig_op.same_as(builtin::tvm_warp_shuffle_up())) {
-      static const Op& metal_simd_shuffle_up_op = Op::Get("tirx.metal.simd_shuffle_up");
+      static const Op metal_simd_shuffle_up_op = Op::Get("tirx.metal.simd_shuffle_up");
       return metal_simd_shuffle_up_op;
     } else {
       TVM_FFI_ICHECK(orig_op.same_as(builtin::tvm_warp_shuffle_down()));
-      static const Op& metal_simd_shuffle_down_op = Op::Get("tirx.metal.simd_shuffle_down");
+      static const Op metal_simd_shuffle_down_op = Op::Get("tirx.metal.simd_shuffle_down");
       return metal_simd_shuffle_down_op;
     }
   }
@@ -58,128 +58,122 @@ static PrimExpr DispatchMetalShuffle(const PrimExpr& e) {
 
 void RegisterMetalIntrinRules() {
   // clang-format off
-TVM_REGISTER_OP("prim.clz")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("prim.clz")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.floor")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.floor")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("prim.ceil")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("prim.ceil")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.trunc")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.trunc")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.fabs")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.fabs")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.round")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
-      // Metal's rint() uses ties-to-even, matching constant-folding semantics.
-      const CallNode* call = e.as<CallNode>();
-      TVM_FFI_ICHECK(call != nullptr);
-      ffi::Array<Expr> new_args = {StringImm("rint")};
-      for (const PrimExpr& arg : call->args.as_or_throw<ffi::Array<PrimExpr>>()) {
-        new_args.push_back(arg);
-      }
-      return Call(e.ty(), tirx::builtin::call_pure_extern(), new_args).as_or_throw<PrimExpr>();
-    });
-
-TVM_REGISTER_OP("tirx.nearbyint")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.exp")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.exp2")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.exp10")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.log")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("prim.log2")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.log10")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.tanh")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchNumericalStableTanh);
-
-TVM_REGISTER_OP("tirx.sqrt")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.pow")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.popcount")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.fmod")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.sin")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.sinh")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.cos")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.cosh")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.erf").set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchFastErf);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchMetalShuffle<MetalWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_up")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchMetalShuffle<MetalWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_down")
-    .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchMetalShuffle<MetalWarpIntrinsic>);
-
-// Register low-level Metal device intrinsics.
-TVM_REGISTER_OP("tirx.metal.simd_shuffle")
-    .set_num_inputs(2)
-    .add_argument("var", "Expr", "The variable to sync.")
-    .add_argument("lane", "Expr", "The source thread id.")
-    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
-    .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("metal"),
-                                               10)
-    .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("metal.simd_shuffle"),
-                                        10)
-    .set_attr<TGlobalSymbol>("TGlobalSymbol", "simd_shuffle")
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TVM_REGISTER_OP("tirx.metal.simd_shuffle_up")
-    .set_num_inputs(2)
-    .add_argument("var", "Expr", "The variable to sync.")
-    .add_argument("delta", "Expr", "The source lane id offset to be added.")
-    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
-    .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("metal"),
-                                               10)
-    .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
-                                        ffi::String("metal.simd_shuffle_up"), 10)
-    .set_attr<TGlobalSymbol>("TGlobalSymbol", "simd_shuffle_up")
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TVM_REGISTER_OP("tirx.metal.simd_shuffle_down")
-    .set_num_inputs(2)
-    .add_argument("var", "Expr", "The variable to sync.")
-    .add_argument("delta", "Expr", "The source lane id offset to be subtracted.")
-    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
-    .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("metal"),
-                                               10)
-    .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
-                                        ffi::String("metal.simd_shuffle_down"), 10)
-    .set_attr<TGlobalSymbol>("TGlobalSymbol", "simd_shuffle_down")
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+  OpDef("tirx.round")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
+        // Metal's rint() uses ties-to-even, matching constant-folding semantics.
+        const CallNode* call = e.as<CallNode>();
+        TVM_FFI_ICHECK(call != nullptr);
+        ffi::Array<Expr> new_args = {StringImm("rint")};
+        for (const PrimExpr& arg : call->args.as_or_throw<ffi::Array<PrimExpr>>()) {
+          new_args.push_back(arg);
+        }
+        return Call(e.ty(), tirx::builtin::call_pure_extern(), new_args).as_or_throw<PrimExpr>();
+      });
+
+  OpDef("tirx.nearbyint")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.exp")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.exp2")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.exp10")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.log")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("prim.log2")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.log10")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.tanh")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchNumericalStableTanh);
+
+  OpDef("tirx.sqrt")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.pow")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.popcount")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.fmod")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.sin")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.sinh")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.cos")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.cosh")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.erf")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchFastErf);
+
+  OpDef("tirx.tvm_warp_shuffle")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchMetalShuffle<MetalWarpIntrinsic>);
+
+  OpDef("tirx.tvm_warp_shuffle_up")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchMetalShuffle<MetalWarpIntrinsic>);
+
+  OpDef("tirx.tvm_warp_shuffle_down")
+      .set_attr<FLowerIntrinsic>("metal.FLowerIntrinsic", DispatchMetalShuffle<MetalWarpIntrinsic>);
+
+  // Register low-level Metal device intrinsics.
+  OpDef("tirx.metal.simd_shuffle")
+      .add_arg("var", "The variable to sync.")
+      .add_arg("lane", "The source thread id.")
+      .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("metal"))
+      .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName", ffi::String("metal.simd_shuffle"))
+      .set_attr<TGlobalSymbol>("TGlobalSymbol", "simd_shuffle")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.metal.simd_shuffle_up")
+      .add_arg("var", "The variable to sync.")
+      .add_arg("delta", "The source lane id offset to be added.")
+      .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("metal"))
+      .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
+                                          ffi::String("metal.simd_shuffle_up"))
+      .set_attr<TGlobalSymbol>("TGlobalSymbol", "simd_shuffle_up")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.metal.simd_shuffle_down")
+      .add_arg("var", "The variable to sync.")
+      .add_arg("delta", "The source lane id offset to be subtracted.")
+      .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("metal"))
+      .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
+                                          ffi::String("metal.simd_shuffle_down"))
+      .set_attr<TGlobalSymbol>("TGlobalSymbol", "simd_shuffle_down")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
   // clang-format on
 }
 
diff --git a/src/backend/metal/op/target_builtin.cc b/src/backend/metal/op/target_builtin.cc
index 08b3bfd95a..e6fdcdbcd4 100644
--- a/src/backend/metal/op/target_builtin.cc
+++ b/src/backend/metal/op/target_builtin.cc
@@ -31,34 +31,61 @@ namespace tvm {
 namespace tirx {
 namespace builtin {
 
-#define TIRX_DEFINE_BUILTIN_FUNC(OpName)                                           \
-  OpRegEntry::RegisterOrGet("tirx." #OpName)                                       \
-      .set_name()                                                                  \
-      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String(#OpName), 1) \
-      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"), /*plevel=*/1)
-
 void RegisterMetalTargetBuiltins() {
-  // clang-format off
-static bool registered = false;
-if (registered) return;
-registered = true;
+  static bool registered = false;
+  if (registered) return;
+  registered = true;
 
-TIRX_DEFINE_BUILTIN_FUNC(make_filled_simdgroup_matrix)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+  OpDef("tirx.make_filled_simdgroup_matrix")
+      .add_arg("d", "The D operand.")
+      .add_arg("index", "The index.")
+      .add_arg("value", "The value to use.")
+      .add_arg("col", "The column index.")
+      .add_arg("row", "The row index.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName",
+                                    ffi::String("metal.make_filled_simdgroup_matrix"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
 
-TIRX_DEFINE_BUILTIN_FUNC(simdgroup_load)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+  OpDef("tirx.simdgroup_load")
+      .add_arg("d", "The D operand.")
+      .add_arg("index", "The index.")
+      .add_arg("ptr", "The pointer.")
+      .add_arg("stride", "The stride.")
+      .add_arg("col", "The column index.")
+      .add_arg("row", "The row index.")
+      .add_arg("transpose_matrix", "Whether to transpose the matrix.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("metal.simdgroup_load"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
 
-TIRX_DEFINE_BUILTIN_FUNC(simdgroup_store)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+  OpDef("tirx.simdgroup_store")
+      .add_arg("d", "The D operand.")
+      .add_arg("index", "The index.")
+      .add_arg("ptr", "The pointer.")
+      .add_arg("stride", "The stride.")
+      .add_arg("col", "The column index.")
+      .add_arg("row", "The row index.")
+      .add_arg("transpose_matrix", "Whether to transpose the matrix.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", ffi::String("metal.simdgroup_store"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
 
-TIRX_DEFINE_BUILTIN_FUNC(simdgroup_multiply_accumulate)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-  // clang-format on
+  OpDef("tirx.simdgroup_multiply_accumulate")
+      .add_arg("d", "The D operand.")
+      .add_arg("index_d", "The D fragment index.")
+      .add_arg("a", "The A operand.")
+      .add_arg("index_a", "The A fragment index.")
+      .add_arg("b", "The B operand.")
+      .add_arg("index_b", "The B fragment index.")
+      .add_arg("c", "The C operand.")
+      .add_arg("index_c", "The C fragment index.")
+      .set_attr<TScriptPrinterName>("TScriptPrinterName",
+                                    ffi::String("metal.simdgroup_multiply_accumulate"))
+      .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"))
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
 }
 
-#undef TIRX_DEFINE_BUILTIN_FUNC
-
 TVM_FFI_STATIC_INIT_BLOCK() { RegisterMetalTargetBuiltins(); }
 
 }  // namespace builtin
diff --git a/src/backend/opencl/codegen/intrin_rule_opencl.cc b/src/backend/opencl/codegen/intrin_rule_opencl.cc
index a911e07873..4a80c424a3 100644
--- a/src/backend/opencl/codegen/intrin_rule_opencl.cc
+++ b/src/backend/opencl/codegen/intrin_rule_opencl.cc
@@ -53,86 +53,86 @@ void RegisterOpenCLIntrinRules() {
   registered = true;
 
   // clang-format off
-TVM_REGISTER_OP("prim.clz")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("prim.clz")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.floor")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.floor")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("prim.ceil")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("prim.ceil")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.trunc")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.trunc")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.fabs")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.fabs")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.round")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
-      // OpenCL's rint() uses ties-to-even, matching constant-folding semantics.
-      const CallNode* call = e.as<CallNode>();
-      TVM_FFI_ICHECK(call != nullptr);
-      ffi::Array<Expr> new_args = {StringImm("rint")};
-      for (const PrimExpr& arg : call->args.as_or_throw<ffi::Array<PrimExpr>>()) {
-        new_args.push_back(arg);
-      }
-      return Call(e.ty(), tirx::builtin::call_pure_extern(), new_args).as_or_throw<PrimExpr>();
-    });
+  OpDef("tirx.round")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
+        // OpenCL's rint() uses ties-to-even, matching constant-folding semantics.
+        const CallNode* call = e.as<CallNode>();
+        TVM_FFI_ICHECK(call != nullptr);
+        ffi::Array<Expr> new_args = {StringImm("rint")};
+        for (const PrimExpr& arg : call->args.as_or_throw<ffi::Array<PrimExpr>>()) {
+          new_args.push_back(arg);
+        }
+        return Call(e.ty(), tirx::builtin::call_pure_extern(), new_args).as_or_throw<PrimExpr>();
+      });
 
-TVM_REGISTER_OP("tirx.nearbyint")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.nearbyint")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.exp")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.exp")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.erf")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.erf")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.exp2")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.exp2")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.exp10")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.exp10")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.log")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.log")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("prim.log2")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("prim.log2")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.log10")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.log10")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.tanh")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.tanh")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.sqrt")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.sqrt")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.pow")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.pow")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.popcount")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.popcount")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.fmod")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.fmod")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.sin")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.sin")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.sinh")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.sinh")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.cos")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.cos")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.cosh")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.cosh")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle")
-    .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchIntelShuffle);
+  OpDef("tirx.tvm_warp_shuffle")
+      .set_attr<FLowerIntrinsic>("opencl.FLowerIntrinsic", DispatchIntelShuffle);
   // clang-format on
 }
 
diff --git a/src/backend/rocm/codegen/llvm/intrin_rule_rocm.cc b/src/backend/rocm/codegen/llvm/intrin_rule_rocm.cc
index 8f7d80ef0a..67b606e5d3 100644
--- a/src/backend/rocm/codegen/llvm/intrin_rule_rocm.cc
+++ b/src/backend/rocm/codegen/llvm/intrin_rule_rocm.cc
@@ -117,110 +117,110 @@ namespace llvm {
 using tirx::FLowerIntrinsic;
 
 void RegisterROCMIntrinRules() {
+  // dummy because we don't have the activemask
   // clang-format off
-// dummy because we don't have the activemask
-TVM_REGISTER_OP("tirx.tvm_warp_activemask")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
-      PrimExpr zero = IntImm::Int32(0);
-      return zero;
-    });
+  OpDef("tirx.tvm_warp_activemask")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", [](const PrimExpr& e) -> PrimExpr {
+        PrimExpr zero = IntImm::Int32(0);
+        return zero;
+      });
 
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchShuffle);
+  OpDef("tirx.tvm_warp_shuffle")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchShuffle);
 
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_up")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchShuffle);
+  OpDef("tirx.tvm_warp_shuffle_up")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchShuffle);
 
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_down")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchShuffle);
+  OpDef("tirx.tvm_warp_shuffle_down")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchShuffle);
 
-TVM_REGISTER_OP("tirx.floor")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::floor, 1>);
+  OpDef("tirx.floor")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::floor, 1>);
 
-TVM_REGISTER_OP("prim.ceil")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::ceil, 1>);
+  OpDef("prim.ceil")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::ceil, 1>);
 
-TVM_REGISTER_OP("tirx.round")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
+  OpDef("tirx.round")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
 
-TVM_REGISTER_OP("tirx.nearbyint")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
+  OpDef("tirx.nearbyint")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::nearbyint, 1>);
 
-TVM_REGISTER_OP("tirx.trunc")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::trunc, 1>);
+  OpDef("tirx.trunc")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::trunc, 1>);
 
-TVM_REGISTER_OP("tirx.fabs")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::fabs, 1>);
+  OpDef("tirx.fabs")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::fabs, 1>);
 
-TVM_REGISTER_OP("tirx.exp")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::exp, 1>);
+  OpDef("tirx.exp")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::exp, 1>);
 
-TVM_REGISTER_OP("tirx.exp2")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::exp2, 1>);
+  OpDef("tirx.exp2")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::exp2, 1>);
 
-TVM_REGISTER_OP("tirx.fma")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::fmuladd, 3>);
+  OpDef("tirx.fma")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::fmuladd, 3>);
 
-TVM_REGISTER_OP("tirx.log")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::log, 1>);
+  OpDef("tirx.log")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::log, 1>);
 
-TVM_REGISTER_OP("prim.log2")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::log2, 1>);
+  OpDef("prim.log2")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::log2, 1>);
 
-TVM_REGISTER_OP("tirx.log10")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::log10, 1>);
+  OpDef("tirx.log10")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::log10, 1>);
 
-TVM_REGISTER_OP("tirx.sqrt")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::sqrt, 1>);
+  OpDef("tirx.sqrt")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::sqrt, 1>);
 
-TVM_REGISTER_OP("tirx.pow")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::pow, 2>);
+  OpDef("tirx.pow")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::pow, 2>);
 
-TVM_REGISTER_OP("tirx.cos")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::cos, 1>);
+  OpDef("tirx.cos")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::cos, 1>);
 
-TVM_REGISTER_OP("tirx.sin")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               DispatchLLVMPureIntrin<::llvm::Intrinsic::sin, 1>);
+  OpDef("tirx.sin")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 DispatchLLVMPureIntrin<::llvm::Intrinsic::sin, 1>);
 
-TVM_REGISTER_OP("tirx.tanh")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-                               ::tvm::codegen::intrin::DispatchNumericalStableTanh);
+  OpDef("tirx.tanh")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+                                 ::tvm::codegen::intrin::DispatchNumericalStableTanh);
 
-TVM_REGISTER_OP("tirx.erf")
-    .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", ::tvm::codegen::intrin::DispatchFastErf);
+  OpDef("tirx.erf")
+      .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", ::tvm::codegen::intrin::DispatchFastErf);
+  // clang-format on
 
-// TVM_REGISTER_OP("tirx.tan").set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-//                                                      DispatchPureExternOCML);
+  // OpDef("tirx.tan").set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+  //                                                      DispatchPureExternOCML);
 
-// TVM_REGISTER_OP("tirx.cosh")
-//     .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchPureExternOCML);
+  // OpDef("tirx.cosh")
+  //     .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchPureExternOCML);
 
-// TVM_REGISTER_OP("tirx.sinh")
-//     .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchPureExternOCML);
+  // OpDef("tirx.sinh")
+  //     .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchPureExternOCML);
 
-// TVM_REGISTER_OP("tirx.atan")
-//     .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchPureExternOCML);
+  // OpDef("tirx.atan")
+  //     .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic", DispatchPureExternOCML);
 
-// TVM_REGISTER_OP("tirx.exp10")
-//     .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
-//                                DispatchLLVMPureIntrin<::llvm::Intrinsic::exp10, 1>);
-  // clang-format on
+  // OpDef("tirx.exp10")
+  //     .set_attr<FLowerIntrinsic>("rocm.FLowerIntrinsic",
+  //                                DispatchLLVMPureIntrin<::llvm::Intrinsic::exp10, 1>);
 }
 
 }  // namespace llvm
diff --git a/src/backend/trn/codegen/codegen_trn.cc b/src/backend/trn/codegen/codegen_trn.cc
index dccb4455a8..9889040c46 100644
--- a/src/backend/trn/codegen/codegen_trn.cc
+++ b/src/backend/trn/codegen/codegen_trn.cc
@@ -370,22 +370,22 @@ void CodeGenTrainium::Dispatch_(const CallNode* op, std::ostream& os) {  // NOLI
   auto is_op = [&](const Op& compat, const char* canonical_name) {
     return op->op.same_as(compat) || (op_node != nullptr && op_node->name == canonical_name);
   };
-  static const Op& nki_matmul_op = Op::Get("tirx.nki.matmul");
-  static const Op& nki_load_op = Op::Get("tirx.nki.load");
-  static const Op& nki_store_op = Op::Get("tirx.nki.store");
-  static const Op& nki_tensor_copy_op = Op::Get("tirx.nki.tensor_copy");
-  static const Op& nki_activation_op = Op::Get("tirx.nki.activation");
-  static const Op& nki_reciprocal_op = Op::Get("tirx.nki.reciprocal");
-  static const Op& nki_tensortensor_op = Op::Get("tirx.nki.tensortensor");
-  static const Op& nki_tensorscalar_op = Op::Get("tirx.nki.tensorscalar");
-  static const Op& nki_memset_op = Op::Get("tirx.nki.memset");
-  static const Op& nki_tensorreduce_op = Op::Get("tirx.nki.tensorreduce");
-  static const Op& nki_activation_reduce_op = Op::Get("tirx.nki.activation_reduce");
-  static const Op& nki_tensorscalar_reduce_op = Op::Get("tirx.nki.tensorscalar_reduce");
-  static const Op& nki_identity_op = Op::Get("tirx.nki.identity");
-  static const Op& nki_scalar_tensor_tensor_op = Op::Get("tirx.nki.scalar_tensor_tensor");
-  static const Op& nki_scalar_tensor_scalar_op = Op::Get("tirx.nki.scalar_tensor_scalar");
-  static const Op& nki_affine_select_op = Op::Get("tirx.nki.affine_select");
+  static const Op nki_matmul_op = Op::Get("tirx.nki.matmul");
+  static const Op nki_load_op = Op::Get("tirx.nki.load");
+  static const Op nki_store_op = Op::Get("tirx.nki.store");
+  static const Op nki_tensor_copy_op = Op::Get("tirx.nki.tensor_copy");
+  static const Op nki_activation_op = Op::Get("tirx.nki.activation");
+  static const Op nki_reciprocal_op = Op::Get("tirx.nki.reciprocal");
+  static const Op nki_tensortensor_op = Op::Get("tirx.nki.tensortensor");
+  static const Op nki_tensorscalar_op = Op::Get("tirx.nki.tensorscalar");
+  static const Op nki_memset_op = Op::Get("tirx.nki.memset");
+  static const Op nki_tensorreduce_op = Op::Get("tirx.nki.tensorreduce");
+  static const Op nki_activation_reduce_op = Op::Get("tirx.nki.activation_reduce");
+  static const Op nki_tensorscalar_reduce_op = Op::Get("tirx.nki.tensorscalar_reduce");
+  static const Op nki_identity_op = Op::Get("tirx.nki.identity");
+  static const Op nki_scalar_tensor_tensor_op = Op::Get("tirx.nki.scalar_tensor_tensor");
+  static const Op nki_scalar_tensor_scalar_op = Op::Get("tirx.nki.scalar_tensor_scalar");
+  static const Op nki_affine_select_op = Op::Get("tirx.nki.affine_select");
 
   if (is_op(nki_matmul_op, "tirx.nki.matmul")) {
     TVM_FFI_ICHECK_EQ(op->args.size(), 4);
diff --git a/src/backend/trn/op/target_builtin.cc b/src/backend/trn/op/target_builtin.cc
index 2341a9683d..64d06b00e0 100644
--- a/src/backend/trn/op/target_builtin.cc
+++ b/src/backend/trn/op/target_builtin.cc
@@ -30,6 +30,7 @@
 #include <tvm/tirx/op.h>
 #include <tvm/tirx/op_attr_types.h>
 
+#include <initializer_list>
 #include <string>
 
 namespace tvm {
@@ -41,20 +42,24 @@ void RegisterNKIIntrinsicAliases();
 }
 
 void RegisterTRNTargetBuiltins() {
-  // clang-format off
-static bool registered = false;
-if (registered) return;
-registered = true;
+  static bool registered = false;
+  if (registered) return;
+  registered = true;
 
-RegisterNKIIntrinsicAliases();
-  // clang-format on
+  RegisterNKIIntrinsicAliases();
 }
 
 namespace {
 
-void RegisterNKIIntrinsic(const char* name) {
+struct NKIIntrinsicRegistration {
+  const char* name;
+  std::initializer_list<const char*> args;
+  bool allow_extra_args;
+};
+
+void RegisterNKIIntrinsic(const NKIIntrinsicRegistration& reg) {
   std::string prefix = "nki_";
-  std::string suffix(name);
+  std::string suffix(reg.name);
   if (suffix.rfind(prefix, 0) == 0) {
     suffix = suffix.substr(prefix.size());
   }
@@ -64,42 +69,50 @@ void RegisterNKIIntrinsic(const char* name) {
   ffi::String printer_name("nki." + suffix);
   int64_t effect = static_cast<int64_t>(CallEffectKind::kOpaque);
 
-  auto register_one = [&](const std::string& op_name) {
-    OpRegEntry::RegisterOrGet(op_name)
-        .set_name()
-        .set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"),
-                                  /*plevel=*/15)
-        .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", namespace_attr,
-                                             /*plevel=*/15)
-        .set_attr<TCallEffectKind>("TCallEffectKind", effect, /*plevel=*/15)
-        .set_attr<TScriptPrinterName>("TScriptPrinterName", printer_name, /*plevel=*/15);
-  };
-
-  register_one(canonical_op_name);
+  OpDef def(canonical_op_name);
+  for (const char* name : reg.args) {
+    def.add_arg(name, "");
+  }
+  if (reg.allow_extra_args) {
+    def.allow_extra_args();
+  }
+  def.set_attr<TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", namespace_attr)
+      .set_attr<TCallEffectKind>("TCallEffectKind", effect)
+      .set_attr<TScriptPrinterName>("TScriptPrinterName", printer_name);
 }
 
-const char* kNKIIntrinsics[] = {
-    "nki_activation",
-    "nki_activation_reduce",
-    "nki_affine_select",
-    "nki_identity",
-    "nki_load",
-    "nki_matmul",
-    "nki_memset",
-    "nki_reciprocal",
-    "nki_scalar_tensor_scalar",
-    "nki_scalar_tensor_tensor",
-    "nki_store",
-    "nki_tensor_copy",
-    "nki_tensorreduce",
-    "nki_tensorscalar",
-    "nki_tensorscalar_reduce",
-    "nki_tensortensor",
+const NKIIntrinsicRegistration kNKIIntrinsics[] = {
+    {"nki_activation", {"result", "data", "opcode", "bias", "scale"}, false},
+    {"nki_activation_reduce",
+     {"reduce_res", "act_res", "data", "opcode", "reduce_opcode", "bias", "scale"},
+     false},
+    {"nki_affine_select", {"result", "pred", "true_value", "false_value"}, false},
+    {"nki_identity", {"result", "size"}, false},
+    {"nki_load", {"res", "data"}, false},
+    {"nki_matmul", {"res", "lhs", "rhs", "accum"}, false},
+    {"nki_memset", {"result", "value"}, false},
+    {"nki_reciprocal", {"result", "data"}, false},
+    {"nki_scalar_tensor_scalar",
+     {"result", "data", "operand0", "operand1", "opcode0", "opcode1", "reverse0", "reverse1"},
+     false},
+    {"nki_scalar_tensor_tensor",
+     {"result", "data", "operand0", "operand1", "opcode0", "opcode1", "reverse0", "reverse1"},
+     false},
+    {"nki_store", {"res", "data"}, false},
+    {"nki_tensor_copy", {"res", "data"}, false},
+    {"nki_tensorreduce", {"result", "data", "opcode", "negate"}, true},
+    {"nki_tensorscalar", {"result", "operand0", "operand1", "opcode", "reverse"}, false},
+    {"nki_tensorscalar_reduce",
+     {"reduce_res", "tensorscalar_res", "operand0", "operand1", "opcode", "reduce_opcode",
+      "reverse"},
+     false},
+    {"nki_tensortensor", {"result", "operand0", "operand1", "opcode"}, false},
 };
 
 void RegisterNKIIntrinsicAliases() {
-  for (const char* op_name : kNKIIntrinsics) {
-    RegisterNKIIntrinsic(op_name);
+  for (const auto& reg : kNKIIntrinsics) {
+    RegisterNKIIntrinsic(reg);
   }
 }
 
diff --git a/src/backend/vulkan/codegen/codegen_spirv.cc b/src/backend/vulkan/codegen/codegen_spirv.cc
index 0c648ac929..99e9a5899a 100644
--- a/src/backend/vulkan/codegen/codegen_spirv.cc
+++ b/src/backend/vulkan/codegen/codegen_spirv.cc
@@ -467,10 +467,10 @@ spirv::Value CodeGenSPIRV::Dispatch_(const CallNode* op) {
     return spirv::Value();
   }
 
-  static const Op& tvm_fill_fragment_op = Op::Get("tirx.tvm_fill_fragment");
-  static const Op& tvm_load_matrix_sync_op = Op::Get("tirx.tvm_load_matrix_sync");
-  static const Op& tvm_mma_sync_op = Op::Get("tirx.tvm_mma_sync");
-  static const Op& tvm_store_matrix_sync_op = Op::Get("tirx.tvm_store_matrix_sync");
+  static const Op tvm_fill_fragment_op = Op::Get("tirx.tvm_fill_fragment");
+  static const Op tvm_load_matrix_sync_op = Op::Get("tirx.tvm_load_matrix_sync");
+  static const Op tvm_mma_sync_op = Op::Get("tirx.tvm_mma_sync");
+  static const Op tvm_store_matrix_sync_op = Op::Get("tirx.tvm_store_matrix_sync");
 
   if (op->op.same_as(tvm_fill_fragment_op)) {
     TVM_FFI_ICHECK_EQ(op->args.size(), 6U);
diff --git a/src/backend/vulkan/codegen/intrin_rule_spirv.cc b/src/backend/vulkan/codegen/intrin_rule_spirv.cc
index bca215bbdb..6086a1d88c 100644
--- a/src/backend/vulkan/codegen/intrin_rule_spirv.cc
+++ b/src/backend/vulkan/codegen/intrin_rule_spirv.cc
@@ -70,87 +70,85 @@ void RegisterVulkanLowerIntrinRules() {
   if (registered) return;
   registered = true;
 
-  // clang-format off
-TVM_REGISTER_OP("tirx.floor")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Floor>);
+  OpDef("tirx.floor")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Floor>);
 
-TVM_REGISTER_OP("prim.ceil")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Ceil>);
+  OpDef("prim.ceil")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Ceil>);
 
-TVM_REGISTER_OP("tirx.round")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic",
-                               DispatchGLSLPureIntrin<GLSLstd450RoundEven>);
+  OpDef("tirx.round")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic",
+                                 DispatchGLSLPureIntrin<GLSLstd450RoundEven>);
 
-TVM_REGISTER_OP("tirx.nearbyint")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic",
-                               DispatchGLSLPureIntrin<GLSLstd450RoundEven>);
+  OpDef("tirx.nearbyint")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic",
+                                 DispatchGLSLPureIntrin<GLSLstd450RoundEven>);
 
-TVM_REGISTER_OP("tirx.trunc")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Trunc>);
+  OpDef("tirx.trunc")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Trunc>);
 
-TVM_REGISTER_OP("tirx.fabs")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450FAbs>);
+  OpDef("tirx.fabs")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450FAbs>);
 
-TVM_REGISTER_OP("tirx.exp")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Exp>);
+  OpDef("tirx.exp")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Exp>);
 
-TVM_REGISTER_OP("tirx.exp2")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Exp2>);
+  OpDef("tirx.exp2")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Exp2>);
 
-TVM_REGISTER_OP("tirx.sin")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Sin>);
+  OpDef("tirx.sin")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Sin>);
 
-TVM_REGISTER_OP("tirx.cos")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Cos>);
+  OpDef("tirx.cos")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Cos>);
 
-TVM_REGISTER_OP("tirx.tan")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Tan>);
+  OpDef("tirx.tan")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Tan>);
 
-TVM_REGISTER_OP("tirx.asin")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Asin>);
+  OpDef("tirx.asin")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Asin>);
 
-TVM_REGISTER_OP("tirx.acos")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Acos>);
+  OpDef("tirx.acos")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Acos>);
 
-TVM_REGISTER_OP("tirx.atan")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Atan>);
+  OpDef("tirx.atan")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Atan>);
 
-TVM_REGISTER_OP("tirx.sinh")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Sinh>);
+  OpDef("tirx.sinh")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Sinh>);
 
-TVM_REGISTER_OP("tirx.cosh")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Cosh>);
+  OpDef("tirx.cosh")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Cosh>);
 
-TVM_REGISTER_OP("tirx.tanh")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Tanh>);
+  OpDef("tirx.tanh")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Tanh>);
 
-TVM_REGISTER_OP("tirx.asinh")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Asinh>);
+  OpDef("tirx.asinh")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Asinh>);
 
-TVM_REGISTER_OP("tirx.acosh")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Acosh>);
+  OpDef("tirx.acosh")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Acosh>);
 
-TVM_REGISTER_OP("tirx.atanh")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Atanh>);
+  OpDef("tirx.atanh")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Atanh>);
 
-TVM_REGISTER_OP("tirx.atan2")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Atan2>);
+  OpDef("tirx.atan2")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Atan2>);
 
-TVM_REGISTER_OP("tirx.log")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Log>);
+  OpDef("tirx.log")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Log>);
 
-TVM_REGISTER_OP("prim.log2")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Log2>);
+  OpDef("prim.log2")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Log2>);
 
-TVM_REGISTER_OP("tirx.sqrt")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Sqrt>);
+  OpDef("tirx.sqrt")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Sqrt>);
 
-TVM_REGISTER_OP("tirx.pow")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Pow>);
+  OpDef("tirx.pow")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", DispatchGLSLPureIntrin<GLSLstd450Pow>);
 
-TVM_REGISTER_OP("tirx.erf")
-    .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", codegen::intrin ::DispatchFastErf);
-  // clang-format on
+  OpDef("tirx.erf")
+      .set_attr<FLowerIntrinsic>("vulkan.FLowerIntrinsic", codegen::intrin ::DispatchFastErf);
 }
 }  // namespace intrin
 
@@ -163,29 +161,29 @@ void RegisterVulkanLegalizeRules() {
   registered = true;
 
   // clang-format off
-TVM_REGISTER_OP("prim.clz")
-    .set_attr<FLegalize>("vulkan.FLegalize", [](const PrimExpr& e) -> PrimExpr {
-      const CallNode* call = e.as<CallNode>();
-      TVM_FFI_ICHECK(call != nullptr);
-      TVM_FFI_ICHECK_EQ(call->args.size(), 1);
-      PrimExpr arg = call->args[0].as_or_throw<PrimExpr>();
-      PrimType arg_ty = arg.ty();
-      PrimExpr msb;
-      if (arg_ty.bits() == 64) {
-        // SPIR-V FindUMsb intrinsic only supports 32 bit input
-        auto int32 = PrimType::Int(32);
-        PrimExpr arg_hi32 = tvm::prim::Cast(int32, arg >> 32);
-        PrimExpr arg_lo32 = tvm::prim::Cast(int32, arg);
-        PrimExpr msb_hi = CallGLSLIntrin<GLSLstd450FindUMsb>(e, {arg_hi32});
-        PrimExpr msb_lo = CallGLSLIntrin<GLSLstd450FindUMsb>(e, {arg_lo32});
-        msb = tvm::if_then_else(arg_hi32 == 0, msb_lo, msb_hi + 32);
-      } else if (arg_ty.bits() == 32) {
-        msb = CallGLSLIntrin<GLSLstd450FindUMsb>(e);
-      } else {
-        TVM_FFI_THROW(InternalError) << "SPIR-V clz only supports a 32 bit or 64 bit integer.";
-      }
-      return PrimExpr(arg_ty.bits() - 1) - msb;
-    });
+  OpDef("prim.clz")
+      .set_attr<FLegalize>("vulkan.FLegalize", [](const PrimExpr& e) -> PrimExpr {
+    const CallNode* call = e.as<CallNode>();
+    TVM_FFI_ICHECK(call != nullptr);
+    TVM_FFI_ICHECK_EQ(call->args.size(), 1);
+    PrimExpr arg = call->args[0].as_or_throw<PrimExpr>();
+    PrimType arg_ty = arg.ty();
+    PrimExpr msb;
+    if (arg_ty.bits() == 64) {
+      // SPIR-V FindUMsb intrinsic only supports 32 bit input
+      auto int32 = PrimType::Int(32);
+      PrimExpr arg_hi32 = tvm::prim::Cast(int32, arg >> 32);
+      PrimExpr arg_lo32 = tvm::prim::Cast(int32, arg);
+      PrimExpr msb_hi = CallGLSLIntrin<GLSLstd450FindUMsb>(e, {arg_hi32});
+      PrimExpr msb_lo = CallGLSLIntrin<GLSLstd450FindUMsb>(e, {arg_lo32});
+      msb = tvm::if_then_else(arg_hi32 == 0, msb_lo, msb_hi + 32);
+    } else if (arg_ty.bits() == 32) {
+      msb = CallGLSLIntrin<GLSLstd450FindUMsb>(e);
+    } else {
+      TVM_FFI_THROW(InternalError) << "SPIR-V clz only supports a 32 bit or 64 bit integer.";
+    }
+    return PrimExpr(arg_ty.bits() - 1) - msb;
+  });
   // clang-format on
 }
 }  // namespace legalize
diff --git a/src/backend/webgpu/codegen/intrin_rule_webgpu.cc b/src/backend/webgpu/codegen/intrin_rule_webgpu.cc
index 75ce8c6367..d030a78417 100644
--- a/src/backend/webgpu/codegen/intrin_rule_webgpu.cc
+++ b/src/backend/webgpu/codegen/intrin_rule_webgpu.cc
@@ -36,14 +36,14 @@ using tirx::FLowerIntrinsic;
 struct WebGPUWarpIntrinsic {
   const Op operator()(PrimType t, const Op& orig_op) const {
     if (orig_op.same_as(builtin::tvm_warp_shuffle())) {
-      static const Op& webgpu_subgroup_shuffle_op = Op::Get("tirx.webgpu.subgroup_shuffle");
+      static const Op webgpu_subgroup_shuffle_op = Op::Get("tirx.webgpu.subgroup_shuffle");
       return webgpu_subgroup_shuffle_op;
     } else if (orig_op.same_as(builtin::tvm_warp_shuffle_up())) {
-      static const Op& webgpu_subgroup_shuffle_up_op = Op::Get("tirx.webgpu.subgroup_shuffle_up");
+      static const Op webgpu_subgroup_shuffle_up_op = Op::Get("tirx.webgpu.subgroup_shuffle_up");
       return webgpu_subgroup_shuffle_up_op;
     } else {
       TVM_FFI_ICHECK(orig_op.same_as(builtin::tvm_warp_shuffle_down()));
-      static const Op& webgpu_subgroup_shuffle_down_op =
+      static const Op webgpu_subgroup_shuffle_down_op =
           Op::Get("tirx.webgpu.subgroup_shuffle_down");
       return webgpu_subgroup_shuffle_down_op;
     }
@@ -66,145 +66,142 @@ void RegisterWebGPUIntrinRules() {
   if (registered) return;
   registered = true;
 
-  // clang-format off
-// See full list of builtin: https://www.w3.org/TR/WGSL/#builtin-functions
-
-struct ReturnAbs {
-  std::string operator()(PrimType t, std::string name) const { return "abs"; }
-};
-
-TVM_REGISTER_OP("tirx.fabs")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<ReturnAbs>);
-
-TVM_REGISTER_OP("tirx.acos")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.acosh")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.asin")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.asinh")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  // See full list of builtin: https://www.w3.org/TR/WGSL/#builtin-functions
 
-TVM_REGISTER_OP("tirx.atan")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  struct ReturnAbs {
+    std::string operator()(PrimType t, std::string name) const { return "abs"; }
+  };
 
-TVM_REGISTER_OP("tirx.atan2")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("prim.ceil")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.cos")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
-
-TVM_REGISTER_OP("tirx.cosh")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  // clang-format off
+  OpDef("tirx.fabs")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<ReturnAbs>);
 
-TVM_REGISTER_OP("tirx.exp")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.acos")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.exp2")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.acosh")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.floor")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.asin")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.fma")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.asinh")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.log")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.atan")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("prim.log2")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.atan2")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.pow")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("prim.ceil")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-struct ReturnRound {
-  std::string operator()(PrimType t, std::string name) const { return "round"; }
-};
+  OpDef("tirx.cos")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-// WGSL round() uses ties-to-even (banker's rounding), matching IEEE 754 and ONNX Round spec.
-TVM_REGISTER_OP("tirx.nearbyint")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<ReturnRound>);
+  OpDef("tirx.cosh")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.round")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.exp")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.sin")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.exp2")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.sinh")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.floor")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.sqrt")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.fma")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.tan")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.log")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.tanh")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchNumericalStableTanh);
+  OpDef("prim.log2")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
 
-TVM_REGISTER_OP("tirx.trunc")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  OpDef("tirx.pow")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+  // clang-format on
 
-// extra dispatch
-TVM_REGISTER_OP("tirx.erf").set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchFastErf);
+  struct ReturnRound {
+    std::string operator()(PrimType t, std::string name) const { return "round"; }
+  };
 
-// warp-level primitives. Follows implementation in intrin_rule_metal.cc
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic",
-                               DispatchWebGPUShuffle<WebGPUWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_up")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic",
-                               DispatchWebGPUShuffle<WebGPUWarpIntrinsic>);
-
-TVM_REGISTER_OP("tirx.tvm_warp_shuffle_down")
-    .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic",
-                               DispatchWebGPUShuffle<WebGPUWarpIntrinsic>);
-
-// Register low-level WebGPU device intrinsics.
-TVM_REGISTER_OP("tirx.webgpu.subgroup_shuffle")
-    .set_num_inputs(2)
-    .add_argument("var", "Expr", "The variable to sync.")
-    .add_argument("lane", "Expr", "The source thread id.")
-    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
-    .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("webgpu"),
-                                               10)
-    .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
-                                        ffi::String("webgpu.subgroup_shuffle"), 10)
-    .set_attr<TGlobalSymbol>("TGlobalSymbol", "subgroupShuffle")
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TVM_REGISTER_OP("tirx.webgpu.subgroup_shuffle_up")
-    .set_num_inputs(2)
-    .add_argument("var", "Expr", "The variable to sync.")
-    .add_argument("delta", "Expr", "The source lane id offset to be added.")
-    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
-    .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("webgpu"),
-                                               10)
-    .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
-                                        ffi::String("webgpu.subgroup_shuffle_up"), 10)
-    .set_attr<TGlobalSymbol>("TGlobalSymbol", "subgroupShuffleUp")
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
-
-TVM_REGISTER_OP("tirx.webgpu.subgroup_shuffle_down")
-    .set_num_inputs(2)
-    .add_argument("var", "Expr", "The variable to sync.")
-    .add_argument("delta", "Expr", "The source lane id offset to be subtracted.")
-    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"), 10)
-    .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("webgpu"),
-                                               10)
-    .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
-                                        ffi::String("webgpu.subgroup_shuffle_down"), 10)
-    .set_attr<TGlobalSymbol>("TGlobalSymbol", "subgroupShuffleDown")
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+  // WGSL round() uses ties-to-even (banker's rounding), matching IEEE 754 and ONNX Round spec.
+  // clang-format off
+  OpDef("tirx.nearbyint")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<ReturnRound>);
+
+  OpDef("tirx.round")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.sin")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.sinh")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.sqrt")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.tan")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  OpDef("tirx.tanh")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchNumericalStableTanh);
+
+  OpDef("tirx.trunc")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchPureExtern<Direct>);
+
+  // extra dispatch
+  OpDef("tirx.erf")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic", DispatchFastErf);
+
+  // warp-level primitives. Follows implementation in intrin_rule_metal.cc
+  OpDef("tirx.tvm_warp_shuffle")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic",
+                                 DispatchWebGPUShuffle<WebGPUWarpIntrinsic>);
+
+  OpDef("tirx.tvm_warp_shuffle_up")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic",
+                                 DispatchWebGPUShuffle<WebGPUWarpIntrinsic>);
+
+  OpDef("tirx.tvm_warp_shuffle_down")
+      .set_attr<FLowerIntrinsic>("webgpu.FLowerIntrinsic",
+                                 DispatchWebGPUShuffle<WebGPUWarpIntrinsic>);
+
+  // Register low-level WebGPU device intrinsics.
+  OpDef("tirx.webgpu.subgroup_shuffle")
+      .add_arg("var", "The variable to sync.")
+      .add_arg("lane", "The source thread id.")
+      .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("webgpu"))
+      .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
+                                          ffi::String("webgpu.subgroup_shuffle"))
+      .set_attr<TGlobalSymbol>("TGlobalSymbol", "subgroupShuffle")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.webgpu.subgroup_shuffle_up")
+      .add_arg("var", "The variable to sync.")
+      .add_arg("delta", "The source lane id offset to be added.")
+      .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("webgpu"))
+      .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
+                                          ffi::String("webgpu.subgroup_shuffle_up"))
+      .set_attr<TGlobalSymbol>("TGlobalSymbol", "subgroupShuffleUp")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
+
+  OpDef("tirx.webgpu.subgroup_shuffle_down")
+      .add_arg("var", "The variable to sync.")
+      .add_arg("delta", "The source lane id offset to be subtracted.")
+      .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("device_intrin"))
+      .set_attr<tirx::TDeviceIntrinsicNamespace>("TDeviceIntrinsicNamespace", ffi::String("webgpu"))
+      .set_attr<tirx::TScriptPrinterName>("TScriptPrinterName",
+                                          ffi::String("webgpu.subgroup_shuffle_down"))
+      .set_attr<TGlobalSymbol>("TGlobalSymbol", "subgroupShuffleDown")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kOpaque));
   // clang-format on
 }
 
diff --git a/src/ir/attr_registry.h b/src/ir/attr_registry.h
deleted file mode 100644
index de80e5054e..0000000000
--- a/src/ir/attr_registry.h
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/*!
- * \file tvm/ir/attr_registry.h
- * \brief Common global registry for objects that also have additional attrs.
- */
-#ifndef TVM_IR_ATTR_REGISTRY_H_
-#define TVM_IR_ATTR_REGISTRY_H_
-
-#include <tvm/ffi/function.h>
-#include <tvm/ir/attr_registry_map.h>
-
-#include <memory>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-
-namespace tvm {
-
-/*!
- * \brief Implementation of registry with attributes.
- *
- * \tparam EntryType The type of the registry entry.
- * \tparam KeyType The actual key that is used to lookup the attributes.
- *                 each entry has a corresponding key by default.
- */
-template <typename EntryType, typename KeyType>
-class AttrRegistry {
- public:
-  using TSelf = AttrRegistry<EntryType, KeyType>;
-  /*!
-   * \brief Get an entry from the registry.
-   * \param name The name of the item.
-   * \return The corresponding entry.
-   */
-  const EntryType* Get(const ffi::String& name) const {
-    auto it = entry_map_.find(name);
-    if (it != entry_map_.end()) return it->second;
-    return nullptr;
-  }
-
-  /*!
-   * \brief Get an entry or register a new one.
-   * \param name The name of the item.
-   * \return The corresponding entry.
-   */
-  EntryType& RegisterOrGet(const ffi::String& name) {
-    auto it = entry_map_.find(name);
-    if (it != entry_map_.end()) return *it->second;
-    uint32_t registry_index = static_cast<uint32_t>(entries_.size());
-    auto entry = std::unique_ptr<EntryType>(new EntryType(registry_index));
-    auto* eptr = entry.get();
-    eptr->name = name;
-    entry_map_[name] = eptr;
-    entries_.emplace_back(std::move(entry));
-    return *eptr;
-  }
-
-  /*!
-   * \brief List all the entry names in the registry.
-   * \return The entry names.
-   */
-  ffi::Array<ffi::String> ListAllNames() const {
-    ffi::Array<ffi::String> names;
-    for (const auto& kv : entry_map_) {
-      names.push_back(kv.first);
-    }
-    return names;
-  }
-
-  /*!
-   * \brief Update the attribute stable.
-   * \param attr_name The name of the attribute.
-   * \param key The key to the attribute table.
-   * \param value The value to be set.
-   * \param plevel The support level.
-   */
-  void UpdateAttr(const ffi::String& attr_name, const KeyType& key, Any value, int plevel) {
-    using ffi::Any;
-    auto& op_map = attrs_[attr_name];
-    if (op_map == nullptr) {
-      op_map.reset(new AttrRegistryMapContainerMap<KeyType>());
-      op_map->attr_name_ = attr_name;
-    }
-
-    uint32_t index = key->AttrRegistryIndex();
-    if (op_map->data_.size() <= index) {
-      op_map->data_.resize(index + 1, std::make_pair(Any(), 0));
-    }
-    std::pair<ffi::Any, int>& p = op_map->data_[index];
-    TVM_FFI_ICHECK(p.second != plevel)
-        << "Attribute " << attr_name << " of " << key->AttrRegistryName()
-        << " is already registered with same plevel=" << plevel;
-    TVM_FFI_ICHECK(value != nullptr) << "Registered packed_func is Null for " << attr_name
-                                     << " of operator " << key->AttrRegistryName();
-    if (p.second < plevel && value != nullptr) {
-      op_map->data_[index] = std::make_pair(value, plevel);
-    }
-  }
-
-  /*!
-   * \brief Reset an attribute table entry.
-   * \param attr_name The name of the attribute.
-   * \param key The key to the attribute table.
-   */
-  void ResetAttr(const ffi::String& attr_name, const KeyType& key) {
-    auto& op_map = attrs_[attr_name];
-    if (op_map == nullptr) {
-      return;
-    }
-    uint32_t index = key->AttrRegistryIndex();
-    if (op_map->data_.size() > index) {
-      op_map->data_[index] = std::make_pair(ffi::Any(), 0);
-    }
-  }
-
-  /*!
-   * \brief Get an internal attribute map.
-   * \param attr_name The name of the attribute.
-   * \return The result attribute map.
-   */
-  const AttrRegistryMapContainerMap<KeyType>& GetAttrMap(const ffi::String& attr_name) {
-    auto it = attrs_.find(attr_name);
-    if (it == attrs_.end()) {
-      TVM_FFI_THROW(InternalError) << "Attribute \'" << attr_name << "\' is not registered";
-    }
-    return *it->second.get();
-  }
-
-  /*!
-   * \brief Check of attribute has been registered.
-   * \param attr_name The name of the attribute.
-   * \return The check result.
-   */
-  bool HasAttrMap(const ffi::String& attr_name) { return attrs_.count(attr_name); }
-
-  /*!
-   * \return a global singleton of the registry.
-   */
-  static TSelf* Global() {
-    static TSelf* inst = new TSelf();
-    return inst;
-  }
-
- private:
-  // entries in the registry
-  std::vector<std::unique_ptr<EntryType>> entries_;
-  // map from name to entries.
-  std::unordered_map<ffi::String, EntryType*> entry_map_;
-  // storage of additional attribute table.
-  std::unordered_map<ffi::String, std::unique_ptr<AttrRegistryMapContainerMap<KeyType>>> attrs_;
-};
-
-}  // namespace tvm
-#endif  // TVM_IR_ATTR_REGISTRY_H_
diff --git a/src/ir/op.cc b/src/ir/op.cc
index 8f7897e502..7f63eb7ca2 100644
--- a/src/ir/op.cc
+++ b/src/ir/op.cc
@@ -21,169 +21,241 @@
  * \file src/ir/op.cc
  * \brief Primitive operators and intrinsics.
  */
+#include <tvm/ffi/container/dict.h>
 #include <tvm/ffi/extra/structural_mutate.h>
 #include <tvm/ffi/extra/structural_visit.h>
-#include <tvm/ffi/function.h>
 #include <tvm/ffi/reflection/registry.h>
 #include <tvm/ir/op.h>
-#include <tvm/ir/type.h>
 
-#include <memory>
+namespace tvm {
 
-#include "attr_registry.h"
+// Owns canonical Ops and mutable attribute columns for the lifetime of the process.
+class OpRegistry {
+ public:
+  static OpRegistry* Global() {
+    static auto* registry = new OpRegistry();
+    return registry;
+  }
 
-namespace tvm {
+  Op Get(const ffi::String& name) const {
+    auto op = ops_.Get(name);
+    TVM_FFI_CHECK(op.has_value(), AttributeError) << "Operator " << name << " is not registered";
+    return *op;
+  }
 
-namespace {
+  Op GetOrCreate(const ffi::String& name) {
+    if (auto op = ops_.Get(name)) return *op;
+    auto node = ffi::make_object<OpNode>();
+    node->name = name;
+    node->index_ = static_cast<uint32_t>(ops_.size());
+    Op op(std::move(node));
+    ops_.Set(name, op);
+    return op;
+  }
 
-TVMFFIAny OpVisit(ffi::StructuralVisitorObj*, ffi::AnyView) noexcept {
-  // Ops are unique registry atoms.  Avoid reflecting through their registry metadata.
-  return ffi::AnyView(nullptr).CopyToTVMFFIAny();
-}
+  bool Contains(const ffi::String& name) const { return ops_.count(name); }
 
-TVMFFIAny OpMutate(ffi::StructuralMutatorObj*, ffi::AnyView) noexcept {
-  return ffi::Unchanged().CopyToTVMFFIAny();
-}
+  ffi::Array<ffi::String> ListNames() const {
+    ffi::Array<ffi::String> names;
+    for (const auto& [name, op] : ops_) names.push_back(name);
+    return names;
+  }
 
-TVMFFIAny OpMaybeInplaceMutate(ffi::StructuralMutatorObj*, ffi::AnyView) noexcept {
-  return ffi::Unchanged().CopyToTVMFFIAny();
-}
+  bool HasAttrMap(const ffi::String& name) const { return attrs_.count(name); }
 
-}  // namespace
+  ffi::List<ffi::Any> GetAttrColumn(const ffi::String& name) const {
+    auto column = attrs_.Get(name);
+    TVM_FFI_CHECK(column.has_value(), InternalError)
+        << "Attribute '" << name << "' is not registered";
+    return *column;
+  }
 
-TVM_FFI_STATIC_INIT_BLOCK() { ArgumentInfoNode::RegisterReflection(); }
+  void SetAttr(const Op& op, const ffi::String& name, ffi::Any value, bool override) {
+    TVM_FFI_CHECK(value != nullptr, ValueError)
+        << "Registered attribute is null for " << name << " of operator " << op->name;
+    auto column = attrs_.Get(name).value_or(ffi::List<ffi::Any>());
+    uint32_t index = op->index_;
+    TVM_FFI_CHECK(override || index >= column.size() || column[index] == nullptr, ValueError)
+        << "Attribute " << name << " of " << op->name << " is already registered";
+    if (index >= column.size()) column.resize(index + 1);
+    column.Set(index, std::move(value));
+    // Store the same List object: cached views must survive both updates and growth.
+    attrs_.Set(name, column);
+  }
 
-using ffi::Any;
-using ffi::Function;
-using ffi::PackedArgs;
+  void ResetAttr(const Op& op, const ffi::String& name) {
+    if (auto column = attrs_.Get(name)) {
+      if (op->index_ < column->size()) column->Set(op->index_, ffi::Any());
+    }
+  }
 
-using OpRegistry = AttrRegistry<OpRegEntry, Op>;
+ private:
+  ffi::Dict<ffi::String, Op> ops_;
+  ffi::Dict<ffi::String, ffi::List<ffi::Any>> attrs_;
+};
 
-// find operator by name
-const Op& Op::Get(const ffi::String& name) {
-  const OpRegEntry* reg = OpRegistry::Global()->Get(name);
-  TVM_FFI_CHECK(reg != nullptr, AttributeError) << "Operator " << name << " is not registered";
-  return reg->op();
+Op Op::Get(const ffi::String& name) { return OpRegistry::Global()->Get(name); }
+
+ffi::Array<ffi::String> Op::ListNames() { return OpRegistry::Global()->ListNames(); }
+
+bool Op::HasAttrMap(const ffi::String& name) { return OpRegistry::Global()->HasAttrMap(name); }
+
+ffi::List<ffi::Any> Op::GetAttrColumn(const ffi::String& name) {
+  return OpRegistry::Global()->GetAttrColumn(name);
 }
 
-Op OpRegEntry::MakeOp(uint32_t reg_index) {
-  ffi::ObjectPtr<OpNode> n = ffi::make_object<OpNode>();
-  n->index_ = reg_index;
-  return Op(n);
+OpDef::OpDef(const ffi::String& name) : op_(OpRegistry::Global()->GetOrCreate(name)) {}
+
+OpDef::OpDef(const ffi::String& name, const ffi::String& doc) : OpDef(name) { get()->doc = doc; }
+
+OpDef& OpDef::add_arg(const ffi::String& name, const ffi::String& doc) {
+  auto node = ffi::make_object<ArgumentInfoNode>();
+  node->name = name;
+  node->doc = doc;
+  get()->args_info.push_back(ArgumentInfo(std::move(node)));
+  return *this;
 }
 
-TVM_FFI_STATIC_INIT_BLOCK() {
-  namespace refl = tvm::ffi::reflection;
-  OpNode::RegisterReflection();
-  refl::TypeAttrDef<OpNode>()
-      .attr(refl::type_attr::kStructuralVisit, reinterpret_cast<void*>(&OpVisit))
-      .attr(refl::type_attr::kStructuralMutate, reinterpret_cast<void*>(&OpMutate))
-      .attr(refl::type_attr::kStructuralMaybeInplaceMutate,
-            reinterpret_cast<void*>(&OpMaybeInplaceMutate));
-  refl::TypeAttrDef<OpNode>()
-      .def("__data_to_json__",
-           [](const OpNode* node) -> ffi::String {
-             // simply save as the string
-             return node->name;
-           })
-      .def("__data_from_json__", [](const ffi::String& name) -> Op { return Op::Get(name); });
+OpDef& OpDef::add_ty_arg(const ffi::String& name, const ffi::String& doc) {
+  auto node = ffi::make_object<ArgumentInfoNode>();
+  node->name = name;
+  node->doc = doc;
+  get()->ty_args_info.push_back(ArgumentInfo(std::move(node)));
+  return *this;
 }
 
-OpRegEntry::OpRegEntry(uint32_t reg_index) : op_(MakeOp(reg_index)) {}
+OpDef& OpDef::set_attrs_type_key(const ffi::String& key) {
+  uint32_t index = ffi::TypeKeyToIndex(key.c_str());
+  get()->attrs_type_key = key;
+  get()->attrs_type_index = index;
+  return *this;
+}
+
+void OpDef::UpdateAttr(const ffi::String& name, ffi::Any value, bool override) {
+  OpRegistry::Global()->SetAttr(op_, name, std::move(value), override);
+}
 
-OpRegEntry& OpRegEntry::RegisterOrGet(const ffi::String& name) {
-  return OpRegistry::Global()->RegisterOrGet(name);
+OpDef& OpDef::reset_attr(const ffi::String& name) {
+  OpRegistry::Global()->ResetAttr(op_, name);
+  return *this;
 }
 
-// Get attribute map by key
-const AttrRegistryMapContainerMap<Op>& Op::GetAttrMapContainer(const ffi::String& attr_name) {
-  return OpRegistry::Global()->GetAttrMap(attr_name);
+void OpNode::RegisterReflection() {
+  namespace refl = ffi::reflection;
+  // clang-format off
+  refl::ObjectDef<OpNode>()
+      .def_ro("index", &OpNode::index_, refl::AttachFieldFlag::SEqHashIgnore())
+      .def_ro("name", &OpNode::name)
+      .def_ro("doc", &OpNode::doc, refl::AttachFieldFlag::SEqHashIgnore())
+      .def_ro("args_info", &OpNode::args_info, refl::AttachFieldFlag::SEqHashIgnore())
+      .def_ro("attrs_type_key", &OpNode::attrs_type_key, refl::AttachFieldFlag::SEqHashIgnore())
+      .def_ro("allow_extra_args", &OpNode::allow_extra_args, refl::AttachFieldFlag::SEqHashIgnore())
+      .def_ro("ty_args_info", &OpNode::ty_args_info, refl::AttachFieldFlag::SEqHashIgnore())
+      .def_static("get", &Op::Get,
+                  "get(op_name: str) -> Op\n\nReturn the canonical named Op; raise AttributeError "
+                  "if unregistered.")
+      .def_static(
+          "list_op_names", &Op::ListNames,
+          "list_op_names() -> list[str]\n\nReturn registered operator names in unspecified order.")
+      .def(
+          "get_attr",
+          [](Op op, ffi::String name) {
+            return Op::GetAttrMap<ffi::Any>(name).get(op, ffi::Any());
+          },
+          "get_attr(attr_name: str) -> object\n\nReturn this Op's attribute or None; an "
+          "unregistered column raises InternalError.")
+      .def(
+          "has_attr", [](Op, ffi::String name) { return Op::HasAttrMap(name); },
+          "has_attr(attr_name: str) -> bool\n\nReturn whether the attribute column exists in the "
+          "registry.")
+      .def("_set_attr", [](Op op, ffi::String name, ffi::Any value, bool override) {
+        OpDef(op->name)
+            .set_attr(name, value, override);
+      })
+      .def(
+          "reset_attr", [](Op op, ffi::String name) {
+            OpDef(op->name)
+                .reset_attr(name);
+          },
+          "reset_attr(attr_name: str) -> None\n\nRemove this Op's current value; missing values "
+          "are ignored and cached views observe removal.")
+      .def(
+          "add_arg",
+          [](Op op, ffi::String name, ffi::String doc) {
+            OpDef(op->name)
+                .add_arg(name, doc);
+          },
+          "add_arg(name: str, doc: str) -> None\n\nAppend an argument's "
+          "name and documentation.")
+      .def(
+          "set_allow_extra_args", [](Op op) {
+            OpDef(op->name)
+                .allow_extra_args();
+          },
+          "set_allow_extra_args() -> None\n\nAllow value arguments after the required prefix.")
+      .def(
+          "add_ty_arg",
+          [](Op op, ffi::String name, ffi::String doc) {
+            OpDef(op->name)
+                .add_ty_arg(name, doc);
+          },
+          "add_ty_arg(name: str, doc: str) -> None\n\nAppend a "
+          "type-argument "
+          "name and documentation without imposing a count rule.")
+      .def(
+          "set_attrs_type_key",
+          [](Op op, ffi::String key) {
+            OpDef(op->name)
+                .set_attrs_type_key(key);
+          },
+          "set_attrs_type_key(key: str) -> None\n\nResolve and set the attribute object type key "
+          "and runtime index together; an unknown key raises before updating.");
+  // clang-format on
 }
 
-// Check if a key is present in the registry.
-bool Op::HasAttrMap(const ffi::String& attr_name) {
-  return OpRegistry::Global()->HasAttrMap(attr_name);
+namespace {
+
+TVMFFIAny OpVisit(ffi::StructuralVisitorObj*, ffi::AnyView) noexcept {
+  // Ops are unique registry atoms.  Avoid reflecting through their registry metadata.
+  return ffi::AnyView(nullptr).CopyToTVMFFIAny();
 }
 
-// Resets attr of the OpAttrMap.
-void OpRegEntry::reset_attr(const std::string& attr_name) {
-  OpRegistry::Global()->ResetAttr(attr_name, op_);
+TVMFFIAny OpMutate(ffi::StructuralMutatorObj*, ffi::AnyView) noexcept {
+  return ffi::Unchanged().CopyToTVMFFIAny();
 }
 
-void OpRegEntry::UpdateAttr(const ffi::String& key, ffi::Any value, int plevel) {
-  OpRegistry::Global()->UpdateAttr(key, op_, value, plevel);
+TVMFFIAny OpMaybeInplaceMutate(ffi::StructuralMutatorObj*, ffi::AnyView) noexcept {
+  return ffi::Unchanged().CopyToTVMFFIAny();
 }
 
-// Frontend APIs
+}  // namespace
+
 TVM_FFI_STATIC_INIT_BLOCK() {
-  namespace refl = tvm::ffi::reflection;
+  namespace refl = ffi::reflection;
+  ArgumentInfoNode::RegisterReflection();
+  OpNode::RegisterReflection();
+  refl::TypeAttrDef<OpNode>()
+      .attr(refl::type_attr::kStructuralVisit, reinterpret_cast<void*>(&OpVisit))
+      .attr(refl::type_attr::kStructuralMutate, reinterpret_cast<void*>(&OpMutate))
+      .attr(refl::type_attr::kStructuralMaybeInplaceMutate,
+            reinterpret_cast<void*>(&OpMaybeInplaceMutate))
+      .def("__data_to_json__", [](const OpNode* node) { return node->name; })
+      .def("__data_from_json__", &Op::Get);
+  // clang-format off
   refl::GlobalDef()
-      .def("ir.ListOpNames", []() { return OpRegistry::Global()->ListAllNames(); })
-      .def("ir.GetOp", [](ffi::String name) -> Op { return Op::Get(name); })
-      .def("ir.OpGetAttr",
-           [](Op op, ffi::String attr_name) -> ffi::Any {
-             auto op_map = Op::GetAttrMap<ffi::Any>(attr_name);
-             ffi::Any rv;
-             if (op_map.count(op)) {
-               rv = op_map[op];
-             }
-             return rv;
-           })
-      .def("ir.OpHasAttr",
-           [](Op op, ffi::String attr_name) -> bool { return Op::HasAttrMap(attr_name); })
-      .def("ir.OpSetAttr",
-           [](Op op, ffi::String attr_name, ffi::AnyView value, int plevel) {
-             auto& reg = OpRegistry::Global()->RegisterOrGet(op->name).set_name();
-             reg.set_attr(attr_name, value, plevel);
-           })
-      .def("ir.OpResetAttr",
-           [](Op op, ffi::String attr_name) {
-             auto& reg = OpRegistry::Global()->RegisterOrGet(op->name);
-             reg.reset_attr(attr_name);
-           })
+      .def("ir.GetOp", &Op::Get)
+      .def("ir.ListOpNames", &Op::ListNames)
       .def("ir.RegisterOp",
-           [](ffi::String op_name, ffi::String descr) {
-             const OpRegEntry* reg = OpRegistry::Global()->Get(op_name);
-             TVM_FFI_CHECK(reg == nullptr, AttributeError)
-                 << "Operator " << op_name << " is registered before";
-             auto& op = OpRegistry::Global()->RegisterOrGet(op_name).set_name();
-             op.describe(descr);
-           })
-      .def("ir.OpAddArgument",
-           [](Op op, ffi::String name, ffi::String type, ffi::String description) {
-             auto& reg = OpRegistry::Global()->RegisterOrGet(op->name).set_name();
-             reg.add_argument(name, type, description);
-           })
-      .def("ir.OpSetSupportLevel",
-           [](Op op, int level) {
-             auto& reg = OpRegistry::Global()->RegisterOrGet(op->name).set_name();
-             reg.set_support_level(level);
+           [](ffi::String name, ffi::String doc) {
+             TVM_FFI_CHECK(!OpRegistry::Global()->Contains(name), AttributeError)
+                 << "Operator " << name << " is registered before";
+             OpDef(name, doc);
            })
-      .def("ir.OpSetNumInputs",
-           [](Op op, int n) {
-             auto& reg = OpRegistry::Global()->RegisterOrGet(op->name).set_name();
-             reg.set_num_inputs(n);
-           })
-      .def("ir.OpSetAttrsTypeKey",
-           [](Op op, ffi::String key) {
-             auto& reg = OpRegistry::Global()->RegisterOrGet(op->name).set_name();
-             reg.set_attrs_type_key(key);
-           })
-      .def("ir.RegisterOpAttr",
-           [](ffi::String op_name, ffi::String attr_key, ffi::AnyView value, int plevel) {
-             auto& reg = OpRegistry::Global()->RegisterOrGet(op_name).set_name();
-             // enable resgiteration and override of certain properties
-             if (attr_key == "num_inputs" && plevel > 128) {
-               reg.set_num_inputs(value.cast<int>());
-             } else if (attr_key == "attrs_type_key" && plevel > 128) {
-               TVM_FFI_THROW(InternalError) << "attrs type key no longer supported";
-             } else {
-               reg.set_attr(attr_key, value, plevel);
-             }
-           });
-  // override OpNode to use name as the repr
+      .def("ir.RegisterOpAttr", [](ffi::String name, ffi::String key, ffi::Any value, bool override) {
+        OpDef(name)
+            .set_attr(key, value, override);
+      });
+  // clang-format on
 }
 
-// Pattern A (RM): auto-default repr from reflection.
-
 }  // namespace tvm
diff --git a/src/ir/prim/builtin.cc b/src/ir/prim/builtin.cc
index 5d1def4789..bb0f6d7d99 100644
--- a/src/ir/prim/builtin.cc
+++ b/src/ir/prim/builtin.cc
@@ -22,37 +22,53 @@
 namespace tvm {
 namespace prim {
 namespace builtin {
-#define PRIM_DEFINE_BUILTIN_FUNC(OpName)            \
-  const Op& OpName() {                              \
-    static const Op& op = Op::Get("prim." #OpName); \
-    return op;                                      \
-  }                                                 \
-  TVM_REGISTER_OP("prim." #OpName)
-
-PRIM_DEFINE_BUILTIN_FUNC(likely)
-    .set_num_inputs(1)
-    .set_attr<TCallEffectKind>("TCallEffectKind",
-                               static_cast<int64_t>(CallEffectKind::kExprAnnotation))
-    .set_attr<bool>("TVectorizable", true);
-PRIM_DEFINE_BUILTIN_FUNC(if_then_else)
-    .set_num_inputs(3)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure));
-PRIM_DEFINE_BUILTIN_FUNC(vscale).set_attr<TCallEffectKind>(
-    "TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure));
-
-PRIM_DEFINE_BUILTIN_FUNC(ceil)
-    .set_num_inputs(1)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
-    .set_attr<bool>("TVectorizable", true);
-PRIM_DEFINE_BUILTIN_FUNC(log2)
-    .set_num_inputs(1)
-    .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
-    .set_attr<bool>("TVectorizable", true);
-
-PRIM_DEFINE_BUILTIN_FUNC(clz).set_num_inputs(1).set_attr<TCallEffectKind>(
-    "TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure));
-
-#undef PRIM_DEFINE_BUILTIN_FUNC
+
+#define TVM_DEFINE_CACHED_OP_GETTER(Name, RegisteredName) \
+  const Op& Name() {                                      \
+    static const Op op = Op::Get(RegisteredName);         \
+    return op;                                            \
+  }
+
+TVM_DEFINE_CACHED_OP_GETTER(likely, "prim.likely")
+TVM_DEFINE_CACHED_OP_GETTER(if_then_else, "prim.if_then_else")
+TVM_DEFINE_CACHED_OP_GETTER(vscale, "prim.vscale")
+TVM_DEFINE_CACHED_OP_GETTER(ceil, "prim.ceil")
+TVM_DEFINE_CACHED_OP_GETTER(log2, "prim.log2")
+TVM_DEFINE_CACHED_OP_GETTER(clz, "prim.clz")
+
+#undef TVM_DEFINE_CACHED_OP_GETTER
+
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("prim.likely")
+      .add_arg("x", "The input value.")
+      .set_attr<TCallEffectKind>("TCallEffectKind",
+                                 static_cast<int64_t>(CallEffectKind::kExprAnnotation))
+      .set_attr<bool>("TVectorizable", true);
+
+  OpDef("prim.if_then_else")
+      .add_arg("condition", "The condition.")
+      .add_arg("true_value", "The value when the condition is true.")
+      .add_arg("false_value", "The value when the condition is false.")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure));
+
+  OpDef("prim.vscale")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure));
+
+  OpDef("prim.ceil")
+      .add_arg("x", "The input value.")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
+      .set_attr<bool>("TVectorizable", true);
+
+  OpDef("prim.log2")
+      .add_arg("x", "The input value.")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure))
+      .set_attr<bool>("TVectorizable", true);
+
+  OpDef("prim.clz")
+      .add_arg("x", "The input value.")
+      .set_attr<TCallEffectKind>("TCallEffectKind", static_cast<int64_t>(CallEffectKind::kPure));
+}
+
 }  // namespace builtin
 }  // namespace prim
 }  // namespace tvm
diff --git a/src/relax/backend/adreno/annotate_custom_storage.cc b/src/relax/backend/adreno/annotate_custom_storage.cc
index d6735510b4..1748149407 100644
--- a/src/relax/backend/adreno/annotate_custom_storage.cc
+++ b/src/relax/backend/adreno/annotate_custom_storage.cc
@@ -337,7 +337,7 @@ class CollectConsumerScopeInfo : public ExprVisitor {
   }
 
   void VisitExpr_(const CallNode* call) final {
-    static const Op& call_tir_op = Op::Get("relax.call_tir");
+    static const Op call_tir_op = Op::Get("relax.call_tir");
     GlobalVar gv;
     ffi::Array<Attrs> op_attrs;
     ffi::Optional<int64_t> op_pattern = static_cast<int64_t>(OpPatternKind::kOpaque);
@@ -495,7 +495,7 @@ class CollectProducerScopeInfo : public ExprVisitor {
   void VisitBinding_(const VarBindingNode* binding, const CallNode* call) final {
     ExprVisitor::VisitBinding_(binding, call);
 
-    static const Op& call_tir_op = Op::Get("relax.call_tir");
+    static const Op call_tir_op = Op::Get("relax.call_tir");
     Type out_ty = Type::Missing();
 
     if (call->op.same_as(call_tir_op)) {
@@ -621,7 +621,7 @@ class DefineVDevice : ExprMutator {
 
   Expr VisitExpr_(const CallNode* call_node) override {
     auto call = ExprMutator::VisitExpr_(call_node).as_or_throw<Call>();
-    static const Op& call_tir_op = Op::Get("relax.call_tir");
+    static const Op call_tir_op = Op::Get("relax.call_tir");
 
     GlobalVar gv;
     Tuple func_args;
@@ -749,7 +749,7 @@ class DefineVDevice : ExprMutator {
     return std::nullopt;
   }
 
-  const Op& hint_on_device_op_ = Op::Get("relax.hint_on_device");
+  const Op hint_on_device_op_ = Op::Get("relax.hint_on_device");
   IRModule mod_;
   IRModule updates_;
   Target target_;
diff --git a/src/relax/backend/adreno/fold_vdevice_scope_change.cc b/src/relax/backend/adreno/fold_vdevice_scope_change.cc
index 0a0341f6a7..de39763fab 100644
--- a/src/relax/backend/adreno/fold_vdevice_scope_change.cc
+++ b/src/relax/backend/adreno/fold_vdevice_scope_change.cc
@@ -133,7 +133,7 @@ class CollectConsumerDetails : public ExprVisitor {
   }
 
   void VisitExpr_(const CallNode* call) final {
-    static const Op& call_tir_op = Op::Get("relax.call_tir");
+    static const Op call_tir_op = Op::Get("relax.call_tir");
     Tuple func_args;
 
     if (call->op.same_as(call_tir_op)) {
diff --git a/src/relax/backend/contrib/tensorrt/codegen.cc b/src/relax/backend/contrib/tensorrt/codegen.cc
index 7a4df792db..c3fe8e15cc 100644
--- a/src/relax/backend/contrib/tensorrt/codegen.cc
+++ b/src/relax/backend/contrib/tensorrt/codegen.cc
@@ -125,7 +125,7 @@ class CollectFromCompositeFunctionBody : public ExprVisitor {
   void SetArgumentAttributes(const CallNode* call_node) {
     const auto* op_node = call_node->op.as<OpNode>();
     if (op_node == nullptr) return;
-    const ffi::Array<ArgumentInfo>& arg_infos = op_node->arguments;
+    const ffi::Array<ArgumentInfo>& arg_infos = op_node->args_info;
     for (size_t i = 0; i < call_node->args.size() && i < arg_infos.size(); ++i) {
       const Expr& arg = call_node->args[i];
       const std::string key = "arg_" + std::string(arg_infos[i]->name);
diff --git a/src/relax/backend/task_extraction.cc b/src/relax/backend/task_extraction.cc
index 108aa6ffb1..1ed4333de5 100644
--- a/src/relax/backend/task_extraction.cc
+++ b/src/relax/backend/task_extraction.cc
@@ -101,7 +101,7 @@ class TaskExtractor : public ExprVisitor {
   }
 
   void VisitExpr_(const CallNode* call) final {
-    static const Op& call_tir_op = Op::Get("relax.call_tir");
+    static const Op call_tir_op = Op::Get("relax.call_tir");
 
     // TODO(@tvm-team): When we differentiate the call for tirx function and packed function,
     // this logic should be changed accordingly.
diff --git a/src/relax/backend/vm/codegen_vm.cc b/src/relax/backend/vm/codegen_vm.cc
index 53d01f3f31..5423fa7936 100644
--- a/src/relax/backend/vm/codegen_vm.cc
+++ b/src/relax/backend/vm/codegen_vm.cc
@@ -431,11 +431,11 @@ class CodeGenVM : public ExprFunctor<Instruction::Arg(const Expr&)> {
   /*! \brief the context module. */
   IRModule ctx_mod_;
   /*! \brief Cache ops that need to be frequently used later to reduce lookup overhead. */
-  const Op& alloc_storage_op_ = Op::Get("relax.vm.alloc_storage");
-  const Op& alloc_tensor_op_ = Op::Get("relax.vm.alloc_tensor");
-  const Op& kill_object_op_ = Op::Get("relax.vm.kill_object");
-  const Op& call_builtin_with_ctx_op_ = Op::Get("relax.call_builtin_with_ctx");
-  const Op& null_value_op_ = Op::Get("relax.null_value");
+  const Op alloc_storage_op_ = Op::Get("relax.vm.alloc_storage");
+  const Op alloc_tensor_op_ = Op::Get("relax.vm.alloc_tensor");
+  const Op kill_object_op_ = Op::Get("relax.vm.kill_object");
+  const Op call_builtin_with_ctx_op_ = Op::Get("relax.call_builtin_with_ctx");
+  const Op null_value_op_ = Op::Get("relax.null_value");
 };
 
 /*!
diff --git a/src/relax/backend/vm/lower_runtime_builtin.cc b/src/relax/backend/vm/lower_runtime_builtin.cc
index f96cd1a8a2..7cde67fc5e 100644
--- a/src/relax/backend/vm/lower_runtime_builtin.cc
+++ b/src/relax/backend/vm/lower_runtime_builtin.cc
@@ -218,27 +218,27 @@ class LowerRuntimeBuiltinMutator : public ExprMutator {
                 Attrs(), {object_ty_});
   }
 
-  const Op& call_builtin_with_ctx_op_ = Op::Get("relax.call_builtin_with_ctx");
+  const Op call_builtin_with_ctx_op_ = Op::Get("relax.call_builtin_with_ctx");
   const Type object_ty_ = AnyType();
   const Type void_ty_ = TupleType(ffi::Array<Type>({}));
   // object to pattern match.
-  const Op& call_tir_dyn_op_ = Op::Get("relax.vm.call_tir_dyn");
-  const Op& reshape_op_ = Op::Get("relax.reshape");
-  const Op& shape_of_op_ = Op::Get("relax.shape_of");
-  const Op& tensor_to_shape_op_ = Op::Get("relax.tensor_to_shape");
-  const Op& call_py_func_op_ = Op::Get("relax.call_py_func");
-  const Op& to_vdevice_op_ = Op::Get("relax.to_vdevice");
-  const Op& make_closure_op_ = Op::Get("relax.make_closure");
-  const Op& invoke_closure_op_ = Op::Get("relax.invoke_closure");
-  const Op& alloc_tensor_op_ = Op::Get("relax.builtin.alloc_tensor");
-  const Op& mem_alloc_storage_op_ = Op::Get("relax.memory.alloc_storage");
-  const Op& mem_alloc_tensor_op_ = Op::Get("relax.memory.alloc_tensor");
-  const Op& mem_kill_storage_op_ = Op::Get("relax.memory.kill_storage");
-  const Op& mem_kill_tensor_op_ = Op::Get("relax.memory.kill_tensor");
+  const Op call_tir_dyn_op_ = Op::Get("relax.vm.call_tir_dyn");
+  const Op reshape_op_ = Op::Get("relax.reshape");
+  const Op shape_of_op_ = Op::Get("relax.shape_of");
+  const Op tensor_to_shape_op_ = Op::Get("relax.tensor_to_shape");
+  const Op call_py_func_op_ = Op::Get("relax.call_py_func");
+  const Op to_vdevice_op_ = Op::Get("relax.to_vdevice");
+  const Op make_closure_op_ = Op::Get("relax.make_closure");
+  const Op invoke_closure_op_ = Op::Get("relax.invoke_closure");
+  const Op alloc_tensor_op_ = Op::Get("relax.builtin.alloc_tensor");
+  const Op mem_alloc_storage_op_ = Op::Get("relax.memory.alloc_storage");
+  const Op mem_alloc_tensor_op_ = Op::Get("relax.memory.alloc_tensor");
+  const Op mem_kill_storage_op_ = Op::Get("relax.memory.kill_storage");
+  const Op mem_kill_tensor_op_ = Op::Get("relax.memory.kill_tensor");
   // functions to lower to
-  const Op& vm_alloc_storage_op_ = Op::Get("relax.vm.alloc_storage");
-  const Op& vm_alloc_tensor_op_ = Op::Get("relax.vm.alloc_tensor");
-  const Op& vm_kill_object_op_ = Op::Get("relax.vm.kill_object");
+  const Op vm_alloc_storage_op_ = Op::Get("relax.vm.alloc_storage");
+  const Op vm_alloc_tensor_op_ = Op::Get("relax.vm.alloc_tensor");
+  const Op vm_kill_object_op_ = Op::Get("relax.vm.kill_object");
   // Function to compute allocated shape.
   const ExternFunc builtin_compute_alloc_shape_{"vm.builtin.compute_alloc_shape"};
   const ExternFunc builtin_call_tir_dyn_{"vm.builtin.call_tir_dyn"};
diff --git a/src/relax/backend/vm/vm_shape_lower.cc b/src/relax/backend/vm/vm_shape_lower.cc
index a8807e2728..8017dc4f49 100644
--- a/src/relax/backend/vm/vm_shape_lower.cc
+++ b/src/relax/backend/vm/vm_shape_lower.cc
@@ -933,8 +933,8 @@ class VMShapeLowerMutator
    */
   std::vector<PrimExprSlot*> ready_vars_;
   // call builtin cop
-  const Op& call_builtin_with_ctx_op_ = Op::Get("relax.call_builtin_with_ctx");
-  const Op& null_value_op_ = Op::Get("relax.null_value");
+  const Op call_builtin_with_ctx_op_ = Op::Get("relax.call_builtin_with_ctx");
+  const Op null_value_op_ = Op::Get("relax.null_value");
   // common type
   const Type object_ty_ = AnyType();
   const Type void_ty_ = TupleType(ffi::Array<Type>({}));
diff --git a/src/relax/distributed/transform/lower_global_view_to_local_view.cc b/src/relax/distributed/transform/lower_global_view_to_local_view.cc
index 02ec388acf..55a5f7f211 100644
--- a/src/relax/distributed/transform/lower_global_view_to_local_view.cc
+++ b/src/relax/distributed/transform/lower_global_view_to_local_view.cc
@@ -383,7 +383,7 @@ class LowerTIRToLocalView : public ExprMutator {
   }
 
   void VisitBinding_(const VarBindingNode* binding, const CallNode* val) final {
-    static const Op& call_tir_op = Op::Get("relax.call_tir");
+    static const Op call_tir_op = Op::Get("relax.call_tir");
     if (!val->op.same_as(call_tir_op)) {
       ExprMutator::VisitBinding_(binding, val);
       return;
diff --git a/src/relax/distributed/transform/propagate_sharding.cc b/src/relax/distributed/transform/propagate_sharding.cc
index a7a1d0c2c6..be1e46d59e 100644
--- a/src/relax/distributed/transform/propagate_sharding.cc
+++ b/src/relax/distributed/transform/propagate_sharding.cc
@@ -47,7 +47,7 @@ void CollectAxisGraphBinary(const VarBindingNode* binding, const CallNode* call,
       "add",     "subtract",      "multiply", "divide",     "power",     "floor_divide", "equal",
       "greater", "greater_equal", "less",     "less_equal", "not_equal", "minimum",      "maximum"};
   for (const auto& op_name : binary_op_names) {
-    const Op& binary_op = Op::Get("relax." + op_name);
+    const Op binary_op = Op::Get("relax." + op_name);
     if (call->op.same_as(binary_op)) {
       BuildAxisGraphBinary(binding->var, ffi::GetRef<Call>(call), axis_group_graph);
       break;
@@ -70,7 +70,7 @@ void CollectAxisGraphUnary(const VarBindingNode* binding, const CallNode* call,
       "isinf",  "isnan",    "dist.annotate_sharding",
       "erf",    "nn.gelu",  "builtin.stop_lift_params"};
   for (const auto& op_name : unary_op_names) {
-    const Op& unary_op = Op::Get("relax." + op_name);
+    const Op unary_op = Op::Get("relax." + op_name);
     if (call->op.same_as(unary_op)) {
       BuildAxisGraphUnary(binding->var, ffi::GetRef<Call>(call), axis_group_graph);
     }
@@ -82,7 +82,7 @@ void CollectAxisGraphReduce(const VarBindingNode* binding, const CallNode* call,
   const std::vector<std::string> reduction_op_names = {"sum",  "max", "min",      "prod",
                                                        "mean", "std", "variance", "nn.softmax"};
   for (const auto& op_name : reduction_op_names) {
-    const Op& reduction_op = Op::Get("relax." + op_name);
+    const Op reduction_op = Op::Get("relax." + op_name);
     if (call->op.same_as(reduction_op)) {
       BuildAxisGraphReduce(binding->var, ffi::GetRef<Call>(call), axis_group_graph);
       break;
@@ -92,7 +92,7 @@ void CollectAxisGraphReduce(const VarBindingNode* binding, const CallNode* call,
 
 void CollectAxisGraphMatmul(const VarBindingNode* binding, const CallNode* call,
                             AxisGroupGraph* axis_group_graph) {
-  static const Op& matmul_op = Op::Get("relax.matmul");
+  static const Op matmul_op = Op::Get("relax.matmul");
   if (call->op.same_as(matmul_op)) {
     BuildAxisGraphMatmul(binding->var, ffi::GetRef<Call>(call), axis_group_graph);
   }
@@ -100,7 +100,7 @@ void CollectAxisGraphMatmul(const VarBindingNode* binding, const CallNode* call,
 
 void CollectAxisGraphPermuteDims(const VarBindingNode* binding, const CallNode* call,
                                  AxisGroupGraph* axis_group_graph) {
-  static const Op& permute_dims_op = Op::Get("relax.permute_dims");
+  static const Op permute_dims_op = Op::Get("relax.permute_dims");
   if (call->op.same_as(permute_dims_op)) {
     BuildAxisGraphPermuteDims(binding->var, ffi::GetRef<Call>(call), axis_group_graph);
   }
@@ -108,7 +108,7 @@ void CollectAxisGraphPermuteDims(const VarBindingNode* binding, const CallNode*
 
 void CollectAxisGraphReshape(const VarBindingNode* binding, const CallNode* call,
                              AxisGroupGraph* axis_group_graph) {
-  static const Op& reshape_op = Op::Get("relax.reshape");
+  static const Op reshape_op = Op::Get("relax.reshape");
   if (call->op.same_as(reshape_op)) {
     BuildAxisGraphReshape(binding->var, ffi::GetRef<Call>(call), axis_group_graph);
   }
@@ -117,7 +117,7 @@ void CollectAxisGraphReshape(const VarBindingNode* binding, const CallNode* call
 void CollectAxisGraphForDeviceMesh(const VarBindingNode* binding, const CallNode* call,
                                    AxisGroupGraph* axis_group_graph) {
   ffi::Array<Expr> tensor_list;
-  static const Op& call_tir_op = Op::Get("relax.call_tir");
+  static const Op call_tir_op = Op::Get("relax.call_tir");
   ffi::Array<Expr> args;
   if (call->op.same_as(call_tir_op)) {
     args = call->args[1].as_or_throw<Tuple>()->fields;
@@ -157,7 +157,7 @@ class AxisGroupGraphBuilder : public ExprVisitor {
     CollectAxisGraphMatmul(binding, val, axis_group_graph_);
     CollectAxisGraphPermuteDims(binding, val, axis_group_graph_);
     CollectAxisGraphReshape(binding, val, axis_group_graph_);
-    static const Op& call_tir_op = Op::Get("relax.call_tir");
+    static const Op call_tir_op = Op::Get("relax.call_tir");
     if (val->op.same_as(call_tir_op)) {
       if (ffi::Optional<tirx::PrimFunc> func = MatchPrimFunc(mod_, val->args[0])) {
         BuildAxisGraphCallTIR(binding->var, ffi::GetRef<Call>(val), func.value(),
@@ -225,7 +225,7 @@ class ShardingAnnotationCollector : public ExprVisitor {
   explicit ShardingAnnotationCollector(AxisGroupGraph* axis_group_graph)
       : axis_group_graph_(axis_group_graph) {}
   void VisitBinding_(const VarBindingNode* binding, const CallNode* val) {
-    static const Op& annotate_sharding_op = Op::Get("relax.dist.annotate_sharding");
+    static const Op annotate_sharding_op = Op::Get("relax.dist.annotate_sharding");
     if (val->op.same_as(annotate_sharding_op)) {
       const auto* attrs = val->attrs.as<DistributionAttrs>();
       TVM_FFI_ICHECK(attrs);
@@ -435,7 +435,7 @@ class DistributedIRBuilder : public ExprMutator {
   }
 
   Expr VisitExpr_(const CallNode* call) final {
-    static const Op& call_tir_op = Op::Get("relax.call_tir");
+    static const Op call_tir_op = Op::Get("relax.call_tir");
     FBuildAxisGraph f = [&](const Var& var, const Call& call, AxisGroupGraph* axis_group_graph) {
       ffi::Optional<tirx::PrimFunc> prim_func =
           MatchPrimFunc(this->builder_->GetContextIRModule(), call->args[0]);
@@ -472,7 +472,7 @@ class DistributedIRBuilder : public ExprMutator {
   }
 
   Expr RemoveAnnotateSharding(Call call) {
-    static const Op& annotate_sharding_op = Op::Get("relax.dist.annotate_sharding");
+    static const Op annotate_sharding_op = Op::Get("relax.dist.annotate_sharding");
     if (call->op.same_as(annotate_sharding_op)) {
       return call->args[0];
     } else {
diff --git a/src/relax/ir/expr.cc b/src/relax/ir/expr.cc
index 5c89eec025..472dee24a3 100644
--- a/src/relax/ir/expr.cc
+++ b/src/relax/ir/expr.cc
@@ -809,7 +809,7 @@ Expr GetShapeOf(const Expr& expr) {
   TVM_FFI_ICHECK(tinfo != nullptr) << "ShapeOf can only be applied to expr with TensorType";
   if (tinfo->shape.has_value()) return tinfo->shape.value();
 
-  static const Op& op = Op::Get("relax.shape_of");
+  static const Op op = Op::Get("relax.shape_of");
   // default case, call shape of, eagerly normalize the expr.
   Call call_shape_of(Type::Missing(), op, {expr}, {}, {});
   UpdateType(call_shape_of, ShapeType(tinfo->ndim));
diff --git a/src/relax/op/ccl/ccl.cc b/src/relax/op/ccl/ccl.cc
index f5b660d1bc..f26578afc1 100644
--- a/src/relax/op/ccl/ccl.cc
+++ b/src/relax/op/ccl/ccl.cc
@@ -42,7 +42,7 @@ Expr allreduce(Expr x, ffi::String op_type, bool in_group) {
   attrs->op_type = std::move(op_type);
   attrs->in_group = std::move(in_group);
 
-  static const Op& op = Op::Get("relax.ccl.allreduce");
+  static const Op op = Op::Get("relax.ccl.allreduce");
   return Call(Type::Missing(), op, {std::move(x)}, Attrs{attrs}, {});
 }
 
@@ -56,13 +56,14 @@ Type InferTypeAllReduce(const Call& call, const BlockBuilder& ctx) {
   return input_ty;
 }
 
-TVM_REGISTER_OP("relax.ccl.allreduce")
-    .set_attrs_type<AllReduceAttrs>()
-    .set_num_inputs(1)
-    .add_argument("x", "Tensor", "Input to which allreduce will be applied.")
-    .set_attr<FInferType>("FInferType", InferTypeAllReduce)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.ccl.allreduce")
+      .add_arg("x", "Input to which allreduce will be applied.")
+      .attrs_type<AllReduceAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeAllReduce)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.ccl.allgather */
 
@@ -71,7 +72,7 @@ Expr allgather(Expr x, int num_workers, bool in_group) {
   attrs->num_workers = std::move(num_workers);
   attrs->in_group = std::move(in_group);
 
-  static const Op& op = Op::Get("relax.ccl.allgather");
+  static const Op op = Op::Get("relax.ccl.allgather");
   return Call(Type::Missing(), op, {std::move(x)}, Attrs{attrs}, {});
 }
 
@@ -96,16 +97,17 @@ Type InferTypeAllGather(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr(output_shape), output_dtype, input_ty->vdevice);
 }
 
-TVM_REGISTER_OP("relax.ccl.allgather")
-    .set_num_inputs(1)
-    .add_argument("x", "Tensor", "Input to which allgather will be applied.")
-    .set_attr<FInferType>("FInferType", InferTypeAllGather)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.ccl.allgather")
+      .add_arg("x", "Input to which allgather will be applied.")
+      .set_attr<FInferType>("FInferType", InferTypeAllGather)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.ccl.broadcast_from_worker0 */
 Expr broadcast_from_worker0(Expr x) {
-  static const Op& op = Op::Get("relax.ccl.broadcast_from_worker0");
+  static const Op op = Op::Get("relax.ccl.broadcast_from_worker0");
   return Call(Type::Missing(), op, {std::move(x)}, {}, {});
 }
 
@@ -119,12 +121,13 @@ Type InferTypeBroadcastFromZero(const Call& call, const BlockBuilder& ctx) {
   return input_ty;
 }
 
-TVM_REGISTER_OP("relax.ccl.broadcast_from_worker0")
-    .set_num_inputs(1)
-    .add_argument("x", "Tensor", "Input to be broadcast.")
-    .set_attr<FInferType>("FInferType", InferTypeBroadcastFromZero)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.ccl.broadcast_from_worker0")
+      .add_arg("x", "Input to be broadcast.")
+      .set_attr<FInferType>("FInferType", InferTypeBroadcastFromZero)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.ccl.scatter_from_worker0 */
 
@@ -132,7 +135,7 @@ Expr scatter_from_worker0(Expr data, int num_workers, int axis) {
   ffi::ObjectPtr<ScatterCollectiveAttrs> attrs = ffi::make_object<ScatterCollectiveAttrs>();
   attrs->num_workers = std::move(num_workers);
   attrs->axis = std::move(axis);
-  static const Op& op = Op::Get("relax.ccl.scatter_from_worker0");
+  static const Op op = Op::Get("relax.ccl.scatter_from_worker0");
 
   return Call(Type::Missing(), op, {std::move(data)}, Attrs{attrs}, {});
 }
@@ -166,13 +169,14 @@ Type InferTypeScatter(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr(output_shape), output_dtype, input_ty->vdevice);
 }
 
-TVM_REGISTER_OP("relax.ccl.scatter_from_worker0")
-    .set_num_inputs(1)
-    .add_argument("x", "Tensor",
-                  "The buffer to be divided into equal parts and sent to each worker accordingly.")
-    .set_attrs_type<ScatterCollectiveAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeScatter)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.ccl.scatter_from_worker0")
+      .add_arg("x",
+               "The buffer to be divided into equal parts and sent to each worker accordingly.")
+      .attrs_type<ScatterCollectiveAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeScatter)
+      .set_attr<bool>("FPurity", true);
+}
 
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/distributed/binary.cc b/src/relax/op/distributed/binary.cc
index 5dca865f78..2ec0550ed4 100644
--- a/src/relax/op/distributed/binary.cc
+++ b/src/relax/op/distributed/binary.cc
@@ -36,42 +36,89 @@ Type InferDistTypeBroadcastCMP(const Call& call, const BlockBuilder& ctx) {
 
 /***************** Arithmetic operators *****************/
 
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(add);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(divide);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(floor_divide);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(multiply);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(power);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(subtract);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(mod);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(floor_mod);
-
-/***************** Comparison operators *****************/
-
-RELAX_REGISTER_CMP_DIST_INFER_TYPE(equal);
-RELAX_REGISTER_CMP_DIST_INFER_TYPE(greater);
-RELAX_REGISTER_CMP_DIST_INFER_TYPE(greater_equal);
-RELAX_REGISTER_CMP_DIST_INFER_TYPE(less);
-RELAX_REGISTER_CMP_DIST_INFER_TYPE(less_equal);
-RELAX_REGISTER_CMP_DIST_INFER_TYPE(not_equal);
-
-/***************** Min/Max operators *****************/
-
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(minimum);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(maximum);
-
-/***************** Logical operators *****************/
-
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(logical_and);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(logical_or);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(logical_xor);
-
-/***************** Bitwise operators *****************/
-
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(bitwise_and);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(bitwise_or);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(bitwise_xor);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(left_shift);
-RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(right_shift);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  // clang-format off
+  OpDef("relax.add")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.divide")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.floor_divide")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.multiply")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.power")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.subtract")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.mod")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.floor_mod")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  /***************** Comparison operators *****************/
+
+  OpDef("relax.equal")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP);
+
+  OpDef("relax.greater")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP);
+
+  OpDef("relax.greater_equal")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP);
+
+  OpDef("relax.less")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP);
+
+  OpDef("relax.less_equal")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP);
+
+  OpDef("relax.not_equal")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP);
+
+  /***************** Min/Max operators *****************/
+
+  OpDef("relax.minimum")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.maximum")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  /***************** Logical operators *****************/
+
+  OpDef("relax.logical_and")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.logical_or")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.logical_xor")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  /***************** Bitwise operators *****************/
+
+  OpDef("relax.bitwise_and")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.bitwise_or")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.bitwise_xor")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.left_shift")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+
+  OpDef("relax.right_shift")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith);
+  // clang-format on
+}
 
 }  // namespace distributed
 }  // namespace relax
diff --git a/src/relax/op/distributed/binary.h b/src/relax/op/distributed/binary.h
index df07bc29e7..f68f2bf153 100644
--- a/src/relax/op/distributed/binary.h
+++ b/src/relax/op/distributed/binary.h
@@ -73,14 +73,6 @@ Type InferDistTypeBroadcastArith(const Call& call, const BlockBuilder& ctx);
 
 Type InferDistTypeBroadcastCMP(const Call& call, const BlockBuilder& ctx);
 
-#define RELAX_REGISTER_BINARY_BROADCAST_DIST_INFER_TYPE(OpName) \
-  TVM_REGISTER_OP("relax." #OpName)                             \
-      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastArith)
-
-#define RELAX_REGISTER_CMP_DIST_INFER_TYPE(OpName) \
-  TVM_REGISTER_OP("relax." #OpName)                \
-      .set_attr<FInferType>("dist.FInferType", InferDistTypeBroadcastCMP)
-
 }  // namespace distributed
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/distributed/ccl.cc b/src/relax/op/distributed/ccl.cc
index 4f899cdec4..8ec459d25d 100644
--- a/src/relax/op/distributed/ccl.cc
+++ b/src/relax/op/distributed/ccl.cc
@@ -37,8 +37,12 @@ Type InferDistTypeAllReduce(const Call& call, const BlockBuilder& ctx) {
                      Placement::FromText(std::string(device_mesh->shape.size(), 'R')));
 }
 
-TVM_REGISTER_OP("relax.ccl.allreduce")
-    .set_attr<FInferType>("dist.FInferType", InferDistTypeAllReduce);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  // clang-format off
+  OpDef("relax.ccl.allreduce")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeAllReduce);
+  // clang-format on
+}
 
 }  // namespace distributed
 }  // namespace relax
diff --git a/src/relax/op/distributed/distributed.cc b/src/relax/op/distributed/distributed.cc
index 0ae767aaf0..5075f8d354 100644
--- a/src/relax/op/distributed/distributed.cc
+++ b/src/relax/op/distributed/distributed.cc
@@ -49,7 +49,7 @@ Expr annotate_sharding(Expr input, distributed::DeviceMesh device_mesh,
   attrs->device_mesh = device_mesh;
   attrs->placement = placement;
 
-  static const Op& op = Op::Get("relax.dist.annotate_sharding");
+  static const Op op = Op::Get("relax.dist.annotate_sharding");
   return Call(Type::Missing(), op, {std::move(input)}, Attrs(attrs), {});
 }
 
@@ -62,12 +62,13 @@ Type InferTypeAnnotateSharding(const Call& call, const BlockBuilder& ctx) {
   return GetType(call->args[0]);
 }
 
-TVM_REGISTER_OP("relax.dist.annotate_sharding")
-    .set_num_inputs(1)
-    .add_argument("input", "Tensor", "The input tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeAnnotateSharding)
-    .set_attr<FInferType>("dist.FInferType", InferTypeAnnotateSharding)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.dist.annotate_sharding")
+      .add_arg("input", "The input tensor.")
+      .set_attr<FInferType>("FInferType", InferTypeAnnotateSharding)
+      .set_attr<FInferType>("dist.FInferType", InferTypeAnnotateSharding)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.dist.redistribute */
 
@@ -77,7 +78,7 @@ Expr redistribute(Expr input, distributed::DeviceMesh device_mesh,
   attrs->device_mesh = device_mesh;
   attrs->placement = placement;
 
-  static const Op& op = Op::Get("relax.dist.redistribute");
+  static const Op op = Op::Get("relax.dist.redistribute");
   return Call(Type::Missing(), op, {std::move(input)}, Attrs(attrs), {});
 }
 
@@ -93,11 +94,12 @@ Type InferDistTypeRedistribute(const Call& call, const BlockBuilder& ctx) {
   return distributed::DTensorType(ty->tensor_ty, attrs->device_mesh, attrs->placement);
 }
 
-TVM_REGISTER_OP("relax.dist.redistribute")
-    .set_num_inputs(1)
-    .add_argument("input", "Tensor", "The input tensor.")
-    .set_attr<FInferType>("dist.FInferType", InferDistTypeRedistribute)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.dist.redistribute")
+      .add_arg("input", "The input tensor.")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeRedistribute)
+      .set_attr<bool>("FPurity", true);
+}
 
 Type InferTypeCallTIRLocalView(const Call& call, const BlockBuilder& ctx) {
   if (call->ty_args.size() != 1) {
@@ -110,12 +112,14 @@ Type InferTypeCallTIRLocalView(const Call& call, const BlockBuilder& ctx) {
   return call->ty_args[0];
 }
 
-TVM_REGISTER_OP("relax.dist.call_tir_local_view")
-    .set_num_inputs(2)
-    .add_argument("func", "Expr", "The destination-passing-style function.")
-    .add_argument("args", "Tuple", "The input arguments.")
-    .set_attr<FInferType>("FInferType", InferTypeCallTIRLocalView)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.dist.call_tir_local_view")
+      .add_arg("func", "The destination-passing-style function.")
+      .add_arg("args", "The input arguments.")
+      .add_ty_arg("out_type", "The output type.")
+      .set_attr<FInferType>("FInferType", InferTypeCallTIRLocalView)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeCallTIRLocalView(Expr func, Tuple args, ffi::Array<distributed::DTensorType> out_ty_list) {
   for (const distributed::DTensorType& ty : out_ty_list) {
@@ -133,7 +137,7 @@ Expr MakeCallTIRLocalView(Expr func, Tuple args, ffi::Array<distributed::DTensor
     out_ty = TupleType({out_ty_list.begin(), out_ty_list.end()});
   }
 
-  static const Op& op = Op::Get("relax.dist.call_tir_local_view");
+  static const Op op = Op::Get("relax.dist.call_tir_local_view");
   return Call(Type::Missing(), op, {func, args}, {}, {out_ty});
 }
 
@@ -204,7 +208,7 @@ Expr redistribute_replica_to_shard(Expr input, int num_workers, int axis) {
   ffi::ObjectPtr<ScatterCollectiveAttrs> attrs = ffi::make_object<ScatterCollectiveAttrs>();
   attrs->num_workers = std::move(num_workers);
   attrs->axis = std::move(axis);
-  static const Op& op = Op::Get("relax.dist.redistribute_replica_to_shard");
+  static const Op op = Op::Get("relax.dist.redistribute_replica_to_shard");
 
   return Call(Type::Missing(), op, {std::move(input)}, Attrs{attrs}, {});
 }
@@ -213,15 +217,14 @@ TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.dist.redistribute_replica_to_shard",
                         redistribute_replica_to_shard);
-}
 
-TVM_REGISTER_OP("relax.dist.redistribute_replica_to_shard")
-    .set_num_inputs(1)
-    .add_argument("input", "Tensor", "The buffer to be sliced.")
-    .set_attrs_type<ScatterCollectiveAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeRtoS)
-    .set_attr<FInferType>("dist.FInferType", InferDistTypeRtoS)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.dist.redistribute_replica_to_shard")
+      .add_arg("input", "The buffer to be sliced.")
+      .attrs_type<ScatterCollectiveAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeRtoS)
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeRtoS)
+      .set_attr<bool>("FPurity", true);
+}
 
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/distributed/linear_algebra.cc b/src/relax/op/distributed/linear_algebra.cc
index 9fc5d29583..e7240c1bc0 100644
--- a/src/relax/op/distributed/linear_algebra.cc
+++ b/src/relax/op/distributed/linear_algebra.cc
@@ -96,7 +96,12 @@ Type InferDistTypeMatmul(const Call& call, const BlockBuilder& ctx) {
   TensorType output_tensor_ty(ShapeExpr(output_shape), out_dtype);
   return InferShardingSpec(call, ctx, output_tensor_ty, distributed::BuildAxisGraphMatmul);
 }
-TVM_REGISTER_OP("relax.matmul").set_attr<FInferType>("dist.FInferType", InferDistTypeMatmul);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  // clang-format off
+  OpDef("relax.matmul")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeMatmul);
+  // clang-format on
+}
 
 }  // namespace distributed
 }  // namespace relax
diff --git a/src/relax/op/distributed/manipulate.cc b/src/relax/op/distributed/manipulate.cc
index 6d30cc879e..2febd5dd6d 100644
--- a/src/relax/op/distributed/manipulate.cc
+++ b/src/relax/op/distributed/manipulate.cc
@@ -77,8 +77,12 @@ Type InferDistTypePermuteDims(const Call& call, const BlockBuilder& ctx) {
   return InferShardingSpec(call, ctx, output_tensor_ty, distributed::BuildAxisGraphPermuteDims);
 }
 
-TVM_REGISTER_OP("relax.permute_dims")
-    .set_attr<FInferType>("dist.FInferType", InferDistTypePermuteDims);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  // clang-format off
+  OpDef("relax.permute_dims")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypePermuteDims);
+  // clang-format on
+}
 
 Type InferDistTypeReshape(const Call& call, const BlockBuilder& ctx) {
   if (call->args.size() != 2) {
@@ -128,7 +132,12 @@ Type InferDistTypeReshape(const Call& call, const BlockBuilder& ctx) {
   return InferShardingSpec(call, ctx, output_tensor_ty, distributed::BuildAxisGraphReshape);
 }
 
-TVM_REGISTER_OP("relax.reshape").set_attr<FInferType>("dist.FInferType", InferDistTypeReshape);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  // clang-format off
+  OpDef("relax.reshape")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeReshape);
+  // clang-format on
+}
 
 }  // namespace distributed
 }  // namespace relax
diff --git a/src/relax/op/distributed/nn.cc b/src/relax/op/distributed/nn.cc
index d87a2c1bfa..48fac7c458 100644
--- a/src/relax/op/distributed/nn.cc
+++ b/src/relax/op/distributed/nn.cc
@@ -46,19 +46,20 @@ Type InferDistTypeSoftmax(const Call& call, const BlockBuilder& ctx) {
   return InferShardingSpec(call, ctx, input_tensor_ty, distributed::BuildAxisGraphReduce);
 }
 
-TVM_REGISTER_OP("relax.nn.softmax").set_attr<FInferType>("dist.FInferType", InferDistTypeSoftmax);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  // clang-format off
+  OpDef("relax.nn.softmax")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeSoftmax);
 
-/* relax.nn.relu */
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.relu, /*require_float_dtype=*/false);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.relu, /*require_float_dtype=*/false);
 
-/* relax.nn.gelu */
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.gelu, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.gelu, /*require_float_dtype=*/true);
 
-/* relax.nn.gelu_tanh */
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.gelu_tanh, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.gelu_tanh, /*require_float_dtype=*/true);
 
-/* relax.nn.silu */
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.silu, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(nn.silu, /*require_float_dtype=*/true);
+  // clang-format on
+}
 
 }  // namespace distributed
 }  // namespace relax
diff --git a/src/relax/op/distributed/op.cc b/src/relax/op/distributed/op.cc
index 4b10b33ed4..697b7fee5d 100644
--- a/src/relax/op/distributed/op.cc
+++ b/src/relax/op/distributed/op.cc
@@ -35,7 +35,12 @@ Type InferDistTypeCallTIR(const Call& call, const BlockBuilder& ctx) {
   return call->ty_args[0];
 }
 
-TVM_REGISTER_OP("relax.call_tir").set_attr<FInferType>("dist.FInferType", InferDistTypeCallTIR);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  // clang-format off
+  OpDef("relax.call_tir")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeCallTIR);
+  // clang-format on
+}
 
 Type InferDistTypeStopLiftParams(const Call& call, const BlockBuilder& ctx) {
   if (call->args.size() != 1) {
@@ -44,8 +49,10 @@ Type InferDistTypeStopLiftParams(const Call& call, const BlockBuilder& ctx) {
   return call->args[0]->ty.as_or_throw<Type>();
 }
 
-TVM_REGISTER_OP("relax.builtin.stop_lift_params")
-    .set_attr<FInferType>("dist.FInferType", InferDistTypeStopLiftParams);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.builtin.stop_lift_params")
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeStopLiftParams);
+}
 
 }  // namespace distributed
 }  // namespace relax
diff --git a/src/relax/op/distributed/statistical.cc b/src/relax/op/distributed/statistical.cc
index d24cc537da..3bdd322d35 100644
--- a/src/relax/op/distributed/statistical.cc
+++ b/src/relax/op/distributed/statistical.cc
@@ -75,13 +75,15 @@ Type InferDistTypeStatistical(const Call& call, const BlockBuilder& ctx) {
 
   return InferShardingSpec(call, ctx, output_tensor_ty, distributed::BuildAxisGraphReduce);
 }
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(max);
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(mean);
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(min);
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(prod);
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(std);
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(sum);
-RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(variance);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(max);
+  RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(mean);
+  RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(min);
+  RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(prod);
+  RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(std);
+  RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(sum);
+  RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(variance);
+}
 
 }  // namespace distributed
 }  // namespace relax
diff --git a/src/relax/op/distributed/statistical.h b/src/relax/op/distributed/statistical.h
index 6aa63ae32a..7d44e1532e 100644
--- a/src/relax/op/distributed/statistical.h
+++ b/src/relax/op/distributed/statistical.h
@@ -33,9 +33,11 @@ namespace distributed {
 
 Type InferDistTypeStatistical(const Call& call, const BlockBuilder& ctx);
 
+// clang-format off
 #define RELAX_REGISTER_STATISTICAL_DIST_INFER_TYPE(OpName) \
-  TVM_REGISTER_OP("relax." #OpName)                        \
+  OpDef("relax." #OpName)                              \
       .set_attr<FInferType>("dist.FInferType", InferDistTypeStatistical)
+// clang-format on
 
 }  // namespace distributed
 }  // namespace relax
diff --git a/src/relax/op/distributed/unary.cc b/src/relax/op/distributed/unary.cc
index 8e4ccce23a..9b59b0ad4e 100644
--- a/src/relax/op/distributed/unary.cc
+++ b/src/relax/op/distributed/unary.cc
@@ -28,37 +28,38 @@ Type InferDistTypeUnaryCheck(const Call& call, const BlockBuilder& ctx) {
                                    [](const TensorType& input_ty) { return PrimType::Bool(); });
 }
 
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(abs, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(acos, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(acosh, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(asin, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(asinh, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(atan, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(atanh, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(bitwise_not, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(ceil, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(cos, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(cosh, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(exp, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(floor, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(log, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(logical_not, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(negative, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(round, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(rsqrt, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sigmoid, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sign, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sin, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sinh, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(square, /*require_float_dtype=*/false);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sqrt, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(tan, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(tanh, /*require_float_dtype=*/true);
-RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(erf, /*require_float_dtype=*/true);
-
-RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(isfinite);
-RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(isinf);
-RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(isnan);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(abs, /*require_float_dtype=*/false);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(acos, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(acosh, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(asin, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(asinh, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(atan, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(atanh, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(bitwise_not, /*require_float_dtype=*/false);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(ceil, /*require_float_dtype=*/false);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(cos, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(cosh, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(exp, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(floor, /*require_float_dtype=*/false);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(log, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(logical_not, /*require_float_dtype=*/false);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(negative, /*require_float_dtype=*/false);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(round, /*require_float_dtype=*/false);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(rsqrt, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sigmoid, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sign, /*require_float_dtype=*/false);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sin, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sinh, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(square, /*require_float_dtype=*/false);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(sqrt, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(tan, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(tanh, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(erf, /*require_float_dtype=*/true);
+  RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(isfinite);
+  RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(isinf);
+  RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(isnan);
+}
 
 }  // namespace distributed
 }  // namespace relax
diff --git a/src/relax/op/distributed/unary.h b/src/relax/op/distributed/unary.h
index 6988a8adae..4e2bb8f64e 100644
--- a/src/relax/op/distributed/unary.h
+++ b/src/relax/op/distributed/unary.h
@@ -64,12 +64,15 @@ Type InferDistTypeUnaryArith(const Call& call, const BlockBuilder& ctx) {
 
 Type InferDistTypeUnaryCheck(const Call& call, const BlockBuilder& ctx);
 
+// clang-format off
 #define RELAX_REGISTER_UNARY_ARITH_DIST_INFER_TYPE(OpName, RequireFloatDtype) \
-  TVM_REGISTER_OP("relax." #OpName)                                           \
+  OpDef("relax." #OpName)                                                     \
       .set_attr<FInferType>("dist.FInferType", InferDistTypeUnaryArith<RequireFloatDtype>)
 
 #define RELAX_REGISTER_UNARY_CHECK_DIST_INFER_TYPE(OpName) \
-  TVM_REGISTER_OP("relax." #OpName).set_attr<FInferType>("dist.FInferType", InferDistTypeUnaryCheck)
+  OpDef("relax." #OpName)                              \
+      .set_attr<FInferType>("dist.FInferType", InferDistTypeUnaryCheck)
+// clang-format on
 
 }  // namespace distributed
 }  // namespace relax
diff --git a/src/relax/op/image/resize.cc b/src/relax/op/image/resize.cc
index 76d1f868ed..1476ddd74b 100644
--- a/src/relax/op/image/resize.cc
+++ b/src/relax/op/image/resize.cc
@@ -33,8 +33,10 @@
 namespace tvm {
 namespace relax {
 
-TVM_FFI_STATIC_INIT_BLOCK() { Resize2DAttrs::RegisterReflection(); }
-TVM_FFI_STATIC_INIT_BLOCK() { Resize3DAttrs::RegisterReflection(); }
+TVM_FFI_STATIC_INIT_BLOCK() {
+  Resize2DAttrs::RegisterReflection();
+  Resize3DAttrs::RegisterReflection();
+}
 
 /* relax.resize2d */
 
@@ -53,7 +55,7 @@ Expr resize2d(Expr data, Expr size, ffi::Array<FloatImm> roi, ffi::String layout
   attrs->extrapolation_value = extrapolation_value;
   attrs->out_dtype = out_dtype;
 
-  static const Op& op = Op::Get("relax.image.resize2d");
+  static const Op op = Op::Get("relax.image.resize2d");
   return Call(Type::Missing(), op, {std::move(data), std::move(size)}, Attrs(attrs), {});
 }
 
@@ -142,15 +144,16 @@ InferLayoutOutput InferLayoutResize2d(
                            Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.image.resize2d")
-    .set_attrs_type<Resize2DAttrs>()
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .add_argument("size", "Shape", "The output image shape.")
-    .set_attr<FInferType>("FInferType", InferTypeResize2D)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutResize2d)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.image.resize2d")
+      .add_arg("data", "The input tensor.")
+      .add_arg("size", "The output image shape.")
+      .attrs_type<Resize2DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeResize2D)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutResize2d)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.resize3d */
 
@@ -169,7 +172,7 @@ Expr resize3d(Expr data, Expr size, ffi::Array<FloatImm> roi, ffi::String layout
   attrs->extrapolation_value = extrapolation_value;
   attrs->out_dtype = out_dtype;
 
-  static const Op& op = Op::Get("relax.image.resize3d");
+  static const Op op = Op::Get("relax.image.resize3d");
   return Call(Type::Missing(), op, {std::move(data), std::move(size)}, Attrs(attrs), {});
 }
 
@@ -256,19 +259,20 @@ InferLayoutOutput InferLayoutResize3d(
                            Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.image.resize3d")
-    .set_attrs_type<Resize3DAttrs>()
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .add_argument("size", "Shape", "The output image shape.")
-    .set_attr<FInferType>("FInferType", InferTypeResize3D)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutResize3d)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
-
-/* relax.grid_sample */
-
-TVM_FFI_STATIC_INIT_BLOCK() { GridSampleAttrs::RegisterReflection(); }
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.image.resize3d")
+      .add_arg("data", "The input tensor.")
+      .add_arg("size", "The output image shape.")
+      .attrs_type<Resize3DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeResize3D)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutResize3d)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+
+  /* relax.grid_sample */
+
+  GridSampleAttrs::RegisterReflection();
+}
 
 Expr grid_sample(Expr data, Expr grid, ffi::String method, ffi::String layout,
                  ffi::String padding_mode, bool align_corners) {
@@ -278,7 +282,7 @@ Expr grid_sample(Expr data, Expr grid, ffi::String method, ffi::String layout,
   attrs->padding_mode = std::move(padding_mode);
   attrs->align_corners = align_corners;
 
-  static const Op& op = Op::Get("relax.image.grid_sample");
+  static const Op op = Op::Get("relax.image.grid_sample");
   return Call(Type::Missing(), op, {std::move(data), std::move(grid)}, Attrs(attrs), {});
 }
 
@@ -347,27 +351,28 @@ Type InferTypeGridSample(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr(out_shape), out_dtype, data_ty->vdevice);
 }
 
-TVM_REGISTER_OP("relax.image.grid_sample")
-    .set_attrs_type<GridSampleAttrs>()
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .add_argument("grid", "Tensor", "The grid tensor for sampling.")
-    .set_attr<FInferType>("FInferType", InferTypeGridSample)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.image.grid_sample")
+      .add_arg("data", "The input tensor.")
+      .add_arg("grid", "The grid tensor for sampling.")
+      .attrs_type<GridSampleAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeGridSample)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.image.affine_grid */
 
 Expr affine_grid(Expr data, Expr size, bool align_corners) {
   ffi::ObjectPtr<AffineGridAttrs> attrs = ffi::make_object<AffineGridAttrs>();
   attrs->align_corners = align_corners;
-  static const Op& op = Op::Get("relax.image.affine_grid");
+  static const Op op = Op::Get("relax.image.affine_grid");
   return Call(Type::Missing(), op, {std::move(data), std::move(size)}, Attrs(attrs), {});
 }
 
-TVM_FFI_STATIC_INIT_BLOCK() { AffineGridAttrs::RegisterReflection(); }
-
 TVM_FFI_STATIC_INIT_BLOCK() {
+  AffineGridAttrs::RegisterReflection();
+
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.image.affine_grid", affine_grid);
 }
@@ -446,14 +451,15 @@ Type InferTypeAffineGrid(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr(out_shape), out_dtype, data_ty->vdevice);
 }
 
-TVM_REGISTER_OP("relax.image.affine_grid")
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "The input affine matrix tensor.")
-    .add_argument("size", "Shape", "The target output shape (H, W).")
-    .set_attrs_type<AffineGridAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeAffineGrid)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.image.affine_grid")
+      .add_arg("data", "The input affine matrix tensor.")
+      .add_arg("size", "The target output shape (H, W).")
+      .attrs_type<AffineGridAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeAffineGrid)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/memory/view.cc b/src/relax/op/memory/view.cc
index 39353d2a35..5d25cf3219 100644
--- a/src/relax/op/memory/view.cc
+++ b/src/relax/op/memory/view.cc
@@ -36,7 +36,7 @@ Expr view(Expr x, ffi::Optional<Expr> shape, ffi::Optional<Expr> dtype,
           ffi::Optional<Expr> relative_byte_offset) {
   Tuple void_expr(ffi::Array<Expr>{});
 
-  static const Op& op = Op::Get("relax.memory.view");
+  static const Op op = Op::Get("relax.memory.view");
   return Call(Type::Missing(), op,
               {
                   x,
@@ -108,7 +108,7 @@ Type InferTypeView(const Call& call, const BlockBuilder& ctx) {
       }
     }
 
-    static const Op& null_value_op = Op::Get("relax.null_value");
+    static const Op null_value_op = Op::Get("relax.null_value");
     if (const CallNode* call_node = arg_value.as<CallNode>()) {
       if (call_node->op.same_as(null_value_op)) {
         // No datatype change is applied.  This is the non-void
@@ -338,7 +338,7 @@ Expr LowerBuiltinView(const BlockBuilder& bb, const Call& call) {
       }
     }
 
-    static const Op& null_value_op = Op::Get("relax.null_value");
+    static const Op null_value_op = Op::Get("relax.null_value");
     if (const CallNode* call_node = arg.as<CallNode>()) {
       return call_node->op.same_as(null_value_op);
     }
@@ -392,20 +392,21 @@ Expr LowerBuiltinView(const BlockBuilder& bb, const Call& call) {
   return Call(Type::Missing(), runtime_view_func, {data, shape, dtype, relative_byte_offset});
 }
 
-TVM_REGISTER_OP("relax.memory.view")
-    .set_num_inputs(4)
-    .add_argument("x", "Tensor", "The input tensor.")
-    .add_argument("shape", "Shape", "The view's shape.")
-    .add_argument("dtype", "DataType", "The view's data type.")
-    .add_argument("relative_byte_offset", "Prim(\"int64\")",
-                  "The view's byte offset, relative to the input tensor's byte offset.")
-    .set_attr<bool>("RequiresArgumentShapes", false)
-    .set_attr<FInferType>("FInferType", InferTypeView)
-    .set_attr<bool>("FPurity", true)
-    .set_attr<FLowerBuiltin>("FLowerBuiltin", LowerBuiltinView);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.memory.view")
+      .add_arg("x", "The input tensor.")
+      .add_arg("shape", "The view's shape.")
+      .add_arg("dtype", "The view's data type.")
+      .add_arg("relative_byte_offset",
+               "The view's byte offset, relative to the input tensor's byte offset.")
+      .set_attr<bool>("RequiresArgumentShapes", false)
+      .set_attr<FInferType>("FInferType", InferTypeView)
+      .set_attr<bool>("FPurity", true)
+      .set_attr<FLowerBuiltin>("FLowerBuiltin", LowerBuiltinView);
+}
 
 Expr ensure_zero_offset(const Expr& x) {
-  static const Op& op = Op::Get("relax.memory.ensure_zero_offset");
+  static const Op op = Op::Get("relax.memory.ensure_zero_offset");
   return Call(Type::Missing(), op, {x});
 }
 
@@ -428,13 +429,14 @@ Expr LowerBuiltinEnsureZeroOffset(const BlockBuilder& bb, const Call& call) {
   return Call(Type::Missing(), builtin_ensure_zero_offset_, call->args, Attrs(), {GetType(call)});
 }
 
-TVM_REGISTER_OP("relax.memory.ensure_zero_offset")
-    .set_num_inputs(1)
-    .add_argument("x", "Tensor", "The input tensor.")
-    .set_attr<bool>("RequiresArgumentShapes", false)
-    .set_attr<FInferType>("FInferType", InferTypeEnsureZeroOffset)
-    .set_attr<bool>("FPurity", true)
-    .set_attr<FLowerBuiltin>("FLowerBuiltin", LowerBuiltinEnsureZeroOffset);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.memory.ensure_zero_offset")
+      .add_arg("x", "The input tensor.")
+      .set_attr<bool>("RequiresArgumentShapes", false)
+      .set_attr<FInferType>("FInferType", InferTypeEnsureZeroOffset)
+      .set_attr<bool>("FPurity", true)
+      .set_attr<FLowerBuiltin>("FLowerBuiltin", LowerBuiltinEnsureZeroOffset);
+}
 
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/nn/attention.cc b/src/relax/op/nn/attention.cc
index f38a0d7a2d..b271db5659 100644
--- a/src/relax/op/nn/attention.cc
+++ b/src/relax/op/nn/attention.cc
@@ -149,45 +149,44 @@ Call InferMixedPrecisionAttention(const Call& call, DLDataType out_dtype) {
       .as_or_throw<Call>();
 }
 
-TVM_REGISTER_OP("relax.nn.attention")
-    .set_attrs_type<AttentionAttrs>()
-    .set_num_inputs(3)
-    .add_argument("query", "Tensor", "The input queries tensor.")
-    .add_argument("key", "Tensor", "The input keys tensor.")
-    .add_argument("value", "Tensor", "The input values tensor.")
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
-    .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionAttention)
-    .set_attr<FInferType>("FInferType", InferTypeAttention)
-    .set_attr<bool>("FPurity", true);
-
-TVM_REGISTER_OP("relax.nn.attention_bias")
-    .set_attrs_type<AttentionAttrs>()
-    .set_num_inputs(4)
-    .add_argument("query", "Tensor", "The input queries tensor.")
-    .add_argument("key", "Tensor", "The input keys tensor.")
-    .add_argument("value", "Tensor", "The input values tensor.")
-    .add_argument("bias", "Tensor", "The input bias tensor.")
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
-    .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionAttention)
-    .set_attr<FInferType>("FInferType", InferTypeAttention)
-    .set_attr<bool>("FPurity", true);
-
-TVM_REGISTER_OP("relax.nn.attention_var_len")
-    .set_attrs_type<AttentionAttrs>()
-    .set_num_inputs(7)
-    .add_argument("query", "Tensor", "The input queries tensor.")
-    .add_argument("key", "Tensor", "The input keys tensor.")
-    .add_argument("value", "Tensor", "The input values tensor.")
-    .add_argument("seqstart_q", "Tensor", "The cumsum of query sequence lengths, prepended with 0.")
-    .add_argument("seqstart_k", "Tensor", "The cumsum of key sequence lengths, prepended with 0.")
-    .add_argument("max_seqlen_q", "Tensor", "The maximum query sequence length in the batch.")
-    .add_argument("max_seqlen_k", "Tensor", "The maximum key sequence length in the batch.")
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
-    .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionAttention)
-    .set_attr<FInferType>("FInferType", InferTypeAttention)
-    .set_attr<bool>("FPurity", true);
-
-TVM_FFI_STATIC_INIT_BLOCK() { AttentionAttrs::RegisterReflection(); }
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.attention")
+      .add_arg("query", "The input queries tensor.")
+      .add_arg("key", "The input keys tensor.")
+      .add_arg("value", "The input values tensor.")
+      .attrs_type<AttentionAttrs>()
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+      .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionAttention)
+      .set_attr<FInferType>("FInferType", InferTypeAttention)
+      .set_attr<bool>("FPurity", true);
+
+  OpDef("relax.nn.attention_bias")
+      .add_arg("query", "The input queries tensor.")
+      .add_arg("key", "The input keys tensor.")
+      .add_arg("value", "The input values tensor.")
+      .add_arg("bias", "The input bias tensor.")
+      .attrs_type<AttentionAttrs>()
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+      .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionAttention)
+      .set_attr<FInferType>("FInferType", InferTypeAttention)
+      .set_attr<bool>("FPurity", true);
+
+  OpDef("relax.nn.attention_var_len")
+      .add_arg("query", "The input queries tensor.")
+      .add_arg("key", "The input keys tensor.")
+      .add_arg("value", "The input values tensor.")
+      .add_arg("seqstart_q", "The cumsum of query sequence lengths, prepended with 0.")
+      .add_arg("seqstart_k", "The cumsum of key sequence lengths, prepended with 0.")
+      .add_arg("max_seqlen_q", "The maximum query sequence length in the batch.")
+      .add_arg("max_seqlen_k", "The maximum key sequence length in the batch.")
+      .attrs_type<AttentionAttrs>()
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+      .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionAttention)
+      .set_attr<FInferType>("FInferType", InferTypeAttention)
+      .set_attr<bool>("FPurity", true);
+
+  AttentionAttrs::RegisterReflection();
+}
 
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/nn/convolution.cc b/src/relax/op/nn/convolution.cc
index 1927a7d5bc..e13541dc10 100644
--- a/src/relax/op/nn/convolution.cc
+++ b/src/relax/op/nn/convolution.cc
@@ -196,16 +196,18 @@ Call InferMixedPrecisionConv1d(const Call& call, DLDataType out_dtype) {
       .as_or_throw<Call>();
 }
 
-TVM_REGISTER_OP("relax.nn.conv1d")
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .add_argument("weight", "Tensor", "The weight tensor.")
-    .set_attrs_type<Conv1DAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeConv1d)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv1d)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
-    .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv1d)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.conv1d")
+      .add_arg("data", "The input tensor.")
+      .add_arg("weight", "The weight tensor.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .attrs_type<Conv1DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeConv1d)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv1d)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+      .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv1d)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.conv2d */
 
@@ -408,16 +410,18 @@ Call InferMixedPrecisionConv2d(const Call& call, DLDataType out_dtype) {
       .as_or_throw<Call>();
 }
 
-TVM_REGISTER_OP("relax.nn.conv2d")
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .add_argument("weight", "Tensor", "The weight tensor.")
-    .set_attrs_type<Conv2DAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeConv2d)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv2d)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
-    .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv2d)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.conv2d")
+      .add_arg("data", "The input tensor.")
+      .add_arg("weight", "The weight tensor.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .attrs_type<Conv2DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeConv2d)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv2d)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+      .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv2d)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.conv3d */
 
@@ -594,16 +598,18 @@ Call InferMixedPrecisionConv3d(const Call& call, DLDataType out_dtype) {
       .as_or_throw<Call>();
 }
 
-TVM_REGISTER_OP("relax.nn.conv3d")
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .add_argument("weight", "Tensor", "The weight tensor.")
-    .set_attrs_type<Conv3DAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeConv3d)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv3d)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
-    .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv3d)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.conv3d")
+      .add_arg("data", "The input tensor.")
+      .add_arg("weight", "The weight tensor.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .attrs_type<Conv3DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeConv3d)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv3d)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+      .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv3d)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr conv1d_transpose(Expr data, Expr weight, ffi::Array<int64_t> strides,
                       ffi::Array<int64_t> padding, ffi::Array<int64_t> output_padding,
@@ -636,7 +642,7 @@ Expr conv1d_transpose(Expr data, Expr weight, ffi::Array<int64_t> strides,
   attrs->kernel_layout = std::move(kernel_layout);
   attrs->out_layout = out_layout.value_or(data_layout);
   attrs->out_dtype = out_dtype;
-  const Op& op = Op::Get("relax.nn.conv1d_transpose");
+  const Op op = Op::Get("relax.nn.conv1d_transpose");
   return Call(Type::Missing(), op, {data, weight}, Attrs(attrs), {});
 }
 
@@ -774,16 +780,18 @@ Call InferMixedPrecisionConv1dTranspose(const Call& call, DLDataType out_dtype)
       .as_or_throw<Call>();
 }
 
-TVM_REGISTER_OP("relax.nn.conv1d_transpose")
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .add_argument("weight", "Tensor", "The weight tensor.")
-    .set_attrs_type<Conv1DTransposeAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeConv1dTranspose)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv1dTranspose)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
-    .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv1dTranspose)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.conv1d_transpose")
+      .add_arg("data", "The input tensor.")
+      .add_arg("weight", "The weight tensor.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .attrs_type<Conv1DTransposeAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeConv1dTranspose)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv1dTranspose)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+      .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv1dTranspose)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.conv2d_transpose */
 
@@ -827,7 +835,7 @@ Expr conv2d_transpose(Expr data, Expr weight, ffi::Array<int64_t> strides,
   attrs->kernel_layout = std::move(kernel_layout);
   attrs->out_layout = out_layout.value_or(data_layout);
   attrs->out_dtype = out_dtype;
-  const Op& op = Op::Get("relax.nn.conv2d_transpose");
+  const Op op = Op::Get("relax.nn.conv2d_transpose");
   return Call(Type::Missing(), op, {data, weight}, Attrs(attrs), {});
 }
 
@@ -1003,16 +1011,18 @@ Call InferMixedPrecisionConv2dTranspose(const Call& call, DLDataType out_dtype)
       .as_or_throw<Call>();
 }
 
-TVM_REGISTER_OP("relax.nn.conv2d_transpose")
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .add_argument("weight", "Tensor", "The weight tensor.")
-    .set_attrs_type<Conv2DTransposeAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeConv2dTranspose)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv2dTranspose)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
-    .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv2dTranspose)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.conv2d_transpose")
+      .add_arg("data", "The input tensor.")
+      .add_arg("weight", "The weight tensor.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .attrs_type<Conv2DTransposeAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeConv2dTranspose)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv2dTranspose)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+      .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv2dTranspose)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.conv3d_transpose */
 
@@ -1059,7 +1069,7 @@ Expr conv3d_transpose(Expr data, Expr weight, ffi::Array<int64_t> strides,
   attrs->kernel_layout = std::move(kernel_layout);
   attrs->out_layout = out_layout.value_or(data_layout);
   attrs->out_dtype = out_dtype;
-  const Op& op = Op::Get("relax.nn.conv3d_transpose");
+  const Op op = Op::Get("relax.nn.conv3d_transpose");
   return Call(Type::Missing(), op, {data, weight}, Attrs(attrs), {});
 }
 
@@ -1243,16 +1253,18 @@ Call InferMixedPrecisionConv3dTranspose(const Call& call, DLDataType out_dtype)
       .as_or_throw<Call>();
 }
 
-TVM_REGISTER_OP("relax.nn.conv3d_transpose")
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .add_argument("weight", "Tensor", "The weight tensor.")
-    .set_attrs_type<Conv3DTransposeAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeConv3dTranspose)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv3dTranspose)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
-    .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv3dTranspose)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.conv3d_transpose")
+      .add_arg("data", "The input tensor.")
+      .add_arg("weight", "The weight tensor.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .attrs_type<Conv3DTransposeAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeConv3dTranspose)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConv3dTranspose)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+      .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionConv3dTranspose)
+      .set_attr<bool>("FPurity", true);
+}
 
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/nn/convolution.h b/src/relax/op/nn/convolution.h
index bf3afe0f41..5b655b797a 100644
--- a/src/relax/op/nn/convolution.h
+++ b/src/relax/op/nn/convolution.h
@@ -49,7 +49,7 @@ inline Expr MakeConv(Expr data, Expr weight, ffi::Array<int64_t> strides,
   attrs->kernel_layout = std::move(kernel_layout);
   attrs->out_layout = std::move(out_layout);
   attrs->out_dtype = out_dtype;
-  const Op& op = Op::Get(op_name);
+  const Op op = Op::Get(op_name);
   return Call(Type::Missing(), op, {data, weight}, Attrs(attrs), {});
 }
 
diff --git a/src/relax/op/nn/nn.cc b/src/relax/op/nn/nn.cc
index 1da6baedd6..38d93b4200 100644
--- a/src/relax/op/nn/nn.cc
+++ b/src/relax/op/nn/nn.cc
@@ -46,40 +46,111 @@ TVM_FFI_STATIC_INIT_BLOCK() {
 }
 
 /* relax.nn.relu */
-RELAX_REGISTER_UNARY_NN_OP_AND_IMPL(relu, "nn.relu", /*require_float_dtype=*/false);
+Expr relu(Expr x) {
+  static const Op op = Op::Get("relax.nn.relu");
+  return Call(Type::Missing(), op, {std::move(x)}, Attrs(), {});
+}
 
-/* relax.nn.gelu */
-RELAX_REGISTER_UNARY_NN_OP_AND_IMPL(gelu, "nn.gelu", /*require_float_dtype=*/true);
+Expr gelu(Expr x) {
+  static const Op op = Op::Get("relax.nn.gelu");
+  return Call(Type::Missing(), op, {std::move(x)}, Attrs(), {});
+}
 
-/* relax.nn.gelu_tanh */
-RELAX_REGISTER_UNARY_NN_OP_AND_IMPL(gelu_tanh, "nn.gelu_tanh", /*require_float_dtype=*/true);
+Expr gelu_tanh(Expr x) {
+  static const Op op = Op::Get("relax.nn.gelu_tanh");
+  return Call(Type::Missing(), op, {std::move(x)}, Attrs(), {});
+}
+
+Expr selu(Expr x) {
+  static const Op op = Op::Get("relax.nn.selu");
+  return Call(Type::Missing(), op, {std::move(x)}, Attrs(), {});
+}
 
-/* relax.nn.selu */
-RELAX_REGISTER_UNARY_NN_OP_AND_IMPL(selu, "nn.selu", /*require_float_dtype=*/true);
+Expr silu(Expr x) {
+  static const Op op = Op::Get("relax.nn.silu");
+  return Call(Type::Missing(), op, {std::move(x)}, Attrs(), {});
+}
 
-/* relax.nn.silu */
-RELAX_REGISTER_UNARY_NN_OP_AND_IMPL(silu, "nn.silu", /*require_float_dtype=*/true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  tvm::ffi::reflection::GlobalDef().def("relax.op.nn.relu", relu);
+
+  OpDef("relax.nn.relu")
+      .add_arg("x", "The input tensor.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true)
+      .set_attr<FInferType>("FInferType", InferTypeUnaryArith<false>);
+
+  /* relax.nn.gelu */
+
+  tvm::ffi::reflection::GlobalDef().def("relax.op.nn.gelu", gelu);
+
+  OpDef("relax.nn.gelu")
+      .add_arg("x", "The input tensor.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true)
+      .set_attr<FInferType>("FInferType", InferTypeUnaryArith<true>);
+
+  /* relax.nn.gelu_tanh */
+
+  tvm::ffi::reflection::GlobalDef().def("relax.op.nn.gelu_tanh", gelu_tanh);
+
+  OpDef("relax.nn.gelu_tanh")
+      .add_arg("x", "The input tensor.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true)
+      .set_attr<FInferType>("FInferType", InferTypeUnaryArith<true>);
+
+  /* relax.nn.selu */
+
+  tvm::ffi::reflection::GlobalDef().def("relax.op.nn.selu", selu);
+
+  OpDef("relax.nn.selu")
+      .add_arg("x", "The input tensor.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true)
+      .set_attr<FInferType>("FInferType", InferTypeUnaryArith<true>);
+
+  /* relax.nn.silu */
+
+  tvm::ffi::reflection::GlobalDef().def("relax.op.nn.silu", silu);
+
+  OpDef("relax.nn.silu")
+      .add_arg("x", "The input tensor.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true)
+      .set_attr<FInferType>("FInferType", InferTypeUnaryArith<true>);
+}
 
 /* relax.nn.leakyrelu */
 
 Expr leakyrelu(Expr data, double alpha) {
   auto attrs = ffi::make_object<LeakyReluAttrs>();
   attrs->alpha = alpha;
-  static const Op& op = Op::Get("relax.nn.leakyrelu");
+  static const Op op = Op::Get("relax.nn.leakyrelu");
   return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.nn.leakyrelu", leakyrelu);
-}
 
-TVM_REGISTER_OP("relax.nn.leakyrelu")
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .set_attrs_type<LeakyReluAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeUnaryArith</*require_float_dtype=*/true>)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.nn.leakyrelu")
+      .add_arg("data", "The input tensor.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .attrs_type<LeakyReluAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeUnaryArith</*require_float_dtype=*/true>)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.softplus */
 
@@ -87,28 +158,28 @@ Expr softplus(Expr data, double beta, double threshold) {
   auto attrs = ffi::make_object<SoftplusAttrs>();
   attrs->beta = beta;
   attrs->threshold = threshold;
-  static const Op& op = Op::Get("relax.nn.softplus");
+  static const Op op = Op::Get("relax.nn.softplus");
   return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.nn.softplus", softplus);
-}
 
-TVM_REGISTER_OP("relax.nn.softplus")
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .set_attrs_type<SoftplusAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeUnaryArith</*require_float_dtype=*/true>)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.nn.softplus")
+      .add_arg("data", "The input tensor.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .attrs_type<SoftplusAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeUnaryArith</*require_float_dtype=*/true>)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.prelu */
 
 Expr prelu(Expr data, Expr alpha, int axis = 1) {
   auto attrs = ffi::make_object<PReluAttrs>();
   attrs->axis = axis;
-  static const Op& op = Op::Get("relax.nn.prelu");
+  static const Op op = Op::Get("relax.nn.prelu");
   return Call(Type::Missing(), op, {data, alpha}, Attrs(attrs), {});
 }
 
@@ -159,21 +230,22 @@ InferLayoutOutput InferLayoutPRelu(
   return InferLayoutOutput({layout, alpha_layout}, {layout}, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.nn.prelu")
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .add_argument("alpha", "Tensor", "The channel-wise learnable slope.")
-    .set_attrs_type<PReluAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypePRelu)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPRelu)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.prelu")
+      .add_arg("data", "The input tensor.")
+      .add_arg("alpha", "The channel-wise learnable slope.")
+      .attrs_type<PReluAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypePRelu)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPRelu)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.softmax */
 
 Expr softmax(Expr data, int axis) {
   auto attrs = ffi::make_object<SoftmaxAttrs>();
   attrs->axis = axis;
-  static const Op& op = Op::Get("relax.nn.softmax");
+  static const Op op = Op::Get("relax.nn.softmax");
   return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
 }
 
@@ -225,33 +297,33 @@ InferLayoutOutput InferLayoutSoftmax(
   return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.nn.softmax")
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .set_attrs_type<SoftmaxAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeSoftmax)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutSoftmax)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.softmax")
+      .add_arg("data", "The input tensor.")
+      .attrs_type<SoftmaxAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeSoftmax)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutSoftmax)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.log_softmax */
 Expr log_softmax(Expr data, int axis) {
   auto attrs = ffi::make_object<SoftmaxAttrs>();
   attrs->axis = axis;
-  static const Op& op = Op::Get("relax.nn.log_softmax");
+  static const Op op = Op::Get("relax.nn.log_softmax");
   return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.nn.log_softmax", log_softmax);
-}
 
-TVM_REGISTER_OP("relax.nn.log_softmax")
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .set_attrs_type<SoftmaxAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeSoftmax)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.nn.log_softmax")
+      .add_arg("data", "The input tensor.")
+      .attrs_type<SoftmaxAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeSoftmax)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.pad */
 
@@ -260,7 +332,7 @@ Expr pad(Expr data, ffi::Array<int64_t> pad_width, ffi::String pad_mode, double
   attrs->pad_width = std::move(pad_width);
   attrs->pad_mode = std::move(pad_mode);
   attrs->pad_value = pad_value;
-  static const Op& op = Op::Get("relax.nn.pad");
+  static const Op op = Op::Get("relax.nn.pad");
   return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
 }
 
@@ -293,19 +365,20 @@ Type InferTypePad(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr(out_shape), input_ty[0]->dtype);
 }
 
-TVM_REGISTER_OP("relax.nn.pad")
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .set_attrs_type<PadAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypePad)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.pad")
+      .add_arg("data", "The input tensor.")
+      .attrs_type<PadAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypePad)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.pixel_shuffle */
 
 Expr pixel_shuffle(Expr data, int upscale_factor) {
   auto attrs = ffi::make_object<PixelShuffleAttrs>();
   attrs->upscale_factor = upscale_factor;
-  static const Op& op = Op::Get("relax.nn.pixel_shuffle");
+  static const Op op = Op::Get("relax.nn.pixel_shuffle");
   return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
 }
 
@@ -365,18 +438,19 @@ Type InferTypePixelShuffle(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr(out_shape), input->dtype);
 }
 
-TVM_REGISTER_OP("relax.nn.pixel_shuffle")
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .set_attrs_type<PixelShuffleAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypePixelShuffle)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.pixel_shuffle")
+      .add_arg("data", "The input tensor.")
+      .attrs_type<PixelShuffleAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypePixelShuffle)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.batchnorm */
 bool NormCheckDtypeAndShape(const Call& call, const BlockBuilder& ctx,
                             const ffi::Array<TensorType>& input_ty, ffi::Array<int64_t> axes) {
   Op op = call->op.as_or_throw<Op>();
-  int n_input = op->arguments.size();
+  int n_input = op->args_info.size();
 
   TensorType data_ty = input_ty[0];
 
@@ -398,11 +472,11 @@ bool NormCheckDtypeAndShape(const Call& call, const BlockBuilder& ctx,
     if (input_ty[i]->dtype != data_ty->dtype) {
       TVM_FFI_VISIT_THROW(TypeError, call)
           << op << " requires all the input tensors to have the same dtype. However, the "
-          << op->arguments[i]->name << " has dtype " << input_ty[i]->dtype
+          << op->args_info[i]->name << " has dtype " << input_ty[i]->dtype
           << " which is other than the input data's dtype " << data_ty->dtype;
     } else if (input_ty[i]->ndim != n_axis) {
       TVM_FFI_VISIT_THROW(ValueError, call)
-          << op << " requires the input " << op->arguments[i]->name
+          << op << " requires the input " << op->args_info[i]->name
           << " to have as many dimensions as the length of input axes. However, the "
              "given one has ndim "
           << input_ty[i]->ndim << ", which is other than the length of axes " << n_axis;
@@ -454,7 +528,7 @@ Expr batch_norm(Expr data, Expr gamma, Expr beta, Expr moving_mean, Expr moving_
   attrs->momentum = momentum;
   attrs->training = training;
 
-  static const Op& op = Op::Get("relax.nn.batch_norm");
+  static const Op op = Op::Get("relax.nn.batch_norm");
   return Call(Type::Missing(), op,
               {std::move(data), std::move(gamma), std::move(beta), std::move(moving_mean),
                std::move(moving_var)},
@@ -512,17 +586,18 @@ InferLayoutOutput InferLayoutBatchNorm(
       {{layout, initial_layouts[3], initial_layouts[4]}}, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.nn.batch_norm")
-    .set_attrs_type<BatchNormAttrs>()
-    .set_num_inputs(5)
-    .add_argument("data", "Tensor", "Input to which batch_norm will be applied.")
-    .add_argument("gamma", "Tensor", "The gamma scale factor.")
-    .add_argument("beta", "Tensor", "The beta offset factor.")
-    .add_argument("moving_mean", "Tensor", "Running mean of input.")
-    .add_argument("moving_var", "Tensor", "Running variance of input.")
-    .set_attr<FInferType>("FInferType", InferTypeBatchNorm)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutBatchNorm)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.batch_norm")
+      .add_arg("data", "Input to which batch_norm will be applied.")
+      .add_arg("gamma", "The gamma scale factor.")
+      .add_arg("beta", "The beta offset factor.")
+      .add_arg("moving_mean", "Running mean of input.")
+      .add_arg("moving_var", "Running variance of input.")
+      .attrs_type<BatchNormAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeBatchNorm)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutBatchNorm)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.layer_norm */
 
@@ -534,7 +609,7 @@ Expr layer_norm(Expr data, Expr gamma, Expr beta, ffi::Array<int64_t> axes, doub
   attrs->center = center;
   attrs->scale = scale;
 
-  static const Op& op = Op::Get("relax.nn.layer_norm");
+  static const Op op = Op::Get("relax.nn.layer_norm");
   return Call(Type::Missing(), op, {std::move(data), std::move(gamma), std::move(beta)},
               Attrs{attrs}, {});
 }
@@ -581,16 +656,17 @@ InferLayoutOutput InferLayoutLayerNorm(
                            Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.nn.layer_norm")
-    .set_attrs_type<LayerNormAttrs>()
-    .set_num_inputs(3)
-    .add_argument("data", "Tensor", "Input to which layer_norm will be applied.")
-    .add_argument("gamma", "Tensor", "The gamma scale factor.")
-    .add_argument("beta", "Tensor", "The beta offset factor.")
-    .set_attr<FInferType>("FInferType", InferTypeLayerNorm)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutLayerNorm)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.layer_norm")
+      .add_arg("data", "Input to which layer_norm will be applied.")
+      .add_arg("gamma", "The gamma scale factor.")
+      .add_arg("beta", "The beta offset factor.")
+      .attrs_type<LayerNormAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeLayerNorm)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutLayerNorm)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.group_norm */
 
@@ -604,7 +680,7 @@ Expr group_norm(Expr data, Expr gamma, Expr beta, int num_groups, int channel_ax
   attrs->center = center;
   attrs->scale = scale;
 
-  static const Op& op = Op::Get("relax.nn.group_norm");
+  static const Op op = Op::Get("relax.nn.group_norm");
   return Call(Type::Missing(), op, {std::move(data), std::move(gamma), std::move(beta)},
               Attrs{attrs}, {});
 }
@@ -644,7 +720,7 @@ Type InferTypeGroupNorm(const Call& call, const BlockBuilder& ctx) {
         << op << " expects that the size of channel_axis must be divisible by " << attrs->num_groups
         << ", but got " << data_shape->values[channel_axis];
   }
-  for (int i = 1; i < static_cast<int>(op->arguments.size()); ++i) {
+  for (int i = 1; i < static_cast<int>(op->args_info.size()); ++i) {
     if (input_ty[i]->dtype != data_ty->dtype) {
       TVM_FFI_VISIT_THROW(TypeError, call)
           << op << " expects that all inputs must have the same dtype, but got "
@@ -695,16 +771,17 @@ InferLayoutOutput InferLayoutGroupNorm(
                            Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.nn.group_norm")
-    .set_attrs_type<GroupNormAttrs>()
-    .set_num_inputs(3)
-    .add_argument("data", "Tensor", "Input to which group_norm will be applied.")
-    .add_argument("gamma", "Tensor", "The gamma scale factor.")
-    .add_argument("beta", "Tensor", "The beta offset factor.")
-    .set_attr<FInferType>("FInferType", InferTypeGroupNorm)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutGroupNorm)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.group_norm")
+      .add_arg("data", "Input to which group_norm will be applied.")
+      .add_arg("gamma", "The gamma scale factor.")
+      .add_arg("beta", "The beta offset factor.")
+      .attrs_type<GroupNormAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeGroupNorm)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutGroupNorm)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.instance_norm */
 
@@ -717,7 +794,7 @@ Expr instance_norm(Expr data, Expr gamma, Expr beta, int channel_axis, ffi::Arra
   attrs->center = center;
   attrs->scale = scale;
 
-  static const Op& op = Op::Get("relax.nn.instance_norm");
+  static const Op op = Op::Get("relax.nn.instance_norm");
   return Call(Type::Missing(), op, {std::move(data), std::move(gamma), std::move(beta)},
               Attrs{attrs}, {});
 }
@@ -747,7 +824,7 @@ Type InferTypeInstanceNorm(const Call& call, const BlockBuilder& ctx) {
   }
   const auto* data_shape = data_ty->shape.as<ShapeExprNode>();
   sym::Analyzer analyzer = ctx->GetAnalyzer();
-  for (int i = 1; i < static_cast<int>(op->arguments.size()); ++i) {
+  for (int i = 1; i < static_cast<int>(op->args_info.size()); ++i) {
     if (input_ty[i]->dtype != data_ty->dtype) {
       TVM_FFI_VISIT_THROW(TypeError, call)
           << op << " expects that all inputs must have the same dtype, but got "
@@ -797,16 +874,17 @@ InferLayoutOutput InferLayoutInstanceNorm(
                            Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.nn.instance_norm")
-    .set_attrs_type<InstanceNormAttrs>()
-    .set_num_inputs(3)
-    .add_argument("data", "Tensor", "Input to which instance_norm will be applied.")
-    .add_argument("gamma", "Tensor", "The gamma scale factor.")
-    .add_argument("beta", "Tensor", "The beta offset factor.")
-    .set_attr<FInferType>("FInferType", InferTypeInstanceNorm)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutInstanceNorm)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.instance_norm")
+      .add_arg("data", "Input to which instance_norm will be applied.")
+      .add_arg("gamma", "The gamma scale factor.")
+      .add_arg("beta", "The beta offset factor.")
+      .attrs_type<InstanceNormAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeInstanceNorm)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutInstanceNorm)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 /* relax.nn.rms_norm */
 
 Expr rms_norm(Expr data, Expr weight, ffi::Array<int64_t> axes, double epsilon) {
@@ -814,7 +892,7 @@ Expr rms_norm(Expr data, Expr weight, ffi::Array<int64_t> axes, double epsilon)
   attrs->axes = std::move(axes);
   attrs->epsilon = epsilon;
 
-  static const Op& op = Op::Get("relax.nn.rms_norm");
+  static const Op op = Op::Get("relax.nn.rms_norm");
   return Call(Type::Missing(), op, {std::move(data), std::move(weight)}, Attrs{attrs}, {});
 }
 
@@ -857,15 +935,16 @@ InferLayoutOutput InferLayoutRMSNorm(
   return InferLayoutOutput({layout, initial_layouts[1]}, {layout}, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.nn.rms_norm")
-    .set_attrs_type<RMSNormAttrs>()
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "Input to which rms_norm will be applied.")
-    .add_argument("weight", "Tensor", "The scale factor.")
-    .set_attr<FInferType>("FInferType", InferTypeRMSNorm)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutRMSNorm)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.rms_norm")
+      .add_arg("data", "Input to which rms_norm will be applied.")
+      .add_arg("weight", "The scale factor.")
+      .attrs_type<RMSNormAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeRMSNorm)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutRMSNorm)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.dropout */
 
@@ -873,7 +952,7 @@ Expr dropout(Expr data, double rate) {
   ffi::ObjectPtr<DropoutAttrs> attrs = ffi::make_object<DropoutAttrs>();
   attrs->rate = rate;
 
-  static const Op& op = Op::Get("relax.nn.dropout");
+  static const Op op = Op::Get("relax.nn.dropout");
   return Call(Type::Missing(), op, {std::move(data)}, Attrs{attrs}, {});
 }
 
@@ -887,14 +966,15 @@ Type InferTypeDropout(const Call& call, const BlockBuilder& ctx) {
   return TupleType({data_ty, data_ty});
 }
 
-TVM_REGISTER_OP("relax.nn.dropout")
-    .set_attrs_type<DropoutAttrs>()
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "Input to which dropout will be applied.")
-    .set_attr<FInferType>("FInferType", InferTypeDropout)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.dropout")
+      .add_arg("data", "Input to which dropout will be applied.")
+      .attrs_type<DropoutAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeDropout)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.cross_entropy_with_logits */
 Type InferTypeCrossEntropy(const Call& call, const BlockBuilder& ctx) {
@@ -942,21 +1022,21 @@ Type InferTypeCrossEntropy(const Call& call, const BlockBuilder& ctx) {
 }
 
 Expr cross_entropy_with_logits(Expr predictions, Expr labels) {
-  static const Op& op = Op::Get("relax.nn.cross_entropy_with_logits");
+  static const Op op = Op::Get("relax.nn.cross_entropy_with_logits");
   return Call(Type::Missing(), op, {std::move(predictions), std::move(labels)}, {}, {});
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.nn.cross_entropy_with_logits", cross_entropy_with_logits);
-}
 
-TVM_REGISTER_OP("relax.nn.cross_entropy_with_logits")
-    .set_num_inputs(2)
-    .add_argument("predictions", "Tensor", "The predictions.")
-    .add_argument("labels", "Tensor", "The labels.")
-    .set_attr<FInferType>("FInferType", InferTypeCrossEntropy)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.nn.cross_entropy_with_logits")
+      .add_arg("predictions", "The predictions.")
+      .add_arg("labels", "The labels.")
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .set_attr<FInferType>("FInferType", InferTypeCrossEntropy)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.nll_loss */
 
@@ -972,7 +1052,7 @@ Expr nll_loss(Expr predictions, Expr targets, ffi::Optional<Expr> weights, ffi::
   attrs->reduction = std::move(reduction);
   attrs->ignore_index = ignore_index;
 
-  static const Op& op = Op::Get("relax.nn.nll_loss");
+  static const Op op = Op::Get("relax.nn.nll_loss");
   if (weights.has_value()) {
     return Call(Type::Missing(), op, {std::move(predictions), std::move(targets), weights.value()},
                 Attrs{attrs}, {});
@@ -1177,19 +1257,21 @@ Type InferTypeNLLLoss(const Call& call, const BlockBuilder& ctx) {
   }
 }
 
-TVM_REGISTER_OP("relax.nn.nll_loss")
-    .set_attrs_type<NLLLossAttrs>()
-    .set_num_inputs(3)
-    .add_argument("predictions", "Tensor", "The prediction tensor.")
-    .add_argument("targets", "Tensor", "The target tensor.")
-    .add_argument("weights", "ffi::Optional<Tensor>", "The weight of each target values.")
-    .set_attr<FInferType>("FInferType", InferTypeNLLLoss)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.nll_loss", "Optional weights: The weight of each target values.")
+      .add_arg("predictions", "The prediction tensor.")
+      .add_arg("targets", "The target tensor.")
+      .allow_extra_args()
+      .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.")
+      .attrs_type<NLLLossAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeNLLLoss)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.batch_flatten */
 
 Expr batch_flatten(Expr data) {
-  static const Op& op = Op::Get("relax.nn.batch_flatten");
+  static const Op op = Op::Get("relax.nn.batch_flatten");
   return Call(Type::Missing(), op, {std::move(data)}, {}, {});
 }
 
@@ -1229,12 +1311,13 @@ Type InferTypeBatchFlatten(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr({batch_dim, flat_dim}), data_ty->dtype, data_ty->vdevice);
 }
 
-TVM_REGISTER_OP("relax.nn.batch_flatten")
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeBatchFlatten)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.batch_flatten")
+      .add_arg("data", "The input tensor.")
+      .set_attr<FInferType>("FInferType", InferTypeBatchFlatten)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/nn/nn.h b/src/relax/op/nn/nn.h
index 068a1462e9..fd36277512 100644
--- a/src/relax/op/nn/nn.h
+++ b/src/relax/op/nn/nn.h
@@ -32,19 +32,6 @@
 namespace tvm {
 namespace relax {
 
-/*!
- * \brief Quick helper macro to
- * - expose a make-function interface which construct the call node.
- * - register op to the registry.
- * \param OpName The name of operator to register.
- * \param OpRegName The identifier of the operator in the registry.
- * \param RequireFloatDtype A boolean indicating if the input is required to have float dtype.
- */
-#define RELAX_REGISTER_UNARY_NN_OP_AND_IMPL(OpName, OpRegName, RequireFloatDtype) \
-  RELAX_UNARY_OP_INTERFACE(OpName, OpRegName)                                     \
-  RELAX_REGISTER_UNARY_OP(OpRegName).set_attr<FInferType>("FInferType",           \
-                                                          InferTypeUnaryArith<RequireFloatDtype>)
-
 /*! \brief Rectified linear unit. */
 Expr relu(Expr data);
 
diff --git a/src/relax/op/nn/pooling.cc b/src/relax/op/nn/pooling.cc
index 50a45d53d0..45f5be6b06 100644
--- a/src/relax/op/nn/pooling.cc
+++ b/src/relax/op/nn/pooling.cc
@@ -64,7 +64,7 @@ Expr MakePool1d(ffi::String op_name, Expr data, ffi::Array<int64_t> pool_size,
   attrs->count_include_pad = count_include_pad;
   attrs->layout = layout;
   attrs->out_layout = out_layout.value_or(layout);
-  const Op& op = Op::Get(op_name);
+  const Op op = Op::Get(op_name);
   return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
 }
 
@@ -144,14 +144,15 @@ InferLayoutOutput InferLayoutPool1d(
   return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.nn.max_pool1d")
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor")
-    .set_attrs_type<Pool1DAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypePool1D)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool1d)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.max_pool1d")
+      .add_arg("data", "The input tensor")
+      .attrs_type<Pool1DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypePool1D)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool1d)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.max_pool2d */
 
@@ -188,7 +189,7 @@ Expr MakePool2d(ffi::String op_name, Expr data, ffi::Array<int64_t> pool_size,
   attrs->count_include_pad = count_include_pad;
   attrs->layout = layout;
   attrs->out_layout = out_layout.value_or(layout);
-  const Op& op = Op::Get(op_name);
+  const Op op = Op::Get(op_name);
   return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
 }
 
@@ -298,14 +299,15 @@ InferLayoutOutput InferLayoutPool2d(
   return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.nn.max_pool2d")
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor")
-    .set_attrs_type<Pool2DAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypePool2D)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool2d)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.max_pool2d")
+      .add_arg("data", "The input tensor")
+      .attrs_type<Pool2DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypePool2D)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool2d)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.max_pool3d */
 
@@ -345,7 +347,7 @@ Expr MakePool3d(ffi::String op_name, Expr data, ffi::Array<int64_t> pool_size,
   attrs->count_include_pad = count_include_pad;
   attrs->layout = layout;
   attrs->out_layout = out_layout.value_or(layout);
-  const Op& op = Op::Get(op_name);
+  const Op op = Op::Get(op_name);
   return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
 }
 
@@ -447,14 +449,15 @@ InferLayoutOutput InferLayoutPool3d(
   return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.nn.max_pool3d")
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor")
-    .set_attrs_type<Pool3DAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypePool3D)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool3d)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.max_pool3d")
+      .add_arg("data", "The input tensor")
+      .attrs_type<Pool3DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypePool3D)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool3d)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.avg_pool1d */
 Expr avg_pool1d(Expr data, ffi::Array<int64_t> pool_size, ffi::Array<int64_t> strides,
@@ -467,16 +470,15 @@ Expr avg_pool1d(Expr data, ffi::Array<int64_t> pool_size, ffi::Array<int64_t> st
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.nn.avg_pool1d", avg_pool1d);
-}
 
-TVM_REGISTER_OP("relax.nn.avg_pool1d")
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor")
-    .set_attrs_type<Pool1DAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypePool1D)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool1d)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.nn.avg_pool1d")
+      .add_arg("data", "The input tensor")
+      .attrs_type<Pool1DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypePool1D)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool1d)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.avg_pool2d */
 Expr avg_pool2d(Expr data, ffi::Array<int64_t> pool_size, ffi::Array<int64_t> strides,
@@ -489,16 +491,15 @@ Expr avg_pool2d(Expr data, ffi::Array<int64_t> pool_size, ffi::Array<int64_t> st
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.nn.avg_pool2d", avg_pool2d);
-}
 
-TVM_REGISTER_OP("relax.nn.avg_pool2d")
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor")
-    .set_attrs_type<Pool2DAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypePool2D)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool2d)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.nn.avg_pool2d")
+      .add_arg("data", "The input tensor")
+      .attrs_type<Pool2DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypePool2D)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool2d)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.avg_pool3d */
 Expr avg_pool3d(Expr data, ffi::Array<int64_t> pool_size, ffi::Array<int64_t> strides,
@@ -511,16 +512,15 @@ Expr avg_pool3d(Expr data, ffi::Array<int64_t> pool_size, ffi::Array<int64_t> st
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.nn.avg_pool3d", avg_pool3d);
-}
 
-TVM_REGISTER_OP("relax.nn.avg_pool3d")
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor")
-    .set_attrs_type<Pool3DAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypePool3D)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool3d)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.nn.avg_pool3d")
+      .add_arg("data", "The input tensor")
+      .attrs_type<Pool3DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypePool3D)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPool3d)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.adaptive_avg_pool1d */
 
@@ -537,7 +537,7 @@ Expr adaptive_avg_pool1d(Expr data, ffi::Optional<ffi::Array<int64_t>> output_si
     attrs->output_size = std::move(_output_size);
   }
 
-  static const Op& op = Op::Get("relax.nn.adaptive_avg_pool1d");
+  static const Op op = Op::Get("relax.nn.adaptive_avg_pool1d");
   return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
 }
 
@@ -595,14 +595,15 @@ InferLayoutOutput InferLayoutAdaptiveAvgPool1D(
   return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.nn.adaptive_avg_pool1d")
-    .set_attrs_type<AdaptivePool1DAttrs>()
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor")
-    .set_attr<FInferType>("FInferType", InferTypeAdaptiveAvgPool1D)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutAdaptiveAvgPool1D)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.adaptive_avg_pool1d")
+      .add_arg("data", "The input tensor")
+      .attrs_type<AdaptivePool1DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeAdaptiveAvgPool1D)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutAdaptiveAvgPool1D)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.adaptive_avg_pool2d */
 
@@ -622,7 +623,7 @@ Expr adaptive_avg_pool2d(Expr data, ffi::Optional<ffi::Array<int64_t>> output_si
     attrs->output_size = std::move(_output_size);
   }
 
-  static const Op& op = Op::Get("relax.nn.adaptive_avg_pool2d");
+  static const Op op = Op::Get("relax.nn.adaptive_avg_pool2d");
   return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
 }
 
@@ -698,14 +699,15 @@ InferLayoutOutput InferLayoutAdaptiveAvgPool2D(
   return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.nn.adaptive_avg_pool2d")
-    .set_attrs_type<AdaptivePool2DAttrs>()
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor")
-    .set_attr<FInferType>("FInferType", InferTypeAdaptiveAvgPool2D)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutAdaptiveAvgPool2D)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.adaptive_avg_pool2d")
+      .add_arg("data", "The input tensor")
+      .attrs_type<AdaptivePool2DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeAdaptiveAvgPool2D)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutAdaptiveAvgPool2D)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.nn.adaptive_avg_pool3d */
 
@@ -725,7 +727,7 @@ Expr adaptive_avg_pool3d(Expr data, ffi::Optional<ffi::Array<int64_t>> output_si
     attrs->output_size = std::move(_output_size);
   }
 
-  static const Op& op = Op::Get("relax.nn.adaptive_avg_pool3d");
+  static const Op op = Op::Get("relax.nn.adaptive_avg_pool3d");
   return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
 }
 
@@ -785,14 +787,15 @@ InferLayoutOutput InferLayoutAdaptiveAvgPool3D(
   return InferLayoutOutput({layout}, {layout}, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.nn.adaptive_avg_pool3d")
-    .set_attrs_type<AdaptivePool3DAttrs>()
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor")
-    .set_attr<FInferType>("FInferType", InferTypeAdaptiveAvgPool3D)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutAdaptiveAvgPool3D)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.nn.adaptive_avg_pool3d")
+      .add_arg("data", "The input tensor")
+      .attrs_type<AdaptivePool3DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeAdaptiveAvgPool3D)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutAdaptiveAvgPool3D)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/op.cc b/src/relax/op/op.cc
index 4c344c18ca..ed913eee58 100644
--- a/src/relax/op/op.cc
+++ b/src/relax/op/op.cc
@@ -113,17 +113,20 @@ Type InferTypeCallPurePacked(const Call& call, const BlockBuilder& ctx) {
   }
 }
 
-TVM_REGISTER_OP("relax.call_pure_packed")
-    .set_num_inputs(-1)
-    .add_argument("args", "ffi::Array<Expr>",
-                  "The first argument is the function being called. The rest are the "
-                  "arguments to that function.")
-    .set_attr<FInferType>("FInferType", InferTypeCallPurePacked)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.call_pure_packed")
+      .add_arg("func",
+               "The first argument is the function being called. The rest are the "
+               "arguments to that function.")
+      .allow_extra_args()
+      .add_ty_arg("type_args", "Optional type arguments forwarded to the callee.")
+      .set_attr<FInferType>("FInferType", InferTypeCallPurePacked)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeCallPurePacked(const Expr& callee, ffi::Array<Expr> args, const Attrs& attrs,
                         ffi::Array<Type> ty_args) {
-  static const Op& op = Op::Get("relax.call_pure_packed");
+  static const Op op = Op::Get("relax.call_pure_packed");
   ffi::Array<Expr> call_args = {callee};
   for (auto arg : args) {
     call_args.push_back(arg);
@@ -224,25 +227,28 @@ Type InferTypeCallInplacePacked(const Call& call, const BlockBuilder& ctx) {
   return ret;
 }
 
-TVM_REGISTER_OP("relax.call_inplace_packed")
-    .set_num_inputs(-1)
-    .set_attrs_type<CallInplacePackedAttrs>()
-    .add_argument("args", "ffi::Array<Expr>",
-                  "The first argument is the function being called. The rest are the "
-                  "arguments to that function.")
-    .set_attr<FInferType>("FInferType", InferTypeCallInplacePacked)
-    // Warning: considered pure, but it has the potential to create visible effects!
-    // This should only be used if it has been *checked* that it is safe (no aliases, in-place
-    // arguments will no longer be live) and the user believes the packed func to have no
-    // side effects other than modifying the arguments specified as "inplace"
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.call_inplace_packed")
+      .add_arg("func",
+               "The first argument is the function being called. The rest are the "
+               "arguments to that function.")
+      .allow_extra_args()
+      .add_ty_arg("type_args", "Optional type arguments forwarded to the callee.")
+      .attrs_type<CallInplacePackedAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeCallInplacePacked)
+      // Warning: considered pure, but it has the potential to create visible effects!
+      // This should only be used if it has been *checked* that it is safe (no aliases, in-place
+      // arguments will no longer be live) and the user believes the packed func to have no
+      // side effects other than modifying the arguments specified as "inplace"
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeCallInplacePacked(Expr func, ffi::Array<Expr> args, ffi::Array<int64_t> inplace_indices,
                            ffi::Array<Type> ty_args) {
   ffi::ObjectPtr<CallInplacePackedAttrs> attrs = ffi::make_object<CallInplacePackedAttrs>();
   attrs->inplace_indices = ffi::Array<int64_t>(inplace_indices.begin(), inplace_indices.end());
 
-  static const Op& op = Op::Get("relax.call_inplace_packed");
+  static const Op op = Op::Get("relax.call_inplace_packed");
   ffi::Array<Expr> call_args = {func};
   call_args.insert(call_args.end(), args.begin(), args.end());
   return Call(Type::Missing(), op, call_args, Attrs(attrs), ty_args);
@@ -511,14 +517,16 @@ void ValidateCallTIR(Call call) {
   }
 }
 
-TVM_REGISTER_OP("relax.call_tir")
-    .set_num_inputs(2)
-    .add_argument("func", "Expr", "The destination-passing-style function.")
-    .add_argument("args", "Tuple", "The input arguments.")
-    .set_attr<FInferType>("FInferType", InferTypeCallTIR)
-    .set_attr<FNormalize>("FNormalize", NormalizeCallTIR)
-    .set_attr<FValidate>("FValidate", ValidateCallTIR)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.call_tir")
+      .add_arg("func", "The destination-passing-style function.")
+      .add_arg("args", "The input arguments.")
+      .add_ty_arg("out_type", "The output type.")
+      .set_attr<FInferType>("FInferType", InferTypeCallTIR)
+      .set_attr<FNormalize>("FNormalize", NormalizeCallTIR)
+      .set_attr<FValidate>("FValidate", ValidateCallTIR)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeCallTIR(Expr func, Tuple args, ffi::Array<TensorType> out_ty_list) {
   for (const TensorType& ty : out_ty_list) {
@@ -536,26 +544,26 @@ Expr MakeCallTIR(Expr func, Tuple args, ffi::Array<TensorType> out_ty_list) {
     out_ty = TupleType({out_ty_list.begin(), out_ty_list.end()});
   }
 
-  static const Op& op = Op::Get("relax.call_tir");
+  static const Op op = Op::Get("relax.call_tir");
   return Call(Type::Missing(), op, {func, args}, {}, {out_ty});
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.call_tir", MakeCallTIR);
-}
 
-// call_tir_with_grad
+  // call_tir_with_grad
 
-TVM_REGISTER_OP("relax.call_tir_with_grad")
-    .set_num_inputs(2)
-    .set_attrs_type<CallTIRWithGradAttrs>()
-    .add_argument("func", "Expr", "The destination-passing-style function.")
-    .add_argument("args", "Tuple", "The input arguments.")
-    .set_attr<FInferType>("FInferType", InferTypeCallTIR)
-    .set_attr<FNormalize>("FNormalize", NormalizeCallTIR)
-    .set_attr<FValidate>("FValidate", ValidateCallTIR)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.call_tir_with_grad")
+      .add_arg("func", "The destination-passing-style function.")
+      .add_arg("args", "The input arguments.")
+      .add_ty_arg("out_type", "The output type.")
+      .attrs_type<CallTIRWithGradAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeCallTIR)
+      .set_attr<FNormalize>("FNormalize", NormalizeCallTIR)
+      .set_attr<FValidate>("FValidate", ValidateCallTIR)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeCallTIRWithGrad(Expr func, Tuple args, ffi::Array<TensorType> out_ty_list,
                          ffi::String te_grad_name, ffi::Map<ffi::String, ffi::Any> te_grad_kwargs) {
@@ -578,7 +586,7 @@ Expr MakeCallTIRWithGrad(Expr func, Tuple args, ffi::Array<TensorType> out_ty_li
   attrs->te_grad_name = te_grad_name;
   attrs->te_grad_kwargs = te_grad_kwargs;
 
-  static const Op& op = Op::Get("relax.call_tir_with_grad");
+  static const Op op = Op::Get("relax.call_tir_with_grad");
   return Call(Type::Missing(), op, {func, args}, Attrs(attrs), {out_ty});
 }
 
@@ -676,18 +684,20 @@ Expr NormalizeCallTIRInPlace(const BlockBuilder& ctx, Call call) {
   return call;
 }
 
-TVM_REGISTER_OP("relax.call_tir_inplace")
-    .set_num_inputs(2)
-    .set_attrs_type<CallTIRInplaceAttrs>()
-    .add_argument("func", "Expr", "The destination-passing-style function.")
-    .add_argument("args", "Tuple", "The input arguments.")
-    .set_attr<FInferType>("FInferType", InferTypeCallTIR)
-    .set_attr<FNormalize>("FNormalize", NormalizeCallTIRInPlace)
-    .set_attr<FValidate>("FValidate", ValidateCallTIR)
-    // Warning: considered pure, but it has the potential to create visible effects!
-    // This should only be used if it has been *checked* that it is safe (no aliases, in-place
-    // arguments will no longer be live)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.call_tir_inplace")
+      .add_arg("func", "The destination-passing-style function.")
+      .add_arg("args", "The input arguments.")
+      .add_ty_arg("out_type", "The output type.")
+      .attrs_type<CallTIRInplaceAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeCallTIR)
+      .set_attr<FNormalize>("FNormalize", NormalizeCallTIRInPlace)
+      .set_attr<FValidate>("FValidate", ValidateCallTIR)
+      // Warning: considered pure, but it has the potential to create visible effects!
+      // This should only be used if it has been *checked* that it is safe (no aliases, in-place
+      // arguments will no longer be live)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeCallTIRInplace(Expr func, Tuple args, ffi::Array<int64_t> inplace_indices,
                         ffi::Array<TensorType> out_ty_list) {
@@ -709,7 +719,7 @@ Expr MakeCallTIRInplace(Expr func, Tuple args, ffi::Array<int64_t> inplace_indic
     out_ty = TupleType({out_ty_list.begin(), out_ty_list.end()});
   }
 
-  static const Op& op = Op::Get("relax.call_tir_inplace");
+  static const Op op = Op::Get("relax.call_tir_inplace");
   return Call(Type::Missing(), op, {func, args}, Attrs(attrs), {out_ty});
 }
 
@@ -727,14 +737,16 @@ Type InferTypeCallDPSPacked(const Call& call, const BlockBuilder& ctx) {
   return call->ty_args[0];
 }
 
-TVM_REGISTER_OP("relax.call_dps_packed")
-    .set_num_inputs(2)
-    .add_argument("func", "Expr", "The destination-passing-style function.")
-    .add_argument("args", "Tuple", "The input arguments.")
-    .set_attr<FInferType>("FInferType", InferTypeCallDPSPacked)
-    // technically, an impure op could be used with this, but there is
-    // little reason to use DPS with an impure op
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.call_dps_packed")
+      .add_arg("func", "The destination-passing-style function.")
+      .add_arg("args", "The input arguments.")
+      .add_ty_arg("out_type", "The output type.")
+      .set_attr<FInferType>("FInferType", InferTypeCallDPSPacked)
+      // technically, an impure op could be used with this, but there is
+      // little reason to use DPS with an impure op
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeCallDPSPacked(Expr func, Tuple args, ffi::Array<TensorType> out_ty_list) {
   for (const TensorType& ty : out_ty_list) {
@@ -752,7 +764,7 @@ Expr MakeCallDPSPacked(Expr func, Tuple args, ffi::Array<TensorType> out_ty_list
     out_ty = TupleType({out_ty_list.begin(), out_ty_list.end()});
   }
 
-  static const Op& op = Op::Get("relax.call_dps_packed");
+  static const Op op = Op::Get("relax.call_dps_packed");
   return Call(Type::Missing(), op, {func, args}, {}, {out_ty});
 }
 
@@ -791,13 +803,15 @@ void ValidateCallPyFunc(Call call) {
       << "nor a variable binding that may be normalized to an in-line tuple.";
 }
 
-TVM_REGISTER_OP("relax.call_py_func")
-    .set_num_inputs(2)
-    .add_argument("func_name", "StringImm", "The name of the Python function to call.")
-    .add_argument("args", "Tuple", "The input arguments.")
-    .set_attr<FInferType>("FInferType", InferTypeCallPyFunc)
-    .set_attr<FValidate>("FValidate", ValidateCallPyFunc)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.call_py_func")
+      .add_arg("func_name", "The name of the Python function to call.")
+      .add_arg("args", "The input arguments.")
+      .add_ty_arg("out_type", "The output type.")
+      .set_attr<FInferType>("FInferType", InferTypeCallPyFunc)
+      .set_attr<FValidate>("FValidate", ValidateCallPyFunc)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeCallPyFunc(StringImm func_name, Tuple args, ffi::Array<TensorType> out_ty_list) {
   for (const TensorType& ty : out_ty_list) {
@@ -815,7 +829,7 @@ Expr MakeCallPyFunc(StringImm func_name, Tuple args, ffi::Array<TensorType> out_
     out_ty = TupleType({out_ty_list.begin(), out_ty_list.end()});
   }
 
-  static const Op& op = Op::Get("relax.call_py_func");
+  static const Op op = Op::Get("relax.call_py_func");
   return Call(Type::Missing(), op, {func_name, args}, {}, {out_ty});
 }
 
@@ -835,49 +849,50 @@ Type InferTypeCallBuiltinWithCtx(const Call& call, const BlockBuilder& ctx) {
   }
 }
 
-TVM_REGISTER_OP("relax.call_builtin_with_ctx")
-    .set_num_inputs(4)
-    .add_argument("func", "Expr", "The builtin packed func.")
-    .add_argument("args", "Tuple", "The input arguments.")
-    .set_attr<FInferType>("FInferType", InferTypeCallBuiltinWithCtx)
-    // Most builtins are pure, but some are not, like `vm.builtin.attention_kv_cache_append`
-    .set_attr<bool>("FPurity", false);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.call_builtin_with_ctx")
+      .add_arg("func", "The builtin packed func.")
+      .add_arg("args", "The input arguments.")
+      .add_ty_arg("out_type", "Optional output type; omitted for void.")
+      .set_attr<FInferType>("FInferType", InferTypeCallBuiltinWithCtx)
+      // Most builtins are pure, but some are not, like `vm.builtin.attention_kv_cache_append`
+      .set_attr<bool>("FPurity", false);
+}
 
 Expr MakeCallBuiltinWithCtx(Expr func, Tuple args, ffi::Array<Type> ty_args) {
-  static const Op& op = Op::Get("relax.call_builtin_with_ctx");
+  static const Op op = Op::Get("relax.call_builtin_with_ctx");
   return Call(Type::Missing(), op, {func, args}, Attrs(), ty_args);
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.call_builtin_with_ctx", MakeCallBuiltinWithCtx);
-}
 
-TVM_REGISTER_OP("relax.null_value")
-    .set_num_inputs(0)
-    .set_attr<FInferType>("FInferType", ReturnAnyType)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.null_value")
+      .set_attr<FInferType>("FInferType", ReturnAnyType)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeCallNullValue() {
-  static const Op& op = Op::Get("relax.null_value");
+  static const Op op = Op::Get("relax.null_value");
   return Call(Type::Missing(), op, {}, {}, {});
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.null_value", MakeCallNullValue);
-}
 
-// print
+  // print
 
-TVM_REGISTER_OP("relax.print")
-    .set_num_inputs(-1)
-    .add_argument("vals", "ffi::Array<Expr>",
-                  "The first value is Python-style format string to use to print. The others "
-                  "are values to print")
-    .set_attr<FInferType>("FInferType", ReturnVoidType)
-    .set_attr<FCallPacked>("FCallPacked", "relax.run.print")
-    .set_attr<bool>("FPurity", false);
+  OpDef("relax.print")
+      .add_arg("format",
+               "The first value is Python-style format string to use to print. The others "
+               "are values to print")
+      .allow_extra_args()
+      .set_attr<FInferType>("FInferType", ReturnVoidType)
+      .set_attr<FCallPacked>("FCallPacked", "relax.run.print")
+      .set_attr<bool>("FPurity", false);
+}
 
 Expr MakePrint(ffi::Array<Expr> vals, StringImm format) {
   ffi::Array<Expr> params;
@@ -885,7 +900,7 @@ Expr MakePrint(ffi::Array<Expr> vals, StringImm format) {
   for (const auto val : vals) {
     params.push_back(val);
   }
-  static const Op& op = Op::Get("relax.print");
+  static const Op op = Op::Get("relax.print");
   return Call(Type::Missing(), op, params);
 }
 
@@ -914,18 +929,20 @@ Type InferAssertType(const Call& call, const BlockBuilder& ctx) {
   return ReturnVoidType(call, ctx);
 }
 
-TVM_REGISTER_OP("relax.assert_op")
-    .set_num_inputs(-1)
-    .add_argument("vals", "ffi::Array<Expr>",
-                  "The first value is used as the assertion condition. The second value is "
-                  "Python-style format string to use for displaying an error message, if the "
-                  "assert fails. The others are used as format arguments if there is an error.")
-    .set_attr<FInferType>("FInferType", InferAssertType)
-    .set_attr<FCallPacked>("FCallPacked", "relax.run.assert_op")
-    .set_attr<bool>("FPurity", false);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.assert_op")
+      .add_arg("condition",
+               "The first value is used as the assertion condition. The second value is "
+               "Python-style format string to use for displaying an error message, if the "
+               "assert fails. The others are used as format arguments if there is an error.")
+      .allow_extra_args()
+      .set_attr<FInferType>("FInferType", InferAssertType)
+      .set_attr<FCallPacked>("FCallPacked", "relax.run.assert_op")
+      .set_attr<bool>("FPurity", false);
+}
 
 Expr MakeAssertOp(Expr condition, ffi::Array<Expr> vals, StringImm format) {
-  static const Op& op = Op::Get("relax.assert_op");
+  static const Op op = Op::Get("relax.assert_op");
   ffi::Array<Expr> args = {condition};
   args.push_back(format);
   for (auto val : vals) {
@@ -937,19 +954,18 @@ Expr MakeAssertOp(Expr condition, ffi::Array<Expr> vals, StringImm format) {
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.assert_op", MakeAssertOp);
-}
 
-// make_closure
+  // make_closure
 
-TVM_REGISTER_OP("relax.make_closure")
-    .set_num_inputs(2)
-    .add_argument("func", "Expr", "The closure.")
-    .add_argument("args", "Tuple", "The captured variables.")
-    .set_attr<FInferType>("FInferType", ReturnAnyType)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.make_closure")
+      .add_arg("func", "The closure.")
+      .add_arg("args", "The captured variables.")
+      .set_attr<FInferType>("FInferType", ReturnAnyType)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeClosure(Expr func, Tuple args) {
-  static const Op& op = Op::Get("relax.make_closure");
+  static const Op op = Op::Get("relax.make_closure");
   return Call(Type::Missing(), op, {func, args}, {}, {});
 }
 
@@ -970,53 +986,54 @@ Type InferTypeInvokeClosure(const Call& call, const BlockBuilder& ctx) {
   }
 }
 
-TVM_REGISTER_OP("relax.invoke_closure")
-    .set_num_inputs(2)
-    .add_argument("closure", "Expr", "The VMClosure.")
-    .add_argument("args", "Tuple", "The captured variables.")
-    .set_attr<FInferType>("FInferType", InferTypeInvokeClosure)
-    // Not all closures are pure. Use invoke_pure_closure for specifying purity
-    .set_attr<bool>("FPurity", false);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.invoke_closure")
+      .add_arg("closure", "The VMClosure.")
+      .add_arg("args", "The captured variables.")
+      .add_ty_arg("out_types", "Zero or more output types; multiple entries form a tuple.")
+      .set_attr<FInferType>("FInferType", InferTypeInvokeClosure)
+      // Not all closures are pure. Use invoke_pure_closure for specifying purity
+      .set_attr<bool>("FPurity", false);
+}
 
 Expr InvokeClosure(Expr closure, Tuple args, ffi::Array<Type> ty_args) {
-  static const Op& op = Op::Get("relax.invoke_closure");
+  static const Op op = Op::Get("relax.invoke_closure");
   return Call(Type::Missing(), op, {closure, args}, {}, ty_args);
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.invoke_closure", InvokeClosure);
-}
 
-// invoke_pure_closure
+  // invoke_pure_closure
 
-TVM_REGISTER_OP("relax.invoke_pure_closure")
-    .set_num_inputs(2)
-    .add_argument("closure", "Expr", "The VMClosure.")
-    .add_argument("args", "Tuple", "The captured variables.")
-    .set_attr<FInferType>("FInferType", InferTypeInvokeClosure)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.invoke_pure_closure")
+      .add_arg("closure", "The VMClosure.")
+      .add_arg("args", "The captured variables.")
+      .add_ty_arg("out_types", "Zero or more output types; multiple entries form a tuple.")
+      .set_attr<FInferType>("FInferType", InferTypeInvokeClosure)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr InvokePureClosure(Expr closure, Tuple args, ffi::Array<Type> ty_args) {
-  static const Op& op = Op::Get("relax.invoke_pure_closure");
+  static const Op op = Op::Get("relax.invoke_pure_closure");
   return Call(Type::Missing(), op, {closure, args}, {}, ty_args);
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.invoke_pure_closure", InvokePureClosure);
-}
 
-// shape_of
+  // shape_of
 
-TVM_REGISTER_OP("relax.shape_of")
-    .set_num_inputs(1)
-    .add_argument("input", "Expr", "The input expression")
-    .set_attr<FInferType>("FInferType", InferTypeShapeOf)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.shape_of")
+      .add_arg("input", "The input expression")
+      .set_attr<FInferType>("FInferType", InferTypeShapeOf)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeShapeOf(Expr expr) {
-  static const Op& op = Op::Get("relax.shape_of");
+  static const Op op = Op::Get("relax.shape_of");
   return Call(Type::Missing(), op, {expr}, {}, {});
 }
 
@@ -1035,14 +1052,15 @@ Type InferTypeSize(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr(ffi::Array<PrimExpr>{}), PrimType::Int(64));
 }
 
-TVM_REGISTER_OP("relax.size")
-    .set_num_inputs(1)
-    .add_argument("input", "Expr", "The input tensor")
-    .set_attr<FInferType>("FInferType", InferTypeSize)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.size")
+      .add_arg("input", "The input tensor")
+      .set_attr<FInferType>("FInferType", InferTypeSize)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeSize(Expr expr) {
-  static const Op& op = Op::Get("relax.size");
+  static const Op op = Op::Get("relax.size");
   return Call(Type::Missing(), op, {expr}, {}, {});
 }
 
@@ -1072,14 +1090,15 @@ Type ReturnTensorToShapeType(const Call& call, const BlockBuilder& ctx) {
   return ShapeType(kUnknownNDim);
 }
 
-TVM_REGISTER_OP("relax.tensor_to_shape")
-    .set_num_inputs(1)
-    .add_argument("input", "Expr", "The input expression")
-    .set_attr<FInferType>("FInferType", ReturnTensorToShapeType)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.tensor_to_shape")
+      .add_arg("input", "The input expression")
+      .set_attr<FInferType>("FInferType", ReturnTensorToShapeType)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeTensorToShape(Expr expr) {
-  static const Op& op = Op::Get("relax.tensor_to_shape");
+  static const Op op = Op::Get("relax.tensor_to_shape");
   return Call(Type::Missing(), op, {expr}, {}, {});
 }
 
@@ -1098,15 +1117,16 @@ Type ReturnShapeToTensorType(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr({PrimExpr(ndim)}), PrimType::Int(64));
 }
 
-TVM_REGISTER_OP("relax.shape_to_tensor")
-    .set_num_inputs(1)
-    .add_argument("input", "Expr", "The input expression")
-    .set_attr<FInferType>("FInferType", ReturnShapeToTensorType)
-    .set_attr<FCallPacked>("FCallPacked", "relax.run.shape_to_tensor")
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.shape_to_tensor")
+      .add_arg("input", "The input expression")
+      .set_attr<FInferType>("FInferType", ReturnShapeToTensorType)
+      .set_attr<FCallPacked>("FCallPacked", "relax.run.shape_to_tensor")
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeShapeToTensor(Expr expr) {
-  static const Op& op = Op::Get("relax.shape_to_tensor");
+  static const Op op = Op::Get("relax.shape_to_tensor");
   return Call(Type::Missing(), op, {expr}, {}, {});
 }
 
@@ -1139,23 +1159,23 @@ Type InferTypeAllocateTensor(const Call& call, const BlockBuilder& ctx) {
   return TensorType(call->args[0], out_dtype);
 }
 
-TVM_REGISTER_OP("relax.builtin.alloc_tensor")
-    .set_num_inputs(4)
-    .add_argument("shape", "Expr", "The shape of the tensor to allocate.")
-    .add_argument("dtype", "DataTypeImm", "The dtype of the tensor to allocate.")
-    .add_argument("runtime_device_index", "PrimExpr",
-                  "The device index indicating on which device the tensor is to be "
-                  "allocated at runtime. Index -1 is reserved for the host device.")
-    .add_argument("storage_scope", "StringImm",
-                  "The storage scope of the storage to allocate. Default is global.")
-    .set_attr<FInferType>("FInferType", InferTypeAllocateTensor)
-    // memory allocation isn't considered a "visible effect" as far as purity is concerned
-    .set_attr<bool>("FPurity", true)
-    .set_attr<bool>("TAllocator", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.builtin.alloc_tensor")
+      .add_arg("shape", "The shape of the tensor to allocate.")
+      .add_arg("dtype", "The dtype of the tensor to allocate.")
+      .add_arg("runtime_device_index",
+               "The device index indicating on which device the tensor is to be "
+               "allocated at runtime. Index -1 is reserved for the host device.")
+      .add_arg("storage_scope", "The storage scope of the storage to allocate. Default is global.")
+      .set_attr<FInferType>("FInferType", InferTypeAllocateTensor)
+      // memory allocation isn't considered a "visible effect" as far as purity is concerned
+      .set_attr<bool>("FPurity", true)
+      .set_attr<bool>("TAllocator", true);
+}
 
 Expr MakeAllocTensor(Expr shape, DataTypeImm dtype, PrimExpr runtime_device_index,
                      StringImm storage_scope) {
-  static const Op& op = Op::Get("relax.builtin.alloc_tensor");
+  static const Op op = Op::Get("relax.builtin.alloc_tensor");
   return Call(Type::Missing(), op, {shape, dtype, runtime_device_index, storage_scope}, Attrs(),
               {});
 }
@@ -1163,28 +1183,26 @@ Expr MakeAllocTensor(Expr shape, DataTypeImm dtype, PrimExpr runtime_device_inde
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.builtin.alloc_tensor", MakeAllocTensor);
-}
 
-// memory planning alloc_storage
+  // memory planning alloc_storage
 
-TVM_REGISTER_OP("relax.memory.alloc_storage")
-    .set_num_inputs(4)
-    .add_argument("total_space", "Expr", "The total space of the storage to allocate.")
-    .add_argument(
-        "virtual_device_index", "PrimExpr",
-        "The virtual device index indicating on which device the storage is to be allocated, "
-        "Index -1 is reserved for the host device.")
-    .add_argument("storage_scope", "StringImm",
-                  "The storage scope of the storage to allocate. Default is global.")
-    .add_argument("dtype", "DataTypeImm", "The dtype of the tensor to allocate.")
-    .set_attr<FInferType>("FInferType", ReturnAnyType)
-    // memory allocation isn't considered a "visible effect" as far as purity is concerned
-    .set_attr<bool>("FPurity", true)
-    .set_attr<bool>("TAllocator", true);
+  OpDef("relax.memory.alloc_storage")
+      .add_arg("total_space", "The total space of the storage to allocate.")
+      .add_arg(
+          "virtual_device_index",
+          "The virtual device index indicating on which device the storage is to be allocated, "
+          "Index -1 is reserved for the host device.")
+      .add_arg("storage_scope", "The storage scope of the storage to allocate. Default is global.")
+      .add_arg("dtype", "The dtype of the tensor to allocate.")
+      .set_attr<FInferType>("FInferType", ReturnAnyType)
+      // memory allocation isn't considered a "visible effect" as far as purity is concerned
+      .set_attr<bool>("FPurity", true)
+      .set_attr<bool>("TAllocator", true);
+}
 
 Expr MakeAllocStorage(Expr size, PrimExpr virtual_device_index, StringImm storage_scope,
                       DataTypeImm dtype) {
-  static const Op& op = Op::Get("relax.memory.alloc_storage");
+  static const Op op = Op::Get("relax.memory.alloc_storage");
   return Call(Type::Missing(), op, {size, virtual_device_index, storage_scope, dtype}, Attrs(), {});
 }
 
@@ -1218,23 +1236,24 @@ Type InferTypeMemAllocTensor(const Call& call, const BlockBuilder& ctx) {
   return TensorType(call->args[2], out_dtype);
 }
 
-TVM_REGISTER_OP("relax.memory.alloc_tensor")
-    .set_num_inputs(5)
-    .add_argument("storage", "Expr", "The storage to allocate the tensor to.")
-    .add_argument("offset", "PrimExpr", "Storage offset to allocate the tensor.")
-    .add_argument("shape", "Expr", "The shape of the tensor to allocate.")
-    .add_argument("dtype", "DataTypeImm", "The dtype of the tensor to allocate.")
-    .add_argument("runtime_device_index", "PrimExpr",
-                  "The device index indicating on which device the tensor is to be "
-                  "allocated at runtime. Index -1 is reserved for the host device.")
-    .set_attr<FInferType>("FInferType", InferTypeMemAllocTensor)
-    // memory allocation isn't considered a "visible effect" as far as purity is concerned
-    .set_attr<bool>("FPurity", true)
-    .set_attr<bool>("TAllocator", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.memory.alloc_tensor")
+      .add_arg("storage", "The storage to allocate the tensor to.")
+      .add_arg("offset", "Storage offset to allocate the tensor.")
+      .add_arg("shape", "The shape of the tensor to allocate.")
+      .add_arg("dtype", "The dtype of the tensor to allocate.")
+      .add_arg("runtime_device_index",
+               "The device index indicating on which device the tensor is to be "
+               "allocated at runtime. Index -1 is reserved for the host device.")
+      .set_attr<FInferType>("FInferType", InferTypeMemAllocTensor)
+      // memory allocation isn't considered a "visible effect" as far as purity is concerned
+      .set_attr<bool>("FPurity", true)
+      .set_attr<bool>("TAllocator", true);
+}
 
 Expr MakeMemAllocTensor(Expr storage, PrimExpr offset, Expr shape, DataTypeImm dtype,
                         PrimExpr virtual_device_index) {
-  static const Op& op = Op::Get("relax.memory.alloc_tensor");
+  static const Op op = Op::Get("relax.memory.alloc_tensor");
   return Call(Type::Missing(), op, {storage, offset, shape, dtype, virtual_device_index}, Attrs(),
               {});
 }
@@ -1253,65 +1272,61 @@ TVM_FFI_STATIC_INIT_BLOCK() {
                                     IntImm::Int64(0));
         }
       });
-}
 
-// memory planning kill_storage
+  // memory planning kill_storage
 
-TVM_REGISTER_OP("relax.memory.kill_storage")
-    .set_num_inputs(1)
-    .add_argument("storage", "Expr", "The storage to be killed.")
-    .set_attr<FInferType>("FInferType", ReturnVoidType)
-    // We mark this as impure so it wouldn't be removed by "remove_all_unused"
-    .set_attr<bool>("FPurity", false);
+  OpDef("relax.memory.kill_storage")
+      .add_arg("storage", "The storage to be killed.")
+      .set_attr<FInferType>("FInferType", ReturnVoidType)
+      // We mark this as impure so it wouldn't be removed by "remove_all_unused"
+      .set_attr<bool>("FPurity", false);
+}
 
 Expr MakeMemKillStorage(Expr storage) {
-  static const Op& op = Op::Get("relax.memory.kill_storage");
+  static const Op op = Op::Get("relax.memory.kill_storage");
   return Call(Type::Missing(), op, {storage}, {}, {});
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.memory.kill_storage", MakeMemKillStorage);
-}
 
-// memory planning kill_tensor
+  // memory planning kill_tensor
 
-TVM_REGISTER_OP("relax.memory.kill_tensor")
-    .set_num_inputs(1)
-    .add_argument("tensor", "Expr", "The tensor to be killed.")
-    .set_attr<FInferType>("FInferType", ReturnVoidType)
-    // We mark this as impure so it wouldn't be removed by "remove_all_unused"
-    .set_attr<bool>("FPurity", false);
+  OpDef("relax.memory.kill_tensor")
+      .add_arg("tensor", "The tensor to be killed.")
+      .set_attr<FInferType>("FInferType", ReturnVoidType)
+      // We mark this as impure so it wouldn't be removed by "remove_all_unused"
+      .set_attr<bool>("FPurity", false);
+}
 
 Expr MakeMemKillTensor(Expr tensor) {
-  static const Op& op = Op::Get("relax.memory.kill_tensor");
+  static const Op op = Op::Get("relax.memory.kill_tensor");
   return Call(Type::Missing(), op, {tensor}, {}, {});
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.memory.kill_tensor", MakeMemKillTensor);
-}
 
-// vm alloc_storage
+  // vm alloc_storage
 
-TVM_REGISTER_OP("relax.vm.alloc_storage")
-    .set_num_inputs(4)
-    .add_argument("size", "Expr", "The size of the storage to allocate.")
-    .add_argument("dtype", "DataTypeImm", "The dtype of the tensor to allocate.")
-    .add_argument("runtime_device_index", "PrimExpr",
-                  "The device index indicating on which device the tensor is "
-                  "to be allocated at runtime.")
-    .add_argument("storage_scope", "StringImm",
-                  "The storage scope of the storage to allocate. Default is global.")
-    .set_attr<FInferType>("FInferType", ReturnAnyType)
-    // memory allocation isn't considered a "visible effect" as far as purity is concerned
-    .set_attr<bool>("FPurity", true)
-    .set_attr<bool>("TAllocator", true);
+  OpDef("relax.vm.alloc_storage")
+      .add_arg("size", "The size of the storage to allocate.")
+      .add_arg("runtime_device_index",
+               "The device index indicating on which device the tensor is "
+               "to be allocated at runtime.")
+      .add_arg("dtype", "The dtype of the tensor to allocate.")
+      .add_arg("storage_scope", "The storage scope of the storage to allocate. Default is global.")
+      .set_attr<FInferType>("FInferType", ReturnAnyType)
+      // memory allocation isn't considered a "visible effect" as far as purity is concerned
+      .set_attr<bool>("FPurity", true)
+      .set_attr<bool>("TAllocator", true);
+}
 
 Expr MakeVMAllocStorage(Expr size, PrimExpr runtime_device_index, DataTypeImm dtype,
                         StringImm storage_scope) {
-  static const Op& op = Op::Get("relax.vm.alloc_storage");
+  static const Op op = Op::Get("relax.vm.alloc_storage");
   return Call(Type::Missing(), op, {size, runtime_device_index, dtype, storage_scope}, Attrs(), {});
 }
 
@@ -1346,23 +1361,24 @@ Type InferTypeVMAllocTensor(const Call& call, const BlockBuilder& ctx) {
   return TensorType(out_dtype, kUnknownNDim, vdevice);
 }
 
-TVM_REGISTER_OP("relax.vm.alloc_tensor")
-    .set_num_inputs(5)
-    .add_argument("storage", "Expr", "The storage to allocate the tensor to.")
-    .add_argument("offset", "PrimExpr", "Storage offset to allocate the tensor.")
-    .add_argument("shape", "Expr", "The shape of the tensor to allocate.")
-    .add_argument("dtype", "DataTypeImm", "The dtype of the tensor to allocate.")
-    .add_argument("runtime_device_index", "PrimExpr",
-                  "The device index indicating on which device the tensor is "
-                  "to be allocated at runtime.")
-    .set_attr<FInferType>("FInferType", InferTypeVMAllocTensor)
-    // memory allocation isn't considered a "visible effect" as far as purity is concerned
-    .set_attr<bool>("FPurity", true)
-    .set_attr<bool>("TAllocator", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.vm.alloc_tensor")
+      .add_arg("storage", "The storage to allocate the tensor to.")
+      .add_arg("offset", "Storage offset to allocate the tensor.")
+      .add_arg("shape", "The shape of the tensor to allocate.")
+      .add_arg("dtype", "The dtype of the tensor to allocate.")
+      .add_arg("runtime_device_index",
+               "The device index indicating on which device the tensor is "
+               "to be allocated at runtime.")
+      .set_attr<FInferType>("FInferType", InferTypeVMAllocTensor)
+      // memory allocation isn't considered a "visible effect" as far as purity is concerned
+      .set_attr<bool>("FPurity", true)
+      .set_attr<bool>("TAllocator", true);
+}
 
 Expr MakeVMAllocTensor(Expr storage, PrimExpr offset, Expr shape, DataTypeImm dtype,
                        PrimExpr runtime_device_index) {
-  static const Op& op = Op::Get("relax.vm.alloc_tensor");
+  static const Op op = Op::Get("relax.vm.alloc_tensor");
   return Call(Type::Missing(), op, {storage, offset, shape, dtype, runtime_device_index}, Attrs(),
               {});
 }
@@ -1378,39 +1394,37 @@ TVM_FFI_STATIC_INIT_BLOCK() {
                                args[3].cast<DataTypeImm>(), IntImm::Int64(0));
     }
   });
-}
 
-// vm kill_object
-TVM_REGISTER_OP("relax.vm.kill_object")
-    .set_num_inputs(1)
-    .add_argument("obj", "Expr", "The object to be killed.")
-    .set_attr<FInferType>("FInferType", ReturnVoidType)
-    // We mark this as impure so it wouldn't be removed by "remove_all_unused"
-    .set_attr<bool>("FPurity", false);
+  // vm kill_object
+
+  OpDef("relax.vm.kill_object")
+      .add_arg("obj", "The object to be killed.")
+      .set_attr<FInferType>("FInferType", ReturnVoidType)
+      // We mark this as impure so it wouldn't be removed by "remove_all_unused"
+      .set_attr<bool>("FPurity", false);
+}
 
 Expr MakeVMKillObject(Expr obj) {
-  static const Op& op = Op::Get("relax.vm.kill_object");
+  static const Op op = Op::Get("relax.vm.kill_object");
   return Call(Type::Missing(), op, {std::move(obj)}, Attrs(), {});
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.vm.kill_object", MakeVMKillObject);
-}
 
-// vm call_tir_dyn
+  // vm call_tir_dyn
 
-TVM_REGISTER_OP("relax.vm.call_tir_dyn")
-    .set_num_inputs(2)
-    .add_argument("func", "Expr", "The destination-passing-style function.")
-    .add_argument("args", "Tuple",
-                  "The input arguments (list of tensors and last argument is ShapeExpr)")
-    .set_attr<FInferType>("FInferType", ReturnVoidType)
-    // "relax.vm.call_tir_dyn" works in an in-place way, which is impure.
-    .set_attr<bool>("FPurity", false);
+  OpDef("relax.vm.call_tir_dyn")
+      .add_arg("func", "The destination-passing-style function.")
+      .add_arg("args", "The input arguments (list of tensors and last argument is ShapeExpr)")
+      .set_attr<FInferType>("FInferType", ReturnVoidType)
+      // "relax.vm.call_tir_dyn" works in an in-place way, which is impure.
+      .set_attr<bool>("FPurity", false);
+}
 
 Expr MakeCallTIRDyn(Expr func, Tuple args) {
-  static const Op& op = Op::Get("relax.vm.call_tir_dyn");
+  static const Op op = Op::Get("relax.vm.call_tir_dyn");
   return Call(Type::Missing(), op, {func, args}, Attrs(), {});
 }
 
@@ -1424,14 +1438,15 @@ Type InferTypeStopLiftParams(const Call& call, const BlockBuilder& ctx) {
   return InferTypeUnaryArith<false>(call, ctx);
 }
 
-TVM_REGISTER_OP("relax.builtin.stop_lift_params")
-    .set_num_inputs(1)
-    .add_argument("x", "Expr", "The input data")
-    .set_attr<FInferType>("FInferType", InferTypeStopLiftParams)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.builtin.stop_lift_params")
+      .add_arg("x", "The input data")
+      .set_attr<FInferType>("FInferType", InferTypeStopLiftParams)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeStopLiftParams(Expr x) {
-  static const Op& op = Op::Get("relax.builtin.stop_lift_params");
+  static const Op op = Op::Get("relax.builtin.stop_lift_params");
   return Call(Type::Missing(), op, {x}, Attrs(), {});
 }
 
@@ -1454,15 +1469,16 @@ Type InferToVDeviceType(const Call& call, const BlockBuilder& ctx) {
   return TensorType(data_ty->dtype, data_ty->ndim, vdev, data_ty->span);
 }
 
-TVM_REGISTER_OP("relax.to_vdevice")
-    .set_num_inputs(1)
-    .set_attrs_type<ToVDeviceAttrs>()
-    .add_argument("data", "Expr", "The input expression to be copied")
-    .set_attr<FInferType>("FInferType", InferToVDeviceType)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.to_vdevice")
+      .add_arg("data", "The input expression to be copied")
+      .attrs_type<ToVDeviceAttrs>()
+      .set_attr<FInferType>("FInferType", InferToVDeviceType)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeToVDevice(Expr data, VDevice dst_vdev) {
-  static const Op& op = Op::Get("relax.to_vdevice");
+  static const Op op = Op::Get("relax.to_vdevice");
   ffi::ObjectPtr<ToVDeviceAttrs> attrs = ffi::make_object<ToVDeviceAttrs>();
   attrs->dst_vdevice = dst_vdev;
   return Call(Type::Missing(), op, {data}, Attrs(attrs), {});
@@ -1482,15 +1498,16 @@ Type InferHintOnDeviceType(const Call& call, const BlockBuilder& ctx) {
   return data_ty;
 }
 
-TVM_REGISTER_OP("relax.hint_on_device")
-    .set_num_inputs(1)
-    .set_attrs_type<HintOnDeviceAttrs>()
-    .add_argument("data", "Expr", "The input expression")
-    .set_attr<FInferType>("FInferType", InferHintOnDeviceType)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.hint_on_device")
+      .add_arg("data", "The input expression")
+      .attrs_type<HintOnDeviceAttrs>()
+      .set_attr<FInferType>("FInferType", InferHintOnDeviceType)
+      .set_attr<bool>("FPurity", true);
+}
 
 Expr MakeHintOnDevice(Expr data, Device device, ffi::String memory_scope = "global") {
-  static const Op& op = Op::Get("relax.hint_on_device");
+  static const Op op = Op::Get("relax.hint_on_device");
   ffi::ObjectPtr<HintOnDeviceAttrs> attrs = ffi::make_object<HintOnDeviceAttrs>();
   attrs->device_type = static_cast<int32_t>(device.device_type);
   attrs->index = device.device_id;
diff --git a/src/relax/op/op_common.cc b/src/relax/op/op_common.cc
index 0c656f9f42..67e23768f7 100644
--- a/src/relax/op/op_common.cc
+++ b/src/relax/op/op_common.cc
@@ -30,7 +30,7 @@ namespace relax {
 using namespace tvm::prim;
 
 ffi::Array<Expr> GetCallArgs(const Call& call) {
-  static const Op& call_tir_op = Op::Get("relax.call_tir");
+  static const Op call_tir_op = Op::Get("relax.call_tir");
   ffi::Array<Expr> args;
   if (call->op.same_as(call_tir_op)) {
     args = call->args[1].as_or_throw<Tuple>()->fields;
@@ -42,10 +42,12 @@ ffi::Array<Expr> GetCallArgs(const Call& call) {
 
 void CheckNumArguments(const Call& call, const BlockBuilder& ctx) {
   Op op = call->op.as_or_throw<Op>();
-  int expected_input = op->arguments.size();
-  if (static_cast<int>(call->args.size()) != expected_input) {
+  int expected_input = op->args_info.size();
+  if (op->allow_extra_args ? call->args.size() < op->args_info.size()
+                           : call->args.size() != op->args_info.size()) {
     TVM_FFI_VISIT_THROW(ValueError, call)
-        << "Operator " << op << " expects " << expected_input << " arguments"
+        << "Operator " << op << " expects " << (op->allow_extra_args ? "at least " : "")
+        << expected_input << " arguments"
         << ", but was called with " << call->args.size() << " arguments";
   }
 }
@@ -53,10 +55,11 @@ void CheckNumArguments(const Call& call, const BlockBuilder& ctx) {
 TensorType GetInputTensorType(const Call& call, size_t i_arg, const BlockBuilder& ctx) {
   Op op = call->op.as_or_throw<Op>();
 
-  TVM_FFI_ICHECK_EQ(op->arguments.size(), call->args.size())
+  TVM_FFI_ICHECK(op->allow_extra_args ? call->args.size() >= op->args_info.size()
+                                      : call->args.size() == op->args_info.size())
       << "Failure caught by this check "
       << "should have previously been caught by `CheckNumArguments`";
-  TVM_FFI_ICHECK_LT(i_arg, op->arguments.size());
+  TVM_FFI_ICHECK_LT(i_arg, call->args.size());
 
   auto arg = call->args[i_arg];
   auto ty = GetType(arg);
@@ -65,8 +68,10 @@ TensorType GetInputTensorType(const Call& call, size_t i_arg, const BlockBuilder
     return tensor_ty.value();
   } else {
     TVM_FFI_VISIT_THROW(TypeError, call)
-        << "Operator " << op << " requires argument " << i_arg << " (" << op->arguments[i_arg]->name
-        << ") to be a tensor.  "
+        << "Operator " << op << " requires argument " << i_arg
+        << (i_arg < op->args_info.size() ? " (" + std::string(op->args_info[i_arg]->name) + ")"
+                                         : "")
+        << " to be a tensor.  "
         << "However, the argument " << arg << " is instead of type " << ty;
     TVM_FFI_UNREACHABLE();
   }
diff --git a/src/relax/op/op_common.h b/src/relax/op/op_common.h
index 953892c1f1..b4a9ac2817 100644
--- a/src/relax/op/op_common.h
+++ b/src/relax/op/op_common.h
@@ -142,50 +142,24 @@ std::tuple<ArgTypes...> GetArgTypeHelper(const Call& call, const Op& op, const B
 template <typename... ArgTypes>
 std::tuple<ArgTypes...> GetArgType(const Call& call, const BlockBuilder& ctx) {
   Op op = call->op.as_or_throw<Op>();
-  size_t n_input = op->arguments.size();
+  size_t n_input = op->args_info.size();
 
-  // Unfortunately, because the `.add_argument()` calls in
-  // TVM_REGISTER_OP occur during initialization of globals and are
+  // Unfortunately, because the `.add_arg()` calls in
+  // OpDef occur during initialization of globals and are
   // not available at compile-time, this cannot be a static_assert.
-  TVM_FFI_ICHECK_EQ(n_input, sizeof...(ArgTypes))
-      << "Internal error: " << op << " op defines " << n_input
-      << " arguments in its TVM_REGISTER_OP() call, "
+  TVM_FFI_ICHECK(op->allow_extra_args ? sizeof...(ArgTypes) >= n_input
+                                      : sizeof...(ArgTypes) == n_input)
+      << "Internal error: " << op << " op defines " << n_input << " arguments in its OpDef() call, "
       << "but GetArgType was given " << sizeof...(ArgTypes) << " template arguments.";
 
+  CheckNumArguments(call, ctx);
+  TVM_FFI_ICHECK_GE(call->args.size(), sizeof...(ArgTypes));
   return detail::GetArgTypeHelper<ArgTypes...>(call, op, ctx,
                                                std::make_index_sequence<sizeof...(ArgTypes)>());
 }
 
 /************ Op registration macro ************/
 
-/*!
- * \brief Quick helper macro to register the operator to registry
- * \param OpRegName The name of operator to register. The name passed in will
- * be prepended with a prefix "relax." as the identifier string in the operator registry.
- */
-#define RELAX_REGISTER_UNARY_OP(OpRegName)                                                         \
-  TVM_REGISTER_OP("relax." OpRegName)                                                              \
-      .set_num_inputs(1)                                                                           \
-      .add_argument("x", "Tensor", "The input tensor.")                                            \
-      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)                     \
-      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow) \
-      .set_attr<bool>("FPurity", true)
-
-/*!
- * \brief Quick helper macro to expose a make-function to construct the operator.
- * \param OpName The name of the operator as well as the make-function name, which will
- * be prepended with a prefix "relax.op." as the FFI identifier string for the make function,
- * \param OpRegName The identifier of the operator in the registry.
- */
-#define RELAX_UNARY_OP_INTERFACE(OpName, OpRegName)                       \
-  Expr OpName(Expr x) {                                                   \
-    static const Op& op = Op::Get("relax." OpRegName);                    \
-    return Call(Type::Missing(), op, {std::move(x)}, Attrs(), {});        \
-  }                                                                       \
-  TVM_FFI_STATIC_INIT_BLOCK() {                                           \
-    tvm::ffi::reflection::GlobalDef().def("relax.op." OpRegName, OpName); \
-  }
-
 /*!
  * \brief Infer the type for unary elementwise ops.
  * \param call The context Call to the operator.
@@ -231,10 +205,8 @@ inline Type InferTypeUnary(const Call& call, const BlockBuilder& ctx, FType f_co
 template <int arg_index>
 Type ReturnTypeFromArg(const Call& call, const BlockBuilder& ctx) {
   Op op = call->op.as_or_throw<Op>();
-  int n_input = op->arguments.size();
-  if (static_cast<int>(call->args.size()) != n_input) {
-    TVM_FFI_VISIT_THROW(ValueError, call) << op << " op should have " << n_input << " arguments";
-  }
+  CheckNumArguments(call, ctx);
+  int n_input = call->args.size();
   if (arg_index >= n_input) {
     TVM_FFI_VISIT_THROW(IndexError, call)
         << op << " op has only " << n_input << "arguments, but try to get the arg with index "
diff --git a/src/relax/op/tensor/binary.cc b/src/relax/op/tensor/binary.cc
index 7bc15b5184..1337db324e 100644
--- a/src/relax/op/tensor/binary.cc
+++ b/src/relax/op/tensor/binary.cc
@@ -35,12 +35,7 @@ using namespace tvm::prim;
 
 template <typename FType>
 Type InferTypeBroadcast(const Call& call, const BlockBuilder& ctx, FType f_compute_out_dtype) {
-  Op op = call->op.as_or_throw<Op>();
-  size_t n_input = op->arguments.size();
-  if (call->args.size() != n_input) {
-    TVM_FFI_VISIT_THROW(ValueError, call)
-        << call->op << " op should have " << n_input << " arguments";
-  }
+  CheckNumArguments(call, ctx);
 
   auto lhs_ty = GetType(call->args[0]);
   auto rhs_ty = GetType(call->args[1]);
diff --git a/src/relax/op/tensor/binary.h b/src/relax/op/tensor/binary.h
index a4a728e076..46ec595b26 100644
--- a/src/relax/op/tensor/binary.h
+++ b/src/relax/op/tensor/binary.h
@@ -29,37 +29,31 @@
 namespace tvm {
 namespace relax {
 
-/*!
- * \brief Quick helper macro
- * - Expose a make function to construct the node.
- * - Register op to the registry.
- * \param OpName The name of operator to register. The name passed in will
- *  1. be prepended with a prefix "relax.op." as the FFI identifier string for the make function,
- *  2. be prepended with a prefix "relax." as the identifier string in the operator registry.
- */
-#define RELAX_REGISTER_BINARY_OP_AND_IMPL(OpName)                                                  \
-  Expr OpName(Expr x1, Expr x2) {                                                                  \
-    static const Op& op = Op::Get("relax." #OpName);                                               \
-    return Call(Type::Missing(), op, {x1, x2}, Attrs(), {});                                       \
-  }                                                                                                \
-  TVM_FFI_STATIC_INIT_BLOCK() {                                                                    \
-    tvm::ffi::reflection::GlobalDef().def("relax.op." #OpName, OpName);                            \
-  }                                                                                                \
-  TVM_REGISTER_OP("relax." #OpName)                                                                \
-      .set_num_inputs(2)                                                                           \
-      .add_argument("x1", "Tensor", "The first input tensor.")                                     \
-      .add_argument("x2", "Tensor", "The second input tensor.")                                    \
-      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutBinaryEwise)                    \
-      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow) \
-      .set_attr<bool>("FPurity", true)
-
-#define RELAX_REGISTER_BINARY_BROADCAST_OP_AND_IMPL(OpName)                    \
-  RELAX_REGISTER_BINARY_OP_AND_IMPL(OpName).set_attr<FInferType>("FInferType", \
-                                                                 InferTypeBroadcastArith)
-
-#define RELAX_REGISTER_CMP_OP_AND_IMPL(OpName)                                 \
-  RELAX_REGISTER_BINARY_OP_AND_IMPL(OpName).set_attr<FInferType>("FInferType", \
-                                                                 InferTypeBroadcastCMP)
+/* Register a binary constructor and its Op metadata together. */
+#define RELAX_REGISTER_BINARY_OP_AND_IMPL(OpName, ...)                                      \
+  Expr OpName(Expr x1, Expr x2) {                                                           \
+    static const Op op = Op::Get("relax." #OpName);                                         \
+    return Call(Type::Missing(), op, {x1, x2}, Attrs(), {});                                \
+  }                                                                                         \
+  TVM_FFI_STATIC_INIT_BLOCK() {                                                             \
+    tvm::ffi::reflection::GlobalDef().def("relax.op." #OpName, OpName);                     \
+    OpDef("relax." #OpName)                                                                 \
+        .add_arg("x1", "The first input tensor.")                                           \
+        .add_arg("x2", "The second input tensor.")                                          \
+        .add_ty_arg("out_type", "Optional output tensor type carrying the virtual device.") \
+        .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutBinaryEwise)           \
+        .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy",                           \
+                                         MixedPrecisionPolicyKind::kFollow)                 \
+        .set_attr<bool>("FPurity", true) __VA_ARGS__;                                       \
+  }
+
+#define RELAX_REGISTER_BINARY_BROADCAST_OP_AND_IMPL(OpName) \
+  RELAX_REGISTER_BINARY_OP_AND_IMPL(OpName,                 \
+                                    .set_attr<FInferType>("FInferType", InferTypeBroadcastArith))
+
+#define RELAX_REGISTER_CMP_OP_AND_IMPL(OpName) \
+  RELAX_REGISTER_BINARY_OP_AND_IMPL(OpName,    \
+                                    .set_attr<FInferType>("FInferType", InferTypeBroadcastCMP))
 
 /***************** Arithmetic operators *****************/
 
diff --git a/src/relax/op/tensor/create.cc b/src/relax/op/tensor/create.cc
index 9776c1cdcd..72504a2d0d 100644
--- a/src/relax/op/tensor/create.cc
+++ b/src/relax/op/tensor/create.cc
@@ -61,7 +61,7 @@ Expr full(ffi::Variant<Expr, ffi::Array<PrimExpr>> shape, Expr fill_value,
   ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
   attrs->dtype = dtype;
 
-  static const Op& op = Op::Get("relax.full");
+  static const Op op = Op::Get("relax.full");
   return Call(Type::Missing(), op, {std::move(shape_in_expr), std::move(fill_value)}, Attrs(attrs),
               {});
 }
@@ -95,22 +95,23 @@ Type InferTypeFull(const Call& call, const BlockBuilder& ctx) {
   return TensorType(/*shape=*/call->args[0], out_dtype, fill_value_ty->vdevice);
 }
 
-TVM_REGISTER_OP("relax.full")
-    .set_attrs_type<InitAttrs>()
-    .set_num_inputs(2)
-    .add_argument("shape", "Shape", "The shape of the created tensor.")
-    .add_argument("fill_value", "Tensor", "The scalar tensor, denoting the value to fill.")
-    .set_attr<FInferType>("FInferType", InferTypeFull)
-    .set_attr<bool>("RequiresArgumentShapes", false)
-    .set_attr<bool>("FDataDependent", true)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.full")
+      .add_arg("shape", "The shape of the created tensor.")
+      .add_arg("fill_value", "The scalar tensor, denoting the value to fill.")
+      .attrs_type<InitAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeFull)
+      .set_attr<bool>("RequiresArgumentShapes", false)
+      .set_attr<bool>("FDataDependent", true)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.full_like */
 Expr full_like(Expr x, Expr fill_value, ffi::Optional<DLDataType> dtype) {
   ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
   attrs->dtype = dtype;
-  static const Op& op = Op::Get("relax.full_like");
+  static const Op op = Op::Get("relax.full_like");
   return Call(Type::Missing(), op, {std::move(x), std::move(fill_value)}, Attrs(attrs), {});
 }
 
@@ -139,14 +140,15 @@ Type InferTypeFullLike(const Call& call, const BlockBuilder& ctx) {
   }
 }
 
-TVM_REGISTER_OP("relax.full_like")
-    .set_attrs_type<InitAttrs>()
-    .set_num_inputs(2)
-    .add_argument("x", "Tensor", "The input tensor.")
-    .add_argument("fill_value", "Tensor", "The scalar value to fill.")
-    .set_attr<FInferType>("FInferType", InferTypeFullLike)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.full_like")
+      .add_arg("x", "The input tensor.")
+      .add_arg("fill_value", "The scalar value to fill.")
+      .attrs_type<InitAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeFullLike)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 // Structure info inference for ones and zeros
 Type InferTypeOnesZeros(const Call& call, const BlockBuilder& ctx) {
@@ -183,85 +185,81 @@ Expr ones(Expr shape, DLDataType dtype) {
   ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
   attrs->dtype = dtype;
 
-  static const Op& op = Op::Get("relax.ones");
+  static const Op op = Op::Get("relax.ones");
   return Call(Type::Missing(), op, {std::move(shape)}, Attrs(attrs), {});
 }
 
 Expr ones_like(Expr x, ffi::Optional<DLDataType> dtype) {
   ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
   attrs->dtype = dtype;
-  static const Op& op = Op::Get("relax.ones_like");
+  static const Op op = Op::Get("relax.ones_like");
   return Call(Type::Missing(), op, {std::move(x)}, Attrs(attrs), {});
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.ones", ones).def("relax.op.ones_like", ones_like);
-}
 
-TVM_REGISTER_OP("relax.ones")
-    .set_attrs_type<InitAttrs>()
-    .set_num_inputs(1)
-    .add_argument("shape", "Shape", "The shape of the created tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeOnesZeros)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
-
-TVM_REGISTER_OP("relax.ones_like")
-    .set_attrs_type<InitAttrs>()
-    .set_num_inputs(1)
-    .add_argument("x", "Tensor", "The input tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeOnesLikeZerosLike)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.ones")
+      .add_arg("shape", "The shape of the created tensor.")
+      .attrs_type<InitAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeOnesZeros)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+
+  OpDef("relax.ones_like")
+      .add_arg("x", "The input tensor.")
+      .attrs_type<InitAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeOnesLikeZerosLike)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.zeros & relax.zeros_like */
 Expr zeros(Expr shape, DLDataType dtype) {
   ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
   attrs->dtype = dtype;
 
-  static const Op& op = Op::Get("relax.zeros");
+  static const Op op = Op::Get("relax.zeros");
   return Call(Type::Missing(), op, {std::move(shape)}, Attrs(attrs), {});
 }
 
 Expr zeros_like(Expr x, ffi::Optional<DLDataType> dtype) {
   ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
   attrs->dtype = dtype;
-  static const Op& op = Op::Get("relax.zeros_like");
+  static const Op op = Op::Get("relax.zeros_like");
   return Call(Type::Missing(), op, {std::move(x)}, Attrs(attrs), {});
 }
 
 TVM_FFI_STATIC_INIT_BLOCK() {
   namespace refl = tvm::ffi::reflection;
   refl::GlobalDef().def("relax.op.zeros", zeros).def("relax.op.zeros_like", zeros_like);
-}
 
-TVM_REGISTER_OP("relax.zeros")
-    .set_attrs_type<InitAttrs>()
-    .set_num_inputs(1)
-    .add_argument("shape", "Shape", "The shape of the created tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeOnesZeros)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
-
-TVM_REGISTER_OP("relax.zeros_like")
-    .set_attrs_type<InitAttrs>()
-    .set_num_inputs(1)
-    .add_argument("x", "Tensor", "The input tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeOnesLikeZerosLike)
-    .set_attr<bool>("FPurity", true);
+  OpDef("relax.zeros")
+      .add_arg("shape", "The shape of the created tensor.")
+      .attrs_type<InitAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeOnesZeros)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+
+  OpDef("relax.zeros_like")
+      .add_arg("x", "The input tensor.")
+      .attrs_type<InitAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeOnesLikeZerosLike)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.eye & relax.eye_like */
 Expr eye(PrimExpr n, PrimExpr m, PrimExpr k, DLDataType dtype) {
   ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
   attrs->dtype = dtype;
-  static const Op& op = Op::Get("relax.eye");
+  static const Op op = Op::Get("relax.eye");
   return Call(Type::Missing(), op, {std::move(n), std::move(m), std::move(k)}, Attrs(attrs), {});
 }
 
 Expr eye_like(Expr x, PrimExpr k, ffi::Optional<DLDataType> dtype) {
   ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
   attrs->dtype = dtype;
-  static const Op& op = Op::Get("relax.eye_like");
+  static const Op op = Op::Get("relax.eye_like");
   return Call(Type::Missing(), op, {std::move(x), std::move(k)}, Attrs(attrs), {});
 }
 
@@ -321,29 +319,29 @@ Type InferTypeEyeLike(const Call& call, const BlockBuilder& ctx) {
   return TensorType(x_ty->shape.value(), out_dtype, x_ty->vdevice);
 }
 
-TVM_REGISTER_OP("relax.eye")
-    .set_attrs_type<InitAttrs>()
-    .set_num_inputs(3)
-    .add_argument("n", "PrimExpr", "Number of rows in the output.")
-    .add_argument("m", "PrimExpr", "Number of columns in the output.")
-    .add_argument("k", "PrimExpr", "Index of the diagonal.")
-    .set_attr<FInferType>("FInferType", InferTypeEye)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
-
-TVM_REGISTER_OP("relax.eye_like")
-    .set_attrs_type<InitAttrs>()
-    .set_num_inputs(2)
-    .add_argument("x", "Tensor", "The input tensor.")
-    .add_argument("k", "PrimExpr", "Index of the diagonal.")
-    .set_attr<FInferType>("FInferType", InferTypeEyeLike)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.eye")
+      .add_arg("n", "Number of rows in the output.")
+      .add_arg("m", "Number of columns in the output.")
+      .add_arg("k", "Index of the diagonal.")
+      .attrs_type<InitAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeEye)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+
+  OpDef("relax.eye_like")
+      .add_arg("x", "The input tensor.")
+      .add_arg("k", "Index of the diagonal.")
+      .attrs_type<InitAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeEyeLike)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.arange */
 Expr arange(PrimExpr start, PrimExpr stop, PrimExpr step, DLDataType dtype) {
   ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
   attrs->dtype = dtype;
-  static const Op& op = Op::Get("relax.arange");
+  static const Op op = Op::Get("relax.arange");
   return Call(Type::Missing(), op, {std::move(start), std::move(stop), std::move(step)},
               Attrs(attrs), {});
 }
@@ -388,22 +386,23 @@ Type InferTypeArange(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr({num_elem}), PrimType(dtype));
 }
 
-TVM_REGISTER_OP("relax.arange")
-    .set_attrs_type<InitAttrs>()
-    .set_num_inputs(3)
-    .add_argument("start", "PrimExpr", "The starting value for the set of points.")
-    .add_argument("end", "PrimExpr", "The ending value for the set of points.")
-    .add_argument("step", "PrimExpr", "The gap between each pair of adjacent points.")
-    .set_attr<FInferType>("FInferType", InferTypeArange)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.arange")
+      .add_arg("start", "The starting value for the set of points.")
+      .add_arg("end", "The ending value for the set of points.")
+      .add_arg("step", "The gap between each pair of adjacent points.")
+      .attrs_type<InitAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeArange)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.hamming_window */
 Expr hamming_window(PrimExpr window_size, PrimExpr periodic, PrimExpr alpha, PrimExpr beta,
                     DLDataType dtype) {
   ffi::ObjectPtr<InitAttrs> attrs = ffi::make_object<InitAttrs>();
   attrs->dtype = dtype;
-  static const Op& op = Op::Get("relax.hamming_window");
+  static const Op op = Op::Get("relax.hamming_window");
   return Call(Type::Missing(), op,
               {std::move(window_size), std::move(periodic), std::move(alpha), std::move(beta)},
               Attrs(attrs), {});
@@ -442,28 +441,29 @@ Type InferTypeHammingWindow(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr({window_size}), PrimType(dtype));
 }
 
-TVM_REGISTER_OP("relax.hamming_window")
-    .set_attrs_type<InitAttrs>()
-    .set_num_inputs(4)
-    .add_argument("window_size", "PrimExpr", "The size of the window")
-    .add_argument("periodic", "PrimExpr",
-                  "If True, returns a window to be used as periodic function. If False, return a "
-                  "symmetric window")
-    .add_argument("alpha", "PrimExpr", "The coefficient alpha")
-    .add_argument("beta", "PrimExpr", "The coefficient beta")
-    .set_attr<FInferType>("FInferType", InferTypeHammingWindow)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.hamming_window")
+      .add_arg("window_size", "The size of the window")
+      .add_arg("periodic",
+               "If True, returns a window to be used as periodic function. If False, return a "
+               "symmetric window")
+      .add_arg("alpha", "The coefficient alpha")
+      .add_arg("beta", "The coefficient beta")
+      .attrs_type<InitAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeHammingWindow)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.tril & relax.triu */
 
 Expr tril(Expr x, Expr k) {
-  static const Op& op = Op::Get("relax.tril");
+  static const Op op = Op::Get("relax.tril");
   return Call(Type::Missing(), op, {x, k});
 }
 
 Expr triu(Expr x, Expr k) {
-  static const Op& op = Op::Get("relax.triu");
+  static const Op op = Op::Get("relax.triu");
   return Call(Type::Missing(), op, {x, k});
 }
 
@@ -486,19 +486,19 @@ Type InferTypeTrilTriu(const Call& call, const BlockBuilder& ctx) {
   return data_ty;
 }
 
-TVM_REGISTER_OP("relax.tril")
-    .set_num_inputs(2)
-    .add_argument("x", "Tensor", "The input tensor.")
-    .add_argument("k", "PrimExpr", "The offset of the diagonal.")
-    .set_attr<FInferType>("FInferType", InferTypeTrilTriu)
-    .set_attr<bool>("FPurity", true);
-
-TVM_REGISTER_OP("relax.triu")
-    .set_num_inputs(2)
-    .add_argument("x", "Tensor", "The input tensor.")
-    .add_argument("k", "PrimExpr", "The offset of the diagonal.")
-    .set_attr<FInferType>("FInferType", InferTypeTrilTriu)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.tril")
+      .add_arg("x", "The input tensor.")
+      .add_arg("k", "The offset of the diagonal.")
+      .set_attr<FInferType>("FInferType", InferTypeTrilTriu)
+      .set_attr<bool>("FPurity", true);
+
+  OpDef("relax.triu")
+      .add_arg("x", "The input tensor.")
+      .add_arg("k", "The offset of the diagonal.")
+      .set_attr<FInferType>("FInferType", InferTypeTrilTriu)
+      .set_attr<bool>("FPurity", true);
+}
 
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/tensor/datatype.cc b/src/relax/op/tensor/datatype.cc
index 5bb18bcb9f..87efbb5d91 100644
--- a/src/relax/op/tensor/datatype.cc
+++ b/src/relax/op/tensor/datatype.cc
@@ -42,7 +42,7 @@ Expr astype(Expr x, DLDataType dtype) {
   ffi::ObjectPtr<AstypeAttrs> attrs = ffi::make_object<AstypeAttrs>();
   attrs->dtype = dtype;
 
-  static const Op& op = Op::Get("relax.astype");
+  static const Op op = Op::Get("relax.astype");
   return Call(Type::Missing(), op, {std::move(x)}, Attrs(attrs), {});
 }
 
@@ -59,14 +59,15 @@ Type InferTypeAstype(const Call& call, const BlockBuilder& ctx) {
   return TensorType(new_ty);
 }
 
-TVM_REGISTER_OP("relax.astype")
-    .set_attrs_type<AstypeAttrs>()
-    .set_num_inputs(1)
-    .add_argument("x", "Tensor", "The input tensor")
-    .set_attr<FInferType>("FInferType", InferTypeAstype)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.astype")
+      .add_arg("x", "The input tensor")
+      .attrs_type<AstypeAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeAstype)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutUnaryEwise)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.wrap_param */
 
@@ -74,7 +75,7 @@ Expr MakeWrapParam(Expr data, DLDataType dtype) {
   ffi::ObjectPtr<WrapParamAttrs> attrs = ffi::make_object<WrapParamAttrs>();
   attrs->dtype = dtype;
 
-  static const Op& op = Op::Get("relax.wrap_param");
+  static const Op op = Op::Get("relax.wrap_param");
   return Call(Type::Missing(), op, {std::move(data)}, Attrs(attrs), {});
 }
 
@@ -91,12 +92,13 @@ Type InferTypeWrapParam(const Call& call, const BlockBuilder& ctx) {
   return TensorType(new_ty);
 }
 
-TVM_REGISTER_OP("relax.wrap_param")
-    .set_attrs_type<WrapParamAttrs>()
-    .set_num_inputs(1)
-    .add_argument("data", "Tensor", "The input tensor")
-    .set_attr<FInferType>("FInferType", InferTypeWrapParam)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.wrap_param")
+      .add_arg("data", "The input tensor")
+      .attrs_type<WrapParamAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeWrapParam)
+      .set_attr<bool>("FPurity", true);
+}
 
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/tensor/grad.cc b/src/relax/op/tensor/grad.cc
index aabede652d..8b3e222a47 100644
--- a/src/relax/op/tensor/grad.cc
+++ b/src/relax/op/tensor/grad.cc
@@ -34,7 +34,7 @@ namespace relax {
 
 /* relax.grad.no_grad */
 Expr no_grad(Expr input) {
-  static const Op& op = Op::Get("relax.grad.no_grad");
+  static const Op op = Op::Get("relax.grad.no_grad");
   return Call(Type::Missing(), op, {std::move(input)}, {}, {});
 }
 
@@ -45,15 +45,16 @@ TVM_FFI_STATIC_INIT_BLOCK() {
 
 Type InferTypeNoGrad(const Call& call, const BlockBuilder& ctx) { return GetType(call->args[0]); }
 
-TVM_REGISTER_OP("relax.grad.no_grad")
-    .set_num_inputs(1)
-    .add_argument("x", "Expr", "The corresponding input tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeNoGrad)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.grad.no_grad")
+      .add_arg("x", "The corresponding input tensor.")
+      .set_attr<FInferType>("FInferType", InferTypeNoGrad)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.grad.start_checkpoint */
 Expr start_checkpoint(Expr input) {
-  static const Op& op = Op::Get("relax.grad.start_checkpoint");
+  static const Op op = Op::Get("relax.grad.start_checkpoint");
   return Call(Type::Missing(), op, {std::move(input)}, {}, {});
 }
 
@@ -70,15 +71,16 @@ Type InferTypeStartCheckpoint(const Call& call, const BlockBuilder& ctx) {
   return GetType(call->args[0]);
 }
 
-TVM_REGISTER_OP("relax.grad.start_checkpoint")
-    .set_num_inputs(1)
-    .add_argument("x", "Expr", "The tensor marking the input of the checkpoint stage.")
-    .set_attr<FInferType>("FInferType", InferTypeStartCheckpoint)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.grad.start_checkpoint")
+      .add_arg("x", "The tensor marking the input of the checkpoint stage.")
+      .set_attr<FInferType>("FInferType", InferTypeStartCheckpoint)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.grad.end_checkpoint */
 Expr end_checkpoint(Expr input) {
-  static const Op& op = Op::Get("relax.grad.end_checkpoint");
+  static const Op op = Op::Get("relax.grad.end_checkpoint");
   return Call(Type::Missing(), op, {std::move(input)}, {}, {});
 }
 
@@ -95,11 +97,12 @@ Type InferTypeEndCheckpoint(const Call& call, const BlockBuilder& ctx) {
   return GetType(call->args[0]);
 }
 
-TVM_REGISTER_OP("relax.grad.end_checkpoint")
-    .set_num_inputs(1)
-    .add_argument("x", "Expr", "The output of the checkpoint stage.")
-    .set_attr<FInferType>("FInferType", InferTypeEndCheckpoint)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.grad.end_checkpoint")
+      .add_arg("x", "The output of the checkpoint stage.")
+      .set_attr<FInferType>("FInferType", InferTypeEndCheckpoint)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.grad.nll_loss_backward */
 Expr nll_loss_backward(Expr output_grad, Expr predictions, Expr targets,
@@ -109,7 +112,7 @@ Expr nll_loss_backward(Expr output_grad, Expr predictions, Expr targets,
   attrs->reduction = reduction;
   attrs->ignore_index = ignore_index;
 
-  static const Op& op = Op::Get("relax.grad.nll_loss_backward");
+  static const Op op = Op::Get("relax.grad.nll_loss_backward");
   if (weights.has_value()) {
     return Call(
         Type::Missing(), op,
@@ -131,15 +134,16 @@ Type InferTypeNLLLossBackward(const Call& call, const BlockBuilder& ctx) {
   return GetType(call->args[1]);
 }
 
-TVM_REGISTER_OP("relax.grad.nll_loss_backward")
-    .set_attrs_type<NLLLossAttrs>()
-    .set_num_inputs(4)
-    .add_argument("output_grad", "Tensor", "The output gradient.")
-    .add_argument("predictions", "Tensor", "The prediction tensor.")
-    .add_argument("targets", "Tensor", "The target tensor.")
-    .add_argument("weights", "ffi::Optional<Tensor>", "The weight of each target values.")
-    .set_attr<FInferType>("FInferType", InferTypeNLLLossBackward)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.grad.nll_loss_backward", "Optional weights: The weight of each target values.")
+      .add_arg("output_grad", "The output gradient.")
+      .add_arg("predictions", "The prediction tensor.")
+      .add_arg("targets", "The target tensor.")
+      .allow_extra_args()
+      .attrs_type<NLLLossAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeNLLLossBackward)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.grad.max_pool2d_backward */
 Expr max_pool2d_backward(Expr output_grad, Expr data, ffi::Array<int64_t> pool_size,
@@ -155,7 +159,7 @@ Expr max_pool2d_backward(Expr output_grad, Expr data, ffi::Array<int64_t> pool_s
   attrs->count_include_pad = count_include_pad;
   attrs->layout = layout;
   attrs->out_layout = out_layout.value_or(layout);
-  static const Op& op = Op::Get("relax.grad.max_pool2d_backward");
+  static const Op op = Op::Get("relax.grad.max_pool2d_backward");
   return Call(Type::Missing(), op, {std::move(output_grad), std::move(data)}, Attrs(attrs), {});
 }
 
@@ -168,13 +172,14 @@ Type InferTypeMaxPool2DBackward(const Call& call, const BlockBuilder& ctx) {
   return GetType(call->args[1]);
 }
 
-TVM_REGISTER_OP("relax.grad.max_pool2d_backward")
-    .set_num_inputs(2)
-    .add_argument("output_grad", "Tensor", "The output gradient.")
-    .add_argument("data", "Tensor", "The input tensor")
-    .set_attrs_type<Pool2DAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeMaxPool2DBackward)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.grad.max_pool2d_backward")
+      .add_arg("output_grad", "The output gradient.")
+      .add_arg("data", "The input tensor")
+      .attrs_type<Pool2DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeMaxPool2DBackward)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.grad.avg_pool2d_backward */
 Expr avg_pool2d_backward(Expr output_grad, Expr data, ffi::Array<int64_t> pool_size,
@@ -190,7 +195,7 @@ Expr avg_pool2d_backward(Expr output_grad, Expr data, ffi::Array<int64_t> pool_s
   attrs->count_include_pad = count_include_pad;
   attrs->layout = layout;
   attrs->out_layout = out_layout.value_or(layout);
-  static const Op& op = Op::Get("relax.grad.avg_pool2d_backward");
+  static const Op op = Op::Get("relax.grad.avg_pool2d_backward");
   return Call(Type::Missing(), op, {std::move(output_grad), std::move(data)}, Attrs(attrs), {});
 }
 
@@ -203,13 +208,14 @@ Type InferTypeAvgPool2DBackward(const Call& call, const BlockBuilder& ctx) {
   return GetType(call->args[1]);
 }
 
-TVM_REGISTER_OP("relax.grad.avg_pool2d_backward")
-    .set_num_inputs(2)
-    .add_argument("output_grad", "Tensor", "The output gradient.")
-    .add_argument("data", "Tensor", "The input tensor")
-    .set_attrs_type<Pool2DAttrs>()
-    .set_attr<FInferType>("FInferType", InferTypeAvgPool2DBackward)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.grad.avg_pool2d_backward")
+      .add_arg("output_grad", "The output gradient.")
+      .add_arg("data", "The input tensor")
+      .attrs_type<Pool2DAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeAvgPool2DBackward)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.grad.take_backward */
 
@@ -217,7 +223,7 @@ Expr take_backward(Expr output_grad, Expr x, Expr indices, ffi::Optional<int64_t
   ffi::ObjectPtr<TakeAttrs> attrs = ffi::make_object<TakeAttrs>();
   attrs->axis = std::move(axis);
 
-  static const Op& op = Op::Get("relax.grad.take_backward");
+  static const Op op = Op::Get("relax.grad.take_backward");
   return Call(Type::Missing(), op, {std::move(output_grad), std::move(x), std::move(indices)},
               Attrs(attrs), {});
 }
@@ -231,14 +237,15 @@ Type InferTypeTakeBackward(const Call& call, const BlockBuilder& ctx) {
   return GetType(call->args[1]);
 }
 
-TVM_REGISTER_OP("relax.grad.take_backward")
-    .set_attrs_type<TakeAttrs>()
-    .set_num_inputs(3)
-    .add_argument("output_grad", "Tensor", "The output gradient.")
-    .add_argument("x", "Tensor", "The source tensor.")
-    .add_argument("indices", "Tensor", "The indices of the values to extract.")
-    .set_attr<FInferType>("FInferType", InferTypeTakeBackward)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.grad.take_backward")
+      .add_arg("output_grad", "The output gradient.")
+      .add_arg("x", "The source tensor.")
+      .add_arg("indices", "The indices of the values to extract.")
+      .attrs_type<TakeAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeTakeBackward)
+      .set_attr<bool>("FPurity", true);
+}
 
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/tensor/index.cc b/src/relax/op/tensor/index.cc
index d5aa64ab86..00ea940555 100644
--- a/src/relax/op/tensor/index.cc
+++ b/src/relax/op/tensor/index.cc
@@ -52,7 +52,7 @@ Expr take(Expr x, Expr indices, ffi::Optional<int64_t> axis, ffi::String mode) {
   attrs->axis = std::move(axis);
   attrs->mode = std::move(mode);
 
-  static const Op& op = Op::Get("relax.take");
+  static const Op op = Op::Get("relax.take");
   return Call(Type::Missing(), op, {std::move(x), std::move(indices)}, Attrs(attrs), {});
 }
 
@@ -127,13 +127,14 @@ Type InferTypeTake(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr(output_shape), data_ty->dtype, data_ty->vdevice);
 }
 
-TVM_REGISTER_OP("relax.take")
-    .set_attrs_type<TakeAttrs>()
-    .set_num_inputs(2)
-    .add_argument("x", "Tensor", "The source tensor.")
-    .add_argument("indices", "Tensor", "The indices of the values to extract.")
-    .set_attr<FInferType>("FInferType", InferTypeTake)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.take")
+      .add_arg("x", "The source tensor.")
+      .add_arg("indices", "The indices of the values to extract.")
+      .attrs_type<TakeAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeTake)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.strided_slice */
 
@@ -173,7 +174,7 @@ Expr strided_slice(Expr x, Expr axes, Expr begin, Expr end, ffi::Optional<Expr>
     args.push_back(strides.value());
   }
 
-  static const Op& op = Op::Get("relax.strided_slice");
+  static const Op op = Op::Get("relax.strided_slice");
   auto call = Call(Type::Missing(), op, args, Attrs(attrs));
 
   return call;
@@ -486,21 +487,26 @@ InferLayoutOutput InferLayoutStridedSlice(
                            {{IntImm::Int32(1), relax::Tuple(new_axes)}});
 }
 
-TVM_REGISTER_OP("relax.strided_slice")
-    .set_attrs_type<StridedSliceAttrs>()
-    .set_num_inputs(1)
-    .add_argument("x", "Tensor", "The source tensor to be sliced.")
-    .set_attr<FInferType>("FInferType", InferTypeStridedSlice)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutStridedSlice)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.strided_slice")
+      .add_arg("x", "The source tensor to be sliced.")
+      .add_arg("axes", "The axes.")
+      .add_arg("begin", "The start index.")
+      .add_arg("end", "The end index.")
+      .allow_extra_args()
+      .attrs_type<StridedSliceAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeStridedSlice)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutStridedSlice)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.dynamic_strided_slice */
 Expr dynamic_strided_slice(Expr x,      //
                            Expr begin,  //
                            Expr end,    //
                            Expr strides) {
-  static const Op& op = Op::Get("relax.dynamic_strided_slice");
+  static const Op op = Op::Get("relax.dynamic_strided_slice");
   return Call(Type::Missing(), op,
               {std::move(x), std::move(begin), std::move(end), std::move(strides)}, {});
 }
@@ -583,17 +589,18 @@ InferLayoutOutput InferLayoutDynStridedSlice(
   return InferLayoutOutput({initial}, {initial}, Attrs());
 }
 
-TVM_REGISTER_OP("relax.dynamic_strided_slice")
-    .set_num_inputs(4)
-    .add_argument("x", "Tensor", "The source tensor to be sliced.")
-    .add_argument("begin", "Tensor", "The indices to begin with in the slicing.")
-    .add_argument("end", "Tensor", "Indices indicating end of the slice.")
-    .add_argument("strides", "Tensor", "The stride values.")
-    .set_attr<FInferType>("FInferType", InferTypeDynStridedSlice)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutDynStridedSlice)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true)
-    .set_attr<bool>("FDataDependent", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.dynamic_strided_slice")
+      .add_arg("x", "The source tensor to be sliced.")
+      .add_arg("begin", "The indices to begin with in the slicing.")
+      .add_arg("end", "Indices indicating end of the slice.")
+      .add_arg("strides", "The stride values.")
+      .set_attr<FInferType>("FInferType", InferTypeDynStridedSlice)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutDynStridedSlice)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true)
+      .set_attr<bool>("FDataDependent", true);
+}
 
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/tensor/inspect.cc b/src/relax/op/tensor/inspect.cc
index 73886a51f5..49bf821dec 100644
--- a/src/relax/op/tensor/inspect.cc
+++ b/src/relax/op/tensor/inspect.cc
@@ -117,7 +117,7 @@ Expr NormalizeToKnownPrimExpr(const BlockBuilder&, Call call) { return call; }
 //// relax.tensor_dtype_code
 
 Expr tensor_dtype_code(Expr expr) {
-  static const Op& op = Op::Get("relax.inspect.tensor_dtype_code");
+  static const Op op = Op::Get("relax.inspect.tensor_dtype_code");
   return Call(Type::Missing(), op, {expr});
 }
 
@@ -134,19 +134,20 @@ Expr LegalizeTensorDtypeCode(const BlockBuilder& bb, const Call& call) {
   return Call(Type::Missing(), gvar_getter, {arg});
 }
 
-TVM_REGISTER_OP("relax.inspect.tensor_dtype_code")
-    .set_num_inputs(1)
-    .add_argument("tensor", "Tensor", "The tensor to be inspected")
-    .set_attr<FInferType>("FInferType", InferTypeTensorDtypeCode)
-    .set_attr<FLegalize>("FLegalize", LegalizeTensorDtypeCode)
-    .set_attr<bool>("RequiresArgumentShapes", false)
-    .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.inspect.tensor_dtype_code")
+      .add_arg("tensor", "The tensor to be inspected")
+      .set_attr<FInferType>("FInferType", InferTypeTensorDtypeCode)
+      .set_attr<FLegalize>("FLegalize", LegalizeTensorDtypeCode)
+      .set_attr<bool>("RequiresArgumentShapes", false)
+      .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+      .set_attr<bool>("FPurity", true);
+}
 
 //// relax.tensor_dtype_bits
 
 Expr tensor_dtype_bits(Expr expr) {
-  static const Op& op = Op::Get("relax.inspect.tensor_dtype_bits");
+  static const Op op = Op::Get("relax.inspect.tensor_dtype_bits");
   return Call(Type::Missing(), op, {expr});
 }
 
@@ -163,19 +164,20 @@ Expr LegalizeTensorDtypeBits(const BlockBuilder& bb, const Call& call) {
   return Call(Type::Missing(), gvar_getter, {arg});
 }
 
-TVM_REGISTER_OP("relax.inspect.tensor_dtype_bits")
-    .set_num_inputs(1)
-    .add_argument("tensor", "Tensor", "The tensor to be inspected")
-    .set_attr<FInferType>("FInferType", InferTypeTensorDtypeBits)
-    .set_attr<FLegalize>("FLegalize", LegalizeTensorDtypeBits)
-    .set_attr<bool>("RequiresArgumentShapes", false)
-    .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.inspect.tensor_dtype_bits")
+      .add_arg("tensor", "The tensor to be inspected")
+      .set_attr<FInferType>("FInferType", InferTypeTensorDtypeBits)
+      .set_attr<FLegalize>("FLegalize", LegalizeTensorDtypeBits)
+      .set_attr<bool>("RequiresArgumentShapes", false)
+      .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+      .set_attr<bool>("FPurity", true);
+}
 
 //// relax.tensor_dtype_lanes
 
 Expr tensor_dtype_lanes(Expr expr) {
-  static const Op& op = Op::Get("relax.inspect.tensor_dtype_lanes");
+  static const Op op = Op::Get("relax.inspect.tensor_dtype_lanes");
   return Call(Type::Missing(), op, {expr});
 }
 
@@ -192,19 +194,20 @@ Expr LegalizeTensorDtypeLanes(const BlockBuilder& bb, const Call& call) {
   return Call(Type::Missing(), gvar_getter, {arg});
 }
 
-TVM_REGISTER_OP("relax.inspect.tensor_dtype_lanes")
-    .set_num_inputs(1)
-    .add_argument("tensor", "Tensor", "The tensor to be inspected")
-    .set_attr<FInferType>("FInferType", InferTypeTensorDtypeLanes)
-    .set_attr<FLegalize>("FLegalize", LegalizeTensorDtypeLanes)
-    .set_attr<bool>("RequiresArgumentShapes", false)
-    .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.inspect.tensor_dtype_lanes")
+      .add_arg("tensor", "The tensor to be inspected")
+      .set_attr<FInferType>("FInferType", InferTypeTensorDtypeLanes)
+      .set_attr<FLegalize>("FLegalize", LegalizeTensorDtypeLanes)
+      .set_attr<bool>("RequiresArgumentShapes", false)
+      .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+      .set_attr<bool>("FPurity", true);
+}
 
 //// relax.tensor_ndim
 
 Expr tensor_ndim(Expr expr) {
-  static const Op& op = Op::Get("relax.inspect.tensor_ndim");
+  static const Op op = Op::Get("relax.inspect.tensor_ndim");
   return Call(Type::Missing(), op, {expr});
 }
 
@@ -221,19 +224,20 @@ Expr LegalizeTensorNDim(const BlockBuilder& bb, const Call& call) {
   return Call(Type::Missing(), gvar_getter, {arg});
 }
 
-TVM_REGISTER_OP("relax.inspect.tensor_ndim")
-    .set_num_inputs(1)
-    .add_argument("tensor", "Tensor", "The tensor to be inspected")
-    .set_attr<FInferType>("FInferType", InferTypeTensorNDim)
-    .set_attr<FLegalize>("FLegalize", LegalizeTensorNDim)
-    .set_attr<bool>("RequiresArgumentShapes", false)
-    .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.inspect.tensor_ndim")
+      .add_arg("tensor", "The tensor to be inspected")
+      .set_attr<FInferType>("FInferType", InferTypeTensorNDim)
+      .set_attr<FLegalize>("FLegalize", LegalizeTensorNDim)
+      .set_attr<bool>("RequiresArgumentShapes", false)
+      .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+      .set_attr<bool>("FPurity", true);
+}
 
 //// relax.tensor_shape_i
 
 Expr tensor_shape_i(Expr expr) {
-  static const Op& op = Op::Get("relax.inspect.tensor_shape_i");
+  static const Op op = Op::Get("relax.inspect.tensor_shape_i");
   return Call(Type::Missing(), op, {expr});
 }
 
@@ -300,20 +304,21 @@ Expr LegalizeTensorShape(const BlockBuilder& bb, const Call& call) {
   return Call(Type::Missing(), gvar_getter, call->args);
 }
 
-TVM_REGISTER_OP("relax.inspect.tensor_shape_i")
-    .set_num_inputs(2)
-    .add_argument("tensor", "Tensor", "The tensor to be inspected")
-    .add_argument("axis", "Prim(int64)", "The axis whose extent should be returned")
-    .set_attr<FInferType>("FInferType", InferTypeTensorShape)
-    .set_attr<FLegalize>("FLegalize", LegalizeTensorShape)
-    .set_attr<bool>("RequiresArgumentShapes", false)
-    .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.inspect.tensor_shape_i")
+      .add_arg("tensor", "The tensor to be inspected")
+      .add_arg("axis", "The axis whose extent should be returned")
+      .set_attr<FInferType>("FInferType", InferTypeTensorShape)
+      .set_attr<FLegalize>("FLegalize", LegalizeTensorShape)
+      .set_attr<bool>("RequiresArgumentShapes", false)
+      .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+      .set_attr<bool>("FPurity", true);
+}
 
 //// relax.tensor_stride_i
 
 Expr tensor_stride_i(Expr expr) {
-  static const Op& op = Op::Get("relax.inspect.tensor_stride_i");
+  static const Op op = Op::Get("relax.inspect.tensor_stride_i");
   return Call(Type::Missing(), op, {expr});
 }
 
@@ -348,19 +353,20 @@ Type InferTypeTensorStride(const Call& call, const BlockBuilder&) {
   }
 }
 
-TVM_REGISTER_OP("relax.inspect.tensor_stride_i")
-    .set_num_inputs(2)
-    .add_argument("tensor", "Tensor", "The tensor to be inspected")
-    .add_argument("axis", "Prim(int64)", "The axis whose extent should be returned")
-    .set_attr<FInferType>("FInferType", InferTypeTensorStride)
-    .set_attr<bool>("RequiresArgumentShapes", false)
-    .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.inspect.tensor_stride_i")
+      .add_arg("tensor", "The tensor to be inspected")
+      .add_arg("axis", "The axis whose extent should be returned")
+      .set_attr<FInferType>("FInferType", InferTypeTensorStride)
+      .set_attr<bool>("RequiresArgumentShapes", false)
+      .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+      .set_attr<bool>("FPurity", true);
+}
 
 //// relax.tensor_byte_offset
 
 Expr tensor_byte_offset(Expr expr) {
-  static const Op& op = Op::Get("relax.inspect.tensor_byte_offset");
+  static const Op op = Op::Get("relax.inspect.tensor_byte_offset");
   return Call(Type::Missing(), op, {expr});
 }
 
@@ -380,18 +386,19 @@ Type InferTypeTensorByteOffset(const Call& call, const BlockBuilder&) {
   }
 }
 
-TVM_REGISTER_OP("relax.inspect.tensor_byte_offset")
-    .set_num_inputs(1)
-    .add_argument("tensor", "Tensor", "The tensor to be inspected")
-    .set_attr<FInferType>("FInferType", InferTypeTensorByteOffset)
-    .set_attr<bool>("RequiresArgumentShapes", false)
-    .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.inspect.tensor_byte_offset")
+      .add_arg("tensor", "The tensor to be inspected")
+      .set_attr<FInferType>("FInferType", InferTypeTensorByteOffset)
+      .set_attr<bool>("RequiresArgumentShapes", false)
+      .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+      .set_attr<bool>("FPurity", true);
+}
 
 //// relax.tensor_elem_offset
 
 Expr tensor_elem_offset(Expr expr) {
-  static const Op& op = Op::Get("relax.inspect.tensor_elem_offset");
+  static const Op op = Op::Get("relax.inspect.tensor_elem_offset");
   return Call(Type::Missing(), op, {expr});
 }
 
@@ -411,13 +418,14 @@ Type InferTypeTensorElemOffset(const Call& call, const BlockBuilder&) {
   }
 }
 
-TVM_REGISTER_OP("relax.inspect.tensor_elem_offset")
-    .set_num_inputs(1)
-    .add_argument("tensor", "Tensor", "The tensor to be inspected")
-    .set_attr<FInferType>("FInferType", InferTypeTensorElemOffset)
-    .set_attr<bool>("RequiresArgumentShapes", false)
-    .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.inspect.tensor_elem_offset")
+      .add_arg("tensor", "The tensor to be inspected")
+      .set_attr<FInferType>("FInferType", InferTypeTensorElemOffset)
+      .set_attr<bool>("RequiresArgumentShapes", false)
+      .set_attr<FNormalize>("FNormalize", NormalizeToKnownPrimExpr)
+      .set_attr<bool>("FPurity", true);
+}
 
 }  // namespace inspect
 }  // namespace relax
diff --git a/src/relax/op/tensor/linear_algebra.cc b/src/relax/op/tensor/linear_algebra.cc
index 3a544a3cea..e9bf495c1b 100644
--- a/src/relax/op/tensor/linear_algebra.cc
+++ b/src/relax/op/tensor/linear_algebra.cc
@@ -46,7 +46,7 @@ Expr matmul(Expr x1, Expr x2, ffi::Optional<DLDataType> out_dtype) {
   ffi::ObjectPtr<MatmulAttrs> attrs = ffi::make_object<MatmulAttrs>();
   attrs->out_dtype = out_dtype;
 
-  static const Op& op = Op::Get("relax.matmul");
+  static const Op op = Op::Get("relax.matmul");
   return Call(Type::Missing(), op, {std::move(x1), std::move(x2)}, Attrs(attrs), {});
 }
 
@@ -162,14 +162,15 @@ Call InferMixedPrecisionMatmul(const Call& call, DLDataType out_dtype) {
   return matmul(call->args[0], call->args[1], out_dtype).as_or_throw<Call>();
 }
 
-TVM_REGISTER_OP("relax.matmul")
-    .set_num_inputs(2)
-    .add_argument("x1", "Tensor", "The first input tensor.")
-    .add_argument("x2", "Tensor", "The second input tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeMatmul)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
-    .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionMatmul)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.matmul")
+      .add_arg("x1", "The first input tensor.")
+      .add_arg("x2", "The second input tensor.")
+      .set_attr<FInferType>("FInferType", InferTypeMatmul)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+      .set_attr<FInferMixedPrecision>("FInferMixedPrecision", InferMixedPrecisionMatmul)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.einsum */
 
@@ -177,7 +178,7 @@ Expr einsum(Expr operands, ffi::String subscripts) {
   ffi::ObjectPtr<EinsumAttrs> attrs = ffi::make_object<EinsumAttrs>();
   attrs->subscripts = std::move(subscripts);
 
-  static const Op& op = Op::Get("relax.einsum");
+  static const Op op = Op::Get("relax.einsum");
   return Call(Type::Missing(), op, {std::move(operands)}, Attrs{attrs}, {});
 }
 
@@ -251,17 +252,18 @@ Type InferTypeEinsum(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr(oshape), operand_ty);
 }
 
-TVM_REGISTER_OP("relax.einsum")
-    .set_attrs_type<EinsumAttrs>()
-    .set_num_inputs(1)
-    .add_argument("operands", "Tensor", "The input tensors.")
-    .set_attr<FInferType>("FInferType", InferTypeEinsum)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.einsum")
+      .add_arg("operands", "The input tensors.")
+      .attrs_type<EinsumAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeEinsum)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.outer */
 
 Expr outer(Expr x1, Expr x2) {
-  static const Op& op = Op::Get("relax.outer");
+  static const Op op = Op::Get("relax.outer");
   return Call(Type::Missing(), op, {std::move(x1), std::move(x2)}, {});
 }
 
@@ -290,13 +292,14 @@ Type InferTypeOuter(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr(output_shape), x1_ty->dtype);
 }
 
-TVM_REGISTER_OP("relax.outer")
-    .set_num_inputs(2)
-    .add_argument("x1", "Tensor", "The first input tensor.")
-    .add_argument("x2", "Tensor", "The second input tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeOuter)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.outer")
+      .add_arg("x1", "The first input tensor.")
+      .add_arg("x2", "The second input tensor.")
+      .set_attr<FInferType>("FInferType", InferTypeOuter)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kAlways)
+      .set_attr<bool>("FPurity", true);
+}
 
 }  // namespace relax
 }  // namespace tvm
diff --git a/src/relax/op/tensor/manipulate.cc b/src/relax/op/tensor/manipulate.cc
index 847cf1f344..64d8d62c75 100644
--- a/src/relax/op/tensor/manipulate.cc
+++ b/src/relax/op/tensor/manipulate.cc
@@ -64,7 +64,7 @@ TVM_FFI_STATIC_INIT_BLOCK() {
 
 /* relax.broadcast_to */
 Expr broadcast_to(Expr x, Expr shape) {
-  static const Op& op = Op::Get("relax.broadcast_to");
+  static const Op op = Op::Get("relax.broadcast_to");
   return Call(Type::Missing(), op, {std::move(x), std::move(shape)}, Attrs(), {});
 }
 
@@ -131,13 +131,14 @@ Type InferTypeBroadcastTo(const Call& call, const BlockBuilder& ctx) {
   return TensorType(/*shape=*/call->args[1], data_ty->dtype, data_ty->vdevice);
 }
 
-TVM_REGISTER_OP("relax.broadcast_to")
-    .set_num_inputs(2)
-    .add_argument("x", "Tensor", "The input tensor.")
-    .add_argument("shape", "Shape", "The target shape.")
-    .set_attr<FInferType>("FInferType", InferTypeBroadcastTo)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.broadcast_to")
+      .add_arg("x", "The input tensor.")
+      .add_arg("shape", "The target shape.")
+      .set_attr<FInferType>("FInferType", InferTypeBroadcastTo)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.concat */
 
@@ -145,7 +146,7 @@ Expr concat(Expr tensors, ffi::Optional<int64_t> axis) {
   ffi::ObjectPtr<ConcatAttrs> attrs = ffi::make_object<ConcatAttrs>();
   attrs->axis = std::move(axis);
 
-  static const Op& op = Op::Get("relax.concat");
+  static const Op op = Op::Get("relax.concat");
   return Call(Type::Missing(), op, {std::move(tensors)}, Attrs(attrs), {});
 }
 
@@ -390,14 +391,15 @@ InferLayoutOutput InferLayoutConcat(
   return InferLayoutOutput({NLayout(input_layouts)}, output_layouts, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.concat")
-    .set_attrs_type<ConcatAttrs>()
-    .set_num_inputs(1)
-    .add_argument("tensors", "Tuple of Tensors", "The input list of tensors.")
-    .set_attr<FInferType>("FInferType", InferTypeConcat)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConcat)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.concat")
+      .add_arg("tensors", "The input list of tensors.")
+      .attrs_type<ConcatAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeConcat)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutConcat)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.expand_dims */
 
@@ -405,7 +407,7 @@ Expr expand_dims(Expr x, ffi::Array<int64_t> axis) {
   ffi::ObjectPtr<ExpandDimsAttrs> attrs = ffi::make_object<ExpandDimsAttrs>();
   attrs->axis = std::move(axis);
 
-  static const Op& op = Op::Get("relax.expand_dims");
+  static const Op op = Op::Get("relax.expand_dims");
   return Call(Type::Missing(), op, {std::move(x)}, Attrs{attrs}, {});
 }
 
@@ -494,14 +496,15 @@ InferLayoutOutput InferLayoutExpandDims(
                            Attrs(call->attrs));
 }
 
-TVM_REGISTER_OP("relax.expand_dims")
-    .set_num_inputs(1)
-    .set_attrs_type<ExpandDimsAttrs>()
-    .add_argument("x", "Tensor", "The input tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeExpandDims)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutExpandDims)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.expand_dims")
+      .add_arg("x", "The input tensor.")
+      .attrs_type<ExpandDimsAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeExpandDims)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutExpandDims)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 // Helper function for flatten and reshape.
 PrimExpr ComputeShapeProduct(const ffi::Array<PrimExpr>& shape_values) {
@@ -514,7 +517,7 @@ PrimExpr ComputeShapeProduct(const ffi::Array<PrimExpr>& shape_values) {
 
 /* relax.flatten */
 Expr flatten(Expr x) {
-  static const Op& op = Op::Get("relax.flatten");
+  static const Op op = Op::Get("relax.flatten");
   return Call(Type::Missing(), op, {std::move(x)}, {}, {});
 }
 
@@ -541,17 +544,18 @@ Type InferTypeFlatten(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr({std::move(shape_prod)}), data_ty->dtype, data_ty->vdevice);
 }
 
-TVM_REGISTER_OP("relax.flatten")
-    .set_num_inputs(1)
-    .add_argument("x", "Tensor", "The input tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeFlatten)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.flatten")
+      .add_arg("x", "The input tensor.")
+      .set_attr<FInferType>("FInferType", InferTypeFlatten)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.index_tensor */
 
 Expr index_tensor(Expr first, Expr tensors) {
-  static const Op& op = Op::Get("relax.index_tensor");
+  static const Op op = Op::Get("relax.index_tensor");
   return Call(Type::Missing(), op, {std::move(first), std::move(tensors)}, Attrs(), {});
 }
 
@@ -690,12 +694,13 @@ Type InferTypeIndexTensor(const Call& call, const BlockBuilder& ctx) {
   return TensorType(output_dtype, out_ndim, vdev);
 }
 
-TVM_REGISTER_OP("relax.index_tensor")
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "The input data.")
-    .add_argument("indices", "List of Tensors", "The indices used to index.")
-    .set_attr<FInferType>("FInferType", InferTypeIndexTensor)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.index_tensor")
+      .add_arg("data", "The input data.")
+      .add_arg("indices", "The indices used to index.")
+      .set_attr<FInferType>("FInferType", InferTypeIndexTensor)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.layout_transform */
 
@@ -704,7 +709,7 @@ Expr layout_transform(Expr x, tirx::IndexMap index_map, ffi::Optional<PrimExpr>
   attrs->index_map = std::move(index_map);
   attrs->pad_value = std::move(pad_value);
 
-  static const Op& op = Op::Get("relax.layout_transform");
+  static const Op op = Op::Get("relax.layout_transform");
   return Call(Type::Missing(), op, {std::move(x)}, Attrs{attrs}, {});
 }
 
@@ -757,13 +762,14 @@ Type InferTypeLayoutTransform(const Call& call, const BlockBuilder& ctx) {
   return TensorType(ShapeExpr(output_shape), data_ty->dtype, data_ty->vdevice);
 }
 
-TVM_REGISTER_OP("relax.layout_transform")
-    .set_num_inputs(1)
-    .set_attrs_type<LayoutTransformAttrs>()
-    .add_argument("x", "Tensor", "The input tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeLayoutTransform)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.layout_transform")
+      .add_arg("x", "The input tensor.")
+      .attrs_type<LayoutTransformAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeLayoutTransform)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.permute_dims */
 
@@ -771,7 +777,7 @@ Expr permute_dims(Expr x, ffi::Optional<ffi::Array<int64_t>> axes) {
   ffi::ObjectPtr<PermuteDimsAttrs> attrs = ffi::make_object<PermuteDimsAttrs>();
   attrs->axes = std::move(axes);
 
-  static const Op& op = Op::Get("relax.permute_dims");
+  static const Op op = Op::Get("relax.permute_dims");
   return Call(Type::Missing(), op, {std::move(x)}, Attrs{attrs}, {});
 }
 
@@ -877,14 +883,15 @@ InferLayoutOutput InferLayoutPermuteDims(
   return InferLayoutOutput({existing_layout}, {InitialLayoutDecision(ndim)}, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.permute_dims")
-    .set_attrs_type<PermuteDimsAttrs>()
-    .set_num_inputs(1)
-    .add_argument("x", "Tensor", "The input tensor.")
-    .set_attr<FInferType>("FInferType", InferTypePermuteDims)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPermuteDims)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.permute_dims")
+      .add_arg("x", "The input tensor.")
+      .attrs_type<PermuteDimsAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypePermuteDims)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutPermuteDims)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.reshape */
 Expr ConvertNewShapeToExpr(const Expr& data,
@@ -990,7 +997,7 @@ Expr ConvertNewShapeToExpr(const Expr& data,
 
 Expr reshape(Expr x, ffi::Variant<Expr, ffi::Array<PrimExpr>> shape) {
   Expr shape_in_expr = ConvertNewShapeToExpr(x, shape);
-  static const Op& op = Op::Get("relax.reshape");
+  static const Op op = Op::Get("relax.reshape");
   return Call(Type::Missing(), op, {std::move(x), std::move(shape_in_expr)}, Attrs(), {});
 }
 
@@ -1042,13 +1049,14 @@ Type InferTypeReshape(const Call& call, const BlockBuilder& ctx) {
   return TensorType(target_shape, data_ty->dtype, data_ty->vdevice);
 }
 
-TVM_REGISTER_OP("relax.reshape")
-    .set_num_inputs(2)
-    .add_argument("x", "Tensor", "The input tensor.")
-    .add_argument("shape", "Shape", "The input new shape.")
-    .set_attr<FInferType>("FInferType", InferTypeReshape)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.reshape")
+      .add_arg("x", "The input tensor.")
+      .add_arg("shape", "The input new shape.")
+      .set_attr<FInferType>("FInferType", InferTypeReshape)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.split */
 
@@ -1080,7 +1088,7 @@ Expr split(Expr x, ffi::Variant<IntImm, ffi::Array<IntImm>> indices_or_sections,
   attrs->indices_or_sections = indices_or_sections_obj;
   attrs->axis = axis;
 
-  static const Op& op = Op::Get("relax.split");
+  static const Op op = Op::Get("relax.split");
   return Call(Type::Missing(), op, {std::move(x)}, Attrs(attrs), {});
 }
 
@@ -1215,14 +1223,15 @@ InferLayoutOutput InferLayoutSplit(
   return InferLayoutOutput({existing_layout}, {tuple_layouts}, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.split")
-    .set_attrs_type<SplitAttrs>()
-    .set_num_inputs(1)
-    .add_argument("x", "Tensor", "The input tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeSplit)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutSplit)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.split")
+      .add_arg("x", "The input tensor.")
+      .attrs_type<SplitAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeSplit)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutSplit)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.squeeze */
 
@@ -1230,7 +1239,7 @@ Expr squeeze(Expr x, ffi::Optional<ffi::Array<int64_t>> axis) {
   ffi::ObjectPtr<SqueezeAttrs> attrs = ffi::make_object<SqueezeAttrs>();
   attrs->axis = std::move(axis);
 
-  static const Op& op = Op::Get("relax.squeeze");
+  static const Op op = Op::Get("relax.squeeze");
   return Call(Type::Missing(), op, {std::move(x)}, Attrs{attrs}, {});
 }
 
@@ -1374,14 +1383,15 @@ InferLayoutOutput InferLayoutSqueeze(
                            Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.squeeze")
-    .set_num_inputs(1)
-    .set_attrs_type<SqueezeAttrs>()
-    .add_argument("x", "Tensor", "The input tensor.")
-    .set_attr<FInferType>("FInferType", InferTypeSqueeze)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutSqueeze)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.squeeze")
+      .add_arg("x", "The input tensor.")
+      .attrs_type<SqueezeAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeSqueeze)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutSqueeze)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 void CheckCollapseShape(const Call& call, const BlockBuilder& ctx,
                         const ffi::Array<PrimExpr>& data_shape,
@@ -1428,7 +1438,7 @@ Expr stack(Expr tensors, ffi::Optional<int64_t> axis) {
   ffi::ObjectPtr<StackAttrs> attrs = ffi::make_object<StackAttrs>();
   attrs->axis = std::move(axis);
 
-  static const Op& op = Op::Get("relax.stack");
+  static const Op op = Op::Get("relax.stack");
   return Call(Type::Missing(), op, {std::move(tensors)}, Attrs(attrs), {});
 }
 
@@ -1626,18 +1636,19 @@ InferLayoutOutput InferLayoutStack(
   return InferLayoutOutput({NLayout(input_layouts)}, output_layouts, Attrs(new_attrs));
 }
 
-TVM_REGISTER_OP("relax.stack")
-    .set_attrs_type<StackAttrs>()
-    .set_num_inputs(1)
-    .add_argument("tensors", "Tuple of Tensors", "The input list of tensors to stack")
-    .set_attr<FInferType>("FInferType", InferTypeStack)
-    .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutStack)
-    .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.stack")
+      .add_arg("tensors", "The input list of tensors to stack")
+      .attrs_type<StackAttrs>()
+      .set_attr<FInferType>("FInferType", InferTypeStack)
+      .set_attr<FRelaxInferLayout>("FRelaxInferLayout", InferLayoutStack)
+      .set_attr<TMixedPrecisionPolicy>("TMixedPrecisionPolicy", MixedPrecisionPolicyKind::kFollow)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.collapse_sum_like */
 Expr collapse_sum_like(Expr data, Expr collapse_target) {
-  static const Op& op = Op::Get("relax.collapse_sum_like");
+  static const Op op = Op::Get("relax.collapse_sum_like");
   return Call(Type::Missing(), op, {std::move(data), std::move(collapse_target)}, Attrs(), {});
 }
 
@@ -1674,17 +1685,17 @@ Type InferTypeCollapseSumLike(const Call& call, const BlockBuilder& ctx) {
   }
 }
 
-TVM_REGISTER_OP("relax.collapse_sum_like")
-    .set_num_inputs(2)
-    .add_argument("data", "Tensor", "The input tensor.")
-    .add_argument("collapse_target", "Tensor",
-                  "The tensor whose shape is the shape to collapse to.")
-    .set_attr<FInferType>("FInferType", InferTypeCollapseSumLike)
-    .set_attr<bool>("FPurity", true);
+TVM_FFI_STATIC_INIT_BLOCK() {
+  OpDef("relax.collapse_sum_like")
+      .add_arg("data", "The input tensor.")
+      .add_arg("collapse_target", "The tensor whose shape is the shape to collapse to.")
+      .set_attr<FInferType>("FInferType", InferTypeCollapseSumLike)
+      .set_attr<bool>("FPurity", true);
+}
 
 /* relax.collapse_sum_to */
 Expr collapse_sum_to(Expr data, Expr shape) {
... 5539 lines suppressed ...