You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by ha...@apache.org on 2020/03/12 03:47:30 UTC

[incubator-tvm] branch master updated: [refactor][relay pass] Separate analysis and transform passes (#5035)

This is an automated email from the ASF dual-hosted git repository.

haichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-tvm.git


The following commit(s) were added to refs/heads/master by this push:
     new 14ba49c  [refactor][relay pass] Separate analysis and transform passes (#5035)
14ba49c is described below

commit 14ba49c60c49474a564f990363de9d114c9b019b
Author: Zhi <51...@users.noreply.github.com>
AuthorDate: Wed Mar 11 20:47:21 2020 -0700

    [refactor][relay pass] Separate analysis and transform passes (#5035)
    
    * [refactor][relay pass] Separate analysis and transform passes into different subfolders
    
    * remove pass folder
---
 CMakeLists.txt                                              |  4 +++-
 python/tvm/relay/analysis.py                                |  9 ++++-----
 src/printer/relay_text_printer.cc                           |  2 +-
 src/relay/{ir => analysis}/alpha_equal.cc                   | 11 +++++------
 src/relay/{pass => analysis}/call_graph.cc                  |  2 +-
 src/relay/{pass => analysis}/call_graph.h                   |  8 ++++----
 src/relay/{pass => analysis}/dependency_graph.cc            |  4 ++--
 src/relay/{pass => analysis}/dependency_graph.h             | 10 +++++-----
 src/relay/{pass => analysis}/feature.cc                     |  2 +-
 src/relay/{pass => analysis}/kind_check.cc                  |  0
 src/relay/{pass => analysis}/mac_count.cc                   |  2 +-
 src/relay/{pass => analysis}/match_exhaustion.cc            |  0
 src/relay/{pass => analysis}/type_solver.cc                 |  0
 src/relay/{pass => analysis}/type_solver.h                  |  7 +++----
 src/relay/{pass => analysis}/util.cc                        |  2 +-
 src/relay/{pass => analysis}/well_formed.cc                 |  0
 src/relay/backend/vm/compiler.cc                            |  2 +-
 src/relay/backend/vm/compiler.h                             |  2 +-
 src/relay/op/annotation/annotation.cc                       |  2 +-
 src/relay/op/device_copy.cc                                 |  2 +-
 src/relay/op/memory/memory.cc                               |  2 +-
 src/relay/op/nn/bitserial.cc                                |  2 +-
 src/relay/op/nn/convolution.cc                              |  2 +-
 src/relay/op/nn/nn.cc                                       |  2 +-
 src/relay/op/nn/pooling.cc                                  |  2 +-
 src/relay/op/nn/sparse.cc                                   |  2 +-
 src/relay/op/op_common.h                                    |  2 +-
 src/relay/op/tensor/transform.cc                            |  4 ++--
 src/relay/qnn/op/add.cc                                     |  2 +-
 src/relay/qnn/op/concatenate.cc                             |  2 +-
 src/relay/qnn/op/convolution.cc                             |  2 +-
 src/relay/qnn/op/dense.cc                                   |  2 +-
 src/relay/qnn/op/dequantize.cc                              |  2 +-
 src/relay/qnn/op/mul.cc                                     |  2 +-
 src/relay/qnn/op/quantize.cc                                |  2 +-
 src/relay/qnn/op/requantize.cc                              |  2 +-
 src/relay/qnn/util.cc                                       |  2 +-
 src/relay/{pass => }/quantize/annotate.cc                   |  0
 src/relay/{pass => }/quantize/calibrate.cc                  |  0
 src/relay/{pass => }/quantize/partition.cc                  |  2 +-
 src/relay/{pass => }/quantize/quantize.cc                   |  0
 src/relay/{pass => }/quantize/quantize.h                    | 10 +++++-----
 src/relay/{pass => }/quantize/realize.cc                    |  4 ++--
 src/relay/{pass => transforms}/alter_op_layout.cc           |  0
 src/relay/{pass => transforms}/annotate_target.cc           |  2 +-
 src/relay/{pass => transforms}/canonicalize_cast.cc         |  2 +-
 src/relay/{pass => transforms}/canonicalize_ops.cc          |  0
 src/relay/{pass => transforms}/combine_parallel_conv2d.cc   |  2 +-
 src/relay/{pass => transforms}/combine_parallel_dense.cc    |  2 +-
 src/relay/{pass => transforms}/combine_parallel_op.cc       |  0
 src/relay/{pass => transforms}/combine_parallel_op.h        |  8 ++++----
 src/relay/{pass => transforms}/combine_parallel_op_batch.cc |  2 +-
 src/relay/{pass => transforms}/combine_parallel_op_batch.h  | 10 +++++-----
 src/relay/{pass => transforms}/convert_layout.cc            |  0
 src/relay/{pass => transforms}/de_duplicate.cc              |  0
 src/relay/{pass => transforms}/dead_code.cc                 |  0
 src/relay/{pass => transforms}/device_annotation.cc         |  5 +----
 src/relay/{pass => transforms}/eliminate_common_subexpr.cc  |  2 +-
 src/relay/{pass => transforms}/eta_expand.cc                |  0
 src/relay/{pass => transforms}/expr_subst.cc                |  0
 src/relay/{pass => transforms}/expr_subst.h                 |  6 +++---
 src/relay/{pass => transforms}/fast_math.cc                 |  0
 src/relay/{pass => transforms}/fold_constant.cc             |  0
 src/relay/{pass => transforms}/fold_scale_axis.cc           |  0
 src/relay/{pass => transforms}/forward_rewrite.cc           |  0
 src/relay/{pass => transforms}/fuse_ops.cc                  |  5 ++---
 src/relay/{pass => transforms}/gradient.cc                  |  0
 src/relay/{pass => transforms}/infer_layout_util.h          |  6 +++---
 src/relay/{pass => transforms}/inline.cc                    |  4 ++--
 src/relay/{pass => transforms}/legalize.cc                  |  0
 src/relay/{pass => transforms}/let_list.h                   |  6 +++---
 src/relay/{pass => transforms}/merge_composite.cc           |  2 +-
 src/relay/{pass => transforms}/partial_eval.cc              |  0
 src/relay/{pass => transforms}/partition_graph.cc           |  2 +-
 src/relay/{pass => transforms}/pass_util.h                  |  8 ++++----
 src/relay/{pass => transforms}/pattern_util.h               |  8 ++++----
 src/relay/{pass => transforms}/print_ir.cc                  |  2 +-
 src/relay/{pass => transforms}/simplify_inference.cc        |  2 +-
 src/relay/{pass => transforms}/to_a_normal_form.cc          |  4 ++--
 src/relay/{pass => transforms}/to_cps.cc                    |  0
 src/relay/{pass => transforms}/to_graph_normal_form.cc      |  0
 src/relay/{pass => transforms}/transform_layout.h           |  6 +++---
 src/relay/{pass => transforms}/type_infer.cc                |  4 ++--
 83 files changed, 109 insertions(+), 114 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index dd59d2d..dafef18 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -146,7 +146,9 @@ file(GLOB_RECURSE RELAY_OP_SRCS
     src/relay/op/*.cc
     )
 file(GLOB_RECURSE RELAY_PASS_SRCS
-    src/relay/pass/*.cc
+    src/relay/analysis/*.cc
+    src/relay/transforms/*.cc
+    src/relay/quantize/*.cc
     )
 file(GLOB RELAY_BACKEND_SRCS
     src/relay/backend/*.cc
diff --git a/python/tvm/relay/analysis.py b/python/tvm/relay/analysis.py
index d0c1724..fc4a037 100644
--- a/python/tvm/relay/analysis.py
+++ b/python/tvm/relay/analysis.py
@@ -23,7 +23,6 @@ configuring the passes and scripting them in Python.
 from tvm.ir import RelayExpr, IRModule
 
 from . import _analysis
-from . import _make
 from .ty import Type
 from .feature import Feature
 
@@ -237,7 +236,7 @@ def alpha_equal(lhs, rhs):
     result : bool
         True iff lhs is alpha equal to rhs.
     """
-    return bool(_make._alpha_equal(lhs, rhs))
+    return bool(_analysis._alpha_equal(lhs, rhs))
 
 
 def assert_alpha_equal(lhs, rhs):
@@ -251,7 +250,7 @@ def assert_alpha_equal(lhs, rhs):
     rhs : tvm.relay.Expr
         One of the input Expression.
     """
-    _make._assert_alpha_equal(lhs, rhs)
+    _analysis._assert_alpha_equal(lhs, rhs)
 
 
 def graph_equal(lhs, rhs):
@@ -273,7 +272,7 @@ def graph_equal(lhs, rhs):
     result : bool
       True iff lhs is data-flow equivalent to rhs.
     """
-    return bool(_make._graph_equal(lhs, rhs))
+    return bool(_analysis._graph_equal(lhs, rhs))
 
 
 def assert_graph_equal(lhs, rhs):
@@ -290,7 +289,7 @@ def assert_graph_equal(lhs, rhs):
     rhs : tvm.relay.Expr
       One of the input Expression.
     """
-    _make._assert_graph_equal(lhs, rhs)
+    _analysis._assert_graph_equal(lhs, rhs)
 
 
 def collect_device_info(expr):
diff --git a/src/printer/relay_text_printer.cc b/src/printer/relay_text_printer.cc
index 86362e0..2799be0 100644
--- a/src/printer/relay_text_printer.cc
+++ b/src/printer/relay_text_printer.cc
@@ -38,7 +38,7 @@
 #include <tvm/relay/pattern_functor.h>
 #include "doc.h"
 #include "meta_data.h"
-#include "../relay/pass/dependency_graph.h"
+#include "../relay/analysis/dependency_graph.h"
 #include "../ir/attr_functor.h"
 
 namespace tvm {
diff --git a/src/relay/ir/alpha_equal.cc b/src/relay/analysis/alpha_equal.cc
similarity index 98%
rename from src/relay/ir/alpha_equal.cc
rename to src/relay/analysis/alpha_equal.cc
index c622599..8a07a19 100644
--- a/src/relay/ir/alpha_equal.cc
+++ b/src/relay/analysis/alpha_equal.cc
@@ -18,7 +18,7 @@
  */
 
 /*!
- * \file src/tvm/relay/ir/alpha_equal.cc
+ * \file src/relay/analysis/alpha_equal.cc
  * \brief Alpha equality check by deep comparing two nodes.
  */
 #include <tvm/ir/type_functor.h>
@@ -593,8 +593,7 @@ bool AlphaEqual(const Expr& lhs, const Expr& rhs) {
   return AlphaEqualHandler(false, false).ExprEqual(lhs, rhs);
 }
 
-// TODO(@jroesch): move to correct namespace?
-TVM_REGISTER_GLOBAL("relay._make._alpha_equal")
+TVM_REGISTER_GLOBAL("relay._analysis._alpha_equal")
 .set_body_typed([](ObjectRef a, ObjectRef b) {
   return AlphaEqualHandler(false, false).Equal(a, b);
 });
@@ -604,18 +603,18 @@ TVM_REGISTER_GLOBAL("ir.type_alpha_equal")
   return AlphaEqual(a, b);
 });
 
-TVM_REGISTER_GLOBAL("relay._make._assert_alpha_equal")
+TVM_REGISTER_GLOBAL("relay._analysis._assert_alpha_equal")
 .set_body_typed([](ObjectRef a, ObjectRef b) {
   bool alpha_equal = AlphaEqualHandler(false, true).Equal(a, b);
   CHECK(alpha_equal) << AsText(a, true) << " and " << AsText(b, true) << " are not alpha equal";
 });
 
-TVM_REGISTER_GLOBAL("relay._make._graph_equal")
+TVM_REGISTER_GLOBAL("relay._analysis._graph_equal")
 .set_body_typed([](ObjectRef a, ObjectRef b) {
   return AlphaEqualHandler(true, false).Equal(a, b);
 });
 
-TVM_REGISTER_GLOBAL("relay._make._assert_graph_equal")
+TVM_REGISTER_GLOBAL("relay._analysis._assert_graph_equal")
 .set_body_typed([](ObjectRef a, ObjectRef b) {
   bool graph_equal = AlphaEqualHandler(true, true).Equal(a, b);
   CHECK(graph_equal) << AsText(a, true) << " and " << AsText(b, true) << " are not graph equal";
diff --git a/src/relay/pass/call_graph.cc b/src/relay/analysis/call_graph.cc
similarity index 99%
rename from src/relay/pass/call_graph.cc
rename to src/relay/analysis/call_graph.cc
index 9c612eb..b9ee8e1 100644
--- a/src/relay/pass/call_graph.cc
+++ b/src/relay/analysis/call_graph.cc
@@ -18,7 +18,7 @@
  */
 
 /*!
- * \file tvm/relay/pass/call_graph.cc
+ * \file src/relay/analysis/call_graph.cc
  * \brief Implementation of APIs to handle the call graph of a Relay module.
  */
 
diff --git a/src/relay/pass/call_graph.h b/src/relay/analysis/call_graph.h
similarity index 98%
rename from src/relay/pass/call_graph.h
rename to src/relay/analysis/call_graph.h
index 684e11a..03c9283 100644
--- a/src/relay/pass/call_graph.h
+++ b/src/relay/analysis/call_graph.h
@@ -18,15 +18,15 @@
  */
 
 /*!
- * \file tvm/relay/pass/call_graph.h
+ * \file src/relay/analysis/call_graph.h
  * \brief Define data structures for the call graph of a IRModule. It borrows
  * the idea how LLVM constructs CallGraph.
  *
  * https://llvm.org/doxygen/CallGraph_8h_source.html
  */
 
-#ifndef TVM_RELAY_PASS_CALL_GRAPH_H_
-#define TVM_RELAY_PASS_CALL_GRAPH_H_
+#ifndef TVM_RELAY_ANALYSIS_CALL_GRAPH_H_
+#define TVM_RELAY_ANALYSIS_CALL_GRAPH_H_
 
 #include <tvm/ir/module.h>
 #include <tvm/relay/expr.h>
@@ -510,4 +510,4 @@ class CallGraphEntry {
 
 }  // namespace relay
 }  // namespace tvm
-#endif  // TVM_RELAY_PASS_CALL_GRAPH_H_
+#endif  // TVM_RELAY_ANALYSIS_CALL_GRAPH_H_
diff --git a/src/relay/pass/dependency_graph.cc b/src/relay/analysis/dependency_graph.cc
similarity index 98%
rename from src/relay/pass/dependency_graph.cc
rename to src/relay/analysis/dependency_graph.cc
index 9dab38c..7e48d12 100644
--- a/src/relay/pass/dependency_graph.cc
+++ b/src/relay/analysis/dependency_graph.cc
@@ -18,8 +18,8 @@
  */
 
 /*!
- * \file tvm/relay/pass/dependency_graph.cc
- * \brief
+ * \file src/relay/analysis/dependency_graph.cc
+ * \brief Implementation of dependency graph APIs.
  */
 #include "dependency_graph.h"
 #include <tvm/relay/expr_functor.h>
diff --git a/src/relay/pass/dependency_graph.h b/src/relay/analysis/dependency_graph.h
similarity index 90%
rename from src/relay/pass/dependency_graph.h
rename to src/relay/analysis/dependency_graph.h
index 4f33e51..5e2dc0c 100644
--- a/src/relay/pass/dependency_graph.h
+++ b/src/relay/analysis/dependency_graph.h
@@ -18,16 +18,16 @@
  */
 
 /*!
- * \file tvm/relay/pass/dependency_graph.h
+ * \file src/relay/analysis/dependency_graph.h
  * \brief create a dependency graph.
  */
-#ifndef TVM_RELAY_PASS_DEPENDENCY_GRAPH_H_
-#define TVM_RELAY_PASS_DEPENDENCY_GRAPH_H_
+#ifndef TVM_RELAY_ANALYSIS_DEPENDENCY_GRAPH_H_
+#define TVM_RELAY_ANALYSIS_DEPENDENCY_GRAPH_H_
 
 #include <tvm/relay/expr.h>
 #include <unordered_map>
 #include <vector>
-#include "let_list.h"
+#include "../transforms/let_list.h"
 #include "../../support/arena.h"
 
 namespace tvm {
@@ -72,4 +72,4 @@ class DependencyGraph {
 
 }  // namespace relay
 }  // namespace tvm
-#endif  // TVM_RELAY_PASS_DEPENDENCY_GRAPH_H_
+#endif  // TVM_RELAY_ANALYSIS_DEPENDENCY_GRAPH_H_
diff --git a/src/relay/pass/feature.cc b/src/relay/analysis/feature.cc
similarity index 98%
rename from src/relay/pass/feature.cc
rename to src/relay/analysis/feature.cc
index 43f28ae..a5a6f19 100644
--- a/src/relay/pass/feature.cc
+++ b/src/relay/analysis/feature.cc
@@ -26,7 +26,7 @@
 #include <tvm/relay/expr.h>
 #include <tvm/relay/expr_functor.h>
 #include <tvm/ir/module.h>
-#include "pass_util.h"
+#include "../transforms/pass_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/pass/kind_check.cc b/src/relay/analysis/kind_check.cc
similarity index 100%
rename from src/relay/pass/kind_check.cc
rename to src/relay/analysis/kind_check.cc
diff --git a/src/relay/pass/mac_count.cc b/src/relay/analysis/mac_count.cc
similarity index 99%
rename from src/relay/pass/mac_count.cc
rename to src/relay/analysis/mac_count.cc
index a03924d..49fe2a3 100644
--- a/src/relay/pass/mac_count.cc
+++ b/src/relay/analysis/mac_count.cc
@@ -31,7 +31,7 @@
 #include <tvm/relay/expr_functor.h>
 #include <tvm/relay/analysis.h>
 #include <tvm/relay/op_attr_types.h>
-#include "pattern_util.h"
+#include "../transforms/pattern_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/pass/match_exhaustion.cc b/src/relay/analysis/match_exhaustion.cc
similarity index 100%
rename from src/relay/pass/match_exhaustion.cc
rename to src/relay/analysis/match_exhaustion.cc
diff --git a/src/relay/pass/type_solver.cc b/src/relay/analysis/type_solver.cc
similarity index 100%
rename from src/relay/pass/type_solver.cc
rename to src/relay/analysis/type_solver.cc
diff --git a/src/relay/pass/type_solver.h b/src/relay/analysis/type_solver.h
similarity index 98%
rename from src/relay/pass/type_solver.h
rename to src/relay/analysis/type_solver.h
index 6eed5b7..8ccc2c7 100644
--- a/src/relay/pass/type_solver.h
+++ b/src/relay/analysis/type_solver.h
@@ -21,8 +21,8 @@
  * \file type_solver.h
  * \brief Solver logic for type inference.
  */
-#ifndef TVM_RELAY_PASS_TYPE_SOLVER_H_
-#define TVM_RELAY_PASS_TYPE_SOLVER_H_
+#ifndef TVM_RELAY_ANALYSIS_TYPE_SOLVER_H_
+#define TVM_RELAY_ANALYSIS_TYPE_SOLVER_H_
 
 #include <tvm/relay/expr.h>
 #include <tvm/relay/type.h>
@@ -34,7 +34,6 @@
 #include <unordered_set>
 #include "../../support/arena.h"
 
-
 namespace tvm {
 namespace relay {
 
@@ -219,4 +218,4 @@ class TypeSolver {
 
 }  // namespace relay
 }  // namespace tvm
-#endif  // TVM_RELAY_PASS_TYPE_SOLVER_H_
+#endif  // TVM_RELAY_ANALYSIS_TYPE_SOLVER_H_
diff --git a/src/relay/pass/util.cc b/src/relay/analysis/util.cc
similarity index 99%
rename from src/relay/pass/util.cc
rename to src/relay/analysis/util.cc
index b341966..88c89df 100644
--- a/src/relay/pass/util.cc
+++ b/src/relay/analysis/util.cc
@@ -28,7 +28,7 @@
 #include <tvm/relay/expr_functor.h>
 #include <tvm/relay/op.h>
 #include <tvm/relay/pattern_functor.h>
-#include "pass_util.h"
+#include "../transforms/pass_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/pass/well_formed.cc b/src/relay/analysis/well_formed.cc
similarity index 100%
rename from src/relay/pass/well_formed.cc
rename to src/relay/analysis/well_formed.cc
diff --git a/src/relay/backend/vm/compiler.cc b/src/relay/backend/vm/compiler.cc
index e3c8d12..5ddcd17 100644
--- a/src/relay/backend/vm/compiler.cc
+++ b/src/relay/backend/vm/compiler.cc
@@ -40,7 +40,7 @@
 #include <vector>
 #include "../utils.h"
 #include "../../backend/compile_engine.h"
-#include "../../pass/pass_util.h"
+#include "../../transforms/pass_util.h"
 #include "../../op/op_common.h"
 #include "compiler.h"
 
diff --git a/src/relay/backend/vm/compiler.h b/src/relay/backend/vm/compiler.h
index 19e1ee8..f18e2c0 100644
--- a/src/relay/backend/vm/compiler.h
+++ b/src/relay/backend/vm/compiler.h
@@ -41,7 +41,7 @@
 #include "../../../runtime/vm/profiler/vm.h"
 #include "../../../runtime/vm/naive_allocator.h"
 #include "../../backend/compile_engine.h"
-#include "../../pass/pass_util.h"
+#include "../../transforms/pass_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/op/annotation/annotation.cc b/src/relay/op/annotation/annotation.cc
index 36f5923..ee9877e 100644
--- a/src/relay/op/annotation/annotation.cc
+++ b/src/relay/op/annotation/annotation.cc
@@ -30,7 +30,7 @@
 #include <tvm/relay/op_attr_types.h>
 #include <topi/elemwise.h>
 
-#include "../../pass/infer_layout_util.h"
+#include "../../transforms/infer_layout_util.h"
 #include "../type_relations.h"
 
 namespace tvm {
diff --git a/src/relay/op/device_copy.cc b/src/relay/op/device_copy.cc
index d15c85c..7e0530c 100644
--- a/src/relay/op/device_copy.cc
+++ b/src/relay/op/device_copy.cc
@@ -33,7 +33,7 @@
 #include <tvm/relay/op_attr_types.h>
 
 #include "type_relations.h"
-#include "../pass/infer_layout_util.h"
+#include "../transforms/infer_layout_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/op/memory/memory.cc b/src/relay/op/memory/memory.cc
index d15099b..dcb50e6 100644
--- a/src/relay/op/memory/memory.cc
+++ b/src/relay/op/memory/memory.cc
@@ -29,7 +29,7 @@
 #include <tvm/relay/attrs/memory.h>
 
 #include "../op_common.h"
-#include "../../pass/infer_layout_util.h"
+#include "../../transforms/infer_layout_util.h"
 #include "../type_relations.h"
 
 namespace tvm {
diff --git a/src/relay/op/nn/bitserial.cc b/src/relay/op/nn/bitserial.cc
index c9e05e1..621fe77 100644
--- a/src/relay/op/nn/bitserial.cc
+++ b/src/relay/op/nn/bitserial.cc
@@ -27,7 +27,7 @@
 #include <tvm/relay/op.h>
 
 #include "../op_common.h"
-#include "../../pass/infer_layout_util.h"
+#include "../../transforms/infer_layout_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/op/nn/convolution.cc b/src/relay/op/nn/convolution.cc
index cd9b5dd..5906d3a 100644
--- a/src/relay/op/nn/convolution.cc
+++ b/src/relay/op/nn/convolution.cc
@@ -27,7 +27,7 @@
 #include <tvm/relay/attrs/nn.h>
 #include <vector>
 
-#include "../../pass/infer_layout_util.h"
+#include "../../transforms/infer_layout_util.h"
 #include "../op_common.h"
 #include "convolution.h"
 
diff --git a/src/relay/op/nn/nn.cc b/src/relay/op/nn/nn.cc
index 10fd4d9..560d06f 100644
--- a/src/relay/op/nn/nn.cc
+++ b/src/relay/op/nn/nn.cc
@@ -33,7 +33,7 @@
 #include <vector>
 #include <string>
 #include "../type_relations.h"
-#include "../../pass/infer_layout_util.h"
+#include "../../transforms/infer_layout_util.h"
 #include "../op_common.h"
 #include "nn.h"
 
diff --git a/src/relay/op/nn/pooling.cc b/src/relay/op/nn/pooling.cc
index f174882..2b02c82 100644
--- a/src/relay/op/nn/pooling.cc
+++ b/src/relay/op/nn/pooling.cc
@@ -27,7 +27,7 @@
 #include <tvm/relay/attrs/nn.h>
 #include <topi/nn/pooling.h>
 #include <vector>
-#include "../../pass/infer_layout_util.h"
+#include "../../transforms/infer_layout_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/op/nn/sparse.cc b/src/relay/op/nn/sparse.cc
index c01b760..e7db125 100644
--- a/src/relay/op/nn/sparse.cc
+++ b/src/relay/op/nn/sparse.cc
@@ -27,7 +27,7 @@
 #include <tvm/relay/attrs/nn.h>
 #include <vector>
 
-#include "../../pass/infer_layout_util.h"
+#include "../../transforms/infer_layout_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/op/op_common.h b/src/relay/op/op_common.h
index d9846cf..25293ad 100644
--- a/src/relay/op/op_common.h
+++ b/src/relay/op/op_common.h
@@ -32,7 +32,7 @@
 #include <string>
 #include <unordered_map>
 #include "type_relations.h"
-#include "../pass/infer_layout_util.h"
+#include "../transforms/infer_layout_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/op/tensor/transform.cc b/src/relay/op/tensor/transform.cc
index fa96d7f..2bdcbcf 100644
--- a/src/relay/op/tensor/transform.cc
+++ b/src/relay/op/tensor/transform.cc
@@ -36,8 +36,8 @@
 #include <vector>
 #include "../op_common.h"
 #include "../../../arith/compute_expr.h"
-#include "../../pass/infer_layout_util.h"
-#include "../../pass/pattern_util.h"
+#include "../../transforms/infer_layout_util.h"
+#include "../../transforms/pattern_util.h"
 #include "transform.h"
 
 namespace tvm {
diff --git a/src/relay/qnn/op/add.cc b/src/relay/qnn/op/add.cc
index 67ca10d..b70954b 100644
--- a/src/relay/qnn/op/add.cc
+++ b/src/relay/qnn/op/add.cc
@@ -24,7 +24,7 @@
 #include <tvm/relay/analysis.h>
 #include <tvm/relay/op_attr_types.h>
 #include <tvm/relay/qnn/attrs.h>
-#include "../../pass/pattern_util.h"
+#include "../../transforms/pattern_util.h"
 #include "../util.h"
 #include "op_common.h"
 
diff --git a/src/relay/qnn/op/concatenate.cc b/src/relay/qnn/op/concatenate.cc
index 360543b..3f4e315 100644
--- a/src/relay/qnn/op/concatenate.cc
+++ b/src/relay/qnn/op/concatenate.cc
@@ -27,7 +27,7 @@
 #include <tvm/relay/op_attr_types.h>
 #include <tvm/relay/qnn/attrs.h>
 #include "../../op/tensor/transform.h"
-#include "../../pass/pattern_util.h"
+#include "../../transforms/pattern_util.h"
 #include "../util.h"
 
 namespace tvm {
diff --git a/src/relay/qnn/op/convolution.cc b/src/relay/qnn/op/convolution.cc
index 6626ce2..30308fd 100644
--- a/src/relay/qnn/op/convolution.cc
+++ b/src/relay/qnn/op/convolution.cc
@@ -28,7 +28,7 @@
 #include <tvm/relay/qnn/attrs.h>
 #include <tvm/relay/transform.h>
 #include "../../op/nn/convolution.h"
-#include "../../pass/pattern_util.h"
+#include "../../transforms/pattern_util.h"
 #include "../util.h"
 
 namespace tvm {
diff --git a/src/relay/qnn/op/dense.cc b/src/relay/qnn/op/dense.cc
index de3c4db..91f6072 100644
--- a/src/relay/qnn/op/dense.cc
+++ b/src/relay/qnn/op/dense.cc
@@ -27,7 +27,7 @@
 #include <tvm/relay/op_attr_types.h>
 #include <tvm/relay/qnn/attrs.h>
 #include "../../op/nn/nn.h"
-#include "../../pass/pattern_util.h"
+#include "../../transforms/pattern_util.h"
 #include "../util.h"
 
 namespace tvm {
diff --git a/src/relay/qnn/op/dequantize.cc b/src/relay/qnn/op/dequantize.cc
index 6233246..087ceac 100644
--- a/src/relay/qnn/op/dequantize.cc
+++ b/src/relay/qnn/op/dequantize.cc
@@ -26,7 +26,7 @@
 #include <tvm/relay/analysis.h>
 #include <tvm/relay/op_attr_types.h>
 #include <tvm/relay/qnn/attrs.h>
-#include "../../pass/pattern_util.h"
+#include "../../transforms/pattern_util.h"
 #include "../util.h"
 
 namespace tvm {
diff --git a/src/relay/qnn/op/mul.cc b/src/relay/qnn/op/mul.cc
index 89193ed..9ab5cad 100644
--- a/src/relay/qnn/op/mul.cc
+++ b/src/relay/qnn/op/mul.cc
@@ -24,7 +24,7 @@
 #include <tvm/relay/analysis.h>
 #include <tvm/relay/op_attr_types.h>
 #include <tvm/relay/qnn/attrs.h>
-#include "../../pass/pattern_util.h"
+#include "../../transforms/pattern_util.h"
 #include "../util.h"
 #include "op_common.h"
 
diff --git a/src/relay/qnn/op/quantize.cc b/src/relay/qnn/op/quantize.cc
index 66c40a2..d3f8ee5 100644
--- a/src/relay/qnn/op/quantize.cc
+++ b/src/relay/qnn/op/quantize.cc
@@ -26,7 +26,7 @@
 #include <tvm/relay/analysis.h>
 #include <tvm/relay/op_attr_types.h>
 #include <tvm/relay/qnn/attrs.h>
-#include "../../pass/pattern_util.h"
+#include "../../transforms/pattern_util.h"
 #include "../util.h"
 
 namespace tvm {
diff --git a/src/relay/qnn/op/requantize.cc b/src/relay/qnn/op/requantize.cc
index 7e99ad7..9b2ed91 100644
--- a/src/relay/qnn/op/requantize.cc
+++ b/src/relay/qnn/op/requantize.cc
@@ -25,7 +25,7 @@
 #include <tvm/relay/analysis.h>
 #include <tvm/relay/op_attr_types.h>
 #include <tvm/relay/qnn/attrs.h>
-#include "../../pass/pattern_util.h"
+#include "../../transforms/pattern_util.h"
 #include "../util.h"
 
 namespace tvm {
diff --git a/src/relay/qnn/util.cc b/src/relay/qnn/util.cc
index fad37bc..648de53 100644
--- a/src/relay/qnn/util.cc
+++ b/src/relay/qnn/util.cc
@@ -23,7 +23,7 @@
  */
 
 #include "util.h"
-#include "../pass/pattern_util.h"
+#include "../transforms/pattern_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/pass/quantize/annotate.cc b/src/relay/quantize/annotate.cc
similarity index 100%
rename from src/relay/pass/quantize/annotate.cc
rename to src/relay/quantize/annotate.cc
diff --git a/src/relay/pass/quantize/calibrate.cc b/src/relay/quantize/calibrate.cc
similarity index 100%
rename from src/relay/pass/quantize/calibrate.cc
rename to src/relay/quantize/calibrate.cc
diff --git a/src/relay/pass/quantize/partition.cc b/src/relay/quantize/partition.cc
similarity index 98%
rename from src/relay/pass/quantize/partition.cc
rename to src/relay/quantize/partition.cc
index 8fb65a4..121e431 100644
--- a/src/relay/pass/quantize/partition.cc
+++ b/src/relay/quantize/partition.cc
@@ -25,7 +25,7 @@
  */
 
 #include <tvm/relay/transform.h>
-#include "../pattern_util.h"
+#include "../transforms/pattern_util.h"
 #include "./quantize.h"
 
 namespace tvm {
diff --git a/src/relay/pass/quantize/quantize.cc b/src/relay/quantize/quantize.cc
similarity index 100%
rename from src/relay/pass/quantize/quantize.cc
rename to src/relay/quantize/quantize.cc
diff --git a/src/relay/pass/quantize/quantize.h b/src/relay/quantize/quantize.h
similarity index 96%
rename from src/relay/pass/quantize/quantize.h
rename to src/relay/quantize/quantize.h
index 4f6cd2d..563f47f 100644
--- a/src/relay/pass/quantize/quantize.h
+++ b/src/relay/quantize/quantize.h
@@ -18,16 +18,16 @@
  */
 
 /*!
- * \file tvm/relay/pass/quantize.h
+ * \file tvm/relay/quantize.h
  * \brief Header of definitions for quantization
  */
-#ifndef TVM_RELAY_PASS_QUANTIZE_QUANTIZE_H_
-#define TVM_RELAY_PASS_QUANTIZE_QUANTIZE_H_
+#ifndef TVM_RELAY_QUANTIZE_QUANTIZE_H_
+#define TVM_RELAY_QUANTIZE_QUANTIZE_H_
 
 #include <tvm/relay/op.h>
 #include <tvm/relay/expr.h>
 #include <string>
-#include "../pattern_util.h"
+#include "../transforms/pattern_util.h"
 
 namespace tvm {
 namespace relay {
@@ -163,4 +163,4 @@ struct QConfigContext {
 }  // namespace quantize
 }  // namespace relay
 }  // namespace tvm
-#endif  // TVM_RELAY_PASS_QUANTIZE_QUANTIZE_H_
+#endif  // TVM_RELAY_QUANTIZE_QUANTIZE_H_
diff --git a/src/relay/pass/quantize/realize.cc b/src/relay/quantize/realize.cc
similarity index 99%
rename from src/relay/pass/quantize/realize.cc
rename to src/relay/quantize/realize.cc
index ae37245..e80b17c 100644
--- a/src/relay/pass/quantize/realize.cc
+++ b/src/relay/quantize/realize.cc
@@ -29,8 +29,8 @@
 #include <tvm/relay/analysis.h>
 #include <tvm/relay/attrs/annotation.h>
 #include "./quantize.h"
-#include "../pattern_util.h"
-#include "../../qnn/util.h"
+#include "../transforms/pattern_util.h"
+#include "../qnn/util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/pass/alter_op_layout.cc b/src/relay/transforms/alter_op_layout.cc
similarity index 100%
rename from src/relay/pass/alter_op_layout.cc
rename to src/relay/transforms/alter_op_layout.cc
diff --git a/src/relay/pass/annotate_target.cc b/src/relay/transforms/annotate_target.cc
similarity index 98%
rename from src/relay/pass/annotate_target.cc
rename to src/relay/transforms/annotate_target.cc
index 7322069..f5bc6c2 100644
--- a/src/relay/pass/annotate_target.cc
+++ b/src/relay/transforms/annotate_target.cc
@@ -18,7 +18,7 @@
  */
 
 /*!
- * \file src/relay/pass/annotate_target.cc
+ * \file src/relay/transforms/annotate_target.cc
  * \brief Wraps a call with compiler_begin and compiler_end to indicate that
  * the op of this call node will use external compiler.
  */
diff --git a/src/relay/pass/canonicalize_cast.cc b/src/relay/transforms/canonicalize_cast.cc
similarity index 100%
rename from src/relay/pass/canonicalize_cast.cc
rename to src/relay/transforms/canonicalize_cast.cc
index dd899df..b65e948 100644
--- a/src/relay/pass/canonicalize_cast.cc
+++ b/src/relay/transforms/canonicalize_cast.cc
@@ -25,8 +25,8 @@
 #include <tvm/relay/expr_functor.h>
 #include <tvm/relay/attrs/nn.h>
 #include <tvm/relay/transform.h>
-#include "pattern_util.h"
 #include "pass_util.h"
+#include "pattern_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/pass/canonicalize_ops.cc b/src/relay/transforms/canonicalize_ops.cc
similarity index 100%
rename from src/relay/pass/canonicalize_ops.cc
rename to src/relay/transforms/canonicalize_ops.cc
diff --git a/src/relay/pass/combine_parallel_conv2d.cc b/src/relay/transforms/combine_parallel_conv2d.cc
similarity index 99%
rename from src/relay/pass/combine_parallel_conv2d.cc
rename to src/relay/transforms/combine_parallel_conv2d.cc
index 5c92637..b4ac997 100644
--- a/src/relay/pass/combine_parallel_conv2d.cc
+++ b/src/relay/transforms/combine_parallel_conv2d.cc
@@ -41,8 +41,8 @@
 #include <unordered_map>
 #include <unordered_set>
 #include "./expr_subst.h"
-#include "./pattern_util.h"
 #include "./combine_parallel_op.h"
+#include "pattern_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/pass/combine_parallel_dense.cc b/src/relay/transforms/combine_parallel_dense.cc
similarity index 99%
rename from src/relay/pass/combine_parallel_dense.cc
rename to src/relay/transforms/combine_parallel_dense.cc
index fdd42be..cd234bb 100644
--- a/src/relay/pass/combine_parallel_dense.cc
+++ b/src/relay/transforms/combine_parallel_dense.cc
@@ -40,7 +40,7 @@
 #include <unordered_map>
 #include <unordered_set>
 #include "./expr_subst.h"
-#include "./pattern_util.h"
+#include "pattern_util.h"
 #include "./combine_parallel_op_batch.h"
 
 namespace tvm {
diff --git a/src/relay/pass/combine_parallel_op.cc b/src/relay/transforms/combine_parallel_op.cc
similarity index 100%
rename from src/relay/pass/combine_parallel_op.cc
rename to src/relay/transforms/combine_parallel_op.cc
diff --git a/src/relay/pass/combine_parallel_op.h b/src/relay/transforms/combine_parallel_op.h
similarity index 97%
rename from src/relay/pass/combine_parallel_op.h
rename to src/relay/transforms/combine_parallel_op.h
index 24f3884..0097e29 100644
--- a/src/relay/pass/combine_parallel_op.h
+++ b/src/relay/transforms/combine_parallel_op.h
@@ -22,8 +22,8 @@
  * \file combine_parallel_op.h
  * \brief Abstract class to combine parallel ops and their successive element-wise ops.
  */
-#ifndef TVM_RELAY_PASS_COMBINE_PARALLEL_OP_H_
-#define TVM_RELAY_PASS_COMBINE_PARALLEL_OP_H_
+#ifndef TVM_RELAY_TRANSFORMS_COMBINE_PARALLEL_OP_H_
+#define TVM_RELAY_TRANSFORMS_COMBINE_PARALLEL_OP_H_
 
 #include <tvm/relay/analysis.h>
 #include <tvm/relay/expr_functor.h>
@@ -36,7 +36,7 @@
 #include <vector>
 #include <string>
 #include "./expr_subst.h"
-#include "./pattern_util.h"
+#include "pattern_util.h"
 
 
 namespace tvm {
@@ -237,4 +237,4 @@ class ParallelOpCombiner {
 
 }  // namespace relay
 }  // namespace tvm
-#endif  // TVM_RELAY_PASS_COMBINE_PARALLEL_OP_H_
+#endif  // TVM_RELAY_TRANSFORMS_COMBINE_PARALLEL_OP_H_
diff --git a/src/relay/pass/combine_parallel_op_batch.cc b/src/relay/transforms/combine_parallel_op_batch.cc
similarity index 99%
rename from src/relay/pass/combine_parallel_op_batch.cc
rename to src/relay/transforms/combine_parallel_op_batch.cc
index a9eefaf..4252cfd 100644
--- a/src/relay/pass/combine_parallel_op_batch.cc
+++ b/src/relay/transforms/combine_parallel_op_batch.cc
@@ -53,9 +53,9 @@
 #include <unordered_map>
 #include <unordered_set>
 #include "./expr_subst.h"
-#include "./pattern_util.h"
 #include "./combine_parallel_op.h"
 #include "./combine_parallel_op_batch.h"
+#include "pattern_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/pass/combine_parallel_op_batch.h b/src/relay/transforms/combine_parallel_op_batch.h
similarity index 95%
rename from src/relay/pass/combine_parallel_op_batch.h
rename to src/relay/transforms/combine_parallel_op_batch.h
index 7dcbe76..6876604 100644
--- a/src/relay/pass/combine_parallel_op_batch.h
+++ b/src/relay/transforms/combine_parallel_op_batch.h
@@ -18,11 +18,11 @@
  */
 
 /*!
- * \file combine_parallel_op_batch.cc
+ * \file combine_parallel_op_batch.h
  * \brief Combine parallel ops into a single batch op.
  */
-#ifndef TVM_RELAY_PASS_COMBINE_PARALLEL_OP_BATCH_H_
-#define TVM_RELAY_PASS_COMBINE_PARALLEL_OP_BATCH_H_
+#ifndef TVM_RELAY_TRANSFORMS_COMBINE_PARALLEL_OP_BATCH_H_
+#define TVM_RELAY_TRANSFORMS_COMBINE_PARALLEL_OP_BATCH_H_
 
 #include <tvm/relay/analysis.h>
 #include <tvm/relay/expr_functor.h>
@@ -34,8 +34,8 @@
 #include <unordered_set>
 #include <string>
 #include "./expr_subst.h"
-#include "./pattern_util.h"
 #include "./combine_parallel_op.h"
+#include "pattern_util.h"
 
 namespace tvm {
 namespace relay {
@@ -145,4 +145,4 @@ class ParallelOpBatchCombiner : public ParallelOpCombiner {
 }  // namespace relay
 }  // namespace tvm
 
-#endif  // TVM_RELAY_PASS_COMBINE_PARALLEL_OP_BATCH_H_
+#endif  // TVM_RELAY_TRANSFORMS_COMBINE_PARALLEL_OP_BATCH_H_
diff --git a/src/relay/pass/convert_layout.cc b/src/relay/transforms/convert_layout.cc
similarity index 100%
rename from src/relay/pass/convert_layout.cc
rename to src/relay/transforms/convert_layout.cc
diff --git a/src/relay/pass/de_duplicate.cc b/src/relay/transforms/de_duplicate.cc
similarity index 100%
rename from src/relay/pass/de_duplicate.cc
rename to src/relay/transforms/de_duplicate.cc
diff --git a/src/relay/pass/dead_code.cc b/src/relay/transforms/dead_code.cc
similarity index 100%
rename from src/relay/pass/dead_code.cc
rename to src/relay/transforms/dead_code.cc
diff --git a/src/relay/pass/device_annotation.cc b/src/relay/transforms/device_annotation.cc
similarity index 99%
rename from src/relay/pass/device_annotation.cc
rename to src/relay/transforms/device_annotation.cc
index c87acd0..93aa108 100644
--- a/src/relay/pass/device_annotation.cc
+++ b/src/relay/transforms/device_annotation.cc
@@ -563,9 +563,6 @@ Map<Expr, Integer> CollectDeviceAnnotationOps(const Expr& expr) {
 TVM_REGISTER_GLOBAL("relay._analysis.CollectDeviceInfo")
 .set_body_typed(CollectDeviceInfo);
 
-TVM_REGISTER_GLOBAL("relay._analysis.RewriteDeviceAnnotation")
-.set_body_typed(RewriteAnnotatedOps);
-
 TVM_REGISTER_GLOBAL("relay._analysis.CollectDeviceAnnotationOps")
 .set_body_typed(CollectDeviceAnnotationOps);
 
@@ -574,7 +571,7 @@ namespace transform {
 Pass RewriteAnnotatedOps(int fallback_device) {
   runtime::TypedPackedFunc<Function(Function, IRModule, PassContext)> pass_func =
     [=](Function f, IRModule m, PassContext pc) {
-    return Downcast<Function>(RewriteAnnotatedOps(f, fallback_device));
+    return Downcast<Function>(relay::RewriteAnnotatedOps(f, fallback_device));
   };
   return CreateFunctionPass(pass_func, 1, "RewriteAnnotatedOps",
                             {tir::StringImmNode::make("InferType")});
diff --git a/src/relay/pass/eliminate_common_subexpr.cc b/src/relay/transforms/eliminate_common_subexpr.cc
similarity index 99%
rename from src/relay/pass/eliminate_common_subexpr.cc
rename to src/relay/transforms/eliminate_common_subexpr.cc
index c041b7e..bb31d32 100644
--- a/src/relay/pass/eliminate_common_subexpr.cc
+++ b/src/relay/transforms/eliminate_common_subexpr.cc
@@ -30,7 +30,7 @@
 #include <tvm/relay/expr_functor.h>
 #include <tvm/relay/transform.h>
 #include <unordered_map>
-#include "./pattern_util.h"
+#include "pattern_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/pass/eta_expand.cc b/src/relay/transforms/eta_expand.cc
similarity index 100%
rename from src/relay/pass/eta_expand.cc
rename to src/relay/transforms/eta_expand.cc
diff --git a/src/relay/pass/expr_subst.cc b/src/relay/transforms/expr_subst.cc
similarity index 100%
rename from src/relay/pass/expr_subst.cc
rename to src/relay/transforms/expr_subst.cc
diff --git a/src/relay/pass/expr_subst.h b/src/relay/transforms/expr_subst.h
similarity index 89%
rename from src/relay/pass/expr_subst.h
rename to src/relay/transforms/expr_subst.h
index 2ffefa2..849ffc2 100644
--- a/src/relay/pass/expr_subst.h
+++ b/src/relay/transforms/expr_subst.h
@@ -21,8 +21,8 @@
  * \file expr_subst.h
  * \brief Utility functions for substituting expressions.
  */
-#ifndef TVM_RELAY_PASS_EXPR_SUBST_H_
-#define TVM_RELAY_PASS_EXPR_SUBST_H_
+#ifndef TVM_RELAY_TRANSFORMS_EXPR_SUBST_H_
+#define TVM_RELAY_TRANSFORMS_EXPR_SUBST_H_
 #include <tvm/relay/expr.h>
 #include <unordered_map>
 
@@ -34,4 +34,4 @@ Expr ExprSubst(const Expr& expr,
 
 }  // namespace relay
 }  // namespace tvm
-#endif  // TVM_RELAY_PASS_EXPR_SUBST_H_
+#endif  // TVM_RELAY_TRANSFORMS_EXPR_SUBST_H_
diff --git a/src/relay/pass/fast_math.cc b/src/relay/transforms/fast_math.cc
similarity index 100%
rename from src/relay/pass/fast_math.cc
rename to src/relay/transforms/fast_math.cc
diff --git a/src/relay/pass/fold_constant.cc b/src/relay/transforms/fold_constant.cc
similarity index 100%
rename from src/relay/pass/fold_constant.cc
rename to src/relay/transforms/fold_constant.cc
diff --git a/src/relay/pass/fold_scale_axis.cc b/src/relay/transforms/fold_scale_axis.cc
similarity index 100%
rename from src/relay/pass/fold_scale_axis.cc
rename to src/relay/transforms/fold_scale_axis.cc
diff --git a/src/relay/pass/forward_rewrite.cc b/src/relay/transforms/forward_rewrite.cc
similarity index 100%
rename from src/relay/pass/forward_rewrite.cc
rename to src/relay/transforms/forward_rewrite.cc
diff --git a/src/relay/pass/fuse_ops.cc b/src/relay/transforms/fuse_ops.cc
similarity index 99%
rename from src/relay/pass/fuse_ops.cc
rename to src/relay/transforms/fuse_ops.cc
index fd8e6fd..0c74159 100644
--- a/src/relay/pass/fuse_ops.cc
+++ b/src/relay/transforms/fuse_ops.cc
@@ -19,7 +19,7 @@
 
 /*!
  *
- * \file src/tvm/relay/pass/fuse_ops.cc
+ * \file src/relay/transforms/fuse_ops.cc
  *
  * \brief This is a backend-aware optimization pass.
  *   Fuse necessary ops into a single one.
@@ -29,10 +29,9 @@
 #include <tvm/relay/expr_functor.h>
 #include <tvm/relay/op_attr_types.h>
 #include <tvm/relay/transform.h>
-#include "./pattern_util.h"
+#include "pattern_util.h"
 #include "../../support/arena.h"
 
-
 namespace tvm {
 namespace relay {
 
diff --git a/src/relay/pass/gradient.cc b/src/relay/transforms/gradient.cc
similarity index 100%
rename from src/relay/pass/gradient.cc
rename to src/relay/transforms/gradient.cc
diff --git a/src/relay/pass/infer_layout_util.h b/src/relay/transforms/infer_layout_util.h
similarity index 98%
rename from src/relay/pass/infer_layout_util.h
rename to src/relay/transforms/infer_layout_util.h
index 9ecd0bf..eeb61e3 100644
--- a/src/relay/pass/infer_layout_util.h
+++ b/src/relay/transforms/infer_layout_util.h
@@ -24,8 +24,8 @@
           custom layouts or other general weight pre-transformation.
  */
 
-#ifndef TVM_RELAY_PASS_INFER_LAYOUT_UTIL_H_
-#define TVM_RELAY_PASS_INFER_LAYOUT_UTIL_H_
+#ifndef TVM_RELAY_TRANSFORMS_INFER_LAYOUT_UTIL_H_
+#define TVM_RELAY_TRANSFORMS_INFER_LAYOUT_UTIL_H_
 
 #include <tvm/tir/data_layout.h>
 #include <tvm/relay/expr.h>
@@ -232,4 +232,4 @@ static inline std::tuple<Array<Layout>, Array<Layout>, bool> InferCorrectLayouts
 }  //  namespace relay
 }  //  namespace tvm
 
-#endif  // TVM_RELAY_PASS_INFER_LAYOUT_UTIL_H_
+#endif  // TVM_RELAY_TRANSFORMS_INFER_LAYOUT_UTIL_H_
diff --git a/src/relay/pass/inline.cc b/src/relay/transforms/inline.cc
similarity index 99%
rename from src/relay/pass/inline.cc
rename to src/relay/transforms/inline.cc
index 6c8caee..914c718 100644
--- a/src/relay/pass/inline.cc
+++ b/src/relay/transforms/inline.cc
@@ -18,7 +18,7 @@
  */
 
 /*!
- * \file tvm/relay/pass/inline.cc
+ * \file src/relay/transforms/inline.cc
  * \brief Global function inliner. It contains the following steps:
  *
  *  - Preprocessing: eligibility checking. Only inline the functions that can
@@ -40,7 +40,7 @@
 #include <string>
 #include <unordered_set>
 
-#include "call_graph.h"
+#include "../analysis/call_graph.h"
 
 using namespace tvm::runtime;
 
diff --git a/src/relay/pass/legalize.cc b/src/relay/transforms/legalize.cc
similarity index 100%
rename from src/relay/pass/legalize.cc
rename to src/relay/transforms/legalize.cc
diff --git a/src/relay/pass/let_list.h b/src/relay/transforms/let_list.h
similarity index 97%
rename from src/relay/pass/let_list.h
rename to src/relay/transforms/let_list.h
index e37ffa9..1fb0947 100644
--- a/src/relay/pass/let_list.h
+++ b/src/relay/transforms/let_list.h
@@ -26,8 +26,8 @@
  *  if one instead write 'b = ll.Push(a + a); c = ll.Push(b + b); d = ll.Get(c + c);',
  *  the AST will contain 2 'a', as b and c are now variables.
  */
-#ifndef TVM_RELAY_PASS_LET_LIST_H_
-#define TVM_RELAY_PASS_LET_LIST_H_
+#ifndef TVM_RELAY_TRANSFORMS_LET_LIST_H_
+#define TVM_RELAY_TRANSFORMS_LET_LIST_H_
 
 #include <tvm/relay/expr.h>
 #include <tvm/relay/analysis.h>
@@ -149,4 +149,4 @@ class LetList {
 
 }  // namespace relay
 }  // namespace tvm
-#endif  // TVM_RELAY_PASS_LET_LIST_H_
+#endif  // TVM_RELAY_TRANSFORMS_LET_LIST_H_
diff --git a/src/relay/pass/merge_composite.cc b/src/relay/transforms/merge_composite.cc
similarity index 99%
rename from src/relay/pass/merge_composite.cc
rename to src/relay/transforms/merge_composite.cc
index 162bf3a..f187825 100644
--- a/src/relay/pass/merge_composite.cc
+++ b/src/relay/transforms/merge_composite.cc
@@ -18,7 +18,7 @@
  */
 
 /*!
- * \file src/relay/pass/merge_composite.cc
+ * \file src/relay/transforms/merge_composite.cc
  * \brief Merges expressions matching patterns into functions marked
  * as 'composite'. This is primarily intended to be used alongside the
  * external codegen infrastructure to support the case where multiple
diff --git a/src/relay/pass/partial_eval.cc b/src/relay/transforms/partial_eval.cc
similarity index 100%
rename from src/relay/pass/partial_eval.cc
rename to src/relay/transforms/partial_eval.cc
diff --git a/src/relay/pass/partition_graph.cc b/src/relay/transforms/partition_graph.cc
similarity index 99%
rename from src/relay/pass/partition_graph.cc
rename to src/relay/transforms/partition_graph.cc
index d9600bd..30a4e51 100644
--- a/src/relay/pass/partition_graph.cc
+++ b/src/relay/transforms/partition_graph.cc
@@ -18,7 +18,7 @@
  */
 
 /*
- * \file src/relay/pass/partition_graph.cc
+ * \file src/relay/transforms/partition_graph.cc
  *
  * \brief Partition an input function into multiple functions according based
  * on the inserted annotation nodes (i.e. compiler_begin and compiler_end).
diff --git a/src/relay/pass/pass_util.h b/src/relay/transforms/pass_util.h
similarity index 96%
rename from src/relay/pass/pass_util.h
rename to src/relay/transforms/pass_util.h
index 2d4722b..963553b 100644
--- a/src/relay/pass/pass_util.h
+++ b/src/relay/transforms/pass_util.h
@@ -19,11 +19,11 @@
 
 /*!
  *
- * \file tvm/relay/pass/pass_util.h
+ * \file tvm/relay/_transforms/pass_util.h
  * \brief Utilities for writing passes
  */
-#ifndef TVM_RELAY_PASS_PASS_UTIL_H_
-#define TVM_RELAY_PASS_PASS_UTIL_H_
+#ifndef TVM_RELAY_TRANSFORMS_PASS_UTIL_H_
+#define TVM_RELAY_TRANSFORMS_PASS_UTIL_H_
 
 #include <tvm/relay/op.h>
 #include <tvm/relay/expr.h>
@@ -167,4 +167,4 @@ struct TreeBranchNode : TreeNode<ConditionObjectPtr> {
 
 }  // namespace relay
 }  // namespace tvm
-#endif  // TVM_RELAY_PASS_PASS_UTIL_H_
+#endif  // TVM_RELAY_TRANSFORMS_PASS_UTIL_H_
diff --git a/src/relay/pass/pattern_util.h b/src/relay/transforms/pattern_util.h
similarity index 99%
rename from src/relay/pass/pattern_util.h
rename to src/relay/transforms/pattern_util.h
index 85750f5..e59d5c9 100644
--- a/src/relay/pass/pattern_util.h
+++ b/src/relay/transforms/pattern_util.h
@@ -19,12 +19,12 @@
 
 /*!
  *
- * \file tvm/relay/pass/pattern_util.h
+ * \file tvm/relay/_pattern_util.h
  * \brief Header of internal operator functions
  *  These can be used for writing passes.
  */
-#ifndef TVM_RELAY_PASS_PATTERN_UTIL_H_
-#define TVM_RELAY_PASS_PATTERN_UTIL_H_
+#ifndef TVM_RELAY_TRANSFORMS_PATTERN_UTIL_H_
+#define TVM_RELAY_TRANSFORMS_PATTERN_UTIL_H_
 
 #include <builtin_fp16.h>
 #include <tvm/tir/data_layout.h>
@@ -606,4 +606,4 @@ Expr CastHint(Expr data, DataType dtype);
 
 }  // namespace relay
 }  // namespace tvm
-#endif  // TVM_RELAY_PASS_PATTERN_UTIL_H_
+#endif  // TVM_RELAY_TRANSFORMS_PATTERN_UTIL_H_
diff --git a/src/relay/pass/print_ir.cc b/src/relay/transforms/print_ir.cc
similarity index 97%
rename from src/relay/pass/print_ir.cc
rename to src/relay/transforms/print_ir.cc
index e7ed89b..cf06b50 100644
--- a/src/relay/pass/print_ir.cc
+++ b/src/relay/transforms/print_ir.cc
@@ -19,7 +19,7 @@
 
 /*!
  *
- * \file src/relay/pass/print_ir.cc
+ * \file src/relay/transforms/print_ir.cc
  *
  * \brief Print the module IR to help debugging.
  */
diff --git a/src/relay/pass/simplify_inference.cc b/src/relay/transforms/simplify_inference.cc
similarity index 99%
rename from src/relay/pass/simplify_inference.cc
rename to src/relay/transforms/simplify_inference.cc
index fbbe635..bc7c15e 100644
--- a/src/relay/pass/simplify_inference.cc
+++ b/src/relay/transforms/simplify_inference.cc
@@ -25,7 +25,7 @@
 #include <tvm/relay/attrs/nn.h>
 #include <tvm/relay/transform.h>
 #include <tvm/relay/op.h>
-#include "./pattern_util.h"
+#include "pattern_util.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/pass/to_a_normal_form.cc b/src/relay/transforms/to_a_normal_form.cc
similarity index 99%
rename from src/relay/pass/to_a_normal_form.cc
rename to src/relay/transforms/to_a_normal_form.cc
index c75afd1..785162c 100644
--- a/src/relay/pass/to_a_normal_form.cc
+++ b/src/relay/transforms/to_a_normal_form.cc
@@ -29,9 +29,9 @@
 #include <tvm/relay/expr_functor.h>
 #include <tvm/support/logging.h>
 #include "let_list.h"
-#include "../../support/arena.h"
 #include "pass_util.h"
-#include "dependency_graph.h"
+#include "../../support/arena.h"
+#include "../analysis/dependency_graph.h"
 
 namespace tvm {
 namespace relay {
diff --git a/src/relay/pass/to_cps.cc b/src/relay/transforms/to_cps.cc
similarity index 100%
rename from src/relay/pass/to_cps.cc
rename to src/relay/transforms/to_cps.cc
diff --git a/src/relay/pass/to_graph_normal_form.cc b/src/relay/transforms/to_graph_normal_form.cc
similarity index 100%
rename from src/relay/pass/to_graph_normal_form.cc
rename to src/relay/transforms/to_graph_normal_form.cc
diff --git a/src/relay/pass/transform_layout.h b/src/relay/transforms/transform_layout.h
similarity index 98%
rename from src/relay/pass/transform_layout.h
rename to src/relay/transforms/transform_layout.h
index f18f5d2..7204a7e 100644
--- a/src/relay/pass/transform_layout.h
+++ b/src/relay/transforms/transform_layout.h
@@ -23,8 +23,8 @@
  * \brief Common infrastructure for transforming the layouts. This is used for AlterOpLayout and
  *        ConvertLayout pass. */
 
-#ifndef TVM_RELAY_PASS_TRANSFORM_LAYOUT_H_
-#define TVM_RELAY_PASS_TRANSFORM_LAYOUT_H_
+#ifndef TVM_RELAY_TRANSFORMS_TRANSFORM_LAYOUT_H_
+#define TVM_RELAY_TRANSFORMS_TRANSFORM_LAYOUT_H_
 
 #include <tvm/tir/data_layout.h>
 #include <tvm/relay/expr.h>
@@ -367,4 +367,4 @@ Expr LayoutRewriter(const Call& ref_call, const Array<Expr>& new_args, const Obj
 }  //  namespace relay
 }  //  namespace tvm
 
-#endif  // TVM_RELAY_PASS_TRANSFORM_LAYOUT_H_
+#endif  // TVM_RELAY_TRANSFORMS_TRANSFORM_LAYOUT_H_
diff --git a/src/relay/pass/type_infer.cc b/src/relay/transforms/type_infer.cc
similarity index 99%
rename from src/relay/pass/type_infer.cc
rename to src/relay/transforms/type_infer.cc
index ed5f91a..3a16d8f 100644
--- a/src/relay/pass/type_infer.cc
+++ b/src/relay/transforms/type_infer.cc
@@ -43,8 +43,8 @@
 #include <tvm/relay/pattern_functor.h>
 #include <tvm/relay/analysis.h>
 #include <tvm/relay/transform.h>
-#include "./pass_util.h"
-#include "type_solver.h"
+#include "pass_util.h"
+#include "../analysis/type_solver.h"
 
 namespace tvm {
 namespace relay {