You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2026/02/05 23:22:33 UTC
(tvm) branch builder-update updated (0af1f136bd -> 68a702d8f3)
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a change to branch builder-update
in repository https://gitbox.apache.org/repos/asf/tvm.git
discard 0af1f136bd [REFACTOR] Migrate old tir.ir_builder to tvmscript or builder
new 68a702d8f3 [REFACTOR] Migrate old tir.ir_builder to tvmscript or builder
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (0af1f136bd)
\
N -- N -- N refs/heads/builder-update (68a702d8f3)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
python/tvm/relax/transform/legalize_ops/grad.py | 78 +++++++++----------
python/tvm/script/ir_builder/tir/utils.py | 3 +-
python/tvm/topi/gpu/sort.py | 69 ++++++++++-------
python/tvm/topi/index_put.py | 89 +++++++++++-----------
python/tvm/topi/scatter.py | 66 ++++++++--------
python/tvm/topi/scatter_elements.py | 42 +++++-----
.../relax/test_transform_legalize_ops_grad.py | 2 +
.../test_transform_legalize_ops_manipulate.py | 3 +
8 files changed, 186 insertions(+), 166 deletions(-)
(tvm) 01/01: [REFACTOR] Migrate old tir.ir_builder to tvmscript or builder
Posted by tq...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch builder-update
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 68a702d8f3350ce17b5e6c9b6d9b365162b1dfa5
Author: tqchen <ti...@gmail.com>
AuthorDate: Wed Feb 4 17:14:15 2026 -0500
[REFACTOR] Migrate old tir.ir_builder to tvmscript or builder
This PR migrates legacy tir.ir_builder infavor of tvmscript or builder.
---
python/tvm/relax/transform/legalize_ops/grad.py | 86 +-
python/tvm/script/ir_builder/tir/__init__.py | 1 +
python/tvm/script/ir_builder/tir/utils.py | 264 ++++++
python/tvm/tir/__init__.py | 1 -
python/tvm/tir/ir_builder.py | 530 ------------
python/tvm/topi/gpu/scan.py | 302 ++++---
python/tvm/topi/gpu/sort.py | 917 +++++++++++++--------
python/tvm/topi/index_put.py | 103 +--
python/tvm/topi/scan.py | 44 +-
python/tvm/topi/scatter.py | 76 +-
python/tvm/topi/scatter_elements.py | 54 +-
python/tvm/topi/searchsorted.py | 106 +--
python/tvm/topi/signal.py | 135 +--
python/tvm/topi/sparse_reshape.py | 252 +++---
python/tvm/topi/unique.py | 21 +-
python/tvm/topi/vision/nms.py | 285 +++----
python/tvm/topi/vision/nms_util.py | 177 ++--
tests/python/codegen/test_target_codegen_extern.py | 130 +--
tests/python/codegen/test_target_codegen_llvm.py | 106 ++-
.../codegen/test_target_codegen_static_init.py | 58 +-
tests/python/codegen/test_target_codegen_vulkan.py | 248 +++---
.../relax/test_transform_legalize_ops_grad.py | 2 +
.../test_transform_legalize_ops_manipulate.py | 3 +
tests/python/runtime/test_runtime_extension.py | 23 +-
tests/python/tir-base/test_tir_base.py | 14 +-
.../tir-base/test_tir_stmt_functor_ir_transform.py | 29 +-
.../tir-base/test_tir_structural_equal_hash.py | 23 +-
.../tir-transform/test_tir_transform_hoist_if.py | 721 +++++++---------
.../test_tir_transform_inject_double_buffer.py | 49 +-
.../test_tir_transform_inject_virtual_thread.py | 206 ++---
.../test_tir_transform_loop_partition.py | 230 +++---
.../test_tir_transform_lower_tvm_builtin.py | 26 +-
.../test_tir_transform_make_unpacked_api.py | 33 +-
.../test_tir_transform_narrow_datatype.py | 157 ++--
.../test_tir_transform_rewrite_unsafe_select.py | 54 +-
.../tir-transform/test_tir_transform_simplify.py | 78 +-
.../test_tir_transform_storage_rewrite.py | 304 +++----
.../test_tir_transform_unroll_loop.py | 89 +-
.../tir-transform/test_tir_transform_vectorize.py | 126 ++-
39 files changed, 2995 insertions(+), 3068 deletions(-)
diff --git a/python/tvm/relax/transform/legalize_ops/grad.py b/python/tvm/relax/transform/legalize_ops/grad.py
index 4fde2a25c3..769ac11743 100644
--- a/python/tvm/relax/transform/legalize_ops/grad.py
+++ b/python/tvm/relax/transform/legalize_ops/grad.py
@@ -19,6 +19,9 @@
import logging
from tvm import te, tir, topi
+from tvm.script.ir_builder import IRBuilder
+from tvm.script.ir_builder import tir as T
+from tvm.script.ir_builder.tir.utils import buffer_proxy
from ...block_builder import BlockBuilder
from ...expr import Call, Expr
from .common import register_legalize
@@ -160,53 +163,58 @@ def _grad_take_backward(bb: BlockBuilder, call: Call) -> Expr:
def te_take_backward(output_grad, x, indices):
def gen_ir(output_grad_ptr, x_ptr, indices_ptr, out_ptr):
# pylint: disable=invalid-name
- ib = tir.ir_builder.create()
-
- output_grad = ib.buffer_ptr(output_grad_ptr)
- indices = ib.buffer_ptr(indices_ptr)
- out = ib.buffer_ptr(out_ptr)
+ # Use buffer_proxy for flat indexing on multi-dimensional buffers
+ out = buffer_proxy(out_ptr)
+ grad = buffer_proxy(output_grad_ptr)
+ idx = buffer_proxy(indices_ptr)
fused_shape = 1
for i in x_ptr.shape:
fused_shape *= i
- with ib.for_range(0, fused_shape) as i:
- out[i] = tir.const(0, dtype=x_ptr.dtype)
-
assert len(indices_ptr.shape) == 1 # indices in take must be 1-dim Tensor
indices_len = indices_ptr.shape[0]
- if axis is not None:
- fused_output_grad_shape_pre = 1
- fused_output_grad_shape_nxt = 1
- for i in range(len(output_grad_ptr.shape)):
- if i < axis:
- fused_output_grad_shape_pre *= output_grad_ptr.shape[i]
- elif i > axis:
- fused_output_grad_shape_nxt *= output_grad_ptr.shape[i]
-
- x_axis_len = x_ptr.shape[axis]
-
- with ib.for_range(
- 0, fused_output_grad_shape_pre * fused_output_grad_shape_nxt, "parallel"
- ) as fused:
- i = fused // fused_output_grad_shape_nxt
- j = fused % fused_output_grad_shape_nxt
- with ib.for_range(0, indices_len, "serial") as l:
- out[
- i * fused_output_grad_shape_nxt * x_axis_len
- + indices[l] * fused_output_grad_shape_nxt
- + j
- ] += output_grad[
- i * fused_output_grad_shape_nxt * indices_len
- + l * fused_output_grad_shape_nxt
- + j
- ]
- else:
- with ib.for_range(0, indices_len, "serial") as l:
- out[indices[l]] += output_grad[l]
-
- return ib.get()
+ with IRBuilder() as ib:
+ with T.seq_scope():
+ # Init loop (zero-fill output buffer)
+ with T.serial(fused_shape) as i:
+ out[i] = tir.const(0, dtype=x_ptr.dtype)
+
+ # Accumulation loop
+ if axis is not None:
+ fused_output_grad_shape_pre = 1
+ fused_output_grad_shape_nxt = 1
+ for i in range(len(output_grad_ptr.shape)):
+ if i < axis:
+ fused_output_grad_shape_pre *= output_grad_ptr.shape[i]
+ elif i > axis:
+ fused_output_grad_shape_nxt *= output_grad_ptr.shape[i]
+
+ x_axis_len = x_ptr.shape[axis]
+
+ with T.serial(
+ fused_output_grad_shape_pre * fused_output_grad_shape_nxt
+ ) as fused:
+ i = fused // fused_output_grad_shape_nxt
+ j = fused % fused_output_grad_shape_nxt
+ with T.serial(indices_len) as loop_l:
+ out_idx = (
+ i * fused_output_grad_shape_nxt * x_axis_len
+ + idx[loop_l] * fused_output_grad_shape_nxt
+ + j
+ )
+ grad_idx = (
+ i * fused_output_grad_shape_nxt * indices_len
+ + loop_l * fused_output_grad_shape_nxt
+ + j
+ )
+ out[out_idx] = out[out_idx] + grad[grad_idx]
+ else:
+ with T.serial(indices_len) as loop_l:
+ out[idx[loop_l]] = out[idx[loop_l]] + grad[loop_l]
+
+ return ib.get()
shape = x.shape
out_buf = tir.decl_buffer(shape, x.dtype, "out_buf")
diff --git a/python/tvm/script/ir_builder/tir/__init__.py b/python/tvm/script/ir_builder/tir/__init__.py
index db2fc6aca0..164f2254de 100644
--- a/python/tvm/script/ir_builder/tir/__init__.py
+++ b/python/tvm/script/ir_builder/tir/__init__.py
@@ -18,3 +18,4 @@
from .ir import * # pylint: disable=wildcard-import,redefined-builtin
from .ir import boolean as bool # pylint: disable=redefined-builtin
from .ir import buffer as Buffer
+from .utils import buffer_proxy, frame_scope, seq_scope
diff --git a/python/tvm/script/ir_builder/tir/utils.py b/python/tvm/script/ir_builder/tir/utils.py
new file mode 100644
index 0000000000..9243887961
--- /dev/null
+++ b/python/tvm/script/ir_builder/tir/utils.py
@@ -0,0 +1,264 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Utility helpers for TIR IRBuilder."""
+
+import contextlib
+from typing import List
+
+from tvm import tir
+from tvm.tir import Buffer
+
+from . import frame, ir as T
+
+
+class _FrameScope:
+ """Context manager to enter multiple IRBuilder frames without deep nesting.
+
+ This class allows entering multiple frames (e.g., T.allocate) in a single
+ `with` statement, avoiding the pyramid of nested context managers.
+
+ Parameters
+ ----------
+ frames : List[IRBuilderFrame]
+ The list of frames to enter.
+
+ Examples
+ --------
+ Instead of deeply nested allocations:
+
+ .. code-block:: python
+
+ with T.allocate([1], "int32", scope="local") as lo:
+ with T.allocate([1], "int32", scope="local") as hi:
+ # code here
+
+ Use frame_scope for flat structure:
+
+ .. code-block:: python
+
+ with frame_scope([
+ T.allocate([1], "int32", scope="local"),
+ T.allocate([1], "int32", scope="local"),
+ ]) as (lo, hi):
+ # code here
+ """
+
+ def __init__(self, frames):
+ self.frames = frames if isinstance(frames, (list, tuple)) else [frames]
+ self._stack = None
+
+ def __enter__(self):
+ self._stack = contextlib.ExitStack()
+ self._stack.__enter__()
+ results = [self._stack.enter_context(f) for f in self.frames]
+ return tuple(results) if len(results) > 1 else results[0]
+
+ def __exit__(self, *args):
+ return self._stack.__exit__(*args)
+
+
+def frame_scope(frames: List[frame.TIRFrame]) -> _FrameScope:
+ """Enter multiple IRBuilder frames without deep nesting.
+
+ This function provides a way to enter multiple frames in a single `with`
+ statement, which is particularly useful when migrating from cases where
+ allocations don't require nested scopes.
+
+ Parameters
+ ----------
+ frames : List[frame.TIRFrame]
+ The list of frames to enter. Each frame's `__enter__` return value
+ will be collected and returned as a tuple.
+
+ Returns
+ -------
+ _FrameScope
+ A context manager that enters all frames and returns their values.
+
+ Examples
+ --------
+ .. code-block:: python
+
+ from tvm.script.ir_builder import IRBuilder
+ from tvm.script.ir_builder import tir as T
+ from tvm.script.ir_builder.tir.utils import frame_scope
+
+ with IRBuilder() as ib:
+ with frame_scope([
+ T.allocate([1], "int32", scope="local"),
+ T.allocate([1], "int32", scope="local"),
+ T.allocate([size], dtype, scope="local"),
+ ]) as (lo, hi, temp):
+ # Use lo, hi, temp directly
+ T.buffer_store(lo, T.int32(0), [0])
+ ...
+ """
+ return _FrameScope(frames)
+
+
+def seq_scope():
+ """Create a scope that allows multiple consecutive statements.
+
+ The IRBuilder requires a parent frame when having multiple consecutive
+ top-level statements (e.g., multiple loops). This function creates a
+ dummy attr frame that serves as a parent scope.
+
+ Returns
+ -------
+ frame.AttrFrame
+ A dummy attribute frame that wraps multiple statements.
+
+ Examples
+ --------
+ Without seq_scope, multiple consecutive loops fail:
+
+ .. code-block:: python
+
+ with IRBuilder() as ib:
+ with T.serial(0, 10) as i:
+ T.evaluate(i)
+ with T.serial(0, 5) as j: # This would fail!
+ T.evaluate(j)
+
+ With seq_scope, multiple consecutive statements work:
+
+ .. code-block:: python
+
+ with IRBuilder() as ib:
+ with seq_scope():
+ with T.serial(0, 10) as i:
+ T.evaluate(i)
+ with T.serial(0, 5) as j:
+ T.evaluate(j)
+ result = ib.get()
+ """
+ return T.attr(tir.const(0, "int32"), "pragma_scope", tir.StringImm("seq"))
+
+
+def _unravel_index(index, shape):
+ """Convert a flat index to multi-dimensional indices.
+
+ Parameters
+ ----------
+ index : PrimExpr
+ The flat index.
+ shape : Tuple
+ The shape of the buffer.
+
+ Returns
+ -------
+ List[PrimExpr]
+ The multi-dimensional indices.
+ """
+ indices = []
+ for i, dim in enumerate(reversed(shape)):
+ if i == len(shape) - 1:
+ # Outermost dimension: use remaining quotient directly (no modulo)
+ indices.append(index)
+ else:
+ indices.append(index % dim)
+ index = index // dim
+ return list(reversed(indices))
+
+
+class _BufferProxy:
+ """Proxy for flat indexing on multi-dimensional buffers.
+
+ This class wraps a TIR Buffer and provides flat indexing that gets
+ automatically converted to multi-dimensional indices. It also supports
+ assignment syntax via __setitem__.
+
+ Parameters
+ ----------
+ buf : Buffer
+ The TIR buffer to wrap.
+
+ Examples
+ --------
+ .. code-block:: python
+
+ buf = tvm.tir.decl_buffer([2, 3], "float32")
+ ptr = buffer_proxy(buf)
+
+ # Read with flat index (converted to [0, 1])
+ val = ptr[1]
+
+ # Write with flat index
+ ptr[1] = 42.0
+
+ # Multi-dimensional access still works
+ val = ptr[0, 2]
+ """
+
+ def __init__(self, buf):
+ self._buffer = buf
+ self.dtype = buf.dtype
+ self.shape = buf.shape
+ self.name = buf.name
+ self.data = buf.data
+
+ def _normalize_index(self, index):
+ """Convert flat index to multi-dimensional indices if needed."""
+ try:
+ index = [*index]
+ except TypeError:
+ index = [index]
+ if len(index) == 1 and len(self._buffer.shape) != 1:
+ index = _unravel_index(index[0], self._buffer.shape)
+ return index
+
+ def __getitem__(self, index):
+ index = self._normalize_index(index)
+ return tir.BufferLoad(self._buffer, index)
+
+ def __setitem__(self, index, value):
+ index = self._normalize_index(index)
+ T.buffer_store(self._buffer, value, index)
+
+
+def buffer_proxy(buf: Buffer) -> _BufferProxy:
+ """Create a buffer proxy for flat indexing on multi-dimensional buffers.
+
+ This provides flat indexing that gets converted to multi-dimensional indices.
+ It also supports assignment syntax via __setitem__.
+
+ Parameters
+ ----------
+ buf : Buffer
+ The TIR buffer to wrap.
+
+ Returns
+ -------
+ _BufferProxy
+ A proxy object that supports flat indexing and assignment.
+
+ Examples
+ --------
+ .. code-block:: python
+
+ from tvm.script.ir_builder.tir.utils import buffer_proxy
+
+ buf = tvm.tir.decl_buffer([2, 3], "float32")
+ ptr = buffer_proxy(buf)
+
+ # Flat indexing (index 1 -> indices [0, 1])
+ val = ptr[1]
+
+ # Assignment syntax
+ ptr[1] = 42.0
+ """
+ return _BufferProxy(buf)
diff --git a/python/tvm/tir/__init__.py b/python/tvm/tir/__init__.py
index 244a26127d..8d50cccc01 100644
--- a/python/tvm/tir/__init__.py
+++ b/python/tvm/tir/__init__.py
@@ -115,7 +115,6 @@ from .op import dp4a
from .op import ignore_loop_partition
from .generic import add, subtract, multiply
-from . import ir_builder
from . import transform
from . import analysis
from . import backend
diff --git a/python/tvm/tir/ir_builder.py b/python/tvm/tir/ir_builder.py
deleted file mode 100644
index 1e9cb07830..0000000000
--- a/python/tvm/tir/ir_builder.py
+++ /dev/null
@@ -1,530 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""Developer API of IR node builder make function."""
-import tvm
-from tvm.runtime import ObjectConvertible, const
-from tvm.ir import container as _container
-
-from . import stmt as _stmt
-from . import expr as _expr
-from . import buffer as _buffer
-from . import op
-
-
-class WithScope(object):
- """Auxiliary scope with"""
-
- def __init__(self, enter_value, exit_cb):
- self._enter_value = enter_value
- self._exit_cb = exit_cb
-
- def __enter__(self):
- return self._enter_value
-
- def __exit__(self, ptype, value, trace):
- self._exit_cb()
-
-
-class BufferVar(ObjectConvertible):
- """Buffer variable with content type, makes load store easily.
-
- Do not create it directly, create use IRBuilder.
-
- Array access through a BufferVar must use the same number of
- indices as the underlying buffer was declared to have.
-
- Examples
- --------
- In the follow example, x is BufferVar.
- :code:`x[0] = ...` directly emit a BufferStore to the IRBuilder,
- :code:`x[10]` translates to BufferLoad.
-
- .. code-block:: python
-
- # The following code generate IR for x[0] = x[10] + 1
- ib = tvm.tir.ir_builder.create()
- x = ib.allocate("float32", 20)
- x[0] = x[10] + 1
-
- # Array access using a multidimensional index
- y = ib.allocate("float32", (32, 32))
- y[2, 31] = 0.
-
- See Also
- --------
- IRBuilder.pointer
- IRBuilder.allocate
-
- """
-
- def __init__(self, builder, buffer, content_type):
- self._builder = builder
- self._buffer = buffer
- self._content_type = content_type
-
- def asobject(self):
- return self._buffer
-
- @property
- def dtype(self):
- return self._content_type
-
- def _normalize_index(self, index):
- try:
- index = [*index]
- except TypeError:
- index = [index]
-
- index = [x.var if isinstance(x, _expr.IterVar) else x for x in index]
-
- # Workaround to support previous behavior of ir_builder
- # indexing by a single index, treating the buffer as if were
- # already flattened.
- if len(index) == 1 and len(self._buffer.shape) != 1:
- index = tvm.topi.utils.unravel_index(index[0], self._buffer.shape)
-
- return index
-
- def __getitem__(self, index):
- index = self._normalize_index(index)
- return _expr.BufferLoad(self._buffer, index)
-
- def __setitem__(self, index, value):
- index = self._normalize_index(index)
-
- if isinstance(value, (int, bool, float)):
- value = tvm.tir.const(value)
-
- value_element = value.dtype.split("x", maxsplit=1)[0]
- content_element = self._content_type.split("x", maxsplit=1)[0]
- if value_element != content_element:
- raise ValueError(
- f"data type does not match content type {value.dtype} vs {self._content_type}"
- )
-
- self._builder.emit(_stmt.BufferStore(self._buffer, value, index))
-
-
-class IRBuilder(object):
- """Auxiliary builder to build IR for testing and dev.
-
- Examples
- --------
- .. code-block:: python
-
- ib = tvm.tir.ir_builder.create()
- n = te.var("n")
- A = ib.allocate("float32", n, name="A")
- with ib.for_range(0, n, name="i") as i:
- with ib.if_scope((i % 2) == 0):
- A[i] = A[i] + 1
- # The result stmt.
- stmt = ib.get()
- """
-
- def __init__(self):
- self._seq_stack = [[]] # type: ignore
- self.nidx = 0
-
- def _pop_seq(self):
- """Pop sequence from stack"""
- seq = self._seq_stack.pop()
- if not seq or callable(seq[-1]):
- seq.append(_stmt.Evaluate(0))
- seqwrap = lambda x: x[0] if len(x) == 1 else _stmt.SeqStmt(list(reversed(x)))
- ret_seq = [seq[-1]]
-
- for s in reversed(seq[:-1]):
- if callable(s):
- ret_seq = [s(seqwrap(ret_seq))]
- else:
- assert isinstance(s, _stmt.Stmt)
- ret_seq.append(s)
- return seqwrap(ret_seq)
-
- def emit(self, stmt):
- """Emit a statement to the end of current scope.
-
- Parameters
- ----------
- stmt : Stmt or callable.
- The statement to be emitted or callable that build stmt given body.
- """
- if isinstance(stmt, _expr.Call):
- stmt = _stmt.Evaluate(stmt)
- assert isinstance(stmt, _stmt.Stmt) or callable(stmt)
- self._seq_stack[-1].append(stmt)
-
- def scope_attr(self, node, attr_key, value):
- """Create an AttrStmt at current scope.
-
- Parameters
- ----------
- attr_key : str
- The key of the attribute type.
-
- node : Node
- The attribute node to annottate on.
-
- value : Expr
- Attribute value.
-
- Examples
- --------
- .. code-block:: python
-
- ib = tvm.tir.ir_builder.create()
- i = te.var("i")
- x = ib.pointer("float32")
- ib.scope_attr(x, "storage_scope", "global")
- x[i] = x[i - 1] + 1
- """
- if isinstance(node, str):
- node = _expr.StringImm(node)
- if isinstance(value, str):
- value = _expr.StringImm(value)
- # thread_extent could be zero for dynamic workloads
- if attr_key == "thread_extent":
- value = op.max(1, value)
- self.emit(lambda x: _stmt.AttrStmt(node, attr_key, value, x))
-
- def for_range(self, begin, end, name="i", dtype=None, kind="serial", step=None):
- """Create a for iteration scope.
-
- Parameters
- ----------
- begin : Expr
- The min iteration scope.
-
- end : Expr
- The end iteration scope
-
- name : str, optional
- The name of iteration variable, if no input names,
- using typical index names i, j, k, then i_nidx
-
- dtype : str, optional
- The data type of iteration variable.
-
- kind : str, optional
- The special tag on the for loop.
-
- step : PrimExpr
- The loop step. Default to none which
- represent one.
-
- Returns
- -------
- loop_scope : With.Scope of Var
- The for scope, when enters returns loop_var
-
- Examples
- --------
- .. code-block:: python
-
- ib = tvm.tir.ir_builder.create()
- x = ib.pointer("float32")
- with ib.for_range(1, 10, name="i") as i:
- x[i] = x[i - 1] + 1
- """
- if name == "i":
- name = chr(ord(name) + self.nidx) if self.nidx < 3 else name + "_" + str(self.nidx - 3)
- self.nidx += 1
- self._seq_stack.append([])
-
- # auto infer dtype when it's not specified
- def get_dtype(expr):
- if isinstance(expr, _expr.PrimExpr):
- if not expr.dtype.startswith("int"):
- raise NotImplementedError(
- f"Infer loop_var dtype failed:"
- f" unsupported dtype in loop begin or end {expr.dtype}"
- )
- return expr.dtype
- if isinstance(expr, int):
- return "int32"
- raise NotImplementedError(
- f"Infer loop_var dtype failed:"
- f" unsupported dtype in loop begin or end {expr.dtype}"
- )
-
- if dtype is None:
- dtype = "int64" if "int64" in [get_dtype(begin), get_dtype(end)] else "int32"
-
- loop_var = _expr.Var(name, dtype=dtype)
- extent = end if begin == 0 else (end - begin)
-
- def _exit_cb():
- if kind == "serial":
- kind_id = _stmt.ForKind.SERIAL
- elif kind == "parallel":
- kind_id = _stmt.ForKind.PARALLEL
- elif kind == "vectorize":
- kind_id = _stmt.ForKind.VECTORIZED
- elif kind == "unroll":
- kind_id = _stmt.ForKind.UNROLLED
- else:
- raise ValueError("Unknown kind")
- self.emit(_stmt.For(loop_var, begin, extent, kind_id, self._pop_seq(), step=step))
-
- return WithScope(loop_var, _exit_cb)
-
- def while_loop(self, condition):
- """Create a while loop scope.
-
- Parameters
- ----------
- condition : Expr
- The termination condition.
-
- Returns
- -------
- loop_scope : With.Scope of Var
- The while scope.
-
- Examples
- --------
- .. code-block:: python
-
- ib = tvm.tir.ir_builder.create()
- iterations = ib.allocate("int32", (1,), name="iterations", scope="local")
- with ib.while_loop(iterations[0] < 10):
- iterations[0] += 1
- """
- self._seq_stack.append([])
-
- def _exit_cb():
- self.emit(_stmt.While(condition, self._pop_seq()))
-
- return WithScope(None, _exit_cb)
-
- def if_scope(self, cond):
- """Create an if scope.
-
- Parameters
- ----------
- cond : Expr
- The condition.
-
- Returns
- -------
- if_scope : WithScope
- The result if scope.
-
- Examples
- --------
- .. code-block:: python
-
- ib = tvm.tir.ir_builder.create()
- i = te.var("i")
- x = ib.pointer("float32")
- with ib.if_scope((i % 2) == 0):
- x[i] = x[i - 1] + 1
- """
- self._seq_stack.append([])
-
- def _exit_cb():
- self.emit(_stmt.IfThenElse(cond, self._pop_seq(), None))
-
- return WithScope(None, _exit_cb)
-
- def else_scope(self):
- """Create an else scope.
-
- This can only be used right after an if scope.
-
- Returns
- -------
- else_scope : WithScope
- The result else scope.
-
- Examples
- --------
- .. code-block:: python
-
- ib = tvm.tir.ir_builder.create()
- i = te.var("i")
- x = ib.pointer("float32")
- with ib.if_scope((i % 2) == 0):
- x[i] = x[i - 1] + 1
- with ib.else_scope():
- x[i] = x[i - 1] + 2
- """
- if not self._seq_stack[-1]:
- raise RuntimeError("else_scope can only follow an if_scope")
- prev = self._seq_stack[-1][-1]
- if not isinstance(prev, _stmt.IfThenElse) or prev.else_case:
- raise RuntimeError("else_scope can only follow an if_scope")
- self._seq_stack[-1].pop()
- self._seq_stack.append([])
-
- def _exit_cb():
- self.emit(_stmt.IfThenElse(prev.condition, prev.then_case, self._pop_seq()))
-
- return WithScope(None, _exit_cb)
-
- def new_scope(self):
- """Create new scope,
-
- this is useful to set boundary of attr and allocate.
-
- Returns
- -------
- new_scope : WithScope
- The result new scope.
- """
- self._seq_stack.append([])
-
- def _exit_cb():
- self.emit(self._pop_seq())
-
- return WithScope(None, _exit_cb)
-
- def let(self, var_name, value):
- """Create a new let stmt binding.
-
- Parameters
- ----------
- var_name : str
- The name of the variable
-
- value : PrimExpr
- The value to be bound
-
- Returns
- -------
- var : tvm.tir.Var
- The var that can be in for future emits.
- """
- var = _expr.Var(var_name, dtype=value.dtype)
- self.emit(lambda x: _stmt.LetStmt(var, value, x))
- return var
-
- def allocate(self, dtype, shape, name="buf", axis_separators=None, scope=""):
- """Create a allocate statement.
-
- Parameters
- ----------
- dtype : str
- The content data type.
-
- shape : tuple of Expr
- The shape of array to be allocated.
-
- name : str, optional
- The name of the buffer.
-
- axis_separators : list of int, optional
-
- If passed, a list of separators between groups of axes,
- each of which is flattened to an output axis. For flat
- memory spaces, should either be None, or an empty list.
-
- scope : str, optional
- The scope of the buffer.
-
-
- Returns
- -------
- buffer : BufferVar
- The buffer var representing the buffer.
-
- """
- if not isinstance(shape, (list, tuple, _container.Array)):
- shape = [shape]
-
- buffer = _buffer.decl_buffer(
- shape, dtype, name, scope=scope, axis_separators=axis_separators
- )
-
- buffer_var = buffer.data
- self.emit(lambda x: _stmt.Allocate(buffer_var, dtype, shape, const(1, dtype="bool"), x))
- return BufferVar(self, buffer, dtype)
-
- def pointer(self, content_type, name="ptr", scope=""):
- """Create pointer variable with content type.
-
- Parameters
- ----------
- content_type : str
- The content data type.
-
- name : str, optional
- The name of the pointer.
-
- scope : str, optional
- The scope of the pointer.
-
- Returns
- -------
- ptr : BufferVar
- The buffer var representing the buffer.
- """
- buffer = _buffer.decl_buffer(shape=[1], dtype=content_type, name=name, scope=scope)
- return BufferVar(self, buffer, content_type)
-
- def buffer_ptr(self, buf):
- """Create pointer variable corresponds to buffer ptr.
-
- Parameters
- ----------
- buf : Buffer
- The buffer to be extracted.
-
- Returns
- -------
- ptr : BufferVar
- The buffer var representing the buffer.
- """
- return BufferVar(self, buf, buf.dtype)
-
- def likely(self, expr):
- """Add likely tag for expression.
- Parameters
- ----------
- expr : Expr
- The expression. Usually a condition expression.
- Returns
- -------
- expr : Expr
- The expression will likely tag.
- """
- return _expr.Call(expr.dtype, "tir.likely", [expr])
-
- def get(self):
- """Return the builded IR.
-
- Returns
- -------
- stmt : Stmt
- The result statement.
- """
- seq = self._pop_seq()
- if self._seq_stack:
- raise RuntimeError("cannot call get inside construction scope")
- return seq
-
-
-def create():
- """Create a new IRBuilder
-
- Returns
- -------
- builder : IRBuilder
- The created IRBuilder
- """
- return IRBuilder()
diff --git a/python/tvm/topi/gpu/scan.py b/python/tvm/topi/gpu/scan.py
index 5be4033e45..643d69d83c 100644
--- a/python/tvm/topi/gpu/scan.py
+++ b/python/tvm/topi/gpu/scan.py
@@ -21,6 +21,8 @@ from typing import Callable, Optional, Union
import tvm
from tvm import te
from tvm.contrib.thrust import can_use_rocthrust, can_use_thrust
+from tvm.script.ir_builder import IRBuilder
+from tvm.script.ir_builder import tir as T
from ..math import cast, ceil_log2
from ..transform import expand_dims, reshape, squeeze, transpose
@@ -77,119 +79,164 @@ def exclusive_scan_ir(data, output, reduction=None, binop=tvm.tir.generic.add, i
batch_size = cast(prod(data.shape[:-1]), "int32")
scan_axis_size = cast(data.shape[-1], "int32")
- ib = tvm.tir.ir_builder.create()
+ with IRBuilder() as ib:
+ data = T.buffer_proxy(data)
+ output = T.buffer_proxy(output)
+ out_dtype = output.dtype
- data = ib.buffer_ptr(data)
- output = ib.buffer_ptr(output)
+ if reduction is not None:
+ reduction = T.buffer_proxy(reduction)
- out_dtype = output.dtype
-
- if reduction is not None:
- reduction = ib.buffer_ptr(reduction)
-
- max_threads = int(tvm.target.Target.current(allow_none=False).max_num_threads)
+ max_threads = int(tvm.target.Target.current(allow_none=False).max_num_threads)
- with ib.if_scope(scan_axis_size == 0):
- with ib.new_scope():
- bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(bx, "thread_extent", batch_size)
- with ib.if_scope(bx < batch_size):
- if reduction is not None:
- reduction[bx] = cast(identity_value, out_dtype)
- with ib.else_scope():
- with ib.new_scope():
- nthread_tx = max_threads
- nthread_bx = ceil_div(scan_axis_size, max_threads)
- nthread_by = batch_size
- tx = te.thread_axis("threadIdx.x")
- bx = te.thread_axis("blockIdx.x")
- by = te.thread_axis("blockIdx.y")
- ib.scope_attr(tx, "thread_extent", nthread_tx)
- ib.scope_attr(bx, "thread_extent", nthread_bx)
- ib.scope_attr(by, "thread_extent", nthread_by)
- tid = bx * nthread_tx + tx
- with ib.if_scope(tid < scan_axis_size):
- output[by * scan_axis_size + tid] = cast(data[by * scan_axis_size + tid], out_dtype)
-
- nthread_tx = max_threads
- nthread_bx = ceil_div(scan_axis_size, max_threads)
- nthread_by = batch_size
-
- # The following algorithm performs parallel exclusive scan
- # Up Sweep of exclusive scan
- lim = ceil_log2(scan_axis_size)
-
- with ib.for_range(0, cast(lim, "int32"), dtype="int32") as l2_width:
- width = 2 << l2_width
-
- with ib.new_scope():
+ with T.If(scan_axis_size == 0):
+ with T.Then():
+ bx = te.thread_axis("blockIdx.x")
+ with T.attr(bx, "thread_extent", batch_size):
+ with T.If(bx < batch_size):
+ with T.Then():
+ if reduction is not None:
+ reduction[bx] = cast(identity_value, out_dtype)
+ with T.Else():
+ nthread_tx = max_threads
+ nthread_bx = ceil_div(scan_axis_size, max_threads)
+ nthread_by = batch_size
+
+ # Copy data to output
tx = te.thread_axis("threadIdx.x")
bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", nthread_tx)
- ib.scope_attr(
- bx,
- "thread_extent",
- tvm.tir.generic.cast(ceil_div(scan_axis_size, max_threads * width), "int32"),
- )
- tid = bx * nthread_tx + tx
-
by = te.thread_axis("blockIdx.y")
- ib.scope_attr(by, "thread_extent", nthread_by)
- start = ib.allocate("int32", (1,), name="start", scope="local")
- middle = ib.allocate("int32", (1,), name="middle", scope="local")
- end = ib.allocate("int32", (1,), name="end", scope="local")
- start[0] = width * tid
- with ib.if_scope(start[0] < scan_axis_size):
- middle[0] = start[0] + tvm.tir.indexdiv(width, 2)
- end[0] = tvm.te.min(start[0] + width, scan_axis_size)
- with ib.if_scope(middle[0] < scan_axis_size):
- output[by * scan_axis_size + end[0] - 1] = binop(
- output[by * scan_axis_size + end[0] - 1],
- output[by * scan_axis_size + middle[0] - 1],
+ with T.frame_scope(
+ [
+ T.attr(tx, "thread_extent", nthread_tx),
+ T.attr(bx, "thread_extent", nthread_bx),
+ T.attr(by, "thread_extent", nthread_by),
+ ]
+ ):
+ tid = bx * nthread_tx + tx
+ with T.If(tid < scan_axis_size):
+ with T.Then():
+ output[by * scan_axis_size + tid] = cast(
+ data[by * scan_axis_size + tid], out_dtype
+ )
+
+ # The following algorithm performs parallel exclusive scan
+ # Up Sweep of exclusive scan
+ lim = ceil_log2(scan_axis_size)
+
+ with T.serial(0, cast(lim, "int32")) as l2_width:
+ width = 2 << l2_width
+
+ tx = te.thread_axis("threadIdx.x")
+ bx = te.thread_axis("blockIdx.x")
+ by = te.thread_axis("blockIdx.y")
+ with T.frame_scope(
+ [
+ T.attr(tx, "thread_extent", nthread_tx),
+ T.attr(
+ bx,
+ "thread_extent",
+ tvm.tir.generic.cast(
+ ceil_div(scan_axis_size, max_threads * width), "int32"
+ ),
+ ),
+ T.attr(by, "thread_extent", nthread_by),
+ T.allocate([1], "int32", scope="local"),
+ T.allocate([1], "int32", scope="local"),
+ T.allocate([1], "int32", scope="local"),
+ ]
+ ) as (_, _, _, start_ptr, middle_ptr, end_ptr):
+ tid = bx * nthread_tx + tx
+ start = T.buffer_proxy(
+ tvm.tir.decl_buffer(
+ [1], "int32", "start", data=start_ptr, scope="local"
+ )
)
-
- # Down Sweep of exclusive scan
- with ib.new_scope():
- bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(bx, "thread_extent", batch_size)
- with ib.if_scope(bx < batch_size):
- if reduction is not None:
- reduction[bx] = output[(bx + 1) * scan_axis_size - 1]
- output[(bx + 1) * scan_axis_size - 1] = cast(identity_value, out_dtype)
-
- with ib.for_range(0, cast(lim, "int32"), dtype="int32") as l2_width:
- width = 2 << (lim - l2_width - 1)
-
- with ib.new_scope():
- tx = te.thread_axis("threadIdx.x")
+ middle = T.buffer_proxy(
+ tvm.tir.decl_buffer(
+ [1], "int32", "middle", data=middle_ptr, scope="local"
+ )
+ )
+ end = T.buffer_proxy(
+ tvm.tir.decl_buffer([1], "int32", "end", data=end_ptr, scope="local")
+ )
+ start[0] = width * tid
+ with T.If(start[0] < scan_axis_size):
+ with T.Then():
+ middle[0] = start[0] + tvm.tir.indexdiv(width, 2)
+ end[0] = tvm.te.min(start[0] + width, scan_axis_size)
+ with T.If(middle[0] < scan_axis_size):
+ with T.Then():
+ output[by * scan_axis_size + end[0] - 1] = binop(
+ output[by * scan_axis_size + end[0] - 1],
+ output[by * scan_axis_size + middle[0] - 1],
+ )
+
+ # Down Sweep of exclusive scan
bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", nthread_tx)
- ib.scope_attr(
- bx,
- "thread_extent",
- tvm.tir.generic.cast(ceil_div(scan_axis_size, max_threads * width), "int32"),
- )
- tid = bx * nthread_tx + tx
-
- by = te.thread_axis("blockIdx.y")
- ib.scope_attr(by, "thread_extent", nthread_by)
- start = ib.allocate("int32", (1,), name="start", scope="local")
- middle = ib.allocate("int32", (1,), name="middle", scope="local")
- end = ib.allocate("int32", (1,), name="end", scope="local")
- tmp = ib.allocate(out_dtype, (1,), name="end", scope="local")
- start[0] = width * tid
- with ib.if_scope(tvm.tir.all(start[0] < scan_axis_size)):
- middle[0] = start[0] + tvm.tir.indexdiv(width, 2)
- end[0] = tvm.tir.min(start[0] + width, scan_axis_size)
- with ib.if_scope(middle[0] < scan_axis_size):
- tmp[0] = output[by * scan_axis_size + middle[0] - 1]
- output[by * scan_axis_size + middle[0] - 1] = output[
- by * scan_axis_size + end[0] - 1
+ with T.attr(bx, "thread_extent", batch_size):
+ with T.If(bx < batch_size):
+ with T.Then():
+ if reduction is not None:
+ reduction[bx] = output[(bx + 1) * scan_axis_size - 1]
+ output[(bx + 1) * scan_axis_size - 1] = cast(identity_value, out_dtype)
+
+ with T.serial(0, cast(lim, "int32")) as l2_width:
+ width = 2 << (lim - l2_width - 1)
+
+ tx = te.thread_axis("threadIdx.x")
+ bx = te.thread_axis("blockIdx.x")
+ by = te.thread_axis("blockIdx.y")
+ with T.frame_scope(
+ [
+ T.attr(tx, "thread_extent", nthread_tx),
+ T.attr(
+ bx,
+ "thread_extent",
+ tvm.tir.generic.cast(
+ ceil_div(scan_axis_size, max_threads * width), "int32"
+ ),
+ ),
+ T.attr(by, "thread_extent", nthread_by),
+ T.allocate([1], "int32", scope="local"),
+ T.allocate([1], "int32", scope="local"),
+ T.allocate([1], "int32", scope="local"),
+ T.allocate([1], out_dtype, scope="local"),
]
- output[by * scan_axis_size + end[0] - 1] = binop(
- output[by * scan_axis_size + end[0] - 1], tmp[0]
+ ) as (_, _, _, start_ptr, middle_ptr, end_ptr, tmp_ptr):
+ tid = bx * nthread_tx + tx
+ start = T.buffer_proxy(
+ tvm.tir.decl_buffer(
+ [1], "int32", "start", data=start_ptr, scope="local"
+ )
+ )
+ middle = T.buffer_proxy(
+ tvm.tir.decl_buffer(
+ [1], "int32", "middle", data=middle_ptr, scope="local"
+ )
+ )
+ end = T.buffer_proxy(
+ tvm.tir.decl_buffer([1], "int32", "end", data=end_ptr, scope="local")
)
- return ib.get()
+ tmp = T.buffer_proxy(
+ tvm.tir.decl_buffer([1], out_dtype, "tmp", data=tmp_ptr, scope="local")
+ )
+ start[0] = width * tid
+ with T.If(tvm.tir.all(start[0] < scan_axis_size)):
+ with T.Then():
+ middle[0] = start[0] + tvm.tir.indexdiv(width, 2)
+ end[0] = tvm.tir.min(start[0] + width, scan_axis_size)
+ with T.If(middle[0] < scan_axis_size):
+ with T.Then():
+ tmp[0] = output[by * scan_axis_size + middle[0] - 1]
+ output[by * scan_axis_size + middle[0] - 1] = output[
+ by * scan_axis_size + end[0] - 1
+ ]
+ output[by * scan_axis_size + end[0] - 1] = binop(
+ output[by * scan_axis_size + end[0] - 1], tmp[0]
+ )
+
+ return ib.get()
def get_reduction_from_exclusive_scan(data, ex_scan_output, binop=tvm.tir.generic.add):
@@ -219,35 +266,40 @@ def get_reduction_from_exclusive_scan(data, ex_scan_output, binop=tvm.tir.generi
data = expand_dims(data, axis=0)
ex_scan_output = expand_dims(ex_scan_output, axis=0)
- def ir(data, data_ex_scan, reduction):
- batch_size = cast(prod(data.shape[:-1]), "int32")
- scan_axis_size = cast(data.shape[-1], "int32")
+ def ir(data_buf, data_ex_scan_buf, reduction_buf):
+ batch_size = cast(prod(data_buf.shape[:-1]), "int32")
+ scan_axis_size = cast(data_buf.shape[-1], "int32")
- ib = tvm.tir.ir_builder.create()
+ max_threads = int(tvm.target.Target.current(allow_none=False).max_num_threads)
- data = ib.buffer_ptr(data)
- data_ex_scan = ib.buffer_ptr(data_ex_scan)
- reduction = ib.buffer_ptr(reduction)
+ with IRBuilder() as ib:
+ data = T.buffer_proxy(data_buf)
+ data_ex_scan = T.buffer_proxy(data_ex_scan_buf)
+ reduction = T.buffer_proxy(reduction_buf)
- max_threads = int(tvm.target.Target.current(allow_none=False).max_num_threads)
- with ib.new_scope():
nthread_tx = max_threads
nthread_bx = ceil_div(batch_size, max_threads)
tx = te.thread_axis("threadIdx.x")
bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", nthread_tx)
- ib.scope_attr(bx, "thread_extent", nthread_bx)
- tid = bx * max_threads + tx
- with ib.if_scope(tid < batch_size):
- with ib.if_scope(scan_axis_size > 0):
- reduction[tid] = binop(
- data_ex_scan[tid * scan_axis_size + scan_axis_size - 1],
- data[tid * scan_axis_size + scan_axis_size - 1],
- )
- with ib.else_scope():
- reduction[tid] = cast(0, reduction.dtype)
-
- return ib.get()
+ with T.frame_scope(
+ [
+ T.attr(tx, "thread_extent", nthread_tx),
+ T.attr(bx, "thread_extent", nthread_bx),
+ ]
+ ):
+ tid = bx * max_threads + tx
+ with T.If(tid < batch_size):
+ with T.Then():
+ with T.If(scan_axis_size > 0):
+ with T.Then():
+ reduction[tid] = binop(
+ data_ex_scan[tid * scan_axis_size + scan_axis_size - 1],
+ data[tid * scan_axis_size + scan_axis_size - 1],
+ )
+ with T.Else():
+ reduction[tid] = cast(0, reduction_buf.dtype)
+
+ return ib.get()
data_buf = tvm.tir.decl_buffer(data.shape, data.dtype, "valid_indices_buf", data_alignment=8)
ex_scan_output_buf = tvm.tir.decl_buffer(
diff --git a/python/tvm/topi/gpu/sort.py b/python/tvm/topi/gpu/sort.py
index 807b23a956..5a25007f12 100644
--- a/python/tvm/topi/gpu/sort.py
+++ b/python/tvm/topi/gpu/sort.py
@@ -15,9 +15,11 @@
# specific language governing permissions and limitations
# under the License.
# pylint: disable=invalid-name, no-member, too-many-locals, too-many-arguments, too-many-statements, singleton-comparison, unused-argument, no-else-return
-"""Sort related operators """
+"""Sort related operators"""
import tvm
from tvm import te
+from tvm.script.ir_builder import IRBuilder
+from tvm.script.ir_builder import tir as T
from ..transform import strided_slice, transpose
from ..utils import ceil_div, swap, prod
@@ -25,19 +27,14 @@ from ..math import cast, ceil_log2
from ..searchsorted import binary_search
-def _get_threads(ib, nthread_tx, nthread_bx, nthread_by):
+def _get_threads(nthread_tx, nthread_bx, nthread_by):
tx = te.thread_axis("threadIdx.x")
bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", nthread_tx)
- ib.scope_attr(bx, "thread_extent", nthread_bx)
-
by = te.thread_axis("blockIdx.y")
- ib.scope_attr(by, "thread_extent", nthread_by)
-
- return tx, bx, by
+ return tx, bx, by, nthread_tx, nthread_bx, nthread_by
-def _sort_init(ib, shape, axis, keys_in, keys_out, values_out=None, value_init_func=None):
+def _sort_init(shape, axis, keys_in, keys_out, values_out=None, value_init_func=None):
"""Initialize the output buffers by copying from inputs"""
axis_mul_before = 1
axis_mul_after = 1
@@ -56,15 +53,23 @@ def _sort_init(ib, shape, axis, keys_in, keys_out, values_out=None, value_init_f
nthread_by = axis_mul_before * axis_mul_after
# Copy the keys_in to initial output
- with ib.new_scope():
- tx, bx, by = _get_threads(ib, nthread_tx, nthread_bx, nthread_by)
+ tx, bx, by, ntx, nbx, nby = _get_threads(nthread_tx, nthread_bx, nthread_by)
+ with T.frame_scope(
+ [
+ T.attr(tx, "thread_extent", ntx),
+ T.attr(bx, "thread_extent", nbx),
+ T.attr(by, "thread_extent", nby),
+ ]
+ ):
tid = bx * nthread_tx + tx
- by, bz = by % axis_mul_before, by // axis_mul_before
- idx = (by * shape[axis] + tid) * axis_mul_after + bz
- with ib.if_scope(tid < shape[axis]):
- keys_out[idx] = keys_in[idx]
- if values_out is not None:
- values_out[idx] = value_init_func(idx, tid)
+ by_val = by % axis_mul_before
+ bz = by // axis_mul_before
+ idx = (by_val * shape[axis] + tid) * axis_mul_after + bz
+ with T.If(tid < shape[axis]):
+ with T.Then():
+ keys_out[idx] = keys_in[idx]
+ if values_out is not None:
+ values_out[idx] = value_init_func(idx, tid)
return axis_mul_before, axis_mul_after
@@ -76,7 +81,6 @@ thread_work = 4
def _odd_even_sort(
- ib,
size,
axis_mul_before,
axis_mul_after,
@@ -89,81 +93,149 @@ def _odd_even_sort(
nthread_tx = block_size // 2
nthread_bx = ceil_div(size, block_size)
nthread_by = axis_mul_before * axis_mul_after
- with ib.new_scope():
- ib.scope_attr(tvm.tir.const(0), "hand_threaded", 0)
- tx, bx, by = _get_threads(ib, nthread_tx, nthread_bx, nthread_by)
- by, bz = by % axis_mul_before, by // axis_mul_before
+
+ tx, bx, by, ntx, nbx, nby = _get_threads(nthread_tx, nthread_bx, nthread_by)
+ with T.frame_scope(
+ [
+ T.attr(tvm.tir.const(0), "hand_threaded", 0),
+ T.attr(tx, "thread_extent", ntx),
+ T.attr(bx, "thread_extent", nbx),
+ T.attr(by, "thread_extent", nby),
+ ]
+ ):
+ by_val = by % axis_mul_before
+ bz = by // axis_mul_before
tid = 2 * tx
start = bx * block_size
- ## Create shared memory as syncable thread scratch space
- tmp_keys_swap = ib.allocate(
- keys_swap.dtype,
- (block_size,),
- name="temp_keys_swap",
- scope="shared",
- )
+ # Build list of allocations
+ alloc_frames = [
+ T.allocate([block_size], keys_swap.dtype, scope="shared"), # tmp_keys_swap
+ T.allocate([1], keys_swap.dtype, scope="local"), # temp_keys
+ T.allocate([1], keys_swap.dtype, scope="local"), # temp_cond1
+ T.allocate([1], keys_swap.dtype, scope="local"), # temp_cond2
+ ]
if values_swap is not None:
- tmp_values_swap = ib.allocate(
- values_swap.dtype,
- (block_size,),
- name="temp_values_swap",
+ alloc_frames.append(
+ T.allocate([block_size], values_swap.dtype, scope="shared")
+ ) # tmp_values_swap
+ alloc_frames.append(T.allocate([1], values_swap.dtype, scope="local")) # temp_values
+
+ with T.frame_scope(alloc_frames) as allocs:
+ if values_swap is not None:
+ (
+ tmp_keys_swap_ptr,
+ temp_keys_ptr,
+ temp_cond1_ptr,
+ temp_cond2_ptr,
+ tmp_values_swap_ptr,
+ temp_values_ptr,
+ ) = allocs
+ else:
+ (
+ tmp_keys_swap_ptr,
+ temp_keys_ptr,
+ temp_cond1_ptr,
+ temp_cond2_ptr,
+ ) = allocs
+ tmp_values_swap_ptr = None
+ temp_values_ptr = None
+
+ # Create buffer views
+ tmp_keys_swap = tvm.tir.decl_buffer(
+ [block_size],
+ keys_swap.dtype,
+ "tmp_keys_swap",
+ data=tmp_keys_swap_ptr,
scope="shared",
)
+ temp_keys = tvm.tir.decl_buffer(
+ [1], keys_swap.dtype, "temp_keys", data=temp_keys_ptr, scope="local"
+ )
+ temp_cond1 = tvm.tir.decl_buffer(
+ [1], keys_swap.dtype, "temp_cond1", data=temp_cond1_ptr, scope="local"
+ )
+ temp_cond2 = tvm.tir.decl_buffer(
+ [1], keys_swap.dtype, "temp_cond2", data=temp_cond2_ptr, scope="local"
+ )
+ if values_swap is not None:
+ tmp_values_swap = tvm.tir.decl_buffer(
+ [block_size],
+ values_swap.dtype,
+ "tmp_values_swap",
+ data=tmp_values_swap_ptr,
+ scope="shared",
+ )
+ temp_values = tvm.tir.decl_buffer(
+ [1],
+ values_swap.dtype,
+ "temp_values",
+ data=temp_values_ptr,
+ scope="local",
+ )
+
+ # Copy data to scratch space
+ base_idx = by_val * size * axis_mul_after + bz
+ with T.serial(0, 2) as n:
+ with T.If((tid + n + start) < size):
+ with T.Then():
+ T.buffer_store(
+ tmp_keys_swap,
+ keys[base_idx + (tid + n + start) * axis_mul_after],
+ [tid + n],
+ )
+ if values_swap is not None:
+ T.buffer_store(
+ tmp_values_swap,
+ values[base_idx + (tid + n + start) * axis_mul_after],
+ [tid + n],
+ )
- ## Create thread local data for swapping
- temp_keys = ib.allocate(keys_swap.dtype, (1,), name="temp_keys", scope="local")
- if values_swap is not None:
- temp_values = ib.allocate(values_swap.dtype, (1,), name="temp_values", scope="local")
-
- temp_cond1 = ib.allocate(keys_swap.dtype, (1,), name="temp_cond1", scope="local")
- temp_cond2 = ib.allocate(keys_swap.dtype, (1,), name="temp_cond2", scope="local")
- # Copy data to scratch space
- base_idx = by * size * axis_mul_after + bz
- with ib.for_range(0, 2) as n:
- with ib.if_scope((tid + n + start) < size):
- tmp_keys_swap[tid + n] = keys[base_idx + (tid + n + start) * axis_mul_after]
- if values_swap is not None:
- tmp_values_swap[tid + n] = values[base_idx + (tid + n + start) * axis_mul_after]
-
- ib.emit(tvm.tir.Call(None, "tir.tvm_storage_sync", tvm.runtime.convert(["shared"])))
-
- idxm = tvm.tir.indexmod
- # OddEvenTransposeSort
- current_sort_num = tvm.tir.min(block_size, size - start)
- with ib.for_range(0, current_sort_num) as k:
- n = idxm(tid + k, 2)
- with ib.if_scope(tid + n < current_sort_num - 1):
- temp_cond1[0] = tmp_keys_swap[tid + n]
- temp_cond2[0] = tmp_keys_swap[tid + n + 1]
- if is_ascend:
- cond = temp_cond1[0] > temp_cond2[0]
- else:
- cond = temp_cond1[0] < temp_cond2[0]
- with ib.if_scope(cond):
- temp_keys[0] = tmp_keys_swap[tid + n]
- tmp_keys_swap[tid + n] = tmp_keys_swap[tid + n + 1]
- tmp_keys_swap[tid + n + 1] = temp_keys[0]
- if values_swap is not None:
- temp_values[0] = tmp_values_swap[tid + n]
- tmp_values_swap[tid + n] = tmp_values_swap[tid + n + 1]
- tmp_values_swap[tid + n + 1] = temp_values[0]
- ib.emit(tvm.tir.Call(None, "tir.tvm_storage_sync", tvm.runtime.convert(["shared"])))
-
- ## Copy sorted data to output
- with ib.for_range(0, 2) as n:
- with ib.if_scope(tid + n + start < size):
- keys[base_idx + (tid + n + start) * axis_mul_after] = tmp_keys_swap[tid + n]
- keys_swap[base_idx + (tid + n + start) * axis_mul_after] = tmp_keys_swap[tid + n]
- if values_swap is not None:
- values[base_idx + (tid + n + start) * axis_mul_after] = tmp_values_swap[tid + n]
- values_swap[base_idx + (tid + n + start) * axis_mul_after] = tmp_values_swap[
- tid + n
- ]
+ T.evaluate(tvm.tir.Call(None, "tir.tvm_storage_sync", tvm.runtime.convert(["shared"])))
+
+ idxm = tvm.tir.indexmod
+ # OddEvenTransposeSort
+ current_sort_num = tvm.tir.min(block_size, size - start)
+ with T.serial(0, current_sort_num) as k:
+ n = idxm(tid + k, 2)
+ with T.If(tid + n < current_sort_num - 1):
+ with T.Then():
+ T.buffer_store(temp_cond1, tmp_keys_swap[tid + n], [0])
+ T.buffer_store(temp_cond2, tmp_keys_swap[tid + n + 1], [0])
+ if is_ascend:
+ cond = temp_cond1[0] > temp_cond2[0]
+ else:
+ cond = temp_cond1[0] < temp_cond2[0]
+ with T.If(cond):
+ with T.Then():
+ T.buffer_store(temp_keys, tmp_keys_swap[tid + n], [0])
+ T.buffer_store(tmp_keys_swap, tmp_keys_swap[tid + n + 1], [tid + n])
+ T.buffer_store(tmp_keys_swap, temp_keys[0], [tid + n + 1])
+ if values_swap is not None:
+ T.buffer_store(temp_values, tmp_values_swap[tid + n], [0])
+ T.buffer_store(
+ tmp_values_swap,
+ tmp_values_swap[tid + n + 1],
+ [tid + n],
+ )
+ T.buffer_store(tmp_values_swap, temp_values[0], [tid + n + 1])
+ T.evaluate(
+ tvm.tir.Call(None, "tir.tvm_storage_sync", tvm.runtime.convert(["shared"]))
+ )
+
+ ## Copy sorted data to output
+ with T.serial(0, 2) as n:
+ with T.If(tid + n + start < size):
+ with T.Then():
+ out_idx = base_idx + (tid + n + start) * axis_mul_after
+ keys[out_idx] = tmp_keys_swap[tid + n]
+ keys_swap[out_idx] = tmp_keys_swap[tid + n]
+ if values_swap is not None:
+ values[out_idx] = tmp_values_swap[tid + n]
+ values_swap[out_idx] = tmp_values_swap[tid + n]
def _sort_common(
- ib,
size,
axis_mul_before,
axis_mul_after,
@@ -186,15 +258,16 @@ def _sort_common(
## finds the start/end locations of the inner mergepath so that we can split
## the merge into more blocks
- max_threads = int(tvm.target.Target.current(allow_none=False).max_num_threads)
+ target = tvm.target.Target.current(allow_none=False)
+ max_threads = int(target.max_num_threads)
+ is_webgpu = "webgpu" in str(target)
+ target_dtype = "int32" if is_webgpu else "int64"
nthread_by = axis_mul_before * axis_mul_after
nthread_tx = max_threads
nthread_bx = ceil_div(size, nthread_tx)
def compare(a, b):
- """
- Compare a and b in proper ascending or descending order
- """
+ """Compare a and b in proper ascending or descending order"""
if is_ascend:
out = a <= b
else:
@@ -205,7 +278,6 @@ def _sort_common(
lower_lim = ceil_log2(block_size)
_odd_even_sort(
- ib,
size,
axis_mul_before * axis_mul_after,
1,
@@ -218,14 +290,7 @@ def _sort_common(
upper_lim = ceil_log2(size)
- def get_merge_begin(source, base_idx, aCount, bCount, aStart, bStart, diag, step_count):
- target = tvm.target.Target.current()
- is_webgpu = "webgpu" in str(target)
- target_dtype = "int32" if is_webgpu else "int64"
-
- first = ib.allocate(target_dtype, (1,), name="first", scope="local")
- mid = ib.allocate(target_dtype, (1,), name="mid", scope="local")
- last = ib.allocate(target_dtype, (1,), name="last", scope="local")
+ def get_merge_begin(source, base_idx, aCount, bCount, aStart, bStart, diag, first, last):
max_val = tvm.te.max(0, diag - bCount)
min_val = tvm.te.min(diag, aCount)
if is_webgpu:
@@ -234,16 +299,15 @@ def _sort_common(
else:
first[0] = max_val
last[0] = min_val
-
- with ib.while_loop(first[0] < last[0]):
+ with T.While(first[0] < last[0]):
mid = (first[0] + last[0]) >> 1
a = source[base_idx + (aStart + mid)]
b = source[base_idx + (bStart + diag - 1 - mid)]
- with ib.if_scope(compare(a, b)):
- first[0] = mid + 1
- with ib.else_scope():
- last[0] = mid
- return first[0], last[0]
+ with T.If(compare(a, b)):
+ with T.Then():
+ first[0] = mid + 1
+ with T.Else():
+ last[0] = mid
def serial_merge(
source,
@@ -260,223 +324,347 @@ def _sort_common(
step_count,
first,
last,
+ i_buf,
+ j_buf,
):
- target = tvm.target.Target.current()
- is_webgpu = "webgpu" in str(target)
- target_dtype = "int32" if is_webgpu else "int64"
- i = ib.allocate(target_dtype, (1,), name="i", scope="local")
- j = ib.allocate(target_dtype, (1,), name="j", scope="local")
- i_val = aStart + first
- j_val = bStart + diag - last
+ i_val = aStart + first[0]
+ j_val = bStart + diag - last[0]
if is_webgpu:
- i[0] = cast(i_val, target_dtype)
- j[0] = cast(j_val, target_dtype)
+ i_buf[0] = cast(i_val, target_dtype)
+ j_buf[0] = cast(j_val, target_dtype)
else:
- i[0] = i_val
- j[0] = j_val
+ i_buf[0] = i_val
+ j_buf[0] = j_val
- with ib.for_range(0, tvm.te.min(aCount + bCount - diag, step_count)) as count:
- i_idx = base_idx + i[0]
- j_idx = base_idx + j[0]
+ with T.serial(0, tvm.te.min(aCount + bCount - diag, step_count)) as count:
+ i_idx = base_idx + i_buf[0]
+ j_idx = base_idx + j_buf[0]
k_idx = base_idx + (kStart + diag + count)
- def assign_i():
- """assign i value to current output"""
- dest[k_idx] = source[i_idx]
- if values is not None:
- dest_idx[k_idx] = source_idx[i_idx]
- i[0] += 1
-
- def assign_j():
- """assign j value to current output"""
- dest[k_idx] = source[j_idx]
- if values is not None:
- dest_idx[k_idx] = source_idx[j_idx]
- j[0] += 1
-
- ## if both of the iterators are in range
- with ib.if_scope(tvm.tir.all(i[0] < aStart + aCount, j[0] < bStart + bCount)):
- # compare them and insert whichever is next into the output
- with ib.if_scope(compare(source[i_idx], source[j_idx])):
- assign_i()
- with ib.else_scope():
- assign_j()
- # otherwise, simply copy the remainder of the valid iterator to the output
- with ib.else_scope():
- with ib.if_scope(i[0] < aStart + aCount):
- assign_i()
- with ib.else_scope():
- assign_j()
-
- target = tvm.target.Target.current()
- target_dtype = "int32" if "webgpu" in str(target) else "int64"
- with ib.for_range(0, cast(upper_lim - lower_lim, target_dtype), dtype=target_dtype) as l2_width:
- width = 2 << (l2_width + lower_lim)
- # Define and launch the cuda kernel
- with ib.new_scope():
- target = tvm.target.Target.current()
- if "vulkan" in str(target):
- # Vulkan can't handle dynamic nthread, so we thread slightly differently
- # for vulkan. We don't do this generally because it causes a 15% perf
- # regression on other platforms
- ntx = max_threads
- nbx = tvm.tir.generic.cast(ceil_div(width, max_threads * thread_work), "int32")
- nbz = tvm.tir.generic.cast(ceil_div(size, width), "int32")
- tx, bx, by = _get_threads(ib, ntx, nbx, nthread_by * nbz)
- else:
- ntx = tvm.tir.generic.cast(tvm.te.min(max_threads, width), "int32")
- nbx = tvm.tir.generic.cast(ceil_div(width, max_threads * thread_work), "int32")
- nbz = tvm.tir.generic.cast(ceil_div(size, width), "int32")
- tx, bx, by = _get_threads(ib, ntx, nbx, nthread_by * nbz)
- by, bz = by % nthread_by, by // nthread_by
-
- def mergepath(
- source,
- dest,
- source_idx,
- dest_idx,
- aCount,
- bCount,
- aStart,
- bStart,
- kStart,
- step_count,
- even,
- ):
- # pylint: disable=arguments-out-of-order
- def merge(source, dest, source_idx, dest_idx):
- diag = tx * step_count
- first, last = get_merge_begin(
+ with T.If(tvm.tir.all(i_buf[0] < aStart + aCount, j_buf[0] < bStart + bCount)):
+ with T.Then():
+ with T.If(compare(source[i_idx], source[j_idx])):
+ with T.Then():
+ dest[k_idx] = source[i_idx]
+ if values is not None:
+ dest_idx[k_idx] = source_idx[i_idx]
+ i_buf[0] = i_buf[0] + 1
+ with T.Else():
+ dest[k_idx] = source[j_idx]
+ if values is not None:
+ dest_idx[k_idx] = source_idx[j_idx]
+ j_buf[0] = j_buf[0] + 1
+ with T.Else():
+ with T.If(i_buf[0] < aStart + aCount):
+ with T.Then():
+ dest[k_idx] = source[i_idx]
+ if values is not None:
+ dest_idx[k_idx] = source_idx[i_idx]
+ i_buf[0] = i_buf[0] + 1
+ with T.Else():
+ dest[k_idx] = source[j_idx]
+ if values is not None:
+ dest_idx[k_idx] = source_idx[j_idx]
+ j_buf[0] = j_buf[0] + 1
+
+ def mergepath(
+ source,
+ dest,
+ source_idx,
+ dest_idx,
+ base_idx,
+ aCount,
+ bCount,
+ aStart,
+ bStart,
+ kStart,
+ tx,
+ step_count,
+ even,
+ ):
+ with T.frame_scope(
+ [
+ T.allocate([1], target_dtype, scope="local"), # first
+ T.allocate([1], target_dtype, scope="local"), # last
+ T.allocate([1], target_dtype, scope="local"), # i_buf
+ T.allocate([1], target_dtype, scope="local"), # j_buf
+ ]
+ ) as (first_ptr, last_ptr, i_ptr, j_ptr):
+ first = T.buffer_proxy(
+ tvm.tir.decl_buffer([1], target_dtype, "first", data=first_ptr, scope="local")
+ )
+ last = T.buffer_proxy(
+ tvm.tir.decl_buffer([1], target_dtype, "last", data=last_ptr, scope="local")
+ )
+ i_buf = T.buffer_proxy(
+ tvm.tir.decl_buffer([1], target_dtype, "i", data=i_ptr, scope="local")
+ )
+ j_buf = T.buffer_proxy(
+ tvm.tir.decl_buffer([1], target_dtype, "j", data=j_ptr, scope="local")
+ )
+
+ diag = tx * step_count
+ with T.If(even):
+ with T.Then():
+ get_merge_begin(
+ source, base_idx, aCount, bCount, aStart, bStart, diag, first, last
+ )
+ serial_merge(
+ source,
+ dest,
+ source_idx,
+ dest_idx,
+ base_idx,
+ aCount,
+ bCount,
+ aStart,
+ bStart,
+ kStart,
+ diag,
+ step_count,
+ first,
+ last,
+ i_buf,
+ j_buf,
+ )
+ with T.Else():
+ get_merge_begin(
+ dest, base_idx, aCount, bCount, aStart, bStart, diag, first, last
+ )
+ # Intentionally swap source/dest for reverse direction merge
+ serial_merge( # pylint: disable=arguments-out-of-order
+ dest,
source,
- by * size,
+ dest_idx,
+ source_idx,
+ base_idx,
aCount,
bCount,
aStart,
bStart,
+ kStart,
diag,
step_count,
+ first,
+ last,
+ i_buf,
+ j_buf,
+ )
+
+ def dual_mergepath(
+ source,
+ dest,
+ source_idx,
+ dest_idx,
+ base_idx,
+ start_pos,
+ middle,
+ end,
+ bx,
+ tx,
+ step_count,
+ even,
+ ):
+ with T.frame_scope(
+ [
+ T.allocate([1], target_dtype, scope="local"), # outer_first
+ T.allocate([1], target_dtype, scope="local"), # outer_last
+ T.allocate([1], target_dtype, scope="local"), # first
+ T.allocate([1], target_dtype, scope="local"), # last
+ T.allocate([1], target_dtype, scope="local"), # i_buf
+ T.allocate([1], target_dtype, scope="local"), # j_buf
+ ]
+ ) as (outer_first_ptr, outer_last_ptr, first_ptr, last_ptr, i_ptr, j_ptr):
+ outer_first = T.buffer_proxy(
+ tvm.tir.decl_buffer(
+ [1], target_dtype, "outer_first", data=outer_first_ptr, scope="local"
+ )
+ )
+ outer_last = T.buffer_proxy(
+ tvm.tir.decl_buffer(
+ [1], target_dtype, "outer_last", data=outer_last_ptr, scope="local"
+ )
+ )
+ first = T.buffer_proxy(
+ tvm.tir.decl_buffer([1], target_dtype, "first", data=first_ptr, scope="local")
+ )
+ last = T.buffer_proxy(
+ tvm.tir.decl_buffer([1], target_dtype, "last", data=last_ptr, scope="local")
+ )
+ i_buf = T.buffer_proxy(
+ tvm.tir.decl_buffer([1], target_dtype, "i", data=i_ptr, scope="local")
+ )
+ j_buf = T.buffer_proxy(
+ tvm.tir.decl_buffer([1], target_dtype, "j", data=j_ptr, scope="local")
+ )
+
+ diag = bx * step_count
+ with T.If(even):
+ with T.Then():
+ get_merge_begin(
+ source,
+ base_idx,
+ middle - start_pos,
+ end - middle,
+ start_pos,
+ middle,
+ diag,
+ outer_first,
+ outer_last,
+ )
+ aStart = start_pos + outer_first[0]
+ bStart = middle + diag - outer_last[0]
+ aCount = tvm.te.min(middle - aStart, step_count)
+ bCount = tvm.te.min(end - bStart, step_count)
+ inner_diag = tx * thread_work
+ get_merge_begin(
+ source, base_idx, aCount, bCount, aStart, bStart, inner_diag, first, last
)
- # iterate over the output loop
serial_merge(
source,
dest,
source_idx,
dest_idx,
- by * size,
+ base_idx,
aCount,
bCount,
aStart,
bStart,
- kStart,
+ start_pos + diag,
+ inner_diag,
+ thread_work,
+ first,
+ last,
+ i_buf,
+ j_buf,
+ )
+ with T.Else():
+ get_merge_begin(
+ dest,
+ base_idx,
+ middle - start_pos,
+ end - middle,
+ start_pos,
+ middle,
diag,
- step_count,
+ outer_first,
+ outer_last,
+ )
+ aStart = start_pos + outer_first[0]
+ bStart = middle + diag - outer_last[0]
+ aCount = tvm.te.min(middle - aStart, step_count)
+ bCount = tvm.te.min(end - bStart, step_count)
+ inner_diag = tx * thread_work
+ get_merge_begin(
+ dest, base_idx, aCount, bCount, aStart, bStart, inner_diag, first, last
+ )
+ serial_merge(
+ dest,
+ source,
+ dest_idx,
+ source_idx,
+ base_idx,
+ aCount,
+ bCount,
+ aStart,
+ bStart,
+ start_pos + diag,
+ inner_diag,
+ thread_work,
first,
last,
+ i_buf,
+ j_buf,
)
- with ib.if_scope(even):
- merge(source, dest, source_idx, dest_idx)
- with ib.else_scope():
- merge(dest, source, dest_idx, source_idx)
-
- def mergesort(source, dest, source_idx, dest_idx, size, width, even):
- # calculate the start, mid, and end points of this section
- start = width * bz
- target = tvm.target.Target.current()
- target_dtype = "int32" if "webgpu" in str(target) else "int64"
- middle = cast(tvm.te.min(start + tvm.tir.indexdiv(width, 2), size), target_dtype)
- end = cast(tvm.te.min(start + width, size), target_dtype)
- with ib.if_scope(start < size):
- with ib.if_scope(nbx == 1):
- ## merge the start->middle and middle->end arrays
- aCount = middle - start
- bCount = end - middle
- mergepath(
- source,
- dest,
- source_idx,
- dest_idx,
- aCount,
- bCount,
- start,
- middle,
- start,
- ceil_div(width, ntx),
- even,
- )
- with ib.else_scope():
- step_count = max_threads * thread_work
- diag = bx * step_count
-
- def do_merge(first, last):
- aStart = start + first
- bStart = middle + diag - last
- aCount = tvm.te.min(middle - aStart, step_count)
- bCount = tvm.te.min(end - bStart, step_count)
+ with T.serial(0, cast(upper_lim - lower_lim, target_dtype)) as l2_width:
+ width = 2 << (l2_width + lower_lim)
+ # Define and launch the cuda kernel
+ target = tvm.target.Target.current()
+ if "vulkan" in str(target):
+ ntx = max_threads
+ nbx = tvm.tir.generic.cast(ceil_div(width, max_threads * thread_work), "int32")
+ nbz = tvm.tir.generic.cast(ceil_div(size, width), "int32")
+ else:
+ ntx = tvm.tir.generic.cast(tvm.te.min(max_threads, width), "int32")
+ nbx = tvm.tir.generic.cast(ceil_div(width, max_threads * thread_work), "int32")
+ nbz = tvm.tir.generic.cast(ceil_div(size, width), "int32")
+
+ tx, bx, by, _, _, _ = _get_threads(ntx, nbx, nthread_by * nbz)
+ with T.frame_scope(
+ [
+ T.attr(tx, "thread_extent", ntx),
+ T.attr(bx, "thread_extent", nbx),
+ T.attr(by, "thread_extent", nthread_by * nbz),
+ ]
+ ):
+ by_val = by % nthread_by
+ bz = by // nthread_by
+ base_idx = by_val * size
+
+ # calculate the start, mid, and end points of this section
+ start_pos = width * bz
+ middle = cast(tvm.te.min(start_pos + tvm.tir.indexdiv(width, 2), size), target_dtype)
+ end = cast(tvm.te.min(start_pos + width, size), target_dtype)
+
+ with T.If(start_pos < size):
+ with T.Then():
+ even = tvm.tir.indexmod(l2_width, 2) == 0
+ with T.If(nbx == 1):
+ with T.Then():
+ ## merge the start->middle and middle->end arrays
+ aCount = middle - start_pos
+ bCount = end - middle
mergepath(
- source,
- dest,
- source_idx,
- dest_idx,
+ keys,
+ keys_swap,
+ values,
+ values_swap,
+ base_idx,
aCount,
bCount,
- aStart,
- bStart,
- start + diag,
- thread_work,
- even,
- )
-
- with ib.if_scope(even):
- first, last = get_merge_begin(
- source,
- by * size,
- middle - start,
- end - middle,
- start,
+ start_pos,
middle,
- diag,
- step_count,
+ start_pos,
+ tx,
+ ceil_div(width, ntx),
+ even,
)
- do_merge(first, last)
- with ib.else_scope():
- first, last = get_merge_begin(
- dest,
- by * size,
- middle - start,
- end - middle,
- start,
+ with T.Else():
+ dual_mergepath(
+ keys,
+ keys_swap,
+ values,
+ values_swap,
+ base_idx,
+ start_pos,
middle,
- diag,
- step_count,
+ end,
+ bx,
+ tx,
+ max_threads * thread_work,
+ even,
)
- do_merge(first, last)
-
- # Call the kernel
- mergesort(
- keys,
- keys_swap,
- values,
- values_swap,
- size,
- width,
- tvm.tir.indexmod(l2_width, 2) == 0,
- )
- nthread_by = axis_mul_before * axis_mul_after
- nthread_tx = max_threads
- nthread_bx = ceil_div(size, nthread_tx)
+
## if the final sorted data ended up in the swap, copy it to the real output
- with ib.if_scope(
- tvm.tir.all(upper_lim > lower_lim, tvm.tir.indexmod(upper_lim - lower_lim, 2) == 1)
- ):
- with ib.new_scope():
- tx, bx, by = _get_threads(ib, nthread_tx, nthread_bx, nthread_by)
- tid = bx * nthread_tx + tx
- idx = by * size + tid
- with ib.if_scope(tid < size):
- keys[idx] = keys_swap[idx]
- if values is not None:
- values[idx] = values_swap[idx]
+ nthread_bx = ceil_div(size, nthread_tx)
+ with T.If(tvm.tir.all(upper_lim > lower_lim, tvm.tir.indexmod(upper_lim - lower_lim, 2) == 1)):
+ with T.Then():
+ tx2, bx2, by2, _, _, _ = _get_threads(nthread_tx, nthread_bx, nthread_by)
+ with T.frame_scope(
+ [
+ T.attr(tx2, "thread_extent", nthread_tx),
+ T.attr(bx2, "thread_extent", nthread_bx),
+ T.attr(by2, "thread_extent", nthread_by),
+ ]
+ ):
+ tid = bx2 * nthread_tx + tx2
+ idx = by2 * size + tid
+ with T.If(tid < size):
+ with T.Then():
+ keys[idx] = keys_swap[idx]
+ if values is not None:
+ values[idx] = values_swap[idx]
def sort_ir(
@@ -512,41 +700,47 @@ def sort_ir(
stmt : Stmt
The result IR statement.
"""
- ib = tvm.tir.ir_builder.create()
- shape = data.shape
-
- data = ib.buffer_ptr(data)
- values_out = ib.buffer_ptr(values_out)
- values_out_swap = ib.buffer_ptr(values_out_swap)
- if indices_out is not None:
- indices_out = ib.buffer_ptr(indices_out)
- assert indices_out_swap is not None
- indices_out_swap = ib.buffer_ptr(indices_out_swap)
-
- with ib.if_scope(shape[axis] > 0):
- axis_mul_before, axis_mul_after = _sort_init(
- ib,
- shape,
- axis,
- data,
- values_out,
- indices_out,
- value_init_func=lambda _, tid: tvm.tir.generic.cast(tid, indices_out.dtype),
- )
-
- _sort_common(
- ib,
- shape[axis],
- axis_mul_before,
- axis_mul_after,
- is_ascend,
- values_out,
- values_out_swap,
- values=indices_out,
- values_swap=indices_out_swap,
- )
+ with IRBuilder() as ib:
+ shape = data.shape
+
+ data = T.buffer_proxy(data)
+ values_out = T.buffer_proxy(values_out)
+ values_out_swap = T.buffer_proxy(values_out_swap)
+ if indices_out is not None:
+ indices_out_orig = indices_out
+ indices_out = T.buffer_proxy(indices_out)
+ assert indices_out_swap is not None
+ indices_out_swap = T.buffer_proxy(indices_out_swap)
+
+ with T.If(shape[axis] > 0):
+ with T.Then():
+ axis_mul_before, axis_mul_after = _sort_init(
+ shape,
+ axis,
+ data,
+ values_out,
+ indices_out,
+ value_init_func=(
+ lambda _, tid: (
+ tvm.tir.generic.cast(tid, indices_out_orig.dtype)
+ if indices_out is not None
+ else None
+ )
+ ),
+ )
+
+ _sort_common(
+ shape[axis],
+ axis_mul_before,
+ axis_mul_after,
+ is_ascend,
+ values_out,
+ values_out_swap,
+ values=indices_out,
+ values_swap=indices_out_swap,
+ )
- return ib.get()
+ return ib.get()
def sort(data, axis=-1, is_ascend=1):
@@ -622,7 +816,6 @@ def sort_thrust(data, axis=-1, is_ascend=1, workspace=None):
The output of this function.
"""
dtype = "float32"
-
ndim = len(data.shape)
axis = ndim + axis if axis < 0 else axis
@@ -824,15 +1017,7 @@ def topk(data, k=1, axis=-1, ret_type="both", is_ascend=False, dtype="int64"):
output = te.extern(
[data.shape, data.shape, data.shape, data.shape],
[data],
- lambda ins, outs: sort_ir(
- ins[0],
- outs[0],
- outs[2],
- -1,
- is_ascend,
- indices_out=outs[1],
- indices_out_swap=outs[3],
- ),
+ lambda ins, outs: sort_ir(ins[0], outs[0], outs[2], -1, is_ascend, outs[1], outs[3]),
out_buffers=[values_buf, indices_buf, values_swap_buf, indices_swap_buf],
name="topk_gpu",
tag="topk_gpu",
@@ -1004,43 +1189,47 @@ def searchsorted(sorted_sequence, values, right=False, out_dtype="int64"):
), "Outer dimensions of sorted_sequence and values must match for N-D searchsorted"
def ir(sorted_sequence_buf, values_buf, indices_buf):
- ib = tvm.tir.ir_builder.create()
- sorted_sequence_shape = sorted_sequence_buf.shape
- values_shape = values_buf.shape
- num_search = prod(values_shape)
- search_range = sorted_sequence_shape[-1]
-
- sorted_sequence_ptr = ib.buffer_ptr(sorted_sequence_buf)
- values_ptr = ib.buffer_ptr(values_buf)
- indices_ptr = ib.buffer_ptr(indices_buf)
-
- max_threads = int(tvm.target.Target.current(allow_none=False).max_num_threads)
- nthread_tx = max_threads
- nthread_bx = ceil_div(num_search, nthread_tx)
- tx = te.thread_axis("threadIdx.x")
- bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", nthread_tx)
- ib.scope_attr(bx, "thread_extent", nthread_bx)
- tid = bx * nthread_tx + tx
-
- with ib.if_scope(tid < num_search):
- if len(sorted_sequence_shape) == 1:
- sequence_offset = 0
- else:
- sequence_id = tid // values_shape[-1]
- sequence_offset = sequence_id * search_range
-
- indices_ptr[tid] = binary_search(
- ib,
- sequence_offset,
- search_range,
- sorted_sequence_ptr,
- values_ptr[tid],
- right,
- out_dtype,
- )
+ with IRBuilder() as ib:
+ sorted_sequence_shape = sorted_sequence_buf.shape
+ values_shape = values_buf.shape
+ num_search = prod(values_shape)
+ search_range = sorted_sequence_shape[-1]
+
+ sorted_sequence_ptr = T.buffer_proxy(sorted_sequence_buf)
+ values_ptr = T.buffer_proxy(values_buf)
+ indices_ptr = T.buffer_proxy(indices_buf)
+
+ max_threads = int(tvm.target.Target.current(allow_none=False).max_num_threads)
+ nthread_tx = max_threads
+ nthread_bx = ceil_div(num_search, nthread_tx)
+ tx = te.thread_axis("threadIdx.x")
+ bx = te.thread_axis("blockIdx.x")
+ with T.frame_scope(
+ [
+ T.attr(tx, "thread_extent", nthread_tx),
+ T.attr(bx, "thread_extent", nthread_bx),
+ ]
+ ):
+ tid = bx * nthread_tx + tx
+
+ with T.If(tid < num_search):
+ with T.Then():
+ if len(sorted_sequence_shape) == 1:
+ sequence_offset = 0
+ else:
+ sequence_id = tid // values_shape[-1]
+ sequence_offset = sequence_id * search_range
+
+ indices_ptr[tid] = binary_search(
+ sequence_offset,
+ search_range,
+ sorted_sequence_ptr,
+ values_ptr[tid],
+ right,
+ out_dtype,
+ )
- return ib.get()
+ return ib.get()
return te.extern(
values.shape,
diff --git a/python/tvm/topi/index_put.py b/python/tvm/topi/index_put.py
index 52406d402c..2156f1191e 100644
--- a/python/tvm/topi/index_put.py
+++ b/python/tvm/topi/index_put.py
@@ -14,9 +14,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+# pylint: disable=invalid-name
"""IndexPut operator"""
from tvm import te
from tvm import tir
+from tvm.script.ir_builder import IRBuilder
+from tvm.script.ir_builder import tir as T
from . import utils
@@ -85,56 +88,56 @@ def index_put(data, indices, values, accumulate=False):
index_len *= dim
def gen_ir(data_ptr, index_ptrs, values_ptr, out_ptr, reduce_func):
- ir_builder = tir.ir_builder.create()
-
- data = ir_builder.buffer_ptr(data_ptr)
- indices = [ir_builder.buffer_ptr(idx) for idx in index_ptrs]
- values = ir_builder.buffer_ptr(values_ptr)
- out = ir_builder.buffer_ptr(out_ptr)
-
- with ir_builder.for_range(0, full_range, "i", kind="parallel") as i:
- out[i] = data[i]
-
- with ir_builder.for_range(0, index_len, "k", kind="parallel") as k:
- # Decompose k into multi-dimensional broadcast index
- k_temp = k
- broadcast_indices = []
- for i in range(broadcast_ndim - 1, -1, -1):
- broadcast_indices.insert(0, k_temp % broadcast_shape[i])
- k_temp = k_temp // broadcast_shape[i]
-
- flat_index = 0
- stride = 1
- for dim in range(len(shape) - 1, -1, -1):
- # Get the index for this dimension using broadcasting
- idx_shape = index_shapes[dim]
- idx_ndim = len(idx_shape)
-
- # Compute the linear index into this index tensor
- idx_offset = 0
- idx_stride = 1
- for i in range(broadcast_ndim - 1, -1, -1):
- # Right-align the index shape with broadcast shape
- dim_idx = idx_ndim - broadcast_ndim + i
- if dim_idx >= 0:
- dim_size = idx_shape[dim_idx]
- # Use broadcasting: if size is 1, use index 0
- # otherwise use broadcast_indices[i]
- if utils.equal_const_int(dim_size, 1):
- idx_in_dim = 0
- else:
- idx_in_dim = broadcast_indices[i]
- idx_offset += idx_in_dim * idx_stride
- idx_stride *= dim_size
-
- idx_val = indices[dim][idx_offset]
- shifted_idx = idx_val + (idx_val < 0) * shape[dim]
- flat_index += shifted_idx * stride
- stride *= shape[dim]
-
- reduce_func(out, flat_index, values[k])
-
- return ir_builder.get()
+ data = T.buffer_proxy(data_ptr)
+ indices = [T.buffer_proxy(idx) for idx in index_ptrs]
+ values = T.buffer_proxy(values_ptr)
+ out = T.buffer_proxy(out_ptr)
+
+ with IRBuilder() as ib:
+ with T.seq_scope():
+ with T.parallel(0, full_range) as i:
+ out[i] = data[i]
+
+ with T.parallel(0, index_len) as k:
+ # Decompose k into multi-dimensional broadcast index
+ k_temp = k
+ broadcast_indices = []
+ for i in range(broadcast_ndim - 1, -1, -1):
+ broadcast_indices.insert(0, k_temp % broadcast_shape[i])
+ k_temp = k_temp // broadcast_shape[i]
+
+ flat_index = 0
+ stride = 1
+ for dim in range(len(shape) - 1, -1, -1):
+ # Get the index for this dimension using broadcasting
+ idx_shape = index_shapes[dim]
+ idx_ndim = len(idx_shape)
+
+ # Compute the linear index into this index tensor
+ idx_offset = 0
+ idx_stride = 1
+ for i in range(broadcast_ndim - 1, -1, -1):
+ # Right-align the index shape with broadcast shape
+ dim_idx = idx_ndim - broadcast_ndim + i
+ if dim_idx >= 0:
+ dim_size = idx_shape[dim_idx]
+ # Use broadcasting: if size is 1, use index 0
+ # otherwise use broadcast_indices[i]
+ if utils.equal_const_int(dim_size, 1):
+ idx_in_dim = 0
+ else:
+ idx_in_dim = broadcast_indices[i]
+ idx_offset += idx_in_dim * idx_stride
+ idx_stride *= dim_size
+
+ idx_val = indices[dim][idx_offset]
+ shifted_idx = idx_val + (idx_val < 0) * shape[dim]
+ flat_index += shifted_idx * stride
+ stride *= shape[dim]
+
+ reduce_func(out, flat_index, values[k])
+
+ return ib.get()
def update_func(dst_ptr, dst_index, update):
dst_ptr[dst_index] = update
diff --git a/python/tvm/topi/scan.py b/python/tvm/topi/scan.py
index 22f9ff58a5..ee47975b8d 100644
--- a/python/tvm/topi/scan.py
+++ b/python/tvm/topi/scan.py
@@ -19,9 +19,11 @@
from typing import Callable, Optional
import tvm
+from tvm.script.ir_builder import IRBuilder
+from tvm.script.ir_builder import tir as T
from ..te import extern
-from ..tir import decl_buffer, generic, ir_builder
+from ..tir import decl_buffer, generic
from .math import cast
from . import utils
@@ -112,28 +114,28 @@ def scanop(
axis_mul_after *= value
def gen_ir(data_buf, out_buf):
- ib = ir_builder.create()
- data_buf = ib.buffer_ptr(data_buf)
- out_buf = ib.buffer_ptr(out_buf)
-
- with ib.for_range(0, axis_mul_before * axis_mul_after, "fused", kind="parallel") as fused:
- i = fused // axis_mul_after
- j = fused % axis_mul_after
- base_idx = i * cumsum_axis_len * axis_mul_after + j
- if exclusive:
- out_buf[base_idx] = cast(identity_value, dtype)
- else:
- out_buf[base_idx] = maybe_cast(data_buf[base_idx])
- with ib.for_range(0, cumsum_axis_len - 1, "_k") as _k:
- k = _k + 1
- cur_idx = base_idx + k * axis_mul_after
- prev_idx = base_idx + (k - 1) * axis_mul_after
+ with IRBuilder() as ib:
+ data_buf = T.buffer_proxy(data_buf)
+ out_buf = T.buffer_proxy(out_buf)
+
+ with T.parallel(0, axis_mul_before * axis_mul_after) as fused:
+ i = fused // axis_mul_after
+ j = fused % axis_mul_after
+ base_idx = i * cumsum_axis_len * axis_mul_after + j
if exclusive:
- out_buf[cur_idx] = binop(out_buf[prev_idx], maybe_cast(data_buf[prev_idx]))
+ out_buf[base_idx] = cast(identity_value, dtype)
else:
- out_buf[cur_idx] = binop(out_buf[prev_idx], maybe_cast(data_buf[cur_idx]))
-
- return ib.get()
+ out_buf[base_idx] = maybe_cast(data_buf[base_idx])
+ with T.serial(0, cumsum_axis_len - 1) as _k:
+ k = _k + 1
+ cur_idx = base_idx + k * axis_mul_after
+ prev_idx = base_idx + (k - 1) * axis_mul_after
+ if exclusive:
+ out_buf[cur_idx] = binop(out_buf[prev_idx], maybe_cast(data_buf[prev_idx]))
+ else:
+ out_buf[cur_idx] = binop(out_buf[prev_idx], maybe_cast(data_buf[cur_idx]))
+
+ return ib.get()
out_buf = decl_buffer(shape, dtype, "out_buf")
diff --git a/python/tvm/topi/scatter.py b/python/tvm/topi/scatter.py
index 9cf19e2e61..311f24229d 100644
--- a/python/tvm/topi/scatter.py
+++ b/python/tvm/topi/scatter.py
@@ -19,6 +19,8 @@
from tvm import te, tir # hide redefinition of min and max
from tvm.tir import expr
from tvm.arith.analyzer import Analyzer
+from tvm.script.ir_builder import IRBuilder
+from tvm.script.ir_builder import tir as T
def _verify_scatter_nd_inputs(data, indices, updates):
@@ -93,12 +95,10 @@ def scatter_nd(data, indices, updates, mode):
def gen_ir(data_ptr, indices_ptr, updates_ptr, out_ptr):
# pylint: disable=invalid-name
- ib = tir.ir_builder.create()
-
- data = ib.buffer_ptr(data_ptr)
- indices = ib.buffer_ptr(indices_ptr)
- updates = ib.buffer_ptr(updates_ptr)
- out = ib.buffer_ptr(out_ptr)
+ data = T.buffer_proxy(data_ptr)
+ indices = T.buffer_proxy(indices_ptr)
+ updates = T.buffer_proxy(updates_ptr)
+ out = T.buffer_proxy(out_ptr)
# We combine all the indices dimensions but the first one into a single
# dimension so we can iterate it in single loop instead of an arbitrary
@@ -115,35 +115,41 @@ def scatter_nd(data, indices, updates, mode):
for i in data_ptr.shape:
fused_shape *= i
- with ib.for_range(0, fused_shape) as i:
- out[i] = data[i]
-
- with ib.for_range(0, fused_indices_dimension) as i:
- with ib.for_range(0, fused_updates_dimension, kind="parallel") as j:
- offset = fused_updates_dimension
- index = j # This is x_M, .. x_{N-1} part of the index into out.
- # Build up the indices[0, y_0, .. y_{K-1}], .. indices[M-1, y_0, .. y_{K-1}] part
- # of the index into out.
- for l in reversed(range(indices_ptr.shape[0].value)):
- # indices[i * l * fused_indices_dimension] = indices[l, y_0, ... y_{k-1}]
- index += offset * indices[i + l * fused_indices_dimension]
- offset *= data_ptr.shape[l]
- if mode == "update":
- out[index] = updates[i * fused_updates_dimension + j]
- elif mode == "add":
- out[index] += updates[i * fused_updates_dimension + j]
- elif mode == "mul":
- out[index] *= updates[i * fused_updates_dimension + j]
- elif mode == "min":
- out[index] = tir.min(out[index], updates[i * fused_updates_dimension + j])
- elif mode == "max":
- out[index] = tir.max(out[index], updates[i * fused_updates_dimension + j])
- else:
- raise NotImplementedError(
- "scatter_nd mode not in [update, add, mul, min, max]:", mode
- )
-
- return ib.get()
+ with IRBuilder() as ib:
+ with T.seq_scope():
+ with T.serial(0, fused_shape) as i:
+ out[i] = data[i]
+
+ with T.serial(0, fused_indices_dimension) as i:
+ with T.parallel(0, fused_updates_dimension) as j:
+ offset = fused_updates_dimension
+ index = j # This is x_M, .. x_{N-1} part of the index into out.
+ # Build up the indices[0, y_0, ..], .. indices[M-1, y_0, ..] part
+ # of the index into out.
+ for l in reversed(range(indices_ptr.shape[0].value)):
+ # indices[l, y_0, ... y_{k-1}]
+ index += offset * indices[i + l * fused_indices_dimension]
+ offset *= data_ptr.shape[l]
+ if mode == "update":
+ out[index] = updates[i * fused_updates_dimension + j]
+ elif mode == "add":
+ out[index] += updates[i * fused_updates_dimension + j]
+ elif mode == "mul":
+ out[index] *= updates[i * fused_updates_dimension + j]
+ elif mode == "min":
+ out[index] = tir.min(
+ out[index], updates[i * fused_updates_dimension + j]
+ )
+ elif mode == "max":
+ out[index] = tir.max(
+ out[index], updates[i * fused_updates_dimension + j]
+ )
+ else:
+ raise NotImplementedError(
+ "scatter_nd mode not in [update, add, mul, min, max]:", mode
+ )
+
+ return ib.get()
out_buf = tir.decl_buffer(data.shape, data.dtype, "out_buf")
return te.extern(
diff --git a/python/tvm/topi/scatter_elements.py b/python/tvm/topi/scatter_elements.py
index 08fcb866b4..b7e9df2c49 100644
--- a/python/tvm/topi/scatter_elements.py
+++ b/python/tvm/topi/scatter_elements.py
@@ -17,6 +17,8 @@
"""ScatterElements operator"""
from tvm import te
from tvm import tir
+from tvm.script.ir_builder import IRBuilder
+from tvm.script.ir_builder import tir as T
from . import utils
from .math import cast
@@ -95,35 +97,33 @@ def scatter_elements(data, indices, updates, axis=0, reduction="update"):
def gen_ir(data_ptr, indices_ptr, updates_ptr, out_ptr, reduce_func):
# pylint: disable=invalid-name
- ib = tir.ir_builder.create()
-
- data = ib.buffer_ptr(data_ptr)
- indices = ib.buffer_ptr(indices_ptr)
- updates = ib.buffer_ptr(updates_ptr)
- out = ib.buffer_ptr(out_ptr)
+ data = T.buffer_proxy(data_ptr)
+ indices = T.buffer_proxy(indices_ptr)
+ updates = T.buffer_proxy(updates_ptr)
+ out = T.buffer_proxy(out_ptr)
# Copy initial input data to output
- with ib.for_range(0, full_range, "i", kind="parallel") as i:
- out[i] = data[i]
-
- with ib.for_range(
- 0, ind_before_axis_range * ind_after_axis_range, "fused", kind="parallel"
- ) as fused:
- i = fused // ind_after_axis_range
- j = fused % ind_after_axis_range
- pre_index1 = i * ind_before_axis_stride + j
- pre_index2 = i * before_axis_stride + j
- with ib.for_range(0, ind_axis_range, "k") as k:
- # Offset along indices or updates
- index1 = pre_index1 + k * ind_after_axis_range
- # Get index and shift to positive side if need
- k_new = indices[index1]
- shifted_index = k_new + (k_new < 0) * axis_range
- # Offset along data
- index2 = pre_index2 + shifted_index * after_axis_range
- reduce_func(out, index2, updates[index1])
-
- return ib.get()
+ with IRBuilder() as ib:
+ with T.seq_scope():
+ with T.parallel(0, full_range) as i:
+ out[i] = data[i]
+
+ with T.parallel(0, ind_before_axis_range * ind_after_axis_range) as fused:
+ i = fused // ind_after_axis_range
+ j = fused % ind_after_axis_range
+ pre_index1 = i * ind_before_axis_stride + j
+ pre_index2 = i * before_axis_stride + j
+ with T.serial(0, ind_axis_range) as k:
+ # Offset along indices or updates
+ index1 = pre_index1 + k * ind_after_axis_range
+ # Get index and shift to positive side if need
+ k_new = indices[index1]
+ shifted_index = k_new + (k_new < 0) * axis_range
+ # Offset along data
+ index2 = pre_index2 + shifted_index * after_axis_range
+ reduce_func(out, index2, updates[index1])
+
+ return ib.get()
def update_func(dst_ptr, dst_index, update):
dst_ptr[dst_index] = update
diff --git a/python/tvm/topi/searchsorted.py b/python/tvm/topi/searchsorted.py
index 28ffd170c9..523c3a01f8 100644
--- a/python/tvm/topi/searchsorted.py
+++ b/python/tvm/topi/searchsorted.py
@@ -16,15 +16,19 @@
# under the License.
# pylint: disable=invalid-name
"""searchsorted operator"""
+import tvm
+from tvm.script.ir_builder import IRBuilder
+from tvm.script.ir_builder import tir as T
from . import utils
from . import te
-from ..tir import ir_builder
from .math import cast
-def binary_search(ib, sequence_offset, search_range, sorted_sequence, value, right, out_dtype):
+def binary_search(sequence_offset, search_range, sorted_sequence, value, right, out_dtype):
"""Common IR generator for binary search used by CPU and GPU backends.
+ Must be called within an active IRBuilder context.
+
`sorted_sequence` is a N-D Buffer whose innermost dimension we want to search for `value`,
and `search_range` is the size of the innermost dimension. `sequence_offset` is
a 1-D linearlized offset specifying which of innermost sequences to search.
@@ -34,24 +38,31 @@ def binary_search(ib, sequence_offset, search_range, sorted_sequence, value, rig
Note that we index N-D Buffer by 1-D linearlized indices.
"""
- lo = ib.allocate(out_dtype, (1,), name="lo", scope="local")
- hi = ib.allocate(out_dtype, (1,), name="hi", scope="local")
-
- lo[0] = cast(0, out_dtype)
- hi[0] = cast(search_range, out_dtype)
-
- # Reference: pytorch/aten/src/ATen/native/cuda/Bucketization.cu
- def condition(current_val, target_val):
- if right:
- return current_val <= target_val
- return current_val < target_val
-
- with ib.while_loop(lo[0] < hi[0]):
- mid = lo[0] + (hi[0] - lo[0] >> 1)
- with ib.if_scope(condition(sorted_sequence[sequence_offset + mid], value)):
- lo[0] = mid + 1
- with ib.else_scope():
- hi[0] = mid
+ with T.frame_scope(
+ [
+ T.allocate([1], out_dtype, scope="local"),
+ T.allocate([1], out_dtype, scope="local"),
+ ]
+ ) as (lo_ptr, hi_ptr):
+ lo = T.buffer_proxy(tvm.tir.decl_buffer([1], out_dtype, "lo", data=lo_ptr, scope="local"))
+ hi = T.buffer_proxy(tvm.tir.decl_buffer([1], out_dtype, "hi", data=hi_ptr, scope="local"))
+
+ lo[0] = cast(0, out_dtype)
+ hi[0] = cast(search_range, out_dtype)
+
+ # Reference: pytorch/aten/src/ATen/native/cuda/Bucketization.cu
+ def condition(current_val, target_val):
+ if right:
+ return current_val <= target_val
+ return current_val < target_val
+
+ with T.While(lo[0] < hi[0]):
+ mid = lo[0] + (hi[0] - lo[0] >> 1)
+ with T.If(condition(sorted_sequence[sequence_offset + mid], value)):
+ with T.Then():
+ lo[0] = mid + 1
+ with T.Else():
+ hi[0] = mid
return lo[0]
@@ -89,34 +100,33 @@ def searchsorted(sorted_sequence, values, right=False, out_dtype="int64"):
"""
def ir(sorted_sequence, values, indices):
- ib = ir_builder.create()
- sorted_sequence_shape = sorted_sequence.shape
- values_shape = values.shape
- num_search = utils.prod(values_shape)
- search_range = sorted_sequence_shape[-1]
-
- sorted_sequence = ib.buffer_ptr(sorted_sequence)
- values = ib.buffer_ptr(values)
- indices = ib.buffer_ptr(indices)
-
- with ib.for_range(0, num_search, name="i", kind="parallel") as i:
- if len(sorted_sequence_shape) == 1:
- sequence_offset = 0
- else:
- sequence_id = i // values_shape[-1]
- sequence_offset = sequence_id * search_range
-
- indices[i] = binary_search(
- ib,
- sequence_offset,
- search_range,
- sorted_sequence,
- values[i],
- right,
- out_dtype,
- )
-
- return ib.get()
+ with IRBuilder() as ib:
+ sorted_sequence_shape = sorted_sequence.shape
+ values_shape = values.shape
+ num_search = utils.prod(values_shape)
+ search_range = sorted_sequence_shape[-1]
+
+ sorted_sequence = T.buffer_proxy(sorted_sequence)
+ values = T.buffer_proxy(values)
+ indices = T.buffer_proxy(indices)
+
+ with T.parallel(0, num_search) as i:
+ if len(sorted_sequence_shape) == 1:
+ sequence_offset = 0
+ else:
+ sequence_id = i // values_shape[-1]
+ sequence_offset = sequence_id * search_range
+
+ indices[i] = binary_search(
+ sequence_offset,
+ search_range,
+ sorted_sequence,
+ values[i],
+ right,
+ out_dtype,
+ )
+
+ return ib.get()
return te.extern(
values.shape,
diff --git a/python/tvm/topi/signal.py b/python/tvm/topi/signal.py
index 162b9a735e..d322c0e921 100644
--- a/python/tvm/topi/signal.py
+++ b/python/tvm/topi/signal.py
@@ -18,6 +18,8 @@
"""STFT operator"""
from math import pi
from tvm import te, tir
+from tvm.script.ir_builder import IRBuilder
+from tvm.script.ir_builder import tir as T
def stft(
@@ -75,37 +77,36 @@ def stft(
output_ptr,
loop_kind,
):
- ib = tir.ir_builder.create()
- data = ib.buffer_ptr(data_ptr)
- window = ib.buffer_ptr(window_ptr)
- output = ib.buffer_ptr(output_ptr)
- # https://librosa.org/doc/0.7.2/_modules/librosa/core/spectrum.html#stft
- with ib.for_range(
- 0, output_ptr.shape[0] * output_ptr.shape[1], kind="parallel"
- ) as batch_row:
- with ib.for_range(0, output_ptr.shape[2], kind=loop_kind) as col:
- batch = ib.allocate("int32", (1), name="batch", scope="local")
- row = ib.allocate("int32", (1), name="row", scope="local")
- batch = tir.floordiv(batch_row, output_ptr.shape[1])
- row = tir.floormod(batch_row, output_ptr.shape[1])
- output[batch, row, col, 0] = tir.Cast(data_ptr.dtype, 0)
- output[batch, row, col, 1] = tir.Cast(data_ptr.dtype, 0)
- with ib.for_range(0, win_length) as wlen:
- output[batch, row, col, 0] += (
- window[wlen]
- * data[batch, col * hop_length + wlen]
- * tir.cos(2 * pi * row * wlen / win_length)
- )
- output[batch, row, col, 1] -= (
- window[wlen]
- * data[batch, col * hop_length + wlen]
- * tir.sin(2 * pi * row * wlen / win_length)
- )
- with ib.if_scope(normalized):
- output[batch, row, col, 0] /= tir.sqrt(tir.const(n_fft, "float32"))
- output[batch, row, col, 1] /= tir.sqrt(tir.const(n_fft, "float32"))
-
- return ib.get()
+ col_loop = T.vectorized if loop_kind == "vectorize" else T.serial
+
+ with IRBuilder() as ib:
+ data = T.buffer_proxy(data_ptr)
+ window = T.buffer_proxy(window_ptr)
+ output = T.buffer_proxy(output_ptr)
+ # https://librosa.org/doc/0.7.2/_modules/librosa/core/spectrum.html#stft
+ with T.parallel(0, output_ptr.shape[0] * output_ptr.shape[1]) as batch_row:
+ with col_loop(0, output_ptr.shape[2]) as col:
+ batch = tir.floordiv(batch_row, output_ptr.shape[1])
+ row = tir.floormod(batch_row, output_ptr.shape[1])
+ output[batch, row, col, 0] = tir.Cast(data_ptr.dtype, 0)
+ output[batch, row, col, 1] = tir.Cast(data_ptr.dtype, 0)
+ with T.serial(0, win_length) as wlen:
+ output[batch, row, col, 0] += (
+ window[wlen]
+ * data[batch, col * hop_length + wlen]
+ * tir.cos(2 * pi * row * wlen / win_length)
+ )
+ output[batch, row, col, 1] -= (
+ window[wlen]
+ * data[batch, col * hop_length + wlen]
+ * tir.sin(2 * pi * row * wlen / win_length)
+ )
+ with T.If(normalized):
+ with T.Then():
+ output[batch, row, col, 0] /= tir.sqrt(tir.const(n_fft, "float32"))
+ output[batch, row, col, 1] /= tir.sqrt(tir.const(n_fft, "float32"))
+
+ return ib.get()
output_buf = tir.decl_buffer(output_shape, data.dtype, "output_buf")
loop_kind = "vectorize"
@@ -160,40 +161,44 @@ def dft(
re_output_buf,
im_output_buf,
):
- ib = tir.ir_builder.create()
- re_data_ptr = ib.buffer_ptr(re_data_buf)
- im_data_ptr = ib.buffer_ptr(im_data_buf)
- re_output_ptr = ib.buffer_ptr(re_output_buf)
- im_output_ptr = ib.buffer_ptr(im_output_buf)
-
- shape = re_data.shape
- n_fft = shape[len(shape) - 1]
- base_range = 1
- for i in range(len(shape) - 1):
- base_range *= shape[i]
-
- sign = -1 if inverse else 1
- factor = 1.0 / n_fft if inverse else 1.0
-
- with ib.for_range(0, base_range, kind="parallel") as i:
- base_idx = i * n_fft
- with ib.for_range(0, n_fft) as n:
- n_idx = base_idx + n
- re_output_ptr[n_idx] = tir.Cast(re_output_ptr.dtype, 0)
- im_output_ptr[n_idx] = tir.Cast(im_output_ptr.dtype, 0)
- _w = sign * -2 * pi * n / n_fft
- with ib.for_range(0, n_fft) as k:
- k_idx = base_idx + k
- w = _w * k
- cos_w = tir.Cast(re_output_ptr.dtype, tir.cos(w))
- sin_w = tir.Cast(re_output_ptr.dtype, tir.sin(w))
- re_output_ptr[n_idx] += re_data_ptr[k_idx] * cos_w - im_data_ptr[k_idx] * sin_w
- im_output_ptr[n_idx] += re_data_ptr[k_idx] * sin_w + im_data_ptr[k_idx] * cos_w
-
- re_output_ptr[n_idx] *= tir.Cast(re_output_ptr.dtype, factor)
- im_output_ptr[n_idx] *= tir.Cast(im_output_ptr.dtype, factor)
-
- return ib.get()
+ with IRBuilder() as ib:
+ re_data_ptr = T.buffer_proxy(re_data_buf)
+ im_data_ptr = T.buffer_proxy(im_data_buf)
+ re_output_ptr = T.buffer_proxy(re_output_buf)
+ im_output_ptr = T.buffer_proxy(im_output_buf)
+
+ shape = re_data.shape
+ n_fft = shape[len(shape) - 1]
+ base_range = 1
+ for i in range(len(shape) - 1):
+ base_range *= shape[i]
+
+ sign = -1 if inverse else 1
+ factor = 1.0 / n_fft if inverse else 1.0
+
+ with T.parallel(0, base_range) as i:
+ base_idx = i * n_fft
+ with T.serial(0, n_fft) as n:
+ n_idx = base_idx + n
+ re_output_ptr[n_idx] = tir.Cast(re_output_ptr.dtype, 0)
+ im_output_ptr[n_idx] = tir.Cast(im_output_ptr.dtype, 0)
+ _w = sign * -2 * pi * n / n_fft
+ with T.serial(0, n_fft) as k:
+ k_idx = base_idx + k
+ w = _w * k
+ cos_w = tir.Cast(re_output_ptr.dtype, tir.cos(w))
+ sin_w = tir.Cast(re_output_ptr.dtype, tir.sin(w))
+ re_output_ptr[n_idx] += (
+ re_data_ptr[k_idx] * cos_w - im_data_ptr[k_idx] * sin_w
+ )
+ im_output_ptr[n_idx] += (
+ re_data_ptr[k_idx] * sin_w + im_data_ptr[k_idx] * cos_w
+ )
+
+ re_output_ptr[n_idx] *= tir.Cast(re_output_ptr.dtype, factor)
+ im_output_ptr[n_idx] *= tir.Cast(im_output_ptr.dtype, factor)
+
+ return ib.get()
output_shape = [re_data.shape] * 2
diff --git a/python/tvm/topi/sparse_reshape.py b/python/tvm/topi/sparse_reshape.py
index 6bf3857e2e..44649515dc 100644
--- a/python/tvm/topi/sparse_reshape.py
+++ b/python/tvm/topi/sparse_reshape.py
@@ -16,8 +16,10 @@
# under the License.
# pylint: disable=invalid-name, too-many-arguments, too-many-nested-blocks
"""Sparse_Reshape operator"""
-from ..tir import decl_buffer, ir_builder, Cast
-from ..te import extern, div, floordiv, floormod
+from tvm.tir import decl_buffer, Cast
+from tvm.te import extern, div, floordiv, floormod
+from tvm.script.ir_builder import IRBuilder
+from tvm.script.ir_builder import tir as T
def sparse_reshape(
@@ -75,105 +77,163 @@ def sparse_reshape(
new_sparse_indices_ptr,
out_new_shape_ptr,
):
- ib = ir_builder.create()
-
- sparse_indices = ib.buffer_ptr(sparse_indices_ptr)
- prev_shape = ib.buffer_ptr(prev_shape_ptr)
-
- new_shape = ib.buffer_ptr(new_shape_ptr)
- out_new_shape = ib.buffer_ptr(out_new_shape_ptr)
- new_sparse_indices = ib.buffer_ptr(new_sparse_indices_ptr)
- out_new_shape = ib.buffer_ptr(out_new_shape_ptr)
-
- prev_shape_size = prev_shape_ptr.shape[0]
- new_shape_size = new_shape_ptr.shape[0]
-
- multipliers = ib.allocate(
- new_shape_ptr.dtype, (prev_shape_size,), name="multipliers", scope="local"
- )
- dividers = ib.allocate(
- new_shape_ptr.dtype, (new_shape_size,), name="dividers", scope="local"
- )
- flattened_indices = ib.allocate(
- new_shape_ptr.dtype,
- (sparse_indices_ptr.shape[0],),
- name="flattened_indices",
- scope="local",
- )
-
- total_ele = ib.allocate(new_shape_ptr.dtype, (1,), name="total_ele", scope="local")
- total_ele[0] = prev_shape[0]
-
- # Cumulative Reverse Exclusive Multiply
- multipliers[prev_shape_size - 1] = Cast(new_shape_ptr.dtype, 1)
- with ib.for_range(0, prev_shape_size - 1) as i_:
- i = i_ + 1
- multipliers[prev_shape_size - 1 - i] = (
- prev_shape[prev_shape_size - i] * multipliers[prev_shape_size - i]
- )
- total_ele[0] *= prev_shape[prev_shape_size - i]
-
- division_total_ele = ib.allocate(
- new_shape_ptr.dtype, (1,), name="division_total_ele", scope="local"
- )
- division_total_ele[0] = Cast(new_shape_ptr.dtype, 1)
- with ib.for_range(0, new_shape_size) as i:
- with ib.if_scope(new_shape[i] != -1):
- division_total_ele[0] *= new_shape[i]
-
- # Compute true output shape (replace negative ones)
- with ib.for_range(0, new_shape_size) as i:
- with ib.if_scope(new_shape[i] == -1):
- out_new_shape[i] = Cast(
- new_shape_ptr.dtype, div(total_ele[0], division_total_ele[0])
+ with IRBuilder() as ib:
+ sparse_indices = T.buffer_proxy(sparse_indices_ptr)
+ prev_shape = T.buffer_proxy(prev_shape_ptr)
+
+ new_shape = T.buffer_proxy(new_shape_ptr)
+ out_new_shape = T.buffer_proxy(out_new_shape_ptr)
+ new_sparse_indices = T.buffer_proxy(new_sparse_indices_ptr)
+
+ prev_shape_size = prev_shape_ptr.shape[0]
+ new_shape_size = new_shape_ptr.shape[0]
+
+ with T.frame_scope(
+ [
+ T.allocate([prev_shape_size], new_shape_ptr.dtype, scope="local"),
+ T.allocate([new_shape_size], new_shape_ptr.dtype, scope="local"),
+ T.allocate([sparse_indices_ptr.shape[0]], new_shape_ptr.dtype, scope="local"),
+ T.allocate([1], new_shape_ptr.dtype, scope="local"),
+ T.allocate([1], new_shape_ptr.dtype, scope="local"),
+ T.allocate([1], "bool", scope="local"),
+ ]
+ ) as (
+ multipliers_ptr,
+ dividers_ptr,
+ flattened_indices_ptr,
+ total_ele_ptr,
+ division_total_ele_ptr,
+ equal_shape_ptr,
+ ):
+ multipliers = T.buffer_proxy(
+ decl_buffer(
+ [prev_shape_size],
+ new_shape_ptr.dtype,
+ "multipliers",
+ data=multipliers_ptr,
+ scope="local",
+ )
)
- with ib.else_scope():
- out_new_shape[i] = new_shape[i]
-
- equal_shape = ib.allocate("bool", (1,), name="equal_shape", scope="local")
-
- # Check if prev_shape and new_shape are equal
- equal_shape[0] = True
- with ib.if_scope(prev_shape_size == new_shape_size):
- with ib.for_range(0, prev_shape_size) as i:
- with ib.if_scope(prev_shape[i] != out_new_shape[i]):
- equal_shape[0] = False
- with ib.else_scope():
- equal_shape[0] = False
-
- # Return same inputs if shapes are equal
- with ib.if_scope(equal_shape[0]):
- with ib.for_range(0, sparse_indices_ptr.shape[0], kind="parallel") as i:
- with ib.for_range(0, sparse_indices_ptr.shape[1]) as j:
- new_sparse_indices[i, j] = sparse_indices[i, j]
-
- # Else compute new_sparse_indices
- with ib.else_scope():
- dividers[new_shape_size - 1] = Cast(new_shape_ptr.dtype, 1)
- with ib.for_range(0, new_shape_size - 1) as i_:
- i = i_ + 1
- dividers[new_shape_size - 1 - i] = (
- dividers[new_shape_size - i] * out_new_shape[new_shape_size - i]
+ dividers = T.buffer_proxy(
+ decl_buffer(
+ [new_shape_size],
+ new_shape_ptr.dtype,
+ "dividers",
+ data=dividers_ptr,
+ scope="local",
+ )
)
-
- with ib.for_range(0, sparse_indices_ptr.shape[0], kind="parallel") as i:
- flattened_indices[i] = Cast(new_shape_ptr.dtype, 0)
- with ib.for_range(0, sparse_indices_ptr.shape[1]) as j:
- flattened_indices[i] += sparse_indices[i, j] * multipliers[j]
-
- with ib.for_range(0, new_sparse_indices_ptr.shape[0], kind="parallel") as i:
- current_element = ib.allocate(
- new_shape_ptr.dtype, (1,), name="current_element", scope="local"
+ flattened_indices = T.buffer_proxy(
+ decl_buffer(
+ [sparse_indices_ptr.shape[0]],
+ new_shape_ptr.dtype,
+ "flattened_indices",
+ data=flattened_indices_ptr,
+ scope="local",
+ )
)
- current_element[0] = flattened_indices[i]
-
- with ib.for_range(0, new_sparse_indices_ptr.shape[1]) as j:
- new_sparse_indices[i, j] = Cast(
- sparse_indices_ptr.dtype, floordiv(current_element[0], dividers[j])
+ total_ele = T.buffer_proxy(
+ decl_buffer(
+ [1], new_shape_ptr.dtype, "total_ele", data=total_ele_ptr, scope="local"
)
- current_element[0] = floormod(current_element[0], dividers[j])
+ )
+ division_total_ele = T.buffer_proxy(
+ decl_buffer(
+ [1],
+ new_shape_ptr.dtype,
+ "division_total_ele",
+ data=division_total_ele_ptr,
+ scope="local",
+ )
+ )
+ equal_shape = T.buffer_proxy(
+ decl_buffer([1], "bool", "equal_shape", data=equal_shape_ptr, scope="local")
+ )
+
+ total_ele[0] = prev_shape[0]
- return ib.get()
+ # Cumulative Reverse Exclusive Multiply
+ multipliers[prev_shape_size - 1] = Cast(new_shape_ptr.dtype, 1)
+ with T.serial(0, prev_shape_size - 1) as i_:
+ i = i_ + 1
+ multipliers[prev_shape_size - 1 - i] = (
+ prev_shape[prev_shape_size - i] * multipliers[prev_shape_size - i]
+ )
+ total_ele[0] *= prev_shape[prev_shape_size - i]
+
+ division_total_ele[0] = Cast(new_shape_ptr.dtype, 1)
+ with T.serial(0, new_shape_size) as i:
+ with T.If(new_shape[i] != -1):
+ with T.Then():
+ division_total_ele[0] *= new_shape[i]
+
+ # Compute true output shape (replace negative ones)
+ with T.serial(0, new_shape_size) as i:
+ with T.If(new_shape[i] == -1):
+ with T.Then():
+ out_new_shape[i] = Cast(
+ new_shape_ptr.dtype, div(total_ele[0], division_total_ele[0])
+ )
+ with T.Else():
+ out_new_shape[i] = new_shape[i]
+
+ # Check if prev_shape and new_shape are equal
+ equal_shape[0] = True
+ with T.If(prev_shape_size == new_shape_size):
+ with T.Then():
+ with T.serial(0, prev_shape_size) as i:
+ with T.If(prev_shape[i] != out_new_shape[i]):
+ with T.Then():
+ equal_shape[0] = False
+ with T.Else():
+ equal_shape[0] = False
+
+ # Return same inputs if shapes are equal
+ with T.If(equal_shape[0]):
+ with T.Then():
+ with T.parallel(0, sparse_indices_ptr.shape[0]) as i:
+ with T.serial(0, sparse_indices_ptr.shape[1]) as j:
+ new_sparse_indices[i, j] = sparse_indices[i, j]
+
+ # Else compute new_sparse_indices
+ with T.Else():
+ dividers[new_shape_size - 1] = Cast(new_shape_ptr.dtype, 1)
+ with T.serial(0, new_shape_size - 1) as i_:
+ i = i_ + 1
+ dividers[new_shape_size - 1 - i] = (
+ dividers[new_shape_size - i] * out_new_shape[new_shape_size - i]
+ )
+
+ with T.parallel(0, sparse_indices_ptr.shape[0]) as i:
+ flattened_indices[i] = Cast(new_shape_ptr.dtype, 0)
+ with T.serial(0, sparse_indices_ptr.shape[1]) as j:
+ flattened_indices[i] += sparse_indices[i, j] * multipliers[j]
+
+ with T.parallel(0, new_sparse_indices_ptr.shape[0]) as i:
+ with T.frame_scope(
+ [
+ T.allocate([1], new_shape_ptr.dtype, scope="local"),
+ ]
+ ) as current_element_ptr:
+ current_element = T.buffer_proxy(
+ decl_buffer(
+ [1],
+ new_shape_ptr.dtype,
+ "current_element",
+ data=current_element_ptr,
+ scope="local",
+ )
+ )
+ current_element[0] = flattened_indices[i]
+
+ with T.serial(0, new_sparse_indices_ptr.shape[1]) as j:
+ new_sparse_indices[i, j] = Cast(
+ sparse_indices_ptr.dtype,
+ floordiv(current_element[0], dividers[j]),
+ )
+ current_element[0] = floormod(current_element[0], dividers[j])
+
+ return ib.get()
new_sparse_indices_buf = decl_buffer(
new_sparse_indices_shape, sparse_indices.dtype, "new_sparse_indices_buf"
diff --git a/python/tvm/topi/unique.py b/python/tvm/topi/unique.py
index 9c97320134..fa5a6d6294 100644
--- a/python/tvm/topi/unique.py
+++ b/python/tvm/topi/unique.py
@@ -17,6 +17,8 @@
# pylint: disable=invalid-name
"""Unique operator"""
from tvm import te, tir
+from tvm.script.ir_builder import IRBuilder
+from tvm.script.ir_builder import tir as T
def _calc_adjacent_diff_ir(data, output, binop=tir.Sub):
@@ -37,15 +39,16 @@ def _calc_adjacent_diff_ir(data, output, binop=tir.Sub):
TIR expressions and produce a new TIR expression. By default it uses tvm.tir.Sub to
compute the adjacent difference.
"""
- ib = tir.ir_builder.create()
- data_ptr = ib.buffer_ptr(data)
- output_ptr = ib.buffer_ptr(output)
- with ib.for_range(0, data.shape[0], kind="parallel") as i:
- with ib.if_scope(i == 0):
- output_ptr[0] = 0
- with ib.else_scope():
- output_ptr[i] = tir.Cast(output.dtype, binop(data_ptr[i], data_ptr[i - 1]))
- return ib.get()
+ with IRBuilder() as ib:
+ data_ptr = T.buffer_proxy(data)
+ output_ptr = T.buffer_proxy(output)
+ with T.parallel(0, data.shape[0]) as i:
+ with T.If(i == 0):
+ with T.Then():
+ output_ptr[0] = 0
+ with T.Else():
+ output_ptr[i] = tir.Cast(output.dtype, binop(data_ptr[i], data_ptr[i - 1]))
+ return ib.get()
def _calc_adjacent_diff(data, out_dtype="int32", binop=tir.Sub):
diff --git a/python/tvm/topi/vision/nms.py b/python/tvm/topi/vision/nms.py
index f4aae45ef9..91e757a715 100644
--- a/python/tvm/topi/vision/nms.py
+++ b/python/tvm/topi/vision/nms.py
@@ -18,6 +18,8 @@
"""Non-maximum suppression operator"""
import tvm
from tvm import te
+from tvm.script.ir_builder import IRBuilder
+from tvm.script.ir_builder import tir as T
from tvm.tir import if_then_else
@@ -71,7 +73,6 @@ def get_valid_counts(
def _nms_loop(
- ib,
batch_size,
top_k,
iou_threshold,
@@ -85,82 +86,91 @@ def _nms_loop(
num_valid_boxes,
score_threshold=None,
):
- def nms_inner_loop(ib, i, j, nkeep, num_valid_boxes_local):
- on_new_valid_box_func(ib, 0, num_valid_boxes_local[0], i, j)
- num_valid_boxes_local[0] += 1
+ """NMS loop using modern IRBuilder. Must be called within IRBuilder context."""
+ out_scores = T.buffer_proxy(out_scores)
+ num_valid_boxes = T.buffer_proxy(num_valid_boxes)
+
+ def nms_inner_loop(i, j, nkeep, num_valid_boxes_local):
+ on_new_valid_box_func(0, num_valid_boxes_local[0], i, j)
+ num_valid_boxes_local[0] = num_valid_boxes_local[0] + 1
num_boxes_to_check = nkeep - (j + 1)
- with ib.for_range(0, num_boxes_to_check, name="_k", kind="parallel") as _k:
+ with T.parallel(0, num_boxes_to_check) as _k:
k = j + 1 + _k
- with ib.if_scope(
+ with T.If(
tvm.tir.all(
k < nkeep,
out_scores[i, k] > 0, # is the box k still valid?
needs_bbox_check_func(i, j, k),
)
):
- iou = calc_overlap_func(i, j, k)
+ with T.Then():
+ iou = calc_overlap_func(i, j, k)
- with ib.if_scope(iou >= iou_threshold):
- out_scores[i, k] = -1.0
- on_new_invalidated_box_func(i, k)
+ with T.If(iou >= iou_threshold):
+ with T.Then():
+ out_scores[i, k] = T.float32(-1.0)
+ on_new_invalidated_box_func(i, k)
- with ib.for_range(0, batch_size, name="i") as i:
+ with T.serial(0, batch_size) as i:
nkeep = if_then_else(tvm.tir.all(top_k > 0, top_k < valid_count[i]), top_k, valid_count[i])
- # Use max_output_size directly without if_then_else
- # max_output_size = if_then_else(max_output_size > te.const(0), max_output_size, nkeep)
- with ib.if_scope(tvm.tir.all(iou_threshold > te.const(0), valid_count[i] > te.const(0))):
- num_valid_boxes_local = ib.allocate(
- "int32", (1,), name="num_valid_boxes_local", scope="local"
- )
- num_valid_boxes_local[0] = 0
-
- # Use for_range to iterate through all boxes, but limit selection count
- with ib.for_range(0, nkeep, name="j") as j:
- with ib.if_scope(
- tvm.tir.all(
- out_scores[i, j] > -1.0, # box is still valid
- num_valid_boxes_local[0] < max_output_size, # haven't reached max limit
+ with T.If(tvm.tir.all(iou_threshold > te.const(0), valid_count[i] > te.const(0))):
+ with T.Then():
+ with T.allocate([1], "int32", scope="local") as num_valid_boxes_local_ptr:
+ num_valid_boxes_local = T.buffer_proxy(
+ tvm.tir.decl_buffer(
+ [1],
+ "int32",
+ "num_valid_boxes_local",
+ data=num_valid_boxes_local_ptr,
+ scope="local",
+ )
)
- ):
- if score_threshold is not None:
- with ib.if_scope(out_scores[i, j] > score_threshold[()]):
- nms_inner_loop(ib, i, j, nkeep, num_valid_boxes_local)
- else:
- nms_inner_loop(ib, i, j, nkeep, num_valid_boxes_local)
-
- num_valid_boxes[i] = num_valid_boxes_local[0]
-
- with ib.else_scope():
- num_valid_boxes[i] = 0
-
- return ib.get()
+ num_valid_boxes_local[0] = T.int32(0)
+
+ with T.serial(0, nkeep) as j:
+ with T.If(
+ tvm.tir.all(
+ out_scores[i, j] > -1.0, # box is still valid
+ num_valid_boxes_local[0]
+ < max_output_size, # haven't reached max limit
+ )
+ ):
+ with T.Then():
+ if score_threshold is not None:
+ with T.If(out_scores[i, j] > score_threshold[()]):
+ with T.Then():
+ nms_inner_loop(i, j, nkeep, num_valid_boxes_local)
+ else:
+ nms_inner_loop(i, j, nkeep, num_valid_boxes_local)
+
+ num_valid_boxes[i] = num_valid_boxes_local[0]
+
+ with T.Else():
+ num_valid_boxes[i] = T.int32(0)
def _get_valid_box_count(scores, score_threshold):
batch_classes, num_boxes = scores.shape
- def searchsorted_ir(scores, score_thresh, valid_count):
- ib = tvm.tir.ir_builder.create()
- scores = ib.buffer_ptr(scores)
- valid_count = ib.buffer_ptr(valid_count)
-
- with ib.for_range(0, batch_classes, name="i", kind="parallel") as i:
- if hasattr(score_threshold, "shape"):
- if len(score_threshold.shape) == 0:
- score_thresh_scalar = score_thresh[()]
- elif len(score_threshold.shape) == 1 and score_threshold.shape[0] > 0:
- score_thresh_scalar = score_thresh[0]
+ def searchsorted_ir(scores_buf, score_thresh_buf, valid_count_buf):
+ with IRBuilder() as ib:
+ with T.parallel(0, batch_classes) as i:
+ if hasattr(score_threshold, "shape"):
+ if len(score_threshold.shape) == 0:
+ score_thresh_scalar = score_thresh_buf[()]
+ elif len(score_threshold.shape) == 1 and score_threshold.shape[0] > 0:
+ score_thresh_scalar = score_thresh_buf[0]
+ else:
+ score_thresh_scalar = tvm.tir.FloatImm("float32", 0.0)
else:
- score_thresh_scalar = tvm.tir.FloatImm("float32", 0.0)
- else:
- score_thresh_scalar = score_threshold
- binary_search(ib, i, num_boxes, scores, score_thresh_scalar, valid_count)
+ score_thresh_scalar = score_threshold
+ binary_search(i, num_boxes, scores_buf, score_thresh_scalar, valid_count_buf)
- return ib.get()
+ return ib.get()
scores_buf = tvm.tir.decl_buffer(scores.shape, scores.dtype, "scores_buf", data_alignment=8)
searchsorted_buf = tvm.tir.decl_buffer(
@@ -183,24 +193,21 @@ def _get_valid_box_count(scores, score_threshold):
)
else:
- def searchsorted_ir_scalar(scores, valid_count):
- ib = tvm.tir.ir_builder.create()
- scores = ib.buffer_ptr(scores)
- valid_count = ib.buffer_ptr(valid_count)
-
- with ib.for_range(0, batch_classes, name="i", kind="parallel") as i:
- if isinstance(score_threshold, te.Tensor):
- if len(score_threshold.shape) == 0:
- score_thresh_tir = score_threshold()
- elif len(score_threshold.shape) == 1 and score_threshold.shape[0] == 1:
- score_thresh_tir = score_threshold[0]
+ def searchsorted_ir_scalar(scores_buf, valid_count_buf):
+ with IRBuilder() as ib:
+ with T.parallel(0, batch_classes) as i:
+ if isinstance(score_threshold, te.Tensor):
+ if len(score_threshold.shape) == 0:
+ score_thresh_tir = score_threshold()
+ elif len(score_threshold.shape) == 1 and score_threshold.shape[0] == 1:
+ score_thresh_tir = score_threshold[0]
+ else:
+ score_thresh_tir = tvm.tir.FloatImm("float32", 0.0)
else:
- score_thresh_tir = tvm.tir.FloatImm("float32", 0.0)
- else:
- score_thresh_tir = tvm.tir.FloatImm("float32", float(score_threshold))
- binary_search(ib, i, num_boxes, scores, score_thresh_tir, valid_count)
+ score_thresh_tir = tvm.tir.FloatImm("float32", float(score_threshold))
+ binary_search(i, num_boxes, scores_buf, score_thresh_tir, valid_count_buf)
- return ib.get()
+ return ib.get()
return te.extern(
[(batch_classes,)],
@@ -219,48 +226,45 @@ def _collect_selected_indices_ir(
):
batch_classes, _ = selected_indices.shape
- ib = tvm.tir.ir_builder.create()
+ with IRBuilder() as ib:
+ with T.seq_scope():
+ out = T.buffer_proxy(out)
- selected_indices = ib.buffer_ptr(selected_indices)
- num_detections = ib.buffer_ptr(num_detections)
- row_offsets = ib.buffer_ptr(row_offsets)
- out = ib.buffer_ptr(out)
-
- # Initialize output buffer to zero
- # Calculate the actual output shape based on max_output_boxes_per_class
- if isinstance(max_output_boxes_per_class, int):
- max_output_rows = batch_classes * max_output_boxes_per_class
- else:
- # Fallback to a reasonable default if max_output_boxes_per_class is not an integer
- max_output_rows = batch_classes * 10
- with ib.for_range(0, max_output_rows, name="init_i") as init_i:
- with ib.for_range(0, 3, name="init_j") as init_j: # 3 columns
- out[init_i, init_j] = cast(0, "int64")
-
- with ib.for_range(0, batch_classes, name="i", kind="parallel") as i:
- i = cast(i, "int64")
- batch_id = i // num_class
- class_id = i % num_class
-
- if isinstance(max_output_boxes_per_class, int):
- limit = tvm.tir.min(
- num_detections[i], tvm.tir.IntImm("int32", max_output_boxes_per_class)
- )
- elif isinstance(max_output_boxes_per_class, te.Tensor):
- if len(max_output_boxes_per_class.shape) == 0:
- max_boxes_val = max_output_boxes_per_class[()]
+ # Initialize output buffer to zero
+ # Calculate the actual output shape based on max_output_boxes_per_class
+ if isinstance(max_output_boxes_per_class, int):
+ max_output_rows = batch_classes * max_output_boxes_per_class
else:
- max_boxes_val = max_output_boxes_per_class[0]
- limit = tvm.tir.min(num_detections[i], max_boxes_val)
- else:
- limit = num_detections[i]
+ # Fallback to a reasonable default if max_output_boxes_per_class is not an integer
+ max_output_rows = batch_classes * 10
+ with T.serial(0, max_output_rows) as init_i:
+ with T.serial(0, 3) as init_j: # 3 columns
+ out[init_i, init_j] = cast(0, "int64")
+
+ with T.parallel(0, batch_classes) as i:
+ i_64 = cast(i, "int64")
+ batch_id = i_64 // num_class
+ class_id = i_64 % num_class
+
+ if isinstance(max_output_boxes_per_class, int):
+ limit = tvm.tir.min(
+ num_detections[i], tvm.tir.IntImm("int32", max_output_boxes_per_class)
+ )
+ elif isinstance(max_output_boxes_per_class, te.Tensor):
+ if len(max_output_boxes_per_class.shape) == 0:
+ max_boxes_val = max_output_boxes_per_class[()]
+ else:
+ max_boxes_val = max_output_boxes_per_class[0]
+ limit = tvm.tir.min(num_detections[i], max_boxes_val)
+ else:
+ limit = num_detections[i]
- with ib.for_range(0, limit, name="j") as j:
- out[row_offsets[i] + j, 0] = batch_id
- out[row_offsets[i] + j, 1] = class_id
- out[row_offsets[i] + j, 2] = cast(selected_indices[i, j], "int64")
+ with T.serial(0, limit) as j:
+ out[row_offsets[i] + j, 0] = batch_id
+ out[row_offsets[i] + j, 1] = class_id
+ out[row_offsets[i] + j, 2] = cast(selected_indices[i, j], "int64")
- return ib.get()
+ return ib.get()
def _collect_selected_indices_and_scores_ir(
@@ -275,41 +279,38 @@ def _collect_selected_indices_and_scores_ir(
batch_size, num_class = row_offsets.shape
num_boxes = selected_indices.shape[1]
- ib = tvm.tir.ir_builder.create()
-
- selected_indices = ib.buffer_ptr(selected_indices)
- selected_scores = ib.buffer_ptr(selected_scores)
- num_detections = ib.buffer_ptr(num_detections)
- row_offsets = ib.buffer_ptr(row_offsets)
- num_total_detections = ib.buffer_ptr(num_total_detections)
- collected_indices = ib.buffer_ptr(collected_indices)
- collected_scores = ib.buffer_ptr(collected_scores)
- zero = cast(0, "int64")
-
- with ib.for_range(0, batch_size * num_class, name="i", kind="parallel") as i:
- i = cast(i, "int64")
- batch_id = i // num_class
- class_id = i % num_class
-
- with ib.for_range(0, num_boxes, name="j") as j:
- with ib.if_scope(j < num_detections[batch_id, class_id]):
- offset = row_offsets[batch_id, class_id] + j
- collected_indices[batch_id, offset, 0] = class_id
- collected_indices[batch_id, offset, 1] = cast(selected_indices[i, j], "int64")
- collected_scores[batch_id, offset] = selected_scores[i, j]
- with ib.else_scope():
- offset = (
- num_total_detections[batch_id]
- + class_id * num_boxes
- - row_offsets[batch_id, class_id]
- + j
- - num_detections[batch_id, class_id]
- )
- collected_indices[batch_id, offset, 0] = zero
- collected_indices[batch_id, offset, 1] = zero
- collected_scores[batch_id, offset] = 0.0
+ with IRBuilder() as ib:
+ collected_indices = T.buffer_proxy(collected_indices)
+ collected_scores = T.buffer_proxy(collected_scores)
+ zero = cast(0, "int64")
+
+ with T.parallel(0, batch_size * num_class) as i:
+ i_64 = cast(i, "int64")
+ batch_id = i_64 // num_class
+ class_id = i_64 % num_class
+
+ with T.serial(0, num_boxes) as j:
+ with T.If(j < num_detections[batch_id, class_id]):
+ with T.Then():
+ offset = row_offsets[batch_id, class_id] + j
+ collected_indices[batch_id, offset, 0] = class_id
+ collected_indices[batch_id, offset, 1] = cast(
+ selected_indices[i, j], "int64"
+ )
+ collected_scores[batch_id, offset] = selected_scores[i, j]
+ with T.Else():
+ offset = (
+ num_total_detections[batch_id]
+ + class_id * num_boxes
+ - row_offsets[batch_id, class_id]
+ + j
+ - num_detections[batch_id, class_id]
+ )
+ collected_indices[batch_id, offset, 0] = zero
+ collected_indices[batch_id, offset, 1] = zero
+ collected_scores[batch_id, offset] = T.float32(0.0)
- return ib.get()
+ return ib.get()
def all_class_non_max_suppression(
diff --git a/python/tvm/topi/vision/nms_util.py b/python/tvm/topi/vision/nms_util.py
index 1633c923e1..f84b6c3762 100644
--- a/python/tvm/topi/vision/nms_util.py
+++ b/python/tvm/topi/vision/nms_util.py
@@ -18,6 +18,8 @@
"""Common utilities used in Non-maximum suppression operators"""
import tvm
from tvm import te
+from tvm.script.ir_builder import IRBuilder
+from tvm.script.ir_builder import tir as T
def _get_boundaries(output, box_idx):
@@ -58,22 +60,30 @@ def calculate_overlap(out_tensor, box_a_idx, box_b_idx):
return tvm.tir.Select(u <= 0.0, 0.0, area / u)
-def binary_search(ib, y, num_boxes, scores, score_threshold, out):
- """Binary search for score_threshold on scores sorted in descending order"""
- lo = ib.allocate("int32", (1,), name="lo", scope="local")
- hi = ib.allocate("int32", (1,), name="hi", scope="local")
+def binary_search(y, num_boxes, scores, score_threshold, out):
+ """Binary search for score_threshold on scores sorted in descending order.
- lo[0] = 0
- hi[0] = num_boxes.astype("int32")
-
- with ib.while_loop(lo[0] < hi[0]):
- mid = (hi[0] + lo[0]) >> 1
- with ib.if_scope(scores[y, mid] > score_threshold):
- lo[0] = mid + 1
- with ib.else_scope():
- hi[0] = mid
-
- out[y] = lo[0]
+ Must be called within an IRBuilder context.
+ """
+ out = T.buffer_proxy(out)
+ with T.frame_scope(
+ [
+ T.allocate([1], "int32", scope="local"),
+ T.allocate([1], "int32", scope="local"),
+ ]
+ ) as (lo_ptr, hi_ptr):
+ lo = T.buffer_proxy(tvm.tir.decl_buffer([1], "int32", "lo", data=lo_ptr, scope="local"))
+ hi = T.buffer_proxy(tvm.tir.decl_buffer([1], "int32", "hi", data=hi_ptr, scope="local"))
+ lo[0] = T.int32(0)
+ hi[0] = tvm.tir.Cast("int32", num_boxes)
+ with T.While(lo[0] < hi[0]):
+ mid = (hi[0] + lo[0]) >> 1
+ with T.If(scores[y, mid] > score_threshold):
+ with T.Then():
+ lo[0] = mid + 1
+ with T.Else():
+ hi[0] = mid
+ out[y] = lo[0]
def _estimate_max_detections(batch_class, input_image_size=None):
@@ -288,77 +298,76 @@ def _all_class_nms_ir(
nms_loop,
score_threshold=None,
):
- ib = tvm.tir.ir_builder.create()
- boxes = ib.buffer_ptr(boxes)
- sorted_scores = ib.buffer_ptr(sorted_scores)
- sorted_indices = ib.buffer_ptr(sorted_indices)
- valid_count = ib.buffer_ptr(valid_count)
- box_indices = ib.buffer_ptr(box_indices)
- num_valid_boxes = ib.buffer_ptr(num_valid_boxes)
-
- if selected_scores is not None:
- selected_scores = ib.buffer_ptr(selected_scores)
-
- if isinstance(iou_threshold, float):
- iou_threshold = tvm.tir.FloatImm("float32", iou_threshold)
- elif isinstance(iou_threshold, te.Tensor):
- if len(iou_threshold.shape) == 0:
- iou_threshold = iou_threshold()
- elif len(iou_threshold.shape) == 1 and iou_threshold.shape[0] == 1:
- iou_threshold = iou_threshold[0]
- else:
- iou_threshold = tvm.tir.FloatImm("float32", 0.5)
-
- if isinstance(max_output_size_per_class, int):
- max_output_size_per_class = tvm.tir.const(max_output_size_per_class)
- elif isinstance(max_output_size_per_class, te.Tensor):
- if len(max_output_size_per_class.shape) == 0:
- max_output_size_per_class = max_output_size_per_class()
- elif len(max_output_size_per_class.shape) == 1 and max_output_size_per_class.shape[0] == 1:
- # Use tensor indexing to get the first element
- max_output_size_per_class = max_output_size_per_class[0]
- else:
- max_output_size_per_class = tvm.tir.const(1000)
-
- def calc_overlap(i, j, k):
- offset_j = sorted_indices[i, j] * 4
- offset_k = sorted_indices[i, k] * 4
- batch_id = i // num_class
- base_bbox_idx = batch_id * num_anchors * 4
- return calculate_overlap(
- boxes,
- base_bbox_idx + offset_j,
- base_bbox_idx + offset_k,
+ with IRBuilder() as ib:
+ # Wrap buffers with T.buffer_proxy for flat indexing support
+ boxes = T.buffer_proxy(boxes)
+ box_indices = T.buffer_proxy(box_indices)
+ if selected_scores is not None:
+ selected_scores = T.buffer_proxy(selected_scores)
+
+ if isinstance(iou_threshold, float):
+ iou_threshold = tvm.tir.FloatImm("float32", iou_threshold)
+ elif isinstance(iou_threshold, te.Tensor):
+ if len(iou_threshold.shape) == 0:
+ iou_threshold = iou_threshold()
+ elif len(iou_threshold.shape) == 1 and iou_threshold.shape[0] == 1:
+ iou_threshold = iou_threshold[0]
+ else:
+ iou_threshold = tvm.tir.FloatImm("float32", 0.5)
+
+ if isinstance(max_output_size_per_class, int):
+ max_output_size_per_class = tvm.tir.const(max_output_size_per_class)
+ elif isinstance(max_output_size_per_class, te.Tensor):
+ if len(max_output_size_per_class.shape) == 0:
+ max_output_size_per_class = max_output_size_per_class()
+ elif (
+ len(max_output_size_per_class.shape) == 1
+ and max_output_size_per_class.shape[0] == 1
+ ):
+ max_output_size_per_class = max_output_size_per_class[0]
+ else:
+ max_output_size_per_class = tvm.tir.const(1000)
+
+ def calc_overlap(i, j, k):
+ offset_j = sorted_indices[i, j] * 4
+ offset_k = sorted_indices[i, k] * 4
+ batch_id = i // num_class
+ base_bbox_idx = batch_id * num_anchors * 4
+ return calculate_overlap(
+ boxes,
+ base_bbox_idx + offset_j,
+ base_bbox_idx + offset_k,
+ )
+
+ def on_new_valid_box(tid, num_current_valid_box, i, j):
+ with T.If(tid + 0 == 0):
+ with T.Then():
+ box_indices[i, num_current_valid_box] = sorted_indices[i, j]
+ if selected_scores is not None:
+ selected_scores[i, num_current_valid_box] = sorted_scores[i, j]
+
+ def on_new_invalidated_box(*_):
+ pass
+
+ def needs_bbox_check(*_):
+ return tvm.tir.const(True)
+
+ nms_loop(
+ batch_class,
+ tvm.tir.IntImm("int32", -1), # top_k
+ iou_threshold,
+ max_output_size_per_class,
+ valid_count,
+ on_new_valid_box,
+ on_new_invalidated_box,
+ needs_bbox_check,
+ calc_overlap,
+ sorted_scores,
+ num_valid_boxes,
+ score_threshold,
)
- def on_new_valid_box(ib, tid, num_current_valid_box, i, j):
- with ib.if_scope(tid + 0 == 0):
- box_indices[i, num_current_valid_box] = sorted_indices[i, j]
-
- if selected_scores is not None:
- selected_scores[i, num_current_valid_box] = sorted_scores[i, j]
-
- def on_new_invalidated_box(*_):
- pass
-
- def needs_bbox_check(*_):
- return tvm.tir.const(True)
-
- return nms_loop(
- ib,
- batch_class,
- tvm.tir.IntImm("int32", -1), # top_k
- iou_threshold,
- max_output_size_per_class,
- valid_count,
- on_new_valid_box,
- on_new_invalidated_box,
- needs_bbox_check,
- calc_overlap,
- sorted_scores,
- num_valid_boxes,
- score_threshold,
- )
+ return ib.get()
def run_all_class_nms(
diff --git a/tests/python/codegen/test_target_codegen_extern.py b/tests/python/codegen/test_target_codegen_extern.py
index 06e0926005..905a4bea80 100644
--- a/tests/python/codegen/test_target_codegen_extern.py
+++ b/tests/python/codegen/test_target_codegen_extern.py
@@ -15,66 +15,47 @@
# specific language governing permissions and limitations
# under the License.
import tvm
-from tvm import te
-import numpy as np
import tvm.testing
-import pytest
-from tvm import tir
+import numpy as np
+from tvm.script import tir as T, ir as I
@tvm.testing.uses_gpu
def test_add_pipeline():
+ """Test extern-style add pipeline with vectorized operations."""
nn = 64
max_threads = 4
- n = tvm.runtime.convert(nn)
- A = te.placeholder((n,), name="A")
-
- def extern_generator(ins, outs):
- """Manually write the IR for the extern function, add pipeline"""
- ib = tvm.tir.ir_builder.create()
- with ib.for_range(0, (n + 1) // 2) as i:
- ib.emit(
- outs[0].vstore(
- i * 2, ins[0].vload(i * 2, "float32x2") + tvm.tir.const(1, "float32x2")
- )
- )
- return ib.get()
-
- def extern_generator_gpu(ins, outs):
- """Manually write the IR for the extern function, add pipeline"""
- ib = tvm.tir.ir_builder.create()
- bx = te.thread_axis("blockIdx.x")
- tx = te.thread_axis("threadIdx.x")
- ib.scope_attr(bx, "thread_extent", (nn + max_threads - 1) // max_threads)
- ib.scope_attr(tx, "thread_extent", max_threads)
- idx = bx.var * max_threads + tx.var
- with ib.if_scope(ib.likely(idx < n)):
- ib.emit(
- outs[0].vstore(
- idx * 2, ins[0].vload(idx * 2, "float32x2") + tvm.tir.const(1, "float32x2")
- )
- )
- return ib.get()
- C_cpu = te.extern(A.shape, [A], extern_generator, name="C")
- C_gpu = te.extern(A.shape, [A], extern_generator_gpu, name="C")
-
- # Create IRModules directly
- mod_cpu = tvm.IRModule.from_expr(te.create_prim_func([A, C_cpu]))
- mod_gpu = tvm.IRModule.from_expr(te.create_prim_func([A, C_gpu]))
+ # CPU version: serial loop with vectorized operations
+ @I.ir_module
+ class ModuleCPU:
+ @T.prim_func
+ def main(A: T.Buffer((64,), "float32"), C: T.Buffer((64,), "float32")):
+ for i in T.serial((64 + 1) // 2):
+ C[T.Ramp(i * 2, 1, 2)] = A[T.Ramp(i * 2, 1, 2)] + T.Broadcast(T.float32(1), 2)
+
+ # GPU version: thread bindings with vectorized operations
+ @I.ir_module
+ class ModuleGPU:
+ @T.prim_func
+ def main(A: T.Buffer((64,), "float32"), C: T.Buffer((64,), "float32")):
+ bx = T.launch_thread("blockIdx.x", (64 + 4 - 1) // 4)
+ tx = T.launch_thread("threadIdx.x", 4)
+ idx = bx * 4 + tx
+ if T.likely(idx < 64):
+ C[T.Ramp(idx * 2, 1, 2)] = A[T.Ramp(idx * 2, 1, 2)] + T.Broadcast(T.float32(1), 2)
def check_target(target):
if not tvm.testing.device_enabled(target):
return
- mod = mod_gpu if target in ["opencl", "cuda"] else mod_cpu
- C = C_gpu if target in ["opencl", "cuda"] else C_cpu
+ mod = ModuleGPU if target in ["opencl", "cuda"] else ModuleCPU
# build and invoke the kernel.
f = tvm.compile(mod, target=target)
dev = tvm.device(target, 0)
# launch the kernel.
n = nn
- a = tvm.runtime.tensor(np.random.uniform(size=n).astype(A.dtype), dev)
- c = tvm.runtime.tensor(np.zeros(n, dtype=C.dtype), dev)
+ a = tvm.runtime.tensor(np.random.uniform(size=n).astype("float32"), dev)
+ c = tvm.runtime.tensor(np.zeros(n, dtype="float32"), dev)
f(a, c)
tvm.testing.assert_allclose(c.numpy(), a.numpy() + 1)
@@ -84,18 +65,14 @@ def test_add_pipeline():
def test_pack_buffer_simple():
+ """Test call_packed with buffer arguments."""
nn = 1024
- n = tvm.runtime.convert(nn)
- A = te.placeholder((n,), name="A")
-
- def extern_generator(ins, outs):
- """Manually write the IR for the extern function, add pipeline."""
- return tvm.tir.call_packed("my_extern_array_func1", ins[0], outs[0])
-
- C = te.extern(A.shape, [A], extern_generator, name="C")
- # Create IRModule directly
- mod = tvm.IRModule.from_expr(te.create_prim_func([A, C]))
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main(A: T.Buffer((1024,), "float32"), C: T.Buffer((1024,), "float32")):
+ T.evaluate(T.call_packed("my_extern_array_func1", A, C))
@tvm.register_global_func
def my_extern_array_func1(aa, bb):
@@ -105,12 +82,12 @@ def test_pack_buffer_simple():
if not tvm.testing.device_enabled(target):
return
# build and invoke the kernel.
- f = tvm.compile(mod, target=target)
+ f = tvm.compile(Module, target=target)
dev = tvm.cpu(0)
# launch the kernel.
n = nn
- a = tvm.runtime.tensor(np.random.uniform(size=n).astype(A.dtype), dev)
- c = tvm.runtime.tensor(np.zeros(n, dtype=C.dtype), dev)
+ a = tvm.runtime.tensor(np.random.uniform(size=n).astype("float32"), dev)
+ c = tvm.runtime.tensor(np.zeros(n, dtype="float32"), dev)
f(a, c)
tvm.testing.assert_allclose(c.numpy(), a.numpy())
@@ -118,44 +95,5 @@ def test_pack_buffer_simple():
check_target("llvm")
-@pytest.mark.skip("LEGACY-TO-FIX: limitation of create_prim_func with intermediate buffer")
-def test_pack_buffer_intermediate():
- nn = 1024
- n = tvm.runtime.convert(nn)
- A = te.placeholder((n,), name="A")
- B = te.compute((n,), lambda i: A[i] + 1, name="B")
-
- def extern_generator(ins, outs):
- """Manually write the IR for the extern function, add pipeline."""
- return tvm.tir.call_packed("my_extern_array_func2", ins[0], outs[0])
-
- C = te.extern(B.shape, [B], extern_generator, name="C")
- mod = tvm.IRModule.from_expr(te.create_prim_func([A, C]))
-
- def check_target(target):
- if not tvm.testing.device_enabled(target):
- return
- # build and invoke the kernel.
- f = tvm.compile(mod, target=target)
- dev = tvm.cpu(0)
- # launch the kernel.
- n = nn
- a = tvm.runtime.tensor(np.random.uniform(size=n).astype(A.dtype), dev)
- c = tvm.runtime.tensor(np.zeros(n, dtype=C.dtype), dev)
-
- @tvm.register_global_func
- def my_extern_array_func2(aa, bb):
- assert aa.shape == a.shape
- tvm.testing.assert_allclose(aa.numpy(), a.numpy() + 1)
- aa.copyto(bb)
-
- f(a, c)
- tvm.testing.assert_allclose(c.numpy(), a.numpy() + 1)
-
- check_target("llvm")
-
-
if __name__ == "__main__":
- test_pack_buffer_simple()
- test_pack_buffer_intermediate()
- test_add_pipeline()
+ tvm.testing.main()
diff --git a/tests/python/codegen/test_target_codegen_llvm.py b/tests/python/codegen/test_target_codegen_llvm.py
index fac7fe9c43..657044246d 100644
--- a/tests/python/codegen/test_target_codegen_llvm.py
+++ b/tests/python/codegen/test_target_codegen_llvm.py
@@ -31,26 +31,24 @@ from tvm.target.codegen import llvm_get_intrinsic_name, llvm_lookup_intrinsic_id
@tvm.testing.requires_llvm
def test_llvm_intrin():
- ib = tvm.tir.ir_builder.create()
- n = tvm.runtime.convert(4)
- A = ib.pointer("float32", name="A")
- args = [tvm.tir.call_intrin("handle", "tir.address_of", A[0]), 0, 3, 1]
- ib.emit(tvm.tir.Evaluate(tvm.tir.Call("void", "tir.prefetch", args)))
- body = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A], body).with_attr("global_symbol", "prefetch"))
+ @T.prim_func
+ def prefetch(A: T.handle("float32")):
+ T.func_attr({"global_symbol": "prefetch"})
+ A_buf = T.Buffer((4,), "float32", data=A)
+ T.evaluate(T.Call("void", "tir.prefetch", [T.address_of(A_buf[0]), 0, 3, 1]))
+
+ mod = tvm.IRModule.from_expr(prefetch)
fcode = tvm.compile(mod)
@tvm.testing.requires_llvm
def test_llvm_void_intrin():
- ib = tvm.tir.ir_builder.create()
- A = ib.pointer("uint8", name="A")
- # Create an intrinsic that returns void.
- x = tvm.tir.call_llvm_intrin("", "llvm.assume", tvm.tir.const(1, "int1"))
- ib.emit(x)
- body = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A], body).with_attr("global_symbol", "main"))
+ @T.prim_func
+ def main(A: T.handle("uint8")):
+ # Create an intrinsic that returns void.
+ T.call_llvm_intrin("", "llvm.assume", T.bool(True))
+
+ mod = tvm.IRModule.from_expr(main)
fcode = tvm.compile(mod)
@@ -69,38 +67,29 @@ def test_llvm_overloaded_intrin():
if tvm.target.codegen.llvm_version_major() < 5:
return
- def use_llvm_intrinsic(A, C):
- ib = tvm.tir.ir_builder.create()
- L = A.vload((0, 0))
- I = tvm.tir.call_llvm_pure_intrin("int32", "llvm.ctlz", L, tvm.tir.const(0, "int1"))
- S = C.vstore((0, 0), I)
- ib.emit(S)
- return ib.get()
-
- A = tvm.te.placeholder((1, 1), dtype="int32", name="A")
- C = tvm.te.extern(
- (1, 1), [A], lambda ins, outs: use_llvm_intrinsic(ins[0], outs[0]), name="C", dtype="int32"
- )
+ # int1 is the type for the is_zero_undef parameter
+ int1_zero = tvm.tir.const(0, "int1")
- # Convert to TIR and create schedule
- mod = te.create_prim_func([A, C])
- sch = tvm.s_tir.Schedule(mod)
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main(A: T.Buffer((1, 1), "int32"), C: T.Buffer((1, 1), "int32")):
+ with T.sblock("C"):
+ T.reads()
+ T.writes()
+ C[0, 0] = T.call_llvm_pure_intrin("int32", "llvm.ctlz", A[0, 0], int1_zero)
- # Build from scheduled TIR
- f = tvm.compile(sch.mod, target="llvm")
+ f = tvm.compile(Module, target="llvm")
@tvm.testing.requires_llvm
def test_llvm_lookup_intrin():
- ib = tvm.tir.ir_builder.create()
- A = ib.pointer("uint8x8", name="A")
- z = tvm.tir.const(0, "int32")
- x = tvm.tir.call_llvm_pure_intrin(
- "uint8x8", "llvm.ctpop.v8i8", tvm.tir.const(1, "uint32"), A[z]
- )
- ib.emit(x)
- body = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A], body).with_attr("global_symbol", "main"))
+ @T.prim_func
+ def main(A: T.handle("uint8x8")):
+ A_buf = T.Buffer((1,), "uint8x8", data=A)
+ T.evaluate(T.call_llvm_pure_intrin("uint8x8", "llvm.ctpop.v8i8", T.uint32(1), A_buf[0]))
+
+ mod = tvm.IRModule.from_expr(main)
fcode = tvm.compile(mod, None)
@@ -813,23 +802,24 @@ def test_llvm_order_functions():
# Note: the order is alphabetical because that's a predictable ordering. Any predictable
# ordering will work fine, but if the ordering changes, this test will need to be updated.
- def make_call_extern(caller, callee):
- # Create a function:
- # float32 caller(float32 v) { return callee(v); }
- ib = tvm.tir.ir_builder.create()
- v = tvm.te.var("v", dtype="float32")
- t = tvm.tir.call_extern("float32", callee, v)
- ib.emit(t)
- return tvm.tir.PrimFunc([v], ib.get()).with_attr("global_symbol", caller)
-
- # Create some functions in a random order.
- functions = {
- "Danny": make_call_extern("Danny", "Dave"),
- "Sammy": make_call_extern("Sammy", "Eve"),
- "Kirby": make_call_extern("Kirby", "Fred"),
- }
- mod = tvm.IRModule(functions=functions)
- ir_text = tvm.tir.build(mod, target="llvm").inspect_source("ll")
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def Danny(v: T.float32) -> T.float32:
+ T.func_attr({"global_symbol": "Danny"})
+ T.ret(T.call_extern("float32", "Dave", v))
+
+ @T.prim_func
+ def Sammy(v: T.float32) -> T.float32:
+ T.func_attr({"global_symbol": "Sammy"})
+ T.ret(T.call_extern("float32", "Eve", v))
+
+ @T.prim_func
+ def Kirby(v: T.float32) -> T.float32:
+ T.func_attr({"global_symbol": "Kirby"})
+ T.ret(T.call_extern("float32", "Fred", v))
+
+ ir_text = tvm.tir.build(Module, target="llvm").inspect_source("ll")
# Skip functions whose names start with _.
matches = re.findall(r"^define[^@]*@([a-zA-Z][a-zA-Z0-9_]*)", ir_text, re.MULTILINE)
assert matches == sorted(matches)
diff --git a/tests/python/codegen/test_target_codegen_static_init.py b/tests/python/codegen/test_target_codegen_static_init.py
index 3016191336..bfa5db0f64 100644
--- a/tests/python/codegen/test_target_codegen_static_init.py
+++ b/tests/python/codegen/test_target_codegen_static_init.py
@@ -16,50 +16,58 @@
# under the License.
import tvm
from tvm import te
+from tvm.script import ir as I, tir as T
import ctypes
import numpy as np
def test_static_callback():
- dtype = "int64"
- n = te.size_var("n")
- Ab = tvm.tir.decl_buffer((n,), dtype)
- i = te.size_var("i")
- ib = tvm.tir.ir_builder.create()
- A = ib.buffer_ptr(Ab)
- cp = te.thread_axis((0, 1), "cop")
- finit = tvm.tir.StringImm("TVMBackendRunOnce")
- ib.scope_attr(cp, "coproc_uop_scope", finit)
- with ib.for_range(0, n, "i", kind="parallel") as i:
- A[i] = A[i] + 1
- stmt = ib.get()
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def ramp(A: T.handle):
+ T.func_attr({"global_symbol": "ramp"})
+ n = T.int64()
+ Ab = T.match_buffer(A, (n,), "int64")
+ # coproc_uop_scope with TVMBackendRunOnce ensures body runs only once
+ with T.attr(
+ T.iter_var(T.int32(), (0, 1), "DataPar", "cop"),
+ "coproc_uop_scope",
+ "TVMBackendRunOnce",
+ ):
+ for i in T.parallel(n):
+ Ab[i] = Ab[i] + T.int64(1)
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([Ab], stmt).with_attr("global_symbol", "ramp"))
+ mod = Module
f = tvm.driver.build(mod, target="llvm")
- a = tvm.runtime.tensor(np.zeros(10, dtype=dtype))
+ a = tvm.runtime.tensor(np.zeros(10, dtype="int64"))
f(a)
f(a)
np.testing.assert_equal(a.numpy(), np.ones(a.shape[0]))
def test_static_init():
- dtype = "int64"
- n = te.size_var("n")
- Ab = tvm.tir.decl_buffer((n,), dtype)
- i = te.size_var("i")
- ib = tvm.tir.ir_builder.create()
- handle = tvm.tir.call_intrin("handle", "tir.tvm_static_handle")
- ib.emit(tvm.tir.call_packed("test_static_callback", handle, Ab))
-
@tvm.register_global_func("test_static_callback")
def test_cb(sh, A):
assert isinstance(sh, ctypes.c_void_p)
return sh
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([Ab], stmt).with_attr("global_symbol", "ramp"))
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def ramp(A: T.handle):
+ T.func_attr({"global_symbol": "ramp"})
+ n = T.int64()
+ Ab = T.match_buffer(A, (n,), "int64")
+ T.call_packed(
+ "test_static_callback",
+ T.call_intrin("handle", "tir.tvm_static_handle"),
+ Ab.data,
+ )
+
+ mod = Module
f = tvm.driver.build(mod, target="llvm")
- a = tvm.runtime.tensor(np.zeros(10, dtype=dtype))
+ a = tvm.runtime.tensor(np.zeros(10, dtype="int64"))
f(a)
diff --git a/tests/python/codegen/test_target_codegen_vulkan.py b/tests/python/codegen/test_target_codegen_vulkan.py
index 73d6597d10..c615d30245 100644
--- a/tests/python/codegen/test_target_codegen_vulkan.py
+++ b/tests/python/codegen/test_target_codegen_vulkan.py
@@ -211,49 +211,45 @@ def test_vulkan_while_if(target, dev):
target = tvm.target.Target(target)
n = 1
dtype = "int32"
- A = te.placeholder((n,), name="A", dtype=dtype)
-
- def do_compute(A, B, n):
- ib = tvm.tir.ir_builder.create()
- A = ib.buffer_ptr(A)
- B = ib.buffer_ptr(B)
- if "gpu" in target.keys:
- ib.scope_attr(te.thread_axis("blockIdx.x"), "thread_extent", 0)
+ def get_module(is_gpu):
+ if is_gpu:
- iterations = ib.allocate("int32", (1,), name="iterations", scope="local")
- iterations[0] = 0
- B[0] = 0
+ @T.prim_func
+ def while_if_gpu(A: T.Buffer((1,), "int32"), B: T.Buffer((1,), "int32")):
+ for bx in T.thread_binding(1, thread="blockIdx.x"):
+ iterations = T.decl_buffer((1,), "int32", scope="local")
+ iterations[0] = 0
+ B[0] = 0
+ while iterations[0] < T.if_then_else(A[0] > 0, 10, 20):
+ iterations[0] = iterations[0] + 1
+ B[0] = B[0] + iterations[0]
- loop_condition = iterations[0] < tvm.tir.if_then_else(A[0] > 0, 10, 20)
- with ib.while_loop(loop_condition):
- iterations[0] += 1
- B[0] += iterations[0]
+ return tvm.IRModule.from_expr(while_if_gpu.with_attr("target", target))
+ else:
- return ib.get()
+ @T.prim_func
+ def while_if_cpu(A: T.Buffer((1,), "int32"), B: T.Buffer((1,), "int32")):
+ iterations = T.decl_buffer((1,), "int32", scope="local")
+ iterations[0] = 0
+ B[0] = 0
+ while iterations[0] < T.if_then_else(A[0] > 0, 10, 20):
+ iterations[0] = iterations[0] + 1
+ B[0] = B[0] + iterations[0]
- B = te.extern(
- A.shape,
- [A],
- lambda ins, outs: do_compute(ins[0], outs[0], n),
- dtype=dtype,
- )
-
- # Create IRModule
- mod = tvm.IRModule.from_expr(te.create_prim_func([A, B]))
- sch = tvm.s_tir.Schedule(mod)
+ return tvm.IRModule.from_expr(while_if_cpu.with_attr("target", target))
- # Build
- func = tvm.compile(sch.mod, target=target)
+ mod = get_module("gpu" in target.keys)
+ compiled_func = tvm.compile(mod, target=target)
- a = tvm.runtime.tensor(np.array([5], dtype=A.dtype), dev)
- b = tvm.runtime.tensor(np.zeros(n, dtype=A.dtype), dev)
- func(a, b)
+ a = tvm.runtime.tensor(np.array([5], dtype=dtype), dev)
+ b = tvm.runtime.tensor(np.zeros(n, dtype=dtype), dev)
+ compiled_func(a, b)
tvm.testing.assert_allclose(b.numpy(), [55])
- a = tvm.runtime.tensor(np.array([-5], dtype=A.dtype), dev)
- b = tvm.runtime.tensor(np.zeros(n, dtype=A.dtype), dev)
- func(a, b)
+ a = tvm.runtime.tensor(np.array([-5], dtype=dtype), dev)
+ b = tvm.runtime.tensor(np.zeros(n, dtype=dtype), dev)
+ compiled_func(a, b)
tvm.testing.assert_allclose(b.numpy(), [210])
@@ -261,41 +257,22 @@ def test_vulkan_while_if(target, dev):
def test_vulkan_local_threadidx(target, dev):
target = tvm.target.Target(target)
n = 32
- A = te.placeholder((n,), name="A", dtype="int32")
-
- def do_compute(A, B, n):
- ib = tvm.tir.ir_builder.create()
- A = ib.buffer_ptr(A)
- B = ib.buffer_ptr(B)
-
- tx = te.thread_axis("threadIdx.x")
- with ib.for_range(0, 1):
- ib.scope_attr(tx, "thread_extent", 16)
- B[tx + 0] = A[tx + 0]
-
- with ib.for_range(0, 1):
- ib.scope_attr(tx, "thread_extent", 16)
- B[tx + 16] = A[tx + 16]
-
- return ib.get()
-
- B = te.extern(
- A.shape,
- [A],
- lambda ins, outs: do_compute(ins[0], outs[0], n),
- dtype="int32",
- )
-
- # Create IRModule
- mod = tvm.IRModule.from_expr(te.create_prim_func([A, B]))
- sch = tvm.s_tir.Schedule(mod)
-
- # Build
- func = tvm.compile(sch.mod, target=target)
-
- n = 32
- a_np = np.arange(n).astype(dtype=A.dtype)
+ @T.prim_func
+ def local_threadidx_func(A: T.Buffer((32,), "int32"), B: T.Buffer((32,), "int32")):
+ # First block with thread extent 16
+ for _ in range(1):
+ for tx in T.thread_binding(16, thread="threadIdx.x"):
+ B[tx + 0] = A[tx + 0]
+ # Second block with thread extent 16
+ for _ in range(1):
+ for tx in T.thread_binding(16, thread="threadIdx.x"):
+ B[tx + 16] = A[tx + 16]
+
+ mod = tvm.IRModule.from_expr(local_threadidx_func)
+ func = tvm.compile(mod, target=target)
+
+ a_np = np.arange(n).astype(dtype="int32")
b_np = np.zeros((n,), dtype="int32")
a = tvm.runtime.tensor(a_np, dev)
b = tvm.runtime.tensor(b_np, dev)
@@ -303,96 +280,67 @@ def test_vulkan_local_threadidx(target, dev):
tvm.testing.assert_allclose(b.numpy(), a_np)
-class TestVectorizedIndices:
- load_type, store_type = tvm.testing.parameters(
- # Load N values, write to N locations.
- # Vectorized copy.
- ("ramp", "ramp"),
- # Load 1 value, write to N locations.
- # Scalar load, vectorized store.
- #
- # Most TVM operations (e.g. schedule[tensor].vectorize(axis)) have
- # the broadcast outside of the index, but it is semantically okay
- # for the broadcast to be inside the index, and it shows up with
- # some optimizations.
- ("broadcast", "ramp"),
- # Load 1 values, write to 1 location.
- # Broadcasting on both sides should be equivalent to a scalar copy.
- ("broadcast", "broadcast"),
- # Loads N values, write to 1 location.
- # Disabled as it would have unclear semantics.
- # ("ramp","broadcoast"),
- )
- indirect_indices = tvm.testing.parameter(True, False, ids=["reorder", "no_reorder"])
-
- @tvm.testing.fixture
- def ref_data(self, load_type, store_type, indirect_indices):
- n = 4
-
- index_map = {
- "ramp": np.arange(n),
- "broadcast": np.zeros(n, dtype="int32"),
- }
-
- a_np = np.random.randint(np.iinfo("int32").max, size=n).astype("int32")
- b_np = np.zeros(shape=n, dtype=a_np.dtype)
- reorder_np = np.arange(n, dtype="int32")[::-1]
-
- load_index = index_map[load_type]
- store_index = index_map[store_type]
-
- if indirect_indices:
- load_index = reorder_np[load_index]
-
- b_np[store_index] = a_np[load_index]
-
- return a_np, reorder_np, b_np
-
- @tvm.testing.fixture
- def mod(self, target, load_type, store_type, indirect_indices):
- target = tvm.target.Target(target)
-
- n = 4
- dtype = "int32"
- A = te.placeholder((n,), dtype=dtype, name="A")
- R = te.placeholder((n,), dtype=dtype, name="R")
-
- def do_compute(ins, outs):
- ib = tvm.tir.ir_builder.create()
- A, R = map(ib.buffer_ptr, ins)
- B = ib.buffer_ptr(outs[0])
-
- if "gpu" in target.keys:
- ib.scope_attr(te.thread_axis("blockIdx.x"), "thread_extent", 0)
-
- index_map = {
- "ramp": tvm.tir.Ramp(0, 1, 4),
- "broadcast": tvm.tir.Broadcast(0, 4),
- }
+@tvm.testing.parametrize_targets("vulkan -from_device=0")
+def test_vectorized_index_ramp(target, dev):
+ """Test vectorized copy with ramp indices (load N values, write to N locations)"""
+ n = 4
+ ramp_index = tvm.tir.Ramp(0, 1, 4)
- load_index = index_map[load_type]
- store_index = index_map[store_type]
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main(var_A: T.handle, var_B: T.handle):
+ T.func_attr({"tir.noalias": True})
+ A = T.match_buffer(var_A, (n,), "int32", offset_factor=1)
+ B = T.match_buffer(var_B, (n,), "int32", offset_factor=1)
+ with T.sblock("compute"):
+ T.reads()
+ T.writes()
+ bx = T.launch_thread("blockIdx.x", 1)
+ B[ramp_index] = A[ramp_index]
- if indirect_indices:
- load_index = R[load_index]
+ f = tvm.compile(Module, target=target)
- B[store_index] = A[load_index]
+ a_np = np.random.randint(np.iinfo("int32").max, size=n).astype("int32")
+ b_np = np.zeros(n, dtype="int32")
- return ib.get()
+ a = tvm.runtime.tensor(a_np, dev)
+ b = tvm.runtime.tensor(b_np, dev)
+ f(a, b)
+ tvm.testing.assert_allclose(b.numpy(), a_np)
- B = te.extern(A.shape, [A, R], do_compute, dtype="int32")
- return tvm.IRModule.from_expr(te.create_prim_func([A, R, B]))
+@tvm.testing.parametrize_targets("vulkan -from_device=0")
+def test_vectorized_index_broadcast(target, dev):
+ """Test broadcast index (load 1 value, write to N locations)"""
+ n = 4
+ broadcast_index = tvm.tir.Broadcast(0, 4)
+ ramp_index = tvm.tir.Ramp(0, 1, 4)
- def test_ramp_broadcast_index(self, target, dev, mod, ref_data):
- f = tvm.compile(mod, target=target)
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main(var_A: T.handle, var_B: T.handle):
+ T.func_attr({"tir.noalias": True})
+ A = T.match_buffer(var_A, (n,), "int32", offset_factor=1)
+ B = T.match_buffer(var_B, (n,), "int32", offset_factor=1)
+ with T.sblock("compute"):
+ T.reads()
+ T.writes()
+ bx = T.launch_thread("blockIdx.x", 1)
+ # Load from broadcast index (single element), store to ramp index
+ B[ramp_index] = A[broadcast_index]
+
+ f = tvm.compile(Module, target=target)
+
+ a_np = np.random.randint(np.iinfo("int32").max, size=n).astype("int32")
+ b_np = np.zeros(n, dtype="int32")
- a_np, reorder_np, b_np = ref_data
- a = tvm.runtime.tensor(a_np, dev)
- r = tvm.runtime.tensor(reorder_np, dev)
- b = tvm.runtime.tensor(np.zeros(shape=b_np.shape, dtype="int32"), dev)
- f(a, r, b)
- tvm.testing.assert_allclose(b.numpy(), b_np)
+ a = tvm.runtime.tensor(a_np, dev)
+ b = tvm.runtime.tensor(b_np, dev)
+ f(a, b)
+ # All elements of b should be a[0] (broadcast load)
+ tvm.testing.assert_allclose(b.numpy(), np.full(n, a_np[0]))
def test_negative_operand_divmod(target, dev):
diff --git a/tests/python/relax/test_transform_legalize_ops_grad.py b/tests/python/relax/test_transform_legalize_ops_grad.py
index d326f751a8..cf9361d7c7 100644
--- a/tests/python/relax/test_transform_legalize_ops_grad.py
+++ b/tests/python/relax/test_transform_legalize_ops_grad.py
@@ -313,6 +313,7 @@ def test_take_backward():
rxplaceholder_1 = T.match_buffer(var_rxplaceholder_1, (T.int64(3), T.int64(4), T.int64(5)), offset_factor=1)
rxplaceholder_2 = T.match_buffer(var_rxplaceholder_2, (T.int64(2),), "int32", offset_factor=1)
with T.sblock("take_backward"):
+ T.attr(0, "pragma_scope", "seq")
for i in range(T.int64(60)):
out_buf[i // T.int64(5) // T.int64(4), i // T.int64(5) % T.int64(4), i % T.int64(5)] = T.float32(0)
for parallel, serial in T.grid(T.int64(15), T.int64(2)):
@@ -352,6 +353,7 @@ def test_take_backward_symbolic():
rxplaceholder_2 = T.match_buffer(var_rxplaceholder_2, (i,), "int32", offset_factor=1)
out_buf = T.match_buffer(var_take_backward, (m, n))
with T.sblock("take_backward"):
+ T.attr(0, "pragma_scope", "seq")
for i_1 in range(m * n):
out_buf[i_1 // n, i_1 % n] = T.float32(0)
for parallel, serial in T.grid(m, i):
diff --git a/tests/python/relax/test_transform_legalize_ops_manipulate.py b/tests/python/relax/test_transform_legalize_ops_manipulate.py
index ec5186f4f0..adfaab64a8 100644
--- a/tests/python/relax/test_transform_legalize_ops_manipulate.py
+++ b/tests/python/relax/test_transform_legalize_ops_manipulate.py
@@ -1389,6 +1389,7 @@ def test_scatter_elements():
var_rxplaceholder_2, (T.int64(2), T.int64(2)), offset_factor=1
)
with T.sblock("scatter_elements_generic"):
+ T.attr(0, "pragma_scope", "seq")
for i in T.parallel(T.int64(16)):
out_buf[i // T.int64(4), i % T.int64(4)] = rxplaceholder[
i // T.int64(4), i % T.int64(4)
@@ -1484,6 +1485,7 @@ def test_scatter_elements_symbolic():
rxplaceholder_2 = T.match_buffer(var_rxplaceholder_2, (m, n), offset_factor=1)
out_buf = T.match_buffer(var_scatter_elements_generic, (a, b))
with T.sblock("scatter_elements_generic"):
+ T.attr(0, "pragma_scope", "seq")
for i in T.parallel(a * b):
out_buf[i // b, i % b] = rxplaceholder[i // b, i % b]
for fused in T.parallel(m):
@@ -1819,6 +1821,7 @@ def test_scatter_nd():
with T.sblock("scatter_nd_generic"):
T.reads()
T.writes()
+ T.attr(0, "pragma_scope", "seq")
for i in range(T.int64(8)):
out_buf[i] = data[i]
for j in range(T.int64(4)):
diff --git a/tests/python/runtime/test_runtime_extension.py b/tests/python/runtime/test_runtime_extension.py
index 44534a6b47..daae64779d 100644
--- a/tests/python/runtime/test_runtime_extension.py
+++ b/tests/python/runtime/test_runtime_extension.py
@@ -15,24 +15,23 @@
# specific language governing permissions and limitations
# under the License.
import tvm
-from tvm import te
import numpy as np
+from tvm.script import ir as I, tir as T
def test_dltensor_compatible():
- dtype = "int64"
- n = te.var("n")
- Ab = tvm.tir.decl_buffer((n,), dtype)
- i = te.var("i")
- ib = tvm.tir.ir_builder.create()
- A = ib.buffer_ptr(Ab)
- with ib.for_range(0, n - 1, "i") as i:
- A[i + 1] = A[i] + 1
- stmt = ib.get()
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def arange(A: T.handle):
+ n = T.int32()
+ Ab = T.match_buffer(A, (n,), "int64")
+ for i in T.serial(n - 1):
+ Ab[i + 1] = Ab[i] + T.int64(1)
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([Ab], stmt).with_attr("global_symbol", "arange"))
+ mod = Module
f = tvm.compile(mod, target="llvm")
- a = tvm.runtime.tensor(np.zeros(10, dtype=dtype))
+ a = tvm.runtime.tensor(np.zeros(10, dtype="int64"))
f(a)
np.testing.assert_equal(a.numpy(), np.arange(a.shape[0]))
diff --git a/tests/python/tir-base/test_tir_base.py b/tests/python/tir-base/test_tir_base.py
index b23c600b15..114b9f784c 100644
--- a/tests/python/tir-base/test_tir_base.py
+++ b/tests/python/tir-base/test_tir_base.py
@@ -102,14 +102,12 @@ def test_ret_const():
def test_control_flow_jump():
- ib = tvm.tir.ir_builder.create()
- a = tir.Var("a", "float32")
- b = tir.Var("b", "float32")
- with ib.if_scope(True):
- ib.emit(tir.Evaluate(tir.ret(a)))
- ib.emit(tir.Evaluate(tir.ret(b)))
- stmt = ib.get()
- func = tir.PrimFunc([a, b], stmt)
+ @T.prim_func
+ def func(a: T.float32, b: T.float32):
+ if True:
+ T.evaluate(T.ret(a))
+ T.evaluate(T.ret(b))
+
func = build_tir_func(func)
out = func(1.0, 2.0)
assert out == 1.0
diff --git a/tests/python/tir-base/test_tir_stmt_functor_ir_transform.py b/tests/python/tir-base/test_tir_stmt_functor_ir_transform.py
index a22e5bf466..6254b34a6e 100644
--- a/tests/python/tir-base/test_tir_stmt_functor_ir_transform.py
+++ b/tests/python/tir-base/test_tir_stmt_functor_ir_transform.py
@@ -15,18 +15,29 @@
# specific language governing permissions and limitations
# under the License.
import tvm
-from tvm import te
+from tvm.script import ir as I, tir as T
def test_ir_transform():
- ib = tvm.tir.ir_builder.create()
- n = te.var("n")
- with ib.for_range(0, n, name="i") as i:
- with ib.for_range(0, 10, name="j") as j:
- x = tvm.tir.call_extern("int32", "TestA", i * 3 + j * 1)
- ib.emit(tvm.tir.call_extern("int32", "TestB", x))
- ib.emit(tvm.tir.call_extern("int32", "TestC", x))
- body = ib.get()
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main(n: T.int32):
+ for i in T.serial(n):
+ for j in T.serial(10):
+ # Inline call_extern to avoid Let binding (x must be the Call node itself)
+ T.evaluate(
+ T.call_extern(
+ "int32", "TestB", T.call_extern("int32", "TestA", i * 3 + j * 1)
+ )
+ )
+ T.evaluate(
+ T.call_extern(
+ "int32", "TestC", T.call_extern("int32", "TestA", i * 3 + j * 1)
+ )
+ )
+
+ body = Module["main"].body
builtin_call_extern = tvm.ir.Op.get("tir.call_extern")
def preorder(op):
diff --git a/tests/python/tir-base/test_tir_structural_equal_hash.py b/tests/python/tir-base/test_tir_structural_equal_hash.py
index 01af60724c..296450b9a2 100644
--- a/tests/python/tir-base/test_tir_structural_equal_hash.py
+++ b/tests/python/tir-base/test_tir_structural_equal_hash.py
@@ -214,20 +214,19 @@ def test_stmt():
ii = te.var("i")
jj = te.var("j")
- Ab = tvm.tir.decl_buffer((n,), name="A")
n = te.var("n")
- def func2():
- ib = tvm.tir.ir_builder.create()
- A = ib.buffer_ptr(Ab)
- with ib.for_range(0, n, name="i") as i:
- A[i] = A[i] + 1
- with ib.for_range(0, 10, name="j") as j:
- A[j] = A[j] + 2
- A[j] = A[j] + 2
- return ib.get()
-
- assert consistent_equal(func2(), func2())
+ @T.prim_func(private=True, check_well_formed=False)
+ def func2(A: T.handle, n_param: T.int32):
+ n_var = T.var("int32")
+ Ab = T.match_buffer(A, (n_var,))
+ for i in T.serial(n_var):
+ Ab[i] = Ab[i] + T.float32(1)
+ for j in T.serial(10):
+ Ab[j] = Ab[j] + T.float32(2)
+ Ab[j] = Ab[j] + T.float32(2)
+
+ assert consistent_equal(func2.body, func2.body)
def test_buffer_storage_scope():
diff --git a/tests/python/tir-transform/test_tir_transform_hoist_if.py b/tests/python/tir-transform/test_tir_transform_hoist_if.py
index 57e42fda24..a941c6c019 100644
--- a/tests/python/tir-transform/test_tir_transform_hoist_if.py
+++ b/tests/python/tir-transform/test_tir_transform_hoist_if.py
@@ -16,6 +16,7 @@
# under the License.
import tvm
from tvm import te
+from tvm.script import tir as T, ir as I
import numpy as np
import pytest
from tvm.testing import enabled_targets
@@ -65,22 +66,17 @@ def _opaque_eval(var):
def test_hoist_top_for():
- ib = tvm.tir.ir_builder.create()
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
- data = ib.pointer("float32", name="data")
-
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- with ib.for_range(0, n, "k") as k:
- with ib.if_scope(ib.likely(i < 2)):
- ib.emit(_opaque_eval(m))
- with ib.else_scope():
- ib.emit(_opaque_eval(n))
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
+ @T.prim_func(private=True)
+ def func(l: T.int32, m: T.int32, n: T.int32):
+ for i in T.serial(l):
+ for j in T.serial(m):
+ for k in T.serial(n):
+ if T.likely(i < 2):
+ T.evaluate(T.call_extern("int32", "dummy", m))
+ else:
+ T.evaluate(T.call_extern("int32", "dummy", n))
+
+ mod = tvm.IRModule.from_expr(func)
new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
expected_struct = {
("tir.For", "k"): (None,),
@@ -92,22 +88,17 @@ def test_hoist_top_for():
def test_hoist_multi_var_if():
- ib = tvm.tir.ir_builder.create()
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
- data = ib.pointer("float32", name="data")
-
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- with ib.for_range(0, n, "k") as k:
- with ib.if_scope(ib.likely(i + j < 2)):
- ib.emit(_opaque_eval(m))
- with ib.else_scope():
- ib.emit(_opaque_eval(n))
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
+ @T.prim_func(private=True)
+ def func(l: T.int32, m: T.int32, n: T.int32):
+ for i in T.serial(l):
+ for j in T.serial(m):
+ for k in T.serial(n):
+ if T.likely(i + j < 2):
+ T.evaluate(T.call_extern("int32", "dummy", m))
+ else:
+ T.evaluate(T.call_extern("int32", "dummy", n))
+
+ mod = tvm.IRModule.from_expr(func)
new_mod = tvm.tir.transform.HoistIfThenElse()(mod)
new_stmt = new_mod["main"].body
expected_struct = {
@@ -120,23 +111,19 @@ def test_hoist_multi_var_if():
def test_hoist_no_match_for():
- ib = tvm.tir.ir_builder.create()
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
- data = ib.pointer("float32", name="data")
-
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- data[i * 3 + j] = data[i * 3 + j] + 0.5
- with ib.for_range(0, n, "k") as k:
- with ib.if_scope(ib.likely(i < 2)):
- ib.emit(_opaque_eval(m))
- with ib.else_scope():
- ib.emit(_opaque_eval(n))
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
+ @T.prim_func(private=True)
+ def func(data: T.handle("float32"), l: T.int32, m: T.int32, n: T.int32):
+ data_ptr = T.decl_buffer(1, "float32", data=data)
+ for i in T.serial(l):
+ for j in T.serial(m):
+ data_ptr[i * 3 + j] = data_ptr[i * 3 + j] + T.float32(0.5)
+ for k in T.serial(n):
+ if T.likely(i < 2):
+ T.evaluate(T.call_extern("int32", "dummy", m))
+ else:
+ T.evaluate(T.call_extern("int32", "dummy", n))
+
+ mod = tvm.IRModule.from_expr(func)
new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
expected_struct = {
("tir.For", "k"): (None,),
@@ -148,19 +135,15 @@ def test_hoist_no_match_for():
def test_no_else():
- ib = tvm.tir.ir_builder.create()
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
-
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- with ib.for_range(0, n, "k") as k:
- with ib.if_scope(ib.likely(i < 2)):
- ib.emit(_opaque_eval(m))
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
+ @T.prim_func(private=True)
+ def func(l: T.int32, m: T.int32, n: T.int32):
+ for i in T.serial(l):
+ for j in T.serial(m):
+ for k in T.serial(n):
+ if T.likely(i < 2):
+ T.evaluate(T.call_extern("int32", "dummy", m))
+
+ mod = tvm.IRModule.from_expr(func)
new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
expected_struct = {
("tir.For", "k"): (None,),
@@ -172,27 +155,26 @@ def test_no_else():
def test_attr_stmt():
- ib = tvm.tir.ir_builder.create()
dshape = (32, 64)
- data = ib.pointer("float32", name="data")
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
-
- tx = te.thread_axis("threadIdx.x")
- bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", dshape[0])
- ib.scope_attr(bx, "thread_extent", dshape[1])
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- with ib.for_range(0, n, "k") as k:
- with ib.if_scope(tvm.tir.any(i < 4, j >= 8)):
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 0.5
- with ib.else_scope():
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 1.0
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
+
+ @T.prim_func(private=True)
+ def func(data: T.handle("float32"), l: T.int32, m: T.int32, n: T.int32):
+ data_ptr = T.decl_buffer(1, "float32", data=data)
+ tx = T.launch_thread("threadIdx.x", dshape[0])
+ bx = T.launch_thread("blockIdx.x", dshape[1])
+ for i in T.serial(l):
+ for j in T.serial(m):
+ for k in T.serial(n):
+ if i < 4 or j >= 8:
+ data_ptr[bx * j + tx * j * k] = data_ptr[bx * j + tx * j * k] + T.float32(
+ 0.5
+ )
+ else:
+ data_ptr[bx * j + tx * j * k] = data_ptr[bx * j + tx * j * k] + T.float32(
+ 1.0
+ )
+
+ mod = tvm.IRModule.from_expr(func)
new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
expected_struct = {
("tir.For", "k"): (None,),
@@ -206,22 +188,25 @@ def test_attr_stmt():
def test_nested_for():
- ib = tvm.tir.ir_builder.create()
- data = ib.pointer("float32", name="data")
-
- with ib.for_range(0, 5, "i") as i:
- with ib.for_range(0, 10, "j") as j:
- with ib.if_scope(i >= 3):
- data[i * 3 + j] = data[i * 3 + j] + 0.5
- with ib.for_range(0, 15, "k") as k:
- with ib.for_range(0, 20, "l") as l:
- with ib.if_scope(tvm.tir.any(i < 4, j >= 8)):
- data[i * 3 + j + k + l] = data[i * 3 + j + k + l] * 2
- with ib.else_scope():
- data[i * 3 + j + k + l] = data[i * 3 + j + k + l] * 1.5
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
+ @T.prim_func(private=True)
+ def func(data: T.handle("float32")):
+ data_ptr = T.decl_buffer(1, "float32", data=data)
+ for i in range(5):
+ for j in range(10):
+ if i >= 3:
+ data_ptr[i * 3 + j] = data_ptr[i * 3 + j] + T.float32(0.5)
+ for k in range(15):
+ for l in range(20):
+ if i < 4 or j >= 8:
+ data_ptr[i * 3 + j + k + l] = data_ptr[
+ i * 3 + j + k + l
+ ] * T.float32(2)
+ else:
+ data_ptr[i * 3 + j + k + l] = data_ptr[
+ i * 3 + j + k + l
+ ] * T.float32(1.5)
+
+ mod = tvm.IRModule.from_expr(func)
new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
expected_struct = {
("tir.For", "l"): (None,),
@@ -235,58 +220,66 @@ def test_nested_for():
def test_if_block():
- ib = tvm.tir.ir_builder.create()
- data = ib.pointer("float32", name="data")
- n = te.var("n")
-
- with ib.for_range(0, 5, "i") as i:
- with ib.for_range(0, 10, "j") as j:
- with ib.if_scope(i >= 3):
- data[i * 3 + j] = data[i * 3 + j] + 0.5
- with ib.for_range(0, 15, "k") as k:
- with ib.for_range(0, 20, "l") as l:
- with ib.if_scope(tvm.tir.any(i < 4, j >= 8)):
- data[i * 3 + j + k + l] = data[i * 3 + j + k + l] * 2
- with ib.else_scope():
- data[i * 3 + j + k + l] = data[i * 3 + j + k + l] * 1.5
- with ib.if_scope(j < 5):
- data[i * 3 + j + k + l] = data[i * 3 + j + k + l] - 1
-
- with ib.for_range(0, 5, "i") as i:
- with ib.for_range(0, 10, "j") as j:
- with ib.for_range(0, 15, "k") as k:
- with ib.if_scope(n >= 3):
- data[i * 3 + j + k] = data[i * 3 + j + k] + 0.6
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
+ # Use different variable names for second loop nest to avoid dict key collision
+ @I.ir_module
+ class Module:
+ @T.prim_func(private=True)
+ def main(data: T.Buffer((1,), "float32"), n: T.int32):
+ # First loop nest: i, j, k, l
+ for i in T.serial(5):
+ for j in T.serial(10):
+ if i >= 3:
+ data[i * 3 + j] = data[i * 3 + j] + T.float32(0.5)
+ for k in T.serial(15):
+ for l in T.serial(20):
+ if i < 4 or j >= 8:
+ data[i * 3 + j + k + l] = data[i * 3 + j + k + l] * T.float32(2)
+ else:
+ data[i * 3 + j + k + l] = data[i * 3 + j + k + l] * T.float32(
+ 1.5
+ )
+ if j < 5:
+ data[i * 3 + j + k + l] = data[i * 3 + j + k + l] - T.float32(1)
+
+ # Second loop nest: i2, j2, k2 (different names)
+ for i2 in T.serial(5):
+ for j2 in T.serial(10):
+ for k2 in T.serial(15):
+ if n >= 3:
+ data[i2 * 3 + j2 + k2] = data[i2 * 3 + j2 + k2] + T.float32(0.6)
+
+ new_stmt = tvm.tir.transform.HoistIfThenElse()(Module)["main"].body
+ # Updated expected_struct with renamed second nest variables
expected_struct = {
("tir.IfThenElse", ("i", "j")): (None, None),
("tir.IfThenElse", ("j",)): (None, None),
("tir.For", "l"): (None,),
- ("tir.For", "k"): (None,),
- ("tir.For", "j"): (("tir.For", "j"),),
+ ("tir.For", "k"): (("tir.For", "l"),),
+ ("tir.For", "j"): (None,),
("tir.IfThenElse", ("i",)): (("tir.For", "j"), None),
("tir.For", "i"): (("tir.IfThenElse", ("i",)),),
- ("tir.IfThenElse", ("n",)): (("tir.For", "j"), None),
+ ("tir.For", "k2"): (None,),
+ ("tir.For", "j2"): (("tir.For", "k2"),),
+ ("tir.For", "i2"): (("tir.For", "j2"),),
+ ("tir.IfThenElse", ("n",)): (("tir.For", "i2"), None),
}
verify_structure(new_stmt, expected_struct)
def test_multi_if():
- ib = tvm.tir.ir_builder.create()
- data = ib.pointer("float32", name="data")
-
- with ib.for_range(0, 10, "i") as i:
- with ib.for_range(0, 10, "j") as j:
- with ib.for_range(0, 10, "k") as k:
- with ib.if_scope(3 <= i):
- with ib.if_scope(3 <= j):
- data[i * 100 + j * 10 + k] = data[i * 100 + j * 10 + k] + 0.5
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
+ @T.prim_func(private=True)
+ def func(data: T.handle("float32")):
+ data_ptr = T.decl_buffer(1, "float32", data=data)
+ for i in range(10):
+ for j in range(10):
+ for k in range(10):
+ if 3 <= i:
+ if 3 <= j:
+ data_ptr[i * 100 + j * 10 + k] = data_ptr[
+ i * 100 + j * 10 + k
+ ] + T.float32(0.5)
+
+ mod = tvm.IRModule.from_expr(func)
new_mod = tvm.tir.transform.HoistIfThenElse()(mod)
new_stmt = new_mod["main"].body
expected_struct = {
@@ -300,18 +293,19 @@ def test_multi_if():
def test_no_hoisting_1():
- ib = tvm.tir.ir_builder.create()
- data = ib.pointer("float32", name="data")
- n = te.var("n")
-
- with ib.for_range(0, 10, "i") as i:
- with ib.for_range(0, 10, "j") as j:
- with ib.for_range(0, 10, "k") as k:
- with ib.if_scope(k <= 3):
- data[i * 100 + j * 10 + k] = data[i * 100 + j * 10 + k] + 0.5
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
+ @T.prim_func(private=True)
+ def func(data: T.handle("float32")):
+ data_ptr = T.decl_buffer(1, "float32", data=data)
+ for i in range(10):
+ for j in range(10):
+ for k in range(10):
+ if k <= 3:
+ data_ptr[i * 100 + j * 10 + k] = data_ptr[i * 100 + j * 10 + k] + T.float32(
+ 0.5
+ )
+
+ mod = tvm.IRModule.from_expr(func)
+ stmt = mod["main"].body
new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
tvm.ir.assert_structural_equal(new_stmt, stmt)
@@ -323,56 +317,20 @@ def test_no_hoisting_1():
def test_no_hoisting_2():
- ib = tvm.tir.ir_builder.create()
- data = ib.pointer("float32", name="data")
- n = te.var("n")
- x = te.var("x")
-
- with ib.for_range(0, 10, "i") as i:
- with ib.for_range(0, 10, "j") as j:
- with ib.for_range(0, 10, "k") as k:
- with ib.if_scope(i <= 3):
- data[i * 100 + j * 10 + k] = data[i * 100 + j * 10 + k] + 0.3
- data[i * 100 + j * 10 + k] = data[i * 100 + j * 10 + k] + 0.5
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
- tvm.ir.assert_structural_equal(new_stmt, stmt)
-
- with tvm.transform.PassContext(
- config={"tir.HoistIfThenElse": {"support_block_scope_hoisting": True}}
- ):
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
- tvm.ir.assert_structural_equal(new_stmt, stmt)
-
-
-@pytest.mark.xfail(reason="Inconsistent thread_extent", strict=True)
-def test_no_hoisting_3():
- ib = tvm.tir.ir_builder.create()
- dshape = (32, 64)
- dshape_inner = (33, 63)
- data = ib.pointer("float32", name="data")
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
-
- tx = te.thread_axis("threadIdx.x")
- bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", dshape[0])
- ib.scope_attr(bx, "thread_extent", dshape[1])
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- with ib.for_range(0, n, "k") as k:
- ib.scope_attr(tx, "thread_extent", dshape_inner[0])
- ib.scope_attr(bx, "thread_extent", dshape_inner[1])
- with ib.if_scope(tx < 3):
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 0.3
- with ib.else_scope():
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 1.3
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
+ @T.prim_func(private=True)
+ def func(data: T.handle("float32")):
+ data_ptr = T.decl_buffer(1, "float32", data=data)
+ for i in range(10):
+ for j in range(10):
+ for k in range(10):
+ if i <= 3:
+ data_ptr[i * 100 + j * 10 + k] = data_ptr[i * 100 + j * 10 + k] + T.float32(
+ 0.3
+ )
+ data_ptr[i * 100 + j * 10 + k] = data_ptr[i * 100 + j * 10 + k] + T.float32(0.5)
+
+ mod = tvm.IRModule.from_expr(func)
+ stmt = mod["main"].body
new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
tvm.ir.assert_structural_equal(new_stmt, stmt)
@@ -384,161 +342,129 @@ def test_no_hoisting_3():
def test_no_hoisting_4():
- ib = tvm.tir.ir_builder.create()
dshape = (32, 64)
dshape_inner = (33, 63)
- data = ib.pointer("float32", name="data")
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
-
- tx = te.thread_axis("threadIdx.x")
- bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(bx, "thread_extent", dshape[1])
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- with ib.for_range(0, n, "k") as k:
- ib.scope_attr(tx, "thread_extent", dshape_inner[0])
- with ib.if_scope(tx < 3):
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 0.3
- with ib.else_scope():
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 1.3
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
- tvm.ir.assert_structural_equal(new_stmt, stmt)
-
- with tvm.transform.PassContext(
- config={"tir.HoistIfThenElse": {"support_block_scope_hoisting": True}}
- ):
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
- tvm.ir.assert_structural_equal(new_stmt, stmt)
+ # Create iter_var for tx (used inside loop with T.attr)
+ tx_var = tvm.tir.Var("threadIdx.x", "int32")
+ tx_iter = tvm.tir.IterVar(
+ tvm.ir.Range(0, dshape_inner[0]), tx_var, tvm.tir.IterVar.ThreadIndex, "threadIdx.x"
+ )
-@pytest.mark.xfail(reason="Inconsistent thread_extent", strict=True)
-def test_no_hoisting_5():
- ib = tvm.tir.ir_builder.create()
- dshape = (32, 64)
- dshape_inner = (33, 63)
- data = ib.pointer("float32", name="data")
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
-
- tx = te.thread_axis("threadIdx.x")
- bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", dshape[0])
- ib.scope_attr(bx, "thread_extent", dshape[1])
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- ib.scope_attr(bx, "thread_extent", dshape_inner[1])
- with ib.for_range(0, n, "k") as k:
- ib.scope_attr(tx, "thread_extent", dshape_inner[0])
- with ib.if_scope(tx < 3):
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 0.3
- with ib.else_scope():
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 1.3
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
+ @I.ir_module
+ class Module:
+ @T.prim_func(private=True)
+ def main(data: T.Buffer((1,), "float32"), l: T.int32, m: T.int32, n: T.int32):
+ bx = T.launch_thread("blockIdx.x", dshape[1])
+ for i in T.serial(l):
+ for j in T.serial(m):
+ for k in T.serial(n):
+ T.attr(tx_iter, "thread_extent", dshape_inner[0])
+ if tx_var < 3:
+ data[bx * j + tx_var * j * k] = data[
+ bx * j + tx_var * j * k
+ ] + T.float32(0.3)
+ else:
+ data[bx * j + tx_var * j * k] = data[
+ bx * j + tx_var * j * k
+ ] + T.float32(1.3)
+
+ stmt = Module["main"].body
+ new_stmt = tvm.tir.transform.HoistIfThenElse()(Module)["main"].body
tvm.ir.assert_structural_equal(new_stmt, stmt)
with tvm.transform.PassContext(
config={"tir.HoistIfThenElse": {"support_block_scope_hoisting": True}}
):
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
+ new_stmt = tvm.tir.transform.HoistIfThenElse()(Module)["main"].body
tvm.ir.assert_structural_equal(new_stmt, stmt)
def test_no_hoisting_6():
- ib = tvm.tir.ir_builder.create()
dshape = (32, 64)
- data = ib.pointer("float32", name="data")
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
-
- tx = te.thread_axis("threadIdx.x")
- bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", dshape[0])
- ib.scope_attr(bx, "thread_extent", dshape[1])
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- with ib.for_range(0, n, "k") as k:
- with ib.if_scope((tx + k) < 3):
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 0.3
- with ib.else_scope():
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 1.3
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
+
+ @I.ir_module
+ class Module:
+ @T.prim_func(private=True)
+ def main(data: T.Buffer((1,), "float32"), l: T.int32, m: T.int32, n: T.int32):
+ tx = T.launch_thread("threadIdx.x", dshape[0])
+ bx = T.launch_thread("blockIdx.x", dshape[1])
+ for i in T.serial(l):
+ for j in T.serial(m):
+ for k in T.serial(n):
+ if tx + k < 3:
+ data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + T.float32(0.3)
+ else:
+ data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + T.float32(1.3)
+
+ stmt = Module["main"].body
+ new_stmt = tvm.tir.transform.HoistIfThenElse()(Module)["main"].body
tvm.ir.assert_structural_equal(new_stmt, stmt)
with tvm.transform.PassContext(
config={"tir.HoistIfThenElse": {"support_block_scope_hoisting": True}}
):
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
+ new_stmt = tvm.tir.transform.HoistIfThenElse()(Module)["main"].body
tvm.ir.assert_structural_equal(new_stmt, stmt)
def test_no_hoisting_7():
- ib = tvm.tir.ir_builder.create()
dshape = (32, 64)
- data = ib.pointer("float32", name="data")
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
-
- tx = te.thread_axis("threadIdx.x")
- bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", dshape[0])
- ib.scope_attr(bx, "thread_extent", dshape[1])
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- with ib.if_scope((tx + j) < 9):
- with ib.for_range(0, n, "k") as k:
- with ib.if_scope((tx + k) < 3):
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 0.3
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
+
+ @I.ir_module
+ class Module:
+ @T.prim_func(private=True)
+ def main(data: T.Buffer((1,), "float32"), l: T.int32, m: T.int32, n: T.int32):
+ tx = T.launch_thread("threadIdx.x", dshape[0])
+ bx = T.launch_thread("blockIdx.x", dshape[1])
+ for i in T.serial(l):
+ for j in T.serial(m):
+ if tx + j < 9:
+ for k in T.serial(n):
+ if tx + k < 3:
+ data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + T.float32(
+ 0.3
+ )
+
+ stmt = Module["main"].body
+ new_stmt = tvm.tir.transform.HoistIfThenElse()(Module)["main"].body
tvm.ir.assert_structural_equal(new_stmt, stmt)
with tvm.transform.PassContext(
config={"tir.HoistIfThenElse": {"support_block_scope_hoisting": True}}
):
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
+ new_stmt = tvm.tir.transform.HoistIfThenElse()(Module)["main"].body
tvm.ir.assert_structural_equal(new_stmt, stmt)
def test_hoisting_block_scope_2():
- ib = tvm.tir.ir_builder.create()
dshape = (32, 64)
- dshape_inner = (33, 63)
- data = ib.pointer("float32", name="data")
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
-
- tx = te.thread_axis("threadIdx.x")
- bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", dshape[0])
- # ib.scope_attr(bx, "thread_extent", dshape[1])
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- with ib.for_range(0, n, "k") as k:
- ib.scope_attr(bx, "thread_extent", dshape[1])
- with ib.if_scope(tx < 3):
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 0.3
- with ib.else_scope():
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 1.3
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
+
+ # Create iter_var for bx (used inside loop with T.attr)
+ bx_var = tvm.tir.Var("blockIdx.x", "int32")
+ bx_iter = tvm.tir.IterVar(
+ tvm.ir.Range(0, dshape[1]), bx_var, tvm.tir.IterVar.ThreadIndex, "blockIdx.x"
+ )
+
+ @I.ir_module
+ class Module:
+ @T.prim_func(private=True)
+ def main(data: T.Buffer((1,), "float32"), l: T.int32, m: T.int32, n: T.int32):
+ tx = T.launch_thread("threadIdx.x", dshape[0])
+ for i in T.serial(l):
+ for j in T.serial(m):
+ for k in T.serial(n):
+ T.attr(bx_iter, "thread_extent", dshape[1])
+ if tx < 3:
+ data[bx_var * j + tx * j * k] = data[
+ bx_var * j + tx * j * k
+ ] + T.float32(0.3)
+ else:
+ data[bx_var * j + tx * j * k] = data[
+ bx_var * j + tx * j * k
+ ] + T.float32(1.3)
+
+ mod = Module
mod = tvm.tir.transform.Simplify()(mod)
mod = tvm.tir.transform.RemoveNoOp()(mod)
stmt = mod["main"].body
@@ -553,66 +479,25 @@ def test_hoisting_block_scope_2():
assert not tvm.ir.structural_equal(new_stmt, stmt)
-@pytest.mark.xfail(reason="Inconsistent thread_extent", strict=True)
-def test_hoisting_block_scope_3():
- ib = tvm.tir.ir_builder.create()
- dshape = (32, 64)
- dshape_inner = (33, 63)
- data = ib.pointer("float32", name="data")
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
-
- tx = te.thread_axis("threadIdx.x")
- bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", dshape[0])
- ib.scope_attr(bx, "thread_extent", dshape[1])
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- ib.scope_attr(tx, "thread_extent", dshape_inner[0])
- ib.scope_attr(bx, "thread_extent", dshape_inner[1])
- with ib.for_range(0, n, "k") as k:
- with ib.if_scope(tx < 3):
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 0.3
- with ib.else_scope():
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 1.3
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
- tvm.ir.assert_structural_equal(new_stmt, stmt)
-
- with tvm.transform.PassContext(
- config={"tir.HoistIfThenElse": {"support_block_scope_hoisting": True}}
- ):
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
- assert not tvm.ir.structural_equal(new_stmt, stmt)
-
-
def test_hoisting_block_scope_5():
- ib = tvm.tir.ir_builder.create()
- data = ib.pointer("float32", name="data", scope="global")
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
- g = te.var("g")
-
- ib.scope_attr(data, "storage_scope", "global")
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- with ib.for_range(0, n, "k") as k:
- with ib.if_scope(data[g] < 3):
- data[9 * j + 3 * j * k] = data[9 * j + 3 * j * k] + 0.3
- with ib.else_scope():
- data[9 * j + 3 * j * k] = data[9 * j + 3 * j * k] + 1.3
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
+ @I.ir_module
+ class Module:
+ @T.prim_func(private=True)
+ def main(data: T.Buffer((1,), "float32"), l: T.int32, m: T.int32, n: T.int32, g: T.int32):
+ for i in T.serial(l):
+ for j in T.serial(m):
+ for k in T.serial(n):
+ if data[g] < T.float32(3):
+ data[9 * j + 3 * j * k] = data[9 * j + 3 * j * k] + T.float32(0.3)
+ else:
+ data[9 * j + 3 * j * k] = data[9 * j + 3 * j * k] + T.float32(1.3)
+
+ stmt = Module["main"].body
+ new_stmt = tvm.tir.transform.HoistIfThenElse()(Module)["main"].body
assert not tvm.ir.structural_equal(new_stmt, stmt)
+ mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], new_stmt))
stmt = new_stmt
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
with tvm.transform.PassContext(
config={"tir.HoistIfThenElse": {"support_block_scope_hoisting": True}}
@@ -622,66 +507,58 @@ def test_hoisting_block_scope_5():
def test_hoisting_block_scope_6():
- ib = tvm.tir.ir_builder.create()
dshape = (32, 64)
- data = ib.pointer("float32", name="data")
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
-
- tx = te.thread_axis("threadIdx.x")
- bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", dshape[0])
- ib.scope_attr(bx, "thread_extent", dshape[1])
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- with ib.for_range(0, n, "k") as k:
- with ib.if_scope((tx + n) < 3):
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 0.3
- with ib.else_scope():
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 1.3
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
+
+ @I.ir_module
+ class Module:
+ @T.prim_func(private=True)
+ def main(data: T.Buffer((1,), "float32"), l: T.int32, m: T.int32, n: T.int32):
+ tx = T.launch_thread("threadIdx.x", dshape[0])
+ bx = T.launch_thread("blockIdx.x", dshape[1])
+ for i in T.serial(l):
+ for j in T.serial(m):
+ for k in T.serial(n):
+ if tx + n < 3:
+ data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + T.float32(0.3)
+ else:
+ data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + T.float32(1.3)
+
+ stmt = Module["main"].body
+ new_stmt = tvm.tir.transform.HoistIfThenElse()(Module)["main"].body
tvm.ir.assert_structural_equal(new_stmt, stmt)
with tvm.transform.PassContext(
config={"tir.HoistIfThenElse": {"support_block_scope_hoisting": True}}
):
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
+ new_stmt = tvm.tir.transform.HoistIfThenElse()(Module)["main"].body
assert not tvm.ir.structural_equal(new_stmt, stmt)
def test_hoisting_block_scope_7():
- ib = tvm.tir.ir_builder.create()
dshape = (32, 64)
- data = ib.pointer("float32", name="data")
- l = te.var("l")
- m = te.var("m")
- n = te.var("n")
-
- tx = te.thread_axis("threadIdx.x")
- bx = te.thread_axis("blockIdx.x")
- ib.scope_attr(tx, "thread_extent", dshape[0])
- ib.scope_attr(bx, "thread_extent", dshape[1])
- with ib.for_range(0, l, "i") as i:
- with ib.for_range(0, m, "j") as j:
- with ib.for_range(0, n, "k") as k:
- with ib.if_scope((tx + i) < 3):
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 0.3
- with ib.else_scope():
- data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + 1.3
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt))
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
+
+ @I.ir_module
+ class Module:
+ @T.prim_func(private=True)
+ def main(data: T.Buffer((1,), "float32"), l: T.int32, m: T.int32, n: T.int32):
+ tx = T.launch_thread("threadIdx.x", dshape[0])
+ bx = T.launch_thread("blockIdx.x", dshape[1])
+ for i in T.serial(l):
+ for j in T.serial(m):
+ for k in T.serial(n):
+ if tx + i < 3:
+ data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + T.float32(0.3)
+ else:
+ data[bx * j + tx * j * k] = data[bx * j + tx * j * k] + T.float32(1.3)
+
+ stmt = Module["main"].body
+ new_stmt = tvm.tir.transform.HoistIfThenElse()(Module)["main"].body
tvm.ir.assert_structural_equal(new_stmt, stmt)
with tvm.transform.PassContext(
config={"tir.HoistIfThenElse": {"support_block_scope_hoisting": True}}
):
- new_stmt = tvm.tir.transform.HoistIfThenElse()(mod)["main"].body
+ new_stmt = tvm.tir.transform.HoistIfThenElse()(Module)["main"].body
assert not tvm.ir.structural_equal(new_stmt, stmt)
diff --git a/tests/python/tir-transform/test_tir_transform_inject_double_buffer.py b/tests/python/tir-transform/test_tir_transform_inject_double_buffer.py
index a0deacf674..3c90476794 100644
--- a/tests/python/tir-transform/test_tir_transform_inject_double_buffer.py
+++ b/tests/python/tir-transform/test_tir_transform_inject_double_buffer.py
@@ -19,29 +19,29 @@ import tvm
import tvm.testing
from tvm.script import tir as T, ir as I
-from tvm import te
def test_double_buffer():
- dtype = "int64"
n = 100
m = 4
- tx = te.thread_axis("threadIdx.x")
- ib = tvm.tir.ir_builder.create()
- A = ib.pointer("float32", name="A")
- C = ib.pointer("float32", name="C")
- ib.scope_attr(tx, "thread_extent", 1)
- with ib.for_range(0, n) as i:
- B = ib.allocate("float32", m, name="B", scope="shared")
- with ib.new_scope():
- ib.scope_attr(B.asobject().data, "double_buffer_scope", 1)
- with ib.for_range(0, m) as j:
- B[j] = A[i * 4 + j]
- with ib.for_range(0, m) as j:
- C[j] = B[j] + 1
-
- stmt = ib.get()
- mod = tvm.IRModule({"db": tvm.tir.PrimFunc([A.asobject(), C.asobject()], stmt)})
+
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def db(A: T.handle("float32"), C: T.handle("float32")):
+ A_buf = T.decl_buffer((n * m,), "float32", data=A)
+ C_buf = T.decl_buffer((m,), "float32", data=C)
+ tx = T.launch_thread("threadIdx.x", 1)
+ for i in range(n):
+ B_data = T.allocate([m], "float32", scope="shared")
+ B = T.Buffer([m], "float32", data=B_data, scope="shared")
+ with T.attr(B_data, "double_buffer_scope", 1):
+ for j in range(m):
+ B[j] = A_buf[i * 4 + j]
+ for j in range(m):
+ C_buf[j] = B[j] + T.float32(1.0)
+
+ mod = Module
opt = tvm.transform.Sequential(
[tvm.tir.transform.InjectDoubleBuffer(), tvm.tir.transform.Simplify()]
@@ -51,8 +51,17 @@ def test_double_buffer():
mod = opt(mod)
stmt = mod["db"].body
- assert isinstance(stmt.body, tvm.tir.Allocate)
- assert list(stmt.body.extents) == [m * 2]
+ # After transformation, the buffer allocation should be doubled
+ allocate_node = None
+
+ def visitor(op):
+ nonlocal allocate_node
+ if isinstance(op, tvm.tir.Allocate) and "B" in str(op.buffer_var):
+ allocate_node = op
+
+ tvm.tir.stmt_functor.post_order_visit(stmt, visitor)
+ assert allocate_node is not None
+ assert list(allocate_node.extents) == [m * 2]
f = tvm.tir.transform.ThreadSync("shared")(mod)["db"]
count = [0]
diff --git a/tests/python/tir-transform/test_tir_transform_inject_virtual_thread.py b/tests/python/tir-transform/test_tir_transform_inject_virtual_thread.py
index 593f9447d4..a6d1d4b58d 100644
--- a/tests/python/tir-transform/test_tir_transform_inject_virtual_thread.py
+++ b/tests/python/tir-transform/test_tir_transform_inject_virtual_thread.py
@@ -16,124 +16,144 @@
# under the License.
import tvm
import tvm.testing
-from tvm import te
-from tvm.script import tir as T
+from tvm.script import tir as T, ir as I
-vthread_name = tvm.testing.parameter("vthread", "cthread")
-
-def test_vthread(vthread_name):
- dtype = "int64"
+def test_vthread():
+ """Test virtual thread injection with vthread"""
n = 100
m = 4
nthread = 2
- def get_vthread(name):
- tx = te.thread_axis(name)
- ty = te.thread_axis(name)
- ib = tvm.tir.ir_builder.create()
- A = ib.pointer("float32", name="A")
- C = ib.pointer("float32", name="C")
- with ib.for_range(0, n) as i:
- ib.scope_attr(tx, "virtual_thread", nthread)
- ib.scope_attr(ty, "virtual_thread", nthread)
- B = ib.allocate("float32", m, name="B", scope="shared")
- B[i] = A[i * nthread + tx]
- bbuffer = B.asobject()
- ib.emit(
- tvm.tir.call_extern(
- "int32",
- "Run",
- bbuffer.access_ptr("r"),
- tvm.tir.call_intrin("int32", "tir.tvm_context_id"),
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main(A: T.handle("float32"), C: T.handle("float32")):
+ A_buf = T.decl_buffer((n * nthread,), "float32", data=A)
+ C_buf = T.decl_buffer((n * nthread,), "float32", data=C)
+ for i in range(n):
+ vt_x = T.launch_thread("vthread", nthread)
+ vt_y = T.launch_thread("vthread", nthread)
+ B_data = T.allocate([m], "float32", scope="shared")
+ B = T.Buffer([m], "float32", data=B_data, scope="shared")
+ B[i] = A_buf[i * nthread + vt_x]
+ T.evaluate(
+ T.call_extern(
+ "int32",
+ "Run",
+ B.access_ptr("r"),
+ T.call_intrin("int32", "tir.tvm_context_id"),
+ )
)
- )
- C[i * nthread + tx] = B[i] + 1
- return ib.get()
+ C_buf[i * nthread + vt_x] = B[i] + T.float32(1)
+
+ # For vthread, expected allocation is m * nthread
+ B_expected_alloc = m * nthread
- if vthread_name == "vthread":
- B_expected_alloc = m * nthread
- elif vthread_name == "cthread":
- B_expected_alloc = m * nthread * nthread
+ stmt = tvm.tir.transform.InjectVirtualThread()(Module)["main"]
- stmt = tvm.tir.transform.InjectVirtualThread()(
- tvm.IRModule.from_expr(tvm.tir.PrimFunc([], get_vthread(vthread_name)))
- )["main"]
+ # Find allocate nodes
+ allocates = []
- assert list(stmt.body.body.extents) == [B_expected_alloc]
+ def find_allocates(node):
+ if isinstance(node, tvm.tir.Allocate):
+ allocates.append(node)
+ tvm.tir.stmt_functor.post_order_visit(stmt.body, find_allocates)
+ assert len(allocates) == 1
+ assert list(allocates[0].extents) == [B_expected_alloc]
-def test_vthread_extern(vthread_name):
- dtype = "int64"
+
+def test_vthread_extern():
+ """Test virtual thread injection with extern call"""
n = 100
m = 4
nthread = 2
- def get_vthread(name):
- tx = te.thread_axis(name)
- ty = te.thread_axis(name)
- ib = tvm.tir.ir_builder.create()
- with ib.for_range(0, n) as i:
- ib.scope_attr(tx, "virtual_thread", nthread)
- ib.scope_attr(ty, "virtual_thread", nthread)
- A = ib.allocate("float32", m, name="A", scope="shared")
- B = ib.allocate("float32", m, name="B", scope="shared")
- C = ib.allocate("float32", m, name="C", scope="shared")
- abuffer = A.asobject()
- bbuffer = B.asobject()
- cbuffer = C.asobject()
- A[tx] = tx + 1.0
- B[ty] = ty + 1.0
- ib.emit(
- tvm.tir.call_extern(
- "int32",
- "Run",
- abuffer.access_ptr("r"),
- bbuffer.access_ptr("r"),
- cbuffer.access_ptr("rw"),
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main():
+ T.func_attr({"global_symbol": "main"})
+ for i in range(n):
+ vt_x = T.launch_thread("vthread", nthread)
+ vt_y = T.launch_thread("vthread", nthread)
+ A_data = T.allocate([m], "float32", scope="shared")
+ A = T.Buffer([m], "float32", data=A_data, scope="shared")
+ B_data = T.allocate([m], "float32", scope="shared")
+ B = T.Buffer([m], "float32", data=B_data, scope="shared")
+ C_data = T.allocate([m], "float32", scope="shared")
+ C = T.Buffer([m], "float32", data=C_data, scope="shared")
+ A[vt_x] = T.Cast("float32", vt_x) + T.float32(1)
+ B[vt_y] = T.Cast("float32", vt_y) + T.float32(1)
+ T.evaluate(
+ T.call_extern(
+ "int32",
+ "Run",
+ A.access_ptr("r"),
+ B.access_ptr("r"),
+ C.access_ptr("rw"),
+ )
)
- )
- return ib.get()
-
- if vthread_name == "vthread":
- A_expected_alloc = m * nthread
- elif vthread_name == "cthread":
- A_expected_alloc = m * nthread * nthread
+ # For vthread:
+ # A, B expected allocation is m * nthread (used with single vthread each)
+ A_expected_alloc = m * nthread
+ # C expected allocation is m * nthread * nthread (used in extern with both vthreads)
C_expected_alloc = m * nthread * nthread
- stmt = tvm.tir.transform.InjectVirtualThread()(
- tvm.IRModule.from_expr(
- tvm.tir.PrimFunc([], get_vthread(vthread_name)).with_attr("global_symbol", "main")
- )
- )["main"]
+ stmt = tvm.tir.transform.InjectVirtualThread()(Module)["main"]
- assert list(stmt.body.body.extents) == [A_expected_alloc]
- assert list(stmt.body.body.body.body.extents) == [C_expected_alloc]
+ # Find allocate nodes
+ allocates = []
+
+ def find_allocates(node):
+ if isinstance(node, tvm.tir.Allocate):
+ allocates.append(node)
+
+ tvm.tir.stmt_functor.post_order_visit(stmt.body, find_allocates)
+ assert len(allocates) == 3
+ # Check that we have the expected extents (order may vary)
+ extents = sorted([int(a.extents[0]) for a in allocates])
+ assert extents == sorted([A_expected_alloc, A_expected_alloc, C_expected_alloc])
def test_vthread_if_then_else():
+ """Test virtual thread injection with if-then-else"""
nthread = 2
- tx = te.thread_axis("vthread")
- ib = tvm.tir.ir_builder.create()
- A = ib.pointer("float32", name="A")
- with ib.for_range(0, 100) as i:
- ib.scope_attr(tx, "virtual_thread", nthread)
- B = ib.allocate("float32", 128, name="B", scope="shared")
- with ib.if_scope(i == 0):
- B[i] = A[i * nthread + tx]
- with ib.else_scope():
- B[i] = A[i * nthread + tx] + 1
- with ib.if_scope(i == 0):
- B[i] = A[i * nthread + tx] + 2
- stmt = ib.get()
-
- stmt = tvm.tir.transform.InjectVirtualThread()(
- tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt).with_attr("global_symbol", "main"))
- )["main"]
-
- assert stmt.body.body.body[0].else_case != None
- assert stmt.body.body.body[1].else_case == None
+
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main(A: T.handle("float32")):
+ T.func_attr({"global_symbol": "main"})
+ A_buf = T.decl_buffer((100 * nthread,), "float32", data=A)
+ for i in range(100):
+ vt = T.launch_thread("vthread", nthread)
+ B_data = T.allocate([128], "float32", scope="shared")
+ B = T.Buffer([128], "float32", data=B_data, scope="shared")
+ if i == 0:
+ B[i] = A_buf[i * nthread + vt]
+ else:
+ B[i] = A_buf[i * nthread + vt] + T.float32(1)
+ if i == 0:
+ B[i] = A_buf[i * nthread + vt] + T.float32(2)
+
+ stmt = tvm.tir.transform.InjectVirtualThread()(Module)["main"]
+
+ # Find IfThenElse nodes
+ if_nodes = []
+
+ def find_ifs(node):
+ if isinstance(node, tvm.tir.IfThenElse):
+ if_nodes.append(node)
+
+ tvm.tir.stmt_functor.post_order_visit(stmt.body, find_ifs)
+
+ assert len(if_nodes) == 2
+ # First if has else_case, second does not
+ assert if_nodes[0].else_case is not None
+ assert if_nodes[1].else_case is None
def test_vthread_simplified():
diff --git a/tests/python/tir-transform/test_tir_transform_loop_partition.py b/tests/python/tir-transform/test_tir_transform_loop_partition.py
index 1e079ada55..63bbf457e5 100644
--- a/tests/python/tir-transform/test_tir_transform_loop_partition.py
+++ b/tests/python/tir-transform/test_tir_transform_loop_partition.py
@@ -30,19 +30,17 @@ def collect_visit(stmt, f):
def test_multi_loop():
- ib = tvm.tir.ir_builder.create()
- m = te.size_var("m")
- n = te.size_var("n")
- with ib.for_range(0, 4, "i") as i:
- with ib.for_range(0, n, "j") as j:
- with ib.for_range(0, m, "k") as k:
- with ib.if_scope(ib.likely(i * m + j + k < n)):
- ib.emit(tvm.tir.Evaluate(m))
- with ib.else_scope():
- ib.emit(tvm.tir.Evaluate(n))
- stmt = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([n, m], stmt).with_attr("global_symbol", "main"))
+ @T.prim_func
+ def func(n: T.int64, m: T.int64):
+ for i in range(4):
+ for j in T.serial(n):
+ for k in T.serial(m):
+ if T.likely(i * m + j + k < n):
+ T.evaluate(m)
+ else:
+ T.evaluate(n)
+
+ mod = tvm.IRModule.from_expr(func.with_attr("global_symbol", "main"))
mod = tvm.tir.transform.LoopPartition()(mod)
stmt = tvm.tir.transform.Simplify()(mod)["main"].body
@@ -50,23 +48,21 @@ def test_multi_loop():
def test_multi_if():
- ib = tvm.tir.ir_builder.create()
- m = te.size_var("m")
- n = te.size_var("n")
- with ib.for_range(0, 4, "i") as i:
- with ib.for_range(0, n, "j") as j:
- with ib.for_range(0, m, "k") as k:
- with ib.if_scope(ib.likely(i * m + j + k < n)):
- ib.emit(tvm.tir.Evaluate(m))
- with ib.else_scope():
- ib.emit(tvm.tir.Evaluate(n))
- with ib.if_scope(ib.likely(i * m + j - k < n)):
- ib.emit(tvm.tir.Evaluate(m))
- with ib.else_scope():
- ib.emit(tvm.tir.Evaluate(n))
- stmt = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt).with_attr("global_symbol", "main"))
+ @T.prim_func
+ def func(n: T.int64, m: T.int64):
+ for i in range(4):
+ for j in T.serial(n):
+ for k in T.serial(m):
+ if T.likely(i * m + j + k < n):
+ T.evaluate(m)
+ else:
+ T.evaluate(n)
+ if T.likely(i * m + j - k < n):
+ T.evaluate(m)
+ else:
+ T.evaluate(n)
+
+ mod = tvm.IRModule.from_expr(func.with_attr("global_symbol", "main"))
mod = tvm.tir.transform.LoopPartition()(mod)
stmt = tvm.tir.transform.Simplify()(mod)["main"].body
@@ -74,15 +70,13 @@ def test_multi_if():
def test_condition():
- ib = tvm.tir.ir_builder.create()
- m = te.size_var("m")
- n = te.size_var("n")
- with ib.for_range(0, tvm.tir.truncdiv(n + 3, 4), "i") as i:
- with ib.for_range(0, 4, "j") as j:
- ib.emit(tvm.tir.Evaluate(tvm.tir.Select(ib.likely(i * 4 + j < n), m, n)))
- stmt = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([m, n], stmt).with_attr("global_symbol", "main"))
+ @T.prim_func
+ def func(m: T.int64, n: T.int64):
+ for i in T.serial(T.truncdiv(n + 3, 4)):
+ for j in range(4):
+ T.evaluate(T.Select(T.likely(i * 4 + j < n), m, n))
+
+ mod = tvm.IRModule.from_expr(func.with_attr("global_symbol", "main"))
mod = tvm.tir.transform.LoopPartition()(mod)
stmt = tvm.tir.transform.Simplify()(mod)["main"].body
@@ -90,14 +84,12 @@ def test_condition():
def test_condition_EQ():
- ib = tvm.tir.ir_builder.create()
- m = te.size_var("m")
- n = te.size_var("n")
- with ib.for_range(0, 10, "i") as i:
- ib.emit(tvm.tir.Evaluate(tvm.tir.Select(ib.likely(tvm.tir.EQ(i, 5)), m, n)))
- stmt = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([m, n], stmt).with_attr("global_symbol", "main"))
+ @T.prim_func
+ def func(m: T.int64, n: T.int64):
+ for i in range(10):
+ T.evaluate(T.Select(T.likely(i == 5), m, n))
+
+ mod = tvm.IRModule.from_expr(func.with_attr("global_symbol", "main"))
with tvm.transform.PassContext(config={"tir.LoopPartition": {"partition_const_loop": True}}):
mod = tvm.tir.transform.LoopPartition()(mod)
stmt = tvm.tir.transform.Simplify()(mod)["main"].body
@@ -106,17 +98,15 @@ def test_condition_EQ():
def test_everything_during_deduction():
- m = te.size_var("m")
- n = te.size_var("n")
- ib = tvm.tir.ir_builder.create()
- with ib.for_range(0, n, "i") as i:
- with ib.for_range(0, 32, "j") as j:
- with ib.if_scope(ib.likely(tvm.tir.truncdiv(i, j) < m)):
- # this guard will produce everything during deduction
- ib.emit(tvm.tir.Evaluate(m))
- stmt = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([m, n], stmt).with_attr("global_symbol", "main"))
+ @T.prim_func
+ def func(m: T.int64, n: T.int64):
+ for i in T.serial(n):
+ for j in range(32):
+ if T.likely(T.truncdiv(i, j) < m):
+ # this guard will produce everything during deduction
+ T.evaluate(m)
+
+ mod = tvm.IRModule.from_expr(func.with_attr("global_symbol", "main"))
mod = tvm.tir.transform.LoopPartition()(mod)
stmt = tvm.tir.transform.Simplify()(mod)["main"].body
@@ -124,32 +114,27 @@ def test_everything_during_deduction():
def test_oneD_pool():
- m = te.size_var("m")
- ib = tvm.tir.ir_builder.create()
- # data = te.placeholder((16,), name = 'data')
- data = ib.pointer("float32", name="A")
- out = ib.pointer("float32", name="A")
- with ib.for_range(0, 16, "ow") as ow:
- with ib.for_range(0, 3, "kw") as kw:
- with ib.if_scope(ib.likely(ow > 0)):
- with ib.if_scope(ib.likely(ow < 15)):
- out[ow] = tvm.te.max(out[ow], data[ow + kw - 1])
- with ib.for_range(0, 16, "ow") as ow:
- with ib.for_range(0, 3, "kw") as kw:
- with ib.if_scope(ib.likely(ow < 1)):
- with ib.if_scope(ib.likely(kw > 0)):
- out[ow] = tvm.te.max(out[ow], data[ow + kw - 1])
- with ib.for_range(0, 16, "ow") as ow:
- with ib.for_range(0, 3, "kw") as kw:
- with ib.if_scope(ib.likely(ow > 14)):
- with ib.if_scope(ib.likely(kw < 2)):
- out[ow] = tvm.te.max(out[ow], data[ow + kw - 1])
-
- stmt = ib.get()
-
- mod = tvm.IRModule.from_expr(
- tvm.tir.PrimFunc([m, data, out], stmt).with_attr("global_symbol", "main")
- )
+ @T.prim_func
+ def func(m: T.int64, data: T.handle("float32"), out: T.handle("float32")):
+ data_ptr = T.Buffer((16,), "float32", data=data)
+ out_ptr = T.Buffer((16,), "float32", data=out)
+ for ow in range(16):
+ for kw in range(3):
+ if T.likely(ow > 0):
+ if T.likely(ow < 15):
+ out_ptr[ow] = T.max(out_ptr[ow], data_ptr[ow + kw - 1])
+ for ow in range(16):
+ for kw in range(3):
+ if T.likely(ow < 1):
+ if T.likely(kw > 0):
+ out_ptr[ow] = T.max(out_ptr[ow], data_ptr[ow + kw - 1])
+ for ow in range(16):
+ for kw in range(3):
+ if T.likely(ow > 14):
+ if T.likely(kw < 2):
+ out_ptr[ow] = T.max(out_ptr[ow], data_ptr[ow + kw - 1])
+
+ mod = tvm.IRModule.from_expr(func.with_attr("global_symbol", "main"))
with tvm.transform.PassContext(config={"tir.LoopPartition": {"partition_const_loop": True}}):
mod = tvm.tir.transform.LoopPartition()(mod)
@@ -159,28 +144,19 @@ def test_oneD_pool():
def test_cce_loop_1():
- ib = tvm.tir.ir_builder.create()
- dtype = "float16"
n = 514
m = 514
- _A = te.placeholder((n * m,), name="A")
- Ab = tvm.tir.decl_buffer((n * m,), dtype, name="A")
- A = ib.buffer_ptr(Ab)
- _B = te.placeholder((n * m,), name="B")
- Bb = tvm.tir.decl_buffer((n * m,), dtype, name="B")
- B = ib.buffer_ptr(Bb)
- # for i in 0 to n-1:
- with ib.for_range(0, 11, name="i") as i:
- with ib.for_range(0, 160, name="j") as j:
- with ib.if_scope(ib.likely(((i * 160) + j) < 1600)):
- A[(i + 1) * m + j + 1] = (
- B[(i) * m + j + 1] + B[(i + 1) * m + j + 1] + B[(i + 2) * m + j + 1]
- )
- stmt = ib.get()
-
- mod = tvm.IRModule.from_expr(
- tvm.tir.PrimFunc([Ab, Bb], stmt).with_attr("global_symbol", "main")
- )
+
+ @T.prim_func
+ def func(A: T.Buffer((n * m,), "float16"), B: T.Buffer((n * m,), "float16")):
+ for i in range(11):
+ for j in range(160):
+ if T.likely(i * 160 + j < 1600):
+ A[(i + 1) * m + j + 1] = (
+ B[i * m + j + 1] + B[(i + 1) * m + j + 1] + B[(i + 2) * m + j + 1]
+ )
+
+ mod = tvm.IRModule.from_expr(func.with_attr("global_symbol", "main"))
with tvm.transform.PassContext(config={"tir.LoopPartition": {"partition_const_loop": True}}):
mod = tvm.tir.transform.LoopPartition()(mod)
stmt = tvm.tir.transform.Simplify()(mod)["main"].body
@@ -189,22 +165,19 @@ def test_cce_loop_1():
def test_cce_loop_2():
- ib = tvm.tir.ir_builder.create()
- len = 112
+ length = 112
tile = 32
- loop = (len + tile - 1) // tile
- with ib.for_range(0, loop, "i") as i:
- head = i * tile
- with ib.if_scope(ib.likely(head + tile > len)):
- tail = len
- ib.emit(tvm.tir.call_extern("float32", "cce_intrisic", head, tail))
- with ib.else_scope():
- tail = head + tile
- ib.emit(tvm.tir.call_extern("float32", "cce_intrisic", head, tail))
-
- stmt = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt).with_attr("global_symbol", "main"))
+ loop = (length + tile - 1) // tile
+
+ @T.prim_func(private=True)
+ def func():
+ for i in range(loop):
+ if T.likely(i * tile + tile > length):
+ T.evaluate(T.call_extern("float32", "cce_intrisic", i * tile, length))
+ else:
+ T.evaluate(T.call_extern("float32", "cce_intrisic", i * tile, i * tile + tile))
+
+ mod = tvm.IRModule.from_expr(func.with_attr("global_symbol", "main"))
with tvm.transform.PassContext(config={"tir.LoopPartition": {"partition_const_loop": True}}):
mod = tvm.tir.transform.LoopPartition()(mod)
stmt = tvm.tir.transform.Simplify()(mod)["main"].body
@@ -213,19 +186,18 @@ def test_cce_loop_2():
def test_cce_loop_3():
- ib = tvm.tir.ir_builder.create()
loop1 = 4
loop2 = 9998
tile = 39991
- with ib.for_range(0, loop2, "i") as i:
- with ib.for_range(0, loop1, "j") as j:
- head1 = i
- head2 = j
- with ib.if_scope(ib.likely(head1 * loop1 + head2 < tile)):
- ib.emit(tvm.tir.call_extern("float16", "cce_intrisic", head1))
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], stmt).with_attr("global_symbol", "main"))
+
+ @T.prim_func(private=True)
+ def func():
+ for i in range(loop2):
+ for j in range(loop1):
+ if T.likely(i * loop1 + j < tile):
+ T.evaluate(T.call_extern("float16", "cce_intrisic", i))
+
+ mod = tvm.IRModule.from_expr(func.with_attr("global_symbol", "main"))
with tvm.transform.PassContext(config={"tir.LoopPartition": {"partition_const_loop": True}}):
mod = tvm.tir.transform.LoopPartition()(mod)
diff --git a/tests/python/tir-transform/test_tir_transform_lower_tvm_builtin.py b/tests/python/tir-transform/test_tir_transform_lower_tvm_builtin.py
index c65e9011ac..6050b07140 100644
--- a/tests/python/tir-transform/test_tir_transform_lower_tvm_builtin.py
+++ b/tests/python/tir-transform/test_tir_transform_lower_tvm_builtin.py
@@ -128,17 +128,21 @@ def test_call_packed_return_non_i32():
def build_tir():
Ab = tvm.tir.decl_buffer((2,), "float32")
- ib = tvm.tir.ir_builder.create()
- Aptr = ib.buffer_ptr(Ab)
- # return f32
- # Aptr[0] = testing.echo(expected_value[0])
- Aptr[0] = packed_echo(tvm.tir.const(expected_value[0], "float32"))
- # return handle
- # let Aptr_var = testing.echo(Aptr) in Aptr_var[1] = expected_value[1]
- Aptr_var = ib.let("Aptr_dup", packed_echo(Aptr.asobject().data))
- ib.emit(tvm.tir.BufferStore(Aptr, tvm.tir.const(expected_value[1], "float32"), [1]))
-
- stmt = ib.get()
+
+ # Build statements using direct TIR construction (no ir_builder)
+ # 1. Store packed_echo(const) result into Ab[0]
+ store0 = tvm.tir.BufferStore(
+ Ab, packed_echo(tvm.tir.const(expected_value[0], "float32")), [0]
+ )
+
+ # 2. Let binding: Aptr_dup = packed_echo(Ab.data), then store const into Ab[1]
+ Aptr_dup = tvm.tir.Var("Aptr_dup", "handle")
+ store1 = tvm.tir.BufferStore(Ab, tvm.tir.const(expected_value[1], "float32"), [1])
+ let_stmt = tvm.tir.LetStmt(Aptr_dup, packed_echo(Ab.data), store1)
+
+ # Combine into sequence
+ stmt = tvm.tir.SeqStmt([store0, let_stmt])
+
return tvm.IRModule.from_expr(
tvm.tir.PrimFunc([Ab], stmt).with_attr("global_symbol", "packed_test")
)
diff --git a/tests/python/tir-transform/test_tir_transform_make_unpacked_api.py b/tests/python/tir-transform/test_tir_transform_make_unpacked_api.py
index c17e47daf8..b8420e40e4 100644
--- a/tests/python/tir-transform/test_tir_transform_make_unpacked_api.py
+++ b/tests/python/tir-transform/test_tir_transform_make_unpacked_api.py
@@ -24,10 +24,9 @@ import numpy
@pytest.fixture
def mod_without_attrs():
- ib = tvm.tir.ir_builder.create()
+ # Use direct TIR construction to preserve exact parameter names
A = tvm.tir.decl_buffer(name="A", shape=[1])
- stmt = ib.get()
- return tvm.IRModule.from_expr(tvm.tir.PrimFunc([A], stmt))
+ return tvm.IRModule.from_expr(tvm.tir.PrimFunc([A], tvm.tir.Evaluate(0)))
@pytest.fixture
@@ -72,10 +71,9 @@ def test_device_setup(mod, target, dev):
def test_no_buffers_no_device_setup():
- ib = tvm.tir.ir_builder.create()
- A = ib.pointer("float32", name="A")
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A], stmt))
+ # Use direct TIR construction - pointer parameter (not buffer)
+ A = tvm.tir.Var("A", "handle")
+ mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A], tvm.tir.Evaluate(0)))
mod = tvm.tir.transform.Apply(lambda f: f.with_attr("target", tvm.target.Target("llvm")))(mod)
mod = tvm.tir.transform.Apply(lambda f: f.with_attr("global_symbol", "main"))(mod)
@@ -91,12 +89,10 @@ def test_argument_mapping(mod):
def test_argument_mapping_multiple():
- ib = tvm.tir.ir_builder.create()
+ # Use direct TIR construction to preserve exact parameter names
A = tvm.tir.decl_buffer(name="A", shape=[1])
B = tvm.tir.decl_buffer(name="B", shape=[1])
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A, B], stmt))
+ mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A, B], tvm.tir.Evaluate(0)))
mod = tvm.tir.transform.Apply(lambda f: f.with_attr("target", tvm.target.Target("llvm")))(mod)
mod = tvm.tir.transform.Apply(lambda f: f.with_attr("global_symbol", "main"))(mod)
@@ -107,11 +103,9 @@ def test_argument_mapping_multiple():
def test_argument_mapping_multiple_matching():
- ib = tvm.tir.ir_builder.create()
+ # Test passing the same buffer twice (duplicate parameter)
A = tvm.tir.decl_buffer(name="A", shape=[1])
- B = tvm.tir.decl_buffer(name="B", shape=[1])
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A, A], stmt))
+ mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A, A], tvm.tir.Evaluate(0)))
mod = tvm.tir.transform.Apply(lambda f: f.with_attr("target", tvm.target.Target("llvm")))(mod)
mod = tvm.tir.transform.Apply(lambda f: f.with_attr("global_symbol", "main"))(mod)
@@ -122,13 +116,12 @@ def test_argument_mapping_multiple_matching():
def test_body():
- ib = tvm.tir.ir_builder.create()
+ # Test with 3 params: A, B, and a buffer ptr derived from A (has same name)
A = tvm.tir.decl_buffer(name="A", shape=[1])
B = tvm.tir.decl_buffer(name="B", shape=[1])
- C = ib.buffer_ptr(A)
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A, B, C], stmt))
+ # Create a buffer pointer variable that points to A's data
+ C = tvm.tir.Var("A", "handle")
+ mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A, B, C], tvm.tir.Evaluate(0)))
mod = tvm.tir.transform.Apply(lambda f: f.with_attr("target", tvm.target.Target("llvm")))(mod)
mod = tvm.tir.transform.Apply(lambda f: f.with_attr("global_symbol", "main"))(mod)
f = tvm.tir.transform.MakeUnpackedAPI()(mod)["main"]
diff --git a/tests/python/tir-transform/test_tir_transform_narrow_datatype.py b/tests/python/tir-transform/test_tir_transform_narrow_datatype.py
index aa1a5d57e4..594f8a0d5b 100644
--- a/tests/python/tir-transform/test_tir_transform_narrow_datatype.py
+++ b/tests/python/tir-transform/test_tir_transform_narrow_datatype.py
@@ -28,80 +28,124 @@ def lower_stmt(params, stmt, target_bits):
return stmt
+def lower_func_body(func, target_bits):
+ """Lower a TVMScript function and return the body (navigating past DeclBuffer)."""
+ mod = tvm.IRModule.from_expr(func)
+ gvar = list(mod.functions.keys())[0]
+ func = tvm.tir.transform.NarrowDataType(target_bits)(mod)[gvar]
+ body = func.body
+ while hasattr(body, "body") and not isinstance(body, tvm.tir.For):
+ body = body.body
+ return body
+
+
def test_basic():
- def check(m, n, target_bits, target_dtype):
- ib = tvm.tir.ir_builder.create()
- Ab = tvm.tir.decl_buffer([m * n], name="A")
- A = ib.buffer_ptr(Ab)
- Bb = tvm.tir.decl_buffer([m * n], name="B")
- B = ib.buffer_ptr(Bb)
- with ib.for_range(0, m, name="i") as i:
- with ib.for_range(0, n, name="j") as j:
- B[i * n + j] = A[i * n + j] + 1
- stmt = ib.get()
- stmt = lower_stmt([Ab, Bb], stmt, target_bits)
+ def check_const(m, n, target_bits, target_dtype):
+ """Check with constant values using TVMScript closure."""
+
+ @T.prim_func
+ def func(A: T.Buffer((m * n,), "float32"), B: T.Buffer((m * n,), "float32")):
+ for i in T.serial(m):
+ for j in T.serial(n):
+ B[i * n + j] = A[i * n + j] + T.float32(1)
+
+ stmt = lower_func_body(func, target_bits)
+ assert stmt.loop_var.dtype == target_dtype
+ assert stmt.body.loop_var.dtype == target_dtype
+
+ def check_symbolic(m_dtype, n_dtype, target_bits, target_dtype):
+ """Check with symbolic shapes as function parameters."""
+ if m_dtype == "int32":
+
+ @T.prim_func
+ def func(A: T.handle("float32"), B: T.handle("float32"), m: T.int32, n: T.int32):
+ A_buf = T.decl_buffer((m * n,), "float32", data=A)
+ B_buf = T.decl_buffer((m * n,), "float32", data=B)
+ for i in T.serial(m):
+ for j in T.serial(n):
+ B_buf[i * n + j] = A_buf[i * n + j] + T.float32(1)
+
+ else:
+
+ @T.prim_func
+ def func(A: T.handle("float32"), B: T.handle("float32"), m: T.int64, n: T.int64):
+ A_buf = T.decl_buffer((m * n,), "float32", data=A)
+ B_buf = T.decl_buffer((m * n,), "float32", data=B)
+ for i in T.serial(m):
+ for j in T.serial(n):
+ B_buf[i * n + j] = A_buf[i * n + j] + T.float32(1)
+
+ stmt = lower_func_body(func, target_bits)
assert stmt.loop_var.dtype == target_dtype
assert stmt.body.loop_var.dtype == target_dtype
# const shape
# i32 -> i32
- check(2, 2, 32, "int32")
+ check_const(2, 2, 32, "int32")
# i64 -> i32
- check(const(2, dtype="int64"), const(2, dtype="int64"), 32, "int32")
- check(const(2**16, dtype="int64"), const(2**16, dtype="int64"), 32, "int64")
+ check_const(const(2, dtype="int64"), const(2, dtype="int64"), 32, "int32")
+ check_const(const(2**16, dtype="int64"), const(2**16, dtype="int64"), 32, "int64")
# i32 -> i16
- check(2, 2, 16, "int16")
- check(2**10, 2**10, 16, "int32")
+ check_const(2, 2, 16, "int16")
+ check_const(2**10, 2**10, 16, "int32")
# symbolic shape
- check(te.size_var(name="m", dtype="int32"), te.size_var(name="n", dtype="int32"), 32, "int32")
- check(te.size_var(name="m", dtype="int64"), te.size_var(name="n", dtype="int64"), 32, "int64")
+ check_symbolic("int32", "int32", 32, "int32")
+ check_symbolic("int64", "int64", 32, "int64")
def test_thread_axis():
- def check(m, n, target_bits, target_dtype):
- ib = tvm.tir.ir_builder.create()
- Ab = tvm.tir.decl_buffer([m * n], name="A")
- A = ib.buffer_ptr(Ab)
- Bb = tvm.tir.decl_buffer([m * n], name="B")
- B = ib.buffer_ptr(Bb)
- bx = te.thread_axis("blockIdx.x")
- tx = te.thread_axis("threadIdx.x")
- ib.scope_attr(bx, "thread_extent", m)
- ib.scope_attr(tx, "thread_extent", n)
- B[bx * n + tx] = A[bx * n + tx] + 1
- stmt = ib.get()
- stmt = lower_stmt([Ab, Bb], stmt, target_bits)
+ # This test uses launch_thread to create AttrStmt nodes with "thread_extent"
+ # and checks the dtype of thread axis variables after narrowing.
+ def check_const(m, n, target_bits, target_dtype):
+ @T.prim_func
+ def func(A: T.Buffer((m * n,), "float32"), B: T.Buffer((m * n,), "float32")):
+ bx = T.launch_thread("blockIdx.x", m)
+ tx = T.launch_thread("threadIdx.x", n)
+ B[bx * n + tx] = A[bx * n + tx] + T.float32(1)
+
+ mod = tvm.IRModule.from_expr(func)
+ gvar = list(mod.functions.keys())[0]
+ func_narrowed = tvm.tir.transform.NarrowDataType(target_bits)(mod)[gvar]
+ stmt = func_narrowed.body
assert stmt.node.var.dtype == target_dtype
assert stmt.body.node.var.dtype == target_dtype
# i32 -> i32
- check(2, 32, target_bits=32, target_dtype="int32")
+ check_const(2, 32, target_bits=32, target_dtype="int32")
# i64 -> i32
- check(const(2, dtype="int64"), const(32, dtype="int64"), target_bits=32, target_dtype="int32")
- check(
+ check_const(
+ const(2, dtype="int64"), const(32, dtype="int64"), target_bits=32, target_dtype="int32"
+ )
+ check_const(
const(2**30, dtype="int64"),
const(32, dtype="int64"),
target_bits=32,
target_dtype="int64",
)
# i32 -> i16
- check(2, 32, target_bits=16, target_dtype="int16")
- check(2**14, 32, target_bits=16, target_dtype="int32")
+ check_const(2, 32, target_bits=16, target_dtype="int16")
+ check_const(2**14, 32, target_bits=16, target_dtype="int32")
def test_multilanes():
+ # Test narrowing with vector types. Uses closure capture for m and lanes.
def check(m, lanes, target_bits, target_dtype):
- ib = tvm.tir.ir_builder.create()
- Ab = tvm.tir.decl_buffer((m,), dtype="float32x{}".format(lanes), name="A")
- A = ib.buffer_ptr(Ab)
- Bb = tvm.tir.decl_buffer((m,), dtype="float32x{}".format(lanes), name="B")
- B = ib.buffer_ptr(Bb)
- with ib.for_range(0, m, name="i", dtype=m.dtype) as i:
- B[i] = A[i] + 1
- A[0] = B[1]
- stmt = ib.get()
- stmt = lower_stmt([Ab, Bb], stmt, target_bits)
+ vec_dtype = "float32x{}".format(lanes)
+
+ @T.prim_func
+ def func(
+ A: T.Buffer((m,), vec_dtype),
+ B: T.Buffer((m,), vec_dtype),
+ ):
+ for i in T.serial(m):
+ B[i] = A[i] + T.Broadcast(T.float32(1), lanes)
+ A[0] = B[1]
+
+ mod = tvm.IRModule.from_expr(func)
+ gvar = list(mod.functions.keys())[0]
+ func_narrowed = tvm.tir.transform.NarrowDataType(target_bits)(mod)[gvar]
+ stmt = func_narrowed.body
assert stmt.seq[0].loop_var.dtype == target_dtype
# i32 -> i32
@@ -115,18 +159,19 @@ def test_multilanes():
def test_slice():
+ # Test narrowing with slice indexing where buffer B has different index ranges.
def check(m, n, target_bits, target_dtype):
# The index may overflow in B, while not in A
- ib = tvm.tir.ir_builder.create()
- Ab = tvm.tir.decl_buffer([m * n], name="A")
- A = ib.buffer_ptr(Ab)
- Bb = tvm.tir.decl_buffer([m * n * 2], name="B")
- B = ib.buffer_ptr(Bb)
- with ib.for_range(0, m, name="i") as i:
- with ib.for_range(0, n, name="j") as j:
- A[i * n + j] = B[i * 2 * n + 2 * j] + 1
- stmt = ib.get()
- stmt = lower_stmt([Ab, Bb], stmt, target_bits)
+ @T.prim_func
+ def func(
+ A: T.Buffer((m * n,), "float32"),
+ B: T.Buffer((m * n * 2,), "float32"),
+ ):
+ for i in T.serial(m):
+ for j in T.serial(n):
+ A[i * n + j] = B[i * 2 * n + 2 * j] + T.float32(1)
+
+ stmt = lower_func_body(func, target_bits)
assert stmt.loop_var.dtype == target_dtype
assert stmt.body.loop_var.dtype == target_dtype
diff --git a/tests/python/tir-transform/test_tir_transform_rewrite_unsafe_select.py b/tests/python/tir-transform/test_tir_transform_rewrite_unsafe_select.py
index 1c5f2f4027..eee05315c9 100644
--- a/tests/python/tir-transform/test_tir_transform_rewrite_unsafe_select.py
+++ b/tests/python/tir-transform/test_tir_transform_rewrite_unsafe_select.py
@@ -15,26 +15,54 @@
# specific language governing permissions and limitations
# under the License.
import tvm
-from tvm import te
+from tvm.script import ir as I, tir as T
def test_rewrite_Select():
- ib = tvm.tir.ir_builder.create()
- A = ib.allocate("float32", 100, name="A", scope="global")
- i = te.var("i")
- y = tvm.tir.Select(i > 1, A[i - 1], 1.0)
+ @I.ir_module
+ class ModuleY:
+ @T.prim_func
+ def main(i: T.int32):
+ A_data = T.allocate([100], "float32", "global")
+ A = T.Buffer(100, "float32", data=A_data)
+ T.evaluate(T.Select(i > 1, A[i - 1], T.float32(1.0)))
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([i], tvm.tir.Evaluate(y)))
- yy = tvm.tir.transform.RewriteUnsafeSelect()(mod)["main"].body.value
+ yy = tvm.tir.transform.RewriteUnsafeSelect()(ModuleY)["main"].body.body.value
- z = tvm.tir.Select(tvm.tir.Select(i > 1, A[i - 1], 1.0) > 0.0, A[i], 0.1)
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([i], tvm.tir.Evaluate(z)))
- zz = tvm.tir.transform.RewriteUnsafeSelect()(mod)["main"].body.value
+ @I.ir_module
+ class ModuleZ:
+ @T.prim_func
+ def main(i: T.int32):
+ A_data = T.allocate([100], "float32", "global")
+ A = T.Buffer(100, "float32", data=A_data)
+ T.evaluate(
+ T.Select(
+ T.Select(i > 1, A[i - 1], T.float32(1.0)) > T.float32(0.0), A[i], T.float32(0.1)
+ )
+ )
- a = tvm.tir.Select(tvm.tir.floordiv(i, 4) > 10, y, z)
+ zz = tvm.tir.transform.RewriteUnsafeSelect()(ModuleZ)["main"].body.body.value
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([i], tvm.tir.Evaluate(a)))
- aa = tvm.tir.transform.RewriteUnsafeSelect()(mod)["main"].body.value
+ @I.ir_module
+ class ModuleA:
+ @T.prim_func
+ def main(i: T.int32):
+ A_data = T.allocate([100], "float32", "global")
+ A = T.Buffer(100, "float32", data=A_data)
+ # Inline y and z to avoid Let bindings - outer Select condition is safe (no buffer access)
+ T.evaluate(
+ T.Select(
+ T.floordiv(i, 4) > 10,
+ T.Select(i > 1, A[i - 1], T.float32(1.0)),
+ T.Select(
+ T.Select(i > 1, A[i - 1], T.float32(1.0)) > T.float32(0.0),
+ A[i],
+ T.float32(0.1),
+ ),
+ )
+ )
+
+ aa = tvm.tir.transform.RewriteUnsafeSelect()(ModuleA)["main"].body.body.value
builtin_if_then_else = tvm.ir.Op.get("tir.if_then_else")
assert yy.op.same_as(builtin_if_then_else)
diff --git a/tests/python/tir-transform/test_tir_transform_simplify.py b/tests/python/tir-transform/test_tir_transform_simplify.py
index 7e66359e29..ae1aca9093 100644
--- a/tests/python/tir-transform/test_tir_transform_simplify.py
+++ b/tests/python/tir-transform/test_tir_transform_simplify.py
@@ -16,59 +16,59 @@
# under the License.
import tvm
import tvm.testing
-
-from tvm import te
from tvm.script import tir as T, ir as I
def test_stmt_simplify():
- ib = tvm.tir.ir_builder.create()
- A = ib.pointer("float32", name="A")
- C = ib.pointer("float32", name="C")
- n = te.size_var("n")
- with ib.for_range(0, n, name="i") as i:
- with ib.if_scope(i < 12):
- A[i] = C[i]
-
- body = tvm.tir.LetStmt(n, 10, ib.get())
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A, C, n], body))
+ @T.prim_func(private=True)
+ def func(A: T.handle("float32"), C: T.handle("float32"), n: T.int32):
+ A_ptr = T.Buffer((10,), "float32", data=A)
+ C_ptr = T.Buffer((10,), "float32", data=C)
+ n_val: T.int32 = 10
+ for i in T.serial(n_val):
+ if i < 12:
+ A_ptr[i] = C_ptr[i]
+
+ mod = tvm.IRModule.from_expr(func)
body = tvm.tir.transform.Simplify()(mod)["main"].body
+ # After simplification, LetStmt -> For -> BufferStore (if is eliminated since i < 12 is always true for i in 0..10)
assert isinstance(body.body, tvm.tir.BufferStore)
def test_thread_extent_simplify():
- ib = tvm.tir.ir_builder.create()
- A = ib.pointer("float32", name="A")
- C = ib.pointer("float32", name="C")
- n = te.size_var("n")
- tx = te.thread_axis("threadIdx.x")
- ty = te.thread_axis("threadIdx.y")
- ib.scope_attr(tx, "thread_extent", n)
- ib.scope_attr(tx, "thread_extent", n)
- ib.scope_attr(ty, "thread_extent", 1)
- with ib.if_scope(tx + ty < 12):
- A[tx] = C[tx + ty]
- body = tvm.tir.LetStmt(n, 10, ib.get())
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A, C, n], body))
+ @T.prim_func(private=True)
+ def func(A: T.handle("float32"), C: T.handle("float32"), n: T.int32):
+ A_ptr = T.Buffer((10,), "float32", data=A)
+ C_ptr = T.Buffer((10,), "float32", data=C)
+ n_val: T.int32 = 10
+ for tx in T.thread_binding(n_val, thread="threadIdx.x"):
+ for ty in T.thread_binding(1, thread="threadIdx.y"):
+ if tx + ty < 12:
+ A_ptr[tx] = C_ptr[tx + ty]
+
+ mod = tvm.IRModule.from_expr(func)
body = tvm.tir.transform.Simplify()(mod)["main"].body
- assert isinstance(body.body.body.body, tvm.tir.BufferStore)
+ # After simplification: For(tx) -> For(ty) -> BufferStore
+ # The LetStmt and if are eliminated since tx + ty < 12 is always true for tx in 0..10 and ty = 0
+ assert isinstance(body, tvm.tir.For) # tx loop
+ assert isinstance(body.body, tvm.tir.For) # ty loop
+ assert isinstance(body.body.body, tvm.tir.BufferStore) # The if was eliminated
def test_if_likely():
- ib = tvm.tir.ir_builder.create()
- A = ib.pointer("float32", name="A")
- C = ib.pointer("float32", name="C")
- n = te.size_var("n")
- tx = te.thread_axis("threadIdx.x")
- ty = te.thread_axis("threadIdx.y")
- ib.scope_attr(tx, "thread_extent", 32)
- ib.scope_attr(ty, "thread_extent", 32)
- with ib.if_scope(ib.likely(tx * 32 + ty < n)):
- with ib.if_scope(ib.likely(tx * 32 + ty < n)):
- A[tx] = C[tx * 32 + ty]
- body = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A, C, n], body))
+ @T.prim_func(private=True)
+ def func(A: T.handle("float32"), C: T.handle("float32"), n: T.int32):
+ A_ptr = T.Buffer((32,), "float32", data=A)
+ C_ptr = T.Buffer((1024,), "float32", data=C)
+ for tx in T.thread_binding(32, thread="threadIdx.x"):
+ for ty in T.thread_binding(32, thread="threadIdx.y"):
+ if T.likely(tx * 32 + ty < n):
+ if T.likely(tx * 32 + ty < n):
+ A_ptr[tx] = C_ptr[tx * 32 + ty]
+
+ mod = tvm.IRModule.from_expr(func)
body = tvm.tir.transform.Simplify()(mod)["main"].body
+ # Structure: For(tx) -> For(ty) -> IfThenElse
assert isinstance(body.body.body, tvm.tir.IfThenElse)
assert not isinstance(body.body.body.then_case, tvm.tir.IfThenElse)
diff --git a/tests/python/tir-transform/test_tir_transform_storage_rewrite.py b/tests/python/tir-transform/test_tir_transform_storage_rewrite.py
index bb24c0bb8d..fc070f3e54 100644
--- a/tests/python/tir-transform/test_tir_transform_storage_rewrite.py
+++ b/tests/python/tir-transform/test_tir_transform_storage_rewrite.py
@@ -43,20 +43,20 @@ def test_alloc_seq():
register_mem(scope_tb, max_bits)
- ib = tvm.tir.ir_builder.create()
- n = te.var("n")
- with ib.for_range(0, n, name="i") as i:
- with ib.for_range(0, 10, name="j") as j:
- A = ib.allocate("float32", 200, name="A", scope=scope_tb)
- A[j] = 1.2
- with ib.for_range(0, 10, name="j") as j:
- A = ib.allocate("float32", 200, name="B", scope=scope_tb)
- A[j] = 1.3
-
- body = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([n], body))
- body = tvm.tir.transform.StorageRewrite()(mod)["main"].body
+ @T.prim_func
+ def func(n: T.int32):
+ for i in T.serial(n):
+ for j in range(10):
+ A_data = T.allocate([200], "float32", scope=scope_tb)
+ A = T.Buffer([200], "float32", data=A_data, scope=scope_tb)
+ A[j] = T.float32(1.2)
+ for j in range(10):
+ B_data = T.allocate([200], "float32", scope=scope_tb)
+ B = T.Buffer([200], "float32", data=B_data, scope=scope_tb)
+ B[j] = T.float32(1.3)
+
+ mod = tvm.IRModule.from_expr(func)
+ body = tvm.tir.transform.StorageRewrite()(mod)["func"].body
num_alloc = [0]
@@ -70,32 +70,41 @@ def test_alloc_seq():
def test_alloc_different_dtypes():
- def stmt_generater(dtype_list, length):
- ib = tvm.tir.ir_builder.create()
- base_dtype = dtype_list[0]
- global_a = te.placeholder((length,), name="global_a", dtype=base_dtype)
+ # Test cross-loop buffer access with buffers allocated in parent scope
+ def make_mod(dtype_list, length):
assert len(dtype_list) == 4
- with ib.for_range(0, length, name="j") as j:
- dtype = dtype_list[0]
- A = ib.allocate(dtype, length, name="A", scope="local.L0A")
- A[j] = tvm.tir.const(1, dtype=dtype)
- with ib.for_range(0, length, name="j") as j:
- dtype = dtype_list[1]
- B = ib.allocate(dtype, length, name="B", scope="local.L0A")
- B[j] = tvm.tir.const(1, dtype=dtype)
- with ib.for_range(0, length, name="j") as j:
- dtype = dtype_list[2]
- C = ib.allocate(dtype, length, name="C", scope="local.L0A")
- C[j] = tvm.tir.const(1, dtype=dtype)
- with ib.for_range(0, length, name="j") as j:
- dtype = dtype_list[3]
- D = ib.allocate(dtype, length, name="D", scope="local.L0A")
- D[j] = tvm.tir.const(1, dtype=dtype)
- with ib.for_range(0, length, name="j") as j:
- dtype = "int8"
- E = ib.allocate(dtype, length, name="E", scope="local.L0A")
- E[j] = A[j].astype(dtype) + B[j].astype(dtype) + C[j].astype(dtype) + D[j].astype(dtype)
- return ib.get()
+
+ @T.prim_func
+ def func():
+ # Allocate all buffers in parent scope (before any loops)
+ A_data = T.allocate([length], dtype_list[0], scope="local.L0A")
+ A = T.Buffer([length], dtype_list[0], data=A_data, scope="local.L0A")
+ B_data = T.allocate([length], dtype_list[1], scope="local.L0A")
+ B = T.Buffer([length], dtype_list[1], data=B_data, scope="local.L0A")
+ C_data = T.allocate([length], dtype_list[2], scope="local.L0A")
+ C = T.Buffer([length], dtype_list[2], data=C_data, scope="local.L0A")
+ D_data = T.allocate([length], dtype_list[3], scope="local.L0A")
+ D = T.Buffer([length], dtype_list[3], data=D_data, scope="local.L0A")
+ E_data = T.allocate([length], "int8", scope="local.L0A")
+ E = T.Buffer([length], "int8", data=E_data, scope="local.L0A")
+
+ for j in range(length):
+ A[j] = T.Cast(dtype_list[0], 1)
+ for j in range(length):
+ B[j] = T.Cast(dtype_list[1], 1)
+ for j in range(length):
+ C[j] = T.Cast(dtype_list[2], 1)
+ for j in range(length):
+ D[j] = T.Cast(dtype_list[3], 1)
+ for j in range(length):
+ E[j] = (
+ T.Cast("int8", A[j])
+ + T.Cast("int8", B[j])
+ + T.Cast("int8", C[j])
+ + T.Cast("int8", D[j])
+ )
+
+ return tvm.IRModule.from_expr(func)
def dtype_bit_len(dtype):
index = 0
@@ -115,12 +124,10 @@ def test_alloc_different_dtypes():
if isinstance(n, tvm.tir.Allocate):
assert n.extents[0].value == offset
- body = stmt_generater(dtype_list, length)
+ mod = make_mod(dtype_list, length)
offset = offset_generater(dtype_list, length)
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([], body))
- body = tvm.tir.transform.StorageRewrite()(mod)["main"].body
-
+ body = tvm.tir.transform.StorageRewrite()(mod)["func"].body
tvm.tir.stmt_functor.post_order_visit(body, verify)
length = 1024
@@ -191,53 +198,61 @@ def test_address_of():
def test_parallel_alloc():
- ib = tvm.tir.ir_builder.create()
- n = te.var("n")
- with ib.for_range(0, n, name="i", kind="parallel") as i:
- with ib.for_range(0, 10, name="j") as j:
- A = ib.allocate("float32", n, name="A", scope="global")
- A[j] = A[j] + 2
+ @T.prim_func
+ def func1(n: T.int32):
+ for i in T.parallel(n):
+ for j in range(10):
+ A_data = T.allocate([n], "float32", scope="global")
+ A = T.Buffer([n], "float32", data=A_data, scope="global")
+ A[j] = A[j] + T.float32(2)
- body = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([n], body))
- body = tvm.tir.transform.StorageRewrite()(mod)["main"]
+ mod = tvm.IRModule.from_expr(func1)
+ body = tvm.tir.transform.StorageRewrite()(mod)["func1"]
assert isinstance(body.body.body, tvm.tir.Allocate)
- ib = tvm.tir.ir_builder.create()
- n = te.var("n")
- with ib.for_range(0, n, name="t") as i:
- ib.scope_attr(
- tvm.tir.const(1, "int32"), "pragma_scope", tvm.tir.StringImm("parallel_launch_point")
- )
- with ib.for_range(0, n, name="i", kind="parallel") as i:
- with ib.for_range(0, 10, name="j") as j:
- A = ib.allocate("float32", n, name="A", scope="global")
- A[j] = A[j] + 2
- body = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([n], body))
- body = tvm.tir.transform.StorageRewrite()(mod)["main"]
+ @T.prim_func
+ def func2(n: T.int32):
+ for t in T.serial(n):
+ with T.attr(T.int32(1), "pragma_scope", "parallel_launch_point"):
+ for i in T.parallel(n):
+ for j in range(10):
+ A_data = T.allocate([n], "float32", scope="global")
+ A = T.Buffer([n], "float32", data=A_data, scope="global")
+ A[j] = A[j] + T.float32(2)
+
+ mod = tvm.IRModule.from_expr(func2)
+ body = tvm.tir.transform.StorageRewrite()(mod)["func2"]
assert isinstance(body.body.body.body.body, tvm.tir.Allocate)
def test_while_alloc():
- def get_mod(kind="serial"):
- ib = tvm.tir.ir_builder.create()
- n = te.var("n")
- with ib.for_range(0, n, name="i", kind=kind) as i:
- j = ib.allocate("int32", 1, name="j", scope="global")
+ @T.prim_func
+ def func_parallel(n: T.int32):
+ for i in T.parallel(n):
+ j_data = T.allocate([1], "int32", scope="global")
+ j = T.Buffer([1], "int32", data=j_data, scope="global")
j[0] = 0
- with ib.while_loop(j[0] < 10):
- A = ib.allocate("float32", n, name="A", scope="global")
- A[j[0]] = A[j[0]] + 2
- j[0] += j[0] + 1
+ while j[0] < 10:
+ A_data = T.allocate([n], "float32", scope="global")
+ A = T.Buffer([n], "float32", data=A_data, scope="global")
+ A[j[0]] = A[j[0]] + T.float32(2)
+ j[0] = j[0] + j[0] + 1
- body = ib.get()
- return tvm.IRModule.from_expr(tvm.tir.PrimFunc([n], body))
+ @T.prim_func
+ def func_serial(n: T.int32):
+ for i in T.serial(n):
+ j_data = T.allocate([1], "int32", scope="global")
+ j = T.Buffer([1], "int32", data=j_data, scope="global")
+ j[0] = 0
+ while j[0] < 10:
+ A_data = T.allocate([n], "float32", scope="global")
+ A = T.Buffer([n], "float32", data=A_data, scope="global")
+ A[j[0]] = A[j[0]] + T.float32(2)
+ j[0] = j[0] + j[0] + 1
- mod = get_mod(kind="parallel")
+ mod = tvm.IRModule.from_expr(func_parallel)
# parallel (i, 0, n) {
# allocate j[int32 * 1]
# j[0] = 0
@@ -248,7 +263,7 @@ def test_while_alloc():
# j[0] = (j[0] + (j[0] + 1))
# }
# }
- body = tvm.tir.transform.StorageRewrite()(mod)["main"]
+ body = tvm.tir.transform.StorageRewrite()(mod)["func_parallel"]
# parallel (i, 0, n) {
# allocate j[int32 * 1]
# allocate A[float32 * n]
@@ -261,7 +276,7 @@ def test_while_alloc():
assert isinstance(body.body.body, tvm.tir.Allocate) # j
assert isinstance(body.body.body.body, tvm.tir.Allocate) # A
- mod = get_mod(kind="serial")
+ mod = tvm.IRModule.from_expr(func_serial)
# for (i, 0, n) {
# allocate j[int32 * 1]
# j[0] = 0
@@ -272,7 +287,7 @@ def test_while_alloc():
# j[0] = (j[0] + (j[0] + 1))
# }
# }
- body = tvm.tir.transform.StorageRewrite()(mod)["main"]
+ body = tvm.tir.transform.StorageRewrite()(mod)["func_serial"]
# allocate j[int32 * 1]
# allocate A[float32 * n]
# for (i, 0, n) {
@@ -287,27 +302,31 @@ def test_while_alloc():
def test_alloc_seq_type():
- ib = tvm.tir.ir_builder.create()
- n = te.var("n")
- with ib.for_range(0, n, name="i") as i:
- with ib.for_range(0, 10, name="j") as j:
- A = ib.allocate("float32", 200, name="A", scope="local.L0A")
- A1 = ib.allocate("float32", 200, name="A1", scope="local.L0A")
- A[j] = 1.2
- A1[j] = 1.3
- B = ib.allocate("int16", 200, name="B", scope="local.L0A")
- B[j] = tvm.tir.const(1, "int16")
- C = ib.allocate("int16", 200, name="C", scope="local.L0A")
- C[j] = tvm.tir.const(1, "int16")
- D = ib.allocate("int16", 200, name="D", scope="local.L0A")
- D[j] = B[j] + C[j]
- A2 = ib.allocate("float32", 200, name="A2", scope="local.L0A")
- A2[j] = A[j]
-
- body = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([n], body))
- body = tvm.tir.transform.StorageRewrite()(mod)["main"].body
+ @T.prim_func
+ def func(n: T.int32):
+ for i in T.serial(n):
+ for j in range(10):
+ A_data = T.allocate([200], "float32", scope="local.L0A")
+ A = T.Buffer([200], "float32", data=A_data, scope="local.L0A")
+ A1_data = T.allocate([200], "float32", scope="local.L0A")
+ A1 = T.Buffer([200], "float32", data=A1_data, scope="local.L0A")
+ A[j] = T.float32(1.2)
+ A1[j] = T.float32(1.3)
+ B_data = T.allocate([200], "int16", scope="local.L0A")
+ B = T.Buffer([200], "int16", data=B_data, scope="local.L0A")
+ B[j] = T.int16(1)
+ C_data = T.allocate([200], "int16", scope="local.L0A")
+ C = T.Buffer([200], "int16", data=C_data, scope="local.L0A")
+ C[j] = T.int16(1)
+ D_data = T.allocate([200], "int16", scope="local.L0A")
+ D = T.Buffer([200], "int16", data=D_data, scope="local.L0A")
+ D[j] = B[j] + C[j]
+ A2_data = T.allocate([200], "float32", scope="local.L0A")
+ A2 = T.Buffer([200], "float32", data=A2_data, scope="local.L0A")
+ A2[j] = A[j]
+
+ mod = tvm.IRModule.from_expr(func)
+ body = tvm.tir.transform.StorageRewrite()(mod)["func"].body
num_alloc = [0]
@@ -326,23 +345,24 @@ def test_alloc_seq_type2():
register_mem(scope_tb, max_bits)
- ib = tvm.tir.ir_builder.create()
- n = te.var("n")
- with ib.for_range(0, n, name="i") as i:
- with ib.for_range(0, 10, name="j") as j:
- A = ib.allocate("float32", 200, name="A", scope=scope_tb)
- A[j] = 1.2
- with ib.for_range(0, 20, name="j") as j:
- B = ib.allocate("int16", 400, name="B", scope=scope_tb)
- B[j] = tvm.tir.const(1, "int16")
- with ib.for_range(0, 10, name="j") as j:
- C = ib.allocate("float32", 200, name="C", scope=scope_tb)
- C[j] = 1.2
-
- body = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([n], body))
- body = tvm.tir.transform.StorageRewrite()(mod)["main"].body
+ @T.prim_func
+ def func(n: T.int32):
+ for i in T.serial(n):
+ for j in range(10):
+ A_data = T.allocate([200], "float32", scope=scope_tb)
+ A = T.Buffer([200], "float32", data=A_data, scope=scope_tb)
+ A[j] = T.float32(1.2)
+ for j in range(20):
+ B_data = T.allocate([400], "int16", scope=scope_tb)
+ B = T.Buffer([400], "int16", data=B_data, scope=scope_tb)
+ B[j] = T.int16(1)
+ for j in range(10):
+ C_data = T.allocate([200], "float32", scope=scope_tb)
+ C = T.Buffer([200], "float32", data=C_data, scope=scope_tb)
+ C[j] = T.float32(1.2)
+
+ mod = tvm.IRModule.from_expr(func)
+ body = tvm.tir.transform.StorageRewrite()(mod)["func"].body
num_alloc = [0]
@@ -356,27 +376,31 @@ def test_alloc_seq_type2():
def test_reuse_small_buffer():
- ib = tvm.tir.ir_builder.create()
- n = te.var("n")
- with ib.for_range(0, n, name="i") as i:
- with ib.for_range(0, 10, name="j") as j:
- A = ib.allocate("int16", 200, name="A", scope="local.L0A")
- A[j] = tvm.tir.const(1, "int16")
- B = ib.allocate("int16", 200, name="B", scope="local.L0A")
- B[j] = tvm.tir.const(1, "int16")
- B1 = ib.allocate("int16", 200, name="B1", scope="local.L0A")
- B1[j] = A[j] + B[j]
- C = ib.allocate("int16", 400, name="C", scope="local.L0A")
- C[j] = tvm.tir.const(1, "int16")
- D = ib.allocate("int16", 400, name="D", scope="local.L0A")
- D[j] = tvm.tir.const(1, "int16")
- E = ib.allocate("int16", 400, name="E", scope="local.L0A")
- E[j] = C[j]
-
- body = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([n], body))
- body = tvm.tir.transform.StorageRewrite()(mod)["main"].body
+ @T.prim_func
+ def func(n: T.int32):
+ for i in T.serial(n):
+ for j in range(10):
+ A_data = T.allocate([200], "int16", scope="local.L0A")
+ A = T.Buffer([200], "int16", data=A_data, scope="local.L0A")
+ A[j] = T.int16(1)
+ B_data = T.allocate([200], "int16", scope="local.L0A")
+ B = T.Buffer([200], "int16", data=B_data, scope="local.L0A")
+ B[j] = T.int16(1)
+ B1_data = T.allocate([200], "int16", scope="local.L0A")
+ B1 = T.Buffer([200], "int16", data=B1_data, scope="local.L0A")
+ B1[j] = A[j] + B[j]
+ C_data = T.allocate([400], "int16", scope="local.L0A")
+ C = T.Buffer([400], "int16", data=C_data, scope="local.L0A")
+ C[j] = T.int16(1)
+ D_data = T.allocate([400], "int16", scope="local.L0A")
+ D = T.Buffer([400], "int16", data=D_data, scope="local.L0A")
+ D[j] = T.int16(1)
+ E_data = T.allocate([400], "int16", scope="local.L0A")
+ E = T.Buffer([400], "int16", data=E_data, scope="local.L0A")
+ E[j] = C[j]
+
+ mod = tvm.IRModule.from_expr(func)
+ body = tvm.tir.transform.StorageRewrite()(mod)["func"].body
num_alloc = [0]
diff --git a/tests/python/tir-transform/test_tir_transform_unroll_loop.py b/tests/python/tir-transform/test_tir_transform_unroll_loop.py
index 37dc64a9e7..4da0a15860 100644
--- a/tests/python/tir-transform/test_tir_transform_unroll_loop.py
+++ b/tests/python/tir-transform/test_tir_transform_unroll_loop.py
@@ -15,24 +15,21 @@
# specific language governing permissions and limitations
# under the License.
import tvm
-from tvm import te
-from tvm.script import tir as T
-import os
+from tvm.script import ir as I, tir as T
def test_unroll_loop():
- ib = tvm.tir.ir_builder.create()
- dtype = "int64"
- n = te.size_var("n")
- Ab = tvm.tir.decl_buffer((n,), dtype)
- Aptr = ib.buffer_ptr(Ab)
- # for i in 0 to n-1:
- with ib.for_range(n, n + 2, name="i") as i:
- with ib.for_range(0, 8, name="i", kind="unroll") as j:
- Aptr[j + 1] = Aptr[i] + 1
-
- stmt = ib.get()
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([Ab], stmt))
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main(A: T.handle, n: T.int64):
+ Ab = T.match_buffer(A, (n,), "int64")
+ for i in T.serial(n, n + 2):
+ for j in T.unroll(8):
+ Ab[j + 1] = Ab[i] + T.int64(1)
+
+ mod = Module
+ stmt = mod["main"].body
assert isinstance(stmt, tvm.tir.For)
@@ -51,18 +48,23 @@ def test_unroll_loop():
assert isinstance(ret, tvm.tir.For)
assert ret.kind == tvm.tir.ForKind.UNROLLED
- ib = tvm.tir.ir_builder.create()
- ib.scope_attr(tvm.tir.const(0, "int32"), "pragma_auto_unroll_max_step", 16)
- ib.emit(stmt)
- wrapped = ib.get()
- wrapped = tvm.tir.SeqStmt([wrapped, stmt])
- assert isinstance(ret, tvm.tir.For)
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([Ab], wrapped))
+ @I.ir_module
+ class ModuleWithPragma:
+ @T.prim_func
+ def main(A: T.handle, n: T.int64):
+ Ab = T.match_buffer(A, (n,), "int64")
+ with T.attr(T.int32(0), "pragma_auto_unroll_max_step", 16):
+ for i in T.serial(n, n + 2):
+ for j in T.unroll(8):
+ Ab[j + 1] = Ab[i] + T.int64(1)
+ for i in T.serial(n, n + 2):
+ for j in T.unroll(8):
+ Ab[j + 1] = Ab[i] + T.int64(1)
with tvm.transform.PassContext(
config={"tir.UnrollLoop": {"auto_max_depth": 8, "explicit_unroll": False}}
):
- ret = tvm.tir.transform.UnrollLoop()(mod)["main"].body
+ ret = tvm.tir.transform.UnrollLoop()(ModuleWithPragma)["main"].body
assert isinstance(ret[0], tvm.tir.For)
assert ret[0].kind == tvm.tir.ForKind.UNROLLED
assert isinstance(ret[1], tvm.tir.For)
@@ -70,41 +72,36 @@ def test_unroll_loop():
def test_unroll_fake_loop():
- ib = tvm.tir.ir_builder.create()
- dtype = "int32"
- n = te.size_var("n")
- Ab = tvm.tir.decl_buffer((n,), dtype)
- Aptr = ib.buffer_ptr(Ab)
- # for i in 0 to n-1:
- with ib.for_range(0, 1, name="i") as i:
- Aptr[i * 2] = 3
- with ib.for_range(0, 10, name="j") as j:
- Aptr[j + 1] = Aptr[i] + 1
-
- stmt = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([Ab], stmt))
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main(A: T.handle, n: T.int64):
+ Ab = T.match_buffer(A, (n,), "int32")
+ for i in T.serial(1):
+ Ab[i * 2] = 3
+ for j in T.serial(10):
+ Ab[j + 1] = Ab[i] + 1
with tvm.transform.PassContext(
config={
"tir.UnrollLoop": {"auto_max_depth": 8, "auto_max_extent": 1, "explicit_unroll": False}
}
):
- ret = tvm.tir.transform.UnrollLoop()(mod)["main"].body
+ ret = tvm.tir.transform.UnrollLoop()(Module)["main"].body
assert isinstance(ret[0], tvm.tir.BufferStore)
def test_unroll_allocations():
- @tvm.script.ir_module
- class before:
+ @I.ir_module
+ class Before:
@T.prim_func
def main():
for i in T.unroll(2):
with T.decl_buffer([16], "float32") as buf:
buf[0] = 0.0
- @tvm.script.ir_module
- class expected:
+ @I.ir_module
+ class Expected:
@T.prim_func
def main():
with T.decl_buffer([16], "float32") as buf1:
@@ -112,13 +109,13 @@ def test_unroll_allocations():
with T.decl_buffer([16], "float32") as buf2:
buf2[0] = 0.0
- after = tvm.tir.transform.UnrollLoop()(before)
+ after = tvm.tir.transform.UnrollLoop()(Before)
- tvm.ir.assert_structural_equal(after, expected)
+ tvm.ir.assert_structural_equal(after, Expected)
def test_unroll_local_access():
- @tvm.script.ir_module
+ @I.ir_module
class Before:
@T.prim_func
def main(B: T.Buffer((64,), "float32")):
@@ -129,7 +126,7 @@ def test_unroll_local_access():
for i in T.serial(4):
A_local[i] = T.float32(i)
- @tvm.script.ir_module
+ @I.ir_module
class Expected:
@T.prim_func
def main(B: T.Buffer((64,), "float32")):
diff --git a/tests/python/tir-transform/test_tir_transform_vectorize.py b/tests/python/tir-transform/test_tir_transform_vectorize.py
index 5a4d4ea17d..bba0a500bc 100644
--- a/tests/python/tir-transform/test_tir_transform_vectorize.py
+++ b/tests/python/tir-transform/test_tir_transform_vectorize.py
@@ -18,7 +18,6 @@ import pytest
import tvm
import tvm.testing
-from tvm import te
from tvm.script import ir as I
from tvm.script import tir as T
@@ -47,17 +46,16 @@ def test_vectorize_loop(extent, target):
def test_vectorize_vector():
- n = te.var("n")
- ib = tvm.tir.ir_builder.create()
- A = ib.pointer("float32x4", name="A")
- with ib.for_range(0, n) as i:
- with ib.for_range(0, 4, kind="vectorize") as j:
- A[j] = tvm.tir.const(1, A.dtype)
- stmt = ib.get()
- assert isinstance(stmt.body, tvm.tir.For)
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A, n], stmt))
- stmt = tvm.tir.transform.VectorizeLoop()(mod)["main"].body
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main(A: T.Buffer((4,), "float32x4"), n: T.int32):
+ for i in range(n):
+ for j in T.vectorized(4):
+ A[j] = T.Broadcast(T.float32(1), 4)
+
+ mod = tvm.tir.transform.VectorizeLoop()(Module)
+ stmt = mod["main"].body
assert isinstance(stmt, tvm.tir.For)
assert not isinstance(stmt.body, tvm.tir.For)
@@ -219,39 +217,35 @@ def test_vectorize_let(extent, target):
tvm.ir.assert_structural_equal(mod, After)
-@pytest.mark.parametrize("extent, target", [(4, simple_target), (tvm.tir.vscale() * 4, sve_target)])
+@pytest.mark.parametrize("extent, target", [(4, simple_target), (T.vscale() * 4, sve_target)])
def test_vectorize_with_le_cond(extent, target):
- n = te.var("n")
- ib = tvm.tir.ir_builder.create()
- A = ib.pointer("float32", name="A")
- with ib.for_range(0, extent, kind="vectorize") as i:
- with ib.if_scope(i <= n):
- A[i] = A[i] + 1
- stmt = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A, n], stmt))
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main(A: T.Buffer((16,), "float32"), n: T.int32):
+ for i in T.vectorized(extent):
+ if i <= n:
+ A[i] = A[i] + T.float32(1)
with tvm.target.Target(target):
- stmt = tvm.tir.transform.VectorizeLoop()(mod)["main"].body
+ stmt = tvm.tir.transform.VectorizeLoop()(Module)["main"].body
- # Check that the loop was't vectorised
+ # Check that the loop wasn't vectorised
assert isinstance(stmt, tvm.tir.For)
-@pytest.mark.parametrize("extent, target", [(4, simple_target), (tvm.tir.vscale() * 4, sve_target)])
+@pytest.mark.parametrize("extent, target", [(4, simple_target), (T.vscale() * 4, sve_target)])
def test_vectorize_with_ge_cond(extent, target):
- n = te.var("n")
- ib = tvm.tir.ir_builder.create()
- A = ib.pointer("float32", name="A")
- with ib.for_range(0, extent, kind="vectorize") as i:
- with ib.if_scope(i >= n):
- A[i] = A[i] + 1
- stmt = ib.get()
-
- mod = tvm.IRModule.from_expr(tvm.tir.PrimFunc([A, n], stmt))
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main(A: T.Buffer((16,), "float32"), n: T.int32):
+ for i in T.vectorized(extent):
+ if i >= n:
+ A[i] = A[i] + T.float32(1)
with tvm.target.Target(target):
- stmt = tvm.tir.transform.VectorizeLoop()(mod)["main"].body
+ stmt = tvm.tir.transform.VectorizeLoop()(Module)["main"].body
# Check that the loop wasn't vectorised
assert isinstance(stmt, tvm.tir.For)
@@ -328,42 +322,28 @@ def test_vectorize_let_if_then_else():
def test_vectorize_while_fail():
"""A while loop inside a vectorized loop should fail."""
- n = 64
- num_iter = 10
-
- def test_ir(A, B, C):
- ib = tvm.tir.ir_builder.create()
- n = C.shape[0]
- A = ib.buffer_ptr(A)
- B = ib.buffer_ptr(B)
- C = ib.buffer_ptr(C)
- i = ib.allocate("int32", (1,), name="i", scope="local")
- i[0] = 0
-
- with ib.for_range(0, n) as j:
- C[j] = 0.0
-
- with ib.for_range(0, n, kind="vectorize") as j:
- with ib.while_loop(i[0] < num_iter):
- C[j] += A[j] + B[j]
- i[0] += 1
-
- return ib.get()
-
- dtype = "float32"
- A = te.placeholder((n,), name="A", dtype=dtype)
- B = te.placeholder((n,), name="B", dtype=dtype)
-
- C = te.extern(
- (n,),
- [A, B],
- lambda ins, outs: test_ir(ins[0], ins[1], outs[0]),
- name="while_vectorize",
- dtype=dtype,
- )
+ @I.ir_module
+ class Module:
+ @T.prim_func
+ def main(
+ A: T.Buffer((64,), "float32"),
+ B: T.Buffer((64,), "float32"),
+ C: T.Buffer((64,), "float32"),
+ ):
+ # Initialize C to 0
+ for j in range(64):
+ C[j] = T.float32(0)
+
+ # While loop inside vectorized loop (should fail)
+ i = T.decl_buffer((1,), "int32", scope="local")
+ i[0] = 0
+ for j in T.vectorized(64):
+ while i[0] < 10:
+ C[j] = C[j] + A[j] + B[j]
+ i[0] = i[0] + 1
try:
- tvm.compile(te.create_prim_func([A, B, C]), target="llvm")
+ tvm.compile(Module, target="llvm")
assert False
except tvm.error.TVMError as e:
error_msg = str(e).split("\n")[-1]
@@ -817,11 +797,11 @@ def test_vectorize_llvm_pure_intrin_fail(extent, vec_str, target):
vec_str, "llvm.lround", B[T.Ramp(0, 1, extent)]
)
- with pytest.raises(Exception) as e_info:
- with tvm.target.Target(target):
- mod = tvm.tir.transform.VectorizeLoop()(Before)
+ with tvm.target.Target(target):
+ mod = tvm.tir.transform.VectorizeLoop()(Before)
+ tvm.ir.assert_structural_equal(mod, After)
+ with pytest.raises(Exception) as e_info:
ex = tvm.compile(mod, target=target)
- tvm.ir.assert_structural_equal(mod, After)
assert "Intrinsic does not support vectors" in e_info.value.args[0]