You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2026/10/09 13:12:38 UTC
(tvm) branch main updated: [REFACTOR][IR] Share statement definitions across dialects (#20624)
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new 244700ac6d [REFACTOR][IR] Share statement definitions across dialects (#20624)
244700ac6d is described below
commit 244700ac6d2aacd3456ac8ae62c41573eed81e55
Author: Tianqi Chen <tq...@users.noreply.github.com>
AuthorDate: Fri Oct 9 09:12:30 2026 -0400
[REFACTOR][IR] Share statement definitions across dialects (#20624)
Move common statement definitions, constructors, reflection and
structural traversal into base IR so dialects can share statement nodes.
Keep TIRx scope and tile statements, statement visitors and
thread-binding interpretation with their dialect owners.
Rename statement `IfThenElse` to `If` and Relax conditional `If` to
`IfExpr`, preserving primitive `if_then_else`. Give `TensorStore` an
expression destination and canonical `(dest, indices, value)` ordering,
with dialect-specific type, evaluation and region contracts registered
through existing reflection and operator facilities. Update consumers,
script builders and printers, canonical Python exports and serialized
identities consistently.
---
docs/arch/codegen.rst | 2 +-
docs/arch/relax_vm.rst | 2 +-
include/tvm/{tirx => ir}/stmt.h | 283 ++---
include/tvm/relax/distributed/axis_group_graph.h | 2 +-
include/tvm/relax/expr.h | 26 +-
include/tvm/relax/expr_functor.h | 16 +-
include/tvm/relax/script/ir_builder/frame.h | 2 +-
include/tvm/s_tir/sblock_scope.h | 2 +-
include/tvm/s_tir/script/ir_builder/frame.h | 2 +-
include/tvm/s_tir/stmt.h | 32 +-
include/tvm/s_tir/stmt_functor.h | 19 +-
include/tvm/s_tir/transform.h | 6 +-
include/tvm/s_tir/utils.h | 2 +-
include/tvm/tirx/expr.h | 6 +-
include/tvm/tirx/function.h | 2 +-
include/tvm/tirx/op/gpu.h | 2 +-
include/tvm/tirx/op_attr_types.h | 16 +-
include/tvm/tirx/script/ir_builder/frame.h | 10 +-
include/tvm/tirx/script/ir_builder/ir.h | 16 +-
include/tvm/tirx/stmt.h | 674 +-----------
include/tvm/tirx/stmt_functor.h | 8 +-
include/tvm/topi/detail/extern.h | 2 +-
python/tvm/backend/cuda/lang/alloc_pool.py | 2 +-
python/tvm/backend/cuda/lang/clc.py | 2 +-
.../backend/cuda/tile_primitive/copy/fallback.py | 4 +-
.../cuda/tile_primitive/copy_async/tcgen05_cp.py | 4 +-
.../backend/cuda/tile_primitive/copy_async/tma.py | 4 +-
.../cuda/tile_primitive/gemm_async/tcgen05.py | 14 +-
.../backend/trn/tile_primitive/private_alloc.py | 3 +-
.../tvm/backend/trn/transform/naive_allocator.py | 4 +-
.../backend/trn/transform/private_buffer_alloc.py | 15 +-
python/tvm/ir/__init__.py | 18 +
python/tvm/{tirx => ir}/stmt.py | 316 +-----
python/tvm/relax/__init__.py | 2 +-
python/tvm/relax/expr.py | 6 +-
python/tvm/relax/expr_functor.py | 42 +-
python/tvm/relax/frontend/onnx/onnx_frontend.py | 4 +-
.../tvm/relax/frontend/tflite/tflite_frontend.py | 4 +-
.../frontend/torch/exported_program_translator.py | 8 +-
python/tvm/relax/relax_to_pyfunc_converter.py | 4 +-
python/tvm/relax/script/ir_builder/op.py | 2 +-
python/tvm/relax/testing/ast_printer.py | 4 +-
python/tvm/relax/transform/legalize_ops/grad.py | 6 +-
python/tvm/s_tir/analysis/__init__.py | 2 +-
.../tvm/s_tir/dlight/analysis/common_analysis.py | 3 +-
python/tvm/s_tir/dlight/analysis/gemv.py | 5 +-
python/tvm/s_tir/dlight/base/utils.py | 5 +-
python/tvm/s_tir/dlight/gpu/fallback.py | 15 +-
python/tvm/s_tir/dlight/gpu/gemv.py | 3 +-
python/tvm/s_tir/dlight/gpu/low_batch_gemv.py | 7 +-
python/tvm/s_tir/dlight/gpu/reduction.py | 5 +-
python/tvm/s_tir/dlight/gpu/rmsnorm.py | 3 +-
python/tvm/s_tir/dlight/gpu/transpose.py | 3 +-
.../tvm/s_tir/meta_schedule/relax_integration.py | 3 +-
python/tvm/s_tir/sblock_scope.py | 2 +-
python/tvm/s_tir/schedule/analysis.py | 3 +-
python/tvm/s_tir/schedule/schedule.py | 4 +-
python/tvm/s_tir/schedule/state.py | 4 +-
python/tvm/s_tir/stmt.py | 3 +-
python/tvm/s_tir/transform/transform.py | 4 +-
python/tvm/script/printer/doc.py | 2 +-
python/tvm/te/operation.py | 6 +-
python/tvm/te/tensor.py | 3 +-
python/tvm/tirx/__init__.py | 8 -
python/tvm/tirx/analysis/analysis.py | 4 +-
python/tvm/tirx/function.py | 3 +-
python/tvm/tirx/script/ir_builder/ir.py | 3 +-
.../tvm/tirx/script/ir_builder/parser_protocol.py | 48 +-
python/tvm/tirx/stmt.py | 487 +--------
python/tvm/tirx/tile_primitive.py | 3 +-
python/tvm/tirx/transform/common.py | 4 +-
python/tvm/topi/gpu/scan.py | 66 +-
python/tvm/topi/gpu/scatter_elements.py | 14 +-
python/tvm/topi/gpu/scatter_nd.py | 12 +-
python/tvm/topi/gpu/sort.py | 82 +-
python/tvm/topi/index_put.py | 6 +-
python/tvm/topi/scan.py | 8 +-
python/tvm/topi/scatter.py | 12 +-
python/tvm/topi/scatter_elements.py | 14 +-
python/tvm/topi/searchsorted.py | 10 +-
python/tvm/topi/signal.py | 24 +-
python/tvm/topi/sparse_reshape.py | 38 +-
python/tvm/topi/unique.py | 4 +-
python/tvm/topi/vision/nms.py | 144 +--
python/tvm/topi/vision/nms_util.py | 14 +-
src/backend/cuda/codegen/codegen_cuda.cc | 4 +-
src/backend/cuda/transforms/lower_iket.cc | 31 +-
src/backend/trn/codegen/codegen_trn.cc | 2 +-
src/backend/trn/codegen/codegen_trn.h | 2 +-
src/backend/trn/transform/lower_trainium_layout.cc | 7 +-
src/backend/vulkan/codegen/codegen_spirv.cc | 4 +-
src/backend/vulkan/codegen/codegen_spirv.h | 2 +-
src/backend/webgpu/codegen/codegen_webgpu.cc | 6 +-
src/{tirx => }/ir/seq_stmt_mutate.h | 10 +-
src/{tirx => }/ir/stmt.cc | 285 ++---
src/relax/analysis/layout_transformation.cc | 7 +-
src/relax/analysis/tir_op_pattern_kind.cc | 6 +-
src/relax/analysis/well_formed.cc | 12 +-
src/relax/backend/vm/codegen_vm.cc | 4 +-
src/relax/backend/vm/vm_shape_lower.cc | 6 +-
.../transform/lower_global_view_to_local_view.cc | 2 +-
src/relax/ir/block_builder.cc | 8 +-
src/relax/ir/dataflow_expr_rewriter.cc | 2 +-
src/relax/ir/dataflow_matcher.cc | 2 +-
src/relax/ir/expr.cc | 44 +-
src/relax/ir/expr_functor.cc | 16 +-
src/relax/ir/py_expr_functor.cc | 26 +-
src/relax/op/tensor/inspect.cc | 48 +-
src/relax/script/ir_builder/frame.cc | 2 +-
src/relax/script/printer/binding.cc | 8 +-
src/relax/transform/canonicalize_bindings.cc | 4 +-
src/relax/transform/compute_prim_value.cc | 4 +-
src/relax/transform/convert_dataflow.cc | 2 +-
src/relax/transform/dataflow_inplace.cc | 10 +-
src/relax/transform/eliminate_common_subexpr.cc | 4 +-
src/relax/transform/fuse_tir.cc | 6 +-
src/relax/transform/normalize.cc | 4 +-
src/relax/transform/split_call_tir_by_pattern.cc | 12 +-
src/relax/transform/static_plan_block_memory.cc | 4 +-
src/relax/transform/to_mixed_precision.cc | 2 +-
src/relax/utils.cc | 2 +-
src/s_tir/analysis/calculate_allocated_memory.cc | 2 +-
src/s_tir/analysis/domain_touched.cc | 7 +-
src/s_tir/analysis/estimate_flops.cc | 2 +-
src/s_tir/analysis/identify_memcpy.cc | 6 +-
src/s_tir/analysis/is_pure_function.cc | 9 +-
src/s_tir/analysis/oob_checker.cc | 6 +-
.../analysis/sblock_access_region_detector.cc | 6 +-
.../analysis/sblock_buffer_access_lca_detector.cc | 6 +-
src/s_tir/analysis/verify_well_formed.cc | 2 +-
src/s_tir/backend/adreno/texture_flatten.cc | 4 +-
src/s_tir/ir/data_type_rewriter.h | 4 +-
src/s_tir/ir/ir_mutator_with_analyzer.h | 4 +-
.../feature_extractor/per_store_feature.cc | 16 +-
.../postproc/rewrite_cooperative_fetch.cc | 2 +-
.../postproc/rewrite_parallel_vectorize_unroll.cc | 2 +-
.../schedule_rule/cross_thread_reduction.cc | 2 +-
.../schedule_rule/multi_level_tiling.cc | 6 +-
.../multi_level_tiling_tensor_core.cc | 4 +-
.../multi_level_tiling_wide_vector.cc | 2 +-
src/s_tir/schedule/analysis.h | 6 +-
src/s_tir/schedule/analysis/analysis.cc | 20 +-
src/s_tir/schedule/analysis/reducer.cc | 38 +-
src/s_tir/schedule/ir_comparator.cc | 4 +-
src/s_tir/schedule/primitive/block_annotate.cc | 4 +-
src/s_tir/schedule/primitive/cache_index.cc | 2 +-
.../schedule/primitive/cache_index_helpers.cc | 4 +-
src/s_tir/schedule/primitive/cache_index_helpers.h | 2 +-
src/s_tir/schedule/primitive/cache_read_write.cc | 27 +-
src/s_tir/schedule/primitive/compute_inline.cc | 18 +-
.../schedule/primitive/layout_transformation.cc | 17 +-
.../schedule/primitive/loop_transformation.cc | 7 +-
src/s_tir/schedule/primitive/pad_einsum.cc | 11 +-
src/s_tir/schedule/primitive/read_write_at.cc | 6 +-
src/s_tir/schedule/primitive/reduction.cc | 20 +-
src/s_tir/schedule/primitive/rolling_buffer.cc | 6 +-
src/s_tir/schedule/transform.cc | 7 +-
src/s_tir/schedule/utils.h | 4 +-
src/s_tir/script/ir_builder/script_complete.cc | 2 +-
src/s_tir/stmt.cc | 6 +-
src/s_tir/transform/compact_buffer_region.cc | 18 +-
src/s_tir/transform/default_gpu_schedule.cc | 9 +-
src/s_tir/transform/hoist_expression.cc | 6 +-
src/s_tir/transform/inject_permuted_layout.cc | 7 +-
src/s_tir/transform/inject_ptx_async_copy.cc | 41 +-
src/s_tir/transform/inject_software_pipeline.cc | 4 +-
src/s_tir/transform/inject_virtual_thread.cc | 22 +-
src/s_tir/transform/ir_utils.h | 4 +-
src/s_tir/transform/lift_thread_binding.cc | 8 +-
src/s_tir/transform/loop_partition.cc | 22 +-
.../transform/lower_cross_thread_reduction.cc | 46 +-
src/s_tir/transform/lower_init_block.cc | 2 +-
src/s_tir/transform/lower_match_buffer.cc | 4 +-
src/s_tir/transform/lower_opaque_block.cc | 6 +-
src/s_tir/transform/lower_thread_binding.cc | 9 +-
.../manifest_shared_memory_local_stage.cc | 12 +-
src/s_tir/transform/memhammer_coalesce.cc | 10 +-
.../transform/memhammer_intermediate_stage.cc | 25 +-
src/s_tir/transform/memhammer_lower_auto_copy.cc | 26 +-
.../transform/memhammer_tensorcore_rewrite.cc | 46 +-
.../transform/merge_shared_memory_allocations.cc | 20 +-
.../plan_update_buffer_allocation_location.cc | 4 +-
.../remove_weight_layout_rewrite_block.cc | 7 +-
src/s_tir/transform/storage_access.cc | 6 +-
src/s_tir/transform/storage_access.h | 2 +-
src/s_tir/transform/transform_mma_buffer_layout.cc | 4 +-
src/script/printer/doc_printer.cc | 2 +-
src/script/printer/script_printer.cc | 26 +-
src/sym/transitive_comparison_analyzer.cc | 4 +-
src/target/llvm/codegen_cpu.cc | 2 +-
src/target/llvm/codegen_llvm.cc | 7 +-
src/target/llvm/codegen_llvm.h | 2 +-
src/target/source/codegen_c.cc | 11 +-
src/target/source/codegen_c.h | 2 +-
src/te/operation/create_function.cc | 6 +-
src/tirx/analysis/verify_memory.cc | 2 +-
src/tirx/ir/buffer.cc | 2 +-
src/tirx/ir/data_type_rewriter.cc | 14 +-
src/tirx/ir/data_type_rewriter.h | 2 +-
src/tirx/ir/ir_mutator_with_analyzer.cc | 7 +-
src/tirx/ir/ir_mutator_with_analyzer.h | 2 +-
src/tirx/ir/ir_visitor_with_analyzer.cc | 2 +-
src/tirx/ir/ir_visitor_with_analyzer.h | 2 +-
src/tirx/ir/stmt.cc | 1124 +-------------------
src/tirx/ir/stmt_functor.cc | 36 +-
src/tirx/ir/tir_visitor_with_path.cc | 4 +-
src/tirx/ir/tir_visitor_with_path.h | 4 +-
src/tirx/op/builtin.cc | 17 +-
src/tirx/script/ir_builder/frame.cc | 31 +-
src/tirx/script/ir_builder/ir.cc | 131 +--
src/tirx/script/ir_builder/utils.h | 28 +-
src/tirx/script/printer/buffer.cc | 12 +-
src/tirx/script/printer/for_loop.cc | 25 +-
src/tirx/script/printer/stmt.cc | 66 +-
src/tirx/script/printer/utils.h | 2 +-
src/tirx/transform/bind_target.cc | 4 +-
src/tirx/transform/common_subexpr_elim.cc | 8 +-
src/tirx/transform/flatten_buffer.cc | 6 +-
src/tirx/transform/inline_private_functions.cc | 6 +-
src/tirx/transform/ir_utils.cc | 8 +-
src/tirx/transform/ir_utils.h | 2 +-
src/tirx/transform/lower_thread_allreduce.h | 46 +-
src/tirx/transform/lower_tirx_cleanup.cc | 12 +-
src/tirx/transform/lower_tirx_dedup_tensormap.cc | 8 +-
src/tirx/transform/lower_tirx_opaque.cc | 10 +-
src/tirx/transform/lower_tvm_builtin.cc | 38 +-
src/tirx/transform/lower_warp_memory.cc | 6 +-
src/tirx/transform/make_packed_api.cc | 22 +-
src/tirx/transform/remove_no_op.cc | 21 +-
src/tirx/transform/split_host_device.cc | 9 +
src/tirx/transform/stmt_simplify.cc | 12 +-
src/tirx/transform/stmt_simplify.h | 2 +-
src/tirx/transform/storage_rewrite.cc | 67 +-
src/tirx/transform/tile_primitive_dispatch.cc | 4 +-
src/tirx/transform/tvm_ffi_binder.cc | 4 +-
src/tirx/transform/unroll_loop.cc | 4 +-
src/tirx/transform/unsupported_dtype_legalize.cc | 14 +-
src/tirx/transform/vectorize_loop.cc | 45 +-
tests/cpp/expr_test.cc | 4 +-
tests/cpp/ir_functor_test.cc | 15 +-
tests/python/codegen/test_target_codegen_c_host.py | 2 +-
tests/python/codegen/test_target_codegen_vulkan.py | 2 +-
tests/python/ir/test_nonnullable_ir.py | 20 +-
.../relax/script/test_relax_script_basic_usage.py | 4 +-
.../script/test_relax_script_dynamic_shape.py | 6 +-
.../relax/script/test_relax_script_printer.py | 2 +-
tests/python/relax/test_analysis_well_formed.py | 16 +-
tests/python/relax/test_ast_printer.py | 2 +-
tests/python/relax/test_expr_functor.py | 16 +-
tests/python/relax/test_pipeline.py | 2 +-
.../python/relax/test_runtime_builtin_rnn_state.py | 4 +-
.../relax/test_transform_bundle_model_params.py | 2 +-
tests/python/relax/test_transform_normalize.py | 10 +-
tests/python/runtime/test_runtime_module_load.py | 6 +-
.../test_s_tir_analysis_verify_block_scope.py | 6 +-
.../s_tir/analysis/test_sblock_access_region.py | 8 +-
tests/python/s_tir/dlight/test_gpu_gemv.py | 2 +-
.../s_tir/schedule/test_tir_schedule_analysis.py | 14 +-
.../s_tir/schedule/test_tir_schedule_split_fuse.py | 2 +-
.../s_tir/schedule/test_tir_schedule_state.py | 6 +-
.../s_tir/script/test_s_tir_script_basic_usage.py | 21 +-
.../s_tir/script/test_s_tir_script_blocks.py | 40 +-
.../script/test_s_tir_script_error_handling.py | 9 +-
.../s_tir/script/test_s_tir_script_ir_builder.py | 24 +-
.../s_tir/script/test_s_tir_script_printer.py | 20 +-
tests/python/s_tir/test_s_tir_renew_defs.py | 4 +-
.../test_s_tir_transform_decorate_device_scope.py | 2 +-
.../transform/test_s_tir_transform_hoist_if.py | 104 +-
.../test_s_tir_transform_inject_virtual_thread.py | 6 +-
.../test_s_tir_transform_loop_partition.py | 14 +-
...test_s_tir_transform_lower_thread_all_reduce.py | 2 +-
...st_s_tir_transform_memhammer_lower_auto_copy.py | 2 +-
.../test_s_tir_transform_mma_buffer_layout.py | 4 +-
tests/python/script/test_basic_usage.py | 10 +-
tests/python/script/test_script_printer_doc.py | 6 +-
tests/python/script/test_script_printer_entry.py | 4 +-
.../test_tir_analysis_undefined_vars.py | 20 +-
.../tirx-analysis/test_tir_analysis_verify_ssa.py | 8 +-
.../test_tir_analysis_verify_well_formed.py | 10 +-
.../tirx-base/test_binding_type_propagation.py | 12 +-
tests/python/tirx-base/test_tir_base.py | 18 +-
tests/python/tirx-base/test_tir_buffer.py | 2 +-
tests/python/tirx-base/test_tir_constructor.py | 38 +-
tests/python/tirx-base/test_tir_nodes.py | 36 +-
.../test_tir_inline_private_functions.py | 6 +-
.../test_tir_transform_bf16_legalize.py | 4 +-
.../test_tir_transform_common_subexpr_elim.py | 30 +-
.../test_tir_transform_convert_ssa.py | 56 +-
.../test_tir_transform_function_pass.py | 6 +-
.../test_tir_transform_lower_intrin.py | 24 +-
.../test_tir_transform_lower_tvm_builtin.py | 10 +-
.../test_tir_transform_make_packed_api.py | 6 +-
.../test_tir_transform_narrow_datatype.py | 6 +-
...est_tir_transform_pointer_value_type_rewrite.py | 6 +-
.../test_tir_transform_remove_no_op.py | 30 +-
.../tirx-transform/test_tir_transform_simplify.py | 40 +-
.../test_tir_transform_split_host_device.py | 6 +-
.../test_tir_transform_storage_rewrite.py | 2 +-
.../test_tir_transform_unroll_loop.py | 20 +-
.../tirx-transform/test_tir_transform_vectorize.py | 8 +-
.../python/tirx/codegen/test_codegen_blackwell.py | 4 +-
tests/python/tirx/codegen/test_codegen_cuda.py | 6 +-
tests/python/tirx/codegen/test_codegen_dsmem.py | 4 +-
.../operator/tile_primitive/cuda/copy/test_reg.py | 6 +-
.../tile_primitive/cuda/copy_async/test_dsmem.py | 2 +-
.../tile_primitive/cuda/copy_async/test_tma.py | 2 +-
.../cuda/gemm/test_gemm_mma_m16n8k_.py | 10 +-
.../operator/tile_primitive/trn/test_binary_trn.py | 4 +-
.../tile_primitive/trn/test_compose_op_trn.py | 4 +-
.../operator/tile_primitive/trn/test_copy_trn.py | 4 +-
.../operator/tile_primitive/trn/test_gemm_trn.py | 4 +-
.../tile_primitive/trn/test_reduction_trn.py | 4 +-
.../operator/tile_primitive/trn/test_select_trn.py | 4 +-
.../operator/tile_primitive/trn/test_unary_trn.py | 4 +-
.../tirx/script/test_tirx_script_basic_usage.py | 27 +-
.../python/tirx/script/test_tirx_script_buffers.py | 4 +-
.../tirx/script/test_tirx_script_error_handling.py | 2 +-
.../tirx/script/test_tirx_script_ir_builder.py | 80 +-
.../python/tirx/script/test_tirx_script_printer.py | 18 +-
.../script/test_tirx_script_source_locations.py | 14 +-
tests/python/tirx/test_op.py | 9 +-
.../transform/test_transform_flatten_buffer.py | 8 +-
.../tirx/transform/test_transform_lower_tirx.py | 8 +-
323 files changed, 2163 insertions(+), 4795 deletions(-)
diff --git a/docs/arch/codegen.rst b/docs/arch/codegen.rst
index 2b3c975cf1..db6ec91351 100644
--- a/docs/arch/codegen.rst
+++ b/docs/arch/codegen.rst
@@ -152,7 +152,7 @@ backend (x86, ARM, NVPTX, AMDGPU, etc.).
``Cast`` → LLVM type conversions, ``Call`` → intrinsic or extern function calls.
- **Statements** (``VisitStmt_``) emit LLVM IR side effects:
``TensorStore`` → store instructions, ``For`` → loop basic blocks with branches,
- ``IfThenElse`` → conditional branches, ``tirx.alloc_tensor`` calls → stack or heap allocation.
+ ``If`` → conditional branches, ``tirx.alloc_tensor`` calls → stack or heap allocation.
The key methods on ``CodeGenLLVM`` are:
diff --git a/docs/arch/relax_vm.rst b/docs/arch/relax_vm.rst
index ec3cbfcab5..91ca6d107a 100644
--- a/docs/arch/relax_vm.rst
+++ b/docs/arch/relax_vm.rst
@@ -104,7 +104,7 @@ each Relax expression and emits instructions through the ``ExecBuilder``:
- Each binding in a ``SeqExpr`` generates one or more instructions; the result is stored in a
new register.
- Function calls (``R.call_tir``, ``R.call_packed``, operator calls) become ``Call`` instructions.
-- Conditional expressions (``relax.If``, written as Python ``if`` in TVMScript) become an ``If``
+- Conditional expressions (``relax.IfExpr``, written as Python ``if`` in TVMScript) become an ``If``
instruction followed by ``Goto`` to skip branches.
- The function body ends with a ``Ret`` instruction.
diff --git a/include/tvm/tirx/stmt.h b/include/tvm/ir/stmt.h
similarity index 67%
copy from include/tvm/tirx/stmt.h
copy to include/tvm/ir/stmt.h
index 0e61eb7b10..206e9fdb58 100644
--- a/include/tvm/tirx/stmt.h
+++ b/include/tvm/ir/stmt.h
@@ -16,30 +16,42 @@
* specific language governing permissions and limitations
* under the License.
*/
-/*!
- * \file tvm/tirx/stmt.h
- * \brief TIR statements.
+/*! \file tvm/ir/stmt.h
+ * \brief Statements shared across IR dialects.
*/
-// Acknowledgement: Many low-level stmts originate from Halide.
-#ifndef TVM_TIRX_STMT_H_
-#define TVM_TIRX_STMT_H_
+#ifndef TVM_IR_STMT_H_
+#define TVM_IR_STMT_H_
#include <tvm/ffi/reflection/registry.h>
#include <tvm/ir/op.h>
#include <tvm/ir/prim/expr.h>
-#include <tvm/tirx/exec_scope.h>
-#include <tvm/tirx/expr.h>
-#include <tvm/tirx/layout.h>
#include <initializer_list>
#include <optional>
#include <string>
-#include <type_traits>
#include <utility>
namespace tvm {
-namespace tirx {
+class RegionStmtNode;
+
+/*!
+ * \brief Construct fresh typed variables for a region's lexical body parameters.
+ *
+ * The input carries only the operation, operands and attributes, without a body
+ * or builder state. Parameters are ordered, distinct definitions and may have
+ * name hints. Every region operation must register this hook, returning an empty
+ * array when it has no body parameters. Presence of the attribute identifies
+ * region support without invoking the hook or allocating variables.
+ */
+using FRegionGetBodyParams =
+ ffi::reflection::NativeFunctionView<ffi::Array<Var>(const CallNode* call)>;
+
+/*! \brief Shared FRegionGetBodyParams implementation for regions without body parameters. */
+TVM_DLL ffi::Array<Var> RegionNoBodyParams(const CallNode* call);
+
+/*! \brief Optional dialect validation of a complete region statement. */
+using FRegionValidate = ffi::reflection::NativeFunctionView<void(const RegionStmtNode*)>;
/*! \brief Base node of all statements. */
class StmtNode : public ffi::Object {
public:
@@ -62,7 +74,7 @@ class StmtNode : public ffi::Object {
static constexpr bool _type_s_eq_hash_subclass_kind_fixed = true;
static constexpr const uint32_t _type_child_slots = 15;
- TVM_FFI_DECLARE_OBJECT_INFO("tirx.Stmt", StmtNode, ffi::Object);
+ TVM_FFI_DECLARE_OBJECT_INFO("ir.Stmt", StmtNode, ffi::Object);
};
/*! \brief Container of all statements */
@@ -92,7 +104,7 @@ class SeqStmtNode : public StmtNode {
Stmt operator[](size_t index) const { return seq[index]; }
TVM_DLL static void RegisterReflection();
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.SeqStmt", SeqStmtNode, StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.SeqStmt", SeqStmtNode, StmtNode);
};
/*! \brief Sequence statement. */
@@ -121,23 +133,16 @@ class SeqStmt : public Stmt {
TVM_DEFINE_OBJECT_REF_COW_METHOD(SeqStmtNode);
};
-} // namespace tirx
namespace ffi {
template <>
-inline constexpr bool use_default_type_traits_v<tirx::SeqStmt> = false;
+inline constexpr bool use_default_type_traits_v<SeqStmt> = false;
template <>
-struct TypeTraits<tirx::SeqStmt>
- : public ObjectRefWithFallbackTraitsBase<tirx::SeqStmt, tirx::Stmt, Array<tirx::Stmt>> {
- static tirx::SeqStmt ConvertFallbackValue(tirx::Stmt stmt) {
- return tirx::SeqStmt(std::move(stmt));
- }
- static tirx::SeqStmt ConvertFallbackValue(Array<tirx::Stmt> seq) {
- return tirx::SeqStmt(std::move(seq));
- }
+struct TypeTraits<SeqStmt> : public ObjectRefWithFallbackTraitsBase<SeqStmt, Stmt, Array<Stmt>> {
+ static SeqStmt ConvertFallbackValue(Stmt stmt) { return SeqStmt(std::move(stmt)); }
+ static SeqStmt ConvertFallbackValue(Array<Stmt> seq) { return SeqStmt(std::move(seq)); }
};
} // namespace ffi
-namespace tirx {
/*!
* \brief Bind a variable to a value in the enclosing scope.
@@ -163,7 +168,7 @@ class BindNode : public StmtNode {
.def_ro("var", &BindNode::var, refl::AttachFieldFlag::SEqHashDefSimple())
.def_ro("value", &BindNode::value);
}
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Bind", BindNode, StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.Bind", BindNode, StmtNode);
};
/*!
@@ -231,7 +236,7 @@ class RegionStmtNode : public StmtNode {
.def_ro("result_vars", &RegionStmtNode::result_vars,
refl::AttachFieldFlag::SEqHashDefSimple());
}
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.RegionStmt", RegionStmtNode, StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.RegionStmt", RegionStmtNode, StmtNode);
};
/*! \brief Managed reference to RegionStmtNode. */
@@ -276,7 +281,7 @@ class AssertStmtNode : public StmtNode {
.def_ro("error_kind", &AssertStmtNode::error_kind)
.def_ro("message_parts", &AssertStmtNode::message_parts);
}
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.AssertStmt", AssertStmtNode, StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.AssertStmt", AssertStmtNode, StmtNode);
};
/*!
@@ -295,37 +300,35 @@ class AssertStmt : public Stmt {
};
/*!
- * \brief Store value to the high dimension buffer.
+ * \brief Store a primitive value to an indexed destination expression.
*
* \code
*
* buffer[i, j] = value;
*
* \endcode
- * \sa MakeTensorLoad
*/
class TensorStoreNode : public StmtNode {
public:
- explicit TensorStoreNode(ffi::UnsafeInit tag) : buffer(tag), value(tag) {}
+ explicit TensorStoreNode(ffi::UnsafeInit tag) : dest(tag), value(tag) {}
- TensorStoreNode(TensorVar buffer, PrimExpr value)
- : buffer(std::move(buffer)), value(std::move(value)) {}
+ TensorStoreNode(Expr dest, PrimExpr value) : dest(std::move(dest)), value(std::move(value)) {}
- /*! \brief The buffer variable. */
- TensorVar buffer;
- /*! \brief The value to be stored. */
- PrimExpr value;
+ /*! \brief The indexed destination expression. */
+ Expr dest;
/*! \brief The indices location to be stored. */
ffi::Array<PrimExpr> indices;
+ /*! \brief The value to be stored. */
+ PrimExpr value;
static void RegisterReflection() {
namespace refl = tvm::ffi::reflection;
refl::ObjectDef<TensorStoreNode>()
- .def_ro("buffer", &TensorStoreNode::buffer)
- .def_ro("value", &TensorStoreNode::value)
- .def_ro("indices", &TensorStoreNode::indices);
+ .def_ro("dest", &TensorStoreNode::dest)
+ .def_ro("indices", &TensorStoreNode::indices)
+ .def_ro("value", &TensorStoreNode::value);
}
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.TensorStore", TensorStoreNode, StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.TensorStore", TensorStoreNode, StmtNode);
};
/*!
@@ -334,7 +337,7 @@ class TensorStoreNode : public StmtNode {
*/
class TensorStore : public Stmt {
public:
- TVM_DLL explicit TensorStore(TensorVar buffer, PrimExpr value, ffi::Array<PrimExpr> indices,
+ TVM_DLL explicit TensorStore(Expr dest, ffi::Array<PrimExpr> indices, PrimExpr value,
Span span = Span());
explicit TensorStore(ffi::ObjectPtr<TensorStoreNode> node) : Stmt(std::move(node)) {}
@@ -362,7 +365,7 @@ class EvaluateNode : public StmtNode {
namespace refl = tvm::ffi::reflection;
refl::ObjectDef<EvaluateNode>().def_ro("value", &EvaluateNode::value);
}
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Evaluate", EvaluateNode, StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.Evaluate", EvaluateNode, StmtNode);
};
/*!
@@ -382,13 +385,13 @@ class Evaluate : public Stmt {
};
/*!
- * \brief IfThenElse statement.
+ * \brief If statement.
*/
-class IfThenElseNode : public StmtNode {
+class IfNode : public StmtNode {
public:
- explicit IfThenElseNode(ffi::UnsafeInit tag) : condition(tag), then_case(tag) {}
+ explicit IfNode(ffi::UnsafeInit tag) : condition(tag), then_case(tag) {}
- IfThenElseNode(PrimExpr condition, SeqStmt then_case)
+ IfNode(PrimExpr condition, SeqStmt then_case)
: condition(std::move(condition)), then_case(std::move(then_case)) {}
/*! \brief The condition. */
@@ -400,27 +403,27 @@ class IfThenElseNode : public StmtNode {
static void RegisterReflection() {
namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<IfThenElseNode>()
- .def_ro("condition", &IfThenElseNode::condition)
- .def_ro("then_case", &IfThenElseNode::then_case)
- .def_ro("else_case", &IfThenElseNode::else_case);
+ refl::ObjectDef<IfNode>()
+ .def_ro("condition", &IfNode::condition)
+ .def_ro("then_case", &IfNode::then_case)
+ .def_ro("else_case", &IfNode::else_case);
}
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.IfThenElse", IfThenElseNode, StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.If", IfNode, StmtNode);
};
/*!
- * \brief Managed reference to IfThenElseNode.
- * \sa IfThenElseNode
+ * \brief Managed reference to IfNode.
+ * \sa IfNode
*/
-class IfThenElse : public Stmt {
+class If : public Stmt {
public:
- TVM_DLL IfThenElse(PrimExpr condition, SeqStmt then_case,
- ffi::Optional<SeqStmt> else_case = std::nullopt, Span span = Span());
+ TVM_DLL If(PrimExpr condition, SeqStmt then_case, ffi::Optional<SeqStmt> else_case = std::nullopt,
+ Span span = Span());
- explicit IfThenElse(ffi::ObjectPtr<IfThenElseNode> node) : Stmt(std::move(node)) {}
+ explicit If(ffi::ObjectPtr<IfNode> node) : Stmt(std::move(node)) {}
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(IfThenElse, Stmt, IfThenElseNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(IfThenElseNode);
+ TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(If, Stmt, IfNode);
+ TVM_DEFINE_OBJECT_REF_COW_METHOD(IfNode);
};
/*!
@@ -433,7 +436,7 @@ class IfThenElse : public Stmt {
enum class ForKind : int {
/*! \brief Ordinary loop with sequential iteration semantics. */
kDefault = 0,
- /*! \brief Parallel execution, optionally placed by the thread_binding annotation. */
+ /*! \brief Parallel execution. */
kParallel = 1,
/*!
* \brief Vector SIMD loop.
@@ -477,9 +480,8 @@ class ForNode : public StmtNode {
/*!
* \brief Additional annotations about the loop.
*
- * The reserved thread_binding entry is a string specifying the thread tag
- * for a parallel loop. It determines execution placement and must be
- * preserved by transformations. Other entries provide auxiliary hints.
+ * Entries are interpreted by the consuming dialect. Transformations preserve
+ * annotations unless they explicitly handle the corresponding semantics.
*/
ffi::Map<ffi::String, ffi::Any> annotations;
/*!
@@ -499,18 +501,10 @@ class ForNode : public StmtNode {
.def_ro("step", &ForNode::step);
}
- /*! \brief Return the explicit thread placement, if present. */
- ffi::Optional<ffi::String> GetThreadBinding() const {
- if (auto value = annotations.Get("thread_binding")) {
- return value->cast<ffi::String>();
- }
- return std::nullopt;
- }
-
/*! \brief Check it is a loop without nontrivial loop step. */
bool HasTrivialStep() const;
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.For", ForNode, StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.For", ForNode, StmtNode);
};
/*!
@@ -557,7 +551,7 @@ class WhileNode : public StmtNode {
.def_ro("condition", &WhileNode::condition)
.def_ro("body", &WhileNode::body);
}
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.While", WhileNode, StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.While", WhileNode, StmtNode);
};
/*!
@@ -591,7 +585,7 @@ class ReturnNode : public StmtNode {
refl::ObjectDef<ReturnNode>().def_ro("value", &ReturnNode::value);
}
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Return", ReturnNode, StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.Return", ReturnNode, StmtNode);
};
/*!
@@ -618,7 +612,7 @@ class BreakNode : public StmtNode {
refl::ObjectDef<BreakNode>();
}
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Break", BreakNode, StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.Break", BreakNode, StmtNode);
};
/*!
@@ -645,7 +639,7 @@ class ContinueNode : public StmtNode {
refl::ObjectDef<ContinueNode>();
}
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Continue", ContinueNode, StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("ir.Continue", ContinueNode, StmtNode);
};
/*!
@@ -662,65 +656,6 @@ class Continue : public Stmt {
TVM_DEFINE_OBJECT_REF_COW_METHOD(ContinueNode);
};
-/*!
- * \brief Standalone statement that declares a scope-id binding (e.g. cta_id,
- * warp_id, lane_id). Carries a ``ScopeIdDef`` value.
- *
- * Each declaration is a flat stmt within the device-region body. The declared
- * ``Var``\ s are visible in subsequent stmts in the same enclosing scope
- * (the ``tirx.device_entry`` region body), analogous to ``BindNode``.
- */
-class ScopeIdDefStmtNode : public StmtNode {
- public:
- /*! \brief The scope-id definition (Vars + extents + binding). */
- ScopeIdDef def;
-
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<ScopeIdDefStmtNode>().def_ro("def", &ScopeIdDefStmtNode::def);
- }
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.ScopeIdDefStmt", ScopeIdDefStmtNode, StmtNode);
-};
-
-/*! \brief Managed reference to ScopeIdDefStmtNode. */
-class ScopeIdDefStmt : public Stmt {
- public:
- TVM_DLL ScopeIdDefStmt(ScopeIdDef def, Span span = Span());
-
- explicit ScopeIdDefStmt(ffi::ObjectPtr<ScopeIdDefStmtNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(ScopeIdDefStmt, Stmt, ScopeIdDefStmtNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(ScopeIdDefStmtNode);
-};
-
-/*! \brief Statement attribute and loop annotation keys. */
-namespace attr {
-/*!
- * \brief For annotation: maximum work for automatic unrolling.
- *
- * Integer policy inherited by nested loops unless they override it. Consumed by UnrollLoop.
- */
-constexpr const char* auto_unroll_max_step = "auto_unroll_max_step";
-/*!
- * \brief For annotation: expand unrolled bodies instead of preserving unrolled loops.
- *
- * Integer policy inherited by nested loops unless they override it. Consumed by UnrollLoop.
- */
-constexpr const char* unroll_explicit = "unroll_explicit";
-/*! \brief Annotation key on AllocTensor marking the allocation as volatile. */
-constexpr const char* kVolatile = "tirx.volatile";
-/*! \brief Mark buffer initial addr alignment in bytes */
-constexpr const char* buffer_data_alignment = "buffer_data_alignment";
-/*! \brief Mark buffer allocated addr in bytes */
-constexpr const char* buffer_allocated_addr = "buffer_allocated_addr";
-
-/*!
- * \brief Mark the kernel as persistent.
- */
-constexpr const char* kPersistentKernel = "tirx.persistent_kernel";
-
-} // namespace attr
-
// overload printing of for type.
TVM_DLL std::ostream& operator<<(std::ostream& os, ForKind kind);
@@ -740,81 +675,5 @@ inline const char* ForKind2String(ForKind t) {
TVM_FFI_UNREACHABLE();
}
-/*! \brief Whether stmt is undefined, an integer Evaluate, or an empty sequence. */
-inline bool IsNoOp(const Stmt& stmt) {
- if (!stmt.defined()) return true;
- if (const auto* op = stmt.as<EvaluateNode>()) return op->value.as<IntImmNode>() != nullptr;
- if (const auto* op = stmt.as<SeqStmtNode>()) return op->seq.empty();
- return false;
-}
-
-/*!
- * \brief TIRX TileOpCall stmt.
- */
-class TileOpCallNode : public StmtNode {
- public:
- explicit TileOpCallNode(ffi::UnsafeInit tag) : op(tag) {}
-
- TileOpCallNode(tvm::Op op, ffi::Array<Expr> args, ffi::Map<ffi::String, TensorVar> workspace,
- ffi::Map<ffi::String, Expr> config, ffi::Optional<ffi::String> dispatch,
- ExecScope scope)
- : op(std::move(op)),
- args(std::move(args)),
- workspace(std::move(workspace)),
- config(std::move(config)),
- dispatch(std::move(dispatch)),
- scope(std::move(scope)) {}
-
- // tvm::Op which corresponds to the TIRX operator.
- tvm::Op op;
-
- // Arguments to the operator.
- ffi::Array<Expr> args;
-
- // Workspace (pre-allocated buffers) for the operator.
- ffi::Map<ffi::String, TensorVar> workspace;
-
- // Config for the operator/scheduler.
- ffi::Map<ffi::String, Expr> config;
-
- // Optional dispatch variant name registered via @register_dispatch.
- ffi::Optional<ffi::String> dispatch{std::nullopt};
-
- // Cooperation scope of this call. Default thread (an unscoped call).
- ExecScope scope = ExecScope(ScopeKind::kThread);
-
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<TileOpCallNode>()
- .def_ro("op", &TileOpCallNode::op)
- .def_ro("args", &TileOpCallNode::args)
- .def_ro("workspace", &TileOpCallNode::workspace)
- .def_ro("config", &TileOpCallNode::config)
- .def_ro("dispatch", &TileOpCallNode::dispatch)
- .def_ro("scope", &TileOpCallNode::scope);
- }
-
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.TileOpCall", TileOpCallNode, StmtNode);
-};
-
-/*!
- * \brief Managed reference to TileOpCallNode
- * \sa TileOpCallNode
- */
-class TileOpCall : public Stmt {
- public:
- TVM_DLL TileOpCall(tvm::Op op, ffi::Array<Expr> args,
- ffi::Map<ffi::String, TensorVar> workspace = {},
- ffi::Map<ffi::String, Expr> config = {},
- ffi::Optional<ffi::String> dispatch = std::nullopt,
- ExecScope scope = ExecScope(ScopeKind::kThread));
-
- explicit TileOpCall(ffi::ObjectPtr<TileOpCallNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(TileOpCall, Stmt, TileOpCallNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(TileOpCallNode);
-};
-
-} // namespace tirx
} // namespace tvm
-#endif // TVM_TIR_STMT_H_
+#endif // TVM_IR_STMT_H_
diff --git a/include/tvm/relax/distributed/axis_group_graph.h b/include/tvm/relax/distributed/axis_group_graph.h
index 829933e40d..338349f893 100644
--- a/include/tvm/relax/distributed/axis_group_graph.h
+++ b/include/tvm/relax/distributed/axis_group_graph.h
@@ -121,7 +121,7 @@ class BufferAxisGraphExtractor : public s_tir::StmtExprVisitor {
private:
ffi::Optional<VisitInterrupt> Visit_(const TensorStoreNode* op) final {
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(s_tir::StmtExprVisitor::Visit_(op));
- buffer_access_indices_.push_back({op->buffer, op->indices});
+ buffer_access_indices_.push_back({op->dest.as_or_throw<tvm::tirx::TensorVar>(), op->indices});
return std::nullopt;
}
diff --git a/include/tvm/relax/expr.h b/include/tvm/relax/expr.h
index 9036955302..97b3134631 100644
--- a/include/tvm/relax/expr.h
+++ b/include/tvm/relax/expr.h
@@ -321,13 +321,13 @@ class SeqExpr : public Expr {
*
* \note This is similar to C's ternary operator.
*/
-class IfNode : public ExprNode {
+class IfExprNode : public ExprNode {
public:
- explicit IfNode(Expr cond, SeqExpr true_branch, SeqExpr false_branch)
+ explicit IfExprNode(Expr cond, SeqExpr true_branch, SeqExpr false_branch)
: cond(std::move(cond)),
true_branch(std::move(true_branch)),
false_branch(std::move(false_branch)) {}
- explicit IfNode(ffi::UnsafeInit)
+ explicit IfExprNode(ffi::UnsafeInit)
: cond(ffi::UnsafeInit{}), true_branch(ffi::UnsafeInit{}), false_branch(ffi::UnsafeInit{}) {}
/*! \brief The condition. */
@@ -339,17 +339,17 @@ class IfNode : public ExprNode {
static void RegisterReflection() {
namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<IfNode>()
- .def_ro("cond", &IfNode::cond)
- .def_ro("true_branch", &IfNode::true_branch)
- .def_ro("false_branch", &IfNode::false_branch);
+ refl::ObjectDef<IfExprNode>()
+ .def_ro("cond", &IfExprNode::cond)
+ .def_ro("true_branch", &IfExprNode::true_branch)
+ .def_ro("false_branch", &IfExprNode::false_branch);
}
static constexpr TVMFFISEqHashKind _type_s_eq_hash_kind = kTVMFFISEqHashKindDAGNode;
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("relax.expr.If", IfNode, ExprNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("relax.expr.IfExpr", IfExprNode, ExprNode);
};
-class If : public Expr {
+class IfExpr : public Expr {
public:
/*!
* \brief The constructor
@@ -368,12 +368,12 @@ class If : public Expr {
*
* \param span The source span of the expression.
*/
- TVM_DLL If(Expr cond, Expr true_branch, Expr false_branch, Span span = Span());
+ TVM_DLL IfExpr(Expr cond, Expr true_branch, Expr false_branch, Span span = Span());
- explicit If(ffi::ObjectPtr<IfNode> node) : Expr(std::move(node)) {}
+ explicit IfExpr(ffi::ObjectPtr<IfExprNode> node) : Expr(std::move(node)) {}
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(If, Expr, IfNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(IfNode);
+ TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(IfExpr, Expr, IfExprNode);
+ TVM_DEFINE_OBJECT_REF_COW_METHOD(IfExprNode);
};
/*! \brief A Relax function. */
diff --git a/include/tvm/relax/expr_functor.h b/include/tvm/relax/expr_functor.h
index 1ece6016d6..dc19631976 100644
--- a/include/tvm/relax/expr_functor.h
+++ b/include/tvm/relax/expr_functor.h
@@ -159,7 +159,7 @@ class ExprFunctor<R(const Expr& n, Args...)> {
virtual R VisitExpr_(const tvm::IntImmNode* op, Args... args) EXPR_FUNCTOR_DEFAULT;
virtual R VisitExpr_(const tvm::FloatImmNode* op, Args... args) EXPR_FUNCTOR_DEFAULT;
virtual R VisitExpr_(const SeqExprNode* op, Args... args) EXPR_FUNCTOR_DEFAULT;
- virtual R VisitExpr_(const IfNode* op, Args... args) EXPR_FUNCTOR_DEFAULT;
+ virtual R VisitExpr_(const IfExprNode* op, Args... args) EXPR_FUNCTOR_DEFAULT;
virtual R VisitExpr_(const OpNode* op, Args... args) EXPR_FUNCTOR_DEFAULT;
virtual R VisitExpr_(const TupleGetItemNode* op, Args... args) EXPR_FUNCTOR_DEFAULT;
virtual R VisitExpr_(const StringImmNode* op, Args... args) EXPR_FUNCTOR_DEFAULT;
@@ -248,7 +248,7 @@ class ExprFunctor<R(const Expr& n, Args...)> {
RELAX_EXPR_FUNCTOR_DISPATCH(tvm::IntImmNode);
RELAX_EXPR_FUNCTOR_DISPATCH(tvm::FloatImmNode);
RELAX_EXPR_FUNCTOR_DISPATCH(SeqExprNode);
- RELAX_EXPR_FUNCTOR_DISPATCH(IfNode);
+ RELAX_EXPR_FUNCTOR_DISPATCH(IfExprNode);
RELAX_EXPR_FUNCTOR_DISPATCH(OpNode);
RELAX_EXPR_FUNCTOR_DISPATCH(TupleGetItemNode);
RELAX_EXPR_FUNCTOR_DISPATCH(StringImmNode);
@@ -283,7 +283,7 @@ class ExprVisitor : public ExprFunctor<void(const Expr&)> {
void VisitExpr_(const tvm::IntImmNode* op) override;
void VisitExpr_(const tvm::FloatImmNode* op) override;
void VisitExpr_(const SeqExprNode* op) override;
- void VisitExpr_(const IfNode* op) override;
+ void VisitExpr_(const IfExprNode* op) override;
void VisitExpr_(const OpNode* op) override;
void VisitExpr_(const TupleGetItemNode* op) override;
void VisitExpr_(const StringImmNode* op) override;
@@ -339,7 +339,7 @@ class ExprVisitor : public ExprFunctor<void(const Expr&)> {
virtual void VisitBinding_(const VarBindingNode* binding, const FunctionNode* val);
virtual void VisitBinding_(const VarBindingNode* binding, const CallNode* val);
virtual void VisitBinding_(const VarBindingNode* binding, const SeqExprNode* val);
- virtual void VisitBinding_(const VarBindingNode* binding, const IfNode* val);
+ virtual void VisitBinding_(const VarBindingNode* binding, const IfExprNode* val);
virtual void VisitBinding_(const VarBindingNode* binding, const OpNode* val);
virtual void VisitBinding_(const VarBindingNode* binding, const TupleGetItemNode* val);
virtual void VisitBinding_(const VarBindingNode* binding, const ExprNode* val);
@@ -443,7 +443,7 @@ class ExprMutatorBase : public ExprFunctor<Expr(const Expr&)> {
Expr VisitExpr_(const tvm::IntImmNode* op) override;
Expr VisitExpr_(const tvm::FloatImmNode* op) override;
Expr VisitExpr_(const SeqExprNode* op) override;
- Expr VisitExpr_(const IfNode* op) override;
+ Expr VisitExpr_(const IfExprNode* op) override;
Expr VisitExpr_(const OpNode* op) override;
Expr VisitExpr_(const TupleGetItemNode* op) override;
Expr VisitExpr_(const StringImmNode* op) override;
@@ -572,7 +572,7 @@ class ExprMutator : public ExprMutatorBase {
Expr VisitExpr_(const DataflowVarNode* op) override;
Expr VisitExpr_(const FunctionNode* op) override;
Expr VisitExpr_(const SeqExprNode* op) override;
- Expr VisitExpr_(const IfNode* op) override;
+ Expr VisitExpr_(const IfExprNode* op) override;
/*!
* \brief Generic dispatcher for bindings.
@@ -594,7 +594,7 @@ class ExprMutator : public ExprMutatorBase {
virtual void VisitBinding_(const VarBindingNode* binding, const FunctionNode* val);
virtual void VisitBinding_(const VarBindingNode* binding, const CallNode* val);
virtual void VisitBinding_(const VarBindingNode* binding, const SeqExprNode* val);
- virtual void VisitBinding_(const VarBindingNode* binding, const IfNode* val);
+ virtual void VisitBinding_(const VarBindingNode* binding, const IfExprNode* val);
virtual void VisitBinding_(const VarBindingNode* binding, const OpNode* val);
virtual void VisitBinding_(const VarBindingNode* binding, const TupleGetItemNode* val);
virtual void VisitBinding_(const VarBindingNode* binding, const ExprNode* val);
@@ -651,7 +651,7 @@ class ExprMutator : public ExprMutatorBase {
ffi::Optional<ffi::Array<Var>> params = std::nullopt);
/*!
- * \brief Rewrite the expr with a new scope, used in the branches of If.
+ * \brief Rewrite the expr with a new scope, used in the branches of IfExpr.
*
* Visit an expression that may access variables from the current
* scope, but may not export definitions into the current scope.
diff --git a/include/tvm/relax/script/ir_builder/frame.h b/include/tvm/relax/script/ir_builder/frame.h
index 1e32fec660..151c733f06 100644
--- a/include/tvm/relax/script/ir_builder/frame.h
+++ b/include/tvm/relax/script/ir_builder/frame.h
@@ -57,7 +57,7 @@ class RelaxFrame : public IRBuilderFrame {
};
/*! \brief The base ir_builder frame for frames with SeqExpr
- i.e. Functions, If branches
+ i.e. Functions, IfExpr branches
*/
class SeqExprFrameNode : public RelaxFrameNode {
public:
diff --git a/include/tvm/s_tir/sblock_scope.h b/include/tvm/s_tir/sblock_scope.h
index fb434ee0b3..ff156137fc 100644
--- a/include/tvm/s_tir/sblock_scope.h
+++ b/include/tvm/s_tir/sblock_scope.h
@@ -88,7 +88,7 @@ class StmtSRefNode : public ffi::Object {
* It serves the same purpose as `ffi::ObjectRef::as`, but does not acquire strong reference to
* `stmt`
* \tparam StmtType The type that `this->stmt` to be downcasted to. Presumably
- * tvm::s_tir::SBlockNode or tvm::tirx::ForNode
+ * tvm::s_tir::SBlockNode or tvm::ForNode
* \return nullptr if type check fails, otherwise the casted result for `this->stmt`
*/
template <typename StmtType>
diff --git a/include/tvm/s_tir/script/ir_builder/frame.h b/include/tvm/s_tir/script/ir_builder/frame.h
index 234dad18fb..7112609ae1 100644
--- a/include/tvm/s_tir/script/ir_builder/frame.h
+++ b/include/tvm/s_tir/script/ir_builder/frame.h
@@ -70,7 +70,7 @@ class SBlockFrameNode : public TIRFrameNode {
/*! \brief The write buffer regions of the block. */
ffi::Optional<ffi::Array<tvm::TensorRegion>> writes;
/*! \brief The init statement of the bolck. */
- ffi::Optional<tvm::tirx::SeqStmt> init;
+ ffi::Optional<tvm::SeqStmt> init;
/*! \brief The buffer allocated in the block. */
ffi::Array<tvm::tirx::TensorVar> alloc_buffers;
/*! \brief The match buffer regions. */
diff --git a/include/tvm/s_tir/stmt.h b/include/tvm/s_tir/stmt.h
index 817a1008ba..b89b78db4e 100644
--- a/include/tvm/s_tir/stmt.h
+++ b/include/tvm/s_tir/stmt.h
@@ -100,11 +100,11 @@ class MatchBufferRegion : public ffi::ObjectRef {
*
* \endcode
*/
-class SBlockNode : public tirx::StmtNode {
+class SBlockNode : public StmtNode {
public:
explicit SBlockNode(ffi::UnsafeInit tag) : body(tag) {}
- explicit SBlockNode(tirx::SeqStmt body) : body(std::move(body)) {}
+ explicit SBlockNode(SeqStmt body) : body(std::move(body)) {}
/*! \brief The variables of the block. */
ffi::Array<s_tir::IterVar> iter_vars;
@@ -127,9 +127,9 @@ class SBlockNode : public tirx::StmtNode {
* We also provide primitives to decompose the init into a separate block during scheduling.
* Init field is `std::nullopt` if there is no reduction iter_vars
*/
- ffi::Optional<tirx::SeqStmt> init;
+ ffi::Optional<SeqStmt> init;
/*! \brief The body of the block. */
- tirx::SeqStmt body;
+ SeqStmt body;
static void RegisterReflection() {
namespace refl = tvm::ffi::reflection;
@@ -145,38 +145,38 @@ class SBlockNode : public tirx::StmtNode {
.def_ro("init", &SBlockNode::init)
.def_ro("body", &SBlockNode::body);
}
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("s_tir.SBlock", SBlockNode, tirx::StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("s_tir.SBlock", SBlockNode, StmtNode);
};
/*!
* \brief Managed reference to SBlockNode.
* \sa SBlockNode
*/
-class SBlock : public tirx::Stmt {
+class SBlock : public Stmt {
public:
TVM_DLL explicit SBlock(
ffi::Array<s_tir::IterVar> iter_vars, ffi::Array<TensorRegion> reads,
- ffi::Array<TensorRegion> writes, ffi::String name_hint, tirx::SeqStmt body,
- ffi::Optional<tirx::SeqStmt> init = std::nullopt,
+ ffi::Array<TensorRegion> writes, ffi::String name_hint, SeqStmt body,
+ ffi::Optional<SeqStmt> init = std::nullopt,
ffi::Array<tirx::TensorVar> alloc_buffers = ffi::Array<tirx::TensorVar>(),
ffi::Array<MatchBufferRegion> match_buffers = ffi::Array<MatchBufferRegion>(),
ffi::Map<ffi::String, ffi::Any> annotations = ffi::Map<ffi::String, ffi::Any>(),
Span span = Span());
- TVM_DLL explicit SBlock(ffi::String name_hint, tirx::SeqStmt body,
+ TVM_DLL explicit SBlock(ffi::String name_hint, SeqStmt body,
ffi::Array<tirx::TensorVar> alloc_buffers = ffi::Array<tirx::TensorVar>(),
Span span = Span());
- explicit SBlock(ffi::ObjectPtr<SBlockNode> node) : tirx::Stmt(std::move(node)) {}
+ explicit SBlock(ffi::ObjectPtr<SBlockNode> node) : Stmt(std::move(node)) {}
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(SBlock, tirx::Stmt, SBlockNode);
+ TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(SBlock, Stmt, SBlockNode);
TVM_DEFINE_OBJECT_REF_COW_METHOD(SBlockNode);
};
/*!
* \brief A block realization node represents execution of the block at the binding values.
*/
-class SBlockRealizeNode : public tirx::StmtNode {
+class SBlockRealizeNode : public StmtNode {
public:
explicit SBlockRealizeNode(ffi::UnsafeInit tag) : predicate(tag), block(tag) {}
@@ -200,21 +200,21 @@ class SBlockRealizeNode : public tirx::StmtNode {
.def_ro("predicate", &SBlockRealizeNode::predicate)
.def_ro("block", &SBlockRealizeNode::block);
}
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("s_tir.SBlockRealize", SBlockRealizeNode, tirx::StmtNode);
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("s_tir.SBlockRealize", SBlockRealizeNode, StmtNode);
};
/*!
* \brief Managed reference to BlockRealizeNode
* \sa BlockRealizeNode
*/
-class SBlockRealize : public tirx::Stmt {
+class SBlockRealize : public Stmt {
public:
TVM_DLL explicit SBlockRealize(ffi::Array<PrimExpr> iter_values, PrimExpr predicate, SBlock block,
Span span = Span());
- explicit SBlockRealize(ffi::ObjectPtr<SBlockRealizeNode> node) : tirx::Stmt(std::move(node)) {}
+ explicit SBlockRealize(ffi::ObjectPtr<SBlockRealizeNode> node) : Stmt(std::move(node)) {}
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(SBlockRealize, tirx::Stmt, SBlockRealizeNode);
+ TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(SBlockRealize, Stmt, SBlockRealizeNode);
TVM_DEFINE_OBJECT_REF_COW_METHOD(SBlockRealizeNode);
};
diff --git a/include/tvm/s_tir/stmt_functor.h b/include/tvm/s_tir/stmt_functor.h
index 8d4734c07c..e697e3bc08 100644
--- a/include/tvm/s_tir/stmt_functor.h
+++ b/include/tvm/s_tir/stmt_functor.h
@@ -37,9 +37,8 @@ template <typename FType>
class StmtFunctor;
template <typename R, typename... Args>
-class StmtFunctor<R(const tirx::Stmt&, Args...)>
- : public tirx::StmtFunctor<R(const tirx::Stmt&, Args...)> {
- using Parent = tirx::StmtFunctor<R(const tirx::Stmt&, Args...)>;
+class StmtFunctor<R(const Stmt&, Args...)> : public tirx::StmtFunctor<R(const Stmt&, Args...)> {
+ using Parent = tirx::StmtFunctor<R(const Stmt&, Args...)>;
public:
TVM_DEFINE_OBJECT_FUNCTOR_DEFAULT_CONSTRUCTOR(StmtFunctor, Parent)
@@ -108,15 +107,15 @@ class TVM_DLL StmtExprMutator : public tirx::StmtExprMutator {
using tirx::StmtExprMutator::Mutate;
using tirx::StmtExprMutator::Mutate_;
- virtual UnchangedOr<tirx::Stmt> Mutate_(const SBlockNode* op, InplaceMode inplace_mode);
- virtual UnchangedOr<tirx::Stmt> Mutate_(const SBlockRealizeNode* op, InplaceMode inplace_mode);
+ virtual UnchangedOr<Stmt> Mutate_(const SBlockNode* op, InplaceMode inplace_mode);
+ virtual UnchangedOr<Stmt> Mutate_(const SBlockRealizeNode* op, InplaceMode inplace_mode);
// Share block ownership and binder rules with specialized TIRX helpers.
- static UnchangedOr<tirx::Stmt> MutateBlock(tirx::StmtExprMutator* mutator, const SBlockNode* op,
- InplaceMode inplace_mode);
- static UnchangedOr<tirx::Stmt> MutateBlockRealize(tirx::StmtExprMutator* mutator,
- const SBlockRealizeNode* op,
- InplaceMode inplace_mode);
+ static UnchangedOr<Stmt> MutateBlock(tirx::StmtExprMutator* mutator, const SBlockNode* op,
+ InplaceMode inplace_mode);
+ static UnchangedOr<Stmt> MutateBlockRealize(tirx::StmtExprMutator* mutator,
+ const SBlockRealizeNode* op,
+ InplaceMode inplace_mode);
protected:
explicit StmtExprMutator(const VTable* vtable) : tirx::StmtExprMutator(vtable) {}
diff --git a/include/tvm/s_tir/transform.h b/include/tvm/s_tir/transform.h
index 5a9c0b7277..1b67ce5084 100644
--- a/include/tvm/s_tir/transform.h
+++ b/include/tvm/s_tir/transform.h
@@ -62,7 +62,7 @@ TVM_DLL Pass CanonicalizeLoop();
TVM_DLL Pass LowerCrossThreadReduction();
/*!
- * \brief Lower block init stmt into IfThenElse stmts
+ * \brief Lower block init stmt into If stmts
* \return The pass.
*/
TVM_DLL Pass LowerInitBlock();
@@ -230,7 +230,7 @@ TVM_DLL Pass LoopPartition();
TVM_DLL Pass InjectVirtualThread();
/*!
- * \brief Hoist loop-invariant IfThenElse nodes to
+ * \brief Hoist loop-invariant If nodes to
* outside the eligible loops.
*
* \param variant The variant of the pass.
@@ -242,7 +242,7 @@ TVM_DLL Pass HoistIfThenElse(tvm::ffi::String variant = "");
/*!
* \brief Hoist loop-invariant expressions to outside the eligible loops.
*
- * Can hoist conditionals used in IfThenElse statements and
+ * Can hoist conditionals used in If statements and
* expressions, bindings of variables in Let statements and
* expressions, or boolean expressions, configurable to enable/disable
* each hoistable type.
diff --git a/include/tvm/s_tir/utils.h b/include/tvm/s_tir/utils.h
index 4d127818e9..4730afbdb3 100644
--- a/include/tvm/s_tir/utils.h
+++ b/include/tvm/s_tir/utils.h
@@ -65,7 +65,7 @@ namespace tirx {
*/
#define TVM_SREF_TO_FOR(SRef) \
[&]() { \
- auto result = TVM_SREF_AS_OR_ERR(result, (SRef), ::tvm::tirx::ForNode) \
+ auto result = TVM_SREF_AS_OR_ERR(result, (SRef), ::tvm::ForNode) \
<< "Expects StmtSRef `" << #SRef << "` points to `Loop`, but gets: " \
<< ((SRef)->stmt ? (SRef)->stmt->GetTypeKey() : "None"); \
return result; \
diff --git a/include/tvm/tirx/expr.h b/include/tvm/tirx/expr.h
index 88bbb9f6ce..afde910a03 100644
--- a/include/tvm/tirx/expr.h
+++ b/include/tvm/tirx/expr.h
@@ -33,11 +33,11 @@
#include <string>
namespace tvm {
-namespace tirx {
-
-// forward declare Stmt
+// forward declare shared statement.
class Stmt;
+namespace tirx {
+
/*!
* \brief Checked zero-state view over an ordinary VarNode with TensorType.
*
diff --git a/include/tvm/tirx/function.h b/include/tvm/tirx/function.h
index 4970002ca5..b4a77fa3f0 100644
--- a/include/tvm/tirx/function.h
+++ b/include/tvm/tirx/function.h
@@ -53,7 +53,7 @@ class FunctionNode : public BaseFuncNode {
/*! \brief The return type of the function. */
Type ret_type = Type::Missing();
/*! \brief The body of the function, absent for a declaration. */
- ffi::Optional<tirx::SeqStmt> body;
+ ffi::Optional<SeqStmt> body;
static void RegisterReflection() {
namespace refl = tvm::ffi::reflection;
diff --git a/include/tvm/tirx/op/gpu.h b/include/tvm/tirx/op/gpu.h
index fac7402e1b..be1ea4ea6a 100644
--- a/include/tvm/tirx/op/gpu.h
+++ b/include/tvm/tirx/op/gpu.h
@@ -48,7 +48,7 @@ TVM_DLL PrimExpr gpu_thread_return(Span span = Span());
*
* Arguments, in order:
* - args[0]: var, The ScopeIdDef-declared thread-axis variable.
- * - args[1]: pred, The runtime predicate used as an IfThenElse condition.
+ * - args[1]: pred, The runtime predicate used as an If condition.
* Canonical thread predicates can be used directly without this wrapper.
*/
TVM_DLL const Op& gpu_thread_filter_op();
diff --git a/include/tvm/tirx/op_attr_types.h b/include/tvm/tirx/op_attr_types.h
index 09e24dd6a3..8ee6a8bd36 100644
--- a/include/tvm/tirx/op_attr_types.h
+++ b/include/tvm/tirx/op_attr_types.h
@@ -32,24 +32,10 @@
#include <tvm/ffi/reflection/native_function.h>
#include <tvm/ffi/string.h>
#include <tvm/ir/expr.h>
+#include <tvm/ir/stmt.h>
namespace tvm {
namespace tirx {
-/*!
- * \brief Construct fresh typed variables for a region's lexical body parameters.
- *
- * The input carries only the operation, operands and attributes, without a body
- * or builder state. Parameters are ordered, distinct definitions and may have
- * name hints. Every region operation must register this hook, returning an empty
- * array when it has no body parameters. Presence of the attribute identifies
- * region support without invoking the hook or allocating variables.
- */
-using FRegionGetBodyParams =
- ffi::reflection::NativeFunctionView<ffi::Array<Var>(const CallNode* call)>;
-
-/*! \brief Shared FRegionGetBodyParams implementation for regions without body parameters. */
-TVM_DLL ffi::Array<Var> RegionNoBodyParams(const CallNode* call);
-
/*!
* \brief Global symbol of the op after lowering.
*/
diff --git a/include/tvm/tirx/script/ir_builder/frame.h b/include/tvm/tirx/script/ir_builder/frame.h
index 7a4ba0051b..deee0c4f87 100644
--- a/include/tvm/tirx/script/ir_builder/frame.h
+++ b/include/tvm/tirx/script/ir_builder/frame.h
@@ -41,7 +41,7 @@ namespace tirx {
class TIRFrameNode : public IRBuilderFrameNode {
public:
/*! \brief The Stmt within in this frame. */
- ffi::Array<tvm::tirx::Stmt> stmts;
+ ffi::Array<tvm::Stmt> stmts;
/*! \brief Bind a view in frames that support region aliases. */
virtual void BindBufferRegion(tvm::tirx::TensorVar buffer, tvm::TensorRegion region);
@@ -152,9 +152,9 @@ class ForFrameNode : public TIRFrameNode {
* \param loop_body The loop body
* \return A stmt, the loop nest
*/
- using FMakeForLoop = ffi::TypedFunction<tvm::tirx::Stmt(
+ using FMakeForLoop = ffi::TypedFunction<tvm::Stmt(
ffi::Array<tvm::Var> loop_vars, ffi::Array<Range> loop_extents,
- ffi::Array<ffi::Optional<PrimExpr>> loop_steps, tvm::tirx::SeqStmt loop_body, Span span)>;
+ ffi::Array<ffi::Optional<PrimExpr>> loop_steps, tvm::SeqStmt loop_body, Span span)>;
/*! \brief The loop variable. */
ffi::Array<tvm::Var> vars;
/*! \brief The domains of iteration. */
@@ -346,9 +346,9 @@ class IfFrameNode : public TIRFrameNode {
/*! \brief The condition of the if statement. */
PrimExpr condition;
/*! \brief The statements in the true branch. */
- ffi::Optional<ffi::Array<tvm::tirx::Stmt>> then_stmts;
+ ffi::Optional<ffi::Array<tvm::Stmt>> then_stmts;
/*! \brief The stetements in the false branch. */
- ffi::Optional<ffi::Array<tvm::tirx::Stmt>> else_stmts;
+ ffi::Optional<ffi::Array<tvm::Stmt>> else_stmts;
static void RegisterReflection() {
namespace refl = tvm::ffi::reflection;
diff --git a/include/tvm/tirx/script/ir_builder/ir.h b/include/tvm/tirx/script/ir_builder/ir.h
index 88d477418d..d524647797 100644
--- a/include/tvm/tirx/script/ir_builder/ir.h
+++ b/include/tvm/tirx/script/ir_builder/ir.h
@@ -249,19 +249,19 @@ WhileFrame While(PrimExpr condition);
* \param value The value to return.
* \return The same statement that was added to the parent frame.
*/
-tvm::tirx::Stmt Return(Expr value);
+tvm::Stmt Return(Expr value);
/*!
* \brief Create a break statement.
* \return The same statement that was added to the parent frame.
*/
-tvm::tirx::Stmt Break();
+tvm::Stmt Break();
/*!
* \brief Create a continue statement.
* \return The same statement that was added to the parent frame.
*/
-tvm::tirx::Stmt Continue();
+tvm::Stmt Continue();
/*!
* \brief Create an if statement.
@@ -326,20 +326,20 @@ RegionFrame Region(Op op, ffi::Array<Expr> args,
DictAttrs attrs = DictAttrs());
/*!
- * \brief Store data in a buffer.
- * \param buffer The buffer.
- * \param value The value to be stored.
+ * \brief Store data in a tensor destination.
+ * \param dest The destination expression.
* \param indices The indices location to be stored.
+ * \param value The value to be stored.
* \return The same statement that was added to the parent frame.
*/
-tvm::tirx::Stmt TensorStore(TensorVar buffer, PrimExpr value, ffi::Array<PrimExpr> indices);
+tvm::Stmt TensorStore(Expr dest, ffi::Array<PrimExpr> indices, PrimExpr value);
/*!
* \brief Evaluate the input expression.
* \param value The input expression to evaluate.
* \return The same statement that was added to the parent frame.
*/
-tvm::tirx::Stmt Evaluate(Expr value);
+tvm::Stmt Evaluate(Expr value);
/*!
* \brief Create a TIR var that represents a pointer
diff --git a/include/tvm/tirx/stmt.h b/include/tvm/tirx/stmt.h
index 0e61eb7b10..8bdc98e868 100644
--- a/include/tvm/tirx/stmt.h
+++ b/include/tvm/tirx/stmt.h
@@ -16,651 +16,34 @@
* specific language governing permissions and limitations
* under the License.
*/
-/*!
- * \file tvm/tirx/stmt.h
- * \brief TIR statements.
+/*! \file tvm/tirx/stmt.h
+ * \brief TIRx statements and shared statement entry header.
*/
-// Acknowledgement: Many low-level stmts originate from Halide.
#ifndef TVM_TIRX_STMT_H_
#define TVM_TIRX_STMT_H_
-#include <tvm/ffi/reflection/registry.h>
-#include <tvm/ir/op.h>
-#include <tvm/ir/prim/expr.h>
+#include <tvm/ir/stmt.h>
#include <tvm/tirx/exec_scope.h>
#include <tvm/tirx/expr.h>
-#include <tvm/tirx/layout.h>
-
-#include <initializer_list>
-#include <optional>
-#include <string>
-#include <type_traits>
-#include <utility>
namespace tvm {
namespace tirx {
-/*! \brief Base node of all statements. */
-class StmtNode : public ffi::Object {
- public:
- /*!
- * \brief Span that points to the original source code.
- * Reserved debug information.
- */
- mutable Span span;
-
- StmtNode() = default;
- explicit StmtNode(Span span) : span(span) {}
-
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<StmtNode>().def_ro("span", &StmtNode::span,
- refl::AttachFieldFlag::SEqHashIgnore());
- }
-
- static constexpr TVMFFISEqHashKind _type_s_eq_hash_kind = kTVMFFISEqHashKindTreeNode;
- static constexpr bool _type_s_eq_hash_subclass_kind_fixed = true;
-
- static constexpr const uint32_t _type_child_slots = 15;
- TVM_FFI_DECLARE_OBJECT_INFO("tirx.Stmt", StmtNode, ffi::Object);
-};
-
-/*! \brief Container of all statements */
-class Stmt : public ffi::ObjectRef {
- public:
- explicit Stmt(ffi::ObjectPtr<StmtNode> node) : ffi::ObjectRef(std::move(node)) {
- TVM_FFI_CHECK(defined(), ValueError) << "Stmt expects a defined node";
- }
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(Stmt, ffi::ObjectRef, StmtNode);
-};
-
-/*!
- * \brief The container of seq statement.
- * Represent a sequence of statements.
- */
-class SeqStmtNode : public StmtNode {
- public:
- /*! \brief internal sequence content. */
- ffi::Array<Stmt> seq;
-
- /*! \return get the size of the sequence */
- size_t size() const { return seq.size(); }
- /*!
- * \brief Get the index-th element in the sequence.
- */
- Stmt operator[](size_t index) const { return seq[index]; }
-
- TVM_DLL static void RegisterReflection();
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.SeqStmt", SeqStmtNode, StmtNode);
-};
-
-/*! \brief Sequence statement. */
-class SeqStmt : public Stmt {
- public:
- /*!
- * \brief Construct SeqStmt.
- * \param seq The sequence.
- * \param span The location of this object in the source code.
- */
- TVM_DLL explicit SeqStmt(ffi::Array<Stmt> seq, Span span = Span());
- /*! \brief Wrap a statement, reusing an existing sequence when possible. */
- TVM_DLL SeqStmt(Stmt stmt, Span span = Span());
- SeqStmt(std::initializer_list<Stmt> seq, Span span = Span())
- : SeqStmt(ffi::Array<Stmt>(seq), std::move(span)) {}
-
- /*! \return get the size of the sequence */
- size_t size() const { return operator->()->size(); }
- /*!
- * \brief Get the index-th element in the sequence.
- */
- Stmt operator[](size_t index) const { return (*(operator->()))[index]; }
- explicit SeqStmt(ffi::ObjectPtr<SeqStmtNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(SeqStmt, Stmt, SeqStmtNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(SeqStmtNode);
-};
-
-} // namespace tirx
-namespace ffi {
-template <>
-inline constexpr bool use_default_type_traits_v<tirx::SeqStmt> = false;
-
-template <>
-struct TypeTraits<tirx::SeqStmt>
- : public ObjectRefWithFallbackTraitsBase<tirx::SeqStmt, tirx::Stmt, Array<tirx::Stmt>> {
- static tirx::SeqStmt ConvertFallbackValue(tirx::Stmt stmt) {
- return tirx::SeqStmt(std::move(stmt));
- }
- static tirx::SeqStmt ConvertFallbackValue(Array<tirx::Stmt> seq) {
- return tirx::SeqStmt(std::move(seq));
- }
-};
-} // namespace ffi
-namespace tirx {
-
-/*!
- * \brief Bind a variable to a value in the enclosing scope.
- *
- * BindNode has no body field. The bound variable is visible
- * in all subsequent statements within the same enclosing scope (SeqStmt,
- * ForNode.body, etc.). This enables flat (non-nested) IR sequences.
- */
-class BindNode : public StmtNode {
- public:
- explicit BindNode(ffi::UnsafeInit tag) : var(tag), value(tag) {}
-
- BindNode(Var var, Expr value) : var(std::move(var)), value(std::move(value)) {}
-
- /*! \brief The variable being bound. */
- Var var;
- /*! \brief The value to bind to the variable. */
- Expr value;
-
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<BindNode>()
- .def_ro("var", &BindNode::var, refl::AttachFieldFlag::SEqHashDefSimple())
- .def_ro("value", &BindNode::value);
- }
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Bind", BindNode, StmtNode);
-};
-
-/*!
- * \brief Managed reference to BindNode.
- * \sa BindNode
- */
-class Bind : public Stmt {
- public:
- TVM_DLL Bind(Var var, Expr value, Span span = Span());
-
- explicit Bind(ffi::ObjectPtr<BindNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(Bind, Stmt, BindNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(BindNode);
-};
-
-/*!
- * \brief Check FRegionGetBodyParams registration without invoking it or creating variables.
- */
-TVM_DLL bool IsRegionOp(const Op& op);
-
-/*!
- * \brief Validate region operands and construct fresh typed body parameters.
- * \param op The region operation.
- * \param args Operands evaluated in the enclosing scope.
- * \param attrs Attributes evaluated in the enclosing scope.
- * \return Parameters from the required FRegionGetBodyParams hook.
- * \throws ValueError if the operation has no registered region hook.
- */
-TVM_DLL ffi::Array<Var> GetRegionBodyParams(Op op, ffi::Array<Expr> args, DictAttrs attrs);
-
-/*!
- * \brief A single-body statement whose semantics are defined by an operator.
- * Operands are evaluated in the enclosing scope. Body parameters are definitions
- * at body entry; result variables are definitions following the region.
- */
-class RegionStmtNode : public StmtNode {
- public:
- /*! \brief Operator defining the region's semantics. */
- Op op;
- /*! \brief Operands evaluated in the enclosing scope. */
- ffi::Array<Expr> args;
- /*! \brief Variables defined at body entry, visible only within the body. */
- ffi::Array<Var> body_params;
- /*! \brief Attributes evaluated in the enclosing scope. */
- DictAttrs attrs;
- /*! \brief Body evaluated with the body parameters in scope. */
- SeqStmt body;
- /*! \brief Variables defined after the region in the enclosing sequence. */
- ffi::Array<Var> result_vars;
-
- explicit RegionStmtNode(ffi::UnsafeInit tag) : op(tag), body(tag) {}
-
- RegionStmtNode(Op op, SeqStmt body) : op(std::move(op)), body(std::move(body)) {}
-
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<RegionStmtNode>()
- .def_ro("op", &RegionStmtNode::op)
- .def_ro("attrs", &RegionStmtNode::attrs)
- .def_ro("args", &RegionStmtNode::args)
- .def_ro("body_params", &RegionStmtNode::body_params,
- refl::AttachFieldFlag::SEqHashDefSimple())
- .def_ro("body", &RegionStmtNode::body)
- .def_ro("result_vars", &RegionStmtNode::result_vars,
- refl::AttachFieldFlag::SEqHashDefSimple());
- }
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.RegionStmt", RegionStmtNode, StmtNode);
-};
-
-/*! \brief Managed reference to RegionStmtNode. */
-class RegionStmt : public Stmt {
- public:
- TVM_DLL RegionStmt(Op op, ffi::Array<Expr> args, ffi::Array<Var> body_params, DictAttrs attrs,
- SeqStmt body, ffi::Array<Var> result_vars = {}, Span span = Span());
-
- explicit RegionStmt(ffi::ObjectPtr<RegionStmtNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(RegionStmt, Stmt, RegionStmtNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(RegionStmtNode);
-};
-
-/*!
- * \brief Assert condition, if an error occurs, return the error message.
- *
- * The error is described by:
- * - \p error_kind: the error kind (e.g. "RuntimeError", "TypeError", "ValueError")
- * - \p message_parts: an array of string fragments that are concatenated at runtime
- * via TVMFFIErrorSetRaisedFromCStrParts. This enables string fragment reuse
- * across multiple assertions to reduce binary size.
- */
-class AssertStmtNode : public StmtNode {
- public:
- explicit AssertStmtNode(ffi::UnsafeInit tag) : condition(tag), error_kind(tag) {}
-
- AssertStmtNode(PrimExpr condition, StringImm error_kind)
- : condition(std::move(condition)), error_kind(std::move(error_kind)) {}
-
- /*! \brief Condition to be checked. */
- PrimExpr condition;
- /*! \brief The error kind, e.g. "RuntimeError", "TypeError", "ValueError". */
- StringImm error_kind;
- /*! \brief Error message fragments, concatenated at runtime when assertion fails. */
- ffi::Array<StringImm> message_parts;
-
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<AssertStmtNode>()
- .def_ro("condition", &AssertStmtNode::condition)
- .def_ro("error_kind", &AssertStmtNode::error_kind)
- .def_ro("message_parts", &AssertStmtNode::message_parts);
- }
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.AssertStmt", AssertStmtNode, StmtNode);
-};
-
-/*!
- * \brief Managed reference to AssertStmtNode.
- * \sa AssertStmtNode
- */
-class AssertStmt : public Stmt {
- public:
- TVM_DLL AssertStmt(PrimExpr condition, StringImm error_kind, ffi::Array<StringImm> message_parts,
- Span span = Span());
-
- explicit AssertStmt(ffi::ObjectPtr<AssertStmtNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(AssertStmt, Stmt, AssertStmtNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(AssertStmtNode);
-};
-
-/*!
- * \brief Store value to the high dimension buffer.
- *
- * \code
- *
- * buffer[i, j] = value;
- *
- * \endcode
- * \sa MakeTensorLoad
- */
-class TensorStoreNode : public StmtNode {
- public:
- explicit TensorStoreNode(ffi::UnsafeInit tag) : buffer(tag), value(tag) {}
-
- TensorStoreNode(TensorVar buffer, PrimExpr value)
- : buffer(std::move(buffer)), value(std::move(value)) {}
-
- /*! \brief The buffer variable. */
- TensorVar buffer;
- /*! \brief The value to be stored. */
- PrimExpr value;
- /*! \brief The indices location to be stored. */
- ffi::Array<PrimExpr> indices;
-
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<TensorStoreNode>()
- .def_ro("buffer", &TensorStoreNode::buffer)
- .def_ro("value", &TensorStoreNode::value)
- .def_ro("indices", &TensorStoreNode::indices);
- }
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.TensorStore", TensorStoreNode, StmtNode);
-};
-
-/*!
- * \brief Managed reference to TensorStoreNode.
- * \sa TensorStoreNode
- */
-class TensorStore : public Stmt {
- public:
- TVM_DLL explicit TensorStore(TensorVar buffer, PrimExpr value, ffi::Array<PrimExpr> indices,
- Span span = Span());
-
- explicit TensorStore(ffi::ObjectPtr<TensorStoreNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(TensorStore, Stmt, TensorStoreNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(TensorStoreNode);
-};
-
-/*!
- * \brief Evaluates an expression.
- * This is mostly used for putting a Call node into Stmt.
- *
- * If value do not have side-effect, this node can be safely removed.
- */
-class EvaluateNode : public StmtNode {
- public:
- explicit EvaluateNode(ffi::UnsafeInit tag) : value(tag) {}
-
- explicit EvaluateNode(Expr value) : value(std::move(value)) {}
-
- /*! \brief The expression to be evaluated. */
- Expr value;
-
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<EvaluateNode>().def_ro("value", &EvaluateNode::value);
- }
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Evaluate", EvaluateNode, StmtNode);
-};
-
-/*!
- * \brief Managed reference to EvaluateNode.
- * \sa EvaluateNode
- */
-class Evaluate : public Stmt {
- public:
- TVM_DLL explicit Evaluate(Expr value, Span span = Span());
-
- explicit Evaluate(int value, Span span = Span()) : Evaluate(PrimExpr(value), span) {}
-
- explicit Evaluate(ffi::ObjectPtr<EvaluateNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(Evaluate, Stmt, EvaluateNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(EvaluateNode);
-};
-
-/*!
- * \brief IfThenElse statement.
- */
-class IfThenElseNode : public StmtNode {
- public:
- explicit IfThenElseNode(ffi::UnsafeInit tag) : condition(tag), then_case(tag) {}
-
- IfThenElseNode(PrimExpr condition, SeqStmt then_case)
- : condition(std::move(condition)), then_case(std::move(then_case)) {}
-
- /*! \brief The condition. */
- PrimExpr condition;
- /*! \brief The branch to be executed when condition is true. */
- SeqStmt then_case;
- /*! \brief The branch to be executed when condition is false, can be null. */
- ffi::Optional<SeqStmt> else_case;
-
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<IfThenElseNode>()
- .def_ro("condition", &IfThenElseNode::condition)
- .def_ro("then_case", &IfThenElseNode::then_case)
- .def_ro("else_case", &IfThenElseNode::else_case);
- }
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.IfThenElse", IfThenElseNode, StmtNode);
-};
-
-/*!
- * \brief Managed reference to IfThenElseNode.
- * \sa IfThenElseNode
- */
-class IfThenElse : public Stmt {
- public:
- TVM_DLL IfThenElse(PrimExpr condition, SeqStmt then_case,
- ffi::Optional<SeqStmt> else_case = std::nullopt, Span span = Span());
-
- explicit IfThenElse(ffi::ObjectPtr<IfThenElseNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(IfThenElse, Stmt, IfThenElseNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(IfThenElseNode);
-};
-
-/*!
- * \brief The kind of the loop.
- *
- * ForKind can change the control flow semantics
- * of the loop. So the kind field needs to be considered
- * in all TIR passes.
- */
-enum class ForKind : int {
- /*! \brief Ordinary loop with sequential iteration semantics. */
- kDefault = 0,
- /*! \brief Parallel execution, optionally placed by the thread_binding annotation. */
- kParallel = 1,
- /*!
- * \brief Vector SIMD loop.
- * The loop body will be vectorized.
- */
- kVectorized = 2,
- /*! \brief The loop body must be unrolled. */
- kUnrolled = 3,
-};
-
-/*!
- * \brief A for loop, with possible type annotations.
- *
- * \code
- *
- * for (loop_var = min; loop_var < min + extent; loop_var += step) {
- * // body
- * }
- * \endcode
- */
-class ForNode : public StmtNode {
- public:
- explicit ForNode(ffi::UnsafeInit tag) : loop_var(tag), min(tag), extent(tag), body(tag) {}
-
- ForNode(PrimVar loop_var, PrimExpr min, PrimExpr extent, SeqStmt body)
- : loop_var(std::move(loop_var)),
- min(std::move(min)),
- extent(std::move(extent)),
- body(std::move(body)) {}
-
- /*! \brief The loop variable. */
- PrimVar loop_var;
- /*! \brief The minimum value of iteration. */
- PrimExpr min;
- /*! \brief The extent of the iteration. */
- PrimExpr extent;
- /*! \brief The kind of the for loop. */
- ForKind kind;
- /*! \brief The body of the for loop. */
- SeqStmt body;
- /*!
- * \brief Additional annotations about the loop.
- *
- * The reserved thread_binding entry is a string specifying the thread tag
- * for a parallel loop. It determines execution placement and must be
- * preserved by transformations. Other entries provide auxiliary hints.
- */
- ffi::Map<ffi::String, ffi::Any> annotations;
- /*!
- * \brief The loop step. It is one if not specified.
- */
- ffi::Optional<PrimExpr> step;
-
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<ForNode>()
- .def_ro("loop_var", &ForNode::loop_var, refl::AttachFieldFlag::SEqHashDefSimple())
- .def_ro("min", &ForNode::min)
- .def_ro("extent", &ForNode::extent)
- .def_ro("kind", &ForNode::kind)
- .def_ro("body", &ForNode::body)
- .def_ro("annotations", &ForNode::annotations)
- .def_ro("step", &ForNode::step);
- }
-
- /*! \brief Return the explicit thread placement, if present. */
- ffi::Optional<ffi::String> GetThreadBinding() const {
- if (auto value = annotations.Get("thread_binding")) {
- return value->cast<ffi::String>();
- }
- return std::nullopt;
- }
-
- /*! \brief Check it is a loop without nontrivial loop step. */
- bool HasTrivialStep() const;
-
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.For", ForNode, StmtNode);
-};
-
-/*!
- * \brief Managed reference to ForNode.
- * \sa ForNode
- */
-class For : public Stmt {
- public:
- TVM_DLL For(PrimVar loop_var, PrimExpr min, PrimExpr extent, ForKind kind, SeqStmt body,
- ffi::Map<ffi::String, ffi::Any> annotations = {},
- ffi::Optional<PrimExpr> step = std::nullopt, Span span = Span());
-
- explicit For(ffi::ObjectPtr<ForNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(For, Stmt, ForNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(ForNode);
-};
-
-/*!
- * \brief A While loop
- *
- * \code
- *
- * while (condition)
- * body
- *
- * \endcode
- */
-class WhileNode : public StmtNode {
- public:
- explicit WhileNode(ffi::UnsafeInit tag) : condition(tag), body(tag) {}
-
- WhileNode(PrimExpr condition, SeqStmt body)
- : condition(std::move(condition)), body(std::move(body)) {}
-
- /*! \brief The termination condition. */
- PrimExpr condition;
- /*! \brief The body of the while loop. */
- SeqStmt body;
-
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<WhileNode>()
- .def_ro("condition", &WhileNode::condition)
- .def_ro("body", &WhileNode::body);
- }
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.While", WhileNode, StmtNode);
-};
-
-/*!
- * \brief Managed reference to WhileNode.
- * \sa WhileNode
- */
-class While : public Stmt {
- public:
- TVM_DLL While(PrimExpr condition, SeqStmt body, Span span = Span());
-
- explicit While(ffi::ObjectPtr<WhileNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(While, Stmt, WhileNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(WhileNode);
-};
-
-/*!
- * \brief A return from the current function.
- */
-class ReturnNode : public StmtNode {
- public:
- explicit ReturnNode(ffi::UnsafeInit tag) : value(tag) {}
-
- explicit ReturnNode(Expr value) : value(std::move(value)) {}
-
- /*! \brief The value to return. */
- Expr value;
-
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<ReturnNode>().def_ro("value", &ReturnNode::value);
- }
-
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Return", ReturnNode, StmtNode);
-};
-
-/*!
- * \brief Managed reference to ReturnNode.
- * \sa ReturnNode
- */
-class Return : public Stmt {
- public:
- TVM_DLL explicit Return(Expr value, Span span = Span());
-
- explicit Return(ffi::ObjectPtr<ReturnNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(Return, Stmt, ReturnNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(ReturnNode);
-};
-
-/*!
- * \brief A Break in control flow.
- */
-class BreakNode : public StmtNode {
- public:
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<BreakNode>();
- }
-
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Break", BreakNode, StmtNode);
-};
-
-/*!
- * \brief Managed reference to BreakNode.
- * \sa BreakNode
- */
-class Break : public Stmt {
- public:
- TVM_DLL explicit Break(Span span);
-
- explicit Break(ffi::ObjectPtr<BreakNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(Break, Stmt, BreakNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(BreakNode);
-};
-
-/*!
- * \brief A Continue in control flow.
- */
-class ContinueNode : public StmtNode {
- public:
- static void RegisterReflection() {
- namespace refl = tvm::ffi::reflection;
- refl::ObjectDef<ContinueNode>();
- }
-
- TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Continue", ContinueNode, StmtNode);
-};
-
-/*!
- * \brief Managed reference to ContinueNode.
- * \sa ContinueNode
- */
-class Continue : public Stmt {
- public:
- TVM_DLL explicit Continue(Span span);
-
- explicit Continue(ffi::ObjectPtr<ContinueNode> node) : Stmt(std::move(node)) {}
-
- TVM_FFI_DEFINE_OBJECT_REF_METHODS_NOTNULLABLE(Continue, Stmt, ContinueNode);
- TVM_DEFINE_OBJECT_REF_COW_METHOD(ContinueNode);
-};
+/*! \brief Interpret and validate TIRx thread placement on a shared loop. */
+inline ffi::Optional<ffi::String> GetThreadBinding(const ForNode* loop) {
+ if (auto tag = loop->annotations.Get("thread_binding")) {
+ TVM_FFI_CHECK(loop->kind == ForKind::kParallel, ValueError)
+ << "thread_binding requires a parallel loop";
+ TVM_FFI_CHECK(tag->as<ffi::String>().has_value(), TypeError)
+ << "thread_binding annotation must be a string";
+ TVM_FFI_CHECK(loop->HasTrivialStep(), ValueError) << "Thread binding loops require a unit step";
+ return tag->cast<ffi::String>();
+ }
+ return std::nullopt;
+}
+inline ffi::Optional<ffi::String> GetThreadBinding(const For& loop) {
+ return GetThreadBinding(loop.get());
+}
/*!
* \brief Standalone statement that declares a scope-id binding (e.g. cta_id,
@@ -721,25 +104,6 @@ constexpr const char* kPersistentKernel = "tirx.persistent_kernel";
} // namespace attr
-// overload printing of for type.
-TVM_DLL std::ostream& operator<<(std::ostream& os, ForKind kind);
-
-// inline implementations
-inline const char* ForKind2String(ForKind t) {
- switch (t) {
- case ForKind::kDefault:
- return "serial";
- case ForKind::kParallel:
- return "parallel";
- case ForKind::kVectorized:
- return "vectorized";
- case ForKind::kUnrolled:
- return "unroll";
- }
- TVM_FFI_THROW(InternalError) << "Unknown ForKind" << t;
- TVM_FFI_UNREACHABLE();
-}
-
/*! \brief Whether stmt is undefined, an integer Evaluate, or an empty sequence. */
inline bool IsNoOp(const Stmt& stmt) {
if (!stmt.defined()) return true;
@@ -817,4 +181,4 @@ class TileOpCall : public Stmt {
} // namespace tirx
} // namespace tvm
-#endif // TVM_TIR_STMT_H_
+#endif // TVM_TIRX_STMT_H_
diff --git a/include/tvm/tirx/stmt_functor.h b/include/tvm/tirx/stmt_functor.h
index b0eb6789c7..87488c6c45 100644
--- a/include/tvm/tirx/stmt_functor.h
+++ b/include/tvm/tirx/stmt_functor.h
@@ -78,7 +78,7 @@ class StmtFunctor<R(const Stmt&, Args...)> {
virtual R Dispatch_(const RegionStmtNode* node, Args... args) {
return DispatchDefault_(node, std::forward<Args>(args)...);
}
- virtual R Dispatch_(const IfThenElseNode* node, Args... args) {
+ virtual R Dispatch_(const IfNode* node, Args... args) {
return DispatchDefault_(node, std::forward<Args>(args)...);
}
virtual R Dispatch_(const ForNode* node, Args... args) {
@@ -133,7 +133,7 @@ class StmtFunctor<R(const Stmt&, Args...)> {
});
SetDispatch<TSelf, BindNode>(vtable);
SetDispatch<TSelf, RegionStmtNode>(vtable);
- SetDispatch<TSelf, IfThenElseNode>(vtable);
+ SetDispatch<TSelf, IfNode>(vtable);
SetDispatch<TSelf, ForNode>(vtable);
SetDispatch<TSelf, WhileNode>(vtable);
SetDispatch<TSelf, ReturnNode>(vtable);
@@ -190,7 +190,7 @@ class TVM_DLL StmtExprVisitor : public tvm::ExprVisitor {
virtual ffi::Optional<VisitInterrupt> Visit_(const BindNode* op);
virtual ffi::Optional<VisitInterrupt> Visit_(const RegionStmtNode* op);
- virtual ffi::Optional<VisitInterrupt> Visit_(const IfThenElseNode* op);
+ virtual ffi::Optional<VisitInterrupt> Visit_(const IfNode* op);
virtual ffi::Optional<VisitInterrupt> Visit_(const ForNode* op);
virtual ffi::Optional<VisitInterrupt> Visit_(const WhileNode* op);
virtual ffi::Optional<VisitInterrupt> Visit_(const ReturnNode* op);
@@ -274,7 +274,7 @@ class TVM_DLL StmtExprMutator : public tvm::ExprMutator {
virtual UnchangedOr<Stmt> Mutate_(const BindNode* op, InplaceMode inplace_mode);
virtual UnchangedOr<Stmt> Mutate_(const RegionStmtNode* op, InplaceMode inplace_mode);
- virtual UnchangedOr<Stmt> Mutate_(const IfThenElseNode* op, InplaceMode inplace_mode);
+ virtual UnchangedOr<Stmt> Mutate_(const IfNode* op, InplaceMode inplace_mode);
virtual UnchangedOr<Stmt> Mutate_(const ForNode* op, InplaceMode inplace_mode);
virtual UnchangedOr<Stmt> Mutate_(const WhileNode* op, InplaceMode inplace_mode);
virtual UnchangedOr<Stmt> Mutate_(const ReturnNode* op, InplaceMode inplace_mode);
diff --git a/include/tvm/topi/detail/extern.h b/include/tvm/topi/detail/extern.h
index 9da8a436a6..c65d5d1083 100644
--- a/include/tvm/topi/detail/extern.h
+++ b/include/tvm/topi/detail/extern.h
@@ -80,7 +80,7 @@ inline ffi::Array<Tensor> make_extern(const ffi::Array<ffi::Array<PrimExpr>>& ou
}
auto body = fextern(input_placeholders, output_placeholders);
- auto body_stmt = tvm::tirx::Evaluate(body);
+ auto body_stmt = tvm::Evaluate(body);
auto op = ExternOp(name, tag, attrs, inputs, input_placeholders, output_placeholders, body_stmt);
diff --git a/python/tvm/backend/cuda/lang/alloc_pool.py b/python/tvm/backend/cuda/lang/alloc_pool.py
index 6f305f731b..c26ff2973a 100644
--- a/python/tvm/backend/cuda/lang/alloc_pool.py
+++ b/python/tvm/backend/cuda/lang/alloc_pool.py
@@ -498,4 +498,4 @@ class SMEMPool:
from tvm.backend.cuda.op import dyn_smem_bytes
from tvm.tirx.script.ir_builder.parser_protocol import add_to_parent
- add_to_parent(tvm.tirx.Evaluate(dyn_smem_bytes(tvm.tirx.IntImm("int64", resolved))))
+ add_to_parent(tvm.ir.Evaluate(dyn_smem_bytes(tvm.tirx.IntImm("int64", resolved))))
diff --git a/python/tvm/backend/cuda/lang/clc.py b/python/tvm/backend/cuda/lang/clc.py
index 1b958cd240..b66b285503 100644
--- a/python/tvm/backend/cuda/lang/clc.py
+++ b/python/tvm/backend/cuda/lang/clc.py
@@ -43,7 +43,7 @@ def query_cancel_first_ctaid_x(first_ctaid_x, handle, *, use_ld_acquire=True):
response, handle
)
T.ptx.clusterlaunchcontrol.query_cancel.is_canceled.pred.b128(canceled, response)
- T.tensor_store(first_ctaid_x.source, T.uint32(0xFFFFFFFF), first_ctaid_x.indices)
+ T.tensor_store(first_ctaid_x.source, first_ctaid_x.indices, T.uint32(0xFFFFFFFF))
T.ptx.clusterlaunchcontrol.query_cancel.get_first_ctaid__x.b32.b128(
first_ctaid_x, response, pred=canceled
)
diff --git a/python/tvm/backend/cuda/tile_primitive/copy/fallback.py b/python/tvm/backend/cuda/tile_primitive/copy/fallback.py
index 74b7e312f0..fa1bafe597 100644
--- a/python/tvm/backend/cuda/tile_primitive/copy/fallback.py
+++ b/python/tvm/backend/cuda/tile_primitive/copy/fallback.py
@@ -93,11 +93,11 @@ def _emit_fallback(op_call: TileOpCall, sctx: DispatchContext) -> Function:
return coord
if not copy_extents:
- T.tensor_store(dst_buf, src_buf[tuple(src_st)], dst_st)
+ T.tensor_store(dst_buf, dst_st, src_buf[tuple(src_st)])
return
with T.grid(*copy_extents) as lvs:
- T.tensor_store(dst_buf, src_buf[tuple(_src_coord(lvs))], _dst_coord(lvs))
+ T.tensor_store(dst_buf, _dst_coord(lvs), src_buf[tuple(_src_coord(lvs))])
scope_kind = sctx.scope_kind
diff --git a/python/tvm/backend/cuda/tile_primitive/copy_async/tcgen05_cp.py b/python/tvm/backend/cuda/tile_primitive/copy_async/tcgen05_cp.py
index 655056d459..18c36b5e33 100644
--- a/python/tvm/backend/cuda/tile_primitive/copy_async/tcgen05_cp.py
+++ b/python/tvm/backend/cuda/tile_primitive/copy_async/tcgen05_cp.py
@@ -109,7 +109,7 @@ import functools
import operator
import tvm
-from tvm.ir import Call, DataTypeImm, DictAttrs, StringImm, Tuple
+from tvm.ir import Bind, Call, DataTypeImm, DictAttrs, Evaluate, SeqStmt, StringImm, Tuple
from tvm.runtime import DataType
from tvm.script import tirx as T
from tvm.sym import Analyzer
@@ -117,7 +117,7 @@ from tvm.tirx import Function, Var
from tvm.tirx.layout import ComposeLayout, TCol, TileLayout, TLane
from tvm.tirx.layout import m as m_axis
from tvm.tirx.operator.tile_primitive import DispatchContext, predicate, register_dispatch
-from tvm.tirx.stmt import Bind, Evaluate, SeqStmt, TileOpCall
+from tvm.tirx.stmt import TileOpCall
from ..copy import _single_thread_exec
diff --git a/python/tvm/backend/cuda/tile_primitive/copy_async/tma.py b/python/tvm/backend/cuda/tile_primitive/copy_async/tma.py
index 661604ba7f..7edde08c3c 100644
--- a/python/tvm/backend/cuda/tile_primitive/copy_async/tma.py
+++ b/python/tvm/backend/cuda/tile_primitive/copy_async/tma.py
@@ -2111,7 +2111,7 @@ def _selected_descriptor(main_map, candidate_maps):
selected_expr,
)
selected = T.Var("selected_tensormap", "uint64")
- return selected, tvm.tirx.Bind(selected, selected_expr), True
+ return selected, tvm.ir.Bind(selected, selected_expr), True
def _emit_plan(
@@ -2215,7 +2215,7 @@ def _emit_plan(
# fmt: on
if selector_bind is not None:
- body = tvm.tirx.SeqStmt([selector_bind, impl.body])
+ body = tvm.ir.SeqStmt([selector_bind, impl.body])
impl = Function([], body, ret_type=None).with_attr("global_symbol", "impl")
return impl
diff --git a/python/tvm/backend/cuda/tile_primitive/gemm_async/tcgen05.py b/python/tvm/backend/cuda/tile_primitive/gemm_async/tcgen05.py
index dd9e8a5cb1..f798828467 100644
--- a/python/tvm/backend/cuda/tile_primitive/gemm_async/tcgen05.py
+++ b/python/tvm/backend/cuda/tile_primitive/gemm_async/tcgen05.py
@@ -26,7 +26,17 @@ import functools
import operator
import tvm
-from tvm.ir import Call, DataTypeImm, DictAttrs, StringImm, TensorRegion, Tuple
+from tvm.ir import (
+ Bind,
+ Call,
+ DataTypeImm,
+ DictAttrs,
+ Evaluate,
+ SeqStmt,
+ StringImm,
+ TensorRegion,
+ Tuple,
+)
from tvm.runtime import DataType
from tvm.script import tirx as T
from tvm.sym.analyzer import Analyzer
@@ -44,7 +54,7 @@ from tvm.tirx.layout import (
tmem_mma_operand_layout,
)
from tvm.tirx.operator.tile_primitive import DispatchContext, predicate, register_dispatch
-from tvm.tirx.stmt import Bind, Evaluate, SeqStmt, TileOpCall
+from tvm.tirx.stmt import TileOpCall
from ...cpp.descriptors import (
_check_tcgen05_mma_matrix_shape,
diff --git a/python/tvm/backend/trn/tile_primitive/private_alloc.py b/python/tvm/backend/trn/tile_primitive/private_alloc.py
index eebe890943..0edf02fab9 100644
--- a/python/tvm/backend/trn/tile_primitive/private_alloc.py
+++ b/python/tvm/backend/trn/tile_primitive/private_alloc.py
@@ -20,8 +20,9 @@ from typing import Any
from tvm.backend.trn.tile_primitive.common import init_analyzer, nki_dim
from tvm.backend.trn.tile_primitive.dim_utils import get_ewise_dim_map
from tvm.backend.trn.tile_primitive.instruction_generator import InstructionGenerator
+from tvm.ir import Stmt
from tvm.script import tirx as T
-from tvm.tirx import FloatImm, IntImm, Stmt, Var
+from tvm.tirx import FloatImm, IntImm, Var
from tvm.tirx.op.tile import (
BinaryReduce,
Copy,
diff --git a/python/tvm/backend/trn/transform/naive_allocator.py b/python/tvm/backend/trn/transform/naive_allocator.py
index 30f87ffeac..0c29d41631 100644
--- a/python/tvm/backend/trn/transform/naive_allocator.py
+++ b/python/tvm/backend/trn/transform/naive_allocator.py
@@ -19,8 +19,8 @@ import functools
import tvm_ffi
-from tvm.ir import Call, Op, Var
-from tvm.tirx import Bind, IntImm
+from tvm.ir import Bind, Call, Op, Var
+from tvm.tirx import IntImm
from tvm.tirx.transform.function_pass import function_pass
diff --git a/python/tvm/backend/trn/transform/private_buffer_alloc.py b/python/tvm/backend/trn/transform/private_buffer_alloc.py
index d3f3372bbc..6b1fe68400 100644
--- a/python/tvm/backend/trn/transform/private_buffer_alloc.py
+++ b/python/tvm/backend/trn/transform/private_buffer_alloc.py
@@ -16,16 +16,23 @@
# under the License.
import tvm_ffi
-from tvm.ir import Call, DataTypeImm, DictAttrs, Op, Range, StringImm, Tuple, Var
-from tvm.target import Target
-from tvm.tirx.stmt import (
+from tvm.ir import (
Bind,
+ Call,
+ DataTypeImm,
+ DictAttrs,
For,
+ Op,
+ Range,
RegionStmt,
SeqStmt,
Stmt,
- TileOpCall,
+ StringImm,
+ Tuple,
+ Var,
)
+from tvm.target import Target
+from tvm.tirx.stmt import TileOpCall
from tvm.tirx.tile_primitive import DispatchContext
from tvm.tirx.transform.common import seek_kernel_replace_point
from tvm.tirx.transform.function_pass import function_pass
diff --git a/python/tvm/ir/__init__.py b/python/tvm/ir/__init__.py
index 6aa8a4ae7a..d0df7854ea 100644
--- a/python/tvm/ir/__init__.py
+++ b/python/tvm/ir/__init__.py
@@ -75,5 +75,23 @@ from .function import BaseFunc, CallingConv
from .global_info import GlobalInfo
from .module import IRModule
from .op import Op, register_op_attr
+from .stmt import (
+ Stmt,
+ SeqStmt,
+ Bind,
+ Evaluate,
+ Return,
+ If,
+ ForKind,
+ For,
+ While,
+ Break,
+ Continue,
+ AssertStmt,
+ RegionStmt,
+ TensorStore,
+ stmt_seq,
+ stmt_list,
+)
from tvm_ffi import Array, Map
diff --git a/python/tvm/tirx/stmt.py b/python/tvm/ir/stmt.py
similarity index 53%
copy from python/tvm/tirx/stmt.py
copy to python/tvm/ir/stmt.py
index f8132b8d9a..d93c53cbea 100644
--- a/python/tvm/tirx/stmt.py
+++ b/python/tvm/ir/stmt.py
@@ -14,52 +14,26 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""Statement AST Node in TVM.
-
-Each statement node have subfields that can be visited from python side.
-
-.. code-block:: python
-
- x = tvm.tirx.Var("n", "int32")
- buffer = tvm.tirx.decl_tensor((16,), "float32")
- st = tvm.tirx.stmt.TensorStore(buffer, 1, (x,))
- assert isinstance(st, tvm.tirx.stmt.TensorStore)
- assert(st.buffer == buffer)
-"""
+"""Shared statement nodes and sequence helpers."""
from collections.abc import Mapping, Sequence
from enum import IntEnum
-from typing import TYPE_CHECKING, Any, ClassVar
+from typing import Any
import tvm_ffi
-from tvm.ir import (
- DictAttrs,
- Expr,
- Op,
- Range,
- Scriptable,
- Span,
- StringImm,
- TensorRegion,
- Var,
- make_node,
-)
+from tvm.ir import DictAttrs, Expr, Op, Scriptable, Span, StringImm, Var, make_node
from tvm.runtime import Object
from . import _ffi_api
-from .exec_scope import ExecScope, ScopeIdDef
-
-if TYPE_CHECKING:
- from .tile_primitive import DispatchContext
-@tvm_ffi.register_object("tirx.Stmt")
+@tvm_ffi.register_object("ir.Stmt")
class Stmt(Object, Scriptable):
"""Base class of all the statements."""
-@tvm_ffi.register_object("tirx.Bind")
+@tvm_ffi.register_object("ir.Bind")
class Bind(Stmt):
"""Bind node.
@@ -93,7 +67,7 @@ class Bind(Stmt):
)
-@tvm_ffi.register_object("tirx.AssertStmt")
+@tvm_ffi.register_object("ir.AssertStmt")
class AssertStmt(Stmt):
"""AssertStmt node.
@@ -150,7 +124,7 @@ class ForKind(IntEnum):
UNROLLED = 3
-@tvm_ffi.register_object("tirx.For")
+@tvm_ffi.register_object("ir.For")
class For(Stmt):
"""For node.
@@ -172,9 +146,7 @@ class For(Stmt):
The body statement.
annotations: Optional[Mapping[str, Object]]
- Additional loop annotations. For parallel loops, the optional
- ``thread_binding`` entry is a string naming the bound thread.
- Parallel loops without this entry use CPU parallel execution.
+ Additional loop annotations interpreted by the consuming dialect.
step : Expr
The loop step. Defaults to None, which represents one.
@@ -216,7 +188,7 @@ class For(Stmt):
)
-@tvm_ffi.register_object("tirx.While")
+@tvm_ffi.register_object("ir.While")
class While(Stmt):
"""While node.
@@ -242,47 +214,47 @@ class While(Stmt):
self.__init_handle_by_constructor__(_ffi_api.While, condition, body, span) # type: ignore
-@tvm_ffi.register_object("tirx.TensorStore")
+@tvm_ffi.register_object("ir.TensorStore")
class TensorStore(Stmt):
- """Store a value into a tensor variable.
+ """Store a primitive value into a tensor destination.
Parameters
----------
- buffer : Var
- The buffer.
+ dest : Expr
+ The destination expression with a supported concrete store type.
- value : Expr
- The value we to be stored.
-
- indices : List[Expr]
+ indices : list[Expr]
The indices location to be stored.
+ value : Expr
+ The primitive value to be stored.
+
span : Optional[Span]
The location of the stmt in the source code.
"""
- buffer: Var
- value: Expr
+ dest: Expr
indices: list[Expr]
+ value: Expr
span: Span | None
def __init__(
self,
- buffer: Var,
- value: Expr,
+ dest: Expr,
indices: list[Expr],
+ value: Expr,
span: Span | None = None,
) -> None:
self.__init_handle_by_constructor__(
_ffi_api.TensorStore,
- buffer,
- value,
+ dest,
indices,
- span, # type: ignore
+ value,
+ span,
)
-@tvm_ffi.register_object("tirx.RegionStmt")
+@tvm_ffi.register_object("ir.RegionStmt")
class RegionStmt(Stmt):
"""An operation with enclosing-scope operands and one lexical body.
@@ -331,7 +303,7 @@ class RegionStmt(Stmt):
)
-@tvm_ffi.register_object("tirx.SeqStmt")
+@tvm_ffi.register_object("ir.SeqStmt")
class SeqStmt(Stmt):
"""Sequence of statements.
@@ -357,9 +329,9 @@ class SeqStmt(Stmt):
return len(self.seq)
-@tvm_ffi.register_object("tirx.IfThenElse")
-class IfThenElse(Stmt):
- """IfThenElse node.
+@tvm_ffi.register_object("ir.If")
+class If(Stmt):
+ """If node.
Parameters
----------
@@ -388,7 +360,7 @@ class IfThenElse(Stmt):
span: Span | None = None,
) -> None:
self.__init_handle_by_constructor__(
- _ffi_api.IfThenElse,
+ _ffi_api.If,
condition,
then_case,
else_case,
@@ -396,7 +368,7 @@ class IfThenElse(Stmt):
)
-@tvm_ffi.register_object("tirx.Evaluate")
+@tvm_ffi.register_object("ir.Evaluate")
class Evaluate(Stmt):
"""Evaluate node.
@@ -416,54 +388,7 @@ class Evaluate(Stmt):
self.__init_handle_by_constructor__(_ffi_api.Evaluate, value, span) # type: ignore
-def BufferRegion(buffer: Var, region: list[Range]) -> TensorRegion:
- """Construct a buffer-backed tensor region with TIRX subscript semantics.
-
- Parameters
- ----------
- buffer : Var
- The source buffer.
-
- region : List[Range]
- The ranges, with one entry for each buffer dimension.
- """
- return _ffi_api.BufferRegion(buffer, region)
-
-
-@tvm_ffi.register_object("tirx.ScopeIdDefStmt")
-class ScopeIdDefStmt(Stmt):
- """ScopeIdDefStmt node.
-
- Leaf statement that introduces scope-identifier vars
- (``wg_id = Tx.warpgroup_id([N])``, ``warp_id = Tx.warp_id_in_wg([4])``,
- ``lane_id = Tx.lane_id([32])``, …) at the kernel-body top level. The
- underlying ``ScopeIdDef`` carries the def vars, their extents, and
- the parent/child scope binding.
-
- Note: the C++ field is named ``def`` (a Python keyword). Access it
- via ``getattr(stmt, "def")`` or ``stmt.__getattribute__("def")`` —
- the type-annotation alias here is purely for documentation.
-
- Parameters
- ----------
- def_ : ScopeIdDef
- The scope-id definition (def vars, extents, scope binding).
-
- span : Optional[Span]
- The location of this statement in the source code.
- """
-
- span: Span | None
-
- def __init__(self, def_: ScopeIdDef, span: Span | None = None) -> None:
- self.__init_handle_by_constructor__(
- _ffi_api.ScopeIdDefStmt, # type: ignore
- def_,
- span,
- ) # type: ignore
-
-
-@tvm_ffi.register_object("tirx.Break")
+@tvm_ffi.register_object("ir.Break")
class Break(Stmt):
"""Break node.
@@ -475,7 +400,7 @@ class Break(Stmt):
self.__init_handle_by_constructor__(_ffi_api.Break, span) # type: ignore
-@tvm_ffi.register_object("tirx.Return")
+@tvm_ffi.register_object("ir.Return")
class Return(Stmt):
"""Return node.
@@ -495,7 +420,7 @@ class Return(Stmt):
self.__init_handle_by_constructor__(_ffi_api.Return, value, span) # type: ignore
-@tvm_ffi.register_object("tirx.Continue")
+@tvm_ffi.register_object("ir.Continue")
class Continue(Stmt):
"""Continue node.
@@ -544,176 +469,3 @@ def stmt_list(stmt: Stmt) -> list[Stmt]:
if isinstance(stmt, SeqStmt):
return list(stmt.seq)
return [stmt]
-
-
-@tvm_ffi.register_object("tirx.TileOpCall")
-class TileOpCall(Stmt):
- """TileOpCall node.
-
- Parameters
- ----------
- op : Op
- The operator.
-
- args : List[Expr]
- The arguments.
-
- workspace : Map[str, Var]
- The workspace.
-
- config : Map[str, Expr]
- The scheduler/config dictionary. Omit unused keys; explicit None values
- are invalid. max_inst_size=-1 means unbounded, while omission retains
- the backend default. A present gather4 contains exactly four coordinates.
-
- dispatch : Optional[str]
- The explicit variant name to dispatch to.
-
- scope : ExecScope
- The cooperation scope of this call. Defaults to ``thread`` (an unscoped call).
- """
-
- args: list[Expr]
- workspace: dict[str, Var]
- config: dict[str, Expr]
- dispatch: str | None
- scope: ExecScope
- _registry: ClassVar[dict[Op, type["TileOpCall"]]] = {}
-
- def __init__(
- self,
- *args: list[Expr],
- op: Op | None = None,
- workspace: dict[str, Var] | None = None,
- config: dict[str, Any] | None = None,
- dispatch: str | None = None,
- scope: ExecScope | None = None,
- ) -> None:
- if workspace is None:
- workspace = {}
- if config is None:
- config = {}
- if scope is None:
- scope = ExecScope("thread")
- if op is None:
- assert self.__class__ != TileOpCall, (
- "Directly instantiating TileOpCall needs to specify the op"
- )
- op = self.__class__.op
- self.__init_handle_by_constructor__(
- _ffi_api.TileOpCall,
- op,
- args,
- workspace,
- config,
- dispatch,
- scope, # pylint: disable=no-member
- )
-
- def __init_subclass__(cls, **kwargs):
- super().__init_subclass__(**kwargs)
- if hasattr(cls, "op"):
- cls._registry[cls.op] = cls
-
- @classmethod
- def downcast(cls, instance: "TileOpCall") -> "TileOpCall":
- subclass = cls._registry.get(instance.op)
- if subclass is None:
- return instance # Unknown op: return as-is
- new_instance = subclass.__new__(subclass)
- new_instance.__init_handle_by_constructor__(
- _ffi_api.TileOpCallCopyHandle,
- instance, # pylint: disable=no-member
- )
- return new_instance
-
- def replace(self, **changes: Any) -> "TileOpCall":
- """Return a copy of this call with selected fields replaced.
-
- Every field that is not overridden in ``changes`` is preserved from
- ``self`` (including ``scope``), so rebuilds never silently drop fields.
- The returned node is downcast to the registered subclass for ``op``.
-
- Parameters
- ----------
- **changes : Any
- Field overrides; any of ``op``, ``args``, ``workspace``, ``config``,
- ``dispatch``, ``scope``.
-
- Returns
- -------
- new_call : TileOpCall
- A new call with the requested fields replaced.
- """
- unknown = set(changes) - {"op", "args", "workspace", "config", "dispatch", "scope"}
- if unknown:
- raise TypeError(f"Unknown field(s) for TileOpCall.replace: {sorted(unknown)}")
- new_call = TileOpCall(
- *changes.get("args", self.args),
- op=changes.get("op", self.op),
- workspace=changes.get("workspace", self.workspace),
- config=changes.get("config", self.config),
- dispatch=changes.get("dispatch", self.dispatch),
- scope=changes.get("scope", self.scope),
- )
- return TileOpCall.downcast(new_call)
-
- def with_workspace(self, workspace: dict[str, Var]) -> "TileOpCall":
- """Return a copy with ``workspace`` replaced, preserving all other fields."""
- return self.replace(workspace=workspace)
-
- @property
- def srcs(self) -> list[Expr]:
- raise NotImplementedError("Subclass must implement this method")
-
- @property
- def dsts(self) -> list[Expr]:
- raise NotImplementedError("Subclass must implement this method")
-
- def get_private_buffers(
- self, buffer_dict: dict[Any, tuple[Var, Stmt | None]], sctx: "DispatchContext"
- ) -> dict[str, Any]:
- """
- Create private (intermediate) buffers needed in this operator.
-
- Parameters
- ----------
- buffer_dict: Dict[Any, Tuple[Var, Optional[Stmt]]]
- A dictionary containing private buffers (and their init stmts) in other operators.
- Key can be anything to reference the buffer.
- This is used to reuse private buffers in other operators (like identity tensor etc.).
- If the buffer is not found in the buffer_dict, it will be created and added to
- the buffer_dict.
- If the buffer is found in the buffer_dict but smaller than required, it will be
- enlarged and updated.
-
- sctx: DispatchContext
- The dispatch context.
- This is used to get the target and reuse op dispatch implementations.
-
- Returns:
- -------
- private_buffer_refs: Dict[str, Any]
- The references to private buffers created in this operator.
- Key will be the name to add into workspace.
- private buffer can be accessed by buffer_dict[private_buffer_refs[name]]
- """
- if sctx.target.kind.name == "trn":
- return self.get_private_buffers_trn(buffer_dict, sctx)
- elif sctx.target.kind.name == "cuda":
- return self.get_private_buffers_cuda(buffer_dict, sctx)
- else:
- raise ValueError(f"Unsupported target: {sctx.target.kind.name}")
-
- def get_private_buffers_trn(
- self, buffer_dict: dict[Any, tuple[Var, Stmt | None]], sctx: "DispatchContext"
- ) -> dict[str, Any]:
- return {}
-
- def get_private_buffers_cuda(
- self, buffer_dict: dict[Any, tuple[Var, Stmt | None]], sctx: "DispatchContext"
- ) -> dict[str, Any]:
- return {}
-
- def validate(self) -> None:
- pass
diff --git a/python/tvm/relax/__init__.py b/python/tvm/relax/__init__.py
index d91341ee33..6603002283 100644
--- a/python/tvm/relax/__init__.py
+++ b/python/tvm/relax/__init__.py
@@ -43,7 +43,7 @@ from .expr import (
TupleGetItem,
Function,
ExternFunc,
- If,
+ IfExpr,
prim_value,
)
diff --git a/python/tvm/relax/expr.py b/python/tvm/relax/expr.py
index 0795d06a49..2c2b2c8e88 100644
--- a/python/tvm/relax/expr.py
+++ b/python/tvm/relax/expr.py
@@ -120,8 +120,8 @@ def _binary_rhs_helper(rhs: Expr):
raise TypeError(f"type {type(rhs)} not supported")
-@tvm_ffi.register_object("relax.expr.If")
-class If(_CallableExprWithOp):
+@tvm_ffi.register_object("relax.expr.IfExpr")
+class IfExpr(_CallableExprWithOp):
"""A conditional expression in Relax.
Parameters
@@ -149,7 +149,7 @@ class If(_CallableExprWithOp):
def __init__(self, cond: Expr, true_branch: Expr, false_branch: Expr, span: Span | None = None):
self.__init_handle_by_constructor__(
- _ffi_api.If,
+ _ffi_api.IfExpr,
cond,
true_branch,
false_branch,
diff --git a/python/tvm/relax/expr_functor.py b/python/tvm/relax/expr_functor.py
index 513372d547..172fd156c6 100644
--- a/python/tvm/relax/expr_functor.py
+++ b/python/tvm/relax/expr_functor.py
@@ -40,7 +40,7 @@ from .expr import (
ExternFunc,
Function,
GlobalVar,
- If,
+ IfExpr,
MatchCast,
SeqExpr,
ShapeExpr,
@@ -216,8 +216,8 @@ class ExprFunctor:
ret = self.visit_float_imm_(expr)
elif isinstance(expr, SeqExpr):
ret = self.visit_seq_expr_(expr)
- elif isinstance(expr, If): # type: ignore
- ret = self.visit_if_(expr)
+ elif isinstance(expr, IfExpr): # type: ignore
+ ret = self.visit_if_expr_(expr)
elif isinstance(expr, Op):
ret = self.visit_op_(expr)
elif isinstance(expr, TupleGetItem):
@@ -363,7 +363,7 @@ class ExprFunctor:
def visit_seq_expr_(self, op: SeqExpr):
raise NotImplementedError()
- def visit_if_(self, op: If):
+ def visit_if_expr_(self, op: IfExpr):
raise NotImplementedError()
def visit_op_(self, op: Op):
@@ -448,7 +448,7 @@ class _PyExprVisitor(tvm_ffi.core.Object):
f_visit_function_: Callable | None = None,
f_visit_call_: Callable | None = None,
f_visit_seq_expr_: Callable | None = None,
- f_visit_if_: Callable | None = None,
+ f_visit_if_expr_: Callable | None = None,
f_visit_op_: Callable | None = None,
f_visit_tuple_getitem_: Callable | None = None,
f_visit_expr_fallback_: Callable | None = None,
@@ -480,7 +480,7 @@ class _PyExprVisitor(tvm_ffi.core.Object):
f_visit_function_,
f_visit_call_,
f_visit_seq_expr_,
- f_visit_if_,
+ f_visit_if_expr_,
f_visit_op_,
f_visit_tuple_getitem_,
f_visit_expr_fallback_,
@@ -575,7 +575,7 @@ class PyExprVisitor:
"visit_function_",
"visit_call_",
"visit_seq_expr_",
- "visit_if_",
+ "visit_if_expr_",
"visit_op_",
"visit_tuple_getitem_",
"visit_expr_fallback_",
@@ -775,15 +775,15 @@ class PyExprVisitor:
# Using self._outer() to ref _PyExprVisitor
return _ffi_api.ExprVisitorVisitExpr(self._outer(), op) # type: ignore
- def visit_if_(self, op: If) -> None:
- """Visit If.
- Users can customized this function to overwrite VisitExpr_(const IfNode* op)
+ def visit_if_expr_(self, op: IfExpr) -> None:
+ """Visit IfExpr.
+ Users can customized this function to overwrite VisitExpr_(const IfExprNode* op)
on the C++ side.
Parameters
----------
- op : If
- The If to be visited.
+ op : IfExpr
+ The IfExpr to be visited.
"""
# Using self._outer() to ref _PyExprVisitor
return _ffi_api.ExprVisitorVisitExpr(self._outer(), op) # type: ignore
@@ -967,7 +967,7 @@ class _PyExprMutator(Object):
f_visit_function_: Callable | None = None,
f_visit_call_: Callable | None = None,
f_visit_seq_expr_: Callable | None = None,
- f_visit_if_: Callable | None = None,
+ f_visit_if_expr_: Callable | None = None,
f_visit_op_: Callable | None = None,
f_visit_tuple_getitem_: Callable | None = None,
f_visit_expr_fallback_: Callable | None = None,
@@ -1000,7 +1000,7 @@ class _PyExprMutator(Object):
f_visit_function_,
f_visit_call_,
f_visit_seq_expr_,
- f_visit_if_,
+ f_visit_if_expr_,
f_visit_op_,
f_visit_tuple_getitem_,
f_visit_expr_fallback_,
@@ -1111,7 +1111,7 @@ class PyExprMutator:
"visit_function_",
"visit_call_",
"visit_seq_expr_",
- "visit_if_",
+ "visit_if_expr_",
"visit_op_",
"visit_tuple_getitem_",
"visit_expr_fallback_",
@@ -1380,15 +1380,15 @@ class PyExprMutator:
# Using self._outer() to ref _PyExprMutator
return _ffi_api.ExprMutatorVisitExpr(self._outer(), op) # type: ignore
- def visit_if_(self, op: If) -> Expr:
- """Visit If.
- Users can customized this function to overwrite VisitExpr_(const IfNode* op)
+ def visit_if_expr_(self, op: IfExpr) -> Expr:
+ """Visit IfExpr.
+ Users can customized this function to overwrite VisitExpr_(const IfExprNode* op)
on the C++ side.
Parameters
----------
- op : If
- The If to be visited.
+ op : IfExpr
+ The IfExpr to be visited.
Returns
-------
@@ -1647,7 +1647,7 @@ class PyExprMutator:
return _ffi_api.PyExprMutatorGetVarRemap(self._outer(), var) # type: ignore
def visit_with_new_scope(self, expr: Expr) -> Expr:
- """Rewrite the expr with a new scope, used in a Function's body and the branches of If.
+ """Rewrite the expr with a new scope, used in a Function's body and the branches of IfExpr.
Parameters
----------
diff --git a/python/tvm/relax/frontend/onnx/onnx_frontend.py b/python/tvm/relax/frontend/onnx/onnx_frontend.py
index c078073c33..b87e9ae479 100644
--- a/python/tvm/relax/frontend/onnx/onnx_frontend.py
+++ b/python/tvm/relax/frontend/onnx/onnx_frontend.py
@@ -6500,7 +6500,7 @@ class ONNXGraphImporter:
else_expr = self._convert_subgraph(self.bb, attr["else_branch"])
then_seq = relax.SeqExpr(blocks=[], body=then_expr)
else_seq = relax.SeqExpr(blocks=[], body=else_expr)
- if_result = self.bb.emit(relax.If(cond, then_seq, else_seq))
+ if_result = self.bb.emit(relax.IfExpr(cond, then_seq, else_seq))
if len(outputs) == 1:
self._nodes[outputs[0]] = if_result
else:
@@ -6685,7 +6685,7 @@ class ONNXGraphImporter:
else_expr = self._convert_subgraph(bb, attr["else_branch"])
then_seq = relax.SeqExpr(blocks=[], body=then_expr)
else_seq = relax.SeqExpr(blocks=[], body=else_expr)
- op = bb.emit(relax.If(cond, then_seq, else_seq))
+ op = bb.emit(relax.IfExpr(cond, then_seq, else_seq))
outputs = node.output
if len(outputs) == 1:
self._nodes[outputs[0]] = op
diff --git a/python/tvm/relax/frontend/tflite/tflite_frontend.py b/python/tvm/relax/frontend/tflite/tflite_frontend.py
index e73d1cdd3f..23f1a08911 100644
--- a/python/tvm/relax/frontend/tflite/tflite_frontend.py
+++ b/python/tvm/relax/frontend/tflite/tflite_frontend.py
@@ -2903,7 +2903,7 @@ class OperatorConverter:
if_bb = relax.BlockBuilder()
with if_bb.function(if_name, params=params, private=True):
- result = relax.If(
+ result = relax.IfExpr(
cond,
relax.Call(then_func, branch_args),
relax.Call(else_func, branch_args),
@@ -2945,7 +2945,7 @@ class OperatorConverter:
next_args = self._bind_call_outputs(next_state, loop_var_count)
true_branch = relax.Call(loop_gv, next_args)
false_branch = self._make_tuple_or_single(params)
- result = relax.If(cond, true_branch, false_branch)
+ result = relax.IfExpr(cond, true_branch, false_branch)
loop_bb.emit_func_output(result)
loop_func = loop_bb.get()[loop_name]
module_builder.update_func(loop_gv, loop_func)
diff --git a/python/tvm/relax/frontend/torch/exported_program_translator.py b/python/tvm/relax/frontend/torch/exported_program_translator.py
index e42cf012a0..61ae87c723 100644
--- a/python/tvm/relax/frontend/torch/exported_program_translator.py
+++ b/python/tvm/relax/frontend/torch/exported_program_translator.py
@@ -1669,7 +1669,7 @@ class ExportedProgramImporter(BaseFXGraphImporter):
return gv
def _cond(self, node: fx.Node) -> relax.Expr:
- """Convert torch.ops.higher_order.cond to relax.If.
+ """Convert torch.ops.higher_order.cond to relax.IfExpr.
FX graph structure:
%pred = call_function[target=aten.gt.Scalar](...)
@@ -1703,11 +1703,11 @@ class ExportedProgramImporter(BaseFXGraphImporter):
false_gv = self._import_branch_subgraph(false_graph, operands, "cond_false_branch")
# Build the If expression.
- # true/false branches of relax.If are SeqExpr bodies, so we construct
+ # true/false branches of relax.IfExpr are SeqExpr bodies, so we construct
# the call expressions that will be the body of each branch.
true_call = relax.Call(true_gv, operands)
false_call = relax.Call(false_gv, operands)
- if_expr = relax.If(pred, true_call, false_call)
+ if_expr = relax.IfExpr(pred, true_call, false_call)
return self.block_builder.emit(if_expr, name_hint="cond_result")
########## Others ##########
@@ -2273,7 +2273,7 @@ class ExportedProgramImporter(BaseFXGraphImporter):
self._check_unsupported_func_type(nodes)
# Assertions have side effects, including when they occur in a cond branch.
- # Neither these effects nor relax.If may appear inside a dataflow region.
+ # Neither these effects nor relax.IfExpr may appear inside a dataflow region.
is_pure = not self._has_assert_async(exported_program.graph_module)
use_dataflow = is_pure and not self._has_cond_op(nodes)
diff --git a/python/tvm/relax/relax_to_pyfunc_converter.py b/python/tvm/relax/relax_to_pyfunc_converter.py
index 4efaa74181..925caf79cb 100644
--- a/python/tvm/relax/relax_to_pyfunc_converter.py
+++ b/python/tvm/relax/relax_to_pyfunc_converter.py
@@ -403,7 +403,7 @@ class RelaxExpressionConverter:
return self._convert_tuple(expr, args)
elif isinstance(expr, relax.TupleGetItem):
return self._convert_tuple_get_item(expr, args)
- elif isinstance(expr, relax.If):
+ elif isinstance(expr, relax.IfExpr):
return self._convert_if(expr, args)
elif isinstance(expr, relax.ShapeExpr):
return self._convert_shape_expr(expr)
@@ -825,7 +825,7 @@ class RelaxExpressionConverter:
else:
return self._create_fallback_tensor()
- def _convert_if(self, if_expr: relax.If, args: list[Any]) -> Any:
+ def _convert_if(self, if_expr: relax.IfExpr, args: list[Any]) -> Any:
"""Convert a Relax if expression to Python equivalent."""
condition = self.convert_expr(if_expr.cond, args)
true_branch = self.convert_expr(if_expr.true_branch, args)
diff --git a/python/tvm/relax/script/ir_builder/op.py b/python/tvm/relax/script/ir_builder/op.py
index 0e386e87ad..7c4e0e45a1 100644
--- a/python/tvm/relax/script/ir_builder/op.py
+++ b/python/tvm/relax/script/ir_builder/op.py
@@ -492,7 +492,7 @@ def if_then_else_(condition: Any, true_value: Any, false_value: Any) -> Any:
for value in (true_value, false_value)
):
return _tir.if_then_else(condition, true_value, false_value)
- return _relax.If(condition, _value(true_value), _value(false_value))
+ return _relax.IfExpr(condition, _value(true_value), _value(false_value))
def and_(*values: Any) -> Any:
diff --git a/python/tvm/relax/testing/ast_printer.py b/python/tvm/relax/testing/ast_printer.py
index 0ffdb74d96..a5e19a0635 100644
--- a/python/tvm/relax/testing/ast_printer.py
+++ b/python/tvm/relax/testing/ast_printer.py
@@ -198,10 +198,10 @@ class ASTPrinter(ExprFunctor):
body=self.visit_expr(op.body),
)
- def visit_if_(self, op: relax.If) -> str:
+ def visit_if_expr_(self, op: relax.IfExpr) -> str:
return self.build_expr(
op,
- "If",
+ "IfExpr",
cond=self.visit_expr(op.cond),
true_branch=self.visit_expr(op.true_branch),
false_branch=self.visit_expr(op.false_branch),
diff --git a/python/tvm/relax/transform/legalize_ops/grad.py b/python/tvm/relax/transform/legalize_ops/grad.py
index 62f8268a41..78aef625f3 100644
--- a/python/tvm/relax/transform/legalize_ops/grad.py
+++ b/python/tvm/relax/transform/legalize_ops/grad.py
@@ -180,7 +180,7 @@ def _grad_take_backward(bb: BlockBuilder, call: Call) -> Expr:
with IRBuilder() as ib:
# Init loop (zero-fill output buffer)
with T.serial(fused_shape) as i:
- T.tensor_store(out, tirx.const(0, dtype=x_ptr.dtype), T.tensor_indices(out, i))
+ T.tensor_store(out, T.tensor_indices(out, i), tirx.const(0, dtype=x_ptr.dtype))
# Accumulation loop
if axis is not None:
@@ -212,17 +212,17 @@ def _grad_take_backward(bb: BlockBuilder, call: Call) -> Expr:
)
T.tensor_store(
out,
+ T.tensor_indices(out, out_idx),
out[T.tensor_indices(out, out_idx)]
+ grad[T.tensor_indices(grad, grad_idx)],
- T.tensor_indices(out, out_idx),
)
else:
with T.serial(indices_len) as loop_l:
T.tensor_store(
out,
+ T.tensor_indices(out, (idx[T.tensor_indices(idx, loop_l)])),
out[T.tensor_indices(out, (idx[T.tensor_indices(idx, loop_l)]))]
+ grad[T.tensor_indices(grad, loop_l)],
- T.tensor_indices(out, (idx[T.tensor_indices(idx, loop_l)])),
)
return ib.get()
diff --git a/python/tvm/s_tir/analysis/__init__.py b/python/tvm/s_tir/analysis/__init__.py
index 2d44b46ea8..2c4917fa4b 100644
--- a/python/tvm/s_tir/analysis/__init__.py
+++ b/python/tvm/s_tir/analysis/__init__.py
@@ -25,7 +25,7 @@ from tvm.ir import IRModule, TensorRegion
from tvm.tirx.expr import Var
from tvm.s_tir import SBlock
-from tvm.tirx import Stmt
+from tvm.ir import Stmt
from tvm.tirx.function import Function
from . import _ffi_api
diff --git a/python/tvm/s_tir/dlight/analysis/common_analysis.py b/python/tvm/s_tir/dlight/analysis/common_analysis.py
index c79a00438c..5651fa09be 100644
--- a/python/tvm/s_tir/dlight/analysis/common_analysis.py
+++ b/python/tvm/s_tir/dlight/analysis/common_analysis.py
@@ -26,6 +26,7 @@ from typing import Literal
import tvm_ffi
from tvm_ffi import get_global_func
+import tvm
from tvm import ir, s_tir, tirx
from tvm.ir import TensorRegion
from tvm.s_tir import Schedule
@@ -83,7 +84,7 @@ class BufferInfo:
buf_region: TensorRegion
shape: tuple[int]
assoc_lps: list[s_tir.schedule.LoopRV | None]
- assoc_lps_info: list[tirx.For | None]
+ assoc_lps_info: list[tvm.ir.For | None]
def __init__(
self,
diff --git a/python/tvm/s_tir/dlight/analysis/gemv.py b/python/tvm/s_tir/dlight/analysis/gemv.py
index c7beb0e6ae..e20c37844b 100644
--- a/python/tvm/s_tir/dlight/analysis/gemv.py
+++ b/python/tvm/s_tir/dlight/analysis/gemv.py
@@ -18,6 +18,7 @@
import tvm_ffi
+import tvm
from tvm import s_tir, sym, tirx
from .common_analysis import (
@@ -48,13 +49,13 @@ def get_reduction_expr(block: s_tir.SBlock) -> tirx.Expr | None:
if len(block.body) != 1:
return None
tensor_store = block.body[0]
- if not isinstance(tensor_store, tirx.TensorStore):
+ if not isinstance(tensor_store, tvm.ir.TensorStore):
return None
if not isinstance(tensor_store.value, tirx.Add):
return None
if not tvm_ffi.structural_equal(
tensor_store.value.a,
- tirx.TensorLoad(tensor_store.buffer, tensor_store.indices),
+ tirx.TensorLoad(tensor_store.dest, tensor_store.indices),
map_free_vars=True,
):
return None
diff --git a/python/tvm/s_tir/dlight/base/utils.py b/python/tvm/s_tir/dlight/base/utils.py
index 424b00ae70..562d132d2b 100644
--- a/python/tvm/s_tir/dlight/base/utils.py
+++ b/python/tvm/s_tir/dlight/base/utils.py
@@ -17,6 +17,7 @@
# pylint: disable=missing-docstring
"""Utility methods for generic GPU."""
+import tvm
from tvm import DataType, s_tir, tirx
from tvm.ir import PrimType
from tvm.target import Target
@@ -31,7 +32,7 @@ def get_bytes(dtype: DataType | PrimType | str) -> int:
def get_extent(sch: s_tir.Schedule, loop_rv: s_tir.schedule.LoopRV):
- loop: tirx.For = sch.get(loop_rv)
+ loop: tvm.ir.For = sch.get(loop_rv)
return loop.extent.value if isinstance(loop.extent, tirx.IntImm) else loop.extent
@@ -68,7 +69,7 @@ def max_threads_per_block(target: Target) -> int:
def suggest_threads_per_block(
target: Target,
- loops: list[tirx.For],
+ loops: list[tvm.ir.For],
max_threads_for_dynamic_loop: int = 32,
) -> list[int]:
if target.kind.name == "cuda":
diff --git a/python/tvm/s_tir/dlight/gpu/fallback.py b/python/tvm/s_tir/dlight/gpu/fallback.py
index 8fd2c87755..a1dca979e6 100644
--- a/python/tvm/s_tir/dlight/gpu/fallback.py
+++ b/python/tvm/s_tir/dlight/gpu/fallback.py
@@ -19,6 +19,7 @@
import tvm_ffi
+import tvm
from tvm import s_tir, tirx
from tvm.target import Target
@@ -28,41 +29,41 @@ from ..base import try_inline
from .base import GPUScheduleRule
-def _has_internal_thread_env(stmt: tirx.Stmt) -> bool:
+def _has_internal_thread_env(stmt: tvm.ir.Stmt) -> bool:
"""Check whether a statement already launches GPU threads internally,
e.g. via `T.launch_thread` regions or nested
thread-bound loops. Such blocks manage their own thread environment
and must not be wrapped in an additional thread binding."""
found = False
- def visit_region(node: tirx.RegionStmt):
+ def visit_region(node: tvm.ir.RegionStmt):
nonlocal found
if node.op.name == "tirx.launch_thread":
found = True
- def visit_for(node: tirx.For):
+ def visit_for(node: tvm.ir.For):
nonlocal found
if node.annotations.get("thread_binding") is not None:
found = True
tvm_ffi.structural_walk(
stmt,
- [(tirx.RegionStmt, visit_region), (tirx.For, visit_for)],
+ [(tvm.ir.RegionStmt, visit_region), (tvm.ir.For, visit_for)],
order="post",
)
return found
-def _has_zero_extent_loop(stmt: tirx.Stmt) -> bool:
+def _has_zero_extent_loop(stmt: tvm.ir.Stmt) -> bool:
"""Check whether a statement contains a statically empty loop."""
found = False
- def visit_for(node: tirx.For):
+ def visit_for(node: tvm.ir.For):
nonlocal found
if isinstance(node.extent, tirx.IntImm) and node.extent.value == 0:
found = True
- tvm_ffi.structural_walk(stmt, (tirx.For, visit_for), order="post")
+ tvm_ffi.structural_walk(stmt, (tvm.ir.For, visit_for), order="post")
return found
diff --git a/python/tvm/s_tir/dlight/gpu/gemv.py b/python/tvm/s_tir/dlight/gpu/gemv.py
index 19c3408e8f..a7c130b1ed 100644
--- a/python/tvm/s_tir/dlight/gpu/gemv.py
+++ b/python/tvm/s_tir/dlight/gpu/gemv.py
@@ -19,6 +19,7 @@
from functools import reduce
+import tvm
from tvm import s_tir, tirx
from tvm.target import Target
@@ -185,7 +186,7 @@ class GEMV(GPUScheduleRule):
V_shared = sch.cache_read(rf, read_buffer_index=0, storage_scope="shared")
sch.compute_at(V_shared, tr, preserve_unit_loops=True)
l = sch.get_loops(block=V_shared)[-1]
- loop: tirx.For = sch.get(l)
+ loop: tvm.ir.For = sch.get(l)
if isinstance(loop.extent, tirx.IntImm):
# avoid introducing predicates when vector length is too large
vec_length = max(
diff --git a/python/tvm/s_tir/dlight/gpu/low_batch_gemv.py b/python/tvm/s_tir/dlight/gpu/low_batch_gemv.py
index a283152a48..8a341b831d 100644
--- a/python/tvm/s_tir/dlight/gpu/low_batch_gemv.py
+++ b/python/tvm/s_tir/dlight/gpu/low_batch_gemv.py
@@ -22,6 +22,7 @@ from typing import Literal
import tvm_ffi
+import tvm
from tvm import s_tir, sym, tirx
from tvm.target import Target
@@ -42,13 +43,13 @@ def _get_reduction_expr(block: s_tir.SBlock) -> tirx.Expr | None:
if len(block.body) != 1:
return None
tensor_store = block.body[0]
- if not isinstance(tensor_store, tirx.TensorStore):
+ if not isinstance(tensor_store, tvm.ir.TensorStore):
return None
if not isinstance(tensor_store.value, tirx.Add):
return None
if not tvm_ffi.structural_equal(
tensor_store.value.a,
- tirx.TensorLoad(tensor_store.buffer, tensor_store.indices),
+ tirx.TensorLoad(tensor_store.dest, tensor_store.indices),
map_free_vars=True,
):
return None
@@ -398,7 +399,7 @@ class LowBatchGEMV(GPUScheduleRule):
V_shared = sch.cache_read(rf, read_buffer_index=0, storage_scope="shared")
sch.compute_at(V_shared, tr, preserve_unit_loops=True)
l = sch.get_loops(block=V_shared)[-1]
- loop: tirx.For = sch.get(l)
+ loop: tvm.ir.For = sch.get(l)
if isinstance(loop.extent, tirx.IntImm):
# avoid introducing predicates when vector length is too large
vec_length = max(
diff --git a/python/tvm/s_tir/dlight/gpu/reduction.py b/python/tvm/s_tir/dlight/gpu/reduction.py
index 79ab81e574..32f50d4aae 100644
--- a/python/tvm/s_tir/dlight/gpu/reduction.py
+++ b/python/tvm/s_tir/dlight/gpu/reduction.py
@@ -21,6 +21,7 @@ from collections.abc import Mapping
import tvm_ffi
+import tvm
from tvm import s_tir, sym, tirx
from tvm.target import Target
@@ -39,13 +40,13 @@ def _get_reduction_expr(block: s_tir.SBlock) -> tirx.Expr | None:
if len(block.body) != 1:
return None
tensor_store = block.body[0]
- if not isinstance(tensor_store, tirx.TensorStore):
+ if not isinstance(tensor_store, tvm.ir.TensorStore):
return None
if not isinstance(tensor_store.value, tirx.Add):
return None
if not tvm_ffi.structural_equal(
tensor_store.value.a,
- tirx.TensorLoad(tensor_store.buffer, tensor_store.indices),
+ tirx.TensorLoad(tensor_store.dest, tensor_store.indices),
map_free_vars=True,
):
return None
diff --git a/python/tvm/s_tir/dlight/gpu/rmsnorm.py b/python/tvm/s_tir/dlight/gpu/rmsnorm.py
index 26fa956633..c2ce9acc26 100644
--- a/python/tvm/s_tir/dlight/gpu/rmsnorm.py
+++ b/python/tvm/s_tir/dlight/gpu/rmsnorm.py
@@ -19,10 +19,9 @@
import tvm
from tvm import tirx
-from tvm.ir import Call, TensorLoad
+from tvm.ir import Call, TensorLoad, TensorStore
from tvm.s_tir import SBlock
from tvm.target import Target
-from tvm.tirx import TensorStore
from tvm.tirx.expr import Cast
from ..base import ScheduleRule
diff --git a/python/tvm/s_tir/dlight/gpu/transpose.py b/python/tvm/s_tir/dlight/gpu/transpose.py
index 3e35c329db..4a71b5a1c0 100644
--- a/python/tvm/s_tir/dlight/gpu/transpose.py
+++ b/python/tvm/s_tir/dlight/gpu/transpose.py
@@ -16,6 +16,7 @@
# under the License.
"""Reduction rule for operators including softmax, layer norm, RMS norm, etc"""
+import tvm
from tvm import s_tir, sym, tirx
from tvm.ir import TensorLoad
from tvm.s_tir import Schedule
@@ -32,7 +33,7 @@ class Transpose(GPUScheduleRule):
def is_transpose(self, sch: Schedule, block_rv: SBlockRV):
block = sch.get(block_rv)
- if len(block.body) == 1 and isinstance(block.body[0], tirx.TensorStore):
+ if len(block.body) == 1 and isinstance(block.body[0], tvm.ir.TensorStore):
store = block.body[0]
rhs = store.value
if isinstance(rhs, TensorLoad):
diff --git a/python/tvm/s_tir/meta_schedule/relax_integration.py b/python/tvm/s_tir/meta_schedule/relax_integration.py
index 051f63476b..c5b7d9a0a7 100644
--- a/python/tvm/s_tir/meta_schedule/relax_integration.py
+++ b/python/tvm/s_tir/meta_schedule/relax_integration.py
@@ -19,6 +19,8 @@
import warnings
from typing import TYPE_CHECKING, Union
+import tvm
+
# isort: off
from typing import Literal
@@ -410,7 +412,6 @@ def compile_relax(
The built runtime module or vm VMExecutable for the given relax workload.
"""
# pylint: disable=import-outside-toplevel
- import tvm
from tvm import relax
from tvm.relax import build as relax_build
from tvm.relax import pipeline as relax_pipeline_mod
diff --git a/python/tvm/s_tir/sblock_scope.py b/python/tvm/s_tir/sblock_scope.py
index 1b37b8c288..a75c750c08 100644
--- a/python/tvm/s_tir/sblock_scope.py
+++ b/python/tvm/s_tir/sblock_scope.py
@@ -21,9 +21,9 @@ from typing import Optional
from tvm_ffi import register_object
+from tvm.ir import For
from tvm.runtime import Object
from tvm.s_tir import SBlock
-from tvm.tirx import For
from . import _ffi_api
diff --git a/python/tvm/s_tir/schedule/analysis.py b/python/tvm/s_tir/schedule/analysis.py
index 61b23f27c1..33239a27f3 100644
--- a/python/tvm/s_tir/schedule/analysis.py
+++ b/python/tvm/s_tir/schedule/analysis.py
@@ -18,11 +18,10 @@
import tvm_ffi
-from tvm.ir import Var
+from tvm.ir import For, Var
from tvm.runtime import Object
from tvm.tirx.expr import Expr
from tvm.tirx.function import Function, IndexMap
-from tvm.tirx.stmt import For
from . import _ffi_api
from .schedule import SBlockRV, Schedule
diff --git a/python/tvm/s_tir/schedule/schedule.py b/python/tvm/s_tir/schedule/schedule.py
index 785a7f9f94..f59d15bbe3 100644
--- a/python/tvm/s_tir/schedule/schedule.py
+++ b/python/tvm/s_tir/schedule/schedule.py
@@ -23,10 +23,10 @@ from typing import Literal
from tvm_ffi import register_object as _register_object
from tvm.error import register_error
-from tvm.ir import Expr, GlobalVar, IRModule, is_prim_expr
+from tvm.ir import Expr, For, GlobalVar, IRModule, is_prim_expr
from tvm.runtime import DataTypeCode, Object
from tvm.s_tir import SBlock
-from tvm.tirx import FloatImm, For, Function, IntImm, Var, is_tensor_var
+from tvm.tirx import FloatImm, Function, IntImm, Var, is_tensor_var
from tvm.tirx.function import IndexMap
from . import _ffi_api
diff --git a/python/tvm/s_tir/schedule/state.py b/python/tvm/s_tir/schedule/state.py
index b31b76e04f..b8c0085a14 100644
--- a/python/tvm/s_tir/schedule/state.py
+++ b/python/tvm/s_tir/schedule/state.py
@@ -22,10 +22,10 @@ from enum import IntEnum
from tvm_ffi import register_object
-from tvm.ir import IRModule
+from tvm.ir import For, IRModule
from tvm.runtime import Object
from tvm.s_tir import SBlock, SBlockRealize
-from tvm.tirx import For, Function
+from tvm.tirx import Function
from ..sblock_scope import SBlockScope, StmtSRef
from . import _ffi_api
diff --git a/python/tvm/s_tir/stmt.py b/python/tvm/s_tir/stmt.py
index 8ff0317d5f..e0c1e0c51b 100644
--- a/python/tvm/s_tir/stmt.py
+++ b/python/tvm/s_tir/stmt.py
@@ -20,10 +20,9 @@ from collections.abc import Mapping, Sequence
import tvm_ffi
-from tvm.ir import Expr, Scriptable, Span, TensorRegion, Var, const
+from tvm.ir import Expr, Scriptable, SeqStmt, Span, Stmt, TensorRegion, Var, const
from tvm.runtime import Object
from tvm.s_tir import IterVar
-from tvm.tirx.stmt import SeqStmt, Stmt
from . import _ffi_api
diff --git a/python/tvm/s_tir/transform/transform.py b/python/tvm/s_tir/transform/transform.py
index 92b1d34911..8bf9ff3d62 100644
--- a/python/tvm/s_tir/transform/transform.py
+++ b/python/tvm/s_tir/transform/transform.py
@@ -56,7 +56,7 @@ def LowerCrossThreadReduction():
def LowerInitBlock():
- """Lower block init stmt into IfThenElse statements.
+ """Lower block init stmt into If statements.
Returns
-------
@@ -258,7 +258,7 @@ def InjectVirtualThread():
def HoistIfThenElse(variant=None):
- """Hoist loop-invariant IfThenElse nodes to outside the eligible loops.
+ """Hoist loop-invariant If nodes to outside the eligible loops.
Parameters
----------
diff --git a/python/tvm/script/printer/doc.py b/python/tvm/script/printer/doc.py
index 45f55df36d..e9620a652e 100644
--- a/python/tvm/script/printer/doc.py
+++ b/python/tvm/script/printer/doc.py
@@ -259,7 +259,7 @@ class OperationKind(IntEnum):
_BinaryEnd = 27
_SpecialStart = 28
- IfThenElse = 29
+ If = 29
_SpecialEnd = 30
# pylint: enable=invalid-name
diff --git a/python/tvm/te/operation.py b/python/tvm/te/operation.py
index 4aa963f0f5..67b6857754 100644
--- a/python/tvm/te/operation.py
+++ b/python/tvm/te/operation.py
@@ -241,7 +241,7 @@ def extern(
**Returns**
- - **stmt** (:any:`tvm.tirx.Stmt` or a sequence of statements) -
+ - **stmt** (:any:`tvm.ir.Stmt` or a sequence of statements) -
The body that carries out array computation.
name: str, optional
@@ -341,8 +341,8 @@ def extern(
)
body = fcompute(input_placeholders, output_placeholders)
if tvm.ir.is_prim_expr(body):
- body = tvm.tirx.Evaluate(body)
- if not isinstance(body, tvm.tirx.Stmt | list | tuple | Array):
+ body = tvm.ir.Evaluate(body)
+ if not isinstance(body, tvm.ir.Stmt | list | tuple | Array):
raise ValueError(
f"Function '{fcompute.__name__}' should return Expr, Stmt, or a sequence of Stmt, "
"but it returned "
diff --git a/python/tvm/te/tensor.py b/python/tvm/te/tensor.py
index cbd9e90e8e..db9fe3355f 100644
--- a/python/tvm/te/tensor.py
+++ b/python/tvm/te/tensor.py
@@ -19,10 +19,9 @@
# pylint: disable=invalid-name
import tvm_ffi
-from tvm.ir import OpaqueExpr, const, is_prim_expr
+from tvm.ir import OpaqueExpr, SeqStmt, const, is_prim_expr
from tvm.runtime import Object, ObjectConvertible
from tvm.tirx import expr as _expr
-from tvm.tirx.stmt import SeqStmt
from . import _ffi_api, _te_tensor_overload
diff --git a/python/tvm/tirx/__init__.py b/python/tvm/tirx/__init__.py
index a7d9c2a4c8..274f799908 100644
--- a/python/tvm/tirx/__init__.py
+++ b/python/tvm/tirx/__init__.py
@@ -43,15 +43,7 @@ from .expr import Min, Max, EQ, NE, LT, LE, GT, GE, And, Or, Not
from .expr import Select, TensorLoad, Ramp, Broadcast, Shuffle
from .expr import CallEffectKind, Let
-from .stmt import Stmt, Bind, AssertStmt, ForKind, For, While, Return, Break, Continue
-# Legacy alias: LetStmt was folded into the body-less Bind statement.
-LetStmt = Bind
-
-from .stmt import TensorStore, RegionStmt
-
-from .stmt import SeqStmt
-from .stmt import IfThenElse, Evaluate, stmt_seq, stmt_list
from .stmt import BufferRegion
from .stmt import ScopeIdDefStmt
from .tile_primitive import DispatchContext
diff --git a/python/tvm/tirx/analysis/analysis.py b/python/tvm/tirx/analysis/analysis.py
index dbdfa51e3c..6debad5c38 100644
--- a/python/tvm/tirx/analysis/analysis.py
+++ b/python/tvm/tirx/analysis/analysis.py
@@ -18,11 +18,9 @@
# pylint: disable=invalid-name
-from tvm.ir import IRModule
+from tvm.ir import Expr, IRModule, Stmt
from tvm.tirx.expr import Var
-from tvm.tirx.stmt import Expr
-from .. import Stmt
from ..function import Function
from . import _ffi_api
diff --git a/python/tvm/tirx/function.py b/python/tvm/tirx/function.py
index 5761a85fcc..18721e6dba 100644
--- a/python/tvm/tirx/function.py
+++ b/python/tvm/tirx/function.py
@@ -26,13 +26,12 @@ import tvm_ffi
import tvm
import tvm.runtime
-from tvm.ir import BaseFunc, Range, Scriptable, Var
+from tvm.ir import BaseFunc, Range, Scriptable, SeqStmt, Stmt, Var
from tvm.runtime import Object
from ..runtime._tensor import Tensor
from . import _ffi_api
from .expr import Expr
-from .stmt import SeqStmt, Stmt
@tvm_ffi.register_object("tirx.Function")
diff --git a/python/tvm/tirx/script/ir_builder/ir.py b/python/tvm/tirx/script/ir_builder/ir.py
index b2b97e4f77..a30dfaa91a 100644
--- a/python/tvm/tirx/script/ir_builder/ir.py
+++ b/python/tvm/tirx/script/ir_builder/ir.py
@@ -24,6 +24,7 @@ from collections.abc import Callable
from numbers import Integral
from typing import TYPE_CHECKING, Any, TypeVar
+import tvm
from tvm import ir as _ir
from tvm.script.ir_builder.base import annotation_constructor as _annotation_constructor
from tvm.script.ir_builder.base import at as _at
@@ -555,7 +556,7 @@ def alloc_tensor(
attrs=ir.DictAttrs(norm_annotations),
ty=buf.ty,
)
- _ffi_api.AddToParent(tir.Bind(buf, allocation))
+ _ffi_api.AddToParent(tvm.ir.Bind(buf, allocation))
return buf
diff --git a/python/tvm/tirx/script/ir_builder/parser_protocol.py b/python/tvm/tirx/script/ir_builder/parser_protocol.py
index 66e90f6132..949734d98a 100644
--- a/python/tvm/tirx/script/ir_builder/parser_protocol.py
+++ b/python/tvm/tirx/script/ir_builder/parser_protocol.py
@@ -29,6 +29,8 @@ from collections.abc import Sequence
from functools import partial as _partial
from typing import Any
+import tvm
+
# isort: off
# isort: on
from tvm import ir as _ir
@@ -309,20 +311,20 @@ def decl_mutable_cell_(
def set_mutable_cell_(
target: _ir.TensorLoad | _ir.Var, value: Any, *, span: _Span = None
-) -> _base.AlreadyEmitted[_tir.Stmt]:
+) -> _base.AlreadyEmitted[tvm.ir.Stmt]:
"""Implements :func:`tvm.script.ir_builder.parser_protocol.set_mutable_cell_`.
Updates emit a scalar buffer store. Targets must denote scalar storage.
"""
if isinstance(target, _ir.TensorLoad):
- return _base.at_(span, tensor_store(target.source, value, list(target.indices)))
+ return _base.at_(span, tensor_store(target.source, list(target.indices), value))
elif (
_tir.is_tensor_var(target)
and len(target.ty.shape) == 1
and isinstance(target.ty.shape[0], _tir.IntImm)
and target.ty.shape[0].value == 1
):
- return _base.at_(span, tensor_store(target, value, [0]))
+ return _base.at_(span, tensor_store(target, [0], value))
else:
raise TypeError("A mutable assignment requires scalar storage")
@@ -362,7 +364,7 @@ def emit_(value: Any, *, span: _Span = None) -> None:
elif hasattr(value, "frames"):
for frame in value.frames:
_enter_concise(_base.at_(span, frame))
- elif isinstance(value, _tir.Stmt):
+ elif isinstance(value, tvm.ir.Stmt):
add_to_parent(_base.at_(span, value))
else:
# Native conversion owns Python literals; annotate the exact expression
@@ -374,14 +376,14 @@ def emit_(value: Any, *, span: _Span = None) -> None:
def setitem_(
target: Any, key: Any, value: Any, *, span: _Span = None
-) -> _base.AlreadyEmitted[_tir.Stmt]:
+) -> _base.AlreadyEmitted[tvm.ir.Stmt]:
"""Implements :func:`tvm.script.ir_builder.parser_protocol.setitem_`."""
- return _base.at_(span, tensor_store(target, value, key))
+ return _base.at_(span, tensor_store(target, key, value))
def setattr_(
target: Any, name: str, value: Any, *, span: _Span = None
-) -> _base.AlreadyEmitted[_tir.Stmt] | None:
+) -> _base.AlreadyEmitted[tvm.ir.Stmt] | None:
"""Implements :func:`tvm.script.ir_builder.parser_protocol.setattr_`."""
previous = getattr(target, name, _base.MISSING)
buffer = previous.source if isinstance(previous, _ir.TensorLoad) else previous
@@ -428,23 +430,23 @@ def bind( # pylint: disable=invalid-name
def tensor_store(
- buffer: Var, # pylint: disable=redefined-outer-name
- value: Expr,
+ dest: Expr,
indices: list[Expr | slice],
-) -> AlreadyEmitted[_tir.Stmt]:
+ value: Expr,
+) -> AlreadyEmitted[tvm.ir.Stmt]:
"""Emit a buffer store and return a receipt for the stored statement.
Parameters
----------
- buffer : Var
- The buffer.
-
- value : Expr
- The value to be stored.
+ dest : Expr
+ The destination expression.
indices : List[Union[Expr, slice]]
The indices location to be stored.
+ value : Expr
+ The value to be stored.
+
Returns
-------
result : AlreadyEmitted[Stmt]
@@ -469,12 +471,12 @@ def tensor_store(
expr_indices.append(_op.ramp(index.start, step, lanes))
else:
expr_indices.append(index)
- if isinstance(value, bool) and buffer.ty.dtype == "bool":
+ if isinstance(value, bool) and dest.ty.dtype == "bool":
value = IntImm("bool", value)
- return AlreadyEmitted(_ffi_api.TensorStore(buffer, value, expr_indices))
+ return AlreadyEmitted(_ffi_api.TensorStore(dest, expr_indices, value))
-def evaluate(value: Expr) -> AlreadyEmitted[_tir.Stmt]:
+def evaluate(value: Expr) -> AlreadyEmitted[tvm.ir.Stmt]:
"""Emit an evaluation and return a reference to its stored statement.
Parameters
@@ -500,7 +502,7 @@ def evaluate(value: Expr) -> AlreadyEmitted[_tir.Stmt]:
return AlreadyEmitted(_ffi_api.Evaluate(value)) # type: ignore[attr-defined] # pylint: disable=no-member
-def add_to_parent(stmt: _tir.Stmt) -> None:
+def add_to_parent(stmt: tvm.ir.Stmt) -> None:
"""Add a statement to the parent frame."""
_ffi_api.AddToParent(stmt) # type: ignore[attr-defined] # pylint: disable=no-member
@@ -571,7 +573,7 @@ def range_(*args: Any, annotations: dict[str, Any] | None = None) -> frame.ForFr
return serial(args[0], args[1], step=args[2], annotations=annotations)
-def break_(*, span: _Span = None) -> _base.AlreadyEmitted[_tir.Stmt]:
+def break_(*, span: _Span = None) -> _base.AlreadyEmitted[tvm.ir.Stmt]:
"""Implements :func:`tvm.script.ir_builder.parser_protocol.break_`.
Legality is checked on the completed function, across loop and function boundaries.
@@ -579,7 +581,7 @@ def break_(*, span: _Span = None) -> _base.AlreadyEmitted[_tir.Stmt]:
return _base.with_at_group_(span, lambda: _base.AlreadyEmitted(_ffi_api.Break()))
-def continue_(*, span: _Span = None) -> _base.AlreadyEmitted[_tir.Stmt]:
+def continue_(*, span: _Span = None) -> _base.AlreadyEmitted[tvm.ir.Stmt]:
"""Implements :func:`tvm.script.ir_builder.parser_protocol.continue_`.
Legality is checked on the completed function, across loop and function boundaries.
@@ -587,7 +589,7 @@ def continue_(*, span: _Span = None) -> _base.AlreadyEmitted[_tir.Stmt]:
return _base.with_at_group_(span, lambda: _base.AlreadyEmitted(_ffi_api.Continue()))
-def return_(value: Any = None, *, span: _Span = None) -> _base.AlreadyEmitted[_tir.Stmt]:
+def return_(value: Any = None, *, span: _Span = None) -> _base.AlreadyEmitted[tvm.ir.Stmt]:
"""Implements :func:`tvm.script.ir_builder.parser_protocol.return_`."""
if value is None:
raise TypeError("A primitive function return requires an expression")
@@ -916,7 +918,7 @@ def region(
Operands and attributes belong to the enclosing scope. Entering the frame
returns one parameter directly, or a sequence for zero or multiple parameters.
- Result variables are supported by direct ``tirx.RegionStmt`` construction;
+ Result variables are supported by direct ``tvm.ir.RegionStmt`` construction;
outward-result script syntax is not supported.
"""
if isinstance(op, str):
diff --git a/python/tvm/tirx/stmt.py b/python/tvm/tirx/stmt.py
index f8132b8d9a..8b9845c0dd 100644
--- a/python/tvm/tirx/stmt.py
+++ b/python/tvm/tirx/stmt.py
@@ -14,38 +14,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""Statement AST Node in TVM.
+"""TIRx-specific statement nodes and buffer-region construction."""
-Each statement node have subfields that can be visited from python side.
-
-.. code-block:: python
-
- x = tvm.tirx.Var("n", "int32")
- buffer = tvm.tirx.decl_tensor((16,), "float32")
- st = tvm.tirx.stmt.TensorStore(buffer, 1, (x,))
- assert isinstance(st, tvm.tirx.stmt.TensorStore)
- assert(st.buffer == buffer)
-"""
-
-from collections.abc import Mapping, Sequence
-from enum import IntEnum
from typing import TYPE_CHECKING, Any, ClassVar
import tvm_ffi
-from tvm.ir import (
- DictAttrs,
- Expr,
- Op,
- Range,
- Scriptable,
- Span,
- StringImm,
- TensorRegion,
- Var,
- make_node,
-)
-from tvm.runtime import Object
+from tvm import ir as _ir
+from tvm.ir import Expr, Op, Range, Span, TensorRegion, Var
from . import _ffi_api
from .exec_scope import ExecScope, ScopeIdDef
@@ -54,368 +30,6 @@ if TYPE_CHECKING:
from .tile_primitive import DispatchContext
-@tvm_ffi.register_object("tirx.Stmt")
-class Stmt(Object, Scriptable):
- """Base class of all the statements."""
-
-
-@tvm_ffi.register_object("tirx.Bind")
-class Bind(Stmt):
- """Bind node.
-
- Bind a variable to a value in the enclosing scope.
- Bind has no body field.
- The bound variable is visible in all subsequent statements
- within the same enclosing scope (SeqStmt, ForNode.body, etc.).
-
- Parameters
- ----------
- var : Var
- The variable in the binding.
-
- value : Expr
- The value to be bound.
-
- span : Optional[Span]
- The location of the stmt in the source code.
- """
-
- var: Var
- value: Expr
- span: Span | None
-
- def __init__(self, var: Var, value: Expr, span: Span | None = None) -> None:
- self.__init_handle_by_constructor__(
- _ffi_api.Bind,
- var,
- value,
- span, # type: ignore
- )
-
-
-@tvm_ffi.register_object("tirx.AssertStmt")
-class AssertStmt(Stmt):
- """AssertStmt node.
-
- Parameters
- ----------
- kind : StringImm
- The error kind, e.g. "RuntimeError", "TypeError", "ValueError".
-
- condition : Expr
- The assert condition.
-
- message_parts : list[StringImm]
- Error message fragments, concatenated at runtime when assertion fails.
-
- span : Span | None
- The location of the stmt in the source code.
- """
-
- kind: StringImm
- condition: Expr
- message_parts: list
- span: Span | None
-
- def __init__(
- self,
- kind: StringImm,
- condition: Expr,
- message_parts: list | None = None,
- span: Span | None = None,
- ) -> None:
- if message_parts is None:
- message_parts = []
- self.__init_handle_by_constructor__(
- _ffi_api.AssertStmt,
- kind,
- condition,
- message_parts,
- span, # type: ignore
- )
-
-
-class ForKind(IntEnum):
- """The kind of the for loop.
-
- note
- ----
- ForKind can change the control flow semantics
- of the loop and need to be considered in all TIR passes.
- """
-
- DEFAULT = 0 # Ordinary loop with sequential iteration semantics.
- PARALLEL = 1 # Parallel execution, optionally with explicit thread placement.
- VECTORIZED = 2
- UNROLLED = 3
-
-
-@tvm_ffi.register_object("tirx.For")
-class For(Stmt):
- """For node.
-
- Parameters
- ----------
- loop_var : Var
- The loop variable.
-
- min : Expr
- The beginning value.
-
- extent : Expr
- The length of the loop.
-
- kind : ForKind
- The type of the for.
-
- body : Stmt | Sequence[Stmt]
- The body statement.
-
- annotations: Optional[Mapping[str, Object]]
- Additional loop annotations. For parallel loops, the optional
- ``thread_binding`` entry is a string naming the bound thread.
- Parallel loops without this entry use CPU parallel execution.
-
- step : Expr
- The loop step. Defaults to None, which represents one.
-
- span : Optional[Span]
- The location of the stmt in the source code.
- """
-
- loop_var: Var
- min: Expr
- extent: Expr
- kind: ForKind
- body: "SeqStmt"
- annotations: Mapping[str, Object]
- step: Expr | None
- span: Span | None
-
- def __init__(
- self,
- loop_var: Var,
- min: Expr, # pylint: disable=redefined-builtin
- extent: Expr,
- kind: ForKind,
- body: Stmt | Sequence[Stmt],
- annotations: Mapping[str, Object] | None = None,
- step: Expr | None = None,
- span: Span | None = None,
- ) -> None:
- self.__init_handle_by_constructor__(
- _ffi_api.For, # type: ignore
- loop_var,
- min,
- extent,
- kind,
- body,
- annotations,
- step,
- span,
- )
-
-
-@tvm_ffi.register_object("tirx.While")
-class While(Stmt):
- """While node.
-
- Parameters
- ----------
- condition : Expr
- The termination condition.
-
- body : Stmt | Sequence[Stmt]
- The body statement.
-
- span : Optional[Span]
- The location of the stmt in the source code.
- """
-
- condition: Expr
- body: "SeqStmt"
- span: Span | None
-
- def __init__(
- self, condition: Expr, body: Stmt | Sequence[Stmt], span: Span | None = None
- ) -> None:
- self.__init_handle_by_constructor__(_ffi_api.While, condition, body, span) # type: ignore
-
-
-@tvm_ffi.register_object("tirx.TensorStore")
-class TensorStore(Stmt):
- """Store a value into a tensor variable.
-
- Parameters
- ----------
- buffer : Var
- The buffer.
-
- value : Expr
- The value we to be stored.
-
- indices : List[Expr]
- The indices location to be stored.
-
- span : Optional[Span]
- The location of the stmt in the source code.
- """
-
- buffer: Var
- value: Expr
- indices: list[Expr]
- span: Span | None
-
- def __init__(
- self,
- buffer: Var,
- value: Expr,
- indices: list[Expr],
- span: Span | None = None,
- ) -> None:
- self.__init_handle_by_constructor__(
- _ffi_api.TensorStore,
- buffer,
- value,
- indices,
- span, # type: ignore
- )
-
-
-@tvm_ffi.register_object("tirx.RegionStmt")
-class RegionStmt(Stmt):
- """An operation with enclosing-scope operands and one lexical body.
-
- ``body_params`` define variables visible only within ``body``. Their count
- and types must match the operation's required ``FRegionGetBodyParams`` hook.
- Zero-parameter regions register an empty-return hook; operations without a
- hook do not support region construction. Explicit parameter
- identities and their references in ``body`` are preserved. ``result_vars``
- define variables after the region in the enclosing sequence. Attributes are
- evaluated outside the body-parameter scope. Direct construction and JSON
- serialization support result variables; structured script syntax currently
- supports only result-free regions.
- """
-
- op: Op
- args: list[Expr]
- body_params: list[Var]
- attrs: DictAttrs
- body: "SeqStmt"
- result_vars: list[Var]
- span: Span | None
-
- def __init__(
- self,
- op: Op | str,
- args: Sequence[Expr],
- body_params: Sequence[Var],
- attrs: DictAttrs | Mapping[str, Any] | None,
- body: Stmt | Sequence[Stmt],
- result_vars: Sequence[Var] | None = None,
- span: Span | None = None,
- ) -> None:
- if isinstance(op, str):
- op = Op.get(op)
- if attrs is None or isinstance(attrs, Mapping):
- attrs = make_node("ir.DictAttrs", **(attrs or {}))
- self.__init_handle_by_constructor__(
- _ffi_api.RegionStmt,
- op,
- args,
- body_params,
- attrs,
- body,
- [] if result_vars is None else result_vars,
- span,
- )
-
-
-@tvm_ffi.register_object("tirx.SeqStmt")
-class SeqStmt(Stmt):
- """Sequence of statements.
-
- Parameters
- ----------
- seq : Stmt | Sequence[Stmt]
- The statements, flattened into one sequence. Empty and singleton sequences are valid.
-
- span : Optional[Span]
- The location of the stmt in the source code.
- """
-
- seq: list[Stmt]
- span: Span | None
-
- def __init__(self, seq: Stmt | Sequence[Stmt], span: Span | None = None) -> None:
- self.__init_handle_by_constructor__(_ffi_api.SeqStmt, seq, span) # type: ignore
-
- def __getitem__(self, i: int):
- return self.seq[i]
-
- def __len__(self):
- return len(self.seq)
-
-
-@tvm_ffi.register_object("tirx.IfThenElse")
-class IfThenElse(Stmt):
- """IfThenElse node.
-
- Parameters
- ----------
- condition : Expr
- The expression
-
- then_case : Stmt | Sequence[Stmt]
- The statement to execute if condition is true.
-
- else_case : Stmt | Sequence[Stmt] | None
- The statement to execute if condition is false.
-
- span : Optional[Span]
- The location of the stmt in the source code.
- """
-
- condition: Expr
- then_case: SeqStmt
- else_case: SeqStmt | None
-
- def __init__(
- self,
- condition: Expr,
- then_case: Stmt | Sequence[Stmt],
- else_case: Stmt | Sequence[Stmt] | None,
- span: Span | None = None,
- ) -> None:
- self.__init_handle_by_constructor__(
- _ffi_api.IfThenElse,
- condition,
- then_case,
- else_case,
- span, # type: ignore
- )
-
-
-@tvm_ffi.register_object("tirx.Evaluate")
-class Evaluate(Stmt):
- """Evaluate node.
-
- Parameters
- ----------
- value : Expr
- The expression to be evaluated.
-
- span : Optional[Span]
- The location of the stmt in the source code.
- """
-
- value: Expr
- span: Span | None
-
- def __init__(self, value: Expr, span: Span | None = None) -> None:
- self.__init_handle_by_constructor__(_ffi_api.Evaluate, value, span) # type: ignore
-
-
def BufferRegion(buffer: Var, region: list[Range]) -> TensorRegion:
"""Construct a buffer-backed tensor region with TIRX subscript semantics.
@@ -431,7 +45,7 @@ def BufferRegion(buffer: Var, region: list[Range]) -> TensorRegion:
@tvm_ffi.register_object("tirx.ScopeIdDefStmt")
-class ScopeIdDefStmt(Stmt):
+class ScopeIdDefStmt(_ir.Stmt):
"""ScopeIdDefStmt node.
Leaf statement that introduces scope-identifier vars
@@ -463,91 +77,8 @@ class ScopeIdDefStmt(Stmt):
) # type: ignore
-@tvm_ffi.register_object("tirx.Break")
-class Break(Stmt):
- """Break node.
-
- Parameters
- ----------
- """
-
- def __init__(self, span: Span | None = None) -> None:
- self.__init_handle_by_constructor__(_ffi_api.Break, span) # type: ignore
-
-
-@tvm_ffi.register_object("tirx.Return")
-class Return(Stmt):
- """Return node.
-
- Parameters
- ----------
- value : Expr
- The value to return.
-
- span : Optional[Span]
- The location of this statement in the source code.
- """
-
- value: Expr
- span: Span | None
-
- def __init__(self, value: Expr, span: Span | None = None) -> None:
- self.__init_handle_by_constructor__(_ffi_api.Return, value, span) # type: ignore
-
-
-@tvm_ffi.register_object("tirx.Continue")
-class Continue(Stmt):
- """Continue node.
-
- Parameters
- ----------
- """
-
- def __init__(self, span: Span | None = None) -> None:
- self.__init_handle_by_constructor__(_ffi_api.Continue, span) # type: ignore
-
-
-def stmt_seq(*args: Expr | Stmt) -> SeqStmt:
- """Make sequence of statements
-
- Parameters
- ----------
- *args : Union[Expr, Stmt]
- List of statements to be combined as sequence.
-
- Returns
- -------
- stmt : SeqStmt
- The combined sequence.
- """
- ret = []
- for value in args:
- if not isinstance(value, Stmt):
- value = Evaluate(value)
- ret.append(value)
- return SeqStmt(ret)
-
-
-def stmt_list(stmt: Stmt) -> list[Stmt]:
- """Make list of stmt from blocks.
-
- Parameters
- ----------
- stmt : Stmt
- The input statement.
-
- Returns
- -------
- stmt_list : List[Stmt]
- The unpacked list of statements
- """
- if isinstance(stmt, SeqStmt):
- return list(stmt.seq)
- return [stmt]
-
-
@tvm_ffi.register_object("tirx.TileOpCall")
-class TileOpCall(Stmt):
+class TileOpCall(_ir.Stmt):
"""TileOpCall node.
Parameters
@@ -671,14 +202,14 @@ class TileOpCall(Stmt):
raise NotImplementedError("Subclass must implement this method")
def get_private_buffers(
- self, buffer_dict: dict[Any, tuple[Var, Stmt | None]], sctx: "DispatchContext"
+ self, buffer_dict: dict[Any, tuple[Var, _ir.Stmt | None]], sctx: "DispatchContext"
) -> dict[str, Any]:
"""
Create private (intermediate) buffers needed in this operator.
Parameters
----------
- buffer_dict: Dict[Any, Tuple[Var, Optional[Stmt]]]
+ buffer_dict: Dict[Any, Tuple[Var, Optional[tvm.ir.Stmt]]]
A dictionary containing private buffers (and their init stmts) in other operators.
Key can be anything to reference the buffer.
This is used to reuse private buffers in other operators (like identity tensor etc.).
@@ -706,12 +237,12 @@ class TileOpCall(Stmt):
raise ValueError(f"Unsupported target: {sctx.target.kind.name}")
def get_private_buffers_trn(
- self, buffer_dict: dict[Any, tuple[Var, Stmt | None]], sctx: "DispatchContext"
+ self, buffer_dict: dict[Any, tuple[Var, _ir.Stmt | None]], sctx: "DispatchContext"
) -> dict[str, Any]:
return {}
def get_private_buffers_cuda(
- self, buffer_dict: dict[Any, tuple[Var, Stmt | None]], sctx: "DispatchContext"
+ self, buffer_dict: dict[Any, tuple[Var, _ir.Stmt | None]], sctx: "DispatchContext"
) -> dict[str, Any]:
return {}
diff --git a/python/tvm/tirx/tile_primitive.py b/python/tvm/tirx/tile_primitive.py
index 80f916cede..af1b07b41e 100644
--- a/python/tvm/tirx/tile_primitive.py
+++ b/python/tvm/tirx/tile_primitive.py
@@ -22,13 +22,12 @@ Mirrors the C++ header ``include/tvm/tirx/tile_dispatch.h``.
from tvm_ffi import register_object
-from tvm.ir import Expr, Range, Scriptable, Var
+from tvm.ir import Expr, Range, Scriptable, Stmt, Var
from tvm.runtime import Object
from tvm.target import Target
from . import _ffi_api
from .exec_scope import ExecScope
-from .stmt import Stmt
@register_object("tirx.DispatchContext")
diff --git a/python/tvm/tirx/transform/common.py b/python/tvm/tirx/transform/common.py
index 21be39a126..238de848d1 100644
--- a/python/tvm/tirx/transform/common.py
+++ b/python/tvm/tirx/transform/common.py
@@ -18,8 +18,8 @@
import tvm_ffi
-from tvm.ir import Call, Op, Var, is_prim_expr
-from tvm.tirx import Evaluate, Expr, Stmt, TileOpCall, decl_tensor, is_tensor_var
+from tvm.ir import Call, Evaluate, Op, Stmt, Var, is_prim_expr
+from tvm.tirx import Expr, TileOpCall, decl_tensor, is_tensor_var
from tvm.tirx.layout import Iter, TileLayout
diff --git a/python/tvm/topi/gpu/scan.py b/python/tvm/topi/gpu/scan.py
index 5cda86f64e..b62c9499a0 100644
--- a/python/tvm/topi/gpu/scan.py
+++ b/python/tvm/topi/gpu/scan.py
@@ -95,8 +95,8 @@ def exclusive_scan_ir(data, output, reduction=None, binop=operator.add, identity
if reduction is not None:
T.tensor_store(
reduction,
- cast(identity_value, out_dtype),
T.tensor_indices(reduction, bx),
+ cast(identity_value, out_dtype),
)
with T.else_():
nthread_tx = max_threads
@@ -117,11 +117,11 @@ def exclusive_scan_ir(data, output, reduction=None, binop=operator.add, identity
with T.then_():
T.tensor_store(
output,
+ T.tensor_indices(output, batch * scan_axis_size + tid),
cast(
data[T.tensor_indices(data, batch * scan_axis_size + tid)],
out_dtype,
),
- T.tensor_indices(output, batch * scan_axis_size + tid),
)
# The following algorithm performs parallel exclusive scan
@@ -146,25 +146,33 @@ def exclusive_scan_ir(data, output, reduction=None, binop=operator.add, identity
start = start_buf
middle = middle_buf
end = end_buf
- T.tensor_store(start, width * tid, T.tensor_indices(start, 0))
+ T.tensor_store(start, T.tensor_indices(start, 0), width * tid)
with T.if_(start[T.tensor_indices(start, 0)] < scan_axis_size):
with T.then_():
T.tensor_store(
middle,
- start[T.tensor_indices(start, 0)] + tvm.tirx.indexdiv(width, 2),
T.tensor_indices(middle, 0),
+ start[T.tensor_indices(start, 0)] + tvm.tirx.indexdiv(width, 2),
)
T.tensor_store(
end,
+ T.tensor_indices(end, 0),
tvm.te.min(
start[T.tensor_indices(start, 0)] + width, scan_axis_size
),
- T.tensor_indices(end, 0),
)
with T.if_(middle[T.tensor_indices(middle, 0)] < scan_axis_size):
with T.then_():
T.tensor_store(
output,
+ T.tensor_indices(
+ output,
+ (
+ batch * scan_axis_size
+ + end[T.tensor_indices(end, 0)]
+ - 1
+ ),
+ ),
binop(
output[
T.tensor_indices(
@@ -187,14 +195,6 @@ def exclusive_scan_ir(data, output, reduction=None, binop=operator.add, identity
)
],
),
- T.tensor_indices(
- output,
- (
- batch * scan_axis_size
- + end[T.tensor_indices(end, 0)]
- - 1
- ),
- ),
)
# Down Sweep of exclusive scan
@@ -204,15 +204,15 @@ def exclusive_scan_ir(data, output, reduction=None, binop=operator.add, identity
if reduction is not None:
T.tensor_store(
reduction,
+ T.tensor_indices(reduction, bx),
output[
T.tensor_indices(output, ((bx + 1) * scan_axis_size - 1))
],
- T.tensor_indices(reduction, bx),
)
T.tensor_store(
output,
- cast(identity_value, out_dtype),
T.tensor_indices(output, ((bx + 1) * scan_axis_size - 1)),
+ cast(identity_value, out_dtype),
)
with T.serial(0, cast(lim, "int32")) as l2_width:
@@ -235,27 +235,28 @@ def exclusive_scan_ir(data, output, reduction=None, binop=operator.add, identity
middle = middle_buf
end = end_buf
tmp = tmp_buf
- T.tensor_store(start, width * tid, T.tensor_indices(start, 0))
+ T.tensor_store(start, T.tensor_indices(start, 0), width * tid)
with T.if_(
tvm.tirx.all(start[T.tensor_indices(start, 0)] < scan_axis_size)
):
with T.then_():
T.tensor_store(
middle,
- start[T.tensor_indices(start, 0)] + tvm.tirx.indexdiv(width, 2),
T.tensor_indices(middle, 0),
+ start[T.tensor_indices(start, 0)] + tvm.tirx.indexdiv(width, 2),
)
T.tensor_store(
end,
+ T.tensor_indices(end, 0),
tvm.tirx.min(
start[T.tensor_indices(start, 0)] + width, scan_axis_size
),
- T.tensor_indices(end, 0),
)
with T.if_(middle[T.tensor_indices(middle, 0)] < scan_axis_size):
with T.then_():
T.tensor_store(
tmp,
+ T.tensor_indices(tmp, 0),
output[
T.tensor_indices(
output,
@@ -266,10 +267,17 @@ def exclusive_scan_ir(data, output, reduction=None, binop=operator.add, identity
),
)
],
- T.tensor_indices(tmp, 0),
)
T.tensor_store(
output,
+ T.tensor_indices(
+ output,
+ (
+ batch * scan_axis_size
+ + middle[T.tensor_indices(middle, 0)]
+ - 1
+ ),
+ ),
output[
T.tensor_indices(
output,
@@ -280,17 +288,17 @@ def exclusive_scan_ir(data, output, reduction=None, binop=operator.add, identity
),
)
],
+ )
+ T.tensor_store(
+ output,
T.tensor_indices(
output,
(
batch * scan_axis_size
- + middle[T.tensor_indices(middle, 0)]
+ + end[T.tensor_indices(end, 0)]
- 1
),
),
- )
- T.tensor_store(
- output,
binop(
output[
T.tensor_indices(
@@ -304,14 +312,6 @@ def exclusive_scan_ir(data, output, reduction=None, binop=operator.add, identity
],
tmp[T.tensor_indices(tmp, 0)],
),
- T.tensor_indices(
- output,
- (
- batch * scan_axis_size
- + end[T.tensor_indices(end, 0)]
- - 1
- ),
- ),
)
return ib.get()
@@ -370,6 +370,7 @@ def get_reduction_from_exclusive_scan(data, ex_scan_output, binop=operator.add):
with T.then_():
T.tensor_store(
reduction,
+ T.tensor_indices(reduction, tid),
binop(
data_ex_scan[
T.tensor_indices(
@@ -383,13 +384,12 @@ def get_reduction_from_exclusive_scan(data, ex_scan_output, binop=operator.add):
)
],
),
- T.tensor_indices(reduction, tid),
)
with T.else_():
T.tensor_store(
reduction,
- cast(0, reduction_buf.dtype),
T.tensor_indices(reduction, tid),
+ cast(0, reduction_buf.dtype),
)
return ib.get()
diff --git a/python/tvm/topi/gpu/scatter_elements.py b/python/tvm/topi/gpu/scatter_elements.py
index e8362149dd..aaa2a9b297 100644
--- a/python/tvm/topi/gpu/scatter_elements.py
+++ b/python/tvm/topi/gpu/scatter_elements.py
@@ -82,8 +82,8 @@ def scatter_elements(data, indices, updates, axis=0, reduction="update"):
with T.then_():
T.tensor_store(
out,
- data[T.tensor_indices(data, tid)],
T.tensor_indices(out, tid),
+ data[T.tensor_indices(data, tid)],
)
# Scatter
@@ -114,41 +114,41 @@ def scatter_elements(data, indices, updates, axis=0, reduction="update"):
return ib.get()
def update_func(dst_ptr, dst_index, update):
- T.tensor_store(dst_ptr, update, T.tensor_indices(dst_ptr, dst_index))
+ T.tensor_store(dst_ptr, T.tensor_indices(dst_ptr, dst_index), update)
def add_func(dst_ptr, dst_index, update):
T.tensor_store(
dst_ptr,
- dst_ptr[T.tensor_indices(dst_ptr, dst_index)] + (update),
T.tensor_indices(dst_ptr, dst_index),
+ dst_ptr[T.tensor_indices(dst_ptr, dst_index)] + (update),
)
def mul_func(dst_ptr, dst_index, update):
T.tensor_store(
dst_ptr,
- dst_ptr[T.tensor_indices(dst_ptr, dst_index)] * (update),
T.tensor_indices(dst_ptr, dst_index),
+ dst_ptr[T.tensor_indices(dst_ptr, dst_index)] * (update),
)
def mean_func(dst_ptr, dst_index, update):
T.tensor_store(
dst_ptr,
- (dst_ptr[T.tensor_indices(dst_ptr, dst_index)] + update) / 2,
T.tensor_indices(dst_ptr, dst_index),
+ (dst_ptr[T.tensor_indices(dst_ptr, dst_index)] + update) / 2,
)
def min_func(dst_ptr, dst_index, update):
T.tensor_store(
dst_ptr,
- tirx.min(dst_ptr[T.tensor_indices(dst_ptr, dst_index)], update),
T.tensor_indices(dst_ptr, dst_index),
+ tirx.min(dst_ptr[T.tensor_indices(dst_ptr, dst_index)], update),
)
def max_func(dst_ptr, dst_index, update):
T.tensor_store(
dst_ptr,
- tirx.max(dst_ptr[T.tensor_indices(dst_ptr, dst_index)], update),
T.tensor_indices(dst_ptr, dst_index),
+ tirx.max(dst_ptr[T.tensor_indices(dst_ptr, dst_index)], update),
)
reduce_func = None
diff --git a/python/tvm/topi/gpu/scatter_nd.py b/python/tvm/topi/gpu/scatter_nd.py
index 2a1879faf1..6688cdb303 100644
--- a/python/tvm/topi/gpu/scatter_nd.py
+++ b/python/tvm/topi/gpu/scatter_nd.py
@@ -70,8 +70,8 @@ def scatter_nd(data, indices, updates, mode):
with T.then_():
T.tensor_store(
out,
- data[T.tensor_indices(data, tid)],
T.tensor_indices(out, tid),
+ data[T.tensor_indices(data, tid)],
)
# Scatter
@@ -102,14 +102,15 @@ def scatter_nd(data, indices, updates, mode):
if mode == "update":
T.tensor_store(
out,
+ T.tensor_indices(out, index),
updates[
T.tensor_indices(updates, (i * fused_updates_dimension + j))
],
- T.tensor_indices(out, index),
)
elif mode == "add":
T.tensor_store(
out,
+ T.tensor_indices(out, index),
out[T.tensor_indices(out, index)]
+ (
updates[
@@ -118,11 +119,11 @@ def scatter_nd(data, indices, updates, mode):
)
]
),
- T.tensor_indices(out, index),
)
elif mode == "mul":
T.tensor_store(
out,
+ T.tensor_indices(out, index),
out[T.tensor_indices(out, index)]
* (
updates[
@@ -131,11 +132,11 @@ def scatter_nd(data, indices, updates, mode):
)
]
),
- T.tensor_indices(out, index),
)
elif mode == "min":
T.tensor_store(
out,
+ T.tensor_indices(out, index),
tirx.min(
out[T.tensor_indices(out, index)],
updates[
@@ -144,11 +145,11 @@ def scatter_nd(data, indices, updates, mode):
)
],
),
- T.tensor_indices(out, index),
)
elif mode == "max":
T.tensor_store(
out,
+ T.tensor_indices(out, index),
tirx.max(
out[T.tensor_indices(out, index)],
updates[
@@ -157,7 +158,6 @@ def scatter_nd(data, indices, updates, mode):
)
],
),
- T.tensor_indices(out, index),
)
else:
raise NotImplementedError(
diff --git a/python/tvm/topi/gpu/sort.py b/python/tvm/topi/gpu/sort.py
index c68f7f29aa..3a44f6624d 100644
--- a/python/tvm/topi/gpu/sort.py
+++ b/python/tvm/topi/gpu/sort.py
@@ -68,12 +68,12 @@ def _sort_init(shape, axis, keys_in, keys_out, values_out=None, value_init_func=
with T.then_():
T.tensor_store(
keys_out,
- keys_in[T.tensor_indices(keys_in, idx)],
T.tensor_indices(keys_out, idx),
+ keys_in[T.tensor_indices(keys_in, idx)],
)
if values_out is not None:
T.tensor_store(
- values_out, value_init_func(idx, tid), T.tensor_indices(values_out, idx)
+ values_out, T.tensor_indices(values_out, idx), value_init_func(idx, tid)
)
return axis_mul_before, axis_mul_after
@@ -126,20 +126,20 @@ def _odd_even_sort(
with T.then_():
T.tensor_store(
tmp_keys_swap,
+ [tid + n],
keys[
T.tensor_indices(keys, (base_idx + (tid + n + start) * axis_mul_after))
],
- [tid + n],
)
if values_swap is not None:
T.tensor_store(
tmp_values_swap,
+ [tid + n],
values[
T.tensor_indices(
values, (base_idx + (tid + n + start) * axis_mul_after)
)
],
- [tid + n],
)
T.evaluate(tvm.ir.Call("tirx.gpu_storage_sync", [tvm.ir.StringImm("shared")], ty="void"))
@@ -151,25 +151,25 @@ def _odd_even_sort(
n = idxm(tid + k, 2)
with T.if_(tid + n < current_sort_num - 1):
with T.then_():
- T.tensor_store(temp_cond1, tmp_keys_swap[tid + n], [0])
- T.tensor_store(temp_cond2, tmp_keys_swap[tid + n + 1], [0])
+ T.tensor_store(temp_cond1, [0], tmp_keys_swap[tid + n])
+ T.tensor_store(temp_cond2, [0], tmp_keys_swap[tid + n + 1])
if is_ascend:
cond = temp_cond1[0] > temp_cond2[0]
else:
cond = temp_cond1[0] < temp_cond2[0]
with T.if_(cond):
with T.then_():
- T.tensor_store(temp_keys, tmp_keys_swap[tid + n], [0])
- T.tensor_store(tmp_keys_swap, tmp_keys_swap[tid + n + 1], [tid + n])
- T.tensor_store(tmp_keys_swap, temp_keys[0], [tid + n + 1])
+ T.tensor_store(temp_keys, [0], tmp_keys_swap[tid + n])
+ T.tensor_store(tmp_keys_swap, [tid + n], tmp_keys_swap[tid + n + 1])
+ T.tensor_store(tmp_keys_swap, [tid + n + 1], temp_keys[0])
if values_swap is not None:
- T.tensor_store(temp_values, tmp_values_swap[tid + n], [0])
+ T.tensor_store(temp_values, [0], tmp_values_swap[tid + n])
T.tensor_store(
tmp_values_swap,
- tmp_values_swap[tid + n + 1],
[tid + n],
+ tmp_values_swap[tid + n + 1],
)
- T.tensor_store(tmp_values_swap, temp_values[0], [tid + n + 1])
+ T.tensor_store(tmp_values_swap, [tid + n + 1], temp_values[0])
T.evaluate(
tvm.ir.Call("tirx.gpu_storage_sync", [tvm.ir.StringImm("shared")], ty="void")
)
@@ -179,18 +179,18 @@ def _odd_even_sort(
with T.if_(tid + n + start < size):
with T.then_():
out_idx = base_idx + (tid + n + start) * axis_mul_after
- T.tensor_store(keys, tmp_keys_swap[tid + n], T.tensor_indices(keys, out_idx))
+ T.tensor_store(keys, T.tensor_indices(keys, out_idx), tmp_keys_swap[tid + n])
T.tensor_store(
- keys_swap, tmp_keys_swap[tid + n], T.tensor_indices(keys_swap, out_idx)
+ keys_swap, T.tensor_indices(keys_swap, out_idx), tmp_keys_swap[tid + n]
)
if values_swap is not None:
T.tensor_store(
- values, tmp_values_swap[tid + n], T.tensor_indices(values, out_idx)
+ values, T.tensor_indices(values, out_idx), tmp_values_swap[tid + n]
)
T.tensor_store(
values_swap,
- tmp_values_swap[tid + n],
T.tensor_indices(values_swap, out_idx),
+ tmp_values_swap[tid + n],
)
@@ -253,20 +253,20 @@ def _sort_common(
max_val = tvm.te.max(0, diag - bCount)
min_val = tvm.te.min(diag, aCount)
if is_webgpu:
- T.tensor_store(first, cast(max_val, target_dtype), T.tensor_indices(first, 0))
- T.tensor_store(last, cast(min_val, target_dtype), T.tensor_indices(last, 0))
+ T.tensor_store(first, T.tensor_indices(first, 0), cast(max_val, target_dtype))
+ T.tensor_store(last, T.tensor_indices(last, 0), cast(min_val, target_dtype))
else:
- T.tensor_store(first, max_val, T.tensor_indices(first, 0))
- T.tensor_store(last, min_val, T.tensor_indices(last, 0))
+ T.tensor_store(first, T.tensor_indices(first, 0), max_val)
+ T.tensor_store(last, T.tensor_indices(last, 0), min_val)
with T.while_(first[T.tensor_indices(first, 0)] < last[T.tensor_indices(last, 0)]):
mid = (first[T.tensor_indices(first, 0)] + last[T.tensor_indices(last, 0)]) >> 1
a = source[T.tensor_indices(source, (base_idx + (aStart + mid)))]
b = source[T.tensor_indices(source, (base_idx + (bStart + diag - 1 - mid)))]
with T.if_(compare(a, b)):
with T.then_():
- T.tensor_store(first, mid + 1, T.tensor_indices(first, 0))
+ T.tensor_store(first, T.tensor_indices(first, 0), mid + 1)
with T.else_():
- T.tensor_store(last, mid, T.tensor_indices(last, 0))
+ T.tensor_store(last, T.tensor_indices(last, 0), mid)
def serial_merge(
source,
@@ -289,11 +289,11 @@ def _sort_common(
i_val = aStart + first[T.tensor_indices(first, 0)]
j_val = bStart + diag - last[T.tensor_indices(last, 0)]
if is_webgpu:
- T.tensor_store(i_buf, cast(i_val, target_dtype), T.tensor_indices(i_buf, 0))
- T.tensor_store(j_buf, cast(j_val, target_dtype), T.tensor_indices(j_buf, 0))
+ T.tensor_store(i_buf, T.tensor_indices(i_buf, 0), cast(i_val, target_dtype))
+ T.tensor_store(j_buf, T.tensor_indices(j_buf, 0), cast(j_val, target_dtype))
else:
- T.tensor_store(i_buf, i_val, T.tensor_indices(i_buf, 0))
- T.tensor_store(j_buf, j_val, T.tensor_indices(j_buf, 0))
+ T.tensor_store(i_buf, T.tensor_indices(i_buf, 0), i_val)
+ T.tensor_store(j_buf, T.tensor_indices(j_buf, 0), j_val)
with T.serial(0, tvm.te.min(aCount + bCount - diag, step_count)) as count:
i_idx = base_idx + i_buf[T.tensor_indices(i_buf, 0)]
@@ -316,72 +316,72 @@ def _sort_common(
with T.then_():
T.tensor_store(
dest,
- source[T.tensor_indices(source, i_idx)],
T.tensor_indices(dest, k_idx),
+ source[T.tensor_indices(source, i_idx)],
)
if values is not None:
T.tensor_store(
dest_idx,
- source_idx[T.tensor_indices(source_idx, i_idx)],
T.tensor_indices(dest_idx, k_idx),
+ source_idx[T.tensor_indices(source_idx, i_idx)],
)
T.tensor_store(
i_buf,
- i_buf[T.tensor_indices(i_buf, 0)] + 1,
T.tensor_indices(i_buf, 0),
+ i_buf[T.tensor_indices(i_buf, 0)] + 1,
)
with T.else_():
T.tensor_store(
dest,
- source[T.tensor_indices(source, j_idx)],
T.tensor_indices(dest, k_idx),
+ source[T.tensor_indices(source, j_idx)],
)
if values is not None:
T.tensor_store(
dest_idx,
- source_idx[T.tensor_indices(source_idx, j_idx)],
T.tensor_indices(dest_idx, k_idx),
+ source_idx[T.tensor_indices(source_idx, j_idx)],
)
T.tensor_store(
j_buf,
- j_buf[T.tensor_indices(j_buf, 0)] + 1,
T.tensor_indices(j_buf, 0),
+ j_buf[T.tensor_indices(j_buf, 0)] + 1,
)
with T.else_():
with T.if_(i_buf[T.tensor_indices(i_buf, 0)] < aStart + aCount):
with T.then_():
T.tensor_store(
dest,
- source[T.tensor_indices(source, i_idx)],
T.tensor_indices(dest, k_idx),
+ source[T.tensor_indices(source, i_idx)],
)
if values is not None:
T.tensor_store(
dest_idx,
- source_idx[T.tensor_indices(source_idx, i_idx)],
T.tensor_indices(dest_idx, k_idx),
+ source_idx[T.tensor_indices(source_idx, i_idx)],
)
T.tensor_store(
i_buf,
- i_buf[T.tensor_indices(i_buf, 0)] + 1,
T.tensor_indices(i_buf, 0),
+ i_buf[T.tensor_indices(i_buf, 0)] + 1,
)
with T.else_():
T.tensor_store(
dest,
- source[T.tensor_indices(source, j_idx)],
T.tensor_indices(dest, k_idx),
+ source[T.tensor_indices(source, j_idx)],
)
if values is not None:
T.tensor_store(
dest_idx,
- source_idx[T.tensor_indices(source_idx, j_idx)],
T.tensor_indices(dest_idx, k_idx),
+ source_idx[T.tensor_indices(source_idx, j_idx)],
)
T.tensor_store(
j_buf,
- j_buf[T.tensor_indices(j_buf, 0)] + 1,
T.tensor_indices(j_buf, 0),
+ j_buf[T.tensor_indices(j_buf, 0)] + 1,
)
def mergepath(
@@ -656,14 +656,14 @@ def _sort_common(
with T.then_():
T.tensor_store(
keys,
- keys_swap[T.tensor_indices(keys_swap, idx)],
T.tensor_indices(keys, idx),
+ keys_swap[T.tensor_indices(keys_swap, idx)],
)
if values is not None:
T.tensor_store(
values,
- values_swap[T.tensor_indices(values_swap, idx)],
T.tensor_indices(values, idx),
+ values_swap[T.tensor_indices(values_swap, idx)],
)
@@ -1233,6 +1233,7 @@ def searchsorted(sorted_sequence, values, right=False, out_dtype="int64"):
T.tensor_store(
indices_ptr,
+ T.tensor_indices(indices_ptr, tid),
binary_search(
sequence_offset,
search_range,
@@ -1241,7 +1242,6 @@ def searchsorted(sorted_sequence, values, right=False, out_dtype="int64"):
right,
out_dtype,
),
- T.tensor_indices(indices_ptr, tid),
)
return ib.get()
diff --git a/python/tvm/topi/index_put.py b/python/tvm/topi/index_put.py
index e5399e1fc4..d75d1d7795 100644
--- a/python/tvm/topi/index_put.py
+++ b/python/tvm/topi/index_put.py
@@ -96,7 +96,7 @@ def index_put(data, indices, values, accumulate=False):
with IRBuilder() as ib:
with T.parallel(0, full_range) as i:
- T.tensor_store(out, data[T.tensor_indices(data, i)], T.tensor_indices(out, i))
+ T.tensor_store(out, T.tensor_indices(out, i), data[T.tensor_indices(data, i)])
with T.parallel(0, index_len) as k:
# Decompose k into multi-dimensional broadcast index
@@ -140,13 +140,13 @@ def index_put(data, indices, values, accumulate=False):
return ib.get()
def update_func(dst_ptr, dst_index, update):
- T.tensor_store(dst_ptr, update, T.tensor_indices(dst_ptr, dst_index))
+ T.tensor_store(dst_ptr, T.tensor_indices(dst_ptr, dst_index), update)
def add_func(dst_ptr, dst_index, update):
T.tensor_store(
dst_ptr,
- dst_ptr[T.tensor_indices(dst_ptr, dst_index)] + (update),
T.tensor_indices(dst_ptr, dst_index),
+ dst_ptr[T.tensor_indices(dst_ptr, dst_index)] + (update),
)
reduce_func = add_func if accumulate else update_func
diff --git a/python/tvm/topi/scan.py b/python/tvm/topi/scan.py
index 02b5e43453..8396aa3757 100644
--- a/python/tvm/topi/scan.py
+++ b/python/tvm/topi/scan.py
@@ -123,13 +123,13 @@ def scanop(
base_idx = i * cumsum_axis_len * axis_mul_after + j
if exclusive:
T.tensor_store(
- out_buf, cast(identity_value, dtype), T.tensor_indices(out_buf, base_idx)
+ out_buf, T.tensor_indices(out_buf, base_idx), cast(identity_value, dtype)
)
else:
T.tensor_store(
out_buf,
- maybe_cast(data_buf[T.tensor_indices(data_buf, base_idx)]),
T.tensor_indices(out_buf, base_idx),
+ maybe_cast(data_buf[T.tensor_indices(data_buf, base_idx)]),
)
with T.serial(0, cumsum_axis_len - 1) as _k:
k = _k + 1
@@ -138,20 +138,20 @@ def scanop(
if exclusive:
T.tensor_store(
out_buf,
+ T.tensor_indices(out_buf, cur_idx),
binop(
out_buf[T.tensor_indices(out_buf, prev_idx)],
maybe_cast(data_buf[T.tensor_indices(data_buf, prev_idx)]),
),
- T.tensor_indices(out_buf, cur_idx),
)
else:
T.tensor_store(
out_buf,
+ T.tensor_indices(out_buf, cur_idx),
binop(
out_buf[T.tensor_indices(out_buf, prev_idx)],
maybe_cast(data_buf[T.tensor_indices(data_buf, cur_idx)]),
),
- T.tensor_indices(out_buf, cur_idx),
)
return ib.get()
diff --git a/python/tvm/topi/scatter.py b/python/tvm/topi/scatter.py
index 2b7ac9f899..bf2e255370 100644
--- a/python/tvm/topi/scatter.py
+++ b/python/tvm/topi/scatter.py
@@ -118,7 +118,7 @@ def scatter_nd(data, indices, updates, mode):
with IRBuilder() as ib:
with T.serial(0, fused_shape) as i:
- T.tensor_store(out, data[T.tensor_indices(data, i)], T.tensor_indices(out, i))
+ T.tensor_store(out, T.tensor_indices(out, i), data[T.tensor_indices(data, i)])
with T.serial(0, fused_indices_dimension) as i:
with T.parallel(0, fused_updates_dimension) as j:
@@ -136,40 +136,40 @@ def scatter_nd(data, indices, updates, mode):
if mode == "update":
T.tensor_store(
out,
- updates[T.tensor_indices(updates, i * fused_updates_dimension + j)],
T.tensor_indices(out, index),
+ updates[T.tensor_indices(updates, i * fused_updates_dimension + j)],
)
elif mode == "add":
T.tensor_store(
out,
+ T.tensor_indices(out, index),
out[T.tensor_indices(out, index)]
+ (updates[T.tensor_indices(updates, i * fused_updates_dimension + j)]),
- T.tensor_indices(out, index),
)
elif mode == "mul":
T.tensor_store(
out,
+ T.tensor_indices(out, index),
out[T.tensor_indices(out, index)]
* (updates[T.tensor_indices(updates, i * fused_updates_dimension + j)]),
- T.tensor_indices(out, index),
)
elif mode == "min":
T.tensor_store(
out,
+ T.tensor_indices(out, index),
tirx.min(
out[T.tensor_indices(out, index)],
updates[T.tensor_indices(updates, i * fused_updates_dimension + j)],
),
- T.tensor_indices(out, index),
)
elif mode == "max":
T.tensor_store(
out,
+ T.tensor_indices(out, index),
tirx.max(
out[T.tensor_indices(out, index)],
updates[T.tensor_indices(updates, i * fused_updates_dimension + j)],
),
- T.tensor_indices(out, index),
)
else:
raise NotImplementedError(
diff --git a/python/tvm/topi/scatter_elements.py b/python/tvm/topi/scatter_elements.py
index 73e0715d6d..aac5b8153e 100644
--- a/python/tvm/topi/scatter_elements.py
+++ b/python/tvm/topi/scatter_elements.py
@@ -106,7 +106,7 @@ def scatter_elements(data, indices, updates, axis=0, reduction="update"):
# Copy initial input data to output
with IRBuilder() as ib:
with T.parallel(0, full_range) as i:
- T.tensor_store(out, data[T.tensor_indices(data, i)], T.tensor_indices(out, i))
+ T.tensor_store(out, T.tensor_indices(out, i), data[T.tensor_indices(data, i)])
with T.parallel(0, ind_before_axis_range * ind_after_axis_range) as fused:
i = fused // ind_after_axis_range
@@ -126,41 +126,41 @@ def scatter_elements(data, indices, updates, axis=0, reduction="update"):
return ib.get()
def update_func(dst_ptr, dst_index, update):
- T.tensor_store(dst_ptr, update, T.tensor_indices(dst_ptr, dst_index))
+ T.tensor_store(dst_ptr, T.tensor_indices(dst_ptr, dst_index), update)
def add_func(dst_ptr, dst_index, update):
T.tensor_store(
dst_ptr,
- dst_ptr[T.tensor_indices(dst_ptr, dst_index)] + (update),
T.tensor_indices(dst_ptr, dst_index),
+ dst_ptr[T.tensor_indices(dst_ptr, dst_index)] + (update),
)
def mul_func(dst_ptr, dst_index, update):
T.tensor_store(
dst_ptr,
- dst_ptr[T.tensor_indices(dst_ptr, dst_index)] * (update),
T.tensor_indices(dst_ptr, dst_index),
+ dst_ptr[T.tensor_indices(dst_ptr, dst_index)] * (update),
)
def mean_func(dst_ptr, dst_index, update):
T.tensor_store(
dst_ptr,
- (dst_ptr[T.tensor_indices(dst_ptr, dst_index)] + update) / 2,
T.tensor_indices(dst_ptr, dst_index),
+ (dst_ptr[T.tensor_indices(dst_ptr, dst_index)] + update) / 2,
)
def min_func(dst_ptr, dst_index, update):
T.tensor_store(
dst_ptr,
- tirx.min(dst_ptr[T.tensor_indices(dst_ptr, dst_index)], update),
T.tensor_indices(dst_ptr, dst_index),
+ tirx.min(dst_ptr[T.tensor_indices(dst_ptr, dst_index)], update),
)
def max_func(dst_ptr, dst_index, update):
T.tensor_store(
dst_ptr,
- tirx.max(dst_ptr[T.tensor_indices(dst_ptr, dst_index)], update),
T.tensor_indices(dst_ptr, dst_index),
+ tirx.max(dst_ptr[T.tensor_indices(dst_ptr, dst_index)], update),
)
reduce_func = None
diff --git a/python/tvm/topi/searchsorted.py b/python/tvm/topi/searchsorted.py
index d6d11db654..39b4b219a8 100644
--- a/python/tvm/topi/searchsorted.py
+++ b/python/tvm/topi/searchsorted.py
@@ -43,8 +43,8 @@ def binary_search(sequence_offset, search_range, sorted_sequence, value, right,
lo = lo_buf
hi = hi_buf
- T.tensor_store(lo, cast(0, out_dtype), T.tensor_indices(lo, 0))
- T.tensor_store(hi, cast(search_range, out_dtype), T.tensor_indices(hi, 0))
+ T.tensor_store(lo, T.tensor_indices(lo, 0), cast(0, out_dtype))
+ T.tensor_store(hi, T.tensor_indices(hi, 0), cast(search_range, out_dtype))
# Reference: pytorch/aten/src/ATen/native/cuda/Bucketization.cu
def condition(current_val, target_val):
@@ -62,9 +62,9 @@ def binary_search(sequence_offset, search_range, sorted_sequence, value, right,
)
):
with T.then_():
- T.tensor_store(lo, mid + 1, T.tensor_indices(lo, 0))
+ T.tensor_store(lo, T.tensor_indices(lo, 0), mid + 1)
with T.else_():
- T.tensor_store(hi, mid, T.tensor_indices(hi, 0))
+ T.tensor_store(hi, T.tensor_indices(hi, 0), mid)
return lo[T.tensor_indices(lo, 0)]
@@ -117,6 +117,7 @@ def searchsorted(sorted_sequence, values, right=False, out_dtype="int64"):
T.tensor_store(
indices,
+ T.tensor_indices(indices, i),
binary_search(
sequence_offset,
search_range,
@@ -125,7 +126,6 @@ def searchsorted(sorted_sequence, values, right=False, out_dtype="int64"):
right,
out_dtype,
),
- T.tensor_indices(indices, i),
)
return ib.get()
diff --git a/python/tvm/topi/signal.py b/python/tvm/topi/signal.py
index d2ea464ac5..7990fad5fd 100644
--- a/python/tvm/topi/signal.py
+++ b/python/tvm/topi/signal.py
@@ -92,48 +92,48 @@ def stft(
row = tirx.floormod(batch_row, output_ptr.shape[1])
T.tensor_store(
output,
- tirx.Cast(data_ptr.dtype, 0),
(batch, row, col, 0),
+ tirx.Cast(data_ptr.dtype, 0),
)
T.tensor_store(
output,
- tirx.Cast(data_ptr.dtype, 0),
(batch, row, col, 1),
+ tirx.Cast(data_ptr.dtype, 0),
)
with T.serial(0, win_length) as wlen:
T.tensor_store(
output,
+ (batch, row, col, 0),
output[(batch, row, col, 0)]
+ (
window[T.tensor_indices(window, wlen)]
* data[(batch, col * hop_length + wlen)]
* tirx.cos(2 * pi * row * wlen / win_length)
),
- (batch, row, col, 0),
)
T.tensor_store(
output,
+ (batch, row, col, 1),
output[(batch, row, col, 1)]
- (
window[T.tensor_indices(window, wlen)]
* data[(batch, col * hop_length + wlen)]
* tirx.sin(2 * pi * row * wlen / win_length)
),
- (batch, row, col, 1),
)
with T.if_(normalized):
with T.then_():
T.tensor_store(
output,
+ (batch, row, col, 0),
output[(batch, row, col, 0)]
/ (tirx.sqrt(tirx.const(n_fft, "float32"))),
- (batch, row, col, 0),
)
T.tensor_store(
output,
+ (batch, row, col, 1),
output[(batch, row, col, 1)]
/ (tirx.sqrt(tirx.const(n_fft, "float32"))),
- (batch, row, col, 1),
)
return ib.get()
@@ -212,13 +212,13 @@ def dft(
n_idx = base_idx + n
T.tensor_store(
re_output_ptr,
- tirx.Cast(re_output_ptr.dtype, 0),
T.tensor_indices(re_output_ptr, n_idx),
+ tirx.Cast(re_output_ptr.dtype, 0),
)
T.tensor_store(
im_output_ptr,
- tirx.Cast(im_output_ptr.dtype, 0),
T.tensor_indices(im_output_ptr, n_idx),
+ tirx.Cast(im_output_ptr.dtype, 0),
)
_w = sign * -2 * pi * n / n_fft
with T.serial(0, n_fft) as k:
@@ -228,34 +228,34 @@ def dft(
sin_w = tirx.Cast(re_output_ptr.dtype, tirx.sin(w))
T.tensor_store(
re_output_ptr,
+ T.tensor_indices(re_output_ptr, n_idx),
re_output_ptr[T.tensor_indices(re_output_ptr, n_idx)]
+ (
re_data_ptr[T.tensor_indices(re_data_ptr, k_idx)] * cos_w
- im_data_ptr[T.tensor_indices(im_data_ptr, k_idx)] * sin_w
),
- T.tensor_indices(re_output_ptr, n_idx),
)
T.tensor_store(
im_output_ptr,
+ T.tensor_indices(im_output_ptr, n_idx),
im_output_ptr[T.tensor_indices(im_output_ptr, n_idx)]
+ (
re_data_ptr[T.tensor_indices(re_data_ptr, k_idx)] * sin_w
+ im_data_ptr[T.tensor_indices(im_data_ptr, k_idx)] * cos_w
),
- T.tensor_indices(im_output_ptr, n_idx),
)
T.tensor_store(
re_output_ptr,
+ T.tensor_indices(re_output_ptr, n_idx),
re_output_ptr[T.tensor_indices(re_output_ptr, n_idx)]
* (tirx.Cast(re_output_ptr.dtype, factor)),
- T.tensor_indices(re_output_ptr, n_idx),
)
T.tensor_store(
im_output_ptr,
+ T.tensor_indices(im_output_ptr, n_idx),
im_output_ptr[T.tensor_indices(im_output_ptr, n_idx)]
* (tirx.Cast(im_output_ptr.dtype, factor)),
- T.tensor_indices(im_output_ptr, n_idx),
)
return ib.get()
diff --git a/python/tvm/topi/sparse_reshape.py b/python/tvm/topi/sparse_reshape.py
index eb483f80e8..8d4bdc9f01 100644
--- a/python/tvm/topi/sparse_reshape.py
+++ b/python/tvm/topi/sparse_reshape.py
@@ -106,44 +106,44 @@ def sparse_reshape(
T.tensor_store(
total_ele,
- prev_shape[T.tensor_indices(prev_shape, 0)],
T.tensor_indices(total_ele, 0),
+ prev_shape[T.tensor_indices(prev_shape, 0)],
)
# Cumulative Reverse Exclusive Multiply
T.tensor_store(
multipliers,
- Cast(new_shape_ptr.dtype, 1),
T.tensor_indices(multipliers, prev_shape_size - 1),
+ Cast(new_shape_ptr.dtype, 1),
)
with T.serial(0, prev_shape_size - 1) as i_:
i = i_ + 1
T.tensor_store(
multipliers,
+ T.tensor_indices(multipliers, prev_shape_size - 1 - i),
prev_shape[T.tensor_indices(prev_shape, prev_shape_size - i)]
* multipliers[T.tensor_indices(multipliers, prev_shape_size - i)],
- T.tensor_indices(multipliers, prev_shape_size - 1 - i),
)
T.tensor_store(
total_ele,
+ T.tensor_indices(total_ele, 0),
total_ele[T.tensor_indices(total_ele, 0)]
* (prev_shape[T.tensor_indices(prev_shape, prev_shape_size - i)]),
- T.tensor_indices(total_ele, 0),
)
T.tensor_store(
division_total_ele,
- Cast(new_shape_ptr.dtype, 1),
T.tensor_indices(division_total_ele, 0),
+ Cast(new_shape_ptr.dtype, 1),
)
with T.serial(0, new_shape_size) as i:
with T.if_(new_shape[T.tensor_indices(new_shape, i)] != -1):
with T.then_():
T.tensor_store(
division_total_ele,
+ T.tensor_indices(division_total_ele, 0),
division_total_ele[T.tensor_indices(division_total_ele, 0)]
* (new_shape[T.tensor_indices(new_shape, i)]),
- T.tensor_indices(division_total_ele, 0),
)
# Compute true output shape (replace negative ones)
@@ -152,6 +152,7 @@ def sparse_reshape(
with T.then_():
T.tensor_store(
out_new_shape,
+ T.tensor_indices(out_new_shape, i),
Cast(
new_shape_ptr.dtype,
div(
@@ -159,17 +160,16 @@ def sparse_reshape(
division_total_ele[T.tensor_indices(division_total_ele, 0)],
),
),
- T.tensor_indices(out_new_shape, i),
)
with T.else_():
T.tensor_store(
out_new_shape,
- new_shape[T.tensor_indices(new_shape, i)],
T.tensor_indices(out_new_shape, i),
+ new_shape[T.tensor_indices(new_shape, i)],
)
# Check if prev_shape and new_shape are equal
- T.tensor_store(equal_shape, True, T.tensor_indices(equal_shape, 0))
+ T.tensor_store(equal_shape, T.tensor_indices(equal_shape, 0), True)
with T.if_(prev_shape_size == new_shape_size):
with T.then_():
with T.serial(0, prev_shape_size) as i:
@@ -178,9 +178,9 @@ def sparse_reshape(
!= out_new_shape[T.tensor_indices(out_new_shape, i)]
):
with T.then_():
- T.tensor_store(equal_shape, False, T.tensor_indices(equal_shape, 0))
+ T.tensor_store(equal_shape, T.tensor_indices(equal_shape, 0), False)
with T.else_():
- T.tensor_store(equal_shape, False, T.tensor_indices(equal_shape, 0))
+ T.tensor_store(equal_shape, T.tensor_indices(equal_shape, 0), False)
# Return same inputs if shapes are equal
with T.if_(equal_shape[T.tensor_indices(equal_shape, 0)]):
@@ -189,41 +189,41 @@ def sparse_reshape(
with T.serial(0, sparse_indices_ptr.shape[1]) as j:
T.tensor_store(
new_sparse_indices,
- sparse_indices[(i, j)],
(i, j),
+ sparse_indices[(i, j)],
)
# Else compute new_sparse_indices
with T.else_():
T.tensor_store(
dividers,
- Cast(new_shape_ptr.dtype, 1),
T.tensor_indices(dividers, new_shape_size - 1),
+ Cast(new_shape_ptr.dtype, 1),
)
with T.serial(0, new_shape_size - 1) as i_:
i = i_ + 1
T.tensor_store(
dividers,
+ T.tensor_indices(dividers, new_shape_size - 1 - i),
dividers[T.tensor_indices(dividers, new_shape_size - i)]
* out_new_shape[T.tensor_indices(out_new_shape, new_shape_size - i)],
- T.tensor_indices(dividers, new_shape_size - 1 - i),
)
with T.parallel(0, sparse_indices_ptr.shape[0]) as i:
T.tensor_store(
flattened_indices,
- Cast(new_shape_ptr.dtype, 0),
T.tensor_indices(flattened_indices, i),
+ Cast(new_shape_ptr.dtype, 0),
)
with T.serial(0, sparse_indices_ptr.shape[1]) as j:
T.tensor_store(
flattened_indices,
+ T.tensor_indices(flattened_indices, i),
flattened_indices[T.tensor_indices(flattened_indices, i)]
+ (
sparse_indices[(i, j)]
* multipliers[T.tensor_indices(multipliers, j)]
),
- T.tensor_indices(flattened_indices, i),
)
with T.parallel(0, new_sparse_indices_ptr.shape[0]) as i:
@@ -233,13 +233,14 @@ def sparse_reshape(
current_element = current_element_buf
T.tensor_store(
current_element,
- flattened_indices[T.tensor_indices(flattened_indices, i)],
T.tensor_indices(current_element, 0),
+ flattened_indices[T.tensor_indices(flattened_indices, i)],
)
with T.serial(0, new_sparse_indices_ptr.shape[1]) as j:
T.tensor_store(
new_sparse_indices,
+ (i, j),
Cast(
sparse_indices_ptr.dtype,
floordiv(
@@ -247,15 +248,14 @@ def sparse_reshape(
dividers[T.tensor_indices(dividers, j)],
),
),
- (i, j),
)
T.tensor_store(
current_element,
+ T.tensor_indices(current_element, 0),
floormod(
current_element[T.tensor_indices(current_element, 0)],
dividers[T.tensor_indices(dividers, j)],
),
- T.tensor_indices(current_element, 0),
)
return ib.get()
diff --git a/python/tvm/topi/unique.py b/python/tvm/topi/unique.py
index 250f11c790..40085bdbf7 100644
--- a/python/tvm/topi/unique.py
+++ b/python/tvm/topi/unique.py
@@ -46,10 +46,11 @@ def _calc_adjacent_diff_ir(data, output, binop=tirx.Sub):
with T.parallel(0, data.shape[0]) as i:
with T.if_(i == 0):
with T.then_():
- T.tensor_store(output_ptr, 0, T.tensor_indices(output_ptr, 0))
+ T.tensor_store(output_ptr, T.tensor_indices(output_ptr, 0), 0)
with T.else_():
T.tensor_store(
output_ptr,
+ T.tensor_indices(output_ptr, i),
tirx.Cast(
output.dtype,
binop(
@@ -57,7 +58,6 @@ def _calc_adjacent_diff_ir(data, output, binop=tirx.Sub):
data_ptr[T.tensor_indices(data_ptr, i - 1)],
),
),
- T.tensor_indices(output_ptr, i),
)
return ib.get()
diff --git a/python/tvm/topi/vision/nms.py b/python/tvm/topi/vision/nms.py
index 1427c536bf..1c97d5f461 100644
--- a/python/tvm/topi/vision/nms.py
+++ b/python/tvm/topi/vision/nms.py
@@ -46,7 +46,7 @@ def _get_valid_counts_ir(
with IRBuilder() as ib:
with T.parallel(0, batch_size) as i:
- T.tensor_store(valid_count, T.int32(0), T.tensor_indices(valid_count, i))
+ T.tensor_store(valid_count, T.tensor_indices(valid_count, i), T.int32(0))
with T.serial(0, num_anchors) as j:
score = data[(i, j, score_index)]
@@ -61,11 +61,11 @@ def _get_valid_counts_ir(
with T.serial(0, box_data_length) as k:
T.tensor_store(
out_tensor,
- data[(i, j, k)],
(i, cur, k),
+ data[(i, j, k)],
)
- T.tensor_store(out_indices, j, (i, cur))
- T.tensor_store(valid_count, cur + 1, T.tensor_indices(valid_count, i))
+ T.tensor_store(out_indices, (i, cur), j)
+ T.tensor_store(valid_count, T.tensor_indices(valid_count, i), cur + 1)
# Fill remaining slots with -1
with T.serial(0, num_anchors) as j:
@@ -74,10 +74,10 @@ def _get_valid_counts_ir(
with T.serial(0, box_data_length) as k:
T.tensor_store(
out_tensor,
- tvm.tirx.Cast(data.dtype, T.float32(-1.0)),
(i, j, k),
+ tvm.tirx.Cast(data.dtype, T.float32(-1.0)),
)
- T.tensor_store(out_indices, T.int32(-1), (i, j))
+ T.tensor_store(out_indices, (i, j), T.int32(-1))
return ib.get()
@@ -216,14 +216,14 @@ def _classic_nms_ir(
nkeep_local = nkeep_buf
T.tensor_store(
nkeep_local,
- valid_count[T.tensor_indices(valid_count, i)],
T.tensor_indices(nkeep_local, 0),
+ valid_count[T.tensor_indices(valid_count, i)],
)
with T.if_(
tvm.tirx.all(top_k > 0, top_k < nkeep_local[T.tensor_indices(nkeep_local, 0)])
):
with T.then_():
- T.tensor_store(nkeep_local, top_k, T.tensor_indices(nkeep_local, 0))
+ T.tensor_store(nkeep_local, T.tensor_indices(nkeep_local, 0), top_k)
# Copy sorted boxes to output
with T.serial(0, num_anchors) as j:
@@ -233,27 +233,27 @@ def _classic_nms_ir(
with T.serial(0, box_data_length) as k:
T.tensor_store(
out_data,
- data[(i, src_idx, k)],
(i, j, k),
+ data[(i, src_idx, k)],
)
T.tensor_store(
out_box_indices,
- sorted_index[(i, j)],
(i, j),
+ sorted_index[(i, j)],
)
with T.else_():
with T.serial(0, box_data_length) as k:
T.tensor_store(
out_data,
- tvm.tirx.Cast(data.dtype, T.float32(-1.0)),
(i, j, k),
+ tvm.tirx.Cast(data.dtype, T.float32(-1.0)),
)
- T.tensor_store(out_box_indices, T.int32(-1), (i, j))
+ T.tensor_store(out_box_indices, (i, j), T.int32(-1))
# Step 2: Apply NMS - greedy suppression
num_valid_boxes_buf = T.alloc_tensor((1,), "int32", scope="local")
num_valid_boxes = num_valid_boxes_buf
- T.tensor_store(num_valid_boxes, T.int32(0), T.tensor_indices(num_valid_boxes, 0))
+ T.tensor_store(num_valid_boxes, T.tensor_indices(num_valid_boxes, 0), T.int32(0))
best_idx_buf = T.alloc_tensor((1,), "int32", scope="local")
best_idx = best_idx_buf
best_score_buf = T.alloc_tensor((1,), data.dtype, scope="local")
@@ -320,8 +320,8 @@ def _classic_nms_ir(
)
):
with T.then_():
- T.tensor_store(best_idx, T.int32(-1), T.tensor_indices(best_idx, 0))
- T.tensor_store(best_score, thresh, T.tensor_indices(best_score, 0))
+ T.tensor_store(best_idx, T.tensor_indices(best_idx, 0), T.int32(-1))
+ T.tensor_store(best_score, T.tensor_indices(best_score, 0), thresh)
with T.serial(0, nkeep_local[T.tensor_indices(nkeep_local, 0)]) as j:
with T.if_(
@@ -333,11 +333,11 @@ def _classic_nms_ir(
)
):
with T.then_():
- T.tensor_store(best_idx, j, T.tensor_indices(best_idx, 0))
+ T.tensor_store(best_idx, T.tensor_indices(best_idx, 0), j)
T.tensor_store(
best_score,
- out_data[(i, j, score_index)],
T.tensor_indices(best_score, 0),
+ out_data[(i, j, score_index)],
)
with T.if_(best_idx[T.tensor_indices(best_idx, 0)] >= 0):
@@ -349,6 +349,7 @@ def _classic_nms_ir(
with T.then_():
T.tensor_store(
tmp_idx,
+ T.tensor_indices(tmp_idx, 0),
out_box_indices[
(
i,
@@ -357,32 +358,32 @@ def _classic_nms_ir(
],
)
],
- T.tensor_indices(tmp_idx, 0),
)
T.tensor_store(
out_box_indices,
- out_box_indices[
- (
- i,
- best_idx[T.tensor_indices(best_idx, 0)],
- )
- ],
(
i,
num_valid_boxes[
T.tensor_indices(num_valid_boxes, 0)
],
),
+ out_box_indices[
+ (
+ i,
+ best_idx[T.tensor_indices(best_idx, 0)],
+ )
+ ],
)
T.tensor_store(
out_box_indices,
- tmp_idx[T.tensor_indices(tmp_idx, 0)],
(i, best_idx[T.tensor_indices(best_idx, 0)]),
+ tmp_idx[T.tensor_indices(tmp_idx, 0)],
)
with T.serial(0, box_data_length) as k:
T.tensor_store(
tmp_val,
+ T.tensor_indices(tmp_val, 0),
out_data[
(
i,
@@ -394,17 +395,9 @@ def _classic_nms_ir(
k,
)
],
- T.tensor_indices(tmp_val, 0),
)
T.tensor_store(
out_data,
- out_data[
- (
- i,
- best_idx[T.tensor_indices(best_idx, 0)],
- k,
- )
- ],
(
i,
num_valid_boxes[
@@ -412,15 +405,22 @@ def _classic_nms_ir(
],
k,
),
+ out_data[
+ (
+ i,
+ best_idx[T.tensor_indices(best_idx, 0)],
+ k,
+ )
+ ],
)
T.tensor_store(
out_data,
- tmp_val[T.tensor_indices(tmp_val, 0)],
(
i,
best_idx[T.tensor_indices(best_idx, 0)],
k,
),
+ tmp_val[T.tensor_indices(tmp_val, 0)],
)
with T.serial(
@@ -473,18 +473,18 @@ def _classic_nms_ir(
with T.then_():
T.tensor_store(
out_box_indices,
- T.int32(-1),
(i, j),
+ T.int32(-1),
)
with T.if_(iou < iou_threshold):
with T.then_():
T.tensor_store(
out_data,
+ (i, j, score_index),
out_data[(i, j, score_index)]
* tvm.tirx.exp(
soft_nms_scale * iou * iou
),
- (i, j, score_index),
)
with T.if_(
out_data[(i, j, score_index)]
@@ -493,21 +493,21 @@ def _classic_nms_ir(
with T.then_():
T.tensor_store(
out_box_indices,
- T.int32(-1),
(i, j),
+ T.int32(-1),
)
T.tensor_store(
num_valid_boxes,
- num_valid_boxes[T.tensor_indices(num_valid_boxes, 0)] + 1,
T.tensor_indices(num_valid_boxes, 0),
+ num_valid_boxes[T.tensor_indices(num_valid_boxes, 0)] + 1,
)
if return_indices:
T.tensor_store(
out_valid_box_count,
- num_valid_boxes[T.tensor_indices(num_valid_boxes, 0)],
(i, 0),
+ num_valid_boxes[T.tensor_indices(num_valid_boxes, 0)],
)
with T.serial(0, num_anchors) as j:
@@ -516,21 +516,21 @@ def _classic_nms_ir(
orig_idx = out_box_indices[(i, j)]
T.tensor_store(
out_box_indices,
- indices[(i, orig_idx)],
(i, j),
+ indices[(i, orig_idx)],
)
with T.if_(j >= num_valid_boxes[T.tensor_indices(num_valid_boxes, 0)]):
with T.then_():
with T.serial(0, box_data_length) as k:
T.tensor_store(
out_data,
- tvm.tirx.Cast(data.dtype, T.float32(-1.0)),
(i, j, k),
+ tvm.tirx.Cast(data.dtype, T.float32(-1.0)),
)
T.tensor_store(
out_box_indices,
- T.int32(-1),
(i, j),
+ T.int32(-1),
)
else:
with T.serial(0, num_anchors) as j:
@@ -539,8 +539,8 @@ def _classic_nms_ir(
with T.serial(0, box_data_length) as k:
T.tensor_store(
out_data,
- tvm.tirx.Cast(data.dtype, T.float32(-1.0)),
(i, j, k),
+ tvm.tirx.Cast(data.dtype, T.float32(-1.0)),
)
else:
with T.serial(0, nkeep_local[T.tensor_indices(nkeep_local, 0)]) as j:
@@ -558,8 +558,8 @@ def _classic_nms_ir(
with T.then_():
T.tensor_store(
num_valid_boxes,
- num_valid_boxes[T.tensor_indices(num_valid_boxes, 0)] + 1,
T.tensor_indices(num_valid_boxes, 0),
+ num_valid_boxes[T.tensor_indices(num_valid_boxes, 0)] + 1,
)
with T.serial(0, nkeep_local[T.tensor_indices(nkeep_local, 0)]) as k:
@@ -589,34 +589,34 @@ def _classic_nms_ir(
with T.then_():
T.tensor_store(
out_data,
+ (i, k, score_index),
tvm.tirx.Cast(
data.dtype, T.float32(-1.0)
),
- (i, k, score_index),
)
T.tensor_store(
out_box_indices,
- T.int32(-1),
(i, k),
+ T.int32(-1),
)
with T.else_():
with T.serial(0, box_data_length) as k:
T.tensor_store(
out_data,
- tvm.tirx.Cast(data.dtype, T.float32(-1.0)),
(i, j, k),
+ tvm.tirx.Cast(data.dtype, T.float32(-1.0)),
)
T.tensor_store(
out_box_indices,
- T.int32(-1),
(i, j),
+ T.int32(-1),
)
if return_indices:
valid_idx_buf = T.alloc_tensor((1,), "int32", scope="local")
valid_idx = valid_idx_buf
- T.tensor_store(valid_idx, T.int32(0), T.tensor_indices(valid_idx, 0))
+ T.tensor_store(valid_idx, T.tensor_indices(valid_idx, 0), T.int32(0))
with T.serial(0, num_anchors) as j:
with T.if_(out_box_indices[(i, j)] >= 0):
@@ -624,19 +624,19 @@ def _classic_nms_ir(
orig_idx = out_box_indices[(i, j)]
T.tensor_store(
out_box_indices,
- indices[(i, orig_idx)],
(i, valid_idx[T.tensor_indices(valid_idx, 0)]),
+ indices[(i, orig_idx)],
)
T.tensor_store(
valid_idx,
- valid_idx[T.tensor_indices(valid_idx, 0)] + 1,
T.tensor_indices(valid_idx, 0),
+ valid_idx[T.tensor_indices(valid_idx, 0)] + 1,
)
T.tensor_store(
out_valid_box_count,
- valid_idx[T.tensor_indices(valid_idx, 0)],
(i, 0),
+ valid_idx[T.tensor_indices(valid_idx, 0)],
)
with T.serial(0, num_anchors) as j:
@@ -644,8 +644,8 @@ def _classic_nms_ir(
with T.then_():
T.tensor_store(
out_box_indices,
- T.int32(-1),
(i, j),
+ T.int32(-1),
)
return ib.get()
@@ -853,7 +853,7 @@ def _rearrange_out(data, batch_size, num_anchors, box_data_length, score_index):
with T.parallel(0, batch_size) as i:
valid_idx_buf = T.alloc_tensor((1,), "int32", scope="local")
valid_idx = valid_idx_buf
- T.tensor_store(valid_idx, T.int32(0), T.tensor_indices(valid_idx, 0))
+ T.tensor_store(valid_idx, T.tensor_indices(valid_idx, 0), T.int32(0))
with T.serial(0, num_anchors) as j:
with T.if_(
@@ -863,13 +863,13 @@ def _rearrange_out(data, batch_size, num_anchors, box_data_length, score_index):
with T.serial(0, box_data_length) as k:
T.tensor_store(
out,
- data[(i, j, k)],
(i, valid_idx[T.tensor_indices(valid_idx, 0)], k),
+ data[(i, j, k)],
)
T.tensor_store(
valid_idx,
- valid_idx[T.tensor_indices(valid_idx, 0)] + 1,
T.tensor_indices(valid_idx, 0),
+ valid_idx[T.tensor_indices(valid_idx, 0)] + 1,
)
with T.serial(0, num_anchors) as j:
@@ -878,8 +878,8 @@ def _rearrange_out(data, batch_size, num_anchors, box_data_length, score_index):
with T.serial(0, box_data_length) as k:
T.tensor_store(
out,
- tvm.tirx.Cast(data.dtype, T.float32(-1.0)),
(i, j, k),
+ tvm.tirx.Cast(data.dtype, T.float32(-1.0)),
)
return ib.get()
@@ -917,8 +917,8 @@ def _nms_loop(
)
T.tensor_store(
num_valid_boxes_local,
- num_valid_boxes_local[T.tensor_indices(num_valid_boxes_local, 0)] + 1,
T.tensor_indices(num_valid_boxes_local, 0),
+ num_valid_boxes_local[T.tensor_indices(num_valid_boxes_local, 0)] + 1,
)
num_boxes_to_check = nkeep - (j + 1)
@@ -938,7 +938,7 @@ def _nms_loop(
with T.if_(iou >= iou_threshold):
with T.then_():
- T.tensor_store(out_scores, T.float32(-1.0), (i, k))
+ T.tensor_store(out_scores, (i, k), T.float32(-1.0))
on_new_invalidated_box_func(i, k)
with T.serial(0, batch_size) as i:
@@ -949,7 +949,7 @@ def _nms_loop(
num_valid_boxes_local_buf = T.alloc_tensor((1,), "int32", scope="local")
num_valid_boxes_local = num_valid_boxes_local_buf
T.tensor_store(
- num_valid_boxes_local, T.int32(0), T.tensor_indices(num_valid_boxes_local, 0)
+ num_valid_boxes_local, T.tensor_indices(num_valid_boxes_local, 0), T.int32(0)
)
with T.serial(0, nkeep) as j:
@@ -970,12 +970,12 @@ def _nms_loop(
T.tensor_store(
num_valid_boxes,
- num_valid_boxes_local[T.tensor_indices(num_valid_boxes_local, 0)],
T.tensor_indices(num_valid_boxes, i),
+ num_valid_boxes_local[T.tensor_indices(num_valid_boxes_local, 0)],
)
with T.else_():
- T.tensor_store(num_valid_boxes, T.int32(0), T.tensor_indices(num_valid_boxes, i))
+ T.tensor_store(num_valid_boxes, T.tensor_indices(num_valid_boxes, i), T.int32(0))
def _get_valid_box_count(scores, score_threshold):
@@ -1067,7 +1067,7 @@ def _collect_selected_indices_ir(
max_output_rows = batch_classes * 10
with T.serial(0, max_output_rows) as init_i:
with T.serial(0, 3) as init_j: # 3 columns
- T.tensor_store(out, cast(0, "int64"), (init_i, init_j))
+ T.tensor_store(out, (init_i, init_j), cast(0, "int64"))
with T.parallel(0, batch_classes) as i:
i_64 = cast(i, "int64")
@@ -1088,12 +1088,12 @@ def _collect_selected_indices_ir(
limit = num_detections[i]
with T.serial(0, limit) as j:
- T.tensor_store(out, batch_id, (row_offsets[i] + j, 0))
- T.tensor_store(out, class_id, (row_offsets[i] + j, 1))
+ T.tensor_store(out, (row_offsets[i] + j, 0), batch_id)
+ T.tensor_store(out, (row_offsets[i] + j, 1), class_id)
T.tensor_store(
out,
- cast(selected_indices[i, j], "int64"),
(row_offsets[i] + j, 2),
+ cast(selected_indices[i, j], "int64"),
)
return ib.get()
@@ -1125,18 +1125,18 @@ def _collect_selected_indices_and_scores_ir(
offset = row_offsets[batch_id, class_id] + j
T.tensor_store(
collected_indices,
- class_id,
(batch_id, offset, 0),
+ class_id,
)
T.tensor_store(
collected_indices,
- cast(selected_indices[i, j], "int64"),
(batch_id, offset, 1),
+ cast(selected_indices[i, j], "int64"),
)
T.tensor_store(
collected_scores,
- selected_scores[i, j],
(batch_id, offset),
+ selected_scores[i, j],
)
with T.else_():
offset = (
@@ -1148,18 +1148,18 @@ def _collect_selected_indices_and_scores_ir(
)
T.tensor_store(
collected_indices,
- zero,
(batch_id, offset, 0),
+ zero,
)
T.tensor_store(
collected_indices,
- zero,
(batch_id, offset, 1),
+ zero,
)
T.tensor_store(
collected_scores,
- T.float32(0.0),
(batch_id, offset),
+ T.float32(0.0),
)
return ib.get()
diff --git a/python/tvm/topi/vision/nms_util.py b/python/tvm/topi/vision/nms_util.py
index 3227479ff5..681678d494 100644
--- a/python/tvm/topi/vision/nms_util.py
+++ b/python/tvm/topi/vision/nms_util.py
@@ -71,16 +71,16 @@ def binary_search(y, num_boxes, scores, score_threshold, out):
hi_buf = T.decl_tensor([1], "int32", scope="local")
lo = lo_buf
hi = hi_buf
- T.tensor_store(lo, T.int32(0), T.tensor_indices(lo, 0))
- T.tensor_store(hi, tvm.tirx.Cast("int32", num_boxes), T.tensor_indices(hi, 0))
+ T.tensor_store(lo, T.tensor_indices(lo, 0), T.int32(0))
+ T.tensor_store(hi, T.tensor_indices(hi, 0), tvm.tirx.Cast("int32", num_boxes))
with T.while_(lo[T.tensor_indices(lo, 0)] < hi[T.tensor_indices(hi, 0)]):
mid = (hi[T.tensor_indices(hi, 0)] + lo[T.tensor_indices(lo, 0)]) >> 1
with T.if_(scores[y, mid] > score_threshold):
with T.then_():
- T.tensor_store(lo, mid + 1, T.tensor_indices(lo, 0))
+ T.tensor_store(lo, T.tensor_indices(lo, 0), mid + 1)
with T.else_():
- T.tensor_store(hi, mid, T.tensor_indices(hi, 0))
- T.tensor_store(out, lo[T.tensor_indices(lo, 0)], T.tensor_indices(out, y))
+ T.tensor_store(hi, T.tensor_indices(hi, 0), mid)
+ T.tensor_store(out, T.tensor_indices(out, y), lo[T.tensor_indices(lo, 0)])
def _estimate_max_detections(batch_class, input_image_size=None):
@@ -340,14 +340,14 @@ def _all_class_nms_ir(
with T.then_():
T.tensor_store(
box_indices,
- sorted_indices[i, j],
(i, num_current_valid_box),
+ sorted_indices[i, j],
)
if selected_scores is not None:
T.tensor_store(
selected_scores,
- sorted_scores[i, j],
(i, num_current_valid_box),
+ sorted_scores[i, j],
)
def on_new_invalidated_box(*_):
diff --git a/src/backend/cuda/codegen/codegen_cuda.cc b/src/backend/cuda/codegen/codegen_cuda.cc
index ce76e2983e..3704226b7d 100644
--- a/src/backend/cuda/codegen/codegen_cuda.cc
+++ b/src/backend/cuda/codegen/codegen_cuda.cc
@@ -372,7 +372,7 @@ std::string CodeGenCUDA::Finish() {
return CodeGenC::Finish();
}
-void CodeGenCUDA::Dispatch_(const tirx::ForNode* op) {
+void CodeGenCUDA::Dispatch_(const ForNode* op) {
// Materialize the loop bounds before emitting an unroll pragma. PrintExpr
// may introduce temporaries (for example, for a Select expression). CUDA
// requires #pragma unroll to immediately precede the loop it controls; if
@@ -385,7 +385,7 @@ void CodeGenCUDA::Dispatch_(const tirx::ForNode* op) {
if (op->annotations.count("disable_unroll")) {
PrintIndent();
stream << "#pragma unroll 1\n";
- } else if (op->kind == tirx::ForKind::kUnrolled) {
+ } else if (op->kind == ForKind::kUnrolled) {
PrintIndent();
stream << "#pragma unroll\n";
} else if (auto it = op->annotations.find("pragma_unroll"); it != op->annotations.end()) {
diff --git a/src/backend/cuda/transforms/lower_iket.cc b/src/backend/cuda/transforms/lower_iket.cc
index 93ec1484fb..da5d35ac70 100644
--- a/src/backend/cuda/transforms/lower_iket.cc
+++ b/src/backend/cuda/transforms/lower_iket.cc
@@ -369,7 +369,8 @@ class TokenBufferCollector : public StmtExprVisitor {
} else if (const auto* load = store->value.as<TensorLoadNode>()) {
is_token_value = buffers_->count(load->source.as_or_throw<tvm::tirx::TensorVar>().get());
}
- if (is_token_value && buffers_->insert(store->buffer.get()).second) changed = true;
+ if (is_token_value && buffers_->insert(store->dest.as_or_throw<TensorVar>().get()).second)
+ changed = true;
return StmtExprVisitor::Visit_(store);
}
@@ -406,7 +407,7 @@ class TokenDeclarationCollector : public StmtExprVisitor {
if (it != declarations_->end()) possible = it->second;
}
if (!possible.empty()) {
- auto& target = (*declarations_)[store->buffer.get()];
+ auto& target = (*declarations_)[store->dest.as_or_throw<TensorVar>().get()];
size_t old_size = target.size();
target.insert(possible.begin(), possible.end());
changed = changed || target.size() != old_size;
@@ -491,17 +492,17 @@ class TokenVerifier : public StmtExprVisitor {
private:
ffi::Optional<VisitInterrupt> Visit_(const TensorStoreNode* store) final {
- if (!token_buffers_.count(store->buffer.get())) {
+ if (!token_buffers_.count(store->dest.as_or_throw<TensorVar>().get())) {
return StmtExprVisitor::Visit_(store);
}
- TVM_FFI_CHECK(store->buffer->dtype->dtype.code == kDLUInt &&
- store->buffer->dtype->dtype.bits == 32 &&
- store->buffer->dtype->dtype.lanes == 1,
+ TVM_FFI_CHECK(store->dest.as_or_throw<TensorVar>()->dtype->dtype.code == kDLUInt &&
+ store->dest.as_or_throw<TensorVar>()->dtype->dtype.bits == 32 &&
+ store->dest.as_or_throw<TensorVar>()->dtype->dtype.lanes == 1,
TypeError)
<< "IKET RangeToken storage must have dtype uint32";
- TVM_FFI_CHECK(
- store->buffer.scope() == "local" && IsScalarBufferAccess(store->buffer, store->indices),
- ValueError)
+ TVM_FFI_CHECK(store->dest.as_or_throw<TensorVar>().scope() == "local" &&
+ IsScalarBufferAccess(store->dest.as_or_throw<TensorVar>(), store->indices),
+ ValueError)
<< "IKET RangeToken must use element zero of a local uint32[1] buffer";
bool valid_value = false;
if (const auto* call = store->value.as<CallNode>()) {
@@ -581,7 +582,7 @@ class StripIket : public StmtExprMutator {
}
UnchangedOr<Stmt> Mutate_(const TensorStoreNode* store, InplaceMode inplace_mode) final {
- if (token_buffers_.count(store->buffer.get())) return Evaluate(0);
+ if (token_buffers_.count(store->dest.as_or_throw<TensorVar>().get())) return Evaluate(0);
return StmtExprMutator::Mutate_(store, inplace_mode);
}
@@ -623,9 +624,9 @@ class RemoveStrippedIketNoOps : public StmtExprMutator {
return result;
}
- UnchangedOr<Stmt> Mutate_(const IfThenElseNode* branch, InplaceMode inplace_mode) final {
+ UnchangedOr<Stmt> Mutate_(const IfNode* branch, InplaceMode inplace_mode) final {
auto result = StmtExprMutator::Mutate_(branch, inplace_mode);
- if (!result.IsUnchanged()) branch = ffi::AnyView(result).as<IfThenElseNode>();
+ if (!result.IsUnchanged()) branch = ffi::AnyView(result).as<IfNode>();
if (!branch->else_case.has_value()) {
if (branch->then_case->seq.empty()) return PreserveConditionEffects(branch->condition);
return result;
@@ -634,10 +635,10 @@ class RemoveStrippedIketNoOps : public StmtExprMutator {
bool empty_else = branch->else_case.value()->seq.empty();
if (empty_then && empty_else) return PreserveConditionEffects(branch->condition);
if (empty_else) {
- return IfThenElse(branch->condition, branch->then_case, std::nullopt, branch->span);
+ return If(branch->condition, branch->then_case, std::nullopt, branch->span);
}
if (empty_then) {
- return IfThenElse(!branch->condition, branch->else_case.value(), std::nullopt, branch->span);
+ return If(!branch->condition, branch->else_case.value(), std::nullopt, branch->span);
}
return result;
}
@@ -1112,7 +1113,7 @@ class InstrumentOfficialKernel : public StmtExprMutator {
PrimExpr payload = NormalizePayload(
Mutate(call->args[1]).ValueOrUnchanged(call->args[1]).as_or_throw<PrimExpr>(),
payload_type);
- return IfThenElse(not_equal(token, 0), Evaluate(Event(token, std::move(payload))));
+ return If(not_equal(token, 0), Evaluate(Event(token, std::move(payload))));
}
return Evaluate(Event(token));
}
diff --git a/src/backend/trn/codegen/codegen_trn.cc b/src/backend/trn/codegen/codegen_trn.cc
index 0c0ac5db5b..873a56b019 100644
--- a/src/backend/trn/codegen/codegen_trn.cc
+++ b/src/backend/trn/codegen/codegen_trn.cc
@@ -706,7 +706,7 @@ ffi::Module BuildTrainium(IRModule mod, Target target) {
return codegen::DeviceSourceModuleCreate(source_maker.str(), fmt, ExtractFuncInfo(mod), "nki");
}
-void CodeGenTrainium::Dispatch_(const IfThenElseNode* op) {
+void CodeGenTrainium::Dispatch_(const IfNode* op) {
if (ctx_.tensorizing) {
TVM_FFI_ICHECK(!op->else_case.has_value()) << "Else not allowed in tensorized instruction";
TVM_FFI_ICHECK(!ctx_.mask.defined()) << "Only one if stmt allowed in tensorized instruction";
diff --git a/src/backend/trn/codegen/codegen_trn.h b/src/backend/trn/codegen/codegen_trn.h
index c051a10747..b0ca3d043b 100644
--- a/src/backend/trn/codegen/codegen_trn.h
+++ b/src/backend/trn/codegen/codegen_trn.h
@@ -72,7 +72,7 @@ class CodeGenTrainium final : public CodeGenC {
void Dispatch_(const prim::FloorDivNode* op, std::ostream& os) final; // NOLINT(*)
void Dispatch_(const prim::FloorModNode* op, std::ostream& os) final; // NOLINT(*)
void DispatchDeclTensor(const BindNode* op, const CallNode* buffer_call);
- void Dispatch_(const IfThenElseNode* op) final; // NOLINT(*)
+ void Dispatch_(const IfNode* op) final; // NOLINT(*)
void Dispatch_(const prim::AndNode* op, std::ostream& os) final; // NOLINT(*)
void Dispatch_(const prim::OrNode* op, std::ostream& os) final; // NOLINT(*)
diff --git a/src/backend/trn/transform/lower_trainium_layout.cc b/src/backend/trn/transform/lower_trainium_layout.cc
index 9617e55e53..30541a9f81 100644
--- a/src/backend/trn/transform/lower_trainium_layout.cc
+++ b/src/backend/trn/transform/lower_trainium_layout.cc
@@ -215,7 +215,7 @@ class TrainiumLayoutApplier : public tirx::IRMutatorWithAnalyzer {
UnchangedOr<Stmt> Mutate_(const TensorStoreNode* op, InplaceMode inplace_mode) final {
// Index conversion needs the original logical layout after the parent remaps the buffer.
- TensorVar logical_buffer = op->buffer;
+ TensorVar logical_buffer = op->dest.as_or_throw<TensorVar>();
TensorStore store = StmtExprMutator::Mutate_(op, inplace_mode)
.ValueOrUnchanged(ffi::GetRef<Stmt>(op))
.as_or_throw<TensorStore>();
@@ -224,7 +224,8 @@ class TrainiumLayoutApplier : public tirx::IRMutatorWithAnalyzer {
store = VisitBufferAccess(store, logical_buffer);
if (store_returns_bool) {
- TVM_FFI_ICHECK_EQ(store->buffer->dtype->dtype, (DLDataType{kDLInt, 8, 1}))
+ TVM_FFI_ICHECK_EQ(store->dest.as_or_throw<TensorVar>()->dtype->dtype,
+ (DLDataType{kDLInt, 8, 1}))
<< "Expected int8 backing array for boolean tensor";
auto writer = store.CopyOnWrite();
writer->value = tvm::prim::cast(PrimType::Int(8), store->value);
@@ -309,7 +310,7 @@ class TrainiumLayoutApplier : public tirx::IRMutatorWithAnalyzer {
auto flattened_indices = GetSimplifiedElemOffset(logical_buffer, node->indices);
TensorVar flattened_buffer = GetFlattenedTensor(logical_buffer);
auto writer = node.CopyOnWrite();
- writer->buffer = flattened_buffer;
+ writer->dest = flattened_buffer;
writer->indices = flattened_indices;
return node;
}
diff --git a/src/backend/vulkan/codegen/codegen_spirv.cc b/src/backend/vulkan/codegen/codegen_spirv.cc
index 29413a5d46..70b248ae1a 100644
--- a/src/backend/vulkan/codegen/codegen_spirv.cc
+++ b/src/backend/vulkan/codegen/codegen_spirv.cc
@@ -716,7 +716,7 @@ spirv::Value CodeGenSPIRV::Dispatch_(const prim::ShuffleNode* op) {
void CodeGenSPIRV::Dispatch_(const TensorStoreNode* op) {
TVM_FFI_ICHECK_EQ(op->indices.size(), 1) << "SPIR-V codegen expects flat memory buffers";
- Var buffer_var = op->buffer.var();
+ Var buffer_var = op->dest.as_or_throw<TensorVar>().var();
PrimExpr prim_index = op->indices[0];
auto it = storage_info_.find(buffer_var.get());
@@ -853,7 +853,7 @@ void CodeGenSPIRV::Dispatch_(const WhileNode* op) {
builder_->StartLabel(merge_label);
}
-void CodeGenSPIRV::Dispatch_(const IfThenElseNode* op) {
+void CodeGenSPIRV::Dispatch_(const IfNode* op) {
spirv::Value cond = MakeValue(op->condition);
spirv::Label then_label = builder_->NewLabel();
spirv::Label merge_label = builder_->NewLabel();
diff --git a/src/backend/vulkan/codegen/codegen_spirv.h b/src/backend/vulkan/codegen/codegen_spirv.h
index d6e20efbd5..38e1837103 100644
--- a/src/backend/vulkan/codegen/codegen_spirv.h
+++ b/src/backend/vulkan/codegen/codegen_spirv.h
@@ -117,7 +117,7 @@ class CodeGenSPIRV : public tvm::ExprFunctor<spirv::Value(const Expr&)>,
void Dispatch_(const TensorStoreNode* op) override;
void Dispatch_(const ForNode* op) override;
void Dispatch_(const WhileNode* op) override;
- void Dispatch_(const IfThenElseNode* op) override;
+ void Dispatch_(const IfNode* op) override;
void DispatchDeclTensor(const BindNode* op, const CallNode* buffer_call);
void DispatchAllocTensor(const BindNode* op, const CallNode* buffer_call);
void Dispatch_(const RegionStmtNode* op) override;
diff --git a/src/backend/webgpu/codegen/codegen_webgpu.cc b/src/backend/webgpu/codegen/codegen_webgpu.cc
index 46275e4fa9..bee599d67f 100644
--- a/src/backend/webgpu/codegen/codegen_webgpu.cc
+++ b/src/backend/webgpu/codegen/codegen_webgpu.cc
@@ -120,7 +120,7 @@ class WebGPUWorkgroupInfoCollector : public StmtExprVisitor {
ffi::Optional<VisitInterrupt> Visit_(const TensorStoreNode* op) final {
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(StmtExprVisitor::Visit_(op));
- info_.write_access_set.insert(ResolveBuffer(op->buffer.var()));
+ info_.write_access_set.insert(ResolveBuffer(op->dest.as_or_throw<TensorVar>().var()));
return std::nullopt;
}
@@ -711,9 +711,9 @@ void CodeGenWebGPU::Dispatch_(const TensorStoreNode* op) {
TVM_FFI_ICHECK_EQ(op->indices.size(), 1) << "Store to non-flat memory not supported.";
PrimType value_ty = op->value.ty();
- const PrimType& element_ty = op->buffer->dtype;
+ const PrimType& element_ty = op->dest.as_or_throw<TensorVar>()->dtype;
PrimExpr index = op->indices[0];
- Var buffer_var = op->buffer.var();
+ Var buffer_var = op->dest.as_or_throw<TensorVar>().var();
std::string buffer_vid = GetVarID(buffer_var.get());
diff --git a/src/tirx/ir/seq_stmt_mutate.h b/src/ir/seq_stmt_mutate.h
similarity index 97%
rename from src/tirx/ir/seq_stmt_mutate.h
rename to src/ir/seq_stmt_mutate.h
index 7e32394f94..8d6079b065 100644
--- a/src/tirx/ir/seq_stmt_mutate.h
+++ b/src/ir/seq_stmt_mutate.h
@@ -17,11 +17,11 @@
* under the License.
*/
-#ifndef TVM_TIRX_IR_SEQ_STMT_MUTATE_H_
-#define TVM_TIRX_IR_SEQ_STMT_MUTATE_H_
+#ifndef TVM_IR_SEQ_STMT_MUTATE_H_
+#define TVM_IR_SEQ_STMT_MUTATE_H_
#include <tvm/ffi/extra/structural_mutate.h>
-#include <tvm/tirx/stmt.h>
+#include <tvm/ir/stmt.h>
#include <iterator>
#include <type_traits>
@@ -29,7 +29,6 @@
#include <vector>
namespace tvm {
-namespace tirx {
namespace detail {
inline bool IsSeqStmtNoOp(ffi::AnyView stmt) {
@@ -229,7 +228,6 @@ auto MutateSeqStmt(const SeqStmtNode* self, ffi::InplaceMode inplace_mode,
}
} // namespace detail
-} // namespace tirx
} // namespace tvm
-#endif // TVM_TIRX_IR_SEQ_STMT_MUTATE_H_
+#endif // TVM_IR_SEQ_STMT_MUTATE_H_
diff --git a/src/tirx/ir/stmt.cc b/src/ir/stmt.cc
similarity index 82%
copy from src/tirx/ir/stmt.cc
copy to src/ir/stmt.cc
index b0e043be48..470ecabb6f 100644
--- a/src/tirx/ir/stmt.cc
+++ b/src/ir/stmt.cc
@@ -18,7 +18,7 @@
*/
/*!
- * \file tvm/tirx/stmt.cc
+ * \file tvm/ir/stmt.cc
*/
#include <tvm/ffi/dtype.h>
#include <tvm/ffi/extra/structural_mutate.h>
@@ -27,9 +27,7 @@
#include <tvm/ffi/reflection/registry.h>
#include <tvm/ir/op.h>
#include <tvm/ir/prim/op.h>
-#include <tvm/tirx/op/region.h>
-#include <tvm/tirx/op_attr_types.h>
-#include <tvm/tirx/stmt.h>
+#include <tvm/ir/stmt.h>
#include <iterator>
#include <limits>
@@ -37,11 +35,9 @@
#include <utility>
#include <vector>
-#include "buffer_common.h"
#include "seq_stmt_mutate.h"
namespace tvm {
-namespace tirx {
using namespace tvm::prim;
namespace {
@@ -494,20 +490,20 @@ TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> SeqStmtMaybeInplaceMuta
return MutateSeqStmtStructural(mutator, value, ffi::InplaceMode::kAllow);
}
-TVM_FFI_INLINE ffi::Expected<ffi::Optional<ffi::VisitInterrupt>> IfThenElseVisit(
+TVM_FFI_INLINE ffi::Expected<ffi::Optional<ffi::VisitInterrupt>> IfVisit(
ffi::StructuralVisitorObj* visitor, ffi::AnyView value) noexcept {
- const IfThenElseNode* self =
- ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const IfThenElseNode>(value);
+ const IfNode* self =
+ ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const IfNode>(value);
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(visitor->VisitExpected(self->condition));
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(visitor->VisitExpected(self->then_case));
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(visitor->VisitExpected(self->else_case));
return std::nullopt;
}
-TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> IfThenElseMutate(
+TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> IfMutate(
ffi::StructuralMutatorObj* mutator, ffi::AnyView value) noexcept {
- const IfThenElseNode* self =
- ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const IfThenElseNode>(value);
+ const IfNode* self =
+ ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const IfNode>(value);
TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<PrimExpr>, mapped_condition,
mutator->MutateExpected(self->condition));
TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<Stmt>, mapped_then_case,
@@ -519,7 +515,7 @@ TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> IfThenElseMutate(
(mapped_else_case.IsUnchanged() || ffi::AnyView(mapped_else_case).same_as(self->else_case))) {
return ffi::Unchanged();
}
- ffi::ObjectPtr<IfThenElseNode> copy = ffi::make_object<IfThenElseNode>(*self);
+ ffi::ObjectPtr<IfNode> copy = ffi::make_object<IfNode>(*self);
copy->condition = std::move(mapped_condition).ValueOrUnchanged(std::move(copy->condition));
if (!mapped_then_case.IsUnchanged())
copy->then_case = SeqStmt(std::move(mapped_then_case).ValueUnchecked());
@@ -532,10 +528,10 @@ TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> IfThenElseMutate(
return ffi::Any(std::move(copy));
}
-TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> IfThenElseMaybeInplaceMutate(
+TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> IfMaybeInplaceMutate(
ffi::StructuralMutatorObj* mutator, ffi::AnyView value) noexcept {
- IfThenElseNode* self = const_cast<IfThenElseNode*>(
- ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const IfThenElseNode>(value));
+ IfNode* self = const_cast<IfNode*>(
+ ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const IfNode>(value));
TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(
ffi::UnchangedOr<PrimExpr>, mapped_condition,
mutator->MutateExpected(self->condition, ffi::InplaceMode::kAllow));
@@ -602,9 +598,9 @@ TVM_FFI_INLINE ffi::Expected<ffi::Optional<ffi::VisitInterrupt>> TensorStoreVisi
ffi::StructuralVisitorObj* visitor, ffi::AnyView value) noexcept {
const TensorStoreNode* self =
ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const TensorStoreNode>(value);
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(visitor->VisitExpected(self->buffer));
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(visitor->VisitExpected(self->value));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(visitor->VisitExpected(self->dest));
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(visitor->VisitExpected(self->indices));
+ TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(visitor->VisitExpected(self->value));
return std::nullopt;
}
@@ -612,19 +608,18 @@ TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> TensorStoreMutate(
ffi::StructuralMutatorObj* mutator, ffi::AnyView value) noexcept {
const TensorStoreNode* self =
ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const TensorStoreNode>(value);
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<TensorVar>, mapped_buffer,
- mutator->MutateExpected(self->buffer));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<PrimExpr>, mapped_value,
- mutator->MutateExpected(self->value));
+ TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<Expr>, mapped_dest,
+ mutator->MutateExpected(self->dest));
TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<ffi::Array<PrimExpr>>, mapped_indices,
mutator->MutateExpected(self->indices));
- if (mapped_buffer.UnchangedOrSameAs(self->buffer) &&
- mapped_value.UnchangedOrSameAs(self->value) &&
+ TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<PrimExpr>, mapped_value,
+ mutator->MutateExpected(self->value));
+ if (mapped_dest.UnchangedOrSameAs(self->dest) && mapped_value.UnchangedOrSameAs(self->value) &&
mapped_indices.UnchangedOrSameAs(self->indices)) {
return ffi::Unchanged();
}
ffi::ObjectPtr<TensorStoreNode> copy = ffi::make_object<TensorStoreNode>(*self);
- copy->buffer = std::move(mapped_buffer).ValueOrUnchanged(std::move(copy->buffer));
+ copy->dest = std::move(mapped_dest).ValueOrUnchanged(std::move(copy->dest));
copy->value = std::move(mapped_value).ValueOrUnchanged(std::move(copy->value));
copy->indices = std::move(mapped_indices).ValueOrUnchanged(std::move(copy->indices));
return ffi::Any(std::move(copy));
@@ -634,64 +629,30 @@ TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> TensorStoreMaybeInplace
ffi::StructuralMutatorObj* mutator, ffi::AnyView value) noexcept {
TensorStoreNode* self = const_cast<TensorStoreNode*>(
ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const TensorStoreNode>(value));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(
- ffi::UnchangedOr<TensorVar>, mapped_buffer,
- mutator->MutateExpected(self->buffer, ffi::InplaceMode::kAllow));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<PrimExpr>, mapped_value,
- mutator->MutateExpected(self->value, ffi::InplaceMode::kAllow));
+ TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<Expr>, mapped_dest,
+ mutator->MutateExpected(self->dest, ffi::InplaceMode::kAllow));
TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(
ffi::UnchangedOr<ffi::Array<PrimExpr>>, mapped_indices,
mutator->MutateExpected(self->indices, ffi::InplaceMode::kAllow));
- if (mapped_buffer.UnchangedOrSameAs(self->buffer) &&
- mapped_value.UnchangedOrSameAs(self->value) &&
+ TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<PrimExpr>, mapped_value,
+ mutator->MutateExpected(self->value, ffi::InplaceMode::kAllow));
+ if (mapped_dest.UnchangedOrSameAs(self->dest) && mapped_value.UnchangedOrSameAs(self->value) &&
mapped_indices.UnchangedOrSameAs(self->indices)) {
return ffi::Unchanged();
}
- if (!mapped_buffer.IsUnchanged()) self->buffer = std::move(mapped_buffer).ValueUnchecked();
+ if (!mapped_dest.IsUnchanged()) self->dest = std::move(mapped_dest).ValueUnchecked();
if (!mapped_value.IsUnchanged()) self->value = std::move(mapped_value).ValueUnchecked();
if (!mapped_indices.IsUnchanged()) self->indices = std::move(mapped_indices).ValueUnchecked();
return ffi::Unchanged();
}
-TVM_FFI_INLINE ffi::Expected<ffi::Optional<ffi::VisitInterrupt>> ScopeIdDefStmtVisit(
- ffi::StructuralVisitorObj* visitor, ffi::AnyView value) noexcept {
- const ScopeIdDefStmtNode* self =
- ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const ScopeIdDefStmtNode>(value);
- // ScopeIdDef reflection marks only def_ids as definitions; its extent fields remain uses.
- TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(visitor->VisitExpected(self->def));
- return std::nullopt;
-}
-
-TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> ScopeIdDefStmtMutate(
- ffi::StructuralMutatorObj* mutator, ffi::AnyView value) noexcept {
- const ScopeIdDefStmtNode* self =
- ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const ScopeIdDefStmtNode>(value);
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<ScopeIdDef>, mapped_def,
- mutator->MutateExpected(self->def));
- if (mapped_def.UnchangedOrSameAs(self->def)) {
- return ffi::Unchanged();
- }
- ffi::ObjectPtr<ScopeIdDefStmtNode> copy = ffi::make_object<ScopeIdDefStmtNode>(*self);
- copy->def = std::move(mapped_def).ValueOrUnchanged(std::move(copy->def));
- return ffi::Any(std::move(copy));
-}
-
-TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> ScopeIdDefStmtMaybeInplaceMutate(
- ffi::StructuralMutatorObj* mutator, ffi::AnyView value) noexcept {
- ScopeIdDefStmtNode* self = const_cast<ScopeIdDefStmtNode*>(
- ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const ScopeIdDefStmtNode>(value));
- TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<ScopeIdDef>, mapped_def,
- mutator->MutateExpected(self->def, ffi::InplaceMode::kAllow));
- if (mapped_def.UnchangedOrSameAs(self->def)) {
- return ffi::Unchanged();
- }
- if (!mapped_def.IsUnchanged()) self->def = std::move(mapped_def).ValueUnchecked();
- return ffi::Unchanged();
-}
-
} // namespace
-TVM_FFI_STATIC_INIT_BLOCK() { StmtNode::RegisterReflection(); }
+TVM_FFI_STATIC_INIT_BLOCK() {
+ StmtNode::RegisterReflection();
+ ffi::reflection::EnsureTypeAttrColumn("__tensor_store_validate__");
+ ffi::reflection::EnsureTypeAttrColumn("__evaluate_validate__");
+}
// Bind
Bind::Bind(Var var, Expr value, Span span) : Stmt(ffi::UnsafeInit{}) {
@@ -712,7 +673,7 @@ TVM_FFI_STATIC_INIT_BLOCK() {
.attr(refl::type_attr::kStructuralMaybeInplaceMutate,
ffi::FStructuralMutate::FromNative<&BindMaybeInplaceMutate>());
- refl::GlobalDef().def("tirx.Bind",
+ refl::GlobalDef().def("ir.Bind",
[](Var var, Expr value, Span span) { return Bind(var, value, span); });
}
@@ -756,11 +717,6 @@ RegionStmt::RegionStmt(Op op, ffi::Array<Expr> args, ffi::Array<Var> body_params
TVM_FFI_CHECK(ffi::StructuralEqual()(body_params[i]->ty, expected_params[i]->ty), ValueError)
<< op->name << " body parameter " << i << " has a type inconsistent with its contract";
}
- if (op.same_as(tirx::device_context_op()) || op.same_as(tirx::compute_scope_op()) ||
- op.same_as(tirx::parallel_launch_op())) {
- TVM_FFI_CHECK(result_vars.empty() && attrs->dict.empty(), ValueError)
- << op->name << " expects no results or attributes";
- }
std::unordered_set<const VarNode*> definitions;
for (const auto& vars : {body_params, result_vars}) {
for (const Var& var : vars) {
@@ -768,23 +724,16 @@ RegionStmt::RegionStmt(Op op, ffi::Array<Expr> args, ffi::Array<Var> body_params
<< "RegionStmt parameters and results must be distinct definitions";
}
}
- static const Op device_scope = Op::Get("tirx.device_scope");
- if (op.same_as(tirx::device_entry_op()) || op.same_as(device_scope)) {
- TVM_FFI_CHECK(result_vars.empty(), ValueError) << op->name << " expects no results";
- if (op.same_as(tirx::device_entry_op())) {
- TVM_FFI_CHECK(attrs->dict.empty(), ValueError) << "device_entry expects no attrs";
- }
- }
- if (op.same_as(tirx::launch_thread_op())) {
- TVM_FFI_CHECK(result_vars.empty() && attrs->dict.empty(), ValueError)
- << "launch_thread expects no results or attrs";
- }
auto n = ffi::make_object<RegionStmtNode>(std::move(op), std::move(body));
n->args = std::move(args);
n->body_params = std::move(body_params);
n->attrs = std::move(attrs);
n->result_vars = std::move(result_vars);
n->span = std::move(span);
+ if (Op::HasAttrMap("FRegionValidate")) {
+ static auto validate = Op::GetAttrMap<FRegionValidate>("FRegionValidate");
+ if (validate.count(n->op)) validate[n->op].CallExpected(n.get()).value();
+ }
data_ = std::move(n);
}
@@ -798,7 +747,7 @@ TVM_FFI_STATIC_INIT_BLOCK() {
ffi::FStructuralMutate::FromNative<&RegionStmtMutate>())
.attr(refl::type_attr::kStructuralMaybeInplaceMutate,
ffi::FStructuralMutate::FromNative<&RegionStmtMaybeInplaceMutate>());
- refl::GlobalDef().def("tirx.RegionStmt",
+ refl::GlobalDef().def("ir.RegionStmt",
[](Op op, ffi::Array<Expr> args, ffi::Array<Var> body_params,
DictAttrs attrs, SeqStmt body, ffi::Array<Var> result_vars, Span span) {
return RegionStmt(op, args, body_params, attrs, body, result_vars, span);
@@ -834,8 +783,8 @@ TVM_FFI_STATIC_INIT_BLOCK() {
.attr(refl::type_attr::kStructuralMaybeInplaceMutate,
ffi::FStructuralMutate::FromNative<&AssertStmtMaybeInplaceMutate>());
- refl::GlobalDef().def("tirx.AssertStmt", [](PrimExpr condition, StringImm error_kind,
- ffi::Array<StringImm> message_parts, Span span) {
+ refl::GlobalDef().def("ir.AssertStmt", [](PrimExpr condition, StringImm error_kind,
+ ffi::Array<StringImm> message_parts, Span span) {
return AssertStmt(condition, error_kind, message_parts, span);
});
}
@@ -894,14 +843,6 @@ For::For(PrimVar loop_var, PrimExpr min, PrimExpr extent, ForKind kind, SeqStmt
ffi::ObjectPtr<ForNode> node = ffi::make_object<ForNode>(std::move(loop_var), std::move(min),
std::move(extent), std::move(body));
- if (auto tag = annotations.Get("thread_binding")) {
- TVM_FFI_CHECK(kind == ForKind::kParallel, ValueError)
- << "thread_binding requires a parallel loop";
- TVM_FFI_CHECK(tag->as<ffi::String>().has_value(), TypeError)
- << "thread_binding annotation must be a string";
- TVM_FFI_CHECK(!step.has_value() || IsOne(*step), ValueError)
- << "Thread binding loops require a unit step";
- }
node->kind = kind;
node->annotations = std::move(annotations);
node->step = std::move(step);
@@ -919,9 +860,9 @@ TVM_FFI_STATIC_INIT_BLOCK() {
ffi::FStructuralMutate::FromNative<&ForMaybeInplaceMutate>());
refl::GlobalDef().def(
- "tirx.For", [](PrimVar loop_var, PrimExpr min, PrimExpr extent, int kind, SeqStmt body,
- ffi::Optional<ffi::Map<ffi::String, Any>> annotations,
- ffi::Optional<PrimExpr> step, Span span) {
+ "ir.For", [](PrimVar loop_var, PrimExpr min, PrimExpr extent, int kind, SeqStmt body,
+ ffi::Optional<ffi::Map<ffi::String, Any>> annotations,
+ ffi::Optional<PrimExpr> step, Span span) {
return For(loop_var, min, extent, static_cast<ForKind>(kind), body,
annotations.value_or(ffi::Map<ffi::String, Any>()), step, span);
});
@@ -968,7 +909,7 @@ TVM_FFI_STATIC_INIT_BLOCK() {
.attr(refl::type_attr::kStructuralMaybeInplaceMutate,
ffi::FStructuralMutate::FromNative<&WhileMaybeInplaceMutate>());
- refl::GlobalDef().def("tirx.While", [](PrimExpr condition, SeqStmt body, Span span) {
+ refl::GlobalDef().def("ir.While", [](PrimExpr condition, SeqStmt body, Span span) {
return While(condition, body, span);
});
}
@@ -991,7 +932,7 @@ TVM_FFI_STATIC_INIT_BLOCK() {
.attr(refl::type_attr::kStructuralMaybeInplaceMutate,
ffi::FStructuralMutate::FromNative<&ReturnMaybeInplaceMutate>());
- refl::GlobalDef().def("tirx.Return", [](Expr value, Span span) { return Return(value, span); });
+ refl::GlobalDef().def("ir.Return", [](Expr value, Span span) { return Return(value, span); });
}
// Break
@@ -1010,7 +951,7 @@ TVM_FFI_STATIC_INIT_BLOCK() {
.attr(refl::type_attr::kStructuralMaybeInplaceMutate,
ffi::FStructuralMutate::FromNative<&BreakMaybeInplaceMutate>());
- refl::GlobalDef().def("tirx.Break", [](Span span) { return Break(span); });
+ refl::GlobalDef().def("ir.Break", [](Span span) { return Break(span); });
}
// Continue
@@ -1030,7 +971,7 @@ TVM_FFI_STATIC_INIT_BLOCK() {
.attr(refl::type_attr::kStructuralMaybeInplaceMutate,
ffi::FStructuralMutate::FromNative<&ContinueMaybeInplaceMutate>());
- refl::GlobalDef().def("tirx.Continue", [](Span span) { return Continue(span); });
+ refl::GlobalDef().def("ir.Continue", [](Span span) { return Continue(span); });
}
// SeqStmt
@@ -1093,20 +1034,19 @@ TVM_FFI_STATIC_INIT_BLOCK() {
.attr(refl::type_attr::kStructuralMaybeInplaceMutate,
ffi::FStructuralMutate::FromNative<&SeqStmtMaybeInplaceMutate>());
- refl::GlobalDef().def("tirx.SeqStmt", [](SeqStmt seq, Span span) {
+ refl::GlobalDef().def("ir.SeqStmt", [](SeqStmt seq, Span span) {
return SeqStmt(std::move(seq), std::move(span));
});
}
-// IfThenElse
-IfThenElse::IfThenElse(PrimExpr condition, SeqStmt then_case, ffi::Optional<SeqStmt> else_case,
- Span span)
+// If
+If::If(PrimExpr condition, SeqStmt then_case, ffi::Optional<SeqStmt> else_case, Span span)
: Stmt(ffi::UnsafeInit{}) {
TVM_FFI_ICHECK(condition.defined());
TVM_FFI_ICHECK(then_case.defined());
// else_case may be null.
- ffi::ObjectPtr<IfThenElseNode> node =
- ffi::make_object<IfThenElseNode>(std::move(condition), std::move(then_case));
+ ffi::ObjectPtr<IfNode> node =
+ ffi::make_object<IfNode>(std::move(condition), std::move(then_case));
node->else_case = std::move(else_case);
node->span = std::move(span);
data_ = std::move(node);
@@ -1114,27 +1054,25 @@ IfThenElse::IfThenElse(PrimExpr condition, SeqStmt then_case, ffi::Optional<SeqS
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
- IfThenElseNode::RegisterReflection();
- refl::TypeAttrDef<IfThenElseNode>()
- .attr(refl::type_attr::kStructuralVisit,
- ffi::FStructuralVisit::FromNative<&IfThenElseVisit>())
- .attr(refl::type_attr::kStructuralMutate,
- ffi::FStructuralMutate::FromNative<&IfThenElseMutate>())
+ IfNode::RegisterReflection();
+ refl::TypeAttrDef<IfNode>()
+ .attr(refl::type_attr::kStructuralVisit, ffi::FStructuralVisit::FromNative<&IfVisit>())
+ .attr(refl::type_attr::kStructuralMutate, ffi::FStructuralMutate::FromNative<&IfMutate>())
.attr(refl::type_attr::kStructuralMaybeInplaceMutate,
- ffi::FStructuralMutate::FromNative<&IfThenElseMaybeInplaceMutate>());
+ ffi::FStructuralMutate::FromNative<&IfMaybeInplaceMutate>());
- refl::GlobalDef().def("tirx.IfThenElse", [](PrimExpr condition, SeqStmt then_case,
- ffi::Optional<SeqStmt> else_case, Span span) {
- return IfThenElse(condition, then_case, else_case, span);
- });
+ refl::GlobalDef().def("ir.If",
+ [](PrimExpr condition, SeqStmt then_case, ffi::Optional<SeqStmt> else_case,
+ Span span) { return If(condition, then_case, else_case, span); });
}
// Evaluate
Evaluate::Evaluate(Expr value, Span span) : Stmt(ffi::UnsafeInit{}) {
TVM_FFI_ICHECK(value.defined());
- TVM_FFI_ICHECK(!(value->IsInstance<VarNode>() && value->ty.as<TensorTypeNode>()))
- << "A buffer variable cannot be used as a scalar Evaluate value; "
- << "use buffer.data to evaluate its physical pointer";
+ static const ffi::reflection::TypeAttrColumn validate_column("__evaluate_validate__");
+ if (auto validate = validate_column[value->ty->type_index()]; validate != nullptr) {
+ validate.cast<ffi::Function>()(value);
+ }
ffi::ObjectPtr<EvaluateNode> node = ffi::make_object<EvaluateNode>(std::move(value));
node->span = std::move(span);
@@ -1151,66 +1089,19 @@ TVM_FFI_STATIC_INIT_BLOCK() {
.attr(refl::type_attr::kStructuralMaybeInplaceMutate,
ffi::FStructuralMutate::FromNative<&EvaluateMaybeInplaceMutate>());
- refl::GlobalDef().def("tirx.Evaluate",
- [](Expr value, Span span) { return Evaluate(value, span); });
+ refl::GlobalDef().def("ir.Evaluate", [](Expr value, Span span) { return Evaluate(value, span); });
}
// TensorStore
-TVM_FFI_INLINE int GetLanesOrVScaleFactor(const PrimType& ty) {
- return ty.IsScalableVector() ? ty.VScaleFactor() : ty.lanes();
-}
-
-TensorStore::TensorStore(TensorVar buffer, PrimExpr value, ffi::Array<PrimExpr> indices, Span span)
+TensorStore::TensorStore(Expr dest, ffi::Array<PrimExpr> indices, PrimExpr value, Span span)
: Stmt(ffi::UnsafeInit{}) {
- TVM_FFI_ICHECK_EQ(buffer->shape.size(), indices.size())
- << "TensorVar " << buffer.name() << " is " << buffer->shape.size()
- << "-dimensional, cannot be indexed with the " << indices.size()
- << "-dimensional indices provided.";
-
- for (int i = 0; i < static_cast<int>(indices.size()) - 1; i++) {
- TVM_FFI_ICHECK(indices[i].ty().IsScalar())
- << "Only the last index of a buffer access may be a vector type.";
- }
-
- bool is_index_scalable = indices.empty() ? false : indices.back().ty().IsScalableVector();
- int16_t buffer_encoded_lanes = static_cast<int16_t>(buffer->dtype->dtype.lanes);
- bool is_buffer_dtype_scalable = buffer_encoded_lanes < -1;
- PrimType value_ty = value.ty();
- bool is_value_dtype_scalable = value_ty.IsScalableVector();
-
- TVM_FFI_ICHECK(!(is_index_scalable && is_buffer_dtype_scalable))
- << "Index dtype and buffer dtype can't both be scalable.";
-
- if (is_index_scalable || is_buffer_dtype_scalable) {
- TVM_FFI_ICHECK(is_value_dtype_scalable) << "Can't store non-scalable data into scalable buffer";
- }
-
- int index_lanes = indices.empty() ? 1 : GetLanesOrVScaleFactor(indices.back().ty());
- int buffer_lanes = is_buffer_dtype_scalable ? -buffer_encoded_lanes : buffer_encoded_lanes;
- int value_dtype_lanes = GetLanesOrVScaleFactor(value_ty);
-
- TVM_FFI_ICHECK_EQ(index_lanes * buffer_lanes, value_dtype_lanes)
- << "Cannot store value with " << value_dtype_lanes << ", expected value with "
- << index_lanes * buffer_lanes << " (" << index_lanes << " index lanes * " << buffer_lanes
- << " buffer element lanes)";
-
- PrimType buffer_dtype = PrimType::Void();
- if (is_index_scalable || is_buffer_dtype_scalable) {
- buffer_dtype = PrimType::ScalableVector(buffer->dtype.code(), buffer->dtype.bits(),
- buffer_lanes * index_lanes);
- } else {
- buffer_dtype = buffer->dtype.WithLanes(buffer_lanes * index_lanes);
- }
- if (buffer_dtype != value_ty) {
- TVM_FFI_THROW(TypeError) << "dtype mismatch on TensorStore: " //
- << "buffer's dtype is `" << buffer->dtype //
- << "`, the lanes of indexing are: `" << index_lanes //
- << "`, the scalability is: `" << buffer_dtype.IsScalableVector()
- << "`, but RHS's dtype is `" << value_ty << "`";
- }
-
- ffi::ObjectPtr<TensorStoreNode> node =
- ffi::make_object<TensorStoreNode>(std::move(buffer), std::move(value));
+ namespace refl = ffi::reflection;
+ static const refl::TypeAttrColumn validate_column("__tensor_store_validate__");
+ auto validate = validate_column[dest->ty->type_index()];
+ TVM_FFI_CHECK(validate != nullptr, TypeError)
+ << "Type " << dest->ty->GetTypeKey() << " does not support TensorStore";
+ validate.cast<ffi::Function>()(dest, indices, value);
+ auto node = ffi::make_object<TensorStoreNode>(std::move(dest), std::move(value));
node->indices = std::move(indices);
node->span = std::move(span);
data_ = std::move(node);
@@ -1227,34 +1118,10 @@ TVM_FFI_STATIC_INIT_BLOCK() {
.attr(refl::type_attr::kStructuralMaybeInplaceMutate,
ffi::FStructuralMutate::FromNative<&TensorStoreMaybeInplaceMutate>());
- refl::GlobalDef().def("tirx.TensorStore",
- [](TensorVar buffer, PrimExpr value, ffi::Array<PrimExpr> indices,
- Span span) { return TensorStore(buffer, value, indices, span); });
-}
-
-// ScopeIdDefStmt
-ScopeIdDefStmt::ScopeIdDefStmt(ScopeIdDef def, Span span) : Stmt(ffi::UnsafeInit{}) {
- TVM_FFI_ICHECK(def.defined());
- ffi::ObjectPtr<ScopeIdDefStmtNode> node = ffi::make_object<ScopeIdDefStmtNode>();
- node->def = std::move(def);
- node->span = std::move(span);
- data_ = std::move(node);
-}
-
-TVM_FFI_STATIC_INIT_BLOCK() {
- namespace refl = tvm::ffi::reflection;
- ScopeIdDefStmtNode::RegisterReflection();
- refl::TypeAttrDef<ScopeIdDefStmtNode>()
- .attr(refl::type_attr::kStructuralVisit,
- ffi::FStructuralVisit::FromNative<&ScopeIdDefStmtVisit>())
- .attr(refl::type_attr::kStructuralMutate,
- ffi::FStructuralMutate::FromNative<&ScopeIdDefStmtMutate>())
- .attr(refl::type_attr::kStructuralMaybeInplaceMutate,
- ffi::FStructuralMutate::FromNative<&ScopeIdDefStmtMaybeInplaceMutate>());
-
- refl::GlobalDef().def("tirx.ScopeIdDefStmt",
- [](ScopeIdDef def, Span span) { return ScopeIdDefStmt(def, span); });
+ refl::GlobalDef().def("ir.TensorStore",
+ [](Expr dest, ffi::Array<PrimExpr> indices, PrimExpr value, Span span) {
+ return TensorStore(dest, indices, value, span);
+ });
}
-} // namespace tirx
} // namespace tvm
diff --git a/src/relax/analysis/layout_transformation.cc b/src/relax/analysis/layout_transformation.cc
index 2142bcf932..f2b4a898a1 100644
--- a/src/relax/analysis/layout_transformation.cc
+++ b/src/relax/analysis/layout_transformation.cc
@@ -495,17 +495,18 @@ class BlockAnalyzer : public s_tir::StmtExprVisitor {
ffi::Optional<VisitInterrupt> Visit_(const TensorStoreNode* op) final {
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(s_tir::StmtExprVisitor::Visit_(op));
- BufferAccessInfo& access_info = buffer_access_info_[op->buffer];
+ BufferAccessInfo& access_info =
+ buffer_access_info_[op->dest.as_or_throw<tvm::tirx::TensorVar>()];
// Fast path to ignore further analysis if we know that the buffer access is invalid.
if (!access_info.IsValid()) return std::nullopt;
// Only single write buffer is supported for each block.
- if (!op->buffer.same_as(block_->writes[0]->source.as_or_throw<tvm::tirx::TensorVar>())) {
+ if (!op->dest.same_as(block_->writes[0]->source.as_or_throw<tvm::tirx::TensorVar>())) {
access_info.Invalidate();
LOG(WARNING) << "[LayoutInference] Exactly one write buffer is supported for layout "
"inference, found two: "
- << op->buffer << " and "
+ << op->dest << " and "
<< block_->writes[0]->source.as_or_throw<tvm::tirx::TensorVar>();
can_transform_block_ = false;
return std::nullopt;
diff --git a/src/relax/analysis/tir_op_pattern_kind.cc b/src/relax/analysis/tir_op_pattern_kind.cc
index 7adca92d00..4f40976ab7 100644
--- a/src/relax/analysis/tir_op_pattern_kind.cc
+++ b/src/relax/analysis/tir_op_pattern_kind.cc
@@ -201,7 +201,7 @@ class PatternKindAnalyzer : public s_tir::StmtExprVisitor {
*/
static bool IsBroadcastPattern(const TensorStore& store, const TensorLoad& load) {
size_t ndim_load_buf = load->source.as_or_throw<tvm::tirx::TensorVar>()->shape.size();
- size_t ndim_store_buf = store->buffer->shape.size();
+ size_t ndim_store_buf = store->dest.as_or_throw<tvm::tirx::TensorVar>()->shape.size();
for (size_t i = 0, j = 0; i < ndim_load_buf; ++i) {
if (IsConstInt(load->source.as_or_throw<tvm::tirx::TensorVar>()->shape[i], 1) &&
@@ -299,7 +299,7 @@ class PatternKindAnalyzer : public s_tir::StmtExprVisitor {
if (const auto* mul = RemoveCast(add->b).as<prim::MulNode>()) {
const auto* store_lhs = RemoveCast(add->a).as<TensorLoadNode>();
if (!store_lhs ||
- !store->buffer.same_as(store_lhs->source.as_or_throw<tvm::tirx::TensorVar>()) ||
+ !store->dest.same_as(store_lhs->source.as_or_throw<tvm::tirx::TensorVar>()) ||
!IsSameArray(store->indices, store_lhs->indices)) {
return false;
}
@@ -447,7 +447,7 @@ bool HasReshapePattern(const tirx::Function& func) {
}
// Further, we require the buffer being stored and being loaded to
// match the parameter of the tirx::Function, namely `dst_buffer_` and `src_buffer_`.
- if (!(tensor_store->buffer.same_as(dst_buffer_) &&
+ if (!(tensor_store->dest.same_as(dst_buffer_) &&
buffer_load->source.as_or_throw<tvm::tirx::TensorVar>().same_as(src_buffer_))) {
return std::nullopt;
}
diff --git a/src/relax/analysis/well_formed.cc b/src/relax/analysis/well_formed.cc
index f70a0714c6..092997397a 100644
--- a/src/relax/analysis/well_formed.cc
+++ b/src/relax/analysis/well_formed.cc
@@ -36,10 +36,10 @@
* 5. Vars, including those introduced in definitional type positions, are defined before use.
* 6. Vars are defined exactly once.
* 7. DataflowVars cannot be defined inside BindingBlock.
- * 8. Vars defined in IfNode, except the return Var, are invisible
+ * 8. Vars defined in IfExprNode, except the return Var, are invisible
* out of the If body.(May change for new AST designs)
* 9. SeqExpr only serves as function body, or in the true and
- * false branches in IfNode.
+ * false branches in IfExprNode.
* 10. The IR is in ANF:
* (a) Expressions cannot contain nested complex expressions.
* Here are the expressions that may be nested inside other expressions:
@@ -481,11 +481,11 @@ class WellFormedChecker : public relax::ExprVisitor, public relax::TypeVisitor {
TVM_FFI_VISIT_END(ffi::GetRef<Call>(call));
}
- void VisitExpr_(const IfNode* op) final {
+ void VisitExpr_(const IfExprNode* op) final {
TVM_FFI_VISIT_BEGIN();
if (is_dataflow_) {
TVM_FFI_VISIT_THROW(ValueError, ffi::GetRef<Expr>(op))
- << "If nodes are not allowed to appear in dataflow blocks.";
+ << "IfExpr nodes are not allowed to appear in dataflow blocks.";
}
if (IsLeafOrTuple(op->cond)) {
this->VisitExpr(op->cond);
@@ -519,13 +519,13 @@ class WellFormedChecker : public relax::ExprVisitor, public relax::TypeVisitor {
void VisitExpr_(const SeqExprNode* op) final {
TVM_FFI_VISIT_THROW(ValueError, ffi::GetRef<Expr>(op))
<< "SeqExpr only serves as the function body in FunctionNode, "
- "or the true/false branch body in IfNode.";
+ "or the true/false branch body in IfExprNode.";
}
void VisitSeqExpr(const SeqExprNode* op) {
TVM_FFI_VISIT_BEGIN();
// a special call only if SeqExpr is the function body
- // in FunctionNode or the true/false branch body in IfNode
+ // in FunctionNode or the true/false branch body in IfExprNode
for (BindingBlock block : op->blocks) {
this->VisitBindingBlock(block);
}
diff --git a/src/relax/backend/vm/codegen_vm.cc b/src/relax/backend/vm/codegen_vm.cc
index 65408406ff..ae579f0ef0 100644
--- a/src/relax/backend/vm/codegen_vm.cc
+++ b/src/relax/backend/vm/codegen_vm.cc
@@ -182,8 +182,8 @@ class CodeGenVM : public ExprFunctor<Instruction::Arg(const Expr&)> {
return Instruction::Arg::Register(dst_reg);
}
- Instruction::Arg VisitExpr_(const IfNode* op) final {
- const If& ife = ffi::GetRef<If>(op);
+ Instruction::Arg VisitExpr_(const IfExprNode* op) final {
+ const IfExpr& ife = ffi::GetRef<IfExpr>(op);
Instruction::Arg cond_value = this->VisitExpr(ife->cond);
// Reserve a register for cond
diff --git a/src/relax/backend/vm/vm_shape_lower.cc b/src/relax/backend/vm/vm_shape_lower.cc
index b7cb40bab2..969eb1947e 100644
--- a/src/relax/backend/vm/vm_shape_lower.cc
+++ b/src/relax/backend/vm/vm_shape_lower.cc
@@ -728,17 +728,17 @@ class VMShapeLowerMutator
return ffi::Unchanged();
};
- ffi::Array<tirx::Stmt> seq;
+ ffi::Array<tvm::Stmt> seq;
for (PrimExprSlot* slot : to_compute) {
TVM_FFI_ICHECK(!slot->value_computed);
slot->value_computed = true;
PrimExpr value = ffi::StructuralMap<ffi::WalkOrder::kPreOrder>(slot->expr, f_substitute)
.as_or_throw<PrimExpr>();
seq.push_back(
- tirx::TensorStore(buffer, value, {IntImm(tvm::PrimType(ShapeDType()), slot->index)}));
+ tvm::TensorStore(buffer, {IntImm(tvm::PrimType(ShapeDType()), slot->index)}, value));
}
- tirx::SeqStmt body(seq);
+ tvm::SeqStmt body(seq);
ffi::Array<tvm::Var> params{buffer.var()};
Type ret_type = VoidType();
diff --git a/src/relax/distributed/transform/lower_global_view_to_local_view.cc b/src/relax/distributed/transform/lower_global_view_to_local_view.cc
index f54fe46f34..ca55c784ee 100644
--- a/src/relax/distributed/transform/lower_global_view_to_local_view.cc
+++ b/src/relax/distributed/transform/lower_global_view_to_local_view.cc
@@ -56,7 +56,7 @@ class DistBufferReplacer : public s_tir::StmtExprMutator {
class DistSBlockInfoCollector : public s_tir::StmtExprVisitor {
private:
ffi::Optional<VisitInterrupt> Visit_(const TensorStoreNode* op) final {
- buffer_access_indices[op->buffer].push_back(op->indices);
+ buffer_access_indices[op->dest.as_or_throw<tvm::tirx::TensorVar>()].push_back(op->indices);
return s_tir::StmtExprVisitor::Visit_(op);
}
diff --git a/src/relax/ir/block_builder.cc b/src/relax/ir/block_builder.cc
index b6d6fd346e..af9006c4c4 100644
--- a/src/relax/ir/block_builder.cc
+++ b/src/relax/ir/block_builder.cc
@@ -737,17 +737,17 @@ class Normalizer : public BlockBuilderImpl, private ExprFunctor<Expr(const Expr&
return seq_expr;
}
- Expr VisitExpr_(const IfNode* op) final {
+ Expr VisitExpr_(const IfExprNode* op) final {
Expr new_cond = this->NormalizeArgument(op->cond);
Expr new_true = this->VisitWithNewScope(op->true_branch);
Expr new_false = this->VisitWithNewScope(op->false_branch);
- If if_node = [&]() -> If {
+ IfExpr if_node = [&]() -> IfExpr {
if (new_cond.same_as(op->cond) && new_true.same_as(op->true_branch) &&
new_false.same_as(op->false_branch)) {
- return ffi::GetRef<If>(op);
+ return ffi::GetRef<IfExpr>(op);
} else {
- return If(new_cond, new_true, new_false, op->span);
+ return IfExpr(new_cond, new_true, new_false, op->span);
}
}();
if (if_node->ty.as<MissingType>().has_value()) {
diff --git a/src/relax/ir/dataflow_expr_rewriter.cc b/src/relax/ir/dataflow_expr_rewriter.cc
index e948b5ed3c..4d8e1b77af 100644
--- a/src/relax/ir/dataflow_expr_rewriter.cc
+++ b/src/relax/ir/dataflow_expr_rewriter.cc
@@ -956,7 +956,7 @@ class PatternMatchingMutator : public ExprMutator {
for (const auto& binding : block->bindings) {
auto value = GetBoundValue(binding);
- bool is_dataflow = (!value.as<IfNode>()) &&
+ bool is_dataflow = (!value.as<IfExprNode>()) &&
(!(value.as<CallNode>() && IsImpureCall(value.as_or_throw<Call>())));
if (is_dataflow) {
// This binding satisfies the dataflow constraints.
diff --git a/src/relax/ir/dataflow_matcher.cc b/src/relax/ir/dataflow_matcher.cc
index b9f3875311..9f9fc57e0e 100644
--- a/src/relax/ir/dataflow_matcher.cc
+++ b/src/relax/ir/dataflow_matcher.cc
@@ -124,7 +124,7 @@ Expr DFPatternMatcher::UnwrapBindings(Expr expr, const ffi::Map<Var, Expr>& var2
}
// Unwrap SeqExpr with no bindings. These can occur due to Relax
- // IR constraints for the bodies of Function and If nodes.
+ // IR constraints for the bodies of Function and IfExpr nodes.
if (auto seq = expr.as<SeqExprNode>()) {
if (seq->blocks.empty()) {
return seq->body;
diff --git a/src/relax/ir/expr.cc b/src/relax/ir/expr.cc
index 4d4b0d3e02..c0d1811ebf 100644
--- a/src/relax/ir/expr.cc
+++ b/src/relax/ir/expr.cc
@@ -68,10 +68,10 @@ TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> TypeOnlyExprMaybeInplac
return ffi::Unchanged();
}
-TVM_FFI_INLINE ffi::Expected<ffi::Optional<ffi::VisitInterrupt>> IfVisit(
+TVM_FFI_INLINE ffi::Expected<ffi::Optional<ffi::VisitInterrupt>> IfExprVisit(
ffi::StructuralVisitorObj* visitor, ffi::AnyView value) noexcept {
- const IfNode* self =
- ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const IfNode>(value);
+ const IfExprNode* self =
+ ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const IfExprNode>(value);
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(visitor->VisitExpected(self->ty));
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(visitor->VisitExpected(self->cond));
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(visitor->VisitExpected(self->true_branch));
@@ -79,10 +79,10 @@ TVM_FFI_INLINE ffi::Expected<ffi::Optional<ffi::VisitInterrupt>> IfVisit(
return std::nullopt;
}
-TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> IfMutate(
+TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> IfExprMutate(
ffi::StructuralMutatorObj* mutator, ffi::AnyView value) noexcept {
- const IfNode* self =
- ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const IfNode>(value);
+ const IfExprNode* self =
+ ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const IfExprNode>(value);
TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<Type>, mapped_ty,
mutator->MutateExpected(self->ty));
TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<Expr>, mapped_cond,
@@ -96,7 +96,7 @@ TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> IfMutate(
mapped_false_branch.UnchangedOrSameAs(self->false_branch)) {
return ffi::Unchanged();
}
- ffi::ObjectPtr<IfNode> copy = ffi::make_object<IfNode>(*self);
+ ffi::ObjectPtr<IfExprNode> copy = ffi::make_object<IfExprNode>(*self);
copy->ty = std::move(mapped_ty).ValueOrUnchanged(std::move(copy->ty));
copy->cond = std::move(mapped_cond).ValueOrUnchanged(std::move(copy->cond));
copy->true_branch = std::move(mapped_true_branch).ValueOrUnchanged(std::move(copy->true_branch));
@@ -105,10 +105,10 @@ TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> IfMutate(
return ffi::Any(std::move(copy));
}
-TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> IfMaybeInplaceMutate(
+TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> IfExprMaybeInplaceMutate(
ffi::StructuralMutatorObj* mutator, ffi::AnyView value) noexcept {
- IfNode* self = const_cast<IfNode*>(
- ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const IfNode>(value));
+ IfExprNode* self = const_cast<IfExprNode*>(
+ ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const IfExprNode>(value));
TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<Type>, mapped_ty,
mutator->MutateExpected(self->ty, ffi::InplaceMode::kAllow));
TVM_FFI_S_MUTATE_ASSIGN_OR_RETURN(ffi::UnchangedOr<Expr>, mapped_cond,
@@ -399,25 +399,27 @@ TVM_FFI_INLINE ffi::Expected<ffi::UnchangedOr<ffi::Any>> FunctionMaybeInplaceMut
TVM_FFI_STATIC_INIT_BLOCK() { BindingNode::RegisterReflection(); }
-If::If(Expr cond, Expr true_branch, Expr false_branch, Span span) : Expr(ffi::UnsafeInit{}) {
- ffi::ObjectPtr<IfNode> n =
- ffi::make_object<IfNode>(std::move(cond), std::move(true_branch), std::move(false_branch));
+IfExpr::IfExpr(Expr cond, Expr true_branch, Expr false_branch, Span span)
+ : Expr(ffi::UnsafeInit{}) {
+ ffi::ObjectPtr<IfExprNode> n = ffi::make_object<IfExprNode>(
+ std::move(cond), std::move(true_branch), std::move(false_branch));
n->span = std::move(span);
data_ = std::move(n);
}
TVM_FFI_STATIC_INIT_BLOCK() {
namespace refl = tvm::ffi::reflection;
- IfNode::RegisterReflection();
- refl::TypeAttrDef<IfNode>()
- .attr(refl::type_attr::kStructuralVisit, ffi::FStructuralVisit::FromNative<&IfVisit>())
- .attr(refl::type_attr::kStructuralMutate, ffi::FStructuralMutate::FromNative<&IfMutate>())
+ IfExprNode::RegisterReflection();
+ refl::TypeAttrDef<IfExprNode>()
+ .attr(refl::type_attr::kStructuralVisit, ffi::FStructuralVisit::FromNative<&IfExprVisit>())
+ .attr(refl::type_attr::kStructuralMutate, ffi::FStructuralMutate::FromNative<&IfExprMutate>())
.attr(refl::type_attr::kStructuralMaybeInplaceMutate,
- ffi::FStructuralMutate::FromNative<&IfMaybeInplaceMutate>());
+ ffi::FStructuralMutate::FromNative<&IfExprMaybeInplaceMutate>());
- refl::GlobalDef().def("relax.If", [](Expr cond, Expr true_branch, Expr false_branch, Span span) {
- return If(cond, true_branch, false_branch, span);
- });
+ refl::GlobalDef().def("relax.IfExpr",
+ [](Expr cond, Expr true_branch, Expr false_branch, Span span) {
+ return IfExpr(cond, true_branch, false_branch, span);
+ });
}
ShapeExpr::ShapeExpr(ffi::Array<PrimExpr> values, Span span) : Expr(ffi::UnsafeInit{}) {
diff --git a/src/relax/ir/expr_functor.cc b/src/relax/ir/expr_functor.cc
index 1978efe665..48b4aa397c 100644
--- a/src/relax/ir/expr_functor.cc
+++ b/src/relax/ir/expr_functor.cc
@@ -51,7 +51,7 @@
RELAX_VISIT_BINDING_DISPATCH(FunctionNode); \
RELAX_VISIT_BINDING_DISPATCH(CallNode); \
RELAX_VISIT_BINDING_DISPATCH(SeqExprNode); \
- RELAX_VISIT_BINDING_DISPATCH(IfNode); \
+ RELAX_VISIT_BINDING_DISPATCH(IfExprNode); \
RELAX_VISIT_BINDING_DISPATCH(OpNode); \
RELAX_VISIT_BINDING_DISPATCH(TupleGetItemNode); \
RELAX_VISIT_BINDING_DISPATCH(StringImmNode); \
@@ -273,7 +273,7 @@ void ExprVisitor::VisitExpr_(const tvm::IntImmNode* op) { this->VisitSpan(op->sp
void ExprVisitor::VisitExpr_(const tvm::FloatImmNode* op) { this->VisitSpan(op->span); }
-void ExprVisitor::VisitExpr_(const IfNode* op) {
+void ExprVisitor::VisitExpr_(const IfExprNode* op) {
this->VisitSpan(op->span);
this->VisitExpr(op->cond);
this->VisitExpr(op->true_branch);
@@ -339,7 +339,7 @@ RELAX_EXPR_VISITOR_VISIT_BINDING_IMPL(GlobalVarNode);
RELAX_EXPR_VISITOR_VISIT_BINDING_IMPL(FunctionNode);
RELAX_EXPR_VISITOR_VISIT_BINDING_IMPL(CallNode);
RELAX_EXPR_VISITOR_VISIT_BINDING_IMPL(SeqExprNode);
-RELAX_EXPR_VISITOR_VISIT_BINDING_IMPL(IfNode);
+RELAX_EXPR_VISITOR_VISIT_BINDING_IMPL(IfExprNode);
RELAX_EXPR_VISITOR_VISIT_BINDING_IMPL(OpNode);
RELAX_EXPR_VISITOR_VISIT_BINDING_IMPL(TupleGetItemNode);
RELAX_EXPR_VISITOR_VISIT_BINDING_IMPL(ExprNode);
@@ -645,7 +645,7 @@ Expr ExprMutatorBase::VisitExpr_(const tvm::IntImmNode* op) { return ffi::GetRef
Expr ExprMutatorBase::VisitExpr_(const tvm::FloatImmNode* op) { return ffi::GetRef<Expr>(op); }
-Expr ExprMutatorBase::VisitExpr_(const IfNode* op) {
+Expr ExprMutatorBase::VisitExpr_(const IfExprNode* op) {
Expr guard = this->VisitExpr(op->cond);
Expr true_b = this->VisitExpr(op->true_branch);
Expr false_b = this->VisitExpr(op->false_branch);
@@ -653,7 +653,7 @@ Expr ExprMutatorBase::VisitExpr_(const IfNode* op) {
op->false_branch.same_as(false_b) && VisitAndCheckTypeFieldUnchanged(op->ty)) {
return ffi::GetRef<Expr>(op);
} else {
- return If(guard, true_b, false_b, op->span);
+ return IfExpr(guard, true_b, false_b, op->span);
}
}
@@ -809,7 +809,7 @@ Expr ExprMutator::VisitExpr_(const FunctionNode* op) {
}
}
-Expr ExprMutator::VisitExpr_(const IfNode* op) {
+Expr ExprMutator::VisitExpr_(const IfExprNode* op) {
Expr guard = this->VisitExpr(op->cond);
Expr true_b = this->VisitWithInnerScope(op->true_branch);
Expr false_b = this->VisitWithInnerScope(op->false_branch);
@@ -817,7 +817,7 @@ Expr ExprMutator::VisitExpr_(const IfNode* op) {
op->false_branch.same_as(false_b) && VisitAndCheckTypeFieldUnchanged(op->ty)) {
return ffi::GetRef<Expr>(op);
} else {
- return If(guard, true_b, false_b, op->span);
+ return IfExpr(guard, true_b, false_b, op->span);
}
}
@@ -858,7 +858,7 @@ RELAX_EXPR_MUTATOR_VISIT_BINDING_IMPL(GlobalVarNode);
RELAX_EXPR_MUTATOR_VISIT_BINDING_IMPL(FunctionNode);
RELAX_EXPR_MUTATOR_VISIT_BINDING_IMPL(CallNode);
RELAX_EXPR_MUTATOR_VISIT_BINDING_IMPL(SeqExprNode);
-RELAX_EXPR_MUTATOR_VISIT_BINDING_IMPL(IfNode);
+RELAX_EXPR_MUTATOR_VISIT_BINDING_IMPL(IfExprNode);
RELAX_EXPR_MUTATOR_VISIT_BINDING_IMPL(OpNode);
RELAX_EXPR_MUTATOR_VISIT_BINDING_IMPL(TupleGetItemNode);
RELAX_EXPR_MUTATOR_VISIT_BINDING_IMPL(ExprNode);
diff --git a/src/relax/ir/py_expr_functor.cc b/src/relax/ir/py_expr_functor.cc
index 690f8ed9e4..de4139f60e 100644
--- a/src/relax/ir/py_expr_functor.cc
+++ b/src/relax/ir/py_expr_functor.cc
@@ -59,8 +59,8 @@ class PyExprVisitorNode : public ffi::Object, public ExprVisitor {
ffi::Function f_visit_call_{nullptr};
/*! \brief The packed function to the `VisitExpr_(const SeqExprNode* op)` function. */
ffi::Function f_visit_seq_expr_{nullptr};
- /*! \brief The packed function to the `VisitExpr_(const IfNode* op)` function. */
- ffi::Function f_visit_if_{nullptr};
+ /*! \brief The packed function to the `VisitExpr_(const IfExprNode* op)` function. */
+ ffi::Function f_visit_if_expr_{nullptr};
/*! \brief The packed function to the `VisitExpr_(const OpNode* op)` function. */
ffi::Function f_visit_op_{nullptr};
/*! \brief The packed function to the `VisitExpr_(const TupleGetItemNode* op)` function. */
@@ -169,7 +169,7 @@ class PyExprVisitorNode : public ffi::Object, public ExprVisitor {
PY_EXPR_VISITOR_DISPATCH(FunctionNode, f_visit_function_);
PY_EXPR_VISITOR_DISPATCH(CallNode, f_visit_call_);
PY_EXPR_VISITOR_DISPATCH(SeqExprNode, f_visit_seq_expr_);
- PY_EXPR_VISITOR_DISPATCH(IfNode, f_visit_if_);
+ PY_EXPR_VISITOR_DISPATCH(IfExprNode, f_visit_if_expr_);
PY_EXPR_VISITOR_DISPATCH(OpNode, f_visit_op_);
PY_EXPR_VISITOR_DISPATCH(TupleGetItemNode, f_visit_tuple_getitem_);
PY_EXPR_VISITOR_DISPATCH(StringImmNode, f_visit_string_imm_);
@@ -201,7 +201,7 @@ class PyExprVisitor : public ffi::ObjectRef {
* \param f_visit_function_ The packed function of `VisitExpr_(const FunctionNode* op)`.
* \param f_visit_call_ The packed function of `VisitExpr_(const CallNode* op)`.
* \param f_visit_seq_expr_ The packed function of `VisitExpr_(const SeqExprNode* op)`.
- * \param f_visit_if_ The packed function of `VisitExpr_(const IfNode* op)`.
+ * \param f_visit_if_expr_ The packed function of `VisitExpr_(const IfExprNode* op)`.
* \param f_visit_op_ The packed function of `VisitExpr_(const OpNode* op)`.
* \param f_visit_tuple_getitem_ The packed function of `VisitExpr_(const TupleGetItemNode* op)`.
* \param f_visit_expr_fallback_ The packed function of the generic expression fallback.
@@ -230,7 +230,7 @@ class PyExprVisitor : public ffi::ObjectRef {
ffi::Function f_visit_tuple_, ffi::Function f_visit_var_, ffi::Function f_visit_dataflow_var_,
ffi::Function f_visit_shape_expr_, ffi::Function f_visit_extern_func_,
ffi::Function f_visit_global_var_, ffi::Function f_visit_function_,
- ffi::Function f_visit_call_, ffi::Function f_visit_seq_expr_, ffi::Function f_visit_if_,
+ ffi::Function f_visit_call_, ffi::Function f_visit_seq_expr_, ffi::Function f_visit_if_expr_,
ffi::Function f_visit_op_, ffi::Function f_visit_tuple_getitem_,
ffi::Function f_visit_expr_fallback_, ffi::Function f_visit_string_imm_,
ffi::Function f_visit_data_type_imm_, ffi::Function f_visit_binding,
@@ -255,7 +255,7 @@ class PyExprVisitor : public ffi::ObjectRef {
n->f_visit_function_ = f_visit_function_;
n->f_visit_call_ = f_visit_call_;
n->f_visit_seq_expr_ = f_visit_seq_expr_;
- n->f_visit_if_ = f_visit_if_;
+ n->f_visit_if_expr_ = f_visit_if_expr_;
n->f_visit_op_ = f_visit_op_;
n->f_visit_tuple_getitem_ = f_visit_tuple_getitem_;
n->f_visit_expr_fallback_ = f_visit_expr_fallback_;
@@ -304,8 +304,8 @@ class PyExprMutatorNode : public ffi::Object, public ExprMutator {
ffi::Function f_visit_call_{nullptr};
/*! \brief The packed function to the `VisitExpr_(const SeqExprNode* op)` function. */
ffi::Function f_visit_seq_expr_{nullptr};
- /*! \brief The packed function to the `VisitExpr_(const IfNode* op)` function. */
- ffi::Function f_visit_if_{nullptr};
+ /*! \brief The packed function to the `VisitExpr_(const IfExprNode* op)` function. */
+ ffi::Function f_visit_if_expr_{nullptr};
/*! \brief The packed function to the `VisitExpr_(const OpNode* op)` function. */
ffi::Function f_visit_op_{nullptr};
/*! \brief The packed function to the `VisitExpr_(const TupleGetItemNode* op)` function. */
@@ -441,7 +441,7 @@ class PyExprMutatorNode : public ffi::Object, public ExprMutator {
PY_EXPR_MUTATOR_DISPATCH(FunctionNode, f_visit_function_);
PY_EXPR_MUTATOR_DISPATCH(CallNode, f_visit_call_);
PY_EXPR_MUTATOR_DISPATCH(SeqExprNode, f_visit_seq_expr_);
- PY_EXPR_MUTATOR_DISPATCH(IfNode, f_visit_if_);
+ PY_EXPR_MUTATOR_DISPATCH(IfExprNode, f_visit_if_expr_);
PY_EXPR_MUTATOR_DISPATCH(OpNode, f_visit_op_);
PY_EXPR_MUTATOR_DISPATCH(TupleGetItemNode, f_visit_tuple_getitem_);
PY_EXPR_MUTATOR_DISPATCH(StringImmNode, f_visit_string_imm_);
@@ -464,7 +464,7 @@ class PyExprMutatorNode : public ffi::Object, public ExprMutator {
PY_EXPR_MUTATOR_VISIT_EXPR_POST_ORDER_DISPATCH(FunctionNode);
PY_EXPR_MUTATOR_VISIT_EXPR_POST_ORDER_DISPATCH(CallNode);
PY_EXPR_MUTATOR_VISIT_EXPR_POST_ORDER_DISPATCH(SeqExprNode);
- PY_EXPR_MUTATOR_VISIT_EXPR_POST_ORDER_DISPATCH(IfNode);
+ PY_EXPR_MUTATOR_VISIT_EXPR_POST_ORDER_DISPATCH(IfExprNode);
PY_EXPR_MUTATOR_VISIT_EXPR_POST_ORDER_DISPATCH(OpNode);
PY_EXPR_MUTATOR_VISIT_EXPR_POST_ORDER_DISPATCH(TupleGetItemNode);
PY_EXPR_MUTATOR_VISIT_EXPR_POST_ORDER_DISPATCH(StringImmNode);
@@ -496,7 +496,7 @@ class PyExprMutator : public ffi::ObjectRef {
* \param f_visit_function_ The packed function of `VisitExpr_(const FunctionNode* op)`.
* \param f_visit_call_ The packed function of `VisitExpr_(const CallNode* op)`.
* \param f_visit_seq_expr_ The packed function of `VisitExpr_(const SeqExprNode* op)`.
- * \param f_visit_if_ The packed function of `VisitExpr_(const IfNode* op)`.
+ * \param f_visit_if_expr_ The packed function of `VisitExpr_(const IfExprNode* op)`.
* \param f_visit_op_ The packed function of `VisitExpr_(const OpNode* op)`.
* \param f_visit_tuple_getitem_ The packed function of `VisitExpr_(const TupleGetItemNode* op)`.
* \param f_visit_expr_fallback_ The packed function of the generic expression fallback.
@@ -525,7 +525,7 @@ class PyExprMutator : public ffi::ObjectRef {
ffi::Function f_visit_tuple_, ffi::Function f_visit_var_, ffi::Function f_visit_dataflow_var_,
ffi::Function f_visit_shape_expr_, ffi::Function f_visit_extern_func_,
ffi::Function f_visit_global_var_, ffi::Function f_visit_function_,
- ffi::Function f_visit_call_, ffi::Function f_visit_seq_expr_, ffi::Function f_visit_if_,
+ ffi::Function f_visit_call_, ffi::Function f_visit_seq_expr_, ffi::Function f_visit_if_expr_,
ffi::Function f_visit_op_, ffi::Function f_visit_tuple_getitem_,
ffi::Function f_visit_expr_fallback_, ffi::Function f_visit_string_imm_,
ffi::Function f_visit_data_type_imm_, ffi::Function f_visit_binding,
@@ -547,7 +547,7 @@ class PyExprMutator : public ffi::ObjectRef {
n->f_visit_function_ = f_visit_function_;
n->f_visit_call_ = f_visit_call_;
n->f_visit_seq_expr_ = f_visit_seq_expr_;
- n->f_visit_if_ = f_visit_if_;
+ n->f_visit_if_expr_ = f_visit_if_expr_;
n->f_visit_op_ = f_visit_op_;
n->f_visit_tuple_getitem_ = f_visit_tuple_getitem_;
n->f_visit_expr_fallback_ = f_visit_expr_fallback_;
diff --git a/src/relax/op/tensor/inspect.cc b/src/relax/op/tensor/inspect.cc
index 9f8410bbdd..9e2eea0456 100644
--- a/src/relax/op/tensor/inspect.cc
+++ b/src/relax/op/tensor/inspect.cc
@@ -96,11 +96,11 @@ tirx::Function GetDLTensorField(tirx::TVMStructFieldKind field, PrimType field_t
tvm::Var value("value", field_ty);
- tirx::SeqStmt body(
- {tirx::Bind(value, tvm::Call(field_ty, tirx::abi_field_get_op(),
- {dlpack_handle, IntImm::Int32(0), IntImm::Int32(field)})
- .as_or_throw<PrimExpr>()),
- tirx::Return(value)});
+ tvm::SeqStmt body(
+ {tvm::Bind(value, tvm::Call(field_ty, tirx::abi_field_get_op(),
+ {dlpack_handle, IntImm::Int32(0), IntImm::Int32(field)})
+ .as_or_throw<PrimExpr>()),
+ tvm::Return(value)});
DictAttrs attrs({{"tirx.is_scheduled", true}, {"tirx.is_host_func", true}});
@@ -262,29 +262,29 @@ Expr LegalizeTensorShape(const BlockBuilder& bb, const Call& call) {
tvm::Var extent("extent", field_ty);
- tirx::SeqStmt body(
- {tirx::AssertStmt(0 <= axis.as_or_throw<PrimExpr>(), StringImm("RuntimeError"),
- {StringImm("Specified axis may not be negative")}),
- tirx::Bind(ndim, tvm::Call(ndim->ty.as_or_throw<PrimType>(), tirx::abi_field_get_op(),
- {dlpack_handle, IntImm::Int32(0),
- IntImm::Int32(tirx::TVMStructFieldKind::kDLTensorNDim)})
- .as_or_throw<PrimExpr>()),
- tirx::AssertStmt(
+ tvm::SeqStmt body(
+ {tvm::AssertStmt(0 <= axis.as_or_throw<PrimExpr>(), StringImm("RuntimeError"),
+ {StringImm("Specified axis may not be negative")}),
+ tvm::Bind(ndim, tvm::Call(ndim->ty.as_or_throw<PrimType>(), tirx::abi_field_get_op(),
+ {dlpack_handle, IntImm::Int32(0),
+ IntImm::Int32(tirx::TVMStructFieldKind::kDLTensorNDim)})
+ .as_or_throw<PrimExpr>()),
+ tvm::AssertStmt(
axis.as_or_throw<PrimExpr>() <
tvm::prim::cast(axis->ty.as_or_throw<PrimType>(), ndim.as_or_throw<PrimExpr>()),
StringImm("RuntimeError"),
{StringImm("Specified axis may not be larger than the tensor's dimensionality")}),
- tirx::Bind(shape_buffer,
- tvm::Call(shape_buffer.type(), tvm::tirx::decl_tensor_op(),
- {tvm::Call(shape_buffer.DataPointerType(), tirx::abi_field_get_op(),
- {dlpack_handle, IntImm::Int32(0),
- IntImm::Int32(tirx::TVMStructFieldKind::kDLTensorShape)}),
- tvm::Tuple(shape_buffer->shape),
- tvm::DataTypeImm(shape_buffer->dtype->dtype),
- tvm::StringImm(shape_buffer.scope())},
- {})),
- tirx::Bind(extent, tirx::MakeTensorLoad(shape_buffer, {axis.as_or_throw<PrimExpr>()})),
- tirx::Return(extent)});
+ tvm::Bind(shape_buffer,
+ tvm::Call(shape_buffer.type(), tvm::tirx::decl_tensor_op(),
+ {tvm::Call(shape_buffer.DataPointerType(), tirx::abi_field_get_op(),
+ {dlpack_handle, IntImm::Int32(0),
+ IntImm::Int32(tirx::TVMStructFieldKind::kDLTensorShape)}),
+ tvm::Tuple(shape_buffer->shape),
+ tvm::DataTypeImm(shape_buffer->dtype->dtype),
+ tvm::StringImm(shape_buffer.scope())},
+ {})),
+ tvm::Bind(extent, tirx::MakeTensorLoad(shape_buffer, {axis.as_or_throw<PrimExpr>()})),
+ tvm::Return(extent)});
DictAttrs attrs({{"tirx.is_scheduled", true}, {"tirx.is_host_func", true}});
diff --git a/src/relax/script/ir_builder/frame.cc b/src/relax/script/ir_builder/frame.cc
index 2373e1f556..49c4d15959 100644
--- a/src/relax/script/ir_builder/frame.cc
+++ b/src/relax/script/ir_builder/frame.cc
@@ -322,7 +322,7 @@ void IfFrameNode::ExitWithScope() {
<< "The body of then part is expected to be defined before exiting.";
TVM_FFI_CHECK(else_expr.has_value(), ValueError)
<< "The body of else part is expected to be defined before exiting.";
- auto body = tvm::relax::If(condition, then_expr.value(), else_expr.value(), source_span);
+ auto body = tvm::relax::IfExpr(condition, then_expr.value(), else_expr.value(), source_span);
auto binding_var = Emit(body, std::nullopt);
var = binding_var;
// Finalization uses the frame's already-composed location, never the exit context.
diff --git a/src/relax/script/printer/binding.cc b/src/relax/script/printer/binding.cc
index 4a96a4beab..945c36b556 100644
--- a/src/relax/script/printer/binding.cc
+++ b/src/relax/script/printer/binding.cc
@@ -135,7 +135,7 @@ TVM_FFI_STATIC_INIT_BLOCK() {
ffi::Optional<ExprDoc> IfDocTranslate(DocTranslatorObj* d, ffi::AnyView input,
const ffi::Object* destination) {
const auto* branch =
- ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const relax::IfNode>(input);
+ ffi::details::AnyUnsafe::RawObjectPtrFromAnyViewAfterCheck<const relax::IfExprNode>(input);
// A value context needs a usable expression after the conditional. A
// destination supplied by a binding is completed directly on both arms.
ffi::Optional<Var> temporary = std::nullopt;
@@ -145,7 +145,7 @@ ffi::Optional<ExprDoc> IfDocTranslate(DocTranslatorObj* d, ffi::AnyView input,
destination = temporary.value().get();
}
TVM_FFI_CHECK(destination->IsInstance<VarNode>(), TypeError)
- << "printer Relax If destination must be a Var";
+ << "printer Relax IfExpr destination must be a Var";
Var var = ffi::GetRef<Var>(static_cast<const VarNode*>(destination));
ffi::Optional<IdDoc> lhs = VarDoc(d, var);
ffi::Optional<ExprDoc> annotation = std::nullopt;
@@ -158,8 +158,8 @@ ffi::Optional<ExprDoc> IfDocTranslate(DocTranslatorObj* d, ffi::AnyView input,
}
TVM_FFI_STATIC_INIT_BLOCK() {
- ffi::reflection::TypeAttrDef<relax::IfNode>().attr(kDocTranslate,
- FDocTranslate::FromNative<&IfDocTranslate>());
+ ffi::reflection::TypeAttrDef<relax::IfExprNode>().attr(
+ kDocTranslate, FDocTranslate::FromNative<&IfDocTranslate>());
}
} // namespace
diff --git a/src/relax/transform/canonicalize_bindings.cc b/src/relax/transform/canonicalize_bindings.cc
index 412a93da32..3a4d54e609 100644
--- a/src/relax/transform/canonicalize_bindings.cc
+++ b/src/relax/transform/canonicalize_bindings.cc
@@ -167,7 +167,7 @@ class SymbolicVarCanonicalizer : public ExprMutator {
canonicalize_shape_values_ = cached;
}
- Expr VisitExpr_(const IfNode* op) override {
+ Expr VisitExpr_(const IfExprNode* op) override {
Expr guard = this->VisitExpr(op->cond);
auto cached = known_values_;
@@ -214,7 +214,7 @@ class SymbolicVarCanonicalizer : public ExprMutator {
output_var);
}
- return If(guard, true_b, false_b, op->span);
+ return IfExpr(guard, true_b, false_b, op->span);
}
private:
diff --git a/src/relax/transform/compute_prim_value.cc b/src/relax/transform/compute_prim_value.cc
index a6ffbb9fb3..91829e01ef 100644
--- a/src/relax/transform/compute_prim_value.cc
+++ b/src/relax/transform/compute_prim_value.cc
@@ -112,9 +112,9 @@ class PrimExprComputeInjector : public ExprMutator {
tvm::PrimType ret_ty = node.ty();
auto param_vars = tvm::UndefinedVars(node);
- tirx::Stmt body = tirx::Return(node);
+ tvm::Stmt body = tvm::Return(node);
- tirx::Function func(param_vars, tirx::SeqStmt(body), ret_ty,
+ tirx::Function func(param_vars, tvm::SeqStmt(body), ret_ty,
DictAttrs({{tirx::attr::kIsHostFunc, true}, {tvm::attr::kSTir, true}}));
func = tirx::RenewDef(func);
diff --git a/src/relax/transform/convert_dataflow.cc b/src/relax/transform/convert_dataflow.cc
index 4a983418b3..f79500c2e1 100644
--- a/src/relax/transform/convert_dataflow.cc
+++ b/src/relax/transform/convert_dataflow.cc
@@ -115,7 +115,7 @@ class DataflowBlockExtractor : public ExprMutator {
for (const auto& binding : binding_block->bindings) {
Expr value = GetBoundValue(binding);
// dataflow values: not an if node and not an impure call
- bool is_dataflow = (!value.as<IfNode>()) &&
+ bool is_dataflow = (!value.as<IfExprNode>()) &&
(!(value.as<CallNode>() && IsImpureCall(value.as_or_throw<Call>())));
if (is_dataflow) {
// extend the streak
diff --git a/src/relax/transform/dataflow_inplace.cc b/src/relax/transform/dataflow_inplace.cc
index 2eecf0f1ed..871517b748 100644
--- a/src/relax/transform/dataflow_inplace.cc
+++ b/src/relax/transform/dataflow_inplace.cc
@@ -764,8 +764,8 @@ FindInplaceOpportunities(const DataflowBlock& block, const ffi::Array<Var>& inpu
}
// Replace buffers in a tirx::Function according to the mapping.
-tirx::Stmt RemapBuffers(const tirx::Stmt& stmt,
- const ffi::Map<tirx::TensorVar, tirx::TensorVar>& buffer_map) {
+tvm::Stmt RemapBuffers(const tvm::Stmt& stmt,
+ const ffi::Map<tirx::TensorVar, tirx::TensorVar>& buffer_map) {
class BufferMapper : public tirx::StmtExprMutator {
public:
explicit BufferMapper(const ffi::Map<tirx::TensorVar, tirx::TensorVar>& buffer_map) {
@@ -881,7 +881,7 @@ class ModuleInplaceTransformer : public ExprMutator {
auto mod = builder_->GetContextIRModule();
auto old_function = mod->Lookup(legal_op).as_or_throw<tirx::Function>();
- tirx::Stmt new_body = old_function->body.value();
+ tvm::Stmt new_body = old_function->body.value();
size_t num_outs = inplace_indices.size();
size_t num_params = old_function->params.size();
@@ -916,14 +916,14 @@ class ModuleInplaceTransformer : public ExprMutator {
return ffi::Unchanged();
};
new_body = ffi::StructuralMap<ffi::WalkOrder::kPreOrder>(std::move(new_body), f_substitute)
- .as_or_throw<tirx::Stmt>();
+ .as_or_throw<tvm::Stmt>();
// now get rid of the last num_outputs arguments
// (couldn't do earlier or else it would have thrown off the indexing)
ffi::Array<tvm::Var> new_params(old_function->params.begin(),
old_function->params.begin() + (num_params - num_outs));
- tirx::Function new_function(new_params, tirx::SeqStmt(new_body), old_function->ret_type,
+ tirx::Function new_function(new_params, tvm::SeqStmt(new_body), old_function->ret_type,
old_function->attrs, old_function->span);
// note: this might be a good time to get rid of the old legalized function, but we don't do it
diff --git a/src/relax/transform/eliminate_common_subexpr.cc b/src/relax/transform/eliminate_common_subexpr.cc
index 0542d02910..1f8b6a9441 100644
--- a/src/relax/transform/eliminate_common_subexpr.cc
+++ b/src/relax/transform/eliminate_common_subexpr.cc
@@ -165,7 +165,7 @@ class CommonSubexprEliminator : public ExprMutator {
}
}
- Expr VisitExpr_(const IfNode* op) override {
+ Expr VisitExpr_(const IfExprNode* op) override {
Expr cond = VisitExpr(op->cond);
Expr true_branch = VisitWithInnerScope(op->true_branch);
Expr false_branch = VisitWithInnerScope(op->false_branch);
@@ -173,7 +173,7 @@ class CommonSubexprEliminator : public ExprMutator {
op->false_branch.same_as(false_branch) && VisitAndCheckTypeFieldUnchanged(op->ty)) {
return ffi::GetRef<Expr>(op);
} else {
- return If(cond, true_branch, false_branch, op->span);
+ return IfExpr(cond, true_branch, false_branch, op->span);
}
}
diff --git a/src/relax/transform/fuse_tir.cc b/src/relax/transform/fuse_tir.cc
index 0709bae52b..be54c3d404 100644
--- a/src/relax/transform/fuse_tir.cc
+++ b/src/relax/transform/fuse_tir.cc
@@ -874,7 +874,7 @@ class FusedTIRConstructor : public ExprVisitor {
alloc_buffers.push_back(subst->SubstituteAllocatedBuffer(buf));
}
}
- tirx::Stmt body = tirx::SeqStmt(func_info_.bodies);
+ tvm::Stmt body = tvm::SeqStmt(func_info_.bodies);
body = ffi::make_object<tirx::SBlockNameDeduplicator>()->Mutate(body).ValueOrUnchanged(body);
body = subst->Mutate(body).ValueOrUnchanged(body);
@@ -886,7 +886,7 @@ class FusedTIRConstructor : public ExprVisitor {
}
return param;
});
- tirx::Function func(params, tirx::SeqStmt(body), VoidType(), DictAttrs(attr_map));
+ tirx::Function func(params, tvm::SeqStmt(body), VoidType(), DictAttrs(attr_map));
// Renew function defs to prevent using the same symbolic vars in different functions
return tirx::RenewDef(func);
}
@@ -921,7 +921,7 @@ class FusedTIRConstructor : public ExprVisitor {
/*! \brief The buffers to allocate in the fused func*/
ffi::Array<tirx::TensorVar> alloc_buffers;
/*! \brief The bodies of the original funcs, which is also the body of the fused func. */
- ffi::Array<tirx::Stmt> bodies;
+ ffi::Array<tvm::Stmt> bodies;
/*! \brief The params of the fused function*/
ffi::Array<tvm::Var> params;
/*!
diff --git a/src/relax/transform/normalize.cc b/src/relax/transform/normalize.cc
index 765b4df946..6170caafcb 100644
--- a/src/relax/transform/normalize.cc
+++ b/src/relax/transform/normalize.cc
@@ -53,7 +53,7 @@ class NormalizeMutator : public ExprMutatorBase {
}
}
- Expr VisitExpr_(const IfNode* op) final {
+ Expr VisitExpr_(const IfExprNode* op) final {
Expr guard = this->VisitExpr(op->cond);
Expr true_b = this->VisitWithNewScope(op->true_branch);
Expr false_b = this->VisitWithNewScope(op->false_branch);
@@ -61,7 +61,7 @@ class NormalizeMutator : public ExprMutatorBase {
op->false_branch.same_as(false_b)) {
return ffi::GetRef<Expr>(op);
} else {
- return If(guard, true_b, false_b, op->span);
+ return IfExpr(guard, true_b, false_b, op->span);
}
}
diff --git a/src/relax/transform/split_call_tir_by_pattern.cc b/src/relax/transform/split_call_tir_by_pattern.cc
index 6cb97bf817..431f33231e 100644
--- a/src/relax/transform/split_call_tir_by_pattern.cc
+++ b/src/relax/transform/split_call_tir_by_pattern.cc
@@ -256,7 +256,7 @@ class ForMatcher : public TensorizeComparator {
return true;
}
- bool Dispatch_(const tirx::ForNode* op, const Stmt& other) final {
+ bool Dispatch_(const tvm::ForNode* op, const Stmt& other) final {
const auto* rhs = other.as<ForNode>();
loop_stack_lhs_.push_back(ffi::GetRef<For>(op));
loop_stack_rhs_.push_back(ffi::GetRef<For>(rhs));
@@ -273,7 +273,8 @@ class ForMatcher : public TensorizeComparator {
if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
// Only handle the case where the loop start from 0
if (!IsZero(op->min) || !IsZero(rhs->min)) return false;
- if (op->GetThreadBinding().has_value() || rhs->GetThreadBinding().has_value()) return false;
+ if (tvm::tirx::GetThreadBinding(op).has_value() || tvm::tirx::GetThreadBinding(rhs).has_value())
+ return false;
if (op->kind != ForKind::kDefault || op->kind != rhs->kind) return false;
if (!op->annotations.empty() || !rhs->annotations.empty()) return false;
// Match the extents of loops
@@ -365,9 +366,10 @@ class ForMatcher : public TensorizeComparator {
return CompareArray(lhs->region, rhs->region, &ForMatcher::CompareRange);
}
- template <typename T>
- bool CompareBufferAccess(const T* lhs, const T* rhs) {
- if (!CompareBuffer(lhs->buffer, rhs->buffer)) return false;
+ bool CompareBufferAccess(const TensorStoreNode* lhs, const TensorStoreNode* rhs) {
+ if (!CompareBuffer(lhs->dest.as_or_throw<TensorVar>(), rhs->dest.as_or_throw<TensorVar>())) {
+ return false;
+ }
return CompareArray(
lhs->indices, rhs->indices,
static_cast<bool (ForMatcher::*)(const Expr&, const PrimExpr&)>(&ForMatcher::Dispatch));
diff --git a/src/relax/transform/static_plan_block_memory.cc b/src/relax/transform/static_plan_block_memory.cc
index 8b0f05389b..3bcacdd3dc 100644
--- a/src/relax/transform/static_plan_block_memory.cc
+++ b/src/relax/transform/static_plan_block_memory.cc
@@ -93,7 +93,7 @@ using namespace tvm::prim;
* - that is a function parameter,
* - that is a function return value,
* - one of whose use site is a BindingBlock different from its allocation site,
- * - that is used as a condition or branch return of a IfNode,
+ * - that is used as a condition or branch return of a IfExprNode,
* - that is used as the body of a SeqExprNode,
* - that is used as arguments in a Call whose op is not a tirx::Function.
*
@@ -606,7 +606,7 @@ class StorageAllocatorInit : public StorageAllocatorBaseVisitor {
}
}
- void VisitExpr_(const IfNode* if_node) final {
+ void VisitExpr_(const IfExprNode* if_node) final {
Tokens cond_tokens = GetTokens(if_node->cond);
Tokens then_tokens = GetTokens(if_node->true_branch);
Tokens else_tokens = GetTokens(if_node->false_branch);
diff --git a/src/relax/transform/to_mixed_precision.cc b/src/relax/transform/to_mixed_precision.cc
index 71b95fefae..677505fffd 100644
--- a/src/relax/transform/to_mixed_precision.cc
+++ b/src/relax/transform/to_mixed_precision.cc
@@ -251,7 +251,7 @@ class DTypeDecisionCollector : public ExprVisitor {
}
}
- void VisitExpr_(const IfNode* op) final {
+ void VisitExpr_(const IfExprNode* op) final {
this->VisitSpan(op->span);
this->VisitExpr(op->true_branch);
this->VisitExpr(op->false_branch);
diff --git a/src/relax/utils.cc b/src/relax/utils.cc
index e15e62a568..d28fb7a28f 100644
--- a/src/relax/utils.cc
+++ b/src/relax/utils.cc
@@ -233,7 +233,7 @@ bool IsBoolType(const Type& ty, bool permit_unknown_rank, bool permit_unknown_dt
bool IsLeafOrTuple(const Expr& expr) {
return !expr.as<CallNode>() && !expr.as<TupleGetItemNode>() && !expr.as<SeqExprNode>() &&
- !expr.as<IfNode>() && !expr.as<FunctionNode>();
+ !expr.as<IfExprNode>() && !expr.as<FunctionNode>();
}
bool IsImpureCall(const Call& call) {
diff --git a/src/s_tir/analysis/calculate_allocated_memory.cc b/src/s_tir/analysis/calculate_allocated_memory.cc
index 7f2d878f34..6de9ba2a6b 100644
--- a/src/s_tir/analysis/calculate_allocated_memory.cc
+++ b/src/s_tir/analysis/calculate_allocated_memory.cc
@@ -102,7 +102,7 @@ class AllocTensorCalculator : public StmtExprVisitor {
_current_size = snapshot;
return std::nullopt;
}
- ffi::Optional<VisitInterrupt> Visit_(const IfThenElseNode* op) override {
+ ffi::Optional<VisitInterrupt> Visit_(const IfNode* op) override {
auto snapshot = _current_size;
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(StmtExprVisitor::Visit_(op));
_current_size = snapshot;
diff --git a/src/s_tir/analysis/domain_touched.cc b/src/s_tir/analysis/domain_touched.cc
index 82fc9c3623..84eb0dfc5a 100644
--- a/src/s_tir/analysis/domain_touched.cc
+++ b/src/s_tir/analysis/domain_touched.cc
@@ -112,9 +112,12 @@ class BufferTouchedDomain final : public s_tir::IRVisitorWithAnalyzer {
ffi::Optional<VisitInterrupt> Visit_(const TensorStoreNode* op) final {
// Record store-exclusive buffer access
- Touch(&std::get<StoreAccess>(buffer_access_map_[op->buffer.get()]).set, op->indices);
+ Touch(&std::get<StoreAccess>(buffer_access_map_[op->dest.as_or_throw<TensorVar>().get()]).set,
+ op->indices);
// Record load-store inclusive buffer access
- Touch(&std::get<CombinedAccess>(buffer_access_map_[op->buffer.get()]).set, op->indices);
+ Touch(
+ &std::get<CombinedAccess>(buffer_access_map_[op->dest.as_or_throw<TensorVar>().get()]).set,
+ op->indices);
return Parent::Visit_(op);
}
diff --git a/src/s_tir/analysis/estimate_flops.cc b/src/s_tir/analysis/estimate_flops.cc
index 9e4984fb61..5788e99aa3 100644
--- a/src/s_tir/analysis/estimate_flops.cc
+++ b/src/s_tir/analysis/estimate_flops.cc
@@ -191,7 +191,7 @@ class FlopEstimator : private tvm::ExprFunctor<TResult(const Expr& n)>,
return result;
}
- TResult Dispatch_(const IfThenElseNode* branch) override {
+ TResult Dispatch_(const IfNode* branch) override {
TResult cond = Dispatch(branch->condition);
if (branch->else_case) {
cond += Dispatch(branch->then_case).MaxWith(Dispatch(branch->else_case.value()));
diff --git a/src/s_tir/analysis/identify_memcpy.cc b/src/s_tir/analysis/identify_memcpy.cc
index d1f5b904d2..a5d6f8f596 100644
--- a/src/s_tir/analysis/identify_memcpy.cc
+++ b/src/s_tir/analysis/identify_memcpy.cc
@@ -86,7 +86,8 @@ std::variant<MemCpyDetails, std::string> IdentifyMemCpyImpl(const For& loop,
// Now, we have a TensorStore whose value is a TensorLoad. Because
// non-flat physical indices are target-dependent, only handle cases
// where the buffer will be flattened to a 1-d physical buffer.
- ffi::Array<PrimExpr> flattened_dst = store->buffer.OffsetOf(store->indices);
+ ffi::Array<PrimExpr> flattened_dst =
+ store->dest.as_or_throw<TensorVar>().OffsetOf(store->indices);
ffi::Array<PrimExpr> flattened_src =
load->source.as_or_throw<tvm::tirx::TensorVar>().OffsetOf(load->indices);
@@ -277,7 +278,8 @@ std::variant<MemCpyDetails, std::string> IdentifyMemCpyImpl(const For& loop,
load->source.as_or_throw<tvm::tirx::TensorVar>(),
DomainTouched(loop, load->source.as_or_throw<tvm::tirx::TensorVar>(), true, true));
TensorRegion dst_region =
- BufferRegion(store->buffer, DomainTouched(loop, store->buffer, true, true));
+ BufferRegion(store->dest.as_or_throw<TensorVar>(),
+ DomainTouched(loop, store->dest.as_or_throw<TensorVar>(), true, true));
return MemCpyDetails{src_region, dst_region};
}
diff --git a/src/s_tir/analysis/is_pure_function.cc b/src/s_tir/analysis/is_pure_function.cc
index ccf3ea40ef..563ec16e59 100644
--- a/src/s_tir/analysis/is_pure_function.cc
+++ b/src/s_tir/analysis/is_pure_function.cc
@@ -66,12 +66,13 @@ class PurityChecker : TIRVisitorWithPath {
void Dispatch_(const TensorStoreNode* op, ffi::reflection::AccessPath path) override {
TIRVisitorWithPath::Dispatch_(op, path);
- if (!internal_allocations_.count(op->buffer.var())) {
+ if (!internal_allocations_.count(op->dest.as_or_throw<TensorVar>().var())) {
is_pure_ = false;
if (assert_on_error_) {
- TVM_FFI_THROW(AssertionError) << "Pure functions must not write to buffers, "
- << ", but function contains store to " << op->buffer
- << op->indices << " of value " << op->value;
+ TVM_FFI_THROW(AssertionError)
+ << "Pure functions must not write to buffers, "
+ << ", but function contains store to " << op->dest.as_or_throw<TensorVar>()
+ << op->indices << " of value " << op->value;
}
}
}
diff --git a/src/s_tir/analysis/oob_checker.cc b/src/s_tir/analysis/oob_checker.cc
index a27b530285..b2aaae8a73 100644
--- a/src/s_tir/analysis/oob_checker.cc
+++ b/src/s_tir/analysis/oob_checker.cc
@@ -74,8 +74,8 @@ class OOBCheckerVisitor final : public s_tir::IRVisitorWithAnalyzer {
using s_tir::IRVisitorWithAnalyzer::Visit_;
ffi::Optional<VisitInterrupt> Visit_(const TensorStoreNode* node) final {
- for (size_t i = 0; i < node->buffer->shape.size(); i++) {
- CheckBounds(node, node->buffer, i);
+ for (size_t i = 0; i < node->dest.as_or_throw<TensorVar>()->shape.size(); i++) {
+ CheckBounds(node, node->dest.as_or_throw<TensorVar>(), i);
}
return IRVisitorWithAnalyzer::Visit_(node);
}
@@ -92,7 +92,7 @@ class OOBCheckerVisitor final : public s_tir::IRVisitorWithAnalyzer {
auto ind_bounds = analyzer_->int_set(node->indices[i]);
auto shape_bounds = analyzer_->int_set(buffer->shape[i]);
// We would expect that
- // `analyzer_.CanProve(node->indices[i] < 0 || node->indices[i] >= node->buffer->shape[i])`
+ // `analyzer_.CanProve(node->indices[i] < 0 || node->indices[i] >= buffer->shape[i])`
// would be the way to check if any out of bounds access occurs here, but `CanProve` checks if
// the statement is true for all possible values (universal quantification). For a mix of in
// bounds and out of bounds access, no out of bounds access would be reported. We instead want
diff --git a/src/s_tir/analysis/sblock_access_region_detector.cc b/src/s_tir/analysis/sblock_access_region_detector.cc
index b52b8eab39..ee3506754f 100644
--- a/src/s_tir/analysis/sblock_access_region_detector.cc
+++ b/src/s_tir/analysis/sblock_access_region_detector.cc
@@ -138,7 +138,7 @@ class BlockReadWriteDetector : public s_tir::StmtExprVisitor {
}
ffi::Optional<VisitInterrupt> Visit_(const ForNode* op) override;
- ffi::Optional<VisitInterrupt> Visit_(const IfThenElseNode* op) override;
+ ffi::Optional<VisitInterrupt> Visit_(const IfNode* op) override;
ffi::Optional<VisitInterrupt> Visit_(const s_tir::SBlockRealizeNode* op) override;
ffi::Optional<VisitInterrupt> Visit_(const TensorStoreNode* op) override;
ffi::Optional<VisitInterrupt> Visit_(const BindNode* op) override;
@@ -216,7 +216,7 @@ ffi::Optional<VisitInterrupt> BlockReadWriteDetector::Visit_(const ForNode* op)
return std::nullopt;
}
-ffi::Optional<VisitInterrupt> BlockReadWriteDetector::Visit_(const IfThenElseNode* op) {
+ffi::Optional<VisitInterrupt> BlockReadWriteDetector::Visit_(const IfNode* op) {
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(Visit(op->condition));
{
// Visit then branch
@@ -355,7 +355,7 @@ ffi::Optional<VisitInterrupt> BlockReadWriteDetector::Visit_(const TensorStoreNo
}
relaxed_region.push_back(sym::EvalSet(sym::IntSet::Vector(remapped_index), dom_map_));
}
- Update(&writes_buffers_, &write_regions_, op->buffer, relaxed_region);
+ Update(&writes_buffers_, &write_regions_, op->dest.as_or_throw<TensorVar>(), relaxed_region);
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(Visit(op->value));
for (const auto& index : op->indices) {
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(Visit(index));
diff --git a/src/s_tir/analysis/sblock_buffer_access_lca_detector.cc b/src/s_tir/analysis/sblock_buffer_access_lca_detector.cc
index bb91e006c1..363f1e1707 100644
--- a/src/s_tir/analysis/sblock_buffer_access_lca_detector.cc
+++ b/src/s_tir/analysis/sblock_buffer_access_lca_detector.cc
@@ -101,9 +101,9 @@ class LCADetector : public s_tir::StmtExprVisitor {
const ScopeInfo* parent_scope = ancestor_scopes_.back();
auto* current_scope = arena_.make<ScopeInfo>(parent_scope, op, n);
- if (op->GetThreadBinding().has_value()) {
+ if (tvm::tirx::GetThreadBinding(op).has_value()) {
const runtime::ThreadScope& scope =
- runtime::ThreadScope::Create(op->GetThreadBinding().value());
+ runtime::ThreadScope::Create(tvm::tirx::GetThreadBinding(op).value());
if (scope.rank == 0) {
blockidx_scopes_.push_back(current_scope);
}
@@ -296,7 +296,7 @@ class LCADetector : public s_tir::StmtExprVisitor {
}
ffi::Optional<VisitInterrupt> Visit_(const TensorStoreNode* op) final {
- UpdateBufferLCA(op->buffer.get(), ancestor_scopes_.back());
+ UpdateBufferLCA(op->dest.as_or_throw<TensorVar>().get(), ancestor_scopes_.back());
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(Visit(op->value));
for (const auto& index : op->indices) {
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(Visit(index));
diff --git a/src/s_tir/analysis/verify_well_formed.cc b/src/s_tir/analysis/verify_well_formed.cc
index 17b62cfaad..04c500a18c 100644
--- a/src/s_tir/analysis/verify_well_formed.cc
+++ b/src/s_tir/analysis/verify_well_formed.cc
@@ -27,8 +27,8 @@
namespace tvm {
namespace s_tir {
-using tirx::ForNode;
using tirx::Function;
+using tvm::ForNode;
/*! \brief Verify all Expr inside the block does not contain:
* 1. loop vars outside the current block.
diff --git a/src/s_tir/backend/adreno/texture_flatten.cc b/src/s_tir/backend/adreno/texture_flatten.cc
index 96bf745ac3..224fd43e2a 100644
--- a/src/s_tir/backend/adreno/texture_flatten.cc
+++ b/src/s_tir/backend/adreno/texture_flatten.cc
@@ -97,10 +97,10 @@ class TextureFlattener : public TextureLoweringBase {
UnchangedOr<Stmt> Mutate_(const TensorStoreNode* op, InplaceMode inplace_mode) final {
Stmt stmt = StmtExprMutator::Mutate_(op, inplace_mode).ValueOrUnchanged(ffi::GetRef<Stmt>(op));
op = stmt.as<TensorStoreNode>();
- std::string storage_scope = GetStorageScope(op->buffer);
+ std::string storage_scope = GetStorageScope(op->dest.as_or_throw<TensorVar>());
// Lower to two dimensional access
if (IsTextureStorage(storage_scope)) {
- ffi::Array<Expr> args = GetTextureAccessArgs(op, op->buffer);
+ ffi::Array<Expr> args = GetTextureAccessArgs(op, op->dest.as_or_throw<TensorVar>());
args.push_back(op->value);
stmt = Evaluate(Call(args[0]->ty, tvm::backend::opencl::texture2d_store_op(), args));
}
diff --git a/src/s_tir/ir/data_type_rewriter.h b/src/s_tir/ir/data_type_rewriter.h
index fa5f5638bd..ce35dd1622 100644
--- a/src/s_tir/ir/data_type_rewriter.h
+++ b/src/s_tir/ir/data_type_rewriter.h
@@ -38,8 +38,8 @@ class IndexDataTypeNormalizer : public tirx::IndexDataTypeNormalizer {
: Parent(std::move(target_data_type), GlobalVTable()) {}
tirx::Function Rewrite(tirx::Function func);
- virtual UnchangedOr<tirx::Stmt> Mutate_(const SBlockNode* op, InplaceMode inplace_mode);
- virtual UnchangedOr<tirx::Stmt> Mutate_(const SBlockRealizeNode* op, InplaceMode inplace_mode);
+ virtual UnchangedOr<Stmt> Mutate_(const SBlockNode* op, InplaceMode inplace_mode);
+ virtual UnchangedOr<Stmt> Mutate_(const SBlockRealizeNode* op, InplaceMode inplace_mode);
protected:
static void InitVTable(VTable* table) {
diff --git a/src/s_tir/ir/ir_mutator_with_analyzer.h b/src/s_tir/ir/ir_mutator_with_analyzer.h
index 204f9750c5..24ad2b9260 100644
--- a/src/s_tir/ir/ir_mutator_with_analyzer.h
+++ b/src/s_tir/ir/ir_mutator_with_analyzer.h
@@ -34,8 +34,8 @@ class IRMutatorWithAnalyzer : public tirx::IRMutatorWithAnalyzer {
explicit IRMutatorWithAnalyzer(const sym::Analyzer& analyzer)
: IRMutatorWithAnalyzer(analyzer.get()) {}
explicit IRMutatorWithAnalyzer(sym::AnalyzerObj* analyzer) : Parent(analyzer, GlobalVTable()) {}
- virtual UnchangedOr<tirx::Stmt> Mutate_(const SBlockNode* op, InplaceMode inplace_mode);
- virtual UnchangedOr<tirx::Stmt> Mutate_(const SBlockRealizeNode* op, InplaceMode inplace_mode) {
+ virtual UnchangedOr<Stmt> Mutate_(const SBlockNode* op, InplaceMode inplace_mode);
+ virtual UnchangedOr<Stmt> Mutate_(const SBlockRealizeNode* op, InplaceMode inplace_mode) {
return s_tir::StmtExprMutator::MutateBlockRealize(this, op, inplace_mode);
}
diff --git a/src/s_tir/meta_schedule/feature_extractor/per_store_feature.cc b/src/s_tir/meta_schedule/feature_extractor/per_store_feature.cc
index 6499f1b304..6ca9d1b48b 100644
--- a/src/s_tir/meta_schedule/feature_extractor/per_store_feature.cc
+++ b/src/s_tir/meta_schedule/feature_extractor/per_store_feature.cc
@@ -311,8 +311,8 @@ Pass SimplifyForFeatureExtraction(bool normalize_thread_bindings = false) {
if (!normalize_thread_bindings_ && IsZero(loop->extent)) {
return Evaluate(0);
}
- if (normalize_thread_bindings_ && loop->GetThreadBinding().has_value()) {
- const auto tag = loop->GetThreadBinding().value();
+ if (normalize_thread_bindings_ && tvm::tirx::GetThreadBinding(loop).has_value()) {
+ const auto tag = tvm::tirx::GetThreadBinding(loop).value();
if (support::StartsWith(tag, "vthread")) {
// Virtual axes are independent iterations and isolate hardware aliases.
thread_bindings_.push_back(nullptr);
@@ -322,7 +322,7 @@ Pass SimplifyForFeatureExtraction(bool normalize_thread_bindings = false) {
}
for (auto it = thread_bindings_.rbegin(); it != thread_bindings_.rend() && *it; ++it) {
const ForNode* outer = *it;
- if (outer->GetThreadBinding().value() != tag) continue;
+ if (tvm::tirx::GetThreadBinding(outer).value() != tag) continue;
// Feature counts describe hardware work, not repeated lexical names
// for one axis. Normalize only nested aliases in this analysis input.
sym::Analyzer analyzer;
@@ -446,14 +446,14 @@ struct LoopNest {
this->auto_unroll.push_back(*auto_unroll_attr);
}
ForVec* ref_loops = nullptr;
- if (loop->kind == ForKind::kParallel && !loop->GetThreadBinding().has_value()) {
+ if (loop->kind == ForKind::kParallel && !tvm::tirx::GetThreadBinding(loop).has_value()) {
ref_loops = ∥
} else if (loop->kind == ForKind::kVectorized) {
ref_loops = &vectorize;
} else if (loop->kind == ForKind::kUnrolled) {
ref_loops = &unroll;
- } else if (loop->GetThreadBinding().has_value()) {
- std::string thread_tag = loop->GetThreadBinding().value();
+ } else if (tvm::tirx::GetThreadBinding(loop).has_value()) {
+ std::string thread_tag = tvm::tirx::GetThreadBinding(loop).value();
if (thread_tag == "blockIdx.x") {
ref_loops = &blockIdx_x;
} else if (thread_tag == "blockIdx.y") {
@@ -890,7 +890,7 @@ void Feature::Init(const TensorStoreNode* store, int n_loops) {
};
std::unordered_map<TensorVar, Info, ffi::ObjectPtrHash, ffi::ObjectPtrEqual> buffer_info;
{
- Info& info = buffer_info[store->buffer];
+ Info& info = buffer_info[store->dest.as_or_throw<TensorVar>()];
info.access_type = AccessType::kWrite;
info.multi_indices.push_back({store->indices.begin(), store->indices.end()});
}
@@ -1450,7 +1450,7 @@ class PerStoreFeatureCollector : public StmtExprVisitor {
if (store->value->IsInstance<IntImmNode>() || store->value->IsInstance<FloatImmNode>()) {
return std::nullopt;
}
- const VarNode* buffer = store->buffer.get();
+ const VarNode* buffer = store->dest.as_or_throw<TensorVar>().get();
Feature& feature = buffer_features_[buffer];
if (feature.buffer == nullptr) {
feature.buffer = buffer;
diff --git a/src/s_tir/meta_schedule/postproc/rewrite_cooperative_fetch.cc b/src/s_tir/meta_schedule/postproc/rewrite_cooperative_fetch.cc
index eb110f0a7f..34066c6452 100644
--- a/src/s_tir/meta_schedule/postproc/rewrite_cooperative_fetch.cc
+++ b/src/s_tir/meta_schedule/postproc/rewrite_cooperative_fetch.cc
@@ -93,7 +93,7 @@ bool ParseWarpExecutionAnn(const Schedule& sch, const Instruction& inst) {
size_t GetMaxUsedDtypeBytes(SBlock block) {
size_t max_bytes = 1;
- auto visit_store = [&](const tirx::TensorStore& store) -> ffi::Expected<ffi::WalkResult> {
+ auto visit_store = [&](const TensorStore& store) -> ffi::Expected<ffi::WalkResult> {
max_bytes = std::max(max_bytes, store->value.ty().StorageBytes());
return ffi::WalkResult::Advance();
};
diff --git a/src/s_tir/meta_schedule/postproc/rewrite_parallel_vectorize_unroll.cc b/src/s_tir/meta_schedule/postproc/rewrite_parallel_vectorize_unroll.cc
index 02a3feaa03..d4bd2096f7 100644
--- a/src/s_tir/meta_schedule/postproc/rewrite_parallel_vectorize_unroll.cc
+++ b/src/s_tir/meta_schedule/postproc/rewrite_parallel_vectorize_unroll.cc
@@ -33,7 +33,7 @@ using namespace tvm::tirx;
* \return Whether the loop has any annotation
*/
inline bool HasAnnOrBinding(const ForNode* loop) {
- return loop->GetThreadBinding().has_value() || !loop->annotations.empty();
+ return tvm::tirx::GetThreadBinding(loop).has_value() || !loop->annotations.empty();
}
/*! \brief The visitor for extracting the stride of a var in a PrimExpr. */
diff --git a/src/s_tir/meta_schedule/schedule_rule/cross_thread_reduction.cc b/src/s_tir/meta_schedule/schedule_rule/cross_thread_reduction.cc
index 3827f7fd8f..3f52d7698a 100644
--- a/src/s_tir/meta_schedule/schedule_rule/cross_thread_reduction.cc
+++ b/src/s_tir/meta_schedule/schedule_rule/cross_thread_reduction.cc
@@ -137,7 +137,7 @@ class CrossThreadReductionNode : public ScheduleRuleNode {
bool InThreadScope(const s_tir::Schedule& sch, const s_tir::SBlockRV& block) {
const ffi::Array<s_tir::LoopRV>& axes = sch->GetLoops(block);
for (const s_tir::LoopRV& loop_rv : axes) {
- const tirx::For& loop = sch->Get(loop_rv);
+ const For& loop = sch->Get(loop_rv);
runtime::ThreadScope thread_scope = s_tir::GetThreadScope(loop.get());
if (s_tir::IsThreadIdx(thread_scope)) {
return true;
diff --git a/src/s_tir/meta_schedule/schedule_rule/multi_level_tiling.cc b/src/s_tir/meta_schedule/schedule_rule/multi_level_tiling.cc
index 9179ce0c70..c07d870868 100644
--- a/src/s_tir/meta_schedule/schedule_rule/multi_level_tiling.cc
+++ b/src/s_tir/meta_schedule/schedule_rule/multi_level_tiling.cc
@@ -340,13 +340,13 @@ std::vector<State> MultiLevelTilingNode::AddAsyncPipeline(State state) const {
// check the reduce loop contains exactly 3 for loops
// therefore it matches the notation array size in the following code
tirx::StmtSRef r_loop_sref = state->sch->GetSRef(state->tiles[r_indices_[0]].back());
- const tirx::ForNode* r_for_loop = TVM_SREF_TO_FOR(r_loop_sref);
- ffi::Array<tirx::Stmt> seq = r_for_loop->body->seq;
+ const ForNode* r_for_loop = TVM_SREF_TO_FOR(r_loop_sref);
+ ffi::Array<Stmt> seq = r_for_loop->body->seq;
if (seq.size() != 3) {
return {state};
}
for (auto& stmt : seq) {
- if (!stmt.as<tirx::ForNode>()) {
+ if (!stmt.as<ForNode>()) {
return {state};
}
}
diff --git a/src/s_tir/meta_schedule/schedule_rule/multi_level_tiling_tensor_core.cc b/src/s_tir/meta_schedule/schedule_rule/multi_level_tiling_tensor_core.cc
index da4f476320..370f41f9f0 100644
--- a/src/s_tir/meta_schedule/schedule_rule/multi_level_tiling_tensor_core.cc
+++ b/src/s_tir/meta_schedule/schedule_rule/multi_level_tiling_tensor_core.cc
@@ -455,9 +455,9 @@ std::vector<State> MultiLevelTilingTensorCoreNode::TransformIntermediateOutputLa
s_tir::SBlock intrin_block =
desc->body.value()->seq[0].as_or_throw<s_tir::SBlockRealize>()->block;
TVM_FFI_ICHECK_EQ(intrin_block->body->size(), 1);
- tirx::For loop_m = intrin_block->body->seq[0].as_or_throw<tirx::For>();
+ For loop_m = intrin_block->body->seq[0].as_or_throw<For>();
TVM_FFI_ICHECK_EQ(loop_m->body->size(), 1);
- tirx::For loop_n = loop_m->body->seq[0].as_or_throw<tirx::For>();
+ For loop_n = loop_m->body->seq[0].as_or_throw<For>();
return std::make_tuple(loop_m->extent, loop_n->extent);
}();
diff --git a/src/s_tir/meta_schedule/schedule_rule/multi_level_tiling_wide_vector.cc b/src/s_tir/meta_schedule/schedule_rule/multi_level_tiling_wide_vector.cc
index a4d7341255..61f14ffa32 100644
--- a/src/s_tir/meta_schedule/schedule_rule/multi_level_tiling_wide_vector.cc
+++ b/src/s_tir/meta_schedule/schedule_rule/multi_level_tiling_wide_vector.cc
@@ -64,7 +64,7 @@ class MultiLevelTilingWideVectorNode : public MultiLevelTilingNode {
std::pair<ffi::Array<ffi::Optional<s_tir::ExprRV>>, ffi::Array<s_tir::LoopRV>>
MultiLevelTilingWideVectorNode::SplitLoop(const Schedule& sch, SBlockRV block_rv, LoopRV loop_rv,
int n_tiles) const {
- const tirx::ForNode* loop = TVM_SREF_TO_FOR(sch->GetSRef(loop_rv));
+ const ForNode* loop = TVM_SREF_TO_FOR(sch->GetSRef(loop_rv));
const tirx::StmtSRef block_sref = sch->GetSRef(block_rv);
const s_tir::SBlockNode* block_node = block_sref->StmtAs<s_tir::SBlockNode>();
const s_tir::SBlockRealize block_realize = s_tir::GetSBlockRealize(sch->state(), block_sref);
diff --git a/src/s_tir/schedule/analysis.h b/src/s_tir/schedule/analysis.h
index 6e0201b3fe..603592860b 100644
--- a/src/s_tir/schedule/analysis.h
+++ b/src/s_tir/schedule/analysis.h
@@ -618,7 +618,7 @@ bool HasOp(const Stmt& stmt, const ffi::Array<Op>& ops);
/*!
* \brief Checks if the given AST statement contains if-then-else, including
- * 1) IfThenElse statement
+ * 1) If statement
* 2) Select expression
* 3) The operator `tirx.if_then_else`
* 4) non-constant-true SBlock predicates
@@ -744,9 +744,9 @@ PrimExpr SimplifyNonTrivialExpr(const PrimExpr& expr, sym::AnalyzerObj* analyzer
class TensorizeInfoNode : public ffi::Object {
public:
/*! \brief Maps loops in a target block to the ones in an intrinsic description */
- ffi::Map<tirx::StmtSRef, tirx::For> loop_map;
+ ffi::Map<tirx::StmtSRef, For> loop_map;
/*! \brief Maps loops in an intrinsic description to its index, outer to inner */
- ffi::Map<tirx::For, int64_t> desc_loop_indexer;
+ ffi::Map<For, int64_t> desc_loop_indexer;
/*! \brief Optional padded extents of the block iters when padding is needed to match the
* intrinsic description
*/
diff --git a/src/s_tir/schedule/analysis/analysis.cc b/src/s_tir/schedule/analysis/analysis.cc
index 1bf6f05070..931508ec6b 100644
--- a/src/s_tir/schedule/analysis/analysis.cc
+++ b/src/s_tir/schedule/analysis/analysis.cc
@@ -706,8 +706,8 @@ ffi::Map<Var, Range> LoopDomainOfSRefTreePath(const StmtSRef& low_inclusive,
if (extra_relax_scope.rank != runtime::StorageRank::kGlobal) {
for (; p; p = p->parent) {
if (const ForNode* loop = p->StmtAs<ForNode>()) {
- if (loop->GetThreadBinding().has_value()) {
- const ffi::String thread_tag = loop->GetThreadBinding().value();
+ if (tvm::tirx::GetThreadBinding(loop).has_value()) {
+ const ffi::String thread_tag = tvm::tirx::GetThreadBinding(loop).value();
if (CanRelaxStorageUnderThread(extra_relax_scope,
runtime::ThreadScope::Create(thread_tag))) {
result.Set(loop->loop_var, Range::FromMinExtent(loop->min, loop->extent));
@@ -1395,7 +1395,7 @@ bool HasIfThenElse(const Stmt& stmt) {
}
return ffi::WalkResult::Advance();
};
- auto visit_branch = [](const IfThenElse&) -> ffi::Expected<ffi::WalkResult> {
+ auto visit_branch = [](const If&) -> ffi::Expected<ffi::WalkResult> {
return ffi::WalkResult::Interrupt(ffi::VisitInterrupt(true));
};
auto visit_select = [](const Select&) -> ffi::Expected<ffi::WalkResult> {
@@ -1753,7 +1753,7 @@ struct TensorIntrinDescInfo {
*/
const SBlockRealizeNode* desc_block = nullptr;
/*! \brief The loops of the description function, in the order from outer loops to inner ones. */
- std::vector<const tirx::ForNode*> desc_loops;
+ std::vector<const ForNode*> desc_loops;
/*! \brief The loop variables. */
std::unordered_set<const tvm::VarNode*> desc_loop_vars;
};
@@ -1804,11 +1804,11 @@ ffi::Optional<TensorizeInfo> GetTensorizeLoopMapping(const s_tir::ScheduleState&
// Step 2. Collect loops from block_sref
const tirx::StmtSRef& scope_sref = GetScopeRoot(self, block_sref, false);
TVM_SREF_TO_SBLOCK(scope_sref);
- std::vector<const tirx::ForNode*> block_loops;
+ std::vector<const ForNode*> block_loops;
std::unordered_set<const tvm::VarNode*> block_loop_vars;
{
for (const tirx::StmtSRefNode* loop_sref = block_sref->parent;; loop_sref = loop_sref->parent) {
- const auto* loop = loop_sref->StmtAs<tirx::ForNode>();
+ const auto* loop = loop_sref->StmtAs<ForNode>();
if (loop == nullptr || loop->body->size() != 1) {
break;
}
@@ -1869,7 +1869,7 @@ ffi::Optional<TensorizeInfo> GetTensorizeLoopMapping(const s_tir::ScheduleState&
for (int i_desc = n_desc_vars - 1; i_desc >= 0; --i_desc) {
// Step 3.1. Find the corresponding loop of the i_desc-th block var of desc
const PrimExpr& desc_bind = desc_block->iter_values[i_desc];
- const tirx::ForNode* desc_loop = nullptr;
+ const ForNode* desc_loop = nullptr;
IterVarType iter_type_desc = iter_types_desc[i_desc];
for (int i = 0, n = desc_loops.size(); i < n; ++i) {
// Check if desc_bind = loops[i]->loop_var + stuff-irrelevant-of-loop-vars
@@ -1902,7 +1902,7 @@ ffi::Optional<TensorizeInfo> GetTensorizeLoopMapping(const s_tir::ScheduleState&
// Step 3.3. Find the corresponding loop of the target block
for (int i = 0, n = block_loops.size(); i < n; ++i) {
// Check if block_bind = block_loops[i]->loop_var + stuff-irrelevant-of-loop-vars
- const tirx::ForNode* block_loop = block_loops[i];
+ const ForNode* block_loop = block_loops[i];
const tirx::StmtSRef& block_loop_sref = self->stmt2ref[block_loop];
// Skip i-th loop if it has already been mapped
if (ret->loop_map.find(block_loop_sref) != ret->loop_map.end()) continue;
@@ -1933,13 +1933,13 @@ ffi::Optional<TensorizeInfo> GetTensorizeLoopMapping(const s_tir::ScheduleState&
}
}
- ret->loop_map.Set(block_loop_sref, ffi::GetRef<tirx::For>(desc_loop));
+ ret->loop_map.Set(block_loop_sref, ffi::GetRef<For>(desc_loop));
break;
}
}
for (int i = 0, n = desc_loops.size(); i < n; ++i) {
- ret->desc_loop_indexer.Set(ffi::GetRef<tirx::For>(desc_loops[i]), static_cast<int64_t>(i));
+ ret->desc_loop_indexer.Set(ffi::GetRef<For>(desc_loops[i]), static_cast<int64_t>(i));
}
if (!block_index_to_padding.empty()) {
if (!allow_padding) {
diff --git a/src/s_tir/schedule/analysis/reducer.cc b/src/s_tir/schedule/analysis/reducer.cc
index dd2c8eb30a..af99ba0872 100644
--- a/src/s_tir/schedule/analysis/reducer.cc
+++ b/src/s_tir/schedule/analysis/reducer.cc
@@ -446,8 +446,9 @@ void ExtractReductionUpdates(const ffi::Optional<ScheduleState>& self, SBlock bl
if ((*updates)[idx].defined()) {
ErrorRFactorCrossThreadReductionNotApplicable(self, std::move(block), /*violated_cond=*/8);
}
- updates->Set(idx, TensorStore(buf_store->buffer, let_values[idx], buf_store->indices));
- auto insert_result = buf2index->insert(std::make_pair(buf_store->buffer.get(), idx));
+ updates->Set(idx, TensorStore(buf_store->dest, buf_store->indices, let_values[idx]));
+ auto insert_result =
+ buf2index->insert(std::make_pair(buf_store->dest.as_or_throw<TensorVar>().get(), idx));
if (!insert_result.second) {
ErrorRFactorCrossThreadReductionNotApplicable(self, std::move(block), /*violated_cond=*/9);
}
@@ -471,7 +472,7 @@ std::pair<ffi::Array<PrimExpr>, ffi::Array<TensorStore>> GetInitValuesAndUpdates
if (!init) {
ErrorRFactorCrossThreadReductionNotApplicable(self, std::move(block), /*violated_cond=*/1);
}
- auto insert_result = init_buffers.insert(init.value()->buffer.get());
+ auto insert_result = init_buffers.insert(init.value()->dest.as_or_throw<TensorVar>().get());
if (!insert_result.second) {
ErrorRFactorCrossThreadReductionNotApplicable(self, std::move(block), /*violated_cond=*/2);
}
@@ -487,7 +488,7 @@ std::pair<ffi::Array<PrimExpr>, ffi::Array<TensorStore>> GetInitValuesAndUpdates
if (const auto* update =
block->body->size() == 1 ? block->body->seq[0].as<TensorStoreNode>() : nullptr) {
updates.push_back(ffi::GetRef<TensorStore>(update));
- buf2index[update->buffer.get()] = 0;
+ buf2index[update->dest.as_or_throw<TensorVar>().get()] = 0;
} else {
ExtractReductionUpdates(self, block, block->body->seq, n_buffers, &updates, &buf2index);
}
@@ -502,13 +503,13 @@ std::pair<ffi::Array<PrimExpr>, ffi::Array<TensorStore>> GetInitValuesAndUpdates
// - Check all indices of the TensorStores are the same
// - Check buffers written in the block init and the block body can match
// - Check buffers do not duplicate
- const ffi::Array<PrimExpr>& expected_shape = updates[0]->buffer->shape;
+ const ffi::Array<PrimExpr>& expected_shape = updates[0]->dest.as_or_throw<TensorVar>()->shape;
const ffi::Array<PrimExpr>& expected_indices = updates[0]->indices;
TVM_FFI_ICHECK_EQ(expected_shape.size(), expected_indices.size());
int n_dim = expected_indices.size();
sym::Analyzer ana;
for (int i = 0; i < n_buffers; ++i) {
- if (static_cast<int>(updates[i]->buffer->shape.size()) != n_dim) {
+ if (static_cast<int>(updates[i]->dest.as_or_throw<TensorVar>()->shape.size()) != n_dim) {
ErrorRFactorCrossThreadReductionNotApplicable(self, std::move(block), /*violated_cond=*/11);
}
if (static_cast<int>(inits[i]->indices.size()) != n_dim ||
@@ -516,7 +517,8 @@ std::pair<ffi::Array<PrimExpr>, ffi::Array<TensorStore>> GetInitValuesAndUpdates
ErrorRFactorCrossThreadReductionNotApplicable(self, std::move(block), /*violated_cond=*/12);
}
for (int d = 0; d < n_dim; ++d) {
- if (!ana->CanProveEqual(updates[i]->buffer->shape[d], expected_shape[d])) {
+ if (!ana->CanProveEqual(updates[i]->dest.as_or_throw<TensorVar>()->shape[d],
+ expected_shape[d])) {
ErrorRFactorCrossThreadReductionNotApplicable(self, std::move(block), /*violated_cond=*/11);
}
if (!ana->CanProveEqual(inits[i]->indices[d], expected_indices[d]) ||
@@ -525,12 +527,13 @@ std::pair<ffi::Array<PrimExpr>, ffi::Array<TensorStore>> GetInitValuesAndUpdates
}
}
- auto it = buf2index.find(inits[i]->buffer.get());
+ auto it = buf2index.find(inits[i]->dest.as_or_throw<TensorVar>().get());
if (it == buf2index.end()) {
ErrorRFactorCrossThreadReductionNotApplicable(self, std::move(block), /*violated_cond=*/10);
}
int idx = it->second;
- TVM_FFI_ICHECK(updates[idx]->buffer.same_as(inits[i]->buffer));
+ TVM_FFI_ICHECK(updates[idx]->dest.as_or_throw<TensorVar>().same_as(
+ inits[i]->dest.as_or_throw<TensorVar>()));
TVM_FFI_ICHECK(!init_values[idx].defined());
init_values.Set(idx, inits[i]->value);
}
@@ -593,7 +596,7 @@ bool ReductionIterNotIndexOutputBuffer(const SBlock& block) {
}
return ffi::WalkResult::Advance();
};
- auto visit_alloc = [&](const tirx::Bind& alloc) -> ffi::Expected<ffi::WalkResult> {
+ auto visit_alloc = [&](const tvm::Bind& alloc) -> ffi::Expected<ffi::WalkResult> {
// Inline AllocTensor statements (e.g. `T.local_scalar(...)` expansions)
// declare buffer-local scratch storage inside the block body; treat them
// the same as block->alloc_buffers entries for the "write-without-signature"
@@ -606,12 +609,14 @@ bool ReductionIterNotIndexOutputBuffer(const SBlock& block) {
};
auto visit_store = [&](const TensorStore& store) -> ffi::Expected<ffi::WalkResult> {
bool write_is_covered_by_match_buffer =
- match_buffer_sources.count(store->buffer.get()) &&
- buffer_written.count(match_buffer_sources.find(store->buffer.get())->second);
- TVM_FFI_CHECK(buffer_written.count(store->buffer.get()) || write_is_covered_by_match_buffer ||
- buffer_allocated.count(store->buffer.get()),
+ match_buffer_sources.count(store->dest.as_or_throw<TensorVar>().get()) &&
+ buffer_written.count(
+ match_buffer_sources.find(store->dest.as_or_throw<TensorVar>().get())->second);
+ TVM_FFI_CHECK(buffer_written.count(store->dest.as_or_throw<TensorVar>().get()) ||
+ write_is_covered_by_match_buffer ||
+ buffer_allocated.count(store->dest.as_or_throw<TensorVar>().get()),
ValueError)
- << "The buffer \"" << store->buffer
+ << "The buffer \"" << store->dest.as_or_throw<TensorVar>()
<< "\" is written in the block but is not in the block's signature nor is it covered "
"by a match_buffer";
for (const PrimExpr& index : store->indices) {
@@ -722,7 +727,8 @@ bool FromIdentityCombiner(const ffi::Array<PrimExpr>& identities,
stored_values.reserve(n);
for (int i = 0; i < n; ++i) {
- buf_loads.push_back(MakeTensorLoad(combiners[i]->buffer, combiners[i]->indices));
+ buf_loads.push_back(
+ MakeTensorLoad(combiners[i]->dest.as_or_throw<TensorVar>(), combiners[i]->indices));
stored_values.push_back(combiners[i]->value);
}
diff --git a/src/s_tir/schedule/ir_comparator.cc b/src/s_tir/schedule/ir_comparator.cc
index 08afbdec8a..7b0f1fd969 100644
--- a/src/s_tir/schedule/ir_comparator.cc
+++ b/src/s_tir/schedule/ir_comparator.cc
@@ -640,7 +640,9 @@ bool TensorizeComparator::CompareBufferRegion(const TensorRegion& lhs, const Ten
}
// Comparator for TensorStoreNode and TensorLoadNode
-inline TensorVar GetBufferAccessBuffer(const TensorStoreNode* op) { return op->buffer; }
+inline TensorVar GetBufferAccessBuffer(const TensorStoreNode* op) {
+ return op->dest.as_or_throw<TensorVar>();
+}
inline TensorVar GetBufferAccessBuffer(const TensorLoadNode* op) {
return op->source.as_or_throw<tvm::tirx::TensorVar>();
}
diff --git a/src/s_tir/schedule/primitive/block_annotate.cc b/src/s_tir/schedule/primitive/block_annotate.cc
index 1debadc292..3b3f370eaa 100644
--- a/src/s_tir/schedule/primitive/block_annotate.cc
+++ b/src/s_tir/schedule/primitive/block_annotate.cc
@@ -335,12 +335,12 @@ class DTypeMutator : public ReplaceBufferMutator {
}
UnchangedOr<Stmt> Mutate_(const TensorStoreNode* op, InplaceMode inplace_mode) final {
- TensorVar original_buffer = op->buffer;
+ TensorVar original_buffer = op->dest.as_or_throw<TensorVar>();
TensorStore node = StmtExprMutator::Mutate_(op, inplace_mode)
.ValueOrUnchanged(ffi::GetRef<Stmt>(op))
.as_or_throw<TensorStore>();
if (auto replacement = VarRemapGet(original_buffer).as<TensorVar>()) {
- node.CopyOnWrite()->buffer = replacement.value();
+ node.CopyOnWrite()->dest = replacement.value();
node.CopyOnWrite()->value = Cast(tgt_dtype_, node->value);
}
return node;
diff --git a/src/s_tir/schedule/primitive/cache_index.cc b/src/s_tir/schedule/primitive/cache_index.cc
index 1767209112..af699a9c4e 100644
--- a/src/s_tir/schedule/primitive/cache_index.cc
+++ b/src/s_tir/schedule/primitive/cache_index.cc
@@ -344,7 +344,7 @@ ffi::Array<SBlock> MakeIndexCacheStage(IndexInfo* info, const ffi::String& stora
/*writes=*/{BufferRegion(info->cache_buffer[expr_index], access_region)},
/*name_hint=*/"index_" + std::to_string(expr_index),
/*body=*/
- TensorStore(info->cache_buffer[expr_index], new_expr, access_indices),
+ TensorStore(info->cache_buffer[expr_index], access_indices, new_expr),
/*init=*/std::nullopt,
/*alloc_buffers=*/{},
/*match_buffers=*/{},
diff --git a/src/s_tir/schedule/primitive/cache_index_helpers.cc b/src/s_tir/schedule/primitive/cache_index_helpers.cc
index 9a482cdcf3..ee3003486e 100644
--- a/src/s_tir/schedule/primitive/cache_index_helpers.cc
+++ b/src/s_tir/schedule/primitive/cache_index_helpers.cc
@@ -244,9 +244,9 @@ ffi::Optional<VisitInterrupt> ComputationsDoneBy::Visit(ffi::AnyView expr_value)
}
/*!
- * \brief The method which overrides the specific treatment for an IfThenElseNode
+ * \brief The method which overrides the specific treatment for an IfNode
*/
-ffi::Optional<VisitInterrupt> ComputationsDoneBy::Visit_(const IfThenElseNode* op) {
+ffi::Optional<VisitInterrupt> ComputationsDoneBy::Visit_(const IfNode* op) {
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(Visit(op->condition));
ComputationTable computations_done_by_cond = table_of_computations_;
table_of_computations_.clear();
diff --git a/src/s_tir/schedule/primitive/cache_index_helpers.h b/src/s_tir/schedule/primitive/cache_index_helpers.h
index a1d351d194..9f55408605 100644
--- a/src/s_tir/schedule/primitive/cache_index_helpers.h
+++ b/src/s_tir/schedule/primitive/cache_index_helpers.h
@@ -96,7 +96,7 @@ class ComputationsDoneBy : public s_tir::StmtExprVisitor {
protected:
ffi::Optional<VisitInterrupt> Visit(ffi::AnyView expr) override;
- ffi::Optional<VisitInterrupt> Visit_(const IfThenElseNode* op) override;
+ ffi::Optional<VisitInterrupt> Visit_(const IfNode* op) override;
ffi::Optional<VisitInterrupt> Visit_(const ForNode* op) override;
ffi::Optional<VisitInterrupt> Visit_(const WhileNode* op) override;
diff --git a/src/s_tir/schedule/primitive/cache_read_write.cc b/src/s_tir/schedule/primitive/cache_read_write.cc
index ce601f3690..3e3ef04f15 100644
--- a/src/s_tir/schedule/primitive/cache_read_write.cc
+++ b/src/s_tir/schedule/primitive/cache_read_write.cc
@@ -222,8 +222,8 @@ SBlock MakeReindexCacheStage(const TensorRegion& cache_region, ReindexCacheStage
/*name_hint*/ cache_region->source.as_or_throw<tvm::tirx::TensorVar>().name() + "_" +
storage_scope,
/*body=*/
- TensorStore(info->write_buffer, MakeTensorLoad(info->read_buffer, read_access_indices),
- write_access_indices),
+ TensorStore(info->write_buffer, write_access_indices,
+ MakeTensorLoad(info->read_buffer, read_access_indices)),
/*init=*/std::nullopt,
/*alloc_buffers=*/{},
/*match_buffers=*/{},
@@ -325,8 +325,8 @@ SBlock MakeCacheStage(const TensorRegion& cache_region, CacheStageInfo* info,
/*name_hint=*/cache_region->source.as_or_throw<tvm::tirx::TensorVar>().name() + "_" +
storage_scope,
/*body=*/
- TensorStore(info->write_buffer, MakeTensorLoad(info->read_buffer, read_access_indices),
- write_access_indices),
+ TensorStore(info->write_buffer, write_access_indices,
+ MakeTensorLoad(info->read_buffer, read_access_indices)),
/*init=*/std::nullopt,
/*alloc_buffers=*/{},
/*match_buffers=*/{},
@@ -432,7 +432,7 @@ SBlock MakeReIndexStage(const SBlock& block, CacheStageInfo* info,
/*writes=*/{BufferRegionFromPoint(info->write_buffer, dst_indices)},
/*name_hint=*/info->write_buffer.name() + "_reindex",
/*body=*/
- TensorStore(info->write_buffer, MakeTensorLoad(info->read_buffer, src_indices), dst_indices));
+ TensorStore(info->write_buffer, dst_indices, MakeTensorLoad(info->read_buffer, src_indices)));
// Step 4: Create surrounding loops
@@ -1436,7 +1436,7 @@ class CacheWriteRewriter : public StmtExprMutator {
}
UnchangedOr<Stmt> Mutate_(const TensorStoreNode* store, InplaceMode inplace_mode) override {
- bool rewrite_buffer = store->buffer.same_as(info_->write_buffer);
+ bool rewrite_buffer = store->dest.as_or_throw<TensorVar>().same_as(info_->write_buffer);
auto value = Mutate(store->value);
auto indices = Mutate(store->indices).as_or_throw<UnchangedOr<ffi::Array<PrimExpr>>>();
TensorStore stmt = ffi::GetRef<TensorStore>(store);
@@ -1447,7 +1447,7 @@ class CacheWriteRewriter : public StmtExprMutator {
}
if (rewrite_buffer) {
TensorStoreNode* n = stmt.CopyOnWrite();
- n->buffer = info_->read_buffer;
+ n->dest = info_->read_buffer;
if (!cache_full_region_) {
n->indices = RewriteIndices(n->indices);
}
@@ -1591,7 +1591,7 @@ class ReindexCacheWriteRewriter : public CacheWriteRewriter {
private:
UnchangedOr<Stmt> Mutate_(const TensorStoreNode* store, InplaceMode inplace_mode) final {
- bool rewrite_buffer = store->buffer.same_as(info_->write_buffer);
+ bool rewrite_buffer = store->dest.as_or_throw<TensorVar>().same_as(info_->write_buffer);
auto value = Mutate(store->value);
auto indices = Mutate(store->indices).as_or_throw<UnchangedOr<ffi::Array<PrimExpr>>>();
TensorStore stmt = ffi::GetRef<TensorStore>(store);
@@ -1602,7 +1602,7 @@ class ReindexCacheWriteRewriter : public CacheWriteRewriter {
}
if (rewrite_buffer) {
TensorStoreNode* n = stmt.CopyOnWrite();
- n->buffer = info_->read_buffer;
+ n->dest = info_->read_buffer;
n->indices = new_indices_;
return stmt;
} else {
@@ -1753,7 +1753,7 @@ class ReIndexCollector : public StmtExprVisitor {
for (const PrimExpr& index : store->indices) {
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(Visit(index));
}
- if (store->buffer.same_as(buffer_)) {
+ if (store->dest.as_or_throw<TensorVar>().same_as(buffer_)) {
CheckAndUpdateBufferAccessIndices(store->indices);
}
return std::nullopt;
@@ -1861,11 +1861,10 @@ class ReIndexRewriter : public StmtExprMutator {
return old_stmt;
}
- template <typename Node>
- Node VisitBufferAccess(Node node) {
- if (node->buffer.same_as(old_buffer_)) {
+ TensorStore VisitBufferAccess(TensorStore node) {
+ if (node->dest.same_as(old_buffer_)) {
auto* n = node.CopyOnWrite();
- n->buffer = new_buffer_;
+ n->dest = new_buffer_;
n->indices = indices_;
}
return node;
diff --git a/src/s_tir/schedule/primitive/compute_inline.cc b/src/s_tir/schedule/primitive/compute_inline.cc
index 369920ef58..ad2e92864a 100644
--- a/src/s_tir/schedule/primitive/compute_inline.cc
+++ b/src/s_tir/schedule/primitive/compute_inline.cc
@@ -679,8 +679,8 @@ class ReverseComputeInliner : public BaseInliner {
.ValueOrUnchanged(ffi::GetRef<PrimExpr>(_load))
.as_or_throw<TensorLoad>();
if (!load->source.as_or_throw<TensorVar>().same_as(self_->inlined_buffer_)) return load;
- PrimExpr replacement =
- MakeTensorLoad(self_->inlined_store_->buffer, self_->inlined_store_->indices);
+ PrimExpr replacement = MakeTensorLoad(self_->inlined_store_->dest.as_or_throw<TensorVar>(),
+ self_->inlined_store_->indices);
return StmtExprMutator::Mutate(ffi::AnyView(replacement), InplaceMode::kDisallow)
.ValueOrUnchanged(std::move(replacement))
.as_or_throw<PrimExpr>();
@@ -755,7 +755,7 @@ class ReverseComputeInliner : public BaseInliner {
}
const TensorStoreNode* producer_store = nullptr;
const Stmt& producer_body = producer_block_->body->seq[0];
- if (const auto* producer_if = producer_body.as<tirx::IfThenElseNode>()) {
+ if (const auto* producer_if = producer_body.as<IfNode>()) {
if (producer_if->else_case.has_value()) {
return false;
}
@@ -827,7 +827,7 @@ class ReverseComputeInliner : public BaseInliner {
return producer_block_realize;
}
if (const auto* if_ = producer_block->body->size() == 1
- ? producer_block->body->seq[0].as<IfThenElseNode>()
+ ? producer_block->body->seq[0].as<IfNode>()
: nullptr) {
if (!if_->else_case.has_value()) {
PrimExpr if_predicate = analyzer_->Simplify(if_->condition);
@@ -866,7 +866,7 @@ class ReverseComputeInliner : public BaseInliner {
TensorStore store = BaseInliner::Mutate_(_store, inplace_mode)
.ValueOrUnchanged(ffi::GetRef<Stmt>(_store))
.as_or_throw<TensorStore>();
- if (!store->buffer.same_as(inlined_buffer_)) {
+ if (!store->dest.as_or_throw<TensorVar>().same_as(inlined_buffer_)) {
return store;
}
return ReplaceInlinedBuffer(std::move(store));
@@ -1314,7 +1314,7 @@ bool ReductionEpilogueFuser::IsReductionBlock(const SBlockNode* block) {
void ReductionEpilogueFuser::ExtractEpilogueInfo() {
// Extract epilogue output buffer and indices
- epilogue_output_buffer_ = inlined_store_->buffer;
+ epilogue_output_buffer_ = inlined_store_->dest.as_or_throw<TensorVar>();
epilogue_output_indices_ = inlined_store_->indices;
// Extract epilogue output region from epilogue block writes
@@ -1437,7 +1437,7 @@ SBlock ReductionEpilogueFuser::CreateFusedReductionBlock(
return ffi::StructuralMap<ffi::WalkOrder::kPreOrder>(index, f_substitute)
.as_or_throw<PrimExpr>();
});
- TensorStore new_init_store = TensorStore(epilogue_output_buffer_, init_epilogue, init_indices);
+ TensorStore new_init_store = TensorStore(epilogue_output_buffer_, init_indices, init_epilogue);
new_block->init = new_init_store;
// 3. Generalized update transformation: apply epilogue expression with reduction buffer replaced
@@ -1461,7 +1461,7 @@ SBlock ReductionEpilogueFuser::CreateFusedReductionBlock(
TensorStore store = StmtExprMutator::Mutate_(op, inplace_mode)
.ValueOrUnchanged(ffi::GetRef<Stmt>(op))
.as_or_throw<TensorStore>();
- if (store->buffer.same_as(old_buffer_)) {
+ if (store->dest.as_or_throw<TensorVar>().same_as(old_buffer_)) {
// Replace old_buffer_ in store->value with new_buffer_ to get the reduction update
// expression This ensures store->value references new_buffer_ instead of old_buffer_
class ReductionUpdateReplacer : public StmtExprMutator {
@@ -1589,7 +1589,7 @@ SBlock ReductionEpilogueFuser::CreateFusedReductionBlock(
new_value = ffi::StructuralMap<ffi::WalkOrder::kPreOrder>(new_value, f_substitute)
.as_or_throw<PrimExpr>();
- return TensorStore(new_buffer_, new_value, store->indices);
+ return TensorStore(new_buffer_, store->indices, new_value);
}
return store;
}
diff --git a/src/s_tir/schedule/primitive/layout_transformation.cc b/src/s_tir/schedule/primitive/layout_transformation.cc
index edc20bae6b..a575dc5080 100644
--- a/src/s_tir/schedule/primitive/layout_transformation.cc
+++ b/src/s_tir/schedule/primitive/layout_transformation.cc
@@ -154,7 +154,7 @@ class TransformLayoutPlanner : public StmtExprVisitor {
}
ffi::Optional<VisitInterrupt> Visit_(const TensorStoreNode* op) override {
- if (!op->buffer.same_as(old_buffer_)) {
+ if (!op->dest.as_or_throw<TensorVar>().same_as(old_buffer_)) {
return std::nullopt;
}
@@ -370,7 +370,7 @@ class TransformLayoutPlanner : public StmtExprVisitor {
UnchangedOr<Stmt> Mutate_(const TensorStoreNode* op, InplaceMode inplace_mode) final {
bool can_replace = [&]() -> bool {
- if (!op->buffer.same_as(info.store->buffer)) {
+ if (!op->dest.as_or_throw<TensorVar>().same_as(info.store->dest)) {
return false;
}
@@ -392,9 +392,8 @@ class TransformLayoutPlanner : public StmtExprVisitor {
new_indices.Map([](const Var& var) { return var.as_or_throw<PrimExpr>(); });
PrimExpr pad_value_at_index =
pad_value.value()->MapIndices(new_index_exprs, ffi::GetRef<sym::Analyzer>(analyzer))[0];
- store =
- TensorStore(new_buffer, if_then_else(padding_predicate, pad_value_at_index, op->value),
- new_index_exprs);
+ store = TensorStore(new_buffer, new_index_exprs,
+ if_then_else(padding_predicate, pad_value_at_index, op->value));
} else {
all_stores_replaced = false;
}
@@ -622,7 +621,7 @@ class TransformLayoutPlanner : public StmtExprVisitor {
PrimExpr pad_value_at_index =
pad_value.value()->MapIndices(indices, ffi::GetRef<sym::Analyzer>(analyzer))[0];
- Stmt stmt = TensorStore(new_buffer, pad_value_at_index, indices);
+ Stmt stmt = TensorStore(new_buffer, indices, pad_value_at_index);
std::stringstream block_name;
block_name << "buffer_" << new_buffer.name() << "_padding";
@@ -910,9 +909,11 @@ class TransformLayoutRewriter : public s_tir::IRMutatorWithAnalyzer {
TensorStore tensor_store = Parent::Mutate_(op, inplace_mode)
.ValueOrUnchanged(ffi::GetRef<Stmt>(op))
.as_or_throw<TensorStore>();
- if (tensor_store->buffer.same_as(old_buffer_)) {
+ if (tensor_store->dest.as_or_throw<TensorVar>().same_as(old_buffer_)) {
auto* n = tensor_store.CopyOnWrite();
- RewriteBufferAccess(&n->buffer, &n->indices);
+ TensorVar buffer = n->dest.as_or_throw<TensorVar>();
+ RewriteBufferAccess(&buffer, &n->indices);
+ n->dest = std::move(buffer);
}
return tensor_store;
}
diff --git a/src/s_tir/schedule/primitive/loop_transformation.cc b/src/s_tir/schedule/primitive/loop_transformation.cc
index 22a553d1b8..f7a42a9a5e 100644
--- a/src/s_tir/schedule/primitive/loop_transformation.cc
+++ b/src/s_tir/schedule/primitive/loop_transformation.cc
@@ -523,8 +523,9 @@ class BufferIndicesMapExtractor : public StmtExprVisitor {
}
indices.push_back(var.value());
}
- if (buffer_indices_map.find(store->buffer) == buffer_indices_map.end() && !check_)
- buffer_indices_map.Set(store->buffer, indices);
+ if (buffer_indices_map.find(store->dest.as_or_throw<TensorVar>()) == buffer_indices_map.end() &&
+ !check_)
+ buffer_indices_map.Set(store->dest.as_or_throw<TensorVar>(), indices);
return StmtExprVisitor::Visit_(store);
}
@@ -772,7 +773,7 @@ ffi::Array<StmtSRef> LoopPartition(ScheduleState self, const StmtSRef& loop_sref
// Create common block with all the partitioned blocks as its children blocks
SBlockRealize common({}, IntImm::Bool(true),
- SBlock({}, {}, {}, block_name + "_common", tirx::SeqStmt(block_partitions)));
+ SBlock({}, {}, {}, block_name + "_common", SeqStmt(block_partitions)));
// Replace existing loop with the newly created common block
self->Replace(loop_sref, common, {});
diff --git a/src/s_tir/schedule/primitive/pad_einsum.cc b/src/s_tir/schedule/primitive/pad_einsum.cc
index b2b6bbb797..ef59db6a30 100644
--- a/src/s_tir/schedule/primitive/pad_einsum.cc
+++ b/src/s_tir/schedule/primitive/pad_einsum.cc
@@ -184,10 +184,10 @@ struct BufferPadding {
}
}
PrimExpr rhs = MakeTensorLoad(buffer, indices);
- return TensorStore(padded_buffer,
- if_then_else(predicate, rhs, prim::MakeConst(rhs.ty(), 0)), indices);
+ return TensorStore(padded_buffer, indices,
+ if_then_else(predicate, rhs, prim::MakeConst(rhs.ty(), 0)));
} else {
- return TensorStore(buffer, MakeTensorLoad(padded_buffer, indices), indices);
+ return TensorStore(buffer, indices, MakeTensorLoad(padded_buffer, indices));
}
}();
TensorRegion read_region = BufferRegion(buffer, instance_dom);
@@ -359,8 +359,9 @@ class PadEinsumBufferReplacer : public StmtExprMutator {
TensorStore store = StmtExprMutator::Mutate_(old_store_ptr, inplace_mode)
.ValueOrUnchanged(ffi::GetRef<Stmt>(old_store_ptr))
.as_or_throw<TensorStore>();
- if (ffi::Optional<TensorVar> buffer = VarRemapGet(store->buffer).as<TensorVar>()) {
- return TensorStore(buffer.value(), store->value, store->indices);
+ if (ffi::Optional<TensorVar> buffer =
+ VarRemapGet(store->dest.as_or_throw<TensorVar>()).as<TensorVar>()) {
+ return TensorStore(buffer.value(), store->indices, store->value);
} else {
return store;
}
diff --git a/src/s_tir/schedule/primitive/read_write_at.cc b/src/s_tir/schedule/primitive/read_write_at.cc
index b9a46e3354..415a362aa2 100644
--- a/src/s_tir/schedule/primitive/read_write_at.cc
+++ b/src/s_tir/schedule/primitive/read_write_at.cc
@@ -120,9 +120,9 @@ class ReadWriteAtBufferReplacer : public StmtExprMutator {
TensorStore store = StmtExprMutator::Mutate_(_store, inplace_mode)
.ValueOrUnchanged(ffi::GetRef<Stmt>(_store))
.as_or_throw<TensorStore>();
- if (store->buffer.same_as(src_)) {
+ if (store->dest.as_or_throw<TensorVar>().same_as(src_)) {
ffi::ObjectPtr<TensorStoreNode> new_store = ffi::make_object<TensorStoreNode>(*store.get());
- new_store->buffer = dst_;
+ new_store->dest = dst_;
return TensorStore(new_store);
}
return store;
@@ -334,7 +334,7 @@ struct ReadWriteAtImpl {
indices.push_back(domain[i]->min + loop_vars[i].as_or_throw<PrimExpr>());
}
Stmt stmt =
- TensorStore(copy_to, /*value=*/MakeTensorLoad(copy_from, indices), /*indices=*/indices);
+ TensorStore(copy_to, /*indices=*/indices, /*value=*/MakeTensorLoad(copy_from, indices));
for (int i = n - 1; i >= 0; --i) {
stmt = For(loop_vars[i].as_or_throw<PrimVar>(), IntImm::Int32(0), domain[i]->extent,
ForKind::kDefault, stmt);
diff --git a/src/s_tir/schedule/primitive/reduction.cc b/src/s_tir/schedule/primitive/reduction.cc
index 0903f31662..7104ea8f86 100644
--- a/src/s_tir/schedule/primitive/reduction.cc
+++ b/src/s_tir/schedule/primitive/reduction.cc
@@ -739,7 +739,7 @@ ffi::Array<TensorVar> CreateRFactorBuffers(const ffi::Array<TensorStore>& buf_st
ffi::Array<TensorVar> rf_buffers;
rf_buffers.reserve(buf_stores.size());
for (const TensorStore& buf_store : buf_stores) {
- TensorVar buffer = buf_store->buffer;
+ TensorVar buffer = buf_store->dest.as_or_throw<TensorVar>();
ffi::Array<PrimExpr> rf_shape = buffer->shape;
rf_shape.insert(rf_shape.begin() + factor_axis, rf_loop->extent);
@@ -841,7 +841,7 @@ class BaseBlockCreator {
// buffers.
if (!has_reduce_iter) {
for (int i = 0; i < n_buffers_; ++i) {
- buf_stores.push_back(TensorStore(update_buffers_[i], update_rhs_[i], update_indices_[i]));
+ buf_stores.push_back(TensorStore(update_buffers_[i], update_indices_[i], update_rhs_[i]));
}
return SeqStmt(buf_stores);
}
@@ -849,7 +849,7 @@ class BaseBlockCreator {
// Case 2. If the reduction is for single buffer, the block body is a single TensorStore.
ffi::Array<PrimExpr> stored_values = (*reducer_.get())(update_lhs_, update_rhs_);
if (n_buffers_ == 1) {
- return TensorStore(update_buffers_[0], stored_values[0], update_indices_[0]);
+ return TensorStore(update_buffers_[0], update_indices_[0], stored_values[0]);
}
// Case 3. In case the reduction is for multiple buffers, we should create the reduction with
@@ -860,11 +860,11 @@ class BaseBlockCreator {
Var var("v_" + update_buffers_[i].name(), stored_values[i].ty());
let_vars.push_back(var);
buf_stores.push_back(
- TensorStore(update_buffers_[i], var.as_or_throw<PrimExpr>(), update_indices_[i]));
+ TensorStore(update_buffers_[i], update_indices_[i], var.as_or_throw<PrimExpr>()));
}
ffi::Array<Stmt> stmts;
for (int i = 0; i < n_buffers_; ++i) {
- stmts.push_back(tirx::Bind(let_vars[i], stored_values[i]));
+ stmts.push_back(tvm::Bind(let_vars[i], stored_values[i]));
}
for (const auto& store : buf_stores) {
stmts.push_back(store);
@@ -881,7 +881,7 @@ class BaseBlockCreator {
inits.reserve(n_buffers_);
for (int i = 0; i < n_buffers_; ++i) {
inits.push_back(
- TensorStore(update_buffers_[i], reducer_->identity_element[i], update_indices_[i]));
+ TensorStore(update_buffers_[i], update_indices_[i], reducer_->identity_element[i]));
}
return SeqStmt(inits);
}
@@ -1047,7 +1047,7 @@ class RFactorBlockCreator : public BaseBlockCreator {
void CreateReadWriteRegions() final {
ffi::Map<TensorVar, TensorVar> buffer_map;
for (int i = 0; i < n_buffers_; ++i) {
- buffer_map.Set(old_reduction_updates_[i]->buffer, rf_buffers_[i]);
+ buffer_map.Set(old_reduction_updates_[i]->dest.as_or_throw<TensorVar>(), rf_buffers_[i]);
}
const SBlock& old_block = old_block_realize_->block;
auto map_block_var = [this](const Var& var) -> ffi::Expected<ffi::UnchangedOr<ffi::Any>> {
@@ -1158,7 +1158,7 @@ class WriteBackBlockCreator : public BaseBlockCreator {
};
for (int i = 0; i < n_buffers_; ++i) {
PrimExpr rhs = MakeTensorLoad(rf_buffers_[i], rf_buf_access_indices_);
- update_buffers_.push_back(old_reduction_updates_[i]->buffer);
+ update_buffers_.push_back(old_reduction_updates_[i]->dest.as_or_throw<TensorVar>());
update_indices_.push_back(old_reduction_updates_[i]->indices);
update_lhs_.push_back(
ffi::StructuralMap<ffi::WalkOrder::kPreOrder>(combiner_lhs_[i], map_block_var)
@@ -1417,8 +1417,8 @@ StmtSRef RFactor(ScheduleState self, const StmtSRef& rf_loop_sref, int factor_ax
// Step 6. Check whether `factor_axis` is in a correct range, and convert it to non-negative if it
// is negative.
- factor_axis =
- FactorAxisOutOfRangeError::CheckAndUpdate(self->mod, updates[0]->buffer, factor_axis);
+ factor_axis = FactorAxisOutOfRangeError::CheckAndUpdate(
+ self->mod, updates[0]->dest.as_or_throw<TensorVar>(), factor_axis);
// *****************************************************
// * IR Manipulation *
diff --git a/src/s_tir/schedule/primitive/rolling_buffer.cc b/src/s_tir/schedule/primitive/rolling_buffer.cc
index 31c7b2b6ac..385c06a833 100644
--- a/src/s_tir/schedule/primitive/rolling_buffer.cc
+++ b/src/s_tir/schedule/primitive/rolling_buffer.cc
@@ -394,9 +394,11 @@ class RollingBufferRewriter : public StmtExprMutator {
TensorStore stmt = StmtExprMutator::Mutate_(op, inplace_mode)
.ValueOrUnchanged(ffi::GetRef<Stmt>(op))
.as_or_throw<TensorStore>();
- if (stmt->buffer.same_as(info_->old_buffer)) {
+ if (stmt->dest.as_or_throw<TensorVar>().same_as(info_->old_buffer)) {
TensorStoreNode* n = stmt.CopyOnWrite();
- RewriteBufferAccess(&n->buffer, &n->indices);
+ TensorVar buffer = n->dest.as_or_throw<TensorVar>();
+ RewriteBufferAccess(&buffer, &n->indices);
+ n->dest = std::move(buffer);
// Need to add predicate to the current block to avoid recomputing elements.
rewrite_block_predicate_ = true;
}
diff --git a/src/s_tir/schedule/transform.cc b/src/s_tir/schedule/transform.cc
index 7a1c4dc316..e9ca53a999 100644
--- a/src/s_tir/schedule/transform.cc
+++ b/src/s_tir/schedule/transform.cc
@@ -394,8 +394,8 @@ ffi::Optional<LoopRV> TileWithTensorIntrin(const s_tir::Schedule& sch,
for (const auto& kv : info->loop_map) {
// Extract mapping (block_loop => desc_loop)
const tirx::StmtSRef& block_loop_sref = kv.first;
- const tirx::ForNode* block_loop = block_loop_sref->StmtAs<tirx::ForNode>();
- const tirx::ForNode* desc_loop = kv.second.get();
+ const ForNode* block_loop = block_loop_sref->StmtAs<ForNode>();
+ const ForNode* desc_loop = kv.second.get();
TVM_FFI_ICHECK(block_loop != nullptr && desc_loop != nullptr);
// Extract the loop extent
PrimExpr block_extent = analyzer->Simplify(block_loop->extent);
@@ -414,8 +414,7 @@ ffi::Optional<LoopRV> TileWithTensorIntrin(const s_tir::Schedule& sch,
TVM_FFI_ICHECK_EQ(split.size(), 2);
inner_loops.insert(sch->GetSRef(split[1]).operator->());
// The inner split will be reordered to the loop domain that is tensorized
- int desc_loop_index =
- static_cast<int>(info->desc_loop_indexer.at(ffi::GetRef<tirx::For>(desc_loop)));
+ int desc_loop_index = static_cast<int>(info->desc_loop_indexer.at(ffi::GetRef<For>(desc_loop)));
reorder_suffix[desc_loop_index] = split[1];
}
// Reorder the loops
diff --git a/src/s_tir/schedule/utils.h b/src/s_tir/schedule/utils.h
index e7a0553219..3904e0869e 100644
--- a/src/s_tir/schedule/utils.h
+++ b/src/s_tir/schedule/utils.h
@@ -166,8 +166,8 @@ inline IterVar IterVarFromLoop(const For& loop, ffi::String name, IterVarType it
* \return The thread scope bound to the loop
*/
inline runtime::ThreadScope GetThreadScope(const ForNode* loop) {
- if (loop->GetThreadBinding().has_value()) {
- return runtime::ThreadScope::Create(loop->GetThreadBinding().value());
+ if (tvm::tirx::GetThreadBinding(loop).has_value()) {
+ return runtime::ThreadScope::Create(tvm::tirx::GetThreadBinding(loop).value());
}
return runtime::ThreadScope{-1, -1};
}
diff --git a/src/s_tir/script/ir_builder/script_complete.cc b/src/s_tir/script/ir_builder/script_complete.cc
index edd530da26..2af52fae6d 100644
--- a/src/s_tir/script/ir_builder/script_complete.cc
+++ b/src/s_tir/script/ir_builder/script_complete.cc
@@ -176,7 +176,7 @@ Function ScriptComplete(Function func, const ffi::Array<TensorVar>& root_allocat
return func;
} else {
auto fptr = func.CopyOnWrite();
- fptr->body = tvm::tirx::SeqStmt(res);
+ fptr->body = tvm::SeqStmt(res);
return func;
}
}
diff --git a/src/s_tir/stmt.cc b/src/s_tir/stmt.cc
index 24147da716..38a2c9141e 100644
--- a/src/s_tir/stmt.cc
+++ b/src/s_tir/stmt.cc
@@ -398,7 +398,7 @@ SBlock::SBlock(ffi::Array<IterVar> iter_vars, ffi::Array<TensorRegion> reads,
ffi::Optional<SeqStmt> init, ffi::Array<TensorVar> alloc_buffers,
ffi::Array<MatchBufferRegion> match_buffers, ffi::Map<ffi::String, Any> annotations,
Span span)
- : tirx::Stmt(ffi::UnsafeInit{}) {
+ : Stmt(ffi::UnsafeInit{}) {
for (const auto& regions : {reads, writes}) {
for (const TensorRegion& region : regions) {
const auto buffer = region->source.as_or_throw<TensorVar>();
@@ -420,7 +420,7 @@ SBlock::SBlock(ffi::Array<IterVar> iter_vars, ffi::Array<TensorRegion> reads,
}
SBlock::SBlock(ffi::String name_hint, SeqStmt body, ffi::Array<TensorVar> alloc_buffers, Span span)
- : tirx::Stmt(ffi::UnsafeInit{}) {
+ : Stmt(ffi::UnsafeInit{}) {
ffi::ObjectPtr<SBlockNode> node = ffi::make_object<SBlockNode>(std::move(body));
node->iter_vars = {};
node->reads = {};
@@ -457,7 +457,7 @@ TVM_FFI_STATIC_INIT_BLOCK() {
// BlockRealize
SBlockRealize::SBlockRealize(ffi::Array<PrimExpr> values, PrimExpr predicate, SBlock block,
Span span)
- : tirx::Stmt(ffi::UnsafeInit{}) {
+ : Stmt(ffi::UnsafeInit{}) {
TVM_FFI_CHECK_EQ(block->iter_vars.size(), values.size(), ValueError)
<< "BlockRealize needs to have the same number of iter_vars and binding values";
PrimType predicate_ty = predicate.ty();
diff --git a/src/s_tir/transform/compact_buffer_region.cc b/src/s_tir/transform/compact_buffer_region.cc
index 371a0e8c25..d83cad5de7 100644
--- a/src/s_tir/transform/compact_buffer_region.cc
+++ b/src/s_tir/transform/compact_buffer_region.cc
@@ -80,7 +80,7 @@ class Var2BufferCollector : public StmtExprVisitor {
private:
ffi::Optional<VisitInterrupt> Visit_(const TensorStoreNode* op) final {
- var2buffer_[op->buffer.var()].insert(op->buffer);
+ var2buffer_[op->dest.as_or_throw<TensorVar>().var()].insert(op->dest.as_or_throw<TensorVar>());
return StmtExprVisitor::Visit_(op);
}
@@ -160,7 +160,7 @@ class BufferAccessRegionCollector : public StmtExprVisitor {
}
ffi::Optional<VisitInterrupt> Visit_(const TensorStoreNode* op) final {
- VisitBufferAccess(BufferRegionFromPoint(op->buffer, op->indices));
+ VisitBufferAccess(BufferRegionFromPoint(op->dest.as_or_throw<TensorVar>(), op->indices));
return Visit(op->value);
}
@@ -186,9 +186,9 @@ class BufferAccessRegionCollector : public StmtExprVisitor {
ffi::Optional<VisitInterrupt> Visit_(const ForNode* op) final {
Range loop_range = Range::FromMinExtent(op->min, op->extent);
- IterVar iter = op->GetThreadBinding().has_value()
+ IterVar iter = tvm::tirx::GetThreadBinding(op).has_value()
? IterVar(Range(), op->loop_var, IterVarType::kThreadIndex,
- op->GetThreadBinding().value())
+ tvm::tirx::GetThreadBinding(op).value())
: IterVar(Range(), op->loop_var, IterVarType::kDataPar);
ancestor_iters_.push_back(iter);
dom_analyzer_->Bind(op->loop_var, loop_range);
@@ -231,7 +231,7 @@ class BufferAccessRegionCollector : public StmtExprVisitor {
return std::nullopt;
}
- ffi::Optional<VisitInterrupt> Visit_(const IfThenElseNode* op) final {
+ ffi::Optional<VisitInterrupt> Visit_(const IfNode* op) final {
// Visit condition
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(StmtExprVisitor::Visit(op->condition));
{
@@ -619,12 +619,14 @@ class BufferCompactor : public StmtExprMutator {
: buffer_info_(std::move(buffer_info)) {}
UnchangedOr<Stmt> Mutate_(const TensorStoreNode* _op, InplaceMode inplace_mode) final {
- TensorVar original_buffer = _op->buffer;
+ TensorVar original_buffer = _op->dest.as_or_throw<TensorVar>();
TensorStore store = StmtExprMutator::Mutate_(_op, inplace_mode)
.ValueOrUnchanged(ffi::GetRef<Stmt>(_op))
.as_or_throw<TensorStore>();
TensorStoreNode* op = store.CopyOnWrite();
- RewriteBufferAccess(original_buffer, &op->buffer, &op->indices);
+ TensorVar buffer = op->dest.as_or_throw<TensorVar>();
+ RewriteBufferAccess(original_buffer, &buffer, &op->indices);
+ op->dest = std::move(buffer);
return store;
}
@@ -678,7 +680,7 @@ class BufferCompactor : public StmtExprMutator {
auto rewritten = ffi::make_object<BindNode>(*op);
rewritten->value =
Call(new_buffer.type(), call->op, args, call->attrs, call->ty_args, call->span);
- tirx::Bind binding(std::move(rewritten));
+ tvm::Bind binding(std::move(rewritten));
return StmtExprMutator::Mutate_(binding.get(), inplace_mode).ValueOrUnchanged(binding);
}
diff --git a/src/s_tir/transform/default_gpu_schedule.cc b/src/s_tir/transform/default_gpu_schedule.cc
index e06bc1c22e..ebc409c32d 100644
--- a/src/s_tir/transform/default_gpu_schedule.cc
+++ b/src/s_tir/transform/default_gpu_schedule.cc
@@ -40,7 +40,7 @@ void ThreadBind(s_tir::Schedule sch, const s_tir::SBlockRV& block, int64_t max_t
ffi::Array<s_tir::LoopRV> loops = sch->GetLoops(block);
for (const s_tir::LoopRV& loop : loops) {
// skip block if already scheduled
- if (sch->Get(loop)->GetThreadBinding().has_value()) {
+ if (tvm::tirx::GetThreadBinding(sch->Get(loop)).has_value()) {
return;
}
}
@@ -130,8 +130,8 @@ tirx::Function WrapBareSBlockBody(const tirx::Function& func) {
// produced by the rest of the pipeline has an implicit root SBlockRealize
// whose block body is a For loop (or a nested SBlockRealize) — that case
// already has somewhere to put thread bindings, so leave it alone.
- const tirx::SeqStmt& inner = realize->block->body;
- if (inner->size() == 1 && (inner->seq[0]->IsInstance<tirx::ForNode>() ||
+ const SeqStmt& inner = realize->block->body;
+ if (inner->size() == 1 && (inner->seq[0]->IsInstance<ForNode>() ||
inner->seq[0]->IsInstance<s_tir::SBlockRealizeNode>())) {
return func;
}
@@ -146,8 +146,7 @@ tirx::Function WrapBareSBlockBody(const tirx::Function& func) {
s_tir::SBlockRealize inner_realize(
/*iter_values=*/ffi::Array<tvm::PrimExpr>{loop_var.as_or_throw<tvm::PrimExpr>()},
/*predicate=*/realize->predicate, inner_block);
- tirx::Stmt for_stmt =
- tirx::For(loop_var.as_or_throw<PrimVar>(), zero, one, tirx::ForKind::kDefault, inner_realize);
+ Stmt for_stmt = For(loop_var.as_or_throw<PrimVar>(), zero, one, ForKind::kDefault, inner_realize);
s_tir::SBlock root_block(/*iter_vars=*/ffi::Array<s_tir::IterVar>{},
/*reads=*/ffi::Array<tvm::TensorRegion>{},
/*writes=*/ffi::Array<tvm::TensorRegion>{},
diff --git a/src/s_tir/transform/hoist_expression.cc b/src/s_tir/transform/hoist_expression.cc
index 3672e08adf..63b3d374a8 100644
--- a/src/s_tir/transform/hoist_expression.cc
+++ b/src/s_tir/transform/hoist_expression.cc
@@ -390,7 +390,7 @@ class HoistInfoCollector : public StmtExprVisitor {
return std::nullopt;
}
- ffi::Optional<VisitInterrupt> Visit_(const IfThenElseNode* op) final {
+ ffi::Optional<VisitInterrupt> Visit_(const IfNode* op) final {
AttemptHoistConditional(op->condition, HoistedConditionals::kIfElseStmt,
op->else_case.has_value());
return Parent::Visit_(op);
@@ -520,9 +520,9 @@ class ExpressionHoister : public s_tir::IRMutatorWithAnalyzer {
for (auto cond_it = info.conditions.rbegin(); cond_it != info.conditions.rend(); cond_it++) {
if (cond_it->IsEnabled(config_)) {
if (cond_it->generate_else_case) {
- stmt = IfThenElse(cond_it->condition, stmt, SeqStmt(stmt));
+ stmt = If(cond_it->condition, stmt, SeqStmt(stmt));
} else {
- stmt = IfThenElse(cond_it->condition, stmt);
+ stmt = If(cond_it->condition, stmt);
}
}
}
diff --git a/src/s_tir/transform/inject_permuted_layout.cc b/src/s_tir/transform/inject_permuted_layout.cc
index 24c9ffa50f..3c4f6085cf 100644
--- a/src/s_tir/transform/inject_permuted_layout.cc
+++ b/src/s_tir/transform/inject_permuted_layout.cc
@@ -214,17 +214,18 @@ class PermutedLayoutInjector : public IRMutatorWithAnalyzer {
.ValueOrUnchanged(ffi::GetRef<Stmt>(op))
.as_or_throw<TensorStore>();
- if (!permute_ || store->buffer->shape.size() < 2) {
+ if (!permute_ || store->dest.as_or_throw<TensorVar>()->shape.size() < 2) {
return store;
}
- auto scope = StorageScope::Create(store->buffer.scope());
+ auto scope = StorageScope::Create(store->dest.as_or_throw<TensorVar>().scope());
if (scope.rank != StorageRank::kShared) {
return store;
}
auto store_node = store.CopyOnWrite();
- store_node->indices = HandleTensorIndices(store_node->buffer, store_node->indices);
+ store_node->indices =
+ HandleTensorIndices(store_node->dest.as_or_throw<TensorVar>(), store_node->indices);
return store;
}
diff --git a/src/s_tir/transform/inject_ptx_async_copy.cc b/src/s_tir/transform/inject_ptx_async_copy.cc
index c8709819e1..1f20accab6 100644
--- a/src/s_tir/transform/inject_ptx_async_copy.cc
+++ b/src/s_tir/transform/inject_ptx_async_copy.cc
@@ -73,7 +73,7 @@ class PTXAsyncCopyInjector : public StmtExprMutator {
static_cast<int>(load->source.as_or_throw<tvm::tirx::TensorVar>()->dtype.StorageBytes());
if (bytes == 4 || bytes == 8 || bytes == 16) {
- auto dst_elem_type = GetPointerType(store->buffer.DataPointerType());
+ auto dst_elem_type = GetPointerType(store->dest.as_or_throw<TensorVar>().DataPointerType());
auto src_elem_type =
GetPointerType(load->source.as_or_throw<tvm::tirx::TensorVar>().DataPointerType());
TVM_FFI_ICHECK(dst_elem_type.has_value() && src_elem_type.has_value())
@@ -83,7 +83,7 @@ class PTXAsyncCopyInjector : public StmtExprMutator {
if (dst_elem_type.value() != src_elem_type.value()) {
// The only case where src and dst have different dtypes is when the dst shared memory
// is a byte buffer generated by merging dynamic shared memory.
- TVM_FFI_ICHECK(store->buffer.scope() == "shared.dyn");
+ TVM_FFI_ICHECK(store->dest.as_or_throw<TensorVar>().scope() == "shared.dyn");
TVM_FFI_ICHECK((dst_elem_type.value() == PrimType::UInt(8)));
// TensorStore/Load have the "pointer reinterpret" semantics according to their
// "value" dtype. Their "indices" are supposed to be applied after such pointer cast,
@@ -96,16 +96,16 @@ class PTXAsyncCopyInjector : public StmtExprMutator {
if (indices_lanes == 1) {
auto src_offset = load->indices[0];
auto dst_offset = store->indices[0];
- ffi::Array<Expr> args = {store->buffer.data(), mul(dst_offset, PrimExpr(index_factor)),
- load->source.as_or_throw<tvm::tirx::TensorVar>().data(),
- src_offset, PrimExpr(bytes)};
+ ffi::Array<Expr> args = {
+ store->dest.as_or_throw<TensorVar>().data(), mul(dst_offset, PrimExpr(index_factor)),
+ load->source.as_or_throw<tvm::tirx::TensorVar>().data(), src_offset, PrimExpr(bytes)};
// use arguments size to indicate whether or not to use predicated cp.async
if (predicated) {
args.push_back(predicate_value.value());
}
static const Op ptx_cp_async_op = Op::Get("tirx.s_tir.cp_async_raw");
- return Evaluate(
- Call(store->buffer->dtype, ptx_cp_async_op, args).as_or_throw<PrimExpr>());
+ return Evaluate(Call(store->dest.as_or_throw<TensorVar>()->dtype, ptx_cp_async_op, args)
+ .as_or_throw<PrimExpr>());
}
// Predicated load don't support vectorized indexing.
@@ -135,12 +135,12 @@ class PTXAsyncCopyInjector : public StmtExprMutator {
}();
if (src_offset.has_value() && dst_offset.has_value()) {
static const Op ptx_cp_async_op = Op::Get("tirx.s_tir.cp_async_raw");
- return Evaluate(
- Call(store->buffer->dtype, ptx_cp_async_op,
- {store->buffer.data(), mul(dst_offset.value(), PrimExpr(index_factor)),
- load->source.as_or_throw<tvm::tirx::TensorVar>().data(), src_offset.value(),
- PrimExpr(bytes)})
- .as_or_throw<PrimExpr>());
+ return Evaluate(Call(store->dest.as_or_throw<TensorVar>()->dtype, ptx_cp_async_op,
+ {store->dest.as_or_throw<TensorVar>().data(),
+ mul(dst_offset.value(), PrimExpr(index_factor)),
+ load->source.as_or_throw<tvm::tirx::TensorVar>().data(),
+ src_offset.value(), PrimExpr(bytes)})
+ .as_or_throw<PrimExpr>());
}
} else {
// Only some vectorized indexing patterns are supported for now.
@@ -169,12 +169,12 @@ class PTXAsyncCopyInjector : public StmtExprMutator {
if (src_offset.has_value() && dst_offset.has_value()) {
static const Op ptx_cp_async_op = Op::Get("tirx.s_tir.cp_async_raw");
- return Evaluate(
- Call(store->buffer->dtype, ptx_cp_async_op,
- {store->buffer.data(), mul(dst_offset.value(), PrimExpr(index_factor)),
- load->source.as_or_throw<tvm::tirx::TensorVar>().data(), src_offset.value(),
- PrimExpr(bytes), predicate_value.value()})
- .as_or_throw<PrimExpr>());
+ return Evaluate(Call(store->dest.as_or_throw<TensorVar>()->dtype, ptx_cp_async_op,
+ {store->dest.as_or_throw<TensorVar>().data(),
+ mul(dst_offset.value(), PrimExpr(index_factor)),
+ load->source.as_or_throw<tvm::tirx::TensorVar>().data(),
+ src_offset.value(), PrimExpr(bytes), predicate_value.value()})
+ .as_or_throw<PrimExpr>());
}
}
}
@@ -184,7 +184,8 @@ class PTXAsyncCopyInjector : public StmtExprMutator {
}
UnchangedOr<Stmt> Mutate_(const TensorStoreNode* store, InplaceMode inplace_mode) {
- if (in_async && (store->buffer.scope() == "shared" || store->buffer.scope() == "shared.dyn")) {
+ if (in_async && (store->dest.as_or_throw<TensorVar>().scope() == "shared" ||
+ store->dest.as_or_throw<TensorVar>().scope() == "shared.dyn")) {
if (auto* load = store->value.as<TensorLoadNode>()) {
return InjectPTX(load, store);
} else if (auto* call = store->value.as<CallNode>()) {
diff --git a/src/s_tir/transform/inject_software_pipeline.cc b/src/s_tir/transform/inject_software_pipeline.cc
index 22bdfe5b2d..e8563967f3 100644
--- a/src/s_tir/transform/inject_software_pipeline.cc
+++ b/src/s_tir/transform/inject_software_pipeline.cc
@@ -303,7 +303,7 @@ class PipelineBodyRewriter : public StmtExprMutator {
}
UnchangedOr<Stmt> Mutate_(const TensorStoreNode* op, InplaceMode inplace_mode) final {
- auto replacement = VarRemapGet(op->buffer).as<TensorVar>();
+ auto replacement = VarRemapGet(op->dest.as_or_throw<TensorVar>()).as<TensorVar>();
TensorStore store = StmtExprMutator::Mutate_(op, inplace_mode)
.ValueOrUnchanged(ffi::GetRef<Stmt>(op))
.as_or_throw<TensorStore>();
@@ -312,7 +312,7 @@ class PipelineBodyRewriter : public StmtExprMutator {
}
TensorVar new_buffer = replacement.value();
auto* n = store.CopyOnWrite();
- n->buffer = new_buffer;
+ n->dest = new_buffer;
PrimExpr version =
floormod((pipeline_loop_->loop_var - pipeline_loop_->min), new_buffer->shape[0]);
n->indices.insert(n->indices.begin(), version);
diff --git a/src/s_tir/transform/inject_virtual_thread.cc b/src/s_tir/transform/inject_virtual_thread.cc
index 52a6839afe..9b736a85cf 100644
--- a/src/s_tir/transform/inject_virtual_thread.cc
+++ b/src/s_tir/transform/inject_virtual_thread.cc
@@ -170,7 +170,7 @@ class VarTouchedAnalysis : public StmtExprVisitor {
for (const auto& index : op->indices) {
expr_touched_->Visit(index);
}
- Record(op->buffer.get(), *expr_touched_);
+ Record(op->dest.as_or_throw<TensorVar>().get(), *expr_touched_);
return std::nullopt;
}
ffi::Optional<VisitInterrupt> Visit_(const ForNode* op) final {
@@ -326,8 +326,8 @@ class VTInjector : public s_tir::IRMutatorWithAnalyzer {
args.push_back(predicate);
return Call(op->ty, op->op, args, op->attrs, op->ty_args, op->span);
}
- TensorStore access = VisitBufferAccess(TensorStore(buffer, value.value(), indices, op->span));
- ffi::Array<Expr> args{access->buffer.var(), access->value};
+ TensorStore access = VisitBufferAccess(TensorStore(buffer, indices, value.value(), op->span));
+ ffi::Array<Expr> args{access->dest.as_or_throw<TensorVar>().var(), access->value};
for (const PrimExpr& index : access->indices) args.push_back(index);
args.push_back(predicate);
return Call(op->ty, op->op, args, op->attrs, op->ty_args, op->span);
@@ -391,18 +391,18 @@ class VTInjector : public s_tir::IRMutatorWithAnalyzer {
return VisitBufferAccess(std::move(node));
}
- template <typename Node>
- Node VisitBufferAccess(Node node) {
- if (touched_var_.count(node->buffer.get())) {
+ TensorStore VisitBufferAccess(TensorStore node) {
+ TensorVar buffer = node->dest.as_or_throw<TensorVar>();
+ if (touched_var_.count(buffer.get())) {
visit_touched_var_ = true;
}
- auto it = alloc_remap_.find(node->buffer.get());
+ auto it = alloc_remap_.find(buffer.get());
if (it != alloc_remap_.end()) {
TVM_FFI_ICHECK_EQ(node->indices.size(), 1)
<< "InjectVirtualThread expects rewritten allocations to be flat memory.";
auto writer = node.CopyOnWrite();
- writer->buffer = GetRemappedBuffer(node->buffer, it->second);
+ writer->dest = GetRemappedBuffer(buffer, it->second);
writer->indices = {RewriteIndex(node->indices[0], it->second)};
}
@@ -510,8 +510,8 @@ class VTInjector : public s_tir::IRMutatorWithAnalyzer {
}
}
}
- // IfThenElse
- UnchangedOr<Stmt> Mutate_(const IfThenElseNode* op, InplaceMode inplace_mode) final {
+ // If
+ UnchangedOr<Stmt> Mutate_(const IfNode* op, InplaceMode inplace_mode) final {
auto condition_result = this->Mutate(op->condition, inplace_mode);
bool condition_unchanged = condition_result.UnchangedOrSameAs(op->condition);
PrimExpr condition = std::move(condition_result).ValueOrUnchanged(op->condition);
@@ -534,7 +534,7 @@ class VTInjector : public s_tir::IRMutatorWithAnalyzer {
if (condition_unchanged && then_case_unchanged && else_case.same_as(op->else_case)) {
return ffi::Unchanged();
} else {
- return IfThenElse(condition, then_case, else_case);
+ return If(condition, then_case, else_case);
}
}
diff --git a/src/s_tir/transform/ir_utils.h b/src/s_tir/transform/ir_utils.h
index 006e7bb2a7..be72b96d4f 100644
--- a/src/s_tir/transform/ir_utils.h
+++ b/src/s_tir/transform/ir_utils.h
@@ -31,7 +31,7 @@ namespace tvm {
namespace s_tir {
/*! \brief Convert a schedulable statement or module to SSA form. */
-tirx::Stmt ConvertSSA(tirx::Stmt stmt);
+Stmt ConvertSSA(Stmt stmt);
IRModule ConvertSSA(IRModule mod);
/*!
@@ -60,7 +60,7 @@ using StorageAlignAnnotation = ffi::Array<StorageAlignTuple>;
* \return The result dict from buffer var to storage align annotations.
*/
std::unordered_map<tvm::Var, StorageAlignAnnotation> CollectStorageAlignAnnotation(
- const tirx::Stmt& body);
+ const Stmt& body);
} // namespace s_tir
} // namespace tvm
diff --git a/src/s_tir/transform/lift_thread_binding.cc b/src/s_tir/transform/lift_thread_binding.cc
index f4ec631bbd..0e41b2415c 100644
--- a/src/s_tir/transform/lift_thread_binding.cc
+++ b/src/s_tir/transform/lift_thread_binding.cc
@@ -48,7 +48,7 @@ FindLoopLCA(const Stmt& root) {
ffi::Optional<VisitInterrupt> Visit_(const ForNode* op) final {
stack.push_back(ffi::GetRef<Stmt>(op));
TVM_FFI_S_VISIT_MAYBE_EARLY_RETURN(StmtExprVisitor::Visit_(op));
- if (op->GetThreadBinding().has_value()) {
+ if (tvm::tirx::GetThreadBinding(op).has_value()) {
UpdateLCA(op);
}
stack.pop_back();
@@ -56,7 +56,7 @@ FindLoopLCA(const Stmt& root) {
}
void UpdateLCA(const ForNode* loop) {
- std::string thread_tag = loop->GetThreadBinding().value();
+ std::string thread_tag = tvm::tirx::GetThreadBinding(loop).value();
{
ffi::Map<ffi::String, ffi::Any>* tgt = &annotations[thread_tag];
for (const auto& kv : loop->annotations) {
@@ -134,7 +134,7 @@ class ThreadBindingLifter : public StmtExprMutator {
UnchangedOr<Stmt> Mutate_(const ForNode* _op, InplaceMode inplace_mode) final {
For op = ffi::GetRef<For>(_op);
bool is_kernel_root = false;
- if (op->GetThreadBinding().has_value()) {
+ if (tvm::tirx::GetThreadBinding(op).has_value()) {
if (iter_lca.empty()) {
is_kernel_root = true;
SetKernelRoot(_op);
@@ -155,7 +155,7 @@ class ThreadBindingLifter : public StmtExprMutator {
if (is_kernel_root) {
iter_lca.clear();
}
- if (op->GetThreadBinding().has_value()) {
+ if (tvm::tirx::GetThreadBinding(op).has_value()) {
return body;
} else {
new_op.CopyOnWrite()->body = std::move(body);
diff --git a/src/s_tir/transform/loop_partition.cc b/src/s_tir/transform/loop_partition.cc
index e303ca648d..f5eb9d3509 100644
--- a/src/s_tir/transform/loop_partition.cc
+++ b/src/s_tir/transform/loop_partition.cc
@@ -111,8 +111,8 @@ using ExpressionSet = std::unordered_set<PrimExpr, ffi::ObjectPtrHash, ffi::Obje
// Virtual threads are lowered separately and are not hardware partition scopes.
static bool IsVirtualThread(const ForNode* op) {
- if (!op->GetThreadBinding().has_value()) return false;
- const auto tag = op->GetThreadBinding().value();
+ if (!tvm::tirx::GetThreadBinding(op).has_value()) return false;
+ const auto tag = tvm::tirx::GetThreadBinding(op).value();
return std::string(tag).rfind("vthread", 0) == 0;
}
@@ -167,8 +167,8 @@ class CandidateSelector final : public StmtExprVisitor {
candidates.insert(ffi::GetRef<Stmt>(op));
return StmtExprVisitor::Visit_(op);
}
- if (op->GetThreadBinding().has_value() &&
- runtime::ThreadScope::Create(op->GetThreadBinding().value()).rank != 0) {
+ if (tvm::tirx::GetThreadBinding(op).has_value() &&
+ runtime::ThreadScope::Create(tvm::tirx::GetThreadBinding(op).value()).rank != 0) {
return StmtExprVisitor::Visit_(op);
}
// partition const loop when sets partition_const_loop_
@@ -453,7 +453,7 @@ class ThreadPartitionInserter : public StmtExprMutator {
: ps_(ps), cond_(cond), innermost_thread_scope_(false) {}
UnchangedOr<Stmt> Mutate_(const ForNode* op, InplaceMode inplace_mode) final {
- if (!op->GetThreadBinding().has_value() || IsVirtualThread(op)) {
+ if (!tvm::tirx::GetThreadBinding(op).has_value() || IsVirtualThread(op)) {
return StmtExprMutator::Mutate_(op, inplace_mode);
}
innermost_thread_scope_ = true;
@@ -465,7 +465,7 @@ class ThreadPartitionInserter : public StmtExprMutator {
Stmt simplified_body = ffi::make_object<ConditionEliminator>(ps_)
->Mutate(body, InplaceMode::kDisallow)
.ValueOrUnchanged(body);
- loop.CopyOnWrite()->body = IfThenElse(cond_, simplified_body, SeqStmt(body));
+ loop.CopyOnWrite()->body = If(cond_, simplified_body, SeqStmt(body));
}
innermost_thread_scope_ = false;
return loop;
@@ -481,7 +481,7 @@ class ThreadPartitionInserter : public StmtExprMutator {
if (innermost_thread_scope_) {
Stmt simplified_body =
ffi::make_object<ConditionEliminator>(ps_)->Mutate(op->body).ValueOrUnchanged(op->body);
- Stmt body = IfThenElse(cond_, simplified_body, op->body);
+ Stmt body = If(cond_, simplified_body, op->body);
auto region = stmt.as_or_throw<RegionStmt>();
region.CopyOnWrite()->body = body;
stmt = region;
@@ -523,7 +523,7 @@ class LoopPartitioner : public StmtExprMutator {
UnchangedOr<Stmt> Mutate_(const ForNode* op, InplaceMode inplace_mode) final {
if (IsVirtualThread(op)) return StmtExprMutator::Mutate_(op, inplace_mode);
- bool thread_scope = op->GetThreadBinding().has_value();
+ bool thread_scope = tvm::tirx::GetThreadBinding(op).has_value();
analyzer_->Bind(op->loop_var, Range::FromMinExtent(op->min, op->extent), true);
auto fs = ffi::GetRef<Stmt>(op);
if (selector->candidates.count(fs)) {
@@ -533,8 +533,8 @@ class LoopPartitioner : public StmtExprMutator {
}
// Relax threadIdx ranges to avoid introducing divergent partition branches.
- bool relax =
- thread_scope && runtime::ThreadScope::Create(op->GetThreadBinding().value()).rank == 1;
+ bool relax = thread_scope &&
+ runtime::ThreadScope::Create(tvm::tirx::GetThreadBinding(op).value()).rank == 1;
auto& ranges = relax ? relax_map_ : hint_map_;
ranges.insert({op->loop_var.get(), IntSet::Interval(op->min, op->min + op->extent - 1)});
Stmt res = StmtExprMutator::Mutate_(op, InplaceMode::kDisallow)
@@ -926,7 +926,7 @@ inline Stmt LoopPartitioner::MakeFor(const ffi::Object* node, PrimExpr extent, S
};
return ffi::StructuralMap<ffi::WalkOrder::kPreOrder>(body, f_substitute).as_or_throw<Stmt>();
} else {
- TVM_FFI_ICHECK(!for_node->GetThreadBinding().has_value());
+ TVM_FFI_ICHECK(!tvm::tirx::GetThreadBinding(for_node).has_value());
auto new_loop = ffi::make_object<ForNode>(*for_node);
new_loop->min = IntImm(for_node->min.ty(), 0);
new_loop->extent = extent;
diff --git a/src/s_tir/transform/lower_cross_thread_reduction.cc b/src/s_tir/transform/lower_cross_thread_reduction.cc
index 3b5ea235ec..ee3f291e73 100644
--- a/src/s_tir/transform/lower_cross_thread_reduction.cc
+++ b/src/s_tir/transform/lower_cross_thread_reduction.cc
@@ -68,10 +68,11 @@ struct ThreadScopeEqual {
* \return True if the loop is bound to threadIdx.x/y/z
*/
bool IsBoundToThreadIdx(const ForNode* loop) {
- if (!loop->GetThreadBinding().has_value()) {
+ if (!tvm::tirx::GetThreadBinding(loop).has_value()) {
return false;
}
- runtime::ThreadScope scope = runtime::ThreadScope::Create(loop->GetThreadBinding().value());
+ runtime::ThreadScope scope =
+ runtime::ThreadScope::Create(tvm::tirx::GetThreadBinding(loop).value());
return scope.rank == 1 && scope.dim_index >= 0;
}
@@ -196,11 +197,11 @@ class BufferReplacer : public StmtExprMutator {
}
UnchangedOr<Stmt> Mutate_(const TensorStoreNode* store, InplaceMode inplace_mode) final {
- if (auto replacement = VarRemapGet(store->buffer).as<TensorVar>()) {
+ if (auto replacement = VarRemapGet(store->dest.as_or_throw<TensorVar>()).as<TensorVar>()) {
PrimExpr value = StmtExprMutator::Mutate(ffi::AnyView(store->value), inplace_mode)
.ValueOrUnchanged(store->value)
.as_or_throw<PrimExpr>();
- return TensorStore(replacement.value(), std::move(value), {0});
+ return TensorStore(replacement.value(), {0}, std::move(value));
} else {
return StmtExprMutator::Mutate_(store, inplace_mode);
}
@@ -298,7 +299,7 @@ class InThreadReducerMaker : public StmtExprMutator {
if (!body.value().same_as(loop->body)) {
res.CopyOnWrite()->body = body.value();
}
- if (res->GetThreadBinding().has_value() &&
+ if (tvm::tirx::GetThreadBinding(res).has_value() &&
UnderLoopReductionBlockVarCollector::CheckHasReductionBlocks(res)) {
return res->body;
}
@@ -376,7 +377,7 @@ Stmt TransformReductionBlock(const SBlockRealizeNode* realize,
inits.reserve(n_buffers);
for (int i = 0; i < n_buffers; ++i) {
inits.push_back(
- TensorStore(it_buffers.value()[i], reducer->identity_element[i], {IntImm::Int32(0)}));
+ TensorStore(it_buffers.value()[i], {IntImm::Int32(0)}, reducer->identity_element[i]));
}
stmts.push_back(SBlockRealize(/*iter_values=*/{},
/*predicate=*/IntImm::Bool(true),
@@ -431,7 +432,7 @@ Stmt TransformReductionBlock(const SBlockRealizeNode* realize,
}
ffi::Array<PrimExpr> thread_axes;
for (const ForNode* reduction_loop : reduction_loops) {
- if (reduction_loop->GetThreadBinding().has_value()) {
+ if (tvm::tirx::GetThreadBinding(reduction_loop).has_value()) {
thread_axes.push_back(reduction_loop->loop_var);
}
}
@@ -518,8 +519,8 @@ Stmt TransformReductionBlock(const SBlockRealizeNode* realize,
.as_or_throw<PrimExpr>());
}
for (int i = 0; i < n_buffers; ++i) {
- wb_updates.push_back(TensorStore(
- wb_buffers[i], MakeTensorLoad(ct_buffers[i], {IntImm::Int32(0)}), wb_indices));
+ wb_updates.push_back(TensorStore(wb_buffers[i], wb_indices,
+ MakeTensorLoad(ct_buffers[i], {IntImm::Int32(0)})));
wb_regions.push_back(BufferRegion(wb_buffers[i], region));
}
@@ -561,7 +562,7 @@ Stmt TransformReductionBlock(const SBlockRealizeNode* realize,
ffi::StructuralWalk<ffi::WalkOrder::kPostOrder>(realize->predicate, walk_fn);
if (wb_buffers[0].scope() != "local") {
for (const ForNode* loop : reduction_loops) {
- if (loop->GetThreadBinding().has_value()) {
+ if (tvm::tirx::GetThreadBinding(loop).has_value()) {
wb_predicate = wb_predicate &&
(static_cast<PrimExpr>(loop->loop_var) == IntImm(loop->loop_var.ty(), 0));
}
@@ -582,7 +583,7 @@ Stmt TransformReductionBlock(const SBlockRealizeNode* realize,
Stmt new_stmt = SeqStmt(std::move(stmts));
for (auto rit = reduction_loops.rbegin(); rit != reduction_loops.rend(); ++rit) {
const ForNode* loop = *rit;
- if (loop->GetThreadBinding().has_value()) {
+ if (tvm::tirx::GetThreadBinding(loop).has_value()) {
ffi::ObjectPtr<ForNode> n = ffi::make_object<ForNode>(*loop);
n->body = std::move(new_stmt);
new_stmt = For(n);
@@ -629,7 +630,7 @@ class CrossThreadReductionTransformer : public StmtExprMutator {
// Step 3. Collect the loop.
reduction_loops.push_back(loop);
// Step 4. See whether the loop is bound to some thread axis.
- if (loop->GetThreadBinding().has_value()) {
+ if (tvm::tirx::GetThreadBinding(loop).has_value()) {
need = true;
}
}
@@ -671,8 +672,8 @@ class CrossThreadReductionTransformer : public StmtExprMutator {
// Erase those threads which are not free to this block.
for (const ForNode* loop : loop_stack_) {
- if (loop->GetThreadBinding().has_value()) {
- ThreadScope scope = ThreadScope::Create(loop->GetThreadBinding().value());
+ if (tvm::tirx::GetThreadBinding(loop).has_value()) {
+ ThreadScope scope = ThreadScope::Create(tvm::tirx::GetThreadBinding(loop).value());
thread2range.erase(scope);
}
}
@@ -726,7 +727,7 @@ class CrossThreadReductionTransformer : public StmtExprMutator {
// bound to `threadIdx.x/y/z`.
int n_bound_reduction_loops = 0;
for (const ForNode* reduction_loop : reduction_loops) {
- if (reduction_loop->GetThreadBinding().has_value()) {
+ if (tvm::tirx::GetThreadBinding(reduction_loop).has_value()) {
++n_bound_reduction_loops;
TVM_FFI_CHECK(IsBoundToThreadIdx(reduction_loop), ValueError)
<< "Cross-thread reduction requires all the reduction-related loops that "
@@ -753,8 +754,8 @@ class CrossThreadReductionTransformer : public StmtExprMutator {
ffi::Array<TensorVar> reduction_buffers;
reduction_buffers.reserve(updates.size());
for (const TensorStore& buf_store : updates) {
- reduction_buffers.push_back(buf_store->buffer);
- if (buf_store->buffer.scope() == "local") {
+ reduction_buffers.push_back(buf_store->dest.as_or_throw<TensorVar>());
+ if (buf_store->dest.as_or_throw<TensorVar>().scope() == "local") {
TVM_FFI_CHECK_NE(is_local_buf, 0, ValueError)
<< "Cross-thread reduction requires all reduction buffers to be all "
"local or all non-local. However, here some buffer is local while some buffer is "
@@ -809,8 +810,8 @@ class CrossThreadReductionTransformer : public StmtExprMutator {
// - we are careful about thread block boundary for safety.
bool is_block_idx = false;
bool is_thread_idx = false;
- if (loop->GetThreadBinding().has_value()) {
- ThreadScope scope = ThreadScope::Create(loop->GetThreadBinding().value());
+ if (tvm::tirx::GetThreadBinding(loop).has_value()) {
+ ThreadScope scope = ThreadScope::Create(tvm::tirx::GetThreadBinding(loop).value());
if (scope.rank == 1 && scope.dim_index >= 0) {
is_thread_idx = true;
++thread_idx_depth;
@@ -907,9 +908,10 @@ class CrossThreadReductionTransformer : public StmtExprMutator {
std::vector<std::pair<ThreadScope, Range>> reduction_threads;
reduction_threads.reserve(reduction_loops.size());
for (const ForNode* loop : reduction_loops) {
- if (loop->GetThreadBinding().has_value()) {
- reduction_threads.emplace_back(ThreadScope::Create(loop->GetThreadBinding().value()),
- Range::FromMinExtent(loop->min, loop->extent));
+ if (tvm::tirx::GetThreadBinding(loop).has_value()) {
+ reduction_threads.emplace_back(
+ ThreadScope::Create(tvm::tirx::GetThreadBinding(loop).value()),
+ Range::FromMinExtent(loop->min, loop->extent));
}
}
for (const TensorVar& reduction_buf : reduction_buffers) {
diff --git a/src/s_tir/transform/lower_init_block.cc b/src/s_tir/transform/lower_init_block.cc
index ee74906d01..a1a320f482 100644
--- a/src/s_tir/transform/lower_init_block.cc
+++ b/src/s_tir/transform/lower_init_block.cc
@@ -79,7 +79,7 @@ class InitBlockLower : public StmtExprMutator {
for (size_t i = 1; i < conditions.size(); ++i) {
cond = logical_and(cond, conditions[i]);
}
- return IfThenElse(cond, init);
+ return If(cond, init);
}
};
diff --git a/src/s_tir/transform/lower_match_buffer.cc b/src/s_tir/transform/lower_match_buffer.cc
index 83cd2ba181..c7cb232585 100644
--- a/src/s_tir/transform/lower_match_buffer.cc
+++ b/src/s_tir/transform/lower_match_buffer.cc
@@ -124,7 +124,7 @@ class MatchBufferLower : public StmtExprMutator {
UnchangedOr<Stmt> Mutate_(const TensorStoreNode* op, InplaceMode inplace_mode) final {
// Save the original buffer before base class mutation may remap it
- TensorVar orig_buffer = op->buffer;
+ TensorVar orig_buffer = op->dest.as_or_throw<TensorVar>();
TensorStore stmt = StmtExprMutator::Mutate_(op, inplace_mode)
.ValueOrUnchanged(ffi::GetRef<Stmt>(op))
.as_or_throw<TensorStore>();
@@ -141,7 +141,7 @@ class MatchBufferLower : public StmtExprMutator {
auto* n = stmt.CopyOnWrite();
n->indices = ConvertIndices(MatchBufferRegion(buffer, source), op->indices);
- n->buffer = source->source.as_or_throw<tvm::tirx::TensorVar>();
+ n->dest = source->source.as_or_throw<tvm::tirx::TensorVar>();
return stmt;
}
}
diff --git a/src/s_tir/transform/lower_opaque_block.cc b/src/s_tir/transform/lower_opaque_block.cc
index 3c1a77098e..5743683d62 100644
--- a/src/s_tir/transform/lower_opaque_block.cc
+++ b/src/s_tir/transform/lower_opaque_block.cc
@@ -76,7 +76,7 @@ class OpaqueBlockLower : public StmtExprMutator {
// Step 2. Transform the `predicate` to if-then-else
Stmt body = new_block->body;
if (!IsOne(predicate)) {
- body = IfThenElse(predicate, std::move(body));
+ body = If(predicate, std::move(body));
}
// Step 3. Handle allocations in reverse order
for (size_t i = new_block->alloc_buffers.size(); i > 0; --i) {
@@ -112,7 +112,7 @@ class OpaqueBlockLower : public StmtExprMutator {
.as_or_throw<UnchangedOr<ffi::Optional<PrimExpr>>>()
.ValueOrUnchanged(op->step);
auto previous_remap = VarRemapGet(op->loop_var);
- if (!op->GetThreadBinding().has_value() && IsOne(extent) && op->annotations.empty()) {
+ if (!tvm::tirx::GetThreadBinding(op).has_value() && IsOne(extent) && op->annotations.empty()) {
// handling unit loop
VarRemapSet(op->loop_var, prim::cast(op->loop_var.ty(), min));
}
@@ -138,7 +138,7 @@ class OpaqueBlockLower : public StmtExprMutator {
VarRemapSet(op->loop_var, previous_remap);
// Step 2. Keep thread-binding loops until LowerThreadBinding.
- if (!op->GetThreadBinding().has_value() && IsOne(extent) && op->annotations.empty() &&
+ if (!tvm::tirx::GetThreadBinding(op).has_value() && IsOne(extent) && op->annotations.empty() &&
!op->annotations.count(s_tir::attr::irregular_loop_mark)) {
return body;
}
diff --git a/src/s_tir/transform/lower_thread_binding.cc b/src/s_tir/transform/lower_thread_binding.cc
index 6c1b07bb2c..7815d3cb3c 100644
--- a/src/s_tir/transform/lower_thread_binding.cc
+++ b/src/s_tir/transform/lower_thread_binding.cc
@@ -45,7 +45,7 @@ class ThreadBindingLowerer : public StmtExprMutator {
}
UnchangedOr<Stmt> Mutate_(const ForNode* op, InplaceMode inplace_mode) final {
- if (!op->GetThreadBinding().has_value()) {
+ if (!tvm::tirx::GetThreadBinding(op).has_value()) {
auto annotations = Mutate(op->annotations, inplace_mode)
.as_or_throw<UnchangedOr<ffi::Map<ffi::String, ffi::Any>>>();
auto result = StmtExprMutator::Mutate_(op, inplace_mode);
@@ -57,7 +57,7 @@ class ThreadBindingLowerer : public StmtExprMutator {
PrimExpr min = Mutate(op->min, inplace_mode).ValueOrUnchanged(op->min);
PrimExpr extent = Mutate(op->extent, inplace_mode).ValueOrUnchanged(op->extent);
TVM_FFI_ICHECK(IsZero(min)) << "Thread binding loops must start at zero";
- TVM_FFI_ICHECK(op->GetThreadBinding().has_value());
+ TVM_FFI_ICHECK(tvm::tirx::GetThreadBinding(op).has_value());
TVM_FFI_ICHECK(!op->annotations.count("loop_partition_hint") ||
op->annotations.at("loop_partition_hint") == nullptr)
<< "Run LoopPartition before LowerThreadBinding";
@@ -75,8 +75,9 @@ class ThreadBindingLowerer : public StmtExprMutator {
body = For(PrimVar("annotation", ty), IntImm(ty, 0), IntImm(ty, 1), ForKind::kDefault,
std::move(body), std::move(annotations), std::nullopt);
}
- return RegionStmt(tirx::launch_thread_op(), {StringImm(op->GetThreadBinding().value()), extent},
- {launch_var}, DictAttrs(), std::move(body), {}, op->span);
+ return RegionStmt(tirx::launch_thread_op(),
+ {StringImm(tvm::tirx::GetThreadBinding(op).value()), extent}, {launch_var},
+ DictAttrs(), std::move(body), {}, op->span);
}
};
diff --git a/src/s_tir/transform/manifest_shared_memory_local_stage.cc b/src/s_tir/transform/manifest_shared_memory_local_stage.cc
index 1e53f7d691..5cbcea094c 100644
--- a/src/s_tir/transform/manifest_shared_memory_local_stage.cc
+++ b/src/s_tir/transform/manifest_shared_memory_local_stage.cc
@@ -57,12 +57,14 @@ class IntermediateStageRewriter {
std::tuple<TensorVar, TensorVar, SBlock, Stmt> Rewrite(const SBlockNode* block) {
const TensorStoreNode* store =
block->body->size() == 1 ? block->body->seq[0].as<TensorStoreNode>() : nullptr;
- TVM_FFI_CHECK(store != nullptr && runtime::StorageScope::Create(store->buffer.scope()).rank ==
- runtime::StorageRank::kShared,
- ValueError)
+ TVM_FFI_CHECK(
+ store != nullptr &&
+ runtime::StorageScope::Create(store->dest.as_or_throw<TensorVar>().scope()).rank ==
+ runtime::StorageRank::kShared,
+ ValueError)
<< "Expect the body of the block to be TensorStore to shared memory.";
- const TensorVar& target_buffer = store->buffer;
+ const TensorVar& target_buffer = store->dest.as_or_throw<TensorVar>();
// Step 0: Collect relaxed loops
std::vector<const ForNode*> relaxed_loops = CollectRelaxedOuterLoops(block, target_buffer);
@@ -130,7 +132,7 @@ class IntermediateStageRewriter {
ffi::Array<PrimExpr> local_stage_indices,
std::vector<const ForNode*> relaxed_loops, const TensorStoreNode* store) {
// Step 0: Create the body of the local stage, which is TensorStore to the intermediate buffer.
- Stmt local_stage = TensorStore(new_buffer, store->value, local_stage_indices);
+ Stmt local_stage = TensorStore(new_buffer, local_stage_indices, store->value);
// Step 1: Make block and block realize
TensorRegion write_buffer_region = BufferRegionFromPoint(new_buffer, local_stage_indices);
diff --git a/src/s_tir/transform/memhammer_coalesce.cc b/src/s_tir/transform/memhammer_coalesce.cc
index e810fa4d0e..8674610cce 100644
--- a/src/s_tir/transform/memhammer_coalesce.cc
+++ b/src/s_tir/transform/memhammer_coalesce.cc
@@ -129,7 +129,7 @@ Stmt SplitBindVectorize(const Stmt& stmt, const ConstraintSet& constraints) {
ffi::StructuralMap<ffi::WalkOrder::kPreOrder>(loop->body, f_substitute).as_or_throw<Stmt>();
PrimExpr predicate = substitute_value < loop->extent;
if (!analyzer->CanProve(predicate)) {
- body = IfThenElse(predicate, body);
+ body = If(predicate, body);
}
body = For(new_loop_vars.back().as_or_throw<PrimVar>(), 0, vector_len, ForKind::kVectorized,
std::move(body));
@@ -169,9 +169,9 @@ ffi::Array<PrimExpr> GetMapping(const Stmt& stmt, const ConstraintSet& constrain
const TensorStoreNode* buf_store = TVM_TYPE_AS(body, TensorStoreNode);
TensorRegion write_region = constraints.write_region;
const ffi::Array<PrimExpr>& write_index = buf_store->indices;
- TVM_FFI_ICHECK(
- write_region->region.size() == write_index.size() &&
- write_region->source.as_or_throw<tvm::tirx::TensorVar>().same_as(buf_store->buffer));
+ TVM_FFI_ICHECK(write_region->region.size() == write_index.size() &&
+ write_region->source.as_or_throw<tvm::tirx::TensorVar>().same_as(
+ buf_store->dest.as_or_throw<TensorVar>()));
ffi::Array<PrimExpr> result;
sym::Analyzer analyzer;
for (int i = 0; i < static_cast<int>(write_region->region.size()); i++) {
@@ -238,7 +238,7 @@ Stmt InverseMapping::Rewrite(const Stmt& stmt, const ConstraintSet& constraints,
TensorLoad new_buf_load =
MakeTensorLoad(read_region->source.as_or_throw<tvm::tirx::TensorVar>(), read_index);
TensorStore new_buf_store = TensorStore(write_region->source.as_or_throw<tvm::tirx::TensorVar>(),
- new_buf_load, write_index);
+ write_index, new_buf_load);
Stmt ret = new_buf_store;
// Step 3.3 construct loop body
for (int i = static_cast<int>(new_loop_vars.size()) - 1; i >= 0; i--) {
diff --git a/src/s_tir/transform/memhammer_intermediate_stage.cc b/src/s_tir/transform/memhammer_intermediate_stage.cc
index d26a78a0d1..97c8ded087 100644
--- a/src/s_tir/transform/memhammer_intermediate_stage.cc
+++ b/src/s_tir/transform/memhammer_intermediate_stage.cc
@@ -51,7 +51,7 @@ std::pair<Stmt, ffi::Optional<For>> LiftThreadBindingLoops(Stmt stmt) {
std::vector<const ForNode*> thread_binding_loops;
Stmt body = stmt;
while (const ForNode* loop = body.as<ForNode>()) {
- if (loop->GetThreadBinding().has_value()) {
+ if (tvm::tirx::GetThreadBinding(loop).has_value()) {
thread_binding_loops.push_back(loop);
} else {
normal_loops.push_back(loop);
@@ -268,15 +268,15 @@ std::pair<Stmt, SeqStmt> InsertCacheStage(Stmt stmt, bool is_write_cache, ffi::S
body = loop->body->size() == 1 ? loop->body->seq[0] : loop->body;
}
ffi::Optional<PrimExpr> predicate;
- if (const auto* op = body.as<IfThenElseNode>()) {
+ if (const auto* op = body.as<IfNode>()) {
// the predicate is generated by coalescing
predicate = op->condition;
TVM_FFI_ICHECK_EQ(op->then_case->size(), 1);
body = op->then_case->seq[0];
}
for (const For& loop : outer_loops) {
- if (loop->GetThreadBinding().has_value()) {
- const ffi::String thread_tag = loop->GetThreadBinding().value();
+ if (tvm::tirx::GetThreadBinding(loop).has_value()) {
+ const ffi::String thread_tag = tvm::tirx::GetThreadBinding(loop).value();
auto thread_scope = runtime::ThreadScope::Create(thread_tag);
if (CanRelaxStorageUnderThread(runtime::StorageScope::Create(storage_scope), thread_scope)) {
if (is_write_cache && thread_scope.dim_index == 0) {
@@ -382,7 +382,7 @@ std::pair<Stmt, SeqStmt> InsertCacheStage(Stmt stmt, bool is_write_cache, ffi::S
new_buffer =
WithScope(target_buffer_load->source.as_or_throw<tvm::tirx::TensorVar>(), storage_scope);
} else {
- new_buffer = WithScope(buf_store->buffer, storage_scope);
+ new_buffer = WithScope(buf_store->dest.as_or_throw<TensorVar>(), storage_scope);
}
ffi::ObjectPtr<TensorTypeNode> buffer_type = CopyTensorType(new_buffer);
buffer_type->shape = new_shape;
@@ -402,10 +402,9 @@ std::pair<Stmt, SeqStmt> InsertCacheStage(Stmt stmt, bool is_write_cache, ffi::S
ffi::StructuralMap<ffi::WalkOrder::kPreOrder>(generate_body, map_var).as_or_throw<Stmt>();
} else {
generate_body =
- TensorStore(new_buffer,
+ TensorStore(new_buffer, subst_cache_indices,
ffi::StructuralMap<ffi::WalkOrder::kPreOrder>(buf_store->value, map_var)
- .as_or_throw<PrimExpr>(),
- subst_cache_indices);
+ .as_or_throw<PrimExpr>());
}
if (predicate.has_value()) {
@@ -415,7 +414,7 @@ std::pair<Stmt, SeqStmt> InsertCacheStage(Stmt stmt, bool is_write_cache, ffi::S
PrimExpr subst_predicate =
ffi::StructuralMap<ffi::WalkOrder::kPreOrder>(predicate.value(), map_var)
.as_or_throw<PrimExpr>();
- generate_body = IfThenElse(subst_predicate, generate_body);
+ generate_body = If(subst_predicate, generate_body);
}
for (int i = static_cast<int>(loops_under_compute_location.size()) - 1; i >= 0; i--) {
@@ -438,13 +437,13 @@ std::pair<Stmt, SeqStmt> InsertCacheStage(Stmt stmt, bool is_write_cache, ffi::S
if (is_write_cache) {
TensorLoad new_buffer_load = MakeTensorLoad(new_buffer, cache_indices);
rewrite_body =
- TensorStore(new_buffer, ffi::GetRef<TensorLoad>(target_buffer_load), cache_indices);
+ TensorStore(new_buffer, cache_indices, ffi::GetRef<TensorLoad>(target_buffer_load));
} else {
- rewrite_body = TensorStore(buf_store->buffer, MakeTensorLoad(new_buffer, cache_indices),
- buf_store->indices);
+ rewrite_body = TensorStore(buf_store->dest.as_or_throw<TensorVar>(), buf_store->indices,
+ MakeTensorLoad(new_buffer, cache_indices));
}
if (predicate.has_value()) {
- rewrite_body = IfThenElse(predicate.value(), rewrite_body);
+ rewrite_body = If(predicate.value(), rewrite_body);
}
for (int i = static_cast<int>(loops_under_compute_location.size()) - 1; i >= 0; i--) {
const ForNode* orig_loop = loops_under_compute_location[i];
diff --git a/src/s_tir/transform/memhammer_lower_auto_copy.cc b/src/s_tir/transform/memhammer_lower_auto_copy.cc
index 0883b24f21..2477926589 100644
--- a/src/s_tir/transform/memhammer_lower_auto_copy.cc
+++ b/src/s_tir/transform/memhammer_lower_auto_copy.cc
@@ -223,8 +223,8 @@ class AutoPadder {
.ValueOrUnchanged(ffi::GetRef<Stmt>(_op))
.as_or_throw<TensorStore>();
TensorStoreNode* op = store.CopyOnWrite();
- if (auto replacement = VarRemapGet(op->buffer).as<TensorVar>()) {
- op->buffer = replacement.value();
+ if (auto replacement = VarRemapGet(op->dest.as_or_throw<TensorVar>()).as<TensorVar>()) {
+ op->dest = replacement.value();
}
return store;
}
@@ -516,10 +516,11 @@ class AutoPadder {
}
ffi::Optional<VisitInterrupt> Visit_(const ForNode* op) final {
- if (!op->GetThreadBinding().has_value()) {
+ if (!tvm::tirx::GetThreadBinding(op).has_value()) {
substitute_map_.Set(op->loop_var, op->min);
} else {
- int64_t extent = warp_thread_extent_.Get(op->GetThreadBinding().value()).value_or(1);
+ int64_t extent =
+ warp_thread_extent_.Get(tvm::tirx::GetThreadBinding(op).value()).value_or(1);
var_range_.Set(op->loop_var, Range::FromMinExtent(op->min, IntImm::Int64(extent)));
}
if (op->kind == ForKind::kVectorized) {
@@ -530,7 +531,7 @@ class AutoPadder {
if (op->kind == ForKind::kVectorized) {
vector_length_ = -1;
}
- if (!op->GetThreadBinding().has_value()) {
+ if (!tvm::tirx::GetThreadBinding(op).has_value()) {
substitute_map_.erase(op->loop_var);
}
return std::nullopt;
@@ -543,7 +544,8 @@ class AutoPadder {
* \param op the buffer store
*/
ffi::Optional<VisitInterrupt> Visit_(const TensorStoreNode* op) final {
- runtime::StorageScope scope = runtime::StorageScope::Create(op->buffer.scope());
+ runtime::StorageScope scope =
+ runtime::StorageScope::Create(op->dest.as_or_throw<TensorVar>().scope());
if (scope.rank == runtime::StorageRank::kShared) {
ffi::Array<PrimExpr> substitued_indices;
sym::Analyzer analyzer;
@@ -559,12 +561,13 @@ class AutoPadder {
std::vector<std::vector<int>> iter_space =
PatternCollector::CollectIterationSpace(substitued_indices, var_range_, data_bits_);
if (!iter_space.empty()) {
- self->iter_spaces_[op->buffer.get()].push_back(iter_space);
+ self->iter_spaces_[op->dest.as_or_throw<TensorVar>().get()].push_back(iter_space);
}
if (vector_length_ != -1 &&
CheckVarContiguous(op->indices.back(), vector_var.value(), substitute_map_)) {
- int64_t m = self->padding_min_.Get(op->buffer).value_or(1);
- self->padding_min_.Set(op->buffer, std::max(static_cast<int64_t>(vector_length_), m));
+ int64_t m = self->padding_min_.Get(op->dest.as_or_throw<TensorVar>()).value_or(1);
+ self->padding_min_.Set(op->dest.as_or_throw<TensorVar>(),
+ std::max(static_cast<int64_t>(vector_length_), m));
}
}
return StmtExprVisitor::Visit_(op);
@@ -831,9 +834,10 @@ class ThreadExtentCollector : public StmtExprVisitor {
return StmtExprVisitor::Visit_(op);
}
ffi::Optional<VisitInterrupt> Visit_(const ForNode* op) final {
- if (op->GetThreadBinding().has_value()) {
+ if (tvm::tirx::GetThreadBinding(op).has_value()) {
if (const auto* extent = op->extent.as<IntImmNode>()) {
- thread_extent_.Set(op->GetThreadBinding().value(), static_cast<int64_t>(extent->value));
+ thread_extent_.Set(tvm::tirx::GetThreadBinding(op).value(),
+ static_cast<int64_t>(extent->value));
}
}
return StmtExprVisitor::Visit_(op);
diff --git a/src/s_tir/transform/memhammer_tensorcore_rewrite.cc b/src/s_tir/transform/memhammer_tensorcore_rewrite.cc
index cdb10dbd1a..fff568d0d6 100644
--- a/src/s_tir/transform/memhammer_tensorcore_rewrite.cc
+++ b/src/s_tir/transform/memhammer_tensorcore_rewrite.cc
@@ -143,7 +143,7 @@ Stmt RewriteWmmaLoad(Stmt stmt) {
const TensorLoadNode* buf_load = TVM_TYPE_AS(buf_store->value, TensorLoadNode);
TensorVar src_buffer = buf_load->source.as_or_throw<tvm::tirx::TensorVar>();
- TensorVar tgt_buffer = buf_store->buffer;
+ TensorVar tgt_buffer = buf_store->dest.as_or_throw<TensorVar>();
std::string layout = tgt_buffer.scope() == "wmma.matrix_a" ? "row_major" : "col_major";
TensorVar new_src_buffer(
/*name=*/"src", TensorType(/*storage_scope=*/src_buffer.scope(),
@@ -250,7 +250,7 @@ Stmt RewriteWmmaStore(Stmt stmt) {
};
ffi::StructuralWalk<ffi::WalkOrder::kPostOrder>(buf_store->value, walk_fn);
TensorVar src_buffer = buf_load->source.as_or_throw<tvm::tirx::TensorVar>();
- TensorVar tgt_buffer = buf_store->buffer;
+ TensorVar tgt_buffer = buf_store->dest.as_or_throw<TensorVar>();
PrimType dtype_ty = src_buffer->dtype;
const PrimType& dtype = dtype_ty;
@@ -477,7 +477,7 @@ Stmt RewriteMmaStore(Stmt stmt) {
// Step 3.1. Generate new buffer
TensorVar src_buffer = buf_load->source.as_or_throw<tvm::tirx::TensorVar>();
- TensorVar tgt_buffer = buf_store->buffer;
+ TensorVar tgt_buffer = buf_store->dest.as_or_throw<TensorVar>();
PrimType dtype_ty = src_buffer->dtype;
const PrimType& dtype = dtype_ty;
TensorVar new_src_buffer(
@@ -500,26 +500,26 @@ Stmt RewriteMmaStore(Stmt stmt) {
Stmt mma_body = SBlockRealize(
/*iter_values=*/{}, //
/*predicate=*/IntImm::Bool(true),
- SBlock(/*iter_vars=*/{},
- /*reads=*/{BufferRegion(src_buffer, read_region)},
- /*writes=*/{BufferRegion(tgt_buffer, write_region)},
- /*name_hint=*/"mma_store",
- RegionStmt(tirx::launch_thread_op(), {StringImm("threadIdx.x"), IntImm::Int32(32)},
- {tx}, DictAttrs(), /*body=*/
- For(vec.as_or_throw<PrimVar>(), 0, 2, ForKind::kVectorized,
- /*body=*/
- TensorStore(new_tgt_buffer,
- MakeTensorLoad(new_src_buffer, {floordiv(tx, 4),
- floormod(tx, 4) * 2 + vec}),
- {floordiv(tx, 4), floormod(tx, 4) * 2 + vec}))),
- /*init=*/std::nullopt,
- /*alloc_buffers=*/{},
- /*match_buffers=*/
- {
- MatchBufferRegion(new_src_buffer, BufferRegion(src_buffer, read_region)),
- MatchBufferRegion(new_tgt_buffer, BufferRegion(tgt_buffer, write_region)),
- },
- /*annotations=*/{}));
+ SBlock(
+ /*iter_vars=*/{},
+ /*reads=*/{BufferRegion(src_buffer, read_region)},
+ /*writes=*/{BufferRegion(tgt_buffer, write_region)},
+ /*name_hint=*/"mma_store",
+ RegionStmt(tirx::launch_thread_op(), {StringImm("threadIdx.x"), IntImm::Int32(32)}, {tx},
+ DictAttrs(), /*body=*/
+ For(vec.as_or_throw<PrimVar>(), 0, 2, ForKind::kVectorized,
+ /*body=*/
+ TensorStore(new_tgt_buffer, {floordiv(tx, 4), floormod(tx, 4) * 2 + vec},
+ MakeTensorLoad(new_src_buffer, {floordiv(tx, 4),
+ floormod(tx, 4) * 2 + vec})))),
+ /*init=*/std::nullopt,
+ /*alloc_buffers=*/{},
+ /*match_buffers=*/
+ {
+ MatchBufferRegion(new_src_buffer, BufferRegion(src_buffer, read_region)),
+ MatchBufferRegion(new_tgt_buffer, BufferRegion(tgt_buffer, write_region)),
... 7477 lines suppressed ...