You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by GitBox <gi...@apache.org> on 2022/01/07 22:21:37 UTC
[GitHub] [tvm] vinx13 opened a new pull request #9871: [TIR][Schedule] Blockize and Tensorize
vinx13 opened a new pull request #9871:
URL: https://github.com/apache/tvm/pull/9871
This PR added two schedule primitives, blockize, and tensorize to TensorIR.
cc @junrushao1994 @spectrometerHBH @Hzfengsy @masahi @comaniac @tqchen @csullivan
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780723680
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
+ }
+ };
+ std::tie(lhs_expr, lhs_iter) = detect(op->iter_values[i + offset]);
+ std::tie(rhs_expr, rhs_iter) = detect(rhs->iter_values[i]);
+ CHECK((lhs_iter && rhs_iter) || (!lhs_iter && !rhs_iter)) << "Incompatible binding";
+ if (lhs_iter) VisitExpr(lhs_iter.value(), rhs_iter.value());
+ if (is_zero(rhs_expr)) {
+ CHECK(is_zero(lhs_expr)) << "Incompatible binding";
+ } else {
+ const auto* bv = rhs_expr.as<VarNode>();
+ if (!bv) {
+ VisitExpr(lhs_expr, rhs_expr);
+ } else {
+ auto it = equal_map_.find(GetRef<Var>(bv));
+ if (it == equal_map_.end()) {
+ equal_map_[GetRef<Var>(bv)] = lhs_expr;
+ } else {
+ CHECK(it->second->IsInstance<PrimExprNode>());
+ VisitExpr(lhs_expr, Downcast<PrimExpr>(it->second));
+ }
+ }
+ }
+ }
+ } else {
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ if (!VisitExpr(op->iter_values[i + offset], rhs->iter_values[i])) return false;
+ }
+ const Block& block = op->block;
+ for (size_t i = 0; i < offset; ++i) {
+ Var block_var = Downcast<Var>(op->iter_values[i]);
+ auto it = equal_map_.find(block_var);
+ equal_map_[block->iter_vars[i]->var] = (it == equal_map_.end() ? block_var : it->second);
+ }
+ }
+ }
+
+ return VisitExpr(op->predicate, rhs->predicate) && VisitStmt(op->block, rhs->block);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockNode>();
+ // Check block equality.
+ // All iter vars and buffer regions including the order shoudl match.
+ // When checking iter vars, DefEqual is used to remap variables.
+ // Only the inner most several axis are compared. Other iter vars are added to extra_block_vars.
+ if (op->iter_vars.size() < rhs->iter_vars.size()) return false;
+
+ size_t offset = op->iter_vars.size() - rhs->iter_vars.size();
+ for (size_t i = 0; i < rhs->iter_vars.size(); ++i) {
+ auto lhs_var = op->iter_vars[i + offset], rhs_var = rhs->iter_vars[i];
+ // Skip iter dom
+ if (!DefEqual(lhs_var->var, rhs_var->var)) {
+ return false;
+ }
+ if (lhs_var->iter_type != rhs_var->iter_type) {
+ return false;
+ }
+ }
+
+ if (is_scope_block) {
+ for (size_t i = 0; i < offset; ++i) {
+ extra_block_vars_.push_back(op->iter_vars[i]);
+ }
+ }
+
+ if (!is_scope_block) {
+ if (!CompareArray(op->writes, rhs->writes, &TensorizeComparator::CompareBufferRegion)) {
+ return false;
+ }
+ if (!CompareArray(op->reads, rhs->reads, &TensorizeComparator::CompareBufferRegion)) {
+ return false;
+ }
+ if (!CompareAnnotationMap(op->annotations, rhs->annotations)) {
+ return false;
+ }
+ if (!CompareArray(op->alloc_buffers, rhs->alloc_buffers, &TensorizeComparator::CompareBuffer)) {
+ return false;
+ }
+ }
+ is_scope_block = false;
+ return VisitStmt(op->body, rhs->body);
+}
+
+// Exprs
+#define TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(OpName) \
+ bool TensorizeComparator::VisitExpr_(const OpName* op, const PrimExpr& other) { \
+ const auto* rhs = other.as<OpName>(); \
+ return VisitExpr(op->a, rhs->a) && VisitExpr(op->b, rhs->b); \
+ }
+
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(AddNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(SubNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MulNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(DivNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(ModNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(EQNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(NENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(LTNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(LENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(GTNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(GENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(AndNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(OrNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MinNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MaxNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(FloorDivNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(FloorModNode);
+
+bool TensorizeComparator::VisitExpr_(const IntImmNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<IntImmNode>();
+ return CompareType(op->dtype, rhs->dtype) && op->value == rhs->value;
+}
+
+bool TensorizeComparator::VisitExpr_(const FloatImmNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<FloatImmNode>();
+ return CompareType(op->dtype, rhs->dtype) && op->value == rhs->value;
+}
+
+bool TensorizeComparator::VisitExpr_(const CastNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<CastNode>();
+ return CompareType(op->dtype, rhs->dtype) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitExpr_(const VarNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<VarNode>();
+ auto lhs = GetRef<Var>(op);
+ if (lhs.same_as(other)) return true;
+ if (!CompareType(op->dtype, rhs->dtype)) return false;
+ auto it = equal_map_.find(lhs);
+ return it != equal_map_.end() && it->second.same_as(other);
+}
+
+bool TensorizeComparator::VisitExpr_(const BufferLoadNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<BufferLoadNode>();
+ return CompareBufferAccess(op, rhs);
+}
+
+bool TensorizeComparator::DefEqual(const ObjectRef& lhs, const ObjectRef& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ if (lhs->type_index() != rhs->type_index()) return false;
+ auto it = equal_map_.find(lhs);
+ // If there is already a mapping
+ if (it != equal_map_.end()) return it->second.same_as(rhs);
+ equal_map_[lhs] = rhs;
+ return true;
+}
+
+bool TensorizeComparator::CompareAnnotation(const std::pair<String, ObjectRef>& lhs,
+ const std::pair<String, ObjectRef>& rhs) {
+ if (lhs.first != rhs.first) return false;
+ if (!lhs.second.same_as(rhs.second)) return false;
+ return VisitExpr(Downcast<PrimExpr>(lhs.second), Downcast<PrimExpr>(rhs.second));
+}
+
+bool TensorizeComparator::CompareAnnotationMap(const Map<String, ObjectRef>& lhs,
+ const Map<String, ObjectRef>& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ if (lhs.size() != rhs.size()) return false;
+
+ auto sort_map =
+ [](const Map<String, ObjectRef>& map) -> std::vector<std::pair<String, ObjectRef>> {
+ std::vector<std::pair<String, ObjectRef>> ret;
+ ret.reserve(map.size());
+ for (const auto& pair : map) {
+ ret.emplace_back(pair);
+ }
+ sort(ret.begin(), ret.end());
+ return ret;
+ };
+
+ auto lhs_array = sort_map(lhs), rhs_array = sort_map(rhs);
+
+ for (size_t i = 0; i < lhs.size(); ++i) {
+ if (!CompareAnnotation(lhs_array[i], rhs_array[i])) return false;
+ }
+ return true;
+}
+
+bool TensorizeComparator::CompareBuffer(const Buffer& lhs, const Buffer& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ // Remap both buffer itself and buffer data
+ // Skip buffer shape
+ bool equal = DefEqual(lhs, rhs) && DefEqual(lhs->data, rhs->data) &&
+ CompareType(lhs->dtype, rhs->dtype) && lhs.scope() == rhs.scope();
+ if (equal) {
+ rhs_buffer_map_[rhs] = lhs;
+ } else if (assert_mode_) {
+ LOG(FATAL) << "Buffers are not matching between:" << lhs << " and " << rhs;
+ }
+ return equal;
+}
+
+bool TensorizeComparator::CompareBufferRegion(const BufferRegion& lhs, const BufferRegion& rhs) {
+ // Only for block region declaration
+ if (!CompareBuffer(lhs->buffer, rhs->buffer)) return false;
+ // Number of indices in desc_block must be smaller than it in AST
+ if (rhs->region.size() > lhs->region.size()) return false;
+
+ std::vector<Range> lhs_region;
+ for (const auto& range : lhs->region) {
+ lhs_region.push_back(Range::FromMinExtent(range->min, range->extent));
+ }
+ size_t offset = lhs_region.size() - rhs->region.size();
+ // initialize buffer indices
+ bool need_update = false;
+ if (!buffer_indices_.count(lhs->buffer)) {
+ need_update = true;
+ buffer_indices_[lhs->buffer] = std::vector<PrimExpr>();
+ } else {
+ if (offset != buffer_indices_[lhs->buffer].size()) return false;
+ }
+ std::vector<PrimExpr>& indices = buffer_indices_[lhs->buffer];
+ for (size_t i = 0; i < offset; ++i) {
+ const Range& range = lhs_region[i];
+ // High-dim region must be element-wise
+ if (!is_one(range->extent)) return false;
+ if (need_update) {
+ indices.push_back(range->min);
+ } else {
+ // The order matters since we only map inner block_var to outside block_var
+ if (!VisitExpr(range->min, indices[i])) return false;
+ }
+ }
Review comment:
style:
i could see two different code paths coupled together controlled by `need_update`, which is a bit vague logic. Let's do this instead:
```
if (buffer_indices_.count(lhs->buffer)) {
for (size_t i = 0; i < offset; ++i) { /* do one thing */ }
} else {
for (size_t i = 0; i < offset; ++i) { /* do the other thing */ }
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780605351
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
+using StmtComparator = StmtFunctor<bool(const Stmt& n, const Stmt& other)>;
+
+/*! \brief Deep comparison to check if two IR ASTs are equivalent */
+class TensorizeComparator : public ExprComparator, public StmtComparator {
+ public:
+ /*!
+ * \brief Constructor of TensorizeComparator
+ * \param assert_mode Whether to raise an error if the two IR ASTs do not match.
+ */
+ explicit TensorizeComparator(bool assert_mode = true) : assert_mode_(assert_mode) {}
+
+ // Map from rhs buffer to lhs buffer
+ std::unordered_map<Buffer, Buffer, ObjectHash, ObjectEqual> rhs_buffer_map_;
+ // Buffer indices mapping
+ std::unordered_map<Buffer, std::vector<PrimExpr>, ObjectPtrHash, ObjectPtrEqual> buffer_indices_;
+ std::vector<IterVar> extra_block_vars_;
+ // variable remap if any
+ std::unordered_map<ObjectRef, ObjectRef, ObjectPtrHash, ObjectPtrEqual> equal_map_;
Review comment:
Let's move the fields to the end of the class, and document them properly
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780589224
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
Review comment:
Let's write some informative error message here given it's user-facing CHECK
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [WIP][TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r789085645
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
+ const Block& desc_block = desc_block_realize->block;
+ const auto* impl_block_realize =
+ Downcast<BlockRealize>(intrinsic->implementation->body)->block->body.as<BlockRealizeNode>();
+ Block impl_block = impl_block_realize->block;
+
+ // Step 1: Blockize the subtree rooted at the given loop
+ const StmtSRef& block_sref = Blockize(self, loop_sref);
+ const BlockRealize& block_realize = GetBlockRealize(self, block_sref);
+
+ // Step 2: Compare the block with the description of the tensor intrinsic, find the correspondence
+ // between buffers in the block and the description.
+ TensorizeComparator comparator(/*assert_mode=*/true);
+ comparator.VisitStmt(block_realize, GetRef<Stmt>(desc_block_realize));
+
+ // Step 3: Find the correspondence between buffers in the current AST and the implementation of
+ // the tensor intrinsic
+
+ // Step 3.1: Map from intrinsic func buffer to description func buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> intrin_buffer_map;
+ RemapTensorIntrinBuffers(intrinsic, &intrin_buffer_map);
+ // Step 3.2: Map form intrinsic func buffer to current AST buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> buffer_map;
+ for (const auto& pair : intrin_buffer_map) {
+ auto it = comparator.rhs_buffer_map_.find(pair.second);
+ ICHECK(it != comparator.rhs_buffer_map_.end());
+ buffer_map[pair.first] = it->second;
+ }
+
+ // Step 4: Create MatchBufferRegion for the params of the implementation function of the tensor
+ // intrin to make them subregions of the buffer in the original IR.
+ std::unordered_map<Buffer, Array<Range>, ObjectPtrHash, ObjectPtrEqual> buffer_region_map;
+ for (const auto& read : impl_block->reads) {
+ buffer_region_map.emplace(read->buffer, read->region);
+ }
+ for (const auto& write : impl_block->writes) {
+ buffer_region_map.emplace(write->buffer, write->region);
+ }
+ Array<MatchBufferRegion> match_buffer_regions;
+ for (size_t i = 0; i < intrinsic->implementation->params.size(); ++i) {
+ const auto& param = intrinsic->implementation->params[i];
+ const auto& buffer = intrinsic->implementation->buffer_map.at(param);
+ const auto& source = buffer_map.at(buffer);
+ Region region = buffer_region_map.at(buffer);
+ auto extra_indices = comparator.buffer_indices_.at(source);
+ std::vector<Range> extra_buffer_ranges;
+ std::transform(extra_indices.begin(), extra_indices.end(),
+ std::back_inserter(extra_buffer_ranges),
+ [](const PrimExpr& index) { return Range::FromMinExtent(index, 1); });
+ region.insert(region.begin(), extra_buffer_ranges.begin(), extra_buffer_ranges.end());
+ match_buffer_regions.push_back(MatchBufferRegion(buffer, BufferRegion(source, region)));
+ }
+
+ // Step 5: Replace the subtree in the original IR with the tensor intrin implementation.
+ ObjectPtr<BlockNode> new_block_ptr = make_object<BlockNode>(*block_realize->block.get());
+ new_block_ptr->body = impl_block->body;
+ ICHECK(new_block_ptr->match_buffers.empty());
+ new_block_ptr->match_buffers = std::move(match_buffer_regions);
+ Block new_block(new_block_ptr);
+
+ // Substitution map for the intrin block to get the correct bindings.
+ Map<Var, PrimExpr> bv_map;
+ for (size_t i = 0; i < desc_block->iter_vars.size(); ++i) {
+ auto it = comparator.equal_map_.find(desc_block->iter_vars[i]->var);
+ if (it != comparator.equal_map_.end()) {
+ bv_map.Set(impl_block->iter_vars[i]->var, Downcast<PrimExpr>(it->second));
+ } else {
+ bv_map.Set(impl_block->iter_vars[i]->var, Integer(0));
+ }
+ }
+ new_block = Downcast<Block>(SubstituteInScope(new_block, bv_map));
+ self->Replace(block_sref, new_block, {{block_realize->block, new_block}});
Review comment:
Thanks for the clarification! Let's focus on landing this PR first and do misc improvement in the future :-)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785359931
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
+ const Block& desc_block = desc_block_realize->block;
+ const auto* impl_block_realize =
+ Downcast<BlockRealize>(intrinsic->implementation->body)->block->body.as<BlockRealizeNode>();
+ Block impl_block = impl_block_realize->block;
+
+ // Step 1: Blockize the subtree rooted at the given loop
+ const StmtSRef& block_sref = Blockize(self, loop_sref);
+ const BlockRealize& block_realize = GetBlockRealize(self, block_sref);
+
+ // Step 2: Compare the block with the description of the tensor intrinsic, find the correspondence
+ // between buffers in the block and the description.
+ TensorizeComparator comparator(/*assert_mode=*/true);
+ comparator.VisitStmt(block_realize, GetRef<Stmt>(desc_block_realize));
+
+ // Step 3: Find the correspondence between buffers in the current AST and the implementation of
+ // the tensor intrinsic
+
+ // Step 3.1: Map from intrinsic func buffer to description func buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> intrin_buffer_map;
+ RemapTensorIntrinBuffers(intrinsic, &intrin_buffer_map);
+ // Step 3.2: Map form intrinsic func buffer to current AST buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> buffer_map;
+ for (const auto& pair : intrin_buffer_map) {
+ auto it = comparator.rhs_buffer_map_.find(pair.second);
+ ICHECK(it != comparator.rhs_buffer_map_.end());
+ buffer_map[pair.first] = it->second;
+ }
+
+ // Step 4: Create MatchBufferRegion for the params of the implementation function of the tensor
+ // intrin to make them subregions of the buffer in the original IR.
+ std::unordered_map<Buffer, Array<Range>, ObjectPtrHash, ObjectPtrEqual> buffer_region_map;
+ for (const auto& read : impl_block->reads) {
+ buffer_region_map.emplace(read->buffer, read->region);
+ }
+ for (const auto& write : impl_block->writes) {
+ buffer_region_map.emplace(write->buffer, write->region);
+ }
+ Array<MatchBufferRegion> match_buffer_regions;
+ for (size_t i = 0; i < intrinsic->implementation->params.size(); ++i) {
+ const auto& param = intrinsic->implementation->params[i];
+ const auto& buffer = intrinsic->implementation->buffer_map.at(param);
+ const auto& source = buffer_map.at(buffer);
+ Region region = buffer_region_map.at(buffer);
+ auto extra_indices = comparator.buffer_indices_.at(source);
+ std::vector<Range> extra_buffer_ranges;
+ std::transform(extra_indices.begin(), extra_indices.end(),
+ std::back_inserter(extra_buffer_ranges),
+ [](const PrimExpr& index) { return Range::FromMinExtent(index, 1); });
+ region.insert(region.begin(), extra_buffer_ranges.begin(), extra_buffer_ranges.end());
+ match_buffer_regions.push_back(MatchBufferRegion(buffer, BufferRegion(source, region)));
+ }
+
+ // Step 5: Replace the subtree in the original IR with the tensor intrin implementation.
+ ObjectPtr<BlockNode> new_block_ptr = make_object<BlockNode>(*block_realize->block.get());
+ new_block_ptr->body = impl_block->body;
+ ICHECK(new_block_ptr->match_buffers.empty());
+ new_block_ptr->match_buffers = std::move(match_buffer_regions);
+ Block new_block(new_block_ptr);
+
+ // Substitution map for the intrin block to get the correct bindings.
+ Map<Var, PrimExpr> bv_map;
+ for (size_t i = 0; i < desc_block->iter_vars.size(); ++i) {
+ auto it = comparator.equal_map_.find(desc_block->iter_vars[i]->var);
+ if (it != comparator.equal_map_.end()) {
+ bv_map.Set(impl_block->iter_vars[i]->var, Downcast<PrimExpr>(it->second));
+ } else {
+ bv_map.Set(impl_block->iter_vars[i]->var, Integer(0));
+ }
+ }
+ new_block = Downcast<Block>(SubstituteInScope(new_block, bv_map));
+ self->Replace(block_sref, new_block, {{block_realize->block, new_block}});
Review comment:
I know it's hard, but am just curious if we could gather all the mutation logic to one place in the end, so that once there is unsatisfied condition preventing full tensorization, the IR isn't mutated by `Blockize`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785358932
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
+ const Block& desc_block = desc_block_realize->block;
+ const auto* impl_block_realize =
+ Downcast<BlockRealize>(intrinsic->implementation->body)->block->body.as<BlockRealizeNode>();
+ Block impl_block = impl_block_realize->block;
+
+ // Step 1: Blockize the subtree rooted at the given loop
+ const StmtSRef& block_sref = Blockize(self, loop_sref);
+ const BlockRealize& block_realize = GetBlockRealize(self, block_sref);
Review comment:
No need to use reference
```suggestion
StmtSRef block_sref = Blockize(self, loop_sref);
BlockRealize block_realize = GetBlockRealize(self, block_sref);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] MasterJH5574 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
MasterJH5574 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r791521474
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,700 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../ir_comparator.h"
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constraint on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ Array<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+
+ auto make_uses_var = [](const Array<Var>& vars) -> std::function<bool(const PrimExpr& expr)> {
+ std::unordered_set<const VarNode*> var_set;
+ var_set.reserve(vars.size());
+ for (const Var& var : vars) {
+ var_set.insert(var.get());
+ }
+ return [var_set = std::move(var_set)](const PrimExpr& expr) -> bool {
+ return UsesVar(expr, [&var_set](const VarNode* var) {
+ return var_set.count(var); //
+ });
+ };
+ };
+ auto use_outer_loop_vars = make_uses_var(outer_iters);
+ auto use_inner_loop_vars = make_uses_var(inner_iters);
+ arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
Review comment:
I guess we should use `use_outer_loop_vars` and `use_inner_loop_vars` on line 99 and 101 :eyes:?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] MasterJH5574 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
MasterJH5574 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r791521474
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,700 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../ir_comparator.h"
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constraint on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ Array<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+
+ auto make_uses_var = [](const Array<Var>& vars) -> std::function<bool(const PrimExpr& expr)> {
+ std::unordered_set<const VarNode*> var_set;
+ var_set.reserve(vars.size());
+ for (const Var& var : vars) {
+ var_set.insert(var.get());
+ }
+ return [var_set = std::move(var_set)](const PrimExpr& expr) -> bool {
+ return UsesVar(expr, [&var_set](const VarNode* var) {
+ return var_set.count(var); //
+ });
+ };
+ };
+ auto use_outer_loop_vars = make_uses_var(outer_iters);
+ auto use_inner_loop_vars = make_uses_var(inner_iters);
+ arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
Review comment:
I guess we should use `use_outer_loop_vars` and `use_inner_loop_vars` on line 99 and 101 :eyes:?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r784359703
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
Review comment:
nit: to make the logic look more natural
```suggestion
[tgt_var = iter_var->var.get()](const VarNode* var) { return var == tgt_var; })) {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r784366063
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
Review comment:
nit: to better structure the logic
```suggestion
IterVar new_iter_var = init_block_iters[i];
Var old_var = new_iter_var->var;
Var new_var = old_var.copy_with_suffix("_init");
new_iter_var.CopyOnWrite()->var = new_var;
subst_map.Set(old_var, new_var);
init_block_iters.Set(i, std::move(new_iter_var));
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780727564
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
Review comment:
Looks like it's a pair of zeros. What is this used for? Why are we using `Bool(true)` to indicate the extent BTW? Isn't it just `unit_iter_mark`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780718868
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
+ }
+ };
+ std::tie(lhs_expr, lhs_iter) = detect(op->iter_values[i + offset]);
+ std::tie(rhs_expr, rhs_iter) = detect(rhs->iter_values[i]);
+ CHECK((lhs_iter && rhs_iter) || (!lhs_iter && !rhs_iter)) << "Incompatible binding";
+ if (lhs_iter) VisitExpr(lhs_iter.value(), rhs_iter.value());
+ if (is_zero(rhs_expr)) {
+ CHECK(is_zero(lhs_expr)) << "Incompatible binding";
+ } else {
+ const auto* bv = rhs_expr.as<VarNode>();
+ if (!bv) {
+ VisitExpr(lhs_expr, rhs_expr);
+ } else {
+ auto it = equal_map_.find(GetRef<Var>(bv));
+ if (it == equal_map_.end()) {
+ equal_map_[GetRef<Var>(bv)] = lhs_expr;
+ } else {
+ CHECK(it->second->IsInstance<PrimExprNode>());
+ VisitExpr(lhs_expr, Downcast<PrimExpr>(it->second));
+ }
+ }
+ }
+ }
+ } else {
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ if (!VisitExpr(op->iter_values[i + offset], rhs->iter_values[i])) return false;
+ }
+ const Block& block = op->block;
+ for (size_t i = 0; i < offset; ++i) {
+ Var block_var = Downcast<Var>(op->iter_values[i]);
+ auto it = equal_map_.find(block_var);
+ equal_map_[block->iter_vars[i]->var] = (it == equal_map_.end() ? block_var : it->second);
+ }
+ }
+ }
+
+ return VisitExpr(op->predicate, rhs->predicate) && VisitStmt(op->block, rhs->block);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockNode>();
+ // Check block equality.
+ // All iter vars and buffer regions including the order shoudl match.
+ // When checking iter vars, DefEqual is used to remap variables.
+ // Only the inner most several axis are compared. Other iter vars are added to extra_block_vars.
+ if (op->iter_vars.size() < rhs->iter_vars.size()) return false;
+
+ size_t offset = op->iter_vars.size() - rhs->iter_vars.size();
+ for (size_t i = 0; i < rhs->iter_vars.size(); ++i) {
+ auto lhs_var = op->iter_vars[i + offset], rhs_var = rhs->iter_vars[i];
Review comment:
nit:
```suggestion
IterVar lhs_var = op->iter_vars[i + offset];
IterVar rhs_var = rhs->iter_vars[i];
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785346258
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
Review comment:
```suggestion
ICHECK_EQ(iter_vars.size() + 1, division.size());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785354410
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
Review comment:
No need to refer to this class, because it's returned by this method
```suggestion
StmtSRef outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785346551
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
Review comment:
1. Use `IterSumExprNode` instead to be more accurate.
2. `TVM_TYPE_AS` could help with type checking BTW.
3. Use some intermediate variables to make the logic clearer.
```suggestion
arith::IterMark outer_mark = division[i][0];
arith::IterMark inner_mark = division[i][1];
const auto* outer_binding = TVM_TYPE_AS(outer_binding, outer_mark->source, arith::IterSumExprNode);
const auto* inner_binding = TVM_TYPE_AS(inner_binding, inner_mark->source, arith::IterSumExprNode);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785359380
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
+ const Block& desc_block = desc_block_realize->block;
+ const auto* impl_block_realize =
+ Downcast<BlockRealize>(intrinsic->implementation->body)->block->body.as<BlockRealizeNode>();
+ Block impl_block = impl_block_realize->block;
+
+ // Step 1: Blockize the subtree rooted at the given loop
+ const StmtSRef& block_sref = Blockize(self, loop_sref);
+ const BlockRealize& block_realize = GetBlockRealize(self, block_sref);
+
+ // Step 2: Compare the block with the description of the tensor intrinsic, find the correspondence
+ // between buffers in the block and the description.
+ TensorizeComparator comparator(/*assert_mode=*/true);
+ comparator.VisitStmt(block_realize, GetRef<Stmt>(desc_block_realize));
+
+ // Step 3: Find the correspondence between buffers in the current AST and the implementation of
+ // the tensor intrinsic
+
+ // Step 3.1: Map from intrinsic func buffer to description func buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> intrin_buffer_map;
+ RemapTensorIntrinBuffers(intrinsic, &intrin_buffer_map);
+ // Step 3.2: Map form intrinsic func buffer to current AST buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> buffer_map;
+ for (const auto& pair : intrin_buffer_map) {
+ auto it = comparator.rhs_buffer_map_.find(pair.second);
+ ICHECK(it != comparator.rhs_buffer_map_.end());
+ buffer_map[pair.first] = it->second;
+ }
+
+ // Step 4: Create MatchBufferRegion for the params of the implementation function of the tensor
+ // intrin to make them subregions of the buffer in the original IR.
+ std::unordered_map<Buffer, Array<Range>, ObjectPtrHash, ObjectPtrEqual> buffer_region_map;
+ for (const auto& read : impl_block->reads) {
Review comment:
```suggestion
for (const BufferRegion& read : impl_block->reads) {
```
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
+ const Block& desc_block = desc_block_realize->block;
+ const auto* impl_block_realize =
+ Downcast<BlockRealize>(intrinsic->implementation->body)->block->body.as<BlockRealizeNode>();
+ Block impl_block = impl_block_realize->block;
+
+ // Step 1: Blockize the subtree rooted at the given loop
+ const StmtSRef& block_sref = Blockize(self, loop_sref);
+ const BlockRealize& block_realize = GetBlockRealize(self, block_sref);
+
+ // Step 2: Compare the block with the description of the tensor intrinsic, find the correspondence
+ // between buffers in the block and the description.
+ TensorizeComparator comparator(/*assert_mode=*/true);
+ comparator.VisitStmt(block_realize, GetRef<Stmt>(desc_block_realize));
+
+ // Step 3: Find the correspondence between buffers in the current AST and the implementation of
+ // the tensor intrinsic
+
+ // Step 3.1: Map from intrinsic func buffer to description func buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> intrin_buffer_map;
+ RemapTensorIntrinBuffers(intrinsic, &intrin_buffer_map);
+ // Step 3.2: Map form intrinsic func buffer to current AST buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> buffer_map;
+ for (const auto& pair : intrin_buffer_map) {
+ auto it = comparator.rhs_buffer_map_.find(pair.second);
+ ICHECK(it != comparator.rhs_buffer_map_.end());
+ buffer_map[pair.first] = it->second;
+ }
+
+ // Step 4: Create MatchBufferRegion for the params of the implementation function of the tensor
+ // intrin to make them subregions of the buffer in the original IR.
+ std::unordered_map<Buffer, Array<Range>, ObjectPtrHash, ObjectPtrEqual> buffer_region_map;
+ for (const auto& read : impl_block->reads) {
+ buffer_region_map.emplace(read->buffer, read->region);
+ }
+ for (const auto& write : impl_block->writes) {
Review comment:
```suggestion
for (const BufferRegion& write : impl_block->writes) {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r784361131
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
Review comment:
nit
```suggestion
[tgt_var = inner_loop->loop_var.get()](const VarNode* var) { return var == tgt_var; })) {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780589809
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
+
+ // check both functions' bodies are directly block
+ const auto* desc_realize =
+ Downcast<BlockRealize>(desc_func->body)->block->body.as<BlockRealizeNode>();
+ const auto* intrin_realize =
+ Downcast<BlockRealize>(intrin_func->body)->block->body.as<BlockRealizeNode>();
+ CHECK(desc_realize != nullptr) << "description function's body expect a directly block";
+ CHECK(intrin_realize != nullptr) << "intrinsic function's body expect a directly block";
+
+ const Block& desc_block = desc_realize->block;
+ const Block& intrin_block = intrin_realize->block;
+
+ // check block var number and iter type
+ CHECK_EQ(desc_block->iter_vars.size(), intrin_block->iter_vars.size())
+ << "Two blocks should have the same number of block vars";
+ for (size_t i = 0; i < desc_block->iter_vars.size(); i++) {
+ const IterVar& desc_var = desc_block->iter_vars[i];
+ const IterVar& intrin_var = intrin_block->iter_vars[i];
+ CHECK(desc_var->iter_type == intrin_var->iter_type)
+ << "Block iter_type mismatch between " << desc_var->iter_type << " and "
+ << intrin_var->iter_type;
+ }
+
+ auto n = make_object<TensorIntrinNode>();
+ n->description = std::move(desc_func);
+ n->implementation = std::move(intrin_func);
+ data_ = std::move(n);
+}
+
+TensorIntrin TensorIntrin::Register(String name, PrimFunc desc_func, PrimFunc intrin_func) {
+ TensorIntrinManager* manager = TensorIntrinManager::Global();
+ ICHECK_EQ(manager->reg.count(name), 0)
+ << "ValueError: TensorIntrin '" << name << "' has already been registered";
+ TensorIntrin intrin(desc_func, intrin_func);
+ manager->reg.Set(name, intrin);
+ return intrin;
+}
+
+TensorIntrin TensorIntrin::Get(String name) {
+ const TensorIntrinManager* manager = TensorIntrinManager::Global();
+ ICHECK_EQ(manager->reg.count(name), 1)
+ << "ValueError: TensorIntrin '" << name << "' is not registered";
+ return manager->reg.at(name);
Review comment:
Use manager->reg.find to save one lookup :-)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780589562
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
+
+ // check both functions' bodies are directly block
+ const auto* desc_realize =
+ Downcast<BlockRealize>(desc_func->body)->block->body.as<BlockRealizeNode>();
+ const auto* intrin_realize =
+ Downcast<BlockRealize>(intrin_func->body)->block->body.as<BlockRealizeNode>();
+ CHECK(desc_realize != nullptr) << "description function's body expect a directly block";
+ CHECK(intrin_realize != nullptr) << "intrinsic function's body expect a directly block";
+
+ const Block& desc_block = desc_realize->block;
+ const Block& intrin_block = intrin_realize->block;
+
+ // check block var number and iter type
+ CHECK_EQ(desc_block->iter_vars.size(), intrin_block->iter_vars.size())
+ << "Two blocks should have the same number of block vars";
+ for (size_t i = 0; i < desc_block->iter_vars.size(); i++) {
+ const IterVar& desc_var = desc_block->iter_vars[i];
+ const IterVar& intrin_var = intrin_block->iter_vars[i];
+ CHECK(desc_var->iter_type == intrin_var->iter_type)
+ << "Block iter_type mismatch between " << desc_var->iter_type << " and "
+ << intrin_var->iter_type;
+ }
+
+ auto n = make_object<TensorIntrinNode>();
Review comment:
nit
```suggestion
ObjectPtr<TensorIntrinNode> n = make_object<TensorIntrinNode>();
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780723478
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
+ }
+ };
+ std::tie(lhs_expr, lhs_iter) = detect(op->iter_values[i + offset]);
+ std::tie(rhs_expr, rhs_iter) = detect(rhs->iter_values[i]);
+ CHECK((lhs_iter && rhs_iter) || (!lhs_iter && !rhs_iter)) << "Incompatible binding";
+ if (lhs_iter) VisitExpr(lhs_iter.value(), rhs_iter.value());
+ if (is_zero(rhs_expr)) {
+ CHECK(is_zero(lhs_expr)) << "Incompatible binding";
+ } else {
+ const auto* bv = rhs_expr.as<VarNode>();
+ if (!bv) {
+ VisitExpr(lhs_expr, rhs_expr);
+ } else {
+ auto it = equal_map_.find(GetRef<Var>(bv));
+ if (it == equal_map_.end()) {
+ equal_map_[GetRef<Var>(bv)] = lhs_expr;
+ } else {
+ CHECK(it->second->IsInstance<PrimExprNode>());
+ VisitExpr(lhs_expr, Downcast<PrimExpr>(it->second));
+ }
+ }
+ }
+ }
+ } else {
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ if (!VisitExpr(op->iter_values[i + offset], rhs->iter_values[i])) return false;
+ }
+ const Block& block = op->block;
+ for (size_t i = 0; i < offset; ++i) {
+ Var block_var = Downcast<Var>(op->iter_values[i]);
+ auto it = equal_map_.find(block_var);
+ equal_map_[block->iter_vars[i]->var] = (it == equal_map_.end() ? block_var : it->second);
+ }
+ }
+ }
+
+ return VisitExpr(op->predicate, rhs->predicate) && VisitStmt(op->block, rhs->block);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockNode>();
+ // Check block equality.
+ // All iter vars and buffer regions including the order shoudl match.
+ // When checking iter vars, DefEqual is used to remap variables.
+ // Only the inner most several axis are compared. Other iter vars are added to extra_block_vars.
+ if (op->iter_vars.size() < rhs->iter_vars.size()) return false;
+
+ size_t offset = op->iter_vars.size() - rhs->iter_vars.size();
+ for (size_t i = 0; i < rhs->iter_vars.size(); ++i) {
+ auto lhs_var = op->iter_vars[i + offset], rhs_var = rhs->iter_vars[i];
+ // Skip iter dom
+ if (!DefEqual(lhs_var->var, rhs_var->var)) {
+ return false;
+ }
+ if (lhs_var->iter_type != rhs_var->iter_type) {
+ return false;
+ }
+ }
+
+ if (is_scope_block) {
+ for (size_t i = 0; i < offset; ++i) {
+ extra_block_vars_.push_back(op->iter_vars[i]);
+ }
+ }
+
+ if (!is_scope_block) {
+ if (!CompareArray(op->writes, rhs->writes, &TensorizeComparator::CompareBufferRegion)) {
+ return false;
+ }
+ if (!CompareArray(op->reads, rhs->reads, &TensorizeComparator::CompareBufferRegion)) {
+ return false;
+ }
+ if (!CompareAnnotationMap(op->annotations, rhs->annotations)) {
+ return false;
+ }
+ if (!CompareArray(op->alloc_buffers, rhs->alloc_buffers, &TensorizeComparator::CompareBuffer)) {
+ return false;
+ }
+ }
+ is_scope_block = false;
+ return VisitStmt(op->body, rhs->body);
+}
+
+// Exprs
+#define TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(OpName) \
+ bool TensorizeComparator::VisitExpr_(const OpName* op, const PrimExpr& other) { \
+ const auto* rhs = other.as<OpName>(); \
+ return VisitExpr(op->a, rhs->a) && VisitExpr(op->b, rhs->b); \
+ }
+
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(AddNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(SubNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MulNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(DivNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(ModNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(EQNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(NENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(LTNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(LENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(GTNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(GENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(AndNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(OrNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MinNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MaxNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(FloorDivNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(FloorModNode);
+
+bool TensorizeComparator::VisitExpr_(const IntImmNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<IntImmNode>();
+ return CompareType(op->dtype, rhs->dtype) && op->value == rhs->value;
+}
+
+bool TensorizeComparator::VisitExpr_(const FloatImmNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<FloatImmNode>();
+ return CompareType(op->dtype, rhs->dtype) && op->value == rhs->value;
+}
+
+bool TensorizeComparator::VisitExpr_(const CastNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<CastNode>();
+ return CompareType(op->dtype, rhs->dtype) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitExpr_(const VarNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<VarNode>();
+ auto lhs = GetRef<Var>(op);
+ if (lhs.same_as(other)) return true;
+ if (!CompareType(op->dtype, rhs->dtype)) return false;
+ auto it = equal_map_.find(lhs);
+ return it != equal_map_.end() && it->second.same_as(other);
+}
+
+bool TensorizeComparator::VisitExpr_(const BufferLoadNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<BufferLoadNode>();
+ return CompareBufferAccess(op, rhs);
+}
+
+bool TensorizeComparator::DefEqual(const ObjectRef& lhs, const ObjectRef& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ if (lhs->type_index() != rhs->type_index()) return false;
+ auto it = equal_map_.find(lhs);
+ // If there is already a mapping
+ if (it != equal_map_.end()) return it->second.same_as(rhs);
+ equal_map_[lhs] = rhs;
+ return true;
+}
+
+bool TensorizeComparator::CompareAnnotation(const std::pair<String, ObjectRef>& lhs,
+ const std::pair<String, ObjectRef>& rhs) {
+ if (lhs.first != rhs.first) return false;
+ if (!lhs.second.same_as(rhs.second)) return false;
+ return VisitExpr(Downcast<PrimExpr>(lhs.second), Downcast<PrimExpr>(rhs.second));
+}
+
+bool TensorizeComparator::CompareAnnotationMap(const Map<String, ObjectRef>& lhs,
+ const Map<String, ObjectRef>& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ if (lhs.size() != rhs.size()) return false;
+
+ auto sort_map =
+ [](const Map<String, ObjectRef>& map) -> std::vector<std::pair<String, ObjectRef>> {
+ std::vector<std::pair<String, ObjectRef>> ret;
+ ret.reserve(map.size());
+ for (const auto& pair : map) {
+ ret.emplace_back(pair);
+ }
+ sort(ret.begin(), ret.end());
+ return ret;
+ };
+
+ auto lhs_array = sort_map(lhs), rhs_array = sort_map(rhs);
+
+ for (size_t i = 0; i < lhs.size(); ++i) {
+ if (!CompareAnnotation(lhs_array[i], rhs_array[i])) return false;
+ }
+ return true;
+}
+
+bool TensorizeComparator::CompareBuffer(const Buffer& lhs, const Buffer& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ // Remap both buffer itself and buffer data
+ // Skip buffer shape
+ bool equal = DefEqual(lhs, rhs) && DefEqual(lhs->data, rhs->data) &&
+ CompareType(lhs->dtype, rhs->dtype) && lhs.scope() == rhs.scope();
+ if (equal) {
+ rhs_buffer_map_[rhs] = lhs;
+ } else if (assert_mode_) {
+ LOG(FATAL) << "Buffers are not matching between:" << lhs << " and " << rhs;
+ }
+ return equal;
+}
+
+bool TensorizeComparator::CompareBufferRegion(const BufferRegion& lhs, const BufferRegion& rhs) {
+ // Only for block region declaration
+ if (!CompareBuffer(lhs->buffer, rhs->buffer)) return false;
+ // Number of indices in desc_block must be smaller than it in AST
+ if (rhs->region.size() > lhs->region.size()) return false;
+
+ std::vector<Range> lhs_region;
+ for (const auto& range : lhs->region) {
+ lhs_region.push_back(Range::FromMinExtent(range->min, range->extent));
+ }
+ size_t offset = lhs_region.size() - rhs->region.size();
+ // initialize buffer indices
+ bool need_update = false;
+ if (!buffer_indices_.count(lhs->buffer)) {
+ need_update = true;
+ buffer_indices_[lhs->buffer] = std::vector<PrimExpr>();
+ } else {
+ if (offset != buffer_indices_[lhs->buffer].size()) return false;
+ }
Review comment:
nit: style
```suggestion
} else if (offset != buffer_indices_[lhs->buffer].size()) {
return false;
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780718620
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
+ }
+ };
+ std::tie(lhs_expr, lhs_iter) = detect(op->iter_values[i + offset]);
+ std::tie(rhs_expr, rhs_iter) = detect(rhs->iter_values[i]);
+ CHECK((lhs_iter && rhs_iter) || (!lhs_iter && !rhs_iter)) << "Incompatible binding";
+ if (lhs_iter) VisitExpr(lhs_iter.value(), rhs_iter.value());
+ if (is_zero(rhs_expr)) {
+ CHECK(is_zero(lhs_expr)) << "Incompatible binding";
+ } else {
+ const auto* bv = rhs_expr.as<VarNode>();
+ if (!bv) {
+ VisitExpr(lhs_expr, rhs_expr);
+ } else {
+ auto it = equal_map_.find(GetRef<Var>(bv));
+ if (it == equal_map_.end()) {
+ equal_map_[GetRef<Var>(bv)] = lhs_expr;
+ } else {
+ CHECK(it->second->IsInstance<PrimExprNode>());
+ VisitExpr(lhs_expr, Downcast<PrimExpr>(it->second));
+ }
+ }
+ }
+ }
+ } else {
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ if (!VisitExpr(op->iter_values[i + offset], rhs->iter_values[i])) return false;
+ }
+ const Block& block = op->block;
+ for (size_t i = 0; i < offset; ++i) {
+ Var block_var = Downcast<Var>(op->iter_values[i]);
+ auto it = equal_map_.find(block_var);
+ equal_map_[block->iter_vars[i]->var] = (it == equal_map_.end() ? block_var : it->second);
+ }
+ }
+ }
+
+ return VisitExpr(op->predicate, rhs->predicate) && VisitStmt(op->block, rhs->block);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockNode>();
+ // Check block equality.
+ // All iter vars and buffer regions including the order shoudl match.
Review comment:
nit:
```suggestion
// All iter vars and buffer regions including the order should match.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780718904
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
+ }
+ };
+ std::tie(lhs_expr, lhs_iter) = detect(op->iter_values[i + offset]);
+ std::tie(rhs_expr, rhs_iter) = detect(rhs->iter_values[i]);
+ CHECK((lhs_iter && rhs_iter) || (!lhs_iter && !rhs_iter)) << "Incompatible binding";
+ if (lhs_iter) VisitExpr(lhs_iter.value(), rhs_iter.value());
+ if (is_zero(rhs_expr)) {
+ CHECK(is_zero(lhs_expr)) << "Incompatible binding";
+ } else {
+ const auto* bv = rhs_expr.as<VarNode>();
+ if (!bv) {
+ VisitExpr(lhs_expr, rhs_expr);
+ } else {
+ auto it = equal_map_.find(GetRef<Var>(bv));
+ if (it == equal_map_.end()) {
+ equal_map_[GetRef<Var>(bv)] = lhs_expr;
+ } else {
+ CHECK(it->second->IsInstance<PrimExprNode>());
+ VisitExpr(lhs_expr, Downcast<PrimExpr>(it->second));
+ }
+ }
+ }
+ }
+ } else {
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ if (!VisitExpr(op->iter_values[i + offset], rhs->iter_values[i])) return false;
+ }
+ const Block& block = op->block;
+ for (size_t i = 0; i < offset; ++i) {
+ Var block_var = Downcast<Var>(op->iter_values[i]);
+ auto it = equal_map_.find(block_var);
+ equal_map_[block->iter_vars[i]->var] = (it == equal_map_.end() ? block_var : it->second);
+ }
+ }
+ }
+
+ return VisitExpr(op->predicate, rhs->predicate) && VisitStmt(op->block, rhs->block);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockNode>();
+ // Check block equality.
+ // All iter vars and buffer regions including the order shoudl match.
+ // When checking iter vars, DefEqual is used to remap variables.
+ // Only the inner most several axis are compared. Other iter vars are added to extra_block_vars.
+ if (op->iter_vars.size() < rhs->iter_vars.size()) return false;
+
+ size_t offset = op->iter_vars.size() - rhs->iter_vars.size();
+ for (size_t i = 0; i < rhs->iter_vars.size(); ++i) {
+ auto lhs_var = op->iter_vars[i + offset], rhs_var = rhs->iter_vars[i];
+ // Skip iter dom
+ if (!DefEqual(lhs_var->var, rhs_var->var)) {
+ return false;
+ }
+ if (lhs_var->iter_type != rhs_var->iter_type) {
+ return false;
+ }
+ }
+
+ if (is_scope_block) {
+ for (size_t i = 0; i < offset; ++i) {
+ extra_block_vars_.push_back(op->iter_vars[i]);
+ }
+ }
+
+ if (!is_scope_block) {
Review comment:
```suggestion
} else {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [WIP][TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r788408997
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
+ const Block& desc_block = desc_block_realize->block;
+ const auto* impl_block_realize =
+ Downcast<BlockRealize>(intrinsic->implementation->body)->block->body.as<BlockRealizeNode>();
+ Block impl_block = impl_block_realize->block;
+
+ // Step 1: Blockize the subtree rooted at the given loop
+ const StmtSRef& block_sref = Blockize(self, loop_sref);
+ const BlockRealize& block_realize = GetBlockRealize(self, block_sref);
+
+ // Step 2: Compare the block with the description of the tensor intrinsic, find the correspondence
+ // between buffers in the block and the description.
+ TensorizeComparator comparator(/*assert_mode=*/true);
+ comparator.VisitStmt(block_realize, GetRef<Stmt>(desc_block_realize));
+
+ // Step 3: Find the correspondence between buffers in the current AST and the implementation of
+ // the tensor intrinsic
+
+ // Step 3.1: Map from intrinsic func buffer to description func buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> intrin_buffer_map;
+ RemapTensorIntrinBuffers(intrinsic, &intrin_buffer_map);
+ // Step 3.2: Map form intrinsic func buffer to current AST buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> buffer_map;
+ for (const auto& pair : intrin_buffer_map) {
+ auto it = comparator.rhs_buffer_map_.find(pair.second);
+ ICHECK(it != comparator.rhs_buffer_map_.end());
+ buffer_map[pair.first] = it->second;
+ }
+
+ // Step 4: Create MatchBufferRegion for the params of the implementation function of the tensor
+ // intrin to make them subregions of the buffer in the original IR.
+ std::unordered_map<Buffer, Array<Range>, ObjectPtrHash, ObjectPtrEqual> buffer_region_map;
+ for (const auto& read : impl_block->reads) {
+ buffer_region_map.emplace(read->buffer, read->region);
+ }
+ for (const auto& write : impl_block->writes) {
+ buffer_region_map.emplace(write->buffer, write->region);
+ }
+ Array<MatchBufferRegion> match_buffer_regions;
+ for (size_t i = 0; i < intrinsic->implementation->params.size(); ++i) {
+ const auto& param = intrinsic->implementation->params[i];
+ const auto& buffer = intrinsic->implementation->buffer_map.at(param);
+ const auto& source = buffer_map.at(buffer);
+ Region region = buffer_region_map.at(buffer);
+ auto extra_indices = comparator.buffer_indices_.at(source);
+ std::vector<Range> extra_buffer_ranges;
+ std::transform(extra_indices.begin(), extra_indices.end(),
+ std::back_inserter(extra_buffer_ranges),
+ [](const PrimExpr& index) { return Range::FromMinExtent(index, 1); });
+ region.insert(region.begin(), extra_buffer_ranges.begin(), extra_buffer_ranges.end());
+ match_buffer_regions.push_back(MatchBufferRegion(buffer, BufferRegion(source, region)));
+ }
+
+ // Step 5: Replace the subtree in the original IR with the tensor intrin implementation.
+ ObjectPtr<BlockNode> new_block_ptr = make_object<BlockNode>(*block_realize->block.get());
+ new_block_ptr->body = impl_block->body;
+ ICHECK(new_block_ptr->match_buffers.empty());
+ new_block_ptr->match_buffers = std::move(match_buffer_regions);
+ Block new_block(new_block_ptr);
+
+ // Substitution map for the intrin block to get the correct bindings.
+ Map<Var, PrimExpr> bv_map;
+ for (size_t i = 0; i < desc_block->iter_vars.size(); ++i) {
+ auto it = comparator.equal_map_.find(desc_block->iter_vars[i]->var);
+ if (it != comparator.equal_map_.end()) {
+ bv_map.Set(impl_block->iter_vars[i]->var, Downcast<PrimExpr>(it->second));
+ } else {
+ bv_map.Set(impl_block->iter_vars[i]->var, Integer(0));
+ }
+ }
+ new_block = Downcast<Block>(SubstituteInScope(new_block, bv_map));
+ self->Replace(block_sref, new_block, {{block_realize->block, new_block}});
Review comment:
I think you didn't get my ideas, but it's not high priority to do in this PR so I'm just curious.
Given we might care about the "exception-safety" of TIR schedule, i.e. if a schedule primitive fails, then we might want the IR to be unchanged. So in our particular case, during tensorization, if `Blockize` works, but the subsequence operation in `Tensorize` fails, then will it leave the IR changed?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785346873
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
Review comment:
1. Looks like we don't need the `const` here
2. Shall we add an underscore before "o"?
```suggestion
IterVar outer_var(/*dom=*/Range::FromMinExtent(0, division[i][0]->extent),
/*var=*/iter_var->var.copy_with_suffix("_o"),
/*iter_type=*/iter_var->iter_type);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785354948
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
Review comment:
```suggestion
const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [WIP][TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r788408997
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
+ const Block& desc_block = desc_block_realize->block;
+ const auto* impl_block_realize =
+ Downcast<BlockRealize>(intrinsic->implementation->body)->block->body.as<BlockRealizeNode>();
+ Block impl_block = impl_block_realize->block;
+
+ // Step 1: Blockize the subtree rooted at the given loop
+ const StmtSRef& block_sref = Blockize(self, loop_sref);
+ const BlockRealize& block_realize = GetBlockRealize(self, block_sref);
+
+ // Step 2: Compare the block with the description of the tensor intrinsic, find the correspondence
+ // between buffers in the block and the description.
+ TensorizeComparator comparator(/*assert_mode=*/true);
+ comparator.VisitStmt(block_realize, GetRef<Stmt>(desc_block_realize));
+
+ // Step 3: Find the correspondence between buffers in the current AST and the implementation of
+ // the tensor intrinsic
+
+ // Step 3.1: Map from intrinsic func buffer to description func buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> intrin_buffer_map;
+ RemapTensorIntrinBuffers(intrinsic, &intrin_buffer_map);
+ // Step 3.2: Map form intrinsic func buffer to current AST buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> buffer_map;
+ for (const auto& pair : intrin_buffer_map) {
+ auto it = comparator.rhs_buffer_map_.find(pair.second);
+ ICHECK(it != comparator.rhs_buffer_map_.end());
+ buffer_map[pair.first] = it->second;
+ }
+
+ // Step 4: Create MatchBufferRegion for the params of the implementation function of the tensor
+ // intrin to make them subregions of the buffer in the original IR.
+ std::unordered_map<Buffer, Array<Range>, ObjectPtrHash, ObjectPtrEqual> buffer_region_map;
+ for (const auto& read : impl_block->reads) {
+ buffer_region_map.emplace(read->buffer, read->region);
+ }
+ for (const auto& write : impl_block->writes) {
+ buffer_region_map.emplace(write->buffer, write->region);
+ }
+ Array<MatchBufferRegion> match_buffer_regions;
+ for (size_t i = 0; i < intrinsic->implementation->params.size(); ++i) {
+ const auto& param = intrinsic->implementation->params[i];
+ const auto& buffer = intrinsic->implementation->buffer_map.at(param);
+ const auto& source = buffer_map.at(buffer);
+ Region region = buffer_region_map.at(buffer);
+ auto extra_indices = comparator.buffer_indices_.at(source);
+ std::vector<Range> extra_buffer_ranges;
+ std::transform(extra_indices.begin(), extra_indices.end(),
+ std::back_inserter(extra_buffer_ranges),
+ [](const PrimExpr& index) { return Range::FromMinExtent(index, 1); });
+ region.insert(region.begin(), extra_buffer_ranges.begin(), extra_buffer_ranges.end());
+ match_buffer_regions.push_back(MatchBufferRegion(buffer, BufferRegion(source, region)));
+ }
+
+ // Step 5: Replace the subtree in the original IR with the tensor intrin implementation.
+ ObjectPtr<BlockNode> new_block_ptr = make_object<BlockNode>(*block_realize->block.get());
+ new_block_ptr->body = impl_block->body;
+ ICHECK(new_block_ptr->match_buffers.empty());
+ new_block_ptr->match_buffers = std::move(match_buffer_regions);
+ Block new_block(new_block_ptr);
+
+ // Substitution map for the intrin block to get the correct bindings.
+ Map<Var, PrimExpr> bv_map;
+ for (size_t i = 0; i < desc_block->iter_vars.size(); ++i) {
+ auto it = comparator.equal_map_.find(desc_block->iter_vars[i]->var);
+ if (it != comparator.equal_map_.end()) {
+ bv_map.Set(impl_block->iter_vars[i]->var, Downcast<PrimExpr>(it->second));
+ } else {
+ bv_map.Set(impl_block->iter_vars[i]->var, Integer(0));
+ }
+ }
+ new_block = Downcast<Block>(SubstituteInScope(new_block, bv_map));
+ self->Replace(block_sref, new_block, {{block_realize->block, new_block}});
Review comment:
I think you didn't get my ideas, but it's not high priority to do, so I'm just curious.
Given we might care about the "exception-safety" of TIR schedule, i.e. if a schedule primitive fails, then we might want the IR to be unchanged. So in our particular case, during tensorization, if `Blockize` works, but the subsequence operation in `Tensorize` fails, then will it leave the IR changed?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#issuecomment-1016289394
@MasterJH5574 the reason these two primitives are in the same file is because blockization is closely connected with tensorization (or more specifically, a prerequisite of tensorization). It might work to split then though, and I don’t have strong opinion
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785346873
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
Review comment:
Looks like we don't need the `const` here
```suggestion
IterVar outer_var(/*dom=*/Range::FromMinExtent(0, division[i][0]->extent),
/*var=*/iter_var->var.copy_with_suffix("o"),
/*iter_type=*/iter_var->iter_type);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785346873
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
Review comment:
Looks like we don't need the `const` here
```suggestion
IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785346873
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
Review comment:
Looks like we don't need the `const` here
```suggestion
IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] Hzfengsy merged pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
Hzfengsy merged pull request #9871:
URL: https://github.com/apache/tvm/pull/9871
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] Hzfengsy commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
Hzfengsy commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780634559
##########
File path: include/tvm/tir/function.h
##########
@@ -187,6 +187,56 @@ class LinkedParam : public ObjectRef {
TVM_DEFINE_OBJECT_REF_COW_METHOD(LinkedParamNode);
};
+/*!
+ * \brief Tensor intrinsics for tensorization
+ */
+class TensorIntrinNode : public Object {
+ public:
+ /*! \brief The function to describe the computation. */
+ PrimFunc description;
+ /*! \brief The intrinsic function for lower-level implementation. */
+ PrimFunc implementation;
Review comment:
Vote for `desc` and `impl`
##########
File path: include/tvm/tir/schedule/schedule.h
##########
@@ -465,6 +465,25 @@ class ScheduleNode : public runtime::Object {
*/
virtual void SetScope(const BlockRV& block_rv, int buffer_index, const String& storage_scope) = 0;
/******** Schedule: Blockize & Tensorize ********/
+ /*!
+ * \brief Convert the subtree rooted at a specific loop into a block.
+ * \param loop_rv the root of the subtree
+ * \return the new block
+ */
+ virtual BlockRV Blockize(const LoopRV& loop_rv) = 0;
+ /*!
+ * \brief Tensorize the computation enclosed by loop with tensor_intrin
+ * \param loop_rv the loop to be tensorized
+ * \param intrin the tensor intrinsic
+ */
+ virtual void Tensorize(const LoopRV& loop_rv, const TensorIntrin& intrin) = 0;
+ /*!
+ * \brief Tensorize the computation enclosed by loop with tensor_intrin
+ * \param loop_rv The loop to be tensorized
+ * \param intrin_name Name of the tensor intrinsic
+ */
+ virtual void Tensorize(const LoopRV& loop_rv, const String& intrin_name) = 0;
Review comment:
Should we add a tensorize API for a block? i.e. Tensorize it if we manually blockized before.
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
+using StmtComparator = StmtFunctor<bool(const Stmt& n, const Stmt& other)>;
+
+/*! \brief Deep comparison to check if two IR ASTs are equivalent */
+class TensorizeComparator : public ExprComparator, public StmtComparator {
+ public:
+ /*!
+ * \brief Constructor of TensorizeComparator
+ * \param assert_mode Whether to raise an error if the two IR ASTs do not match.
+ */
+ explicit TensorizeComparator(bool assert_mode = true) : assert_mode_(assert_mode) {}
+
+ // Map from rhs buffer to lhs buffer
+ std::unordered_map<Buffer, Buffer, ObjectHash, ObjectEqual> rhs_buffer_map_;
Review comment:
Keep them private if they are internal vars
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
+using StmtComparator = StmtFunctor<bool(const Stmt& n, const Stmt& other)>;
+
+/*! \brief Deep comparison to check if two IR ASTs are equivalent */
+class TensorizeComparator : public ExprComparator, public StmtComparator {
+ public:
+ /*!
+ * \brief Constructor of TensorizeComparator
+ * \param assert_mode Whether to raise an error if the two IR ASTs do not match.
+ */
+ explicit TensorizeComparator(bool assert_mode = true) : assert_mode_(assert_mode) {}
+
+ // Map from rhs buffer to lhs buffer
+ std::unordered_map<Buffer, Buffer, ObjectHash, ObjectEqual> rhs_buffer_map_;
+ // Buffer indices mapping
+ std::unordered_map<Buffer, std::vector<PrimExpr>, ObjectPtrHash, ObjectPtrEqual> buffer_indices_;
+ std::vector<IterVar> extra_block_vars_;
+ // variable remap if any
+ std::unordered_map<ObjectRef, ObjectRef, ObjectPtrHash, ObjectPtrEqual> equal_map_;
+
+ bool VisitExpr(const PrimExpr& n, const PrimExpr& other) override;
+ bool VisitStmt(const Stmt& n, const Stmt& other) override;
+
+ bool VisitStmt_(const ForNode* op, const Stmt& other) override;
+ bool VisitStmt_(const SeqStmtNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BufferStoreNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockRealizeNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockNode* op, const Stmt& other) override;
Review comment:
Yes, only parts of types are supported now. We may add the rest when needed
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
+using StmtComparator = StmtFunctor<bool(const Stmt& n, const Stmt& other)>;
+
+/*! \brief Deep comparison to check if two IR ASTs are equivalent */
+class TensorizeComparator : public ExprComparator, public StmtComparator {
+ public:
+ /*!
+ * \brief Constructor of TensorizeComparator
+ * \param assert_mode Whether to raise an error if the two IR ASTs do not match.
+ */
+ explicit TensorizeComparator(bool assert_mode = true) : assert_mode_(assert_mode) {}
+
+ // Map from rhs buffer to lhs buffer
+ std::unordered_map<Buffer, Buffer, ObjectHash, ObjectEqual> rhs_buffer_map_;
+ // Buffer indices mapping
+ std::unordered_map<Buffer, std::vector<PrimExpr>, ObjectPtrHash, ObjectPtrEqual> buffer_indices_;
+ std::vector<IterVar> extra_block_vars_;
+ // variable remap if any
+ std::unordered_map<ObjectRef, ObjectRef, ObjectPtrHash, ObjectPtrEqual> equal_map_;
+
+ bool VisitExpr(const PrimExpr& n, const PrimExpr& other) override;
+ bool VisitStmt(const Stmt& n, const Stmt& other) override;
+
+ bool VisitStmt_(const ForNode* op, const Stmt& other) override;
+ bool VisitStmt_(const SeqStmtNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BufferStoreNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockRealizeNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockNode* op, const Stmt& other) override;
+
+ bool VisitExpr_(const AddNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const SubNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const MulNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const DivNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const ModNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const EQNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const NENode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const LTNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const LENode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const GTNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const GENode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const AndNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const OrNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const MinNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const MaxNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const FloorDivNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const FloorModNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const IntImmNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const FloatImmNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const CastNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const VarNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const BufferLoadNode* op, const PrimExpr& other) override;
+
+ bool DefEqual(const ObjectRef& lhs, const ObjectRef& rhs);
+ virtual bool CompareBuffer(const Buffer& lhs, const Buffer& rhs);
+ bool CompareBufferRegion(const BufferRegion& lhs, const BufferRegion& rhs);
+ bool CompareAnnotation(const std::pair<String, ObjectRef>& lhs,
+ const std::pair<String, ObjectRef>& rhs);
+ bool CompareAnnotationMap(const Map<String, ObjectRef>& lhs, const Map<String, ObjectRef>& rhs);
+ template <typename T>
+ bool CompareBufferAccess(const T* lhs, const T* rhs);
+ template <typename T, typename F>
+ bool CompareArray(const Array<T>& lhs, const Array<T>& rhs, F cmp);
+ bool CompareRange(const Range& lhs, const Range& rhs);
+ bool CompareType(const DataType& lhs, const DataType& rhs);
+
+ protected:
Review comment:
Why `protected`?
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
Review comment:
Can we use `ScheduleError` here? Since it has a better error renderer
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
+
+ // check both functions' bodies are directly block
+ const auto* desc_realize =
+ Downcast<BlockRealize>(desc_func->body)->block->body.as<BlockRealizeNode>();
+ const auto* intrin_realize =
+ Downcast<BlockRealize>(intrin_func->body)->block->body.as<BlockRealizeNode>();
+ CHECK(desc_realize != nullptr) << "description function's body expect a directly block";
+ CHECK(intrin_realize != nullptr) << "intrinsic function's body expect a directly block";
+
+ const Block& desc_block = desc_realize->block;
+ const Block& intrin_block = intrin_realize->block;
+
+ // check block var number and iter type
+ CHECK_EQ(desc_block->iter_vars.size(), intrin_block->iter_vars.size())
+ << "Two blocks should have the same number of block vars";
+ for (size_t i = 0; i < desc_block->iter_vars.size(); i++) {
+ const IterVar& desc_var = desc_block->iter_vars[i];
+ const IterVar& intrin_var = intrin_block->iter_vars[i];
+ CHECK(desc_var->iter_type == intrin_var->iter_type)
+ << "Block iter_type mismatch between " << desc_var->iter_type << " and "
+ << intrin_var->iter_type;
+ }
+
+ auto n = make_object<TensorIntrinNode>();
Review comment:
The Type `TensorIntrinNode` is embedded in `make_object`. Personally think `auto` is OK here.
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
Review comment:
Do we need all params are Buffers? (To be specific Buffer Handles)
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
Review comment:
+1
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] xqdan commented on pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
xqdan commented on pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#issuecomment-1008498021
@vinx13 @junrushao1994 @Hzfengsy you may want to see if can handle this case in tir tensorize, https://github.com/apache/tvm/pull/1053, which @ZihengJiang and I met long long ago.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] vinx13 commented on pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
vinx13 commented on pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#issuecomment-1020425076
@xqdan Thanks for raising this case. Unfortunately this is not supported, which seems a limitation of match buffer region. Currently, after tensorize, the params of the tensor intrin will be replaced with a buffer subregion. Such buffer subregions can't handle argmax well. We can try to improve this in the future
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780587280
##########
File path: include/tvm/tir/function.h
##########
@@ -187,6 +187,56 @@ class LinkedParam : public ObjectRef {
TVM_DEFINE_OBJECT_REF_COW_METHOD(LinkedParamNode);
};
+/*!
+ * \brief Tensor intrinsics for tensorization
+ */
+class TensorIntrinNode : public Object {
+ public:
+ /*! \brief The function to describe the computation. */
+ PrimFunc description;
+ /*! \brief The intrinsic function for lower-level implementation. */
+ PrimFunc implementation;
Review comment:
Which one do you prefer, shortened name or full name? Like, `desc` and `impl` vs `description` and `implementation`
##########
File path: include/tvm/tir/function.h
##########
@@ -187,6 +187,56 @@ class LinkedParam : public ObjectRef {
TVM_DEFINE_OBJECT_REF_COW_METHOD(LinkedParamNode);
};
+/*!
+ * \brief Tensor intrinsics for tensorization
+ */
+class TensorIntrinNode : public Object {
+ public:
+ /*! \brief The function to describe the computation. */
+ PrimFunc description;
+ /*! \brief The intrinsic function for lower-level implementation. */
+ PrimFunc implementation;
+
+ void VisitAttrs(AttrVisitor* v) {
+ v->Visit("description", &description);
+ v->Visit("implementation", &implementation);
+ }
+
+ static constexpr const char* _type_key = "tir.TensorIntrin";
+ TVM_DECLARE_FINAL_OBJECT_INFO(TensorIntrinNode, Object);
+};
+
+/*!
+ * \brief Managed reference to TensorIntrinNode.
+ */
+class TensorIntrin : public ObjectRef {
+ public:
+ /*!
+ * \brief Constructor
+ * \param desc_func The function to describe the computation.
+ * \param intrin_func The intrinsic function for lower-level implementation.
+ */
+ TVM_DLL explicit TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func);
+
+ /*!
+ * \brief Create and register a TensorIntrin. After registration, the TensorIntrin can be looked
+ * up with its name.
+ * \param name The name of the TensorIntrin to register
+ * \param desc_func The function to describe the computation.
+ * \param intrin_func The intrinsic function for lower-level implementation.
+ * \return The created TensorIntrin.
+ */
+ TVM_DLL static TensorIntrin Register(String name, PrimFunc desc_func, PrimFunc intrin_func);
Review comment:
Let's be more idiomatic in registry design
```suggestion
TVM_DLL static TensorIntrin Register(String name, TensorIntrin intrin);
```
##########
File path: include/tvm/tir/schedule/schedule.h
##########
@@ -465,6 +465,25 @@ class ScheduleNode : public runtime::Object {
*/
virtual void SetScope(const BlockRV& block_rv, int buffer_index, const String& storage_scope) = 0;
/******** Schedule: Blockize & Tensorize ********/
+ /*!
+ * \brief Convert the subtree rooted at a specific loop into a block.
+ * \param loop_rv the root of the subtree
+ * \return the new block
+ */
+ virtual BlockRV Blockize(const LoopRV& loop_rv) = 0;
+ /*!
+ * \brief Tensorize the computation enclosed by loop with tensor_intrin
+ * \param loop_rv the loop to be tensorized
+ * \param intrin the tensor intrinsic
+ */
+ virtual void Tensorize(const LoopRV& loop_rv, const TensorIntrin& intrin) = 0;
Review comment:
Shall we hide this method to internal impl given:
- TracedSchedule is the default go-to
- This method isn't well supported by TracedSchedule
##########
File path: python/tvm/tir/schedule/schedule.py
##########
@@ -1733,6 +1733,258 @@ def after_set_scope(
########## Schedule: Blockize & Tensorize ##########
+ @type_checked
+ def blockize(self, loop: LoopRV) -> BlockRV:
+ """Convert the subtree rooted at a specific loop into a block.
+
+ Parameters
+ ----------
+ loop : LoopRV
+ The root of the subtree.
+
+ Returns
+ -------
+ result : BlockRV
+ The new block.
+
+ Examples
+ --------
+
+ Before blockize, in TensorIR, the IR is:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def before_blockize(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"]
+ ) -> None:
+ for i_0, j_0, i_1, j_1 in T.grid(8, 8, 16, 16):
+ with T.block("B"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1)
+ T.reads(A[vi, vj])
+ T.writes(B[vi, vj])
+ B[vi, vj] = A[vi, vj] * T.float32(2)
+
+ Create the schedule and do set_scope:
+
+ .. code-block:: python
+
+ sch = tir.Schedule(before_blockize)
+ B = sch.get_block("B")
+ _, _, i1, _ = sch.get_loops(B)
+ sch.blockize(i1)
+ print(sch.mod["main"].script())
+
+ After applying blockize, the IR becomes:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def after_blockize(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"]
+ )-> None:
+ for i_0, j_0 in T.grid(8, 8):
+ with T.block("blockized_B"):
+ vio, vjo = T.axis.remap("SS", [i_0, j_0])
+ T.reads(A[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16])
+ T.writes(B[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16])
+ for i_1, j_1 in T.grid(16, 16):
+ with T.block("B"):
+ vi = T.axis.spatial(128, vio * 16 + i_1)
+ vj = T.axis.spatial(128, vjo * 16 + j_1)
+ T.reads(A[vi, vj])
+ T.writes(B[vi, vj])
+ B[vi, vj] = A[vi, vj] * T.float32(2)
+
+ Note
+ ----
+ blockize requires there is exactly one block under the given loop and the bindings of the
+ block are divisible by the subspace represented by the loops starting at the given loop.
+
+ """
+
+ return _ffi_api.ScheduleBlockize(self, loop) # type: ignore # pylint: disable=no-member
+
+ @type_checked
+ def tensorize(self, loop: LoopRV, tensor_intrin: Union[str, TensorIntrin]) -> None:
+ """Tensorize the computation enclosed by loop with the tensor intrinsic.
+
+ Parameters
+ ----------
+ loop : LoopRV
+ The loop to be tensorized.
+ tensor_intrin : Union[str, TensorIntrin]
+ The tensor intrin or the name of the tensor intrin.
+
+ Examples
+ --------
+
+ Before tensorize, in TensorIR, the IR is:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def before_tensorize(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"],
+ C: T.Buffer[(128, 128), "float32"],
+ ) -> None:
+ # body
+ # with T.block("root")
+ for i_0, j_0 in T.grid(8, 8):
+ for i_1_init, j_1_init in T.grid(16, 16):
+ with T.block("init"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1_init)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1_init)
+ T.reads()
+ T.writes(C[vi, vj])
+ C[vi, vj] = T.float32(0)
+ for k_0, i_1, j_1, k_1 in T.grid(8, 16, 16, 16):
+ with T.block("update"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1)
+ vk = T.axis.reduce(128, k_0 * 16 + k_1)
+ T.reads(C[vi, vj], A[vi, vk], B[vj, vk])
+ T.writes(C[vi, vj])
+ C[vi, vj] = C[vi, vj] + A[vi, vk] * B[vj, vk]
+
+ Declare and register the tensor intrinsic:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def mma_desc(a: T.handle, b: T.handle, c: T.handle) -> None:
+ A = T.match_buffer(a, (16, 16), align=128, offset_factor=1)
+ B = T.match_buffer(b, (16, 16), align=128, offset_factor=1)
+ C = T.match_buffer(c, (16, 16), align=128, offset_factor=1)
+
+ with T.block("root"):
+ vi = T.axis.S(16, 0)
+ vj = T.axis.S(16, 0)
+ vk = T.axis.R(16, 0)
+ for i, j, k in T.grid(16, 16, 16):
+ with T.block("update"):
+ vii = T.axis.S(16, vi + i)
+ vjj = T.axis.S(16, vj + j)
+ vkk = T.axis.R(16, vk + k)
+ C[vii, vjj] = C[vii, vjj] + A[vii, vkk] * B[vjj, vkk]
+
+
+ @T.prim_func
+ def mma_intrin(a: T.handle, b: T.handle, c: T.handle) -> None:
+ A = T.match_buffer(a, (16, 16), align=128, offset_factor=1)
+ B = T.match_buffer(b, (16, 16), align=128, offset_factor=1)
+ C = T.match_buffer(c, (16, 16), align=128, offset_factor=1)
+
+ with T.block("root"):
+ vi = T.axis.S(16, 0)
+ vj = T.axis.S(16, 0)
+ vk = T.axis.R(16, 0)
+ T.reads(
+ [
+ C[vi : vi + 16, vj : vj + 16],
+ A[vi : vi + 16, vk : vk + 16],
+ B[vj : vj + 16, vk : vk + 16],
+ ]
+ )
+ T.writes(C[vi : vi + 16, vj : vj + 16])
+ T.evaluate(
+ T.tvm_mma_sync(
+ C.data,
+ C.elem_offset // 256,
+ A.data,
+ A.elem_offset // 256,
+ B.data,
+ B.elem_offset // 256,
+ C.data,
+ C.elem_offset // 256,
+ dtype="handle",
+ )
+ )
+
+ tir.TensorIntrin.register("test_mma_intrin", mma_desc, mma_intrin)
+
+ Create the schedule and do tensorize:
+
+ .. code-block:: python
+
+ sch = tir.Schedule(before_tensorize)
+ update = s.get_block("update")
+ _, _, _, i1, _, _ = s.get_loops(update)
+ s.tensorize(ii, "test_mma_intrin")
+ print(sch.mod["main"].script())
+
+ After applying tensorize, the IR becomes:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def after_tensoirze(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"],
+ C: T.Buffer[(128, 128), "float32"],
+ ) -> None:
+ # body
+ # with T.block("root")
+ for i_0, j_0 in T.grid(8, 8):
+ for i_1_init, j_1_init in T.grid(16, 16):
+ with T.block("init"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1_init)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1_init)
+ T.reads()
+ T.writes(C[vi, vj])
+ C[vi, vj] = T.float32(0)
+ for k_0 in T.serial(8):
+ with T.block("blockized_update"):
+ vio, vjo, vko = T.axis.remap("SSR", [i_0, j_0, k_0])
+ T.reads(
+ C[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16],
+ A[vio * 16 : vio * 16 + 16, vko * 16 : vko * 16 + 16],
+ B[vjo * 16 : vjo * 16 + 16, vko * 16 : vko * 16 + 16],
+ )
+ T.writes(C[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16])
+ A_1 = T.match_buffer(
+ A[vio * 16 : vio * 16 + 16, vko * 16 : vko * 16 + 16],
+ [16, 16],
+ dtype="float32",
+ offset_factor=1,
+ )
+ B_1 = T.match_buffer(
+ B[vjo * 16 : vjo * 16 + 16, vko * 16 : vko * 16 + 16],
+ [16, 16],
+ dtype="float32",
+ offset_factor=1,
+ )
+ C_1 = T.match_buffer(
+ C[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16],
+ [16, 16],
+ dtype="float32",
+ offset_factor=1,
+ )
+ T.evaluate(
+ T.tvm_mma_sync(
+ C_1.data,
+ C_1.elem_offset // 256,
+ A_1.data,
+ A_1.elem_offset // 256,
+ B_1.data,
+ B_1.elem_offset // 256,
+ C_1.data,
+ C_1.elem_offset // 256,
+ dtype="handle",
+ )
+ )
+
+ """
+ if isinstance(tensor_intrin, str):
+ tensor_intrin = String(tensor_intrin)
Review comment:
Do we need this conversion? I suppose `PackedFuncValueConverter` is able to handle this :-)
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
Review comment:
Let's write some informative error message here given it's user-facing CHECK
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
+
+ // check both functions' bodies are directly block
+ const auto* desc_realize =
+ Downcast<BlockRealize>(desc_func->body)->block->body.as<BlockRealizeNode>();
+ const auto* intrin_realize =
+ Downcast<BlockRealize>(intrin_func->body)->block->body.as<BlockRealizeNode>();
+ CHECK(desc_realize != nullptr) << "description function's body expect a directly block";
+ CHECK(intrin_realize != nullptr) << "intrinsic function's body expect a directly block";
+
+ const Block& desc_block = desc_realize->block;
+ const Block& intrin_block = intrin_realize->block;
+
+ // check block var number and iter type
+ CHECK_EQ(desc_block->iter_vars.size(), intrin_block->iter_vars.size())
+ << "Two blocks should have the same number of block vars";
+ for (size_t i = 0; i < desc_block->iter_vars.size(); i++) {
+ const IterVar& desc_var = desc_block->iter_vars[i];
+ const IterVar& intrin_var = intrin_block->iter_vars[i];
+ CHECK(desc_var->iter_type == intrin_var->iter_type)
+ << "Block iter_type mismatch between " << desc_var->iter_type << " and "
+ << intrin_var->iter_type;
Review comment:
Let's use `IterVarType2String`
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
+
+ // check both functions' bodies are directly block
+ const auto* desc_realize =
+ Downcast<BlockRealize>(desc_func->body)->block->body.as<BlockRealizeNode>();
+ const auto* intrin_realize =
+ Downcast<BlockRealize>(intrin_func->body)->block->body.as<BlockRealizeNode>();
+ CHECK(desc_realize != nullptr) << "description function's body expect a directly block";
+ CHECK(intrin_realize != nullptr) << "intrinsic function's body expect a directly block";
+
+ const Block& desc_block = desc_realize->block;
+ const Block& intrin_block = intrin_realize->block;
+
+ // check block var number and iter type
+ CHECK_EQ(desc_block->iter_vars.size(), intrin_block->iter_vars.size())
+ << "Two blocks should have the same number of block vars";
+ for (size_t i = 0; i < desc_block->iter_vars.size(); i++) {
+ const IterVar& desc_var = desc_block->iter_vars[i];
+ const IterVar& intrin_var = intrin_block->iter_vars[i];
+ CHECK(desc_var->iter_type == intrin_var->iter_type)
+ << "Block iter_type mismatch between " << desc_var->iter_type << " and "
+ << intrin_var->iter_type;
+ }
+
+ auto n = make_object<TensorIntrinNode>();
Review comment:
nit
```suggestion
ObjectPtr<TensorIntrinNode> n = make_object<TensorIntrinNode>();
```
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
+
+ // check both functions' bodies are directly block
+ const auto* desc_realize =
+ Downcast<BlockRealize>(desc_func->body)->block->body.as<BlockRealizeNode>();
+ const auto* intrin_realize =
+ Downcast<BlockRealize>(intrin_func->body)->block->body.as<BlockRealizeNode>();
+ CHECK(desc_realize != nullptr) << "description function's body expect a directly block";
+ CHECK(intrin_realize != nullptr) << "intrinsic function's body expect a directly block";
+
+ const Block& desc_block = desc_realize->block;
+ const Block& intrin_block = intrin_realize->block;
+
+ // check block var number and iter type
+ CHECK_EQ(desc_block->iter_vars.size(), intrin_block->iter_vars.size())
+ << "Two blocks should have the same number of block vars";
+ for (size_t i = 0; i < desc_block->iter_vars.size(); i++) {
+ const IterVar& desc_var = desc_block->iter_vars[i];
+ const IterVar& intrin_var = intrin_block->iter_vars[i];
+ CHECK(desc_var->iter_type == intrin_var->iter_type)
+ << "Block iter_type mismatch between " << desc_var->iter_type << " and "
+ << intrin_var->iter_type;
+ }
+
+ auto n = make_object<TensorIntrinNode>();
+ n->description = std::move(desc_func);
+ n->implementation = std::move(intrin_func);
+ data_ = std::move(n);
+}
+
+TensorIntrin TensorIntrin::Register(String name, PrimFunc desc_func, PrimFunc intrin_func) {
+ TensorIntrinManager* manager = TensorIntrinManager::Global();
+ ICHECK_EQ(manager->reg.count(name), 0)
+ << "ValueError: TensorIntrin '" << name << "' has already been registered";
Review comment:
Should be user-facing API
```suggestion
CHECK_EQ(manager->reg.count(name), 0)
<< "ValueError: TensorIntrin '" << name << "' has already been registered";
```
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
+
+ // check both functions' bodies are directly block
+ const auto* desc_realize =
+ Downcast<BlockRealize>(desc_func->body)->block->body.as<BlockRealizeNode>();
+ const auto* intrin_realize =
+ Downcast<BlockRealize>(intrin_func->body)->block->body.as<BlockRealizeNode>();
+ CHECK(desc_realize != nullptr) << "description function's body expect a directly block";
+ CHECK(intrin_realize != nullptr) << "intrinsic function's body expect a directly block";
+
+ const Block& desc_block = desc_realize->block;
+ const Block& intrin_block = intrin_realize->block;
+
+ // check block var number and iter type
+ CHECK_EQ(desc_block->iter_vars.size(), intrin_block->iter_vars.size())
+ << "Two blocks should have the same number of block vars";
+ for (size_t i = 0; i < desc_block->iter_vars.size(); i++) {
+ const IterVar& desc_var = desc_block->iter_vars[i];
+ const IterVar& intrin_var = intrin_block->iter_vars[i];
+ CHECK(desc_var->iter_type == intrin_var->iter_type)
+ << "Block iter_type mismatch between " << desc_var->iter_type << " and "
+ << intrin_var->iter_type;
+ }
+
+ auto n = make_object<TensorIntrinNode>();
+ n->description = std::move(desc_func);
+ n->implementation = std::move(intrin_func);
+ data_ = std::move(n);
+}
+
+TensorIntrin TensorIntrin::Register(String name, PrimFunc desc_func, PrimFunc intrin_func) {
+ TensorIntrinManager* manager = TensorIntrinManager::Global();
+ ICHECK_EQ(manager->reg.count(name), 0)
+ << "ValueError: TensorIntrin '" << name << "' has already been registered";
+ TensorIntrin intrin(desc_func, intrin_func);
+ manager->reg.Set(name, intrin);
+ return intrin;
+}
+
+TensorIntrin TensorIntrin::Get(String name) {
+ const TensorIntrinManager* manager = TensorIntrinManager::Global();
+ ICHECK_EQ(manager->reg.count(name), 1)
+ << "ValueError: TensorIntrin '" << name << "' is not registered";
Review comment:
ditto
```suggestion
CHECK_EQ(manager->reg.count(name), 1)
<< "ValueError: TensorIntrin '" << name << "' is not registered";
```
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
+
+ // check both functions' bodies are directly block
+ const auto* desc_realize =
+ Downcast<BlockRealize>(desc_func->body)->block->body.as<BlockRealizeNode>();
+ const auto* intrin_realize =
+ Downcast<BlockRealize>(intrin_func->body)->block->body.as<BlockRealizeNode>();
+ CHECK(desc_realize != nullptr) << "description function's body expect a directly block";
+ CHECK(intrin_realize != nullptr) << "intrinsic function's body expect a directly block";
+
+ const Block& desc_block = desc_realize->block;
+ const Block& intrin_block = intrin_realize->block;
+
+ // check block var number and iter type
+ CHECK_EQ(desc_block->iter_vars.size(), intrin_block->iter_vars.size())
+ << "Two blocks should have the same number of block vars";
+ for (size_t i = 0; i < desc_block->iter_vars.size(); i++) {
+ const IterVar& desc_var = desc_block->iter_vars[i];
+ const IterVar& intrin_var = intrin_block->iter_vars[i];
+ CHECK(desc_var->iter_type == intrin_var->iter_type)
+ << "Block iter_type mismatch between " << desc_var->iter_type << " and "
+ << intrin_var->iter_type;
+ }
+
+ auto n = make_object<TensorIntrinNode>();
+ n->description = std::move(desc_func);
+ n->implementation = std::move(intrin_func);
+ data_ = std::move(n);
+}
+
+TensorIntrin TensorIntrin::Register(String name, PrimFunc desc_func, PrimFunc intrin_func) {
+ TensorIntrinManager* manager = TensorIntrinManager::Global();
+ ICHECK_EQ(manager->reg.count(name), 0)
+ << "ValueError: TensorIntrin '" << name << "' has already been registered";
+ TensorIntrin intrin(desc_func, intrin_func);
+ manager->reg.Set(name, intrin);
+ return intrin;
+}
+
+TensorIntrin TensorIntrin::Get(String name) {
+ const TensorIntrinManager* manager = TensorIntrinManager::Global();
+ ICHECK_EQ(manager->reg.count(name), 1)
+ << "ValueError: TensorIntrin '" << name << "' is not registered";
+ return manager->reg.at(name);
Review comment:
Use manager->reg.find to save one lookup :-)
##########
File path: python/tvm/tir/schedule/schedule.py
##########
@@ -1733,6 +1733,258 @@ def after_set_scope(
########## Schedule: Blockize & Tensorize ##########
+ @type_checked
+ def blockize(self, loop: LoopRV) -> BlockRV:
+ """Convert the subtree rooted at a specific loop into a block.
+
+ Parameters
+ ----------
+ loop : LoopRV
+ The root of the subtree.
+
+ Returns
+ -------
+ result : BlockRV
+ The new block.
+
+ Examples
+ --------
+
+ Before blockize, in TensorIR, the IR is:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def before_blockize(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"]
+ ) -> None:
+ for i_0, j_0, i_1, j_1 in T.grid(8, 8, 16, 16):
+ with T.block("B"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1)
+ T.reads(A[vi, vj])
+ T.writes(B[vi, vj])
+ B[vi, vj] = A[vi, vj] * T.float32(2)
+
+ Create the schedule and do set_scope:
+
+ .. code-block:: python
+
+ sch = tir.Schedule(before_blockize)
+ B = sch.get_block("B")
+ _, _, i1, _ = sch.get_loops(B)
+ sch.blockize(i1)
+ print(sch.mod["main"].script())
+
+ After applying blockize, the IR becomes:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def after_blockize(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"]
+ )-> None:
+ for i_0, j_0 in T.grid(8, 8):
+ with T.block("blockized_B"):
+ vio, vjo = T.axis.remap("SS", [i_0, j_0])
+ T.reads(A[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16])
+ T.writes(B[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16])
+ for i_1, j_1 in T.grid(16, 16):
+ with T.block("B"):
+ vi = T.axis.spatial(128, vio * 16 + i_1)
+ vj = T.axis.spatial(128, vjo * 16 + j_1)
+ T.reads(A[vi, vj])
+ T.writes(B[vi, vj])
+ B[vi, vj] = A[vi, vj] * T.float32(2)
+
+ Note
+ ----
+ blockize requires there is exactly one block under the given loop and the bindings of the
+ block are divisible by the subspace represented by the loops starting at the given loop.
+
+ """
+
+ return _ffi_api.ScheduleBlockize(self, loop) # type: ignore # pylint: disable=no-member
+
+ @type_checked
+ def tensorize(self, loop: LoopRV, tensor_intrin: Union[str, TensorIntrin]) -> None:
+ """Tensorize the computation enclosed by loop with the tensor intrinsic.
+
+ Parameters
+ ----------
+ loop : LoopRV
+ The loop to be tensorized.
+ tensor_intrin : Union[str, TensorIntrin]
+ The tensor intrin or the name of the tensor intrin.
+
+ Examples
+ --------
+
+ Before tensorize, in TensorIR, the IR is:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def before_tensorize(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"],
+ C: T.Buffer[(128, 128), "float32"],
+ ) -> None:
+ # body
+ # with T.block("root")
+ for i_0, j_0 in T.grid(8, 8):
+ for i_1_init, j_1_init in T.grid(16, 16):
+ with T.block("init"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1_init)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1_init)
+ T.reads()
+ T.writes(C[vi, vj])
+ C[vi, vj] = T.float32(0)
+ for k_0, i_1, j_1, k_1 in T.grid(8, 16, 16, 16):
+ with T.block("update"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1)
+ vk = T.axis.reduce(128, k_0 * 16 + k_1)
+ T.reads(C[vi, vj], A[vi, vk], B[vj, vk])
+ T.writes(C[vi, vj])
+ C[vi, vj] = C[vi, vj] + A[vi, vk] * B[vj, vk]
+
+ Declare and register the tensor intrinsic:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def mma_desc(a: T.handle, b: T.handle, c: T.handle) -> None:
+ A = T.match_buffer(a, (16, 16), align=128, offset_factor=1)
+ B = T.match_buffer(b, (16, 16), align=128, offset_factor=1)
+ C = T.match_buffer(c, (16, 16), align=128, offset_factor=1)
+
+ with T.block("root"):
+ vi = T.axis.S(16, 0)
+ vj = T.axis.S(16, 0)
+ vk = T.axis.R(16, 0)
+ for i, j, k in T.grid(16, 16, 16):
+ with T.block("update"):
+ vii = T.axis.S(16, vi + i)
+ vjj = T.axis.S(16, vj + j)
+ vkk = T.axis.R(16, vk + k)
+ C[vii, vjj] = C[vii, vjj] + A[vii, vkk] * B[vjj, vkk]
+
+
+ @T.prim_func
+ def mma_intrin(a: T.handle, b: T.handle, c: T.handle) -> None:
+ A = T.match_buffer(a, (16, 16), align=128, offset_factor=1)
+ B = T.match_buffer(b, (16, 16), align=128, offset_factor=1)
+ C = T.match_buffer(c, (16, 16), align=128, offset_factor=1)
+
+ with T.block("root"):
+ vi = T.axis.S(16, 0)
+ vj = T.axis.S(16, 0)
+ vk = T.axis.R(16, 0)
+ T.reads(
+ [
+ C[vi : vi + 16, vj : vj + 16],
+ A[vi : vi + 16, vk : vk + 16],
+ B[vj : vj + 16, vk : vk + 16],
+ ]
+ )
+ T.writes(C[vi : vi + 16, vj : vj + 16])
+ T.evaluate(
+ T.tvm_mma_sync(
+ C.data,
+ C.elem_offset // 256,
+ A.data,
+ A.elem_offset // 256,
+ B.data,
+ B.elem_offset // 256,
+ C.data,
+ C.elem_offset // 256,
+ dtype="handle",
+ )
+ )
+
+ tir.TensorIntrin.register("test_mma_intrin", mma_desc, mma_intrin)
+
+ Create the schedule and do tensorize:
+
+ .. code-block:: python
+
+ sch = tir.Schedule(before_tensorize)
+ update = s.get_block("update")
+ _, _, _, i1, _, _ = s.get_loops(update)
+ s.tensorize(ii, "test_mma_intrin")
+ print(sch.mod["main"].script())
+
+ After applying tensorize, the IR becomes:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def after_tensoirze(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"],
+ C: T.Buffer[(128, 128), "float32"],
+ ) -> None:
+ # body
+ # with T.block("root")
+ for i_0, j_0 in T.grid(8, 8):
+ for i_1_init, j_1_init in T.grid(16, 16):
+ with T.block("init"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1_init)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1_init)
+ T.reads()
+ T.writes(C[vi, vj])
+ C[vi, vj] = T.float32(0)
+ for k_0 in T.serial(8):
+ with T.block("blockized_update"):
+ vio, vjo, vko = T.axis.remap("SSR", [i_0, j_0, k_0])
+ T.reads(
+ C[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16],
+ A[vio * 16 : vio * 16 + 16, vko * 16 : vko * 16 + 16],
+ B[vjo * 16 : vjo * 16 + 16, vko * 16 : vko * 16 + 16],
+ )
+ T.writes(C[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16])
+ A_1 = T.match_buffer(
+ A[vio * 16 : vio * 16 + 16, vko * 16 : vko * 16 + 16],
+ [16, 16],
+ dtype="float32",
+ offset_factor=1,
+ )
+ B_1 = T.match_buffer(
+ B[vjo * 16 : vjo * 16 + 16, vko * 16 : vko * 16 + 16],
+ [16, 16],
+ dtype="float32",
+ offset_factor=1,
+ )
+ C_1 = T.match_buffer(
+ C[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16],
+ [16, 16],
+ dtype="float32",
+ offset_factor=1,
+ )
+ T.evaluate(
+ T.tvm_mma_sync(
+ C_1.data,
+ C_1.elem_offset // 256,
+ A_1.data,
+ A_1.elem_offset // 256,
+ B_1.data,
+ B_1.elem_offset // 256,
+ C_1.data,
+ C_1.elem_offset // 256,
+ dtype="handle",
+ )
+ )
+
+ """
+ if isinstance(tensor_intrin, str):
+ tensor_intrin = String(tensor_intrin)
Review comment:
oh i see. It's because our C++ registry are trying to decide between String and TensorIntrin - then it makes sense to me
##########
File path: src/tir/schedule/schedule.cc
##########
@@ -183,6 +183,20 @@ TVM_REGISTER_GLOBAL("tir.schedule.ScheduleStorageAlign")
TVM_REGISTER_GLOBAL("tir.schedule.ScheduleSetScope")
.set_body_method<Schedule>(&ScheduleNode::SetScope);
/******** (FFI) Blockize & Tensorize ********/
+TVM_REGISTER_GLOBAL("tir.schedule.ScheduleBlockize")
+ .set_body_method<Schedule>(&ScheduleNode::Blockize);
+TVM_REGISTER_GLOBAL("tir.schedule.ScheduleTensorize")
+ .set_body_typed([](Schedule self, LoopRV loop_rv, ObjectRef intrin) {
+ if (const auto* str = intrin.as<runtime::StringObj>()) {
+ return self->Tensorize(loop_rv, GetRef<String>(str));
+ }
+ if (const auto* p_intrin = intrin.as<TensorIntrinNode>()) {
+ return self->Tensorize(loop_rv, GetRef<TensorIntrin>(p_intrin));
+ }
+ LOG(FATAL) << "TypeError: Cannot handle type: " << intrin->GetTypeKey();
+ throw;
Review comment:
I'm considering...is it reasonable to use only the `String` API and not expose the `TensorIntrin` one? The reason I'm asking this is that `TensorIntrin` isn't available for `TracedSchedule` anyway, which is the default go-to for python frontend users
##########
File path: src/tir/schedule/state.cc
##########
@@ -201,9 +201,8 @@ class BlockInfoCollector : private StmtVisitor {
bool is_root_block = srefs_.empty();
// Calculate `BlockInfo::scope`
Array<StmtSRef> child_block_srefs = std::move(block_frames_.back());
- BlockInfo& info =
- self_->block_info.emplace(scope_root, BlockInfo(BlockScope(child_block_srefs)))
- .first->second;
+ self_->block_info[scope_root] = BlockInfo(BlockScope(child_block_srefs));
+ BlockInfo& info = self_->block_info[scope_root];
Review comment:
Why this change? I suppose they are logically equivalent and the original one saves one lookup?
##########
File path: src/tir/schedule/state.cc
##########
@@ -201,9 +201,8 @@ class BlockInfoCollector : private StmtVisitor {
bool is_root_block = srefs_.empty();
// Calculate `BlockInfo::scope`
Array<StmtSRef> child_block_srefs = std::move(block_frames_.back());
- BlockInfo& info =
- self_->block_info.emplace(scope_root, BlockInfo(BlockScope(child_block_srefs)))
- .first->second;
+ self_->block_info[scope_root] = BlockInfo(BlockScope(child_block_srefs));
+ BlockInfo& info = self_->block_info[scope_root];
Review comment:
Oh I see - that makes sense to me! Then to save one look-up, let's do
```suggestion
BlockInfo& info = self_->block_info[scope_root] = BlockInfo(BlockScope(child_block_srefs));
```
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
Review comment:
Shall we move this into `src/tir/schedule/ir_comparator.h` and the impl to a corresponding cc file? The reason I'm considering this is that it's relatively independent but takes huge chunk of code
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
+using StmtComparator = StmtFunctor<bool(const Stmt& n, const Stmt& other)>;
+
+/*! \brief Deep comparison to check if two IR ASTs are equivalent */
+class TensorizeComparator : public ExprComparator, public StmtComparator {
+ public:
+ /*!
+ * \brief Constructor of TensorizeComparator
+ * \param assert_mode Whether to raise an error if the two IR ASTs do not match.
+ */
+ explicit TensorizeComparator(bool assert_mode = true) : assert_mode_(assert_mode) {}
+
+ // Map from rhs buffer to lhs buffer
+ std::unordered_map<Buffer, Buffer, ObjectHash, ObjectEqual> rhs_buffer_map_;
+ // Buffer indices mapping
+ std::unordered_map<Buffer, std::vector<PrimExpr>, ObjectPtrHash, ObjectPtrEqual> buffer_indices_;
+ std::vector<IterVar> extra_block_vars_;
+ // variable remap if any
+ std::unordered_map<ObjectRef, ObjectRef, ObjectPtrHash, ObjectPtrEqual> equal_map_;
Review comment:
Let's move the fields to the end of the class, and document them properly
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
+using StmtComparator = StmtFunctor<bool(const Stmt& n, const Stmt& other)>;
+
+/*! \brief Deep comparison to check if two IR ASTs are equivalent */
+class TensorizeComparator : public ExprComparator, public StmtComparator {
+ public:
+ /*!
+ * \brief Constructor of TensorizeComparator
+ * \param assert_mode Whether to raise an error if the two IR ASTs do not match.
+ */
+ explicit TensorizeComparator(bool assert_mode = true) : assert_mode_(assert_mode) {}
+
+ // Map from rhs buffer to lhs buffer
+ std::unordered_map<Buffer, Buffer, ObjectHash, ObjectEqual> rhs_buffer_map_;
+ // Buffer indices mapping
+ std::unordered_map<Buffer, std::vector<PrimExpr>, ObjectPtrHash, ObjectPtrEqual> buffer_indices_;
+ std::vector<IterVar> extra_block_vars_;
+ // variable remap if any
+ std::unordered_map<ObjectRef, ObjectRef, ObjectPtrHash, ObjectPtrEqual> equal_map_;
+
+ bool VisitExpr(const PrimExpr& n, const PrimExpr& other) override;
+ bool VisitStmt(const Stmt& n, const Stmt& other) override;
+
+ bool VisitStmt_(const ForNode* op, const Stmt& other) override;
+ bool VisitStmt_(const SeqStmtNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BufferStoreNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockRealizeNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockNode* op, const Stmt& other) override;
Review comment:
Just curious, does it mean that we only allow 5 types of statements?
##########
File path: src/tir/schedule/transform.cc
##########
@@ -136,5 +136,52 @@ void LeafBlockRemovalPlan(const ScheduleState& self, const StmtSRef& leaf_block_
throw OnlyLeafError(self->mod, GetRef<Block>(leaf_block), GetRef<Block>(scope_block));
}
+/******** IR Substitution ********/
+class IRSubstituteInScope : public StmtExprMutator {
+ public:
+ explicit IRSubstituteInScope(std::function<PrimExpr(const VarNode*)> fmap)
+ : fmap_(std::move(fmap)) {}
+
+ PrimExpr VisitExpr_(const VarNode* op) final {
+ auto it = fmap_(op);
+ if (it.defined()) {
+ return it;
+ } else {
+ return GetRef<PrimExpr>(op);
+ }
+ }
+
+ Stmt VisitStmt_(const BlockRealizeNode* op) final {
+ auto fmutate = [&](const PrimExpr& e) { return this->VisitExpr(e); };
+ Array<PrimExpr> v = op->iter_values;
+ v.MutateByApply(fmutate);
+ PrimExpr pred = this->VisitExpr(op->predicate);
+ if (v.same_as(op->iter_values) && pred.same_as(op->predicate)) {
+ return GetRef<Stmt>(op);
+ } else {
+ auto n = CopyOnWrite(op);
+ n->iter_values = std::move(v);
+ n->predicate = std::move(analyzer.Simplify(pred));
+ return Stmt(n);
+ }
+ }
+
+ private:
+ const std::function<PrimExpr(const VarNode*)> fmap_;
+ arith::Analyzer analyzer;
+};
+
+Stmt SubstituteInScope(const Stmt& stmt, const Map<Var, PrimExpr>& subst_map) {
Review comment:
I'm reading the implementation. Looks like what `SubstituteInScope` really does is that we only substitute block bindings and predicates, stops at the `BlockRealize` boundary and does nothing else. Is that accurate? If so, shall we rename this to `SubstituteBlockBinding`?
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
+using StmtComparator = StmtFunctor<bool(const Stmt& n, const Stmt& other)>;
+
+/*! \brief Deep comparison to check if two IR ASTs are equivalent */
+class TensorizeComparator : public ExprComparator, public StmtComparator {
+ public:
+ /*!
+ * \brief Constructor of TensorizeComparator
+ * \param assert_mode Whether to raise an error if the two IR ASTs do not match.
+ */
+ explicit TensorizeComparator(bool assert_mode = true) : assert_mode_(assert_mode) {}
+
+ // Map from rhs buffer to lhs buffer
+ std::unordered_map<Buffer, Buffer, ObjectHash, ObjectEqual> rhs_buffer_map_;
Review comment:
They are not :-(
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
Review comment:
```suggestion
if (op->thread_binding.defined() != rhs->thread_binding.defined()) return false;
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785351315
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
Review comment:
nit: Let's make the logic more concise here
```suggestion
return BlockRealize(/*iter_values=*/std::move(extractor.outer_bindings),
/*predicate=*/std::move(predicate),
/*block=*/
Block(/*iter_vars=*/std::move(extractor.outer_iter_vars), //
/*reads=*/std::move(new_reads), //
/*writes=*/std::move(new_writes), //
/*name_hint=*/block->name_hint + "_o", //
/*body=*/std::move(outer_block_body), //
/*init=*/std::move(new_init)));
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785359141
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
+ const Block& desc_block = desc_block_realize->block;
+ const auto* impl_block_realize =
+ Downcast<BlockRealize>(intrinsic->implementation->body)->block->body.as<BlockRealizeNode>();
+ Block impl_block = impl_block_realize->block;
Review comment:
Let's use a helper function on to extract the body of tensor intrin's desc/impl
```suggestion
const BlockRealizeNode* desc_block_realize = GetTensorIntrinBody(intrinsic->description);
const BlockRealizeNode* impl_block_realize = GetTensorIntrinBody(intrinsic->implementation);
Block desc_block = desc_block_realize->block;
Block impl_block = impl_block_realize->block;
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785358784
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
Review comment:
looks like both desc&impl of the tensor intrin is the root block's children, which is another block. Is there any specific reason that we don't use the root block directly?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] vinx13 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
vinx13 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r788220637
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
+using StmtComparator = StmtFunctor<bool(const Stmt& n, const Stmt& other)>;
+
+/*! \brief Deep comparison to check if two IR ASTs are equivalent */
+class TensorizeComparator : public ExprComparator, public StmtComparator {
+ public:
+ /*!
+ * \brief Constructor of TensorizeComparator
+ * \param assert_mode Whether to raise an error if the two IR ASTs do not match.
+ */
+ explicit TensorizeComparator(bool assert_mode = true) : assert_mode_(assert_mode) {}
+
+ // Map from rhs buffer to lhs buffer
+ std::unordered_map<Buffer, Buffer, ObjectHash, ObjectEqual> rhs_buffer_map_;
+ // Buffer indices mapping
+ std::unordered_map<Buffer, std::vector<PrimExpr>, ObjectPtrHash, ObjectPtrEqual> buffer_indices_;
+ std::vector<IterVar> extra_block_vars_;
+ // variable remap if any
+ std::unordered_map<ObjectRef, ObjectRef, ObjectPtrHash, ObjectPtrEqual> equal_map_;
+
+ bool VisitExpr(const PrimExpr& n, const PrimExpr& other) override;
+ bool VisitStmt(const Stmt& n, const Stmt& other) override;
+
+ bool VisitStmt_(const ForNode* op, const Stmt& other) override;
+ bool VisitStmt_(const SeqStmtNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BufferStoreNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockRealizeNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockNode* op, const Stmt& other) override;
+
+ bool VisitExpr_(const AddNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const SubNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const MulNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const DivNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const ModNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const EQNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const NENode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const LTNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const LENode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const GTNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const GENode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const AndNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const OrNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const MinNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const MaxNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const FloorDivNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const FloorModNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const IntImmNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const FloatImmNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const CastNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const VarNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const BufferLoadNode* op, const PrimExpr& other) override;
+
+ bool DefEqual(const ObjectRef& lhs, const ObjectRef& rhs);
+ virtual bool CompareBuffer(const Buffer& lhs, const Buffer& rhs);
+ bool CompareBufferRegion(const BufferRegion& lhs, const BufferRegion& rhs);
+ bool CompareAnnotation(const std::pair<String, ObjectRef>& lhs,
+ const std::pair<String, ObjectRef>& rhs);
+ bool CompareAnnotationMap(const Map<String, ObjectRef>& lhs, const Map<String, ObjectRef>& rhs);
+ template <typename T>
+ bool CompareBufferAccess(const T* lhs, const T* rhs);
+ template <typename T, typename F>
+ bool CompareArray(const Array<T>& lhs, const Array<T>& rhs, F cmp);
+ bool CompareRange(const Range& lhs, const Range& rhs);
+ bool CompareType(const DataType& lhs, const DataType& rhs);
+
+ protected:
Review comment:
there might be some use cases to subclass this class, like auto tensorization
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] Hzfengsy commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
Hzfengsy commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780634559
##########
File path: include/tvm/tir/function.h
##########
@@ -187,6 +187,56 @@ class LinkedParam : public ObjectRef {
TVM_DEFINE_OBJECT_REF_COW_METHOD(LinkedParamNode);
};
+/*!
+ * \brief Tensor intrinsics for tensorization
+ */
+class TensorIntrinNode : public Object {
+ public:
+ /*! \brief The function to describe the computation. */
+ PrimFunc description;
+ /*! \brief The intrinsic function for lower-level implementation. */
+ PrimFunc implementation;
Review comment:
Vote for `desc` and `impl`
##########
File path: include/tvm/tir/schedule/schedule.h
##########
@@ -465,6 +465,25 @@ class ScheduleNode : public runtime::Object {
*/
virtual void SetScope(const BlockRV& block_rv, int buffer_index, const String& storage_scope) = 0;
/******** Schedule: Blockize & Tensorize ********/
+ /*!
+ * \brief Convert the subtree rooted at a specific loop into a block.
+ * \param loop_rv the root of the subtree
+ * \return the new block
+ */
+ virtual BlockRV Blockize(const LoopRV& loop_rv) = 0;
+ /*!
+ * \brief Tensorize the computation enclosed by loop with tensor_intrin
+ * \param loop_rv the loop to be tensorized
+ * \param intrin the tensor intrinsic
+ */
+ virtual void Tensorize(const LoopRV& loop_rv, const TensorIntrin& intrin) = 0;
+ /*!
+ * \brief Tensorize the computation enclosed by loop with tensor_intrin
+ * \param loop_rv The loop to be tensorized
+ * \param intrin_name Name of the tensor intrinsic
+ */
+ virtual void Tensorize(const LoopRV& loop_rv, const String& intrin_name) = 0;
Review comment:
Should we add a tensorize API for a block? i.e. Tensorize it if we manually blockized before.
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
+using StmtComparator = StmtFunctor<bool(const Stmt& n, const Stmt& other)>;
+
+/*! \brief Deep comparison to check if two IR ASTs are equivalent */
+class TensorizeComparator : public ExprComparator, public StmtComparator {
+ public:
+ /*!
+ * \brief Constructor of TensorizeComparator
+ * \param assert_mode Whether to raise an error if the two IR ASTs do not match.
+ */
+ explicit TensorizeComparator(bool assert_mode = true) : assert_mode_(assert_mode) {}
+
+ // Map from rhs buffer to lhs buffer
+ std::unordered_map<Buffer, Buffer, ObjectHash, ObjectEqual> rhs_buffer_map_;
Review comment:
Keep them private if they are internal vars
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
+using StmtComparator = StmtFunctor<bool(const Stmt& n, const Stmt& other)>;
+
+/*! \brief Deep comparison to check if two IR ASTs are equivalent */
+class TensorizeComparator : public ExprComparator, public StmtComparator {
+ public:
+ /*!
+ * \brief Constructor of TensorizeComparator
+ * \param assert_mode Whether to raise an error if the two IR ASTs do not match.
+ */
+ explicit TensorizeComparator(bool assert_mode = true) : assert_mode_(assert_mode) {}
+
+ // Map from rhs buffer to lhs buffer
+ std::unordered_map<Buffer, Buffer, ObjectHash, ObjectEqual> rhs_buffer_map_;
+ // Buffer indices mapping
+ std::unordered_map<Buffer, std::vector<PrimExpr>, ObjectPtrHash, ObjectPtrEqual> buffer_indices_;
+ std::vector<IterVar> extra_block_vars_;
+ // variable remap if any
+ std::unordered_map<ObjectRef, ObjectRef, ObjectPtrHash, ObjectPtrEqual> equal_map_;
+
+ bool VisitExpr(const PrimExpr& n, const PrimExpr& other) override;
+ bool VisitStmt(const Stmt& n, const Stmt& other) override;
+
+ bool VisitStmt_(const ForNode* op, const Stmt& other) override;
+ bool VisitStmt_(const SeqStmtNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BufferStoreNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockRealizeNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockNode* op, const Stmt& other) override;
Review comment:
Yes, only parts of types are supported now. We may add the rest when needed
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
+using StmtComparator = StmtFunctor<bool(const Stmt& n, const Stmt& other)>;
+
+/*! \brief Deep comparison to check if two IR ASTs are equivalent */
+class TensorizeComparator : public ExprComparator, public StmtComparator {
+ public:
+ /*!
+ * \brief Constructor of TensorizeComparator
+ * \param assert_mode Whether to raise an error if the two IR ASTs do not match.
+ */
+ explicit TensorizeComparator(bool assert_mode = true) : assert_mode_(assert_mode) {}
+
+ // Map from rhs buffer to lhs buffer
+ std::unordered_map<Buffer, Buffer, ObjectHash, ObjectEqual> rhs_buffer_map_;
+ // Buffer indices mapping
+ std::unordered_map<Buffer, std::vector<PrimExpr>, ObjectPtrHash, ObjectPtrEqual> buffer_indices_;
+ std::vector<IterVar> extra_block_vars_;
+ // variable remap if any
+ std::unordered_map<ObjectRef, ObjectRef, ObjectPtrHash, ObjectPtrEqual> equal_map_;
+
+ bool VisitExpr(const PrimExpr& n, const PrimExpr& other) override;
+ bool VisitStmt(const Stmt& n, const Stmt& other) override;
+
+ bool VisitStmt_(const ForNode* op, const Stmt& other) override;
+ bool VisitStmt_(const SeqStmtNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BufferStoreNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockRealizeNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockNode* op, const Stmt& other) override;
+
+ bool VisitExpr_(const AddNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const SubNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const MulNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const DivNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const ModNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const EQNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const NENode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const LTNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const LENode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const GTNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const GENode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const AndNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const OrNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const MinNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const MaxNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const FloorDivNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const FloorModNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const IntImmNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const FloatImmNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const CastNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const VarNode* op, const PrimExpr& other) override;
+ bool VisitExpr_(const BufferLoadNode* op, const PrimExpr& other) override;
+
+ bool DefEqual(const ObjectRef& lhs, const ObjectRef& rhs);
+ virtual bool CompareBuffer(const Buffer& lhs, const Buffer& rhs);
+ bool CompareBufferRegion(const BufferRegion& lhs, const BufferRegion& rhs);
+ bool CompareAnnotation(const std::pair<String, ObjectRef>& lhs,
+ const std::pair<String, ObjectRef>& rhs);
+ bool CompareAnnotationMap(const Map<String, ObjectRef>& lhs, const Map<String, ObjectRef>& rhs);
+ template <typename T>
+ bool CompareBufferAccess(const T* lhs, const T* rhs);
+ template <typename T, typename F>
+ bool CompareArray(const Array<T>& lhs, const Array<T>& rhs, F cmp);
+ bool CompareRange(const Range& lhs, const Range& rhs);
+ bool CompareType(const DataType& lhs, const DataType& rhs);
+
+ protected:
Review comment:
Why `protected`?
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
Review comment:
Can we use `ScheduleError` here? Since it has a better error renderer
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
+
+ // check both functions' bodies are directly block
+ const auto* desc_realize =
+ Downcast<BlockRealize>(desc_func->body)->block->body.as<BlockRealizeNode>();
+ const auto* intrin_realize =
+ Downcast<BlockRealize>(intrin_func->body)->block->body.as<BlockRealizeNode>();
+ CHECK(desc_realize != nullptr) << "description function's body expect a directly block";
+ CHECK(intrin_realize != nullptr) << "intrinsic function's body expect a directly block";
+
+ const Block& desc_block = desc_realize->block;
+ const Block& intrin_block = intrin_realize->block;
+
+ // check block var number and iter type
+ CHECK_EQ(desc_block->iter_vars.size(), intrin_block->iter_vars.size())
+ << "Two blocks should have the same number of block vars";
+ for (size_t i = 0; i < desc_block->iter_vars.size(); i++) {
+ const IterVar& desc_var = desc_block->iter_vars[i];
+ const IterVar& intrin_var = intrin_block->iter_vars[i];
+ CHECK(desc_var->iter_type == intrin_var->iter_type)
+ << "Block iter_type mismatch between " << desc_var->iter_type << " and "
+ << intrin_var->iter_type;
+ }
+
+ auto n = make_object<TensorIntrinNode>();
Review comment:
The Type `TensorIntrinNode` is embedded in `make_object`. Personally think `auto` is OK here.
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
Review comment:
Do we need all params are Buffers? (To be specific Buffer Handles)
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
Review comment:
+1
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780718594
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
+ }
+ };
+ std::tie(lhs_expr, lhs_iter) = detect(op->iter_values[i + offset]);
+ std::tie(rhs_expr, rhs_iter) = detect(rhs->iter_values[i]);
+ CHECK((lhs_iter && rhs_iter) || (!lhs_iter && !rhs_iter)) << "Incompatible binding";
+ if (lhs_iter) VisitExpr(lhs_iter.value(), rhs_iter.value());
+ if (is_zero(rhs_expr)) {
+ CHECK(is_zero(lhs_expr)) << "Incompatible binding";
+ } else {
+ const auto* bv = rhs_expr.as<VarNode>();
+ if (!bv) {
+ VisitExpr(lhs_expr, rhs_expr);
+ } else {
+ auto it = equal_map_.find(GetRef<Var>(bv));
+ if (it == equal_map_.end()) {
+ equal_map_[GetRef<Var>(bv)] = lhs_expr;
+ } else {
+ CHECK(it->second->IsInstance<PrimExprNode>());
+ VisitExpr(lhs_expr, Downcast<PrimExpr>(it->second));
+ }
+ }
+ }
Review comment:
we should probably restructure this a little bit
```suggestion
} else if (const auto* bv = rhs_expr.as<VarNode>()) {
auto it = equal_map_.find(GetRef<Var>(bv));
if (it == equal_map_.end()) {
equal_map_[GetRef<Var>(bv)] = lhs_expr;
} else {
CHECK(it->second->IsInstance<PrimExprNode>());
VisitExpr(lhs_expr, Downcast<PrimExpr>(it->second));
}
} else {
VisitExpr(lhs_expr, rhs_expr);
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780717860
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
Review comment:
be more cautious when subtracting size_t:
```suggestion
int offset = static_cast<int>(op->iter_values.size()) - static_cast<int>(rhs->iter_values.size());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780719575
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
+ }
+ };
+ std::tie(lhs_expr, lhs_iter) = detect(op->iter_values[i + offset]);
+ std::tie(rhs_expr, rhs_iter) = detect(rhs->iter_values[i]);
+ CHECK((lhs_iter && rhs_iter) || (!lhs_iter && !rhs_iter)) << "Incompatible binding";
+ if (lhs_iter) VisitExpr(lhs_iter.value(), rhs_iter.value());
+ if (is_zero(rhs_expr)) {
+ CHECK(is_zero(lhs_expr)) << "Incompatible binding";
+ } else {
+ const auto* bv = rhs_expr.as<VarNode>();
+ if (!bv) {
+ VisitExpr(lhs_expr, rhs_expr);
+ } else {
+ auto it = equal_map_.find(GetRef<Var>(bv));
+ if (it == equal_map_.end()) {
+ equal_map_[GetRef<Var>(bv)] = lhs_expr;
+ } else {
+ CHECK(it->second->IsInstance<PrimExprNode>());
+ VisitExpr(lhs_expr, Downcast<PrimExpr>(it->second));
+ }
+ }
+ }
+ }
+ } else {
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ if (!VisitExpr(op->iter_values[i + offset], rhs->iter_values[i])) return false;
+ }
+ const Block& block = op->block;
+ for (size_t i = 0; i < offset; ++i) {
+ Var block_var = Downcast<Var>(op->iter_values[i]);
+ auto it = equal_map_.find(block_var);
+ equal_map_[block->iter_vars[i]->var] = (it == equal_map_.end() ? block_var : it->second);
+ }
+ }
+ }
+
+ return VisitExpr(op->predicate, rhs->predicate) && VisitStmt(op->block, rhs->block);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockNode>();
+ // Check block equality.
+ // All iter vars and buffer regions including the order shoudl match.
+ // When checking iter vars, DefEqual is used to remap variables.
+ // Only the inner most several axis are compared. Other iter vars are added to extra_block_vars.
+ if (op->iter_vars.size() < rhs->iter_vars.size()) return false;
+
+ size_t offset = op->iter_vars.size() - rhs->iter_vars.size();
+ for (size_t i = 0; i < rhs->iter_vars.size(); ++i) {
+ auto lhs_var = op->iter_vars[i + offset], rhs_var = rhs->iter_vars[i];
+ // Skip iter dom
+ if (!DefEqual(lhs_var->var, rhs_var->var)) {
+ return false;
+ }
+ if (lhs_var->iter_type != rhs_var->iter_type) {
+ return false;
+ }
+ }
+
+ if (is_scope_block) {
+ for (size_t i = 0; i < offset; ++i) {
+ extra_block_vars_.push_back(op->iter_vars[i]);
+ }
+ }
+
+ if (!is_scope_block) {
+ if (!CompareArray(op->writes, rhs->writes, &TensorizeComparator::CompareBufferRegion)) {
+ return false;
+ }
+ if (!CompareArray(op->reads, rhs->reads, &TensorizeComparator::CompareBufferRegion)) {
+ return false;
+ }
+ if (!CompareAnnotationMap(op->annotations, rhs->annotations)) {
+ return false;
+ }
+ if (!CompareArray(op->alloc_buffers, rhs->alloc_buffers, &TensorizeComparator::CompareBuffer)) {
+ return false;
+ }
+ }
+ is_scope_block = false;
+ return VisitStmt(op->body, rhs->body);
+}
+
+// Exprs
+#define TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(OpName) \
+ bool TensorizeComparator::VisitExpr_(const OpName* op, const PrimExpr& other) { \
+ const auto* rhs = other.as<OpName>(); \
+ return VisitExpr(op->a, rhs->a) && VisitExpr(op->b, rhs->b); \
+ }
+
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(AddNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(SubNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MulNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(DivNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(ModNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(EQNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(NENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(LTNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(LENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(GTNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(GENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(AndNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(OrNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MinNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MaxNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(FloorDivNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(FloorModNode);
+
+bool TensorizeComparator::VisitExpr_(const IntImmNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<IntImmNode>();
+ return CompareType(op->dtype, rhs->dtype) && op->value == rhs->value;
+}
+
+bool TensorizeComparator::VisitExpr_(const FloatImmNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<FloatImmNode>();
+ return CompareType(op->dtype, rhs->dtype) && op->value == rhs->value;
+}
+
+bool TensorizeComparator::VisitExpr_(const CastNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<CastNode>();
+ return CompareType(op->dtype, rhs->dtype) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitExpr_(const VarNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<VarNode>();
+ auto lhs = GetRef<Var>(op);
+ if (lhs.same_as(other)) return true;
+ if (!CompareType(op->dtype, rhs->dtype)) return false;
+ auto it = equal_map_.find(lhs);
+ return it != equal_map_.end() && it->second.same_as(other);
+}
+
+bool TensorizeComparator::VisitExpr_(const BufferLoadNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<BufferLoadNode>();
+ return CompareBufferAccess(op, rhs);
+}
+
+bool TensorizeComparator::DefEqual(const ObjectRef& lhs, const ObjectRef& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ if (lhs->type_index() != rhs->type_index()) return false;
+ auto it = equal_map_.find(lhs);
+ // If there is already a mapping
+ if (it != equal_map_.end()) return it->second.same_as(rhs);
+ equal_map_[lhs] = rhs;
+ return true;
+}
+
+bool TensorizeComparator::CompareAnnotation(const std::pair<String, ObjectRef>& lhs,
+ const std::pair<String, ObjectRef>& rhs) {
+ if (lhs.first != rhs.first) return false;
+ if (!lhs.second.same_as(rhs.second)) return false;
+ return VisitExpr(Downcast<PrimExpr>(lhs.second), Downcast<PrimExpr>(rhs.second));
+}
+
+bool TensorizeComparator::CompareAnnotationMap(const Map<String, ObjectRef>& lhs,
+ const Map<String, ObjectRef>& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ if (lhs.size() != rhs.size()) return false;
+
+ auto sort_map =
+ [](const Map<String, ObjectRef>& map) -> std::vector<std::pair<String, ObjectRef>> {
+ std::vector<std::pair<String, ObjectRef>> ret;
+ ret.reserve(map.size());
+ for (const auto& pair : map) {
+ ret.emplace_back(pair);
+ }
+ sort(ret.begin(), ret.end());
+ return ret;
+ };
+
+ auto lhs_array = sort_map(lhs), rhs_array = sort_map(rhs);
Review comment:
nit: no need to abuse auto
```suggestion
std::vector<std::pair<String, ObjectRef>> lhs_array = sort_map(lhs);
std::vector<std::pair<String, ObjectRef>> rhs_array = sort_map(rhs);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780716472
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
Review comment:
Just to double confirm, it's guaranteed that `(rhs != nullptr)` in `TensorizeComparator::VisitStmt` right?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780718288
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
Review comment:
Just check if binding is a `Var`
```suggestion
if (const auto* var = binding.as<VarNode>()) {
return std::make_pair(0, GetRef<Var>(var));
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780719473
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
+ }
+ };
+ std::tie(lhs_expr, lhs_iter) = detect(op->iter_values[i + offset]);
+ std::tie(rhs_expr, rhs_iter) = detect(rhs->iter_values[i]);
+ CHECK((lhs_iter && rhs_iter) || (!lhs_iter && !rhs_iter)) << "Incompatible binding";
+ if (lhs_iter) VisitExpr(lhs_iter.value(), rhs_iter.value());
+ if (is_zero(rhs_expr)) {
+ CHECK(is_zero(lhs_expr)) << "Incompatible binding";
+ } else {
+ const auto* bv = rhs_expr.as<VarNode>();
+ if (!bv) {
+ VisitExpr(lhs_expr, rhs_expr);
+ } else {
+ auto it = equal_map_.find(GetRef<Var>(bv));
+ if (it == equal_map_.end()) {
+ equal_map_[GetRef<Var>(bv)] = lhs_expr;
+ } else {
+ CHECK(it->second->IsInstance<PrimExprNode>());
+ VisitExpr(lhs_expr, Downcast<PrimExpr>(it->second));
+ }
+ }
+ }
+ }
+ } else {
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ if (!VisitExpr(op->iter_values[i + offset], rhs->iter_values[i])) return false;
+ }
+ const Block& block = op->block;
+ for (size_t i = 0; i < offset; ++i) {
+ Var block_var = Downcast<Var>(op->iter_values[i]);
+ auto it = equal_map_.find(block_var);
+ equal_map_[block->iter_vars[i]->var] = (it == equal_map_.end() ? block_var : it->second);
+ }
+ }
+ }
+
+ return VisitExpr(op->predicate, rhs->predicate) && VisitStmt(op->block, rhs->block);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockNode>();
+ // Check block equality.
+ // All iter vars and buffer regions including the order shoudl match.
+ // When checking iter vars, DefEqual is used to remap variables.
+ // Only the inner most several axis are compared. Other iter vars are added to extra_block_vars.
+ if (op->iter_vars.size() < rhs->iter_vars.size()) return false;
+
+ size_t offset = op->iter_vars.size() - rhs->iter_vars.size();
+ for (size_t i = 0; i < rhs->iter_vars.size(); ++i) {
+ auto lhs_var = op->iter_vars[i + offset], rhs_var = rhs->iter_vars[i];
+ // Skip iter dom
+ if (!DefEqual(lhs_var->var, rhs_var->var)) {
+ return false;
+ }
+ if (lhs_var->iter_type != rhs_var->iter_type) {
+ return false;
+ }
+ }
+
+ if (is_scope_block) {
+ for (size_t i = 0; i < offset; ++i) {
+ extra_block_vars_.push_back(op->iter_vars[i]);
+ }
+ }
+
+ if (!is_scope_block) {
+ if (!CompareArray(op->writes, rhs->writes, &TensorizeComparator::CompareBufferRegion)) {
+ return false;
+ }
+ if (!CompareArray(op->reads, rhs->reads, &TensorizeComparator::CompareBufferRegion)) {
+ return false;
+ }
+ if (!CompareAnnotationMap(op->annotations, rhs->annotations)) {
+ return false;
+ }
+ if (!CompareArray(op->alloc_buffers, rhs->alloc_buffers, &TensorizeComparator::CompareBuffer)) {
+ return false;
+ }
+ }
+ is_scope_block = false;
+ return VisitStmt(op->body, rhs->body);
+}
+
+// Exprs
+#define TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(OpName) \
+ bool TensorizeComparator::VisitExpr_(const OpName* op, const PrimExpr& other) { \
+ const auto* rhs = other.as<OpName>(); \
+ return VisitExpr(op->a, rhs->a) && VisitExpr(op->b, rhs->b); \
+ }
+
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(AddNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(SubNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MulNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(DivNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(ModNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(EQNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(NENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(LTNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(LENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(GTNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(GENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(AndNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(OrNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MinNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MaxNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(FloorDivNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(FloorModNode);
+
+bool TensorizeComparator::VisitExpr_(const IntImmNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<IntImmNode>();
+ return CompareType(op->dtype, rhs->dtype) && op->value == rhs->value;
+}
+
+bool TensorizeComparator::VisitExpr_(const FloatImmNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<FloatImmNode>();
+ return CompareType(op->dtype, rhs->dtype) && op->value == rhs->value;
+}
+
+bool TensorizeComparator::VisitExpr_(const CastNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<CastNode>();
+ return CompareType(op->dtype, rhs->dtype) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitExpr_(const VarNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<VarNode>();
+ auto lhs = GetRef<Var>(op);
+ if (lhs.same_as(other)) return true;
+ if (!CompareType(op->dtype, rhs->dtype)) return false;
+ auto it = equal_map_.find(lhs);
+ return it != equal_map_.end() && it->second.same_as(other);
+}
+
+bool TensorizeComparator::VisitExpr_(const BufferLoadNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<BufferLoadNode>();
+ return CompareBufferAccess(op, rhs);
+}
+
+bool TensorizeComparator::DefEqual(const ObjectRef& lhs, const ObjectRef& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ if (lhs->type_index() != rhs->type_index()) return false;
+ auto it = equal_map_.find(lhs);
+ // If there is already a mapping
+ if (it != equal_map_.end()) return it->second.same_as(rhs);
+ equal_map_[lhs] = rhs;
+ return true;
+}
+
+bool TensorizeComparator::CompareAnnotation(const std::pair<String, ObjectRef>& lhs,
+ const std::pair<String, ObjectRef>& rhs) {
+ if (lhs.first != rhs.first) return false;
+ if (!lhs.second.same_as(rhs.second)) return false;
+ return VisitExpr(Downcast<PrimExpr>(lhs.second), Downcast<PrimExpr>(rhs.second));
Review comment:
looks like the logic here is a bit convoluted...because if `lhs.second == rhs.second` then we don't need to recursively do VisitExpr. Do you want to double check?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780727366
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
Review comment:
Alternatively, there is a cooler way that could potentially make our code more concise:
```C++
auto make_uses_var = [](const Array<Var>& vars) -> std::function<bool(const PrimExpr& expr)> {
std::unordered_set<const VarNode*> var_set;
var_set.reserve(vars.size());
for (const Var& var : vars) {
var_set.insert(var.get());
}
return [var_set = std::move(var_set)](const PrimExpr& expr) -> bool {
return UsesVar(expr, [&var_set](const VarNode* var) {
return var_set.count(var); //
});
};
};
auto use_outer_loop_vars = make_uses_var(outer_iters);
auto use_inner_loop_vars = make_uses_var(inner_iters);
```
By doing so, we can replace the calculation of `outer` and `inner` with:
```C++
bool outer = use_outer_loop_vars(bindings[i]);
bool inner = use_inner_loop_vars(bindings[i]);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780742469
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
Review comment:
should we add a "break" here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780589725
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
+
+ // check both functions' bodies are directly block
+ const auto* desc_realize =
+ Downcast<BlockRealize>(desc_func->body)->block->body.as<BlockRealizeNode>();
+ const auto* intrin_realize =
+ Downcast<BlockRealize>(intrin_func->body)->block->body.as<BlockRealizeNode>();
+ CHECK(desc_realize != nullptr) << "description function's body expect a directly block";
+ CHECK(intrin_realize != nullptr) << "intrinsic function's body expect a directly block";
+
+ const Block& desc_block = desc_realize->block;
+ const Block& intrin_block = intrin_realize->block;
+
+ // check block var number and iter type
+ CHECK_EQ(desc_block->iter_vars.size(), intrin_block->iter_vars.size())
+ << "Two blocks should have the same number of block vars";
+ for (size_t i = 0; i < desc_block->iter_vars.size(); i++) {
+ const IterVar& desc_var = desc_block->iter_vars[i];
+ const IterVar& intrin_var = intrin_block->iter_vars[i];
+ CHECK(desc_var->iter_type == intrin_var->iter_type)
+ << "Block iter_type mismatch between " << desc_var->iter_type << " and "
+ << intrin_var->iter_type;
+ }
+
+ auto n = make_object<TensorIntrinNode>();
+ n->description = std::move(desc_func);
+ n->implementation = std::move(intrin_func);
+ data_ = std::move(n);
+}
+
+TensorIntrin TensorIntrin::Register(String name, PrimFunc desc_func, PrimFunc intrin_func) {
+ TensorIntrinManager* manager = TensorIntrinManager::Global();
+ ICHECK_EQ(manager->reg.count(name), 0)
+ << "ValueError: TensorIntrin '" << name << "' has already been registered";
+ TensorIntrin intrin(desc_func, intrin_func);
+ manager->reg.Set(name, intrin);
+ return intrin;
+}
+
+TensorIntrin TensorIntrin::Get(String name) {
+ const TensorIntrinManager* manager = TensorIntrinManager::Global();
+ ICHECK_EQ(manager->reg.count(name), 1)
+ << "ValueError: TensorIntrin '" << name << "' is not registered";
Review comment:
ditto
```suggestion
CHECK_EQ(manager->reg.count(name), 1)
<< "ValueError: TensorIntrin '" << name << "' is not registered";
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780598498
##########
File path: src/tir/schedule/state.cc
##########
@@ -201,9 +201,8 @@ class BlockInfoCollector : private StmtVisitor {
bool is_root_block = srefs_.empty();
// Calculate `BlockInfo::scope`
Array<StmtSRef> child_block_srefs = std::move(block_frames_.back());
- BlockInfo& info =
- self_->block_info.emplace(scope_root, BlockInfo(BlockScope(child_block_srefs)))
- .first->second;
+ self_->block_info[scope_root] = BlockInfo(BlockScope(child_block_srefs));
+ BlockInfo& info = self_->block_info[scope_root];
Review comment:
Oh I see - that makes sense to me! Then to save one look-up, let's do
```suggestion
BlockInfo& info = self_->block_info[scope_root] = BlockInfo(BlockScope(child_block_srefs));
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] vinx13 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
vinx13 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780598226
##########
File path: src/tir/schedule/state.cc
##########
@@ -201,9 +201,8 @@ class BlockInfoCollector : private StmtVisitor {
bool is_root_block = srefs_.empty();
// Calculate `BlockInfo::scope`
Array<StmtSRef> child_block_srefs = std::move(block_frames_.back());
- BlockInfo& info =
- self_->block_info.emplace(scope_root, BlockInfo(BlockScope(child_block_srefs)))
- .first->second;
+ self_->block_info[scope_root] = BlockInfo(BlockScope(child_block_srefs));
+ BlockInfo& info = self_->block_info[scope_root];
Review comment:
`emplace` will not overwrite the old entry if the key exists. This function may be called in `UpdateScopeBlockInfo`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785346551
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
Review comment:
Use `IterSumExprNode` instead to be more accurate. `TVM_TYPE_AS` could help with type checking BTW.
```suggestion
const auto* outer_binding = TVM_TYPE_AS(outer_binding, division[i][0]->source, arith::IterSumExprNode);
const auto* inner_binding = TVM_TYPE_AS(inner_binding, division[i][1]->source, arith::IterSumExprNode);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] vinx13 commented on pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
vinx13 commented on pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#issuecomment-1018005745
comments have been addressed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [WIP][TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r788408997
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
+ const Block& desc_block = desc_block_realize->block;
+ const auto* impl_block_realize =
+ Downcast<BlockRealize>(intrinsic->implementation->body)->block->body.as<BlockRealizeNode>();
+ Block impl_block = impl_block_realize->block;
+
+ // Step 1: Blockize the subtree rooted at the given loop
+ const StmtSRef& block_sref = Blockize(self, loop_sref);
+ const BlockRealize& block_realize = GetBlockRealize(self, block_sref);
+
+ // Step 2: Compare the block with the description of the tensor intrinsic, find the correspondence
+ // between buffers in the block and the description.
+ TensorizeComparator comparator(/*assert_mode=*/true);
+ comparator.VisitStmt(block_realize, GetRef<Stmt>(desc_block_realize));
+
+ // Step 3: Find the correspondence between buffers in the current AST and the implementation of
+ // the tensor intrinsic
+
+ // Step 3.1: Map from intrinsic func buffer to description func buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> intrin_buffer_map;
+ RemapTensorIntrinBuffers(intrinsic, &intrin_buffer_map);
+ // Step 3.2: Map form intrinsic func buffer to current AST buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> buffer_map;
+ for (const auto& pair : intrin_buffer_map) {
+ auto it = comparator.rhs_buffer_map_.find(pair.second);
+ ICHECK(it != comparator.rhs_buffer_map_.end());
+ buffer_map[pair.first] = it->second;
+ }
+
+ // Step 4: Create MatchBufferRegion for the params of the implementation function of the tensor
+ // intrin to make them subregions of the buffer in the original IR.
+ std::unordered_map<Buffer, Array<Range>, ObjectPtrHash, ObjectPtrEqual> buffer_region_map;
+ for (const auto& read : impl_block->reads) {
+ buffer_region_map.emplace(read->buffer, read->region);
+ }
+ for (const auto& write : impl_block->writes) {
+ buffer_region_map.emplace(write->buffer, write->region);
+ }
+ Array<MatchBufferRegion> match_buffer_regions;
+ for (size_t i = 0; i < intrinsic->implementation->params.size(); ++i) {
+ const auto& param = intrinsic->implementation->params[i];
+ const auto& buffer = intrinsic->implementation->buffer_map.at(param);
+ const auto& source = buffer_map.at(buffer);
+ Region region = buffer_region_map.at(buffer);
+ auto extra_indices = comparator.buffer_indices_.at(source);
+ std::vector<Range> extra_buffer_ranges;
+ std::transform(extra_indices.begin(), extra_indices.end(),
+ std::back_inserter(extra_buffer_ranges),
+ [](const PrimExpr& index) { return Range::FromMinExtent(index, 1); });
+ region.insert(region.begin(), extra_buffer_ranges.begin(), extra_buffer_ranges.end());
+ match_buffer_regions.push_back(MatchBufferRegion(buffer, BufferRegion(source, region)));
+ }
+
+ // Step 5: Replace the subtree in the original IR with the tensor intrin implementation.
+ ObjectPtr<BlockNode> new_block_ptr = make_object<BlockNode>(*block_realize->block.get());
+ new_block_ptr->body = impl_block->body;
+ ICHECK(new_block_ptr->match_buffers.empty());
+ new_block_ptr->match_buffers = std::move(match_buffer_regions);
+ Block new_block(new_block_ptr);
+
+ // Substitution map for the intrin block to get the correct bindings.
+ Map<Var, PrimExpr> bv_map;
+ for (size_t i = 0; i < desc_block->iter_vars.size(); ++i) {
+ auto it = comparator.equal_map_.find(desc_block->iter_vars[i]->var);
+ if (it != comparator.equal_map_.end()) {
+ bv_map.Set(impl_block->iter_vars[i]->var, Downcast<PrimExpr>(it->second));
+ } else {
+ bv_map.Set(impl_block->iter_vars[i]->var, Integer(0));
+ }
+ }
+ new_block = Downcast<Block>(SubstituteInScope(new_block, bv_map));
+ self->Replace(block_sref, new_block, {{block_realize->block, new_block}});
Review comment:
I think you didn't get my ideas, but it's not high priority to do (no action needed in this PR), so I'm just curious.
Given we might care about the "exception-safety" of TIR schedule, i.e. if a schedule primitive fails, then we might want the IR to be unchanged. So in our particular case, during tensorization, if `Blockize` works, but the subsequence operation in `Tensorize` fails, then will it leave the IR changed?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785354878
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
Review comment:
```suggestion
const Var& lhs_var = intrinsic->description->params[i];
const Buffer& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
const Var& rhs_var = intrinsic->implementation->params[i];
const Buffer& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] wrongtest commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
wrongtest commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780873056
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
Review comment:
Would be `std::make_pair(binding, NullOpt);` since first + second == binding?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] vinx13 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
vinx13 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r788221107
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
Review comment:
Yes
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] vinx13 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
vinx13 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r788239076
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
Review comment:
this is how `SubspaceDivide` is implemented in affine analysis https://github.com/apache/tvm/blob/main/include/tvm/arith/iter_affine_map.h#L344
the last element of the array is the predicates, the extent here has a different meaning
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] vinx13 commented on a change in pull request #9871: [WIP][TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
vinx13 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r789123327
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
Review comment:
Updated, we are using root block directly. Previously this was because we need to define root iter vars (the parser will create another block as direct child of root block, because root block can't have iter vars)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780642764
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
Review comment:
```suggestion
if (op->thread_binding.defined() != rhs->thread_binding.defined()) return false;
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780588745
##########
File path: python/tvm/tir/schedule/schedule.py
##########
@@ -1733,6 +1733,258 @@ def after_set_scope(
########## Schedule: Blockize & Tensorize ##########
+ @type_checked
+ def blockize(self, loop: LoopRV) -> BlockRV:
+ """Convert the subtree rooted at a specific loop into a block.
+
+ Parameters
+ ----------
+ loop : LoopRV
+ The root of the subtree.
+
+ Returns
+ -------
+ result : BlockRV
+ The new block.
+
+ Examples
+ --------
+
+ Before blockize, in TensorIR, the IR is:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def before_blockize(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"]
+ ) -> None:
+ for i_0, j_0, i_1, j_1 in T.grid(8, 8, 16, 16):
+ with T.block("B"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1)
+ T.reads(A[vi, vj])
+ T.writes(B[vi, vj])
+ B[vi, vj] = A[vi, vj] * T.float32(2)
+
+ Create the schedule and do set_scope:
+
+ .. code-block:: python
+
+ sch = tir.Schedule(before_blockize)
+ B = sch.get_block("B")
+ _, _, i1, _ = sch.get_loops(B)
+ sch.blockize(i1)
+ print(sch.mod["main"].script())
+
+ After applying blockize, the IR becomes:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def after_blockize(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"]
+ )-> None:
+ for i_0, j_0 in T.grid(8, 8):
+ with T.block("blockized_B"):
+ vio, vjo = T.axis.remap("SS", [i_0, j_0])
+ T.reads(A[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16])
+ T.writes(B[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16])
+ for i_1, j_1 in T.grid(16, 16):
+ with T.block("B"):
+ vi = T.axis.spatial(128, vio * 16 + i_1)
+ vj = T.axis.spatial(128, vjo * 16 + j_1)
+ T.reads(A[vi, vj])
+ T.writes(B[vi, vj])
+ B[vi, vj] = A[vi, vj] * T.float32(2)
+
+ Note
+ ----
+ blockize requires there is exactly one block under the given loop and the bindings of the
+ block are divisible by the subspace represented by the loops starting at the given loop.
+
+ """
+
+ return _ffi_api.ScheduleBlockize(self, loop) # type: ignore # pylint: disable=no-member
+
+ @type_checked
+ def tensorize(self, loop: LoopRV, tensor_intrin: Union[str, TensorIntrin]) -> None:
+ """Tensorize the computation enclosed by loop with the tensor intrinsic.
+
+ Parameters
+ ----------
+ loop : LoopRV
+ The loop to be tensorized.
+ tensor_intrin : Union[str, TensorIntrin]
+ The tensor intrin or the name of the tensor intrin.
+
+ Examples
+ --------
+
+ Before tensorize, in TensorIR, the IR is:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def before_tensorize(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"],
+ C: T.Buffer[(128, 128), "float32"],
+ ) -> None:
+ # body
+ # with T.block("root")
+ for i_0, j_0 in T.grid(8, 8):
+ for i_1_init, j_1_init in T.grid(16, 16):
+ with T.block("init"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1_init)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1_init)
+ T.reads()
+ T.writes(C[vi, vj])
+ C[vi, vj] = T.float32(0)
+ for k_0, i_1, j_1, k_1 in T.grid(8, 16, 16, 16):
+ with T.block("update"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1)
+ vk = T.axis.reduce(128, k_0 * 16 + k_1)
+ T.reads(C[vi, vj], A[vi, vk], B[vj, vk])
+ T.writes(C[vi, vj])
+ C[vi, vj] = C[vi, vj] + A[vi, vk] * B[vj, vk]
+
+ Declare and register the tensor intrinsic:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def mma_desc(a: T.handle, b: T.handle, c: T.handle) -> None:
+ A = T.match_buffer(a, (16, 16), align=128, offset_factor=1)
+ B = T.match_buffer(b, (16, 16), align=128, offset_factor=1)
+ C = T.match_buffer(c, (16, 16), align=128, offset_factor=1)
+
+ with T.block("root"):
+ vi = T.axis.S(16, 0)
+ vj = T.axis.S(16, 0)
+ vk = T.axis.R(16, 0)
+ for i, j, k in T.grid(16, 16, 16):
+ with T.block("update"):
+ vii = T.axis.S(16, vi + i)
+ vjj = T.axis.S(16, vj + j)
+ vkk = T.axis.R(16, vk + k)
+ C[vii, vjj] = C[vii, vjj] + A[vii, vkk] * B[vjj, vkk]
+
+
+ @T.prim_func
+ def mma_intrin(a: T.handle, b: T.handle, c: T.handle) -> None:
+ A = T.match_buffer(a, (16, 16), align=128, offset_factor=1)
+ B = T.match_buffer(b, (16, 16), align=128, offset_factor=1)
+ C = T.match_buffer(c, (16, 16), align=128, offset_factor=1)
+
+ with T.block("root"):
+ vi = T.axis.S(16, 0)
+ vj = T.axis.S(16, 0)
+ vk = T.axis.R(16, 0)
+ T.reads(
+ [
+ C[vi : vi + 16, vj : vj + 16],
+ A[vi : vi + 16, vk : vk + 16],
+ B[vj : vj + 16, vk : vk + 16],
+ ]
+ )
+ T.writes(C[vi : vi + 16, vj : vj + 16])
+ T.evaluate(
+ T.tvm_mma_sync(
+ C.data,
+ C.elem_offset // 256,
+ A.data,
+ A.elem_offset // 256,
+ B.data,
+ B.elem_offset // 256,
+ C.data,
+ C.elem_offset // 256,
+ dtype="handle",
+ )
+ )
+
+ tir.TensorIntrin.register("test_mma_intrin", mma_desc, mma_intrin)
+
+ Create the schedule and do tensorize:
+
+ .. code-block:: python
+
+ sch = tir.Schedule(before_tensorize)
+ update = s.get_block("update")
+ _, _, _, i1, _, _ = s.get_loops(update)
+ s.tensorize(ii, "test_mma_intrin")
+ print(sch.mod["main"].script())
+
+ After applying tensorize, the IR becomes:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def after_tensoirze(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"],
+ C: T.Buffer[(128, 128), "float32"],
+ ) -> None:
+ # body
+ # with T.block("root")
+ for i_0, j_0 in T.grid(8, 8):
+ for i_1_init, j_1_init in T.grid(16, 16):
+ with T.block("init"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1_init)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1_init)
+ T.reads()
+ T.writes(C[vi, vj])
+ C[vi, vj] = T.float32(0)
+ for k_0 in T.serial(8):
+ with T.block("blockized_update"):
+ vio, vjo, vko = T.axis.remap("SSR", [i_0, j_0, k_0])
+ T.reads(
+ C[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16],
+ A[vio * 16 : vio * 16 + 16, vko * 16 : vko * 16 + 16],
+ B[vjo * 16 : vjo * 16 + 16, vko * 16 : vko * 16 + 16],
+ )
+ T.writes(C[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16])
+ A_1 = T.match_buffer(
+ A[vio * 16 : vio * 16 + 16, vko * 16 : vko * 16 + 16],
+ [16, 16],
+ dtype="float32",
+ offset_factor=1,
+ )
+ B_1 = T.match_buffer(
+ B[vjo * 16 : vjo * 16 + 16, vko * 16 : vko * 16 + 16],
+ [16, 16],
+ dtype="float32",
+ offset_factor=1,
+ )
+ C_1 = T.match_buffer(
+ C[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16],
+ [16, 16],
+ dtype="float32",
+ offset_factor=1,
+ )
+ T.evaluate(
+ T.tvm_mma_sync(
+ C_1.data,
+ C_1.elem_offset // 256,
+ A_1.data,
+ A_1.elem_offset // 256,
+ B_1.data,
+ B_1.elem_offset // 256,
+ C_1.data,
+ C_1.elem_offset // 256,
+ dtype="handle",
+ )
+ )
+
+ """
+ if isinstance(tensor_intrin, str):
+ tensor_intrin = String(tensor_intrin)
Review comment:
Do we need this conversion? I suppose `PackedFuncValueConverter` is able to handle this :-)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780589634
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
+
+ // check both functions' bodies are directly block
+ const auto* desc_realize =
+ Downcast<BlockRealize>(desc_func->body)->block->body.as<BlockRealizeNode>();
+ const auto* intrin_realize =
+ Downcast<BlockRealize>(intrin_func->body)->block->body.as<BlockRealizeNode>();
+ CHECK(desc_realize != nullptr) << "description function's body expect a directly block";
+ CHECK(intrin_realize != nullptr) << "intrinsic function's body expect a directly block";
+
+ const Block& desc_block = desc_realize->block;
+ const Block& intrin_block = intrin_realize->block;
+
+ // check block var number and iter type
+ CHECK_EQ(desc_block->iter_vars.size(), intrin_block->iter_vars.size())
+ << "Two blocks should have the same number of block vars";
+ for (size_t i = 0; i < desc_block->iter_vars.size(); i++) {
+ const IterVar& desc_var = desc_block->iter_vars[i];
+ const IterVar& intrin_var = intrin_block->iter_vars[i];
+ CHECK(desc_var->iter_type == intrin_var->iter_type)
+ << "Block iter_type mismatch between " << desc_var->iter_type << " and "
+ << intrin_var->iter_type;
+ }
+
+ auto n = make_object<TensorIntrinNode>();
+ n->description = std::move(desc_func);
+ n->implementation = std::move(intrin_func);
+ data_ = std::move(n);
+}
+
+TensorIntrin TensorIntrin::Register(String name, PrimFunc desc_func, PrimFunc intrin_func) {
+ TensorIntrinManager* manager = TensorIntrinManager::Global();
+ ICHECK_EQ(manager->reg.count(name), 0)
+ << "ValueError: TensorIntrin '" << name << "' has already been registered";
Review comment:
Should be user-facing API
```suggestion
CHECK_EQ(manager->reg.count(name), 0)
<< "ValueError: TensorIntrin '" << name << "' has already been registered";
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780587954
##########
File path: include/tvm/tir/function.h
##########
@@ -187,6 +187,56 @@ class LinkedParam : public ObjectRef {
TVM_DEFINE_OBJECT_REF_COW_METHOD(LinkedParamNode);
};
+/*!
+ * \brief Tensor intrinsics for tensorization
+ */
+class TensorIntrinNode : public Object {
+ public:
+ /*! \brief The function to describe the computation. */
+ PrimFunc description;
+ /*! \brief The intrinsic function for lower-level implementation. */
+ PrimFunc implementation;
+
+ void VisitAttrs(AttrVisitor* v) {
+ v->Visit("description", &description);
+ v->Visit("implementation", &implementation);
+ }
+
+ static constexpr const char* _type_key = "tir.TensorIntrin";
+ TVM_DECLARE_FINAL_OBJECT_INFO(TensorIntrinNode, Object);
+};
+
+/*!
+ * \brief Managed reference to TensorIntrinNode.
+ */
+class TensorIntrin : public ObjectRef {
+ public:
+ /*!
+ * \brief Constructor
+ * \param desc_func The function to describe the computation.
+ * \param intrin_func The intrinsic function for lower-level implementation.
+ */
+ TVM_DLL explicit TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func);
+
+ /*!
+ * \brief Create and register a TensorIntrin. After registration, the TensorIntrin can be looked
+ * up with its name.
+ * \param name The name of the TensorIntrin to register
+ * \param desc_func The function to describe the computation.
+ * \param intrin_func The intrinsic function for lower-level implementation.
+ * \return The created TensorIntrin.
+ */
+ TVM_DLL static TensorIntrin Register(String name, PrimFunc desc_func, PrimFunc intrin_func);
Review comment:
Let's be more idiomatic in registry design
```suggestion
TVM_DLL static TensorIntrin Register(String name, TensorIntrin intrin);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780717884
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
Review comment:
```suggestion
if (offset < 0) return false;
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] wrongtest commented on pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
wrongtest commented on pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#issuecomment-1008267187
Would there be some working tutorials or examples on TensorIR tensorize?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780725406
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
Review comment:
nit: no need to use const here
```suggestion
arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r784447241
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
Review comment:
nit: can save an intermediate variable
```suggestion
Stmt new_init = BlockRealize(
/*iter_values=*/init_bindings,
/*predicate=*/inner_block_realize->predicate,
/*block=*/
Block{/*iter_vars=*/init_block_iters,
/*reads=*/{},
/*writes=*/block->writes,
/*name_hint=*/block->name_hint + "_init",
/*body=*/block->init.value(),
/*init=*/NullOpt});
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785343951
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
Review comment:
```suggestion
* \param loop_sref The sref to the separator loop. The loop itself is counted as an inner loop.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#issuecomment-1013767672
@vinx13 Sorry for the delay! Finally I did a pass :-) Most of my comments are just nits.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] vinx13 commented on pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
vinx13 commented on pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#issuecomment-1020425076
@xqdan Thanks for raising this case. Unfortunately this is not supported, which seems a limitation of match buffer region. Currently, after tensorize, the params of the tensor intrin will be replaced with a buffer subregion. Such buffer subregions can't handle argmax well. We can try to improve this in the future
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785347163
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
Review comment:
In which case `division[i][1]->source` is not an `IterSumExpr`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] driazati commented on pull request #9871: [WIP][TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
driazati commented on pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#issuecomment-1017013390
Please ignore any failures on the `PR / cc-reviewers (pull_request_target)` job, you can still merge fine if it's failing (we had some infra problems deploying a new tool that will be fixed with #9988).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785360038
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
+
+ // check both functions' bodies are directly block
+ const auto* desc_realize =
+ Downcast<BlockRealize>(desc_func->body)->block->body.as<BlockRealizeNode>();
+ const auto* intrin_realize =
+ Downcast<BlockRealize>(intrin_func->body)->block->body.as<BlockRealizeNode>();
+ CHECK(desc_realize != nullptr) << "description function's body expect a directly block";
+ CHECK(intrin_realize != nullptr) << "intrinsic function's body expect a directly block";
+
+ const Block& desc_block = desc_realize->block;
+ const Block& intrin_block = intrin_realize->block;
+
+ // check block var number and iter type
+ CHECK_EQ(desc_block->iter_vars.size(), intrin_block->iter_vars.size())
+ << "Two blocks should have the same number of block vars";
+ for (size_t i = 0; i < desc_block->iter_vars.size(); i++) {
+ const IterVar& desc_var = desc_block->iter_vars[i];
+ const IterVar& intrin_var = intrin_block->iter_vars[i];
+ CHECK(desc_var->iter_type == intrin_var->iter_type)
+ << "Block iter_type mismatch between " << desc_var->iter_type << " and "
+ << intrin_var->iter_type;
+ }
+
+ auto n = make_object<TensorIntrinNode>();
Review comment:
`auto` is fine, just nit
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780588169
##########
File path: include/tvm/tir/schedule/schedule.h
##########
@@ -465,6 +465,25 @@ class ScheduleNode : public runtime::Object {
*/
virtual void SetScope(const BlockRV& block_rv, int buffer_index, const String& storage_scope) = 0;
/******** Schedule: Blockize & Tensorize ********/
+ /*!
+ * \brief Convert the subtree rooted at a specific loop into a block.
+ * \param loop_rv the root of the subtree
+ * \return the new block
+ */
+ virtual BlockRV Blockize(const LoopRV& loop_rv) = 0;
+ /*!
+ * \brief Tensorize the computation enclosed by loop with tensor_intrin
+ * \param loop_rv the loop to be tensorized
+ * \param intrin the tensor intrinsic
+ */
+ virtual void Tensorize(const LoopRV& loop_rv, const TensorIntrin& intrin) = 0;
Review comment:
Shall we hide this method to internal impl given:
- TracedSchedule is the default go-to
- This method isn't well supported by TracedSchedule
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780639158
##########
File path: src/tir/schedule/transform.cc
##########
@@ -136,5 +136,52 @@ void LeafBlockRemovalPlan(const ScheduleState& self, const StmtSRef& leaf_block_
throw OnlyLeafError(self->mod, GetRef<Block>(leaf_block), GetRef<Block>(scope_block));
}
+/******** IR Substitution ********/
+class IRSubstituteInScope : public StmtExprMutator {
+ public:
+ explicit IRSubstituteInScope(std::function<PrimExpr(const VarNode*)> fmap)
+ : fmap_(std::move(fmap)) {}
+
+ PrimExpr VisitExpr_(const VarNode* op) final {
+ auto it = fmap_(op);
+ if (it.defined()) {
+ return it;
+ } else {
+ return GetRef<PrimExpr>(op);
+ }
+ }
+
+ Stmt VisitStmt_(const BlockRealizeNode* op) final {
+ auto fmutate = [&](const PrimExpr& e) { return this->VisitExpr(e); };
+ Array<PrimExpr> v = op->iter_values;
+ v.MutateByApply(fmutate);
+ PrimExpr pred = this->VisitExpr(op->predicate);
+ if (v.same_as(op->iter_values) && pred.same_as(op->predicate)) {
+ return GetRef<Stmt>(op);
+ } else {
+ auto n = CopyOnWrite(op);
+ n->iter_values = std::move(v);
+ n->predicate = std::move(analyzer.Simplify(pred));
+ return Stmt(n);
+ }
+ }
+
+ private:
+ const std::function<PrimExpr(const VarNode*)> fmap_;
+ arith::Analyzer analyzer;
+};
+
+Stmt SubstituteInScope(const Stmt& stmt, const Map<Var, PrimExpr>& subst_map) {
Review comment:
I'm reading the implementation. Looks like what `SubstituteInScope` really does is that we only substitute block bindings and predicates, stops at the `BlockRealize` boundary and does nothing else. Is that accurate? If so, shall we rename this to `SubstituteBlockBinding`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780639226
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
+using StmtComparator = StmtFunctor<bool(const Stmt& n, const Stmt& other)>;
+
+/*! \brief Deep comparison to check if two IR ASTs are equivalent */
+class TensorizeComparator : public ExprComparator, public StmtComparator {
+ public:
+ /*!
+ * \brief Constructor of TensorizeComparator
+ * \param assert_mode Whether to raise an error if the two IR ASTs do not match.
+ */
+ explicit TensorizeComparator(bool assert_mode = true) : assert_mode_(assert_mode) {}
+
+ // Map from rhs buffer to lhs buffer
+ std::unordered_map<Buffer, Buffer, ObjectHash, ObjectEqual> rhs_buffer_map_;
Review comment:
They are not :-(
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780701263
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
Review comment:
Let's do lighter-weight checks first, so let's move this to a few lines above :-)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780717433
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
Review comment:
looks like `is_inner_block` is always `true` - is that correct?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780719152
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
+ }
+ };
+ std::tie(lhs_expr, lhs_iter) = detect(op->iter_values[i + offset]);
+ std::tie(rhs_expr, rhs_iter) = detect(rhs->iter_values[i]);
+ CHECK((lhs_iter && rhs_iter) || (!lhs_iter && !rhs_iter)) << "Incompatible binding";
+ if (lhs_iter) VisitExpr(lhs_iter.value(), rhs_iter.value());
+ if (is_zero(rhs_expr)) {
+ CHECK(is_zero(lhs_expr)) << "Incompatible binding";
+ } else {
+ const auto* bv = rhs_expr.as<VarNode>();
+ if (!bv) {
+ VisitExpr(lhs_expr, rhs_expr);
+ } else {
+ auto it = equal_map_.find(GetRef<Var>(bv));
+ if (it == equal_map_.end()) {
+ equal_map_[GetRef<Var>(bv)] = lhs_expr;
+ } else {
+ CHECK(it->second->IsInstance<PrimExprNode>());
+ VisitExpr(lhs_expr, Downcast<PrimExpr>(it->second));
+ }
+ }
+ }
+ }
+ } else {
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ if (!VisitExpr(op->iter_values[i + offset], rhs->iter_values[i])) return false;
+ }
+ const Block& block = op->block;
+ for (size_t i = 0; i < offset; ++i) {
+ Var block_var = Downcast<Var>(op->iter_values[i]);
+ auto it = equal_map_.find(block_var);
+ equal_map_[block->iter_vars[i]->var] = (it == equal_map_.end() ? block_var : it->second);
+ }
+ }
+ }
+
+ return VisitExpr(op->predicate, rhs->predicate) && VisitStmt(op->block, rhs->block);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockNode>();
+ // Check block equality.
+ // All iter vars and buffer regions including the order shoudl match.
+ // When checking iter vars, DefEqual is used to remap variables.
+ // Only the inner most several axis are compared. Other iter vars are added to extra_block_vars.
+ if (op->iter_vars.size() < rhs->iter_vars.size()) return false;
+
+ size_t offset = op->iter_vars.size() - rhs->iter_vars.size();
+ for (size_t i = 0; i < rhs->iter_vars.size(); ++i) {
+ auto lhs_var = op->iter_vars[i + offset], rhs_var = rhs->iter_vars[i];
+ // Skip iter dom
+ if (!DefEqual(lhs_var->var, rhs_var->var)) {
+ return false;
+ }
+ if (lhs_var->iter_type != rhs_var->iter_type) {
+ return false;
+ }
+ }
+
+ if (is_scope_block) {
+ for (size_t i = 0; i < offset; ++i) {
+ extra_block_vars_.push_back(op->iter_vars[i]);
+ }
+ }
+
+ if (!is_scope_block) {
+ if (!CompareArray(op->writes, rhs->writes, &TensorizeComparator::CompareBufferRegion)) {
+ return false;
+ }
+ if (!CompareArray(op->reads, rhs->reads, &TensorizeComparator::CompareBufferRegion)) {
+ return false;
+ }
+ if (!CompareAnnotationMap(op->annotations, rhs->annotations)) {
+ return false;
+ }
+ if (!CompareArray(op->alloc_buffers, rhs->alloc_buffers, &TensorizeComparator::CompareBuffer)) {
+ return false;
+ }
+ }
+ is_scope_block = false;
+ return VisitStmt(op->body, rhs->body);
+}
+
+// Exprs
+#define TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(OpName) \
+ bool TensorizeComparator::VisitExpr_(const OpName* op, const PrimExpr& other) { \
+ const auto* rhs = other.as<OpName>(); \
+ return VisitExpr(op->a, rhs->a) && VisitExpr(op->b, rhs->b); \
+ }
+
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(AddNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(SubNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MulNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(DivNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(ModNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(EQNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(NENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(LTNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(LENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(GTNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(GENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(AndNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(OrNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MinNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MaxNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(FloorDivNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(FloorModNode);
+
+bool TensorizeComparator::VisitExpr_(const IntImmNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<IntImmNode>();
+ return CompareType(op->dtype, rhs->dtype) && op->value == rhs->value;
+}
+
+bool TensorizeComparator::VisitExpr_(const FloatImmNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<FloatImmNode>();
+ return CompareType(op->dtype, rhs->dtype) && op->value == rhs->value;
+}
+
+bool TensorizeComparator::VisitExpr_(const CastNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<CastNode>();
+ return CompareType(op->dtype, rhs->dtype) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitExpr_(const VarNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<VarNode>();
+ auto lhs = GetRef<Var>(op);
+ if (lhs.same_as(other)) return true;
+ if (!CompareType(op->dtype, rhs->dtype)) return false;
+ auto it = equal_map_.find(lhs);
+ return it != equal_map_.end() && it->second.same_as(other);
+}
+
+bool TensorizeComparator::VisitExpr_(const BufferLoadNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<BufferLoadNode>();
+ return CompareBufferAccess(op, rhs);
+}
+
+bool TensorizeComparator::DefEqual(const ObjectRef& lhs, const ObjectRef& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ if (lhs->type_index() != rhs->type_index()) return false;
+ auto it = equal_map_.find(lhs);
+ // If there is already a mapping
+ if (it != equal_map_.end()) return it->second.same_as(rhs);
+ equal_map_[lhs] = rhs;
+ return true;
+}
+
+bool TensorizeComparator::CompareAnnotation(const std::pair<String, ObjectRef>& lhs,
+ const std::pair<String, ObjectRef>& rhs) {
+ if (lhs.first != rhs.first) return false;
+ if (!lhs.second.same_as(rhs.second)) return false;
+ return VisitExpr(Downcast<PrimExpr>(lhs.second), Downcast<PrimExpr>(rhs.second));
+}
+
+bool TensorizeComparator::CompareAnnotationMap(const Map<String, ObjectRef>& lhs,
+ const Map<String, ObjectRef>& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ if (lhs.size() != rhs.size()) return false;
+
+ auto sort_map =
+ [](const Map<String, ObjectRef>& map) -> std::vector<std::pair<String, ObjectRef>> {
+ std::vector<std::pair<String, ObjectRef>> ret;
+ ret.reserve(map.size());
+ for (const auto& pair : map) {
+ ret.emplace_back(pair);
+ }
+ sort(ret.begin(), ret.end());
+ return ret;
+ };
+
+ auto lhs_array = sort_map(lhs), rhs_array = sort_map(rhs);
+
+ for (size_t i = 0; i < lhs.size(); ++i) {
+ if (!CompareAnnotation(lhs_array[i], rhs_array[i])) return false;
+ }
+ return true;
+}
+
+bool TensorizeComparator::CompareBuffer(const Buffer& lhs, const Buffer& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ // Remap both buffer itself and buffer data
+ // Skip buffer shape
+ bool equal = DefEqual(lhs, rhs) && DefEqual(lhs->data, rhs->data) &&
+ CompareType(lhs->dtype, rhs->dtype) && lhs.scope() == rhs.scope();
+ if (equal) {
+ rhs_buffer_map_[rhs] = lhs;
+ } else if (assert_mode_) {
+ LOG(FATAL) << "Buffers are not matching between:" << lhs << " and " << rhs;
+ }
+ return equal;
+}
+
+bool TensorizeComparator::CompareBufferRegion(const BufferRegion& lhs, const BufferRegion& rhs) {
+ // Only for block region declaration
+ if (!CompareBuffer(lhs->buffer, rhs->buffer)) return false;
+ // Number of indices in desc_block must be smaller than it in AST
+ if (rhs->region.size() > lhs->region.size()) return false;
+
+ std::vector<Range> lhs_region;
+ for (const auto& range : lhs->region) {
+ lhs_region.push_back(Range::FromMinExtent(range->min, range->extent));
+ }
+ size_t offset = lhs_region.size() - rhs->region.size();
+ // initialize buffer indices
+ bool need_update = false;
+ if (!buffer_indices_.count(lhs->buffer)) {
+ need_update = true;
+ buffer_indices_[lhs->buffer] = std::vector<PrimExpr>();
+ } else {
+ if (offset != buffer_indices_[lhs->buffer].size()) return false;
+ }
+ std::vector<PrimExpr>& indices = buffer_indices_[lhs->buffer];
+ for (size_t i = 0; i < offset; ++i) {
+ const Range& range = lhs_region[i];
+ // High-dim region must be element-wise
+ if (!is_one(range->extent)) return false;
+ if (need_update) {
+ indices.push_back(range->min);
+ } else {
+ // The order matters since we only map inner block_var to outside block_var
+ if (!VisitExpr(range->min, indices[i])) return false;
+ }
+ }
+ for (size_t i = 0; i < rhs->region.size(); ++i) {
+ if (!CompareRange(lhs_region[i + offset], rhs->region[i])) return false;
+ }
+ return true;
+}
+
+// Only for BufferStoreNode and BufferLoadNode
+template <typename T>
+bool TensorizeComparator::CompareBufferAccess(const T* lhs, const T* rhs) {
+ if (!CompareBuffer(lhs->buffer, rhs->buffer)) return false;
+
+ if (rhs->indices.size() > lhs->indices.size()) return false;
+ // otherwise
+ size_t offset = lhs->indices.size() - rhs->indices.size();
+ for (size_t i = 0; i < rhs->indices.size(); ++i) {
+ if (!VisitExpr(lhs->indices[i + offset], rhs->indices[i])) return false;
+ }
+ return true;
+}
+
+template <typename T, typename F>
+bool TensorizeComparator::CompareArray(const Array<T>& lhs, const Array<T>& rhs, F cmp) {
+ if (lhs.same_as(rhs)) return true;
+ if (lhs.size() != rhs.size()) return false;
+ for (size_t i = 0; i < lhs.size(); ++i) {
+ if (!(this->*cmp)(lhs[i], rhs[i])) return false;
+ }
+ return true;
+}
+
+bool TensorizeComparator::CompareRange(const Range& lhs, const Range& rhs) {
+ return VisitExpr(lhs->min, rhs->min) && VisitExpr(lhs->extent, rhs->extent);
+}
+
+bool TensorizeComparator::CompareType(const DataType& lhs, const DataType& rhs) {
+ if (lhs == rhs) return true;
+ return lhs.code() == rhs.code() && lhs.bits() == rhs.bits() && lhs.lanes() == rhs.lanes();
+}
+
+// Deep comparison to check if two IR graph are equivalent
+bool TensorizeComparator::VisitExpr(const PrimExpr& n, const PrimExpr& other) {
Review comment:
Shall we do `CompareType` here so that it doesn't need to appear anywhere else?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780719532
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
+ }
+ };
+ std::tie(lhs_expr, lhs_iter) = detect(op->iter_values[i + offset]);
+ std::tie(rhs_expr, rhs_iter) = detect(rhs->iter_values[i]);
+ CHECK((lhs_iter && rhs_iter) || (!lhs_iter && !rhs_iter)) << "Incompatible binding";
+ if (lhs_iter) VisitExpr(lhs_iter.value(), rhs_iter.value());
+ if (is_zero(rhs_expr)) {
+ CHECK(is_zero(lhs_expr)) << "Incompatible binding";
+ } else {
+ const auto* bv = rhs_expr.as<VarNode>();
+ if (!bv) {
+ VisitExpr(lhs_expr, rhs_expr);
+ } else {
+ auto it = equal_map_.find(GetRef<Var>(bv));
+ if (it == equal_map_.end()) {
+ equal_map_[GetRef<Var>(bv)] = lhs_expr;
+ } else {
+ CHECK(it->second->IsInstance<PrimExprNode>());
+ VisitExpr(lhs_expr, Downcast<PrimExpr>(it->second));
+ }
+ }
+ }
+ }
+ } else {
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ if (!VisitExpr(op->iter_values[i + offset], rhs->iter_values[i])) return false;
+ }
+ const Block& block = op->block;
+ for (size_t i = 0; i < offset; ++i) {
+ Var block_var = Downcast<Var>(op->iter_values[i]);
+ auto it = equal_map_.find(block_var);
+ equal_map_[block->iter_vars[i]->var] = (it == equal_map_.end() ? block_var : it->second);
+ }
+ }
+ }
+
+ return VisitExpr(op->predicate, rhs->predicate) && VisitStmt(op->block, rhs->block);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockNode>();
+ // Check block equality.
+ // All iter vars and buffer regions including the order shoudl match.
+ // When checking iter vars, DefEqual is used to remap variables.
+ // Only the inner most several axis are compared. Other iter vars are added to extra_block_vars.
+ if (op->iter_vars.size() < rhs->iter_vars.size()) return false;
+
+ size_t offset = op->iter_vars.size() - rhs->iter_vars.size();
+ for (size_t i = 0; i < rhs->iter_vars.size(); ++i) {
+ auto lhs_var = op->iter_vars[i + offset], rhs_var = rhs->iter_vars[i];
+ // Skip iter dom
+ if (!DefEqual(lhs_var->var, rhs_var->var)) {
+ return false;
+ }
+ if (lhs_var->iter_type != rhs_var->iter_type) {
+ return false;
+ }
+ }
+
+ if (is_scope_block) {
+ for (size_t i = 0; i < offset; ++i) {
+ extra_block_vars_.push_back(op->iter_vars[i]);
+ }
+ }
+
+ if (!is_scope_block) {
+ if (!CompareArray(op->writes, rhs->writes, &TensorizeComparator::CompareBufferRegion)) {
+ return false;
+ }
+ if (!CompareArray(op->reads, rhs->reads, &TensorizeComparator::CompareBufferRegion)) {
+ return false;
+ }
+ if (!CompareAnnotationMap(op->annotations, rhs->annotations)) {
+ return false;
+ }
+ if (!CompareArray(op->alloc_buffers, rhs->alloc_buffers, &TensorizeComparator::CompareBuffer)) {
+ return false;
+ }
+ }
+ is_scope_block = false;
+ return VisitStmt(op->body, rhs->body);
+}
+
+// Exprs
+#define TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(OpName) \
+ bool TensorizeComparator::VisitExpr_(const OpName* op, const PrimExpr& other) { \
+ const auto* rhs = other.as<OpName>(); \
+ return VisitExpr(op->a, rhs->a) && VisitExpr(op->b, rhs->b); \
+ }
+
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(AddNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(SubNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MulNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(DivNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(ModNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(EQNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(NENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(LTNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(LENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(GTNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(GENode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(AndNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(OrNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MinNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(MaxNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(FloorDivNode);
+TVM_DECLARE_TENSORIZE_COMPARATOR_BINOP(FloorModNode);
+
+bool TensorizeComparator::VisitExpr_(const IntImmNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<IntImmNode>();
+ return CompareType(op->dtype, rhs->dtype) && op->value == rhs->value;
+}
+
+bool TensorizeComparator::VisitExpr_(const FloatImmNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<FloatImmNode>();
+ return CompareType(op->dtype, rhs->dtype) && op->value == rhs->value;
+}
+
+bool TensorizeComparator::VisitExpr_(const CastNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<CastNode>();
+ return CompareType(op->dtype, rhs->dtype) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitExpr_(const VarNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<VarNode>();
+ auto lhs = GetRef<Var>(op);
+ if (lhs.same_as(other)) return true;
+ if (!CompareType(op->dtype, rhs->dtype)) return false;
+ auto it = equal_map_.find(lhs);
+ return it != equal_map_.end() && it->second.same_as(other);
+}
+
+bool TensorizeComparator::VisitExpr_(const BufferLoadNode* op, const PrimExpr& other) {
+ const auto* rhs = other.as<BufferLoadNode>();
+ return CompareBufferAccess(op, rhs);
+}
+
+bool TensorizeComparator::DefEqual(const ObjectRef& lhs, const ObjectRef& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ if (lhs->type_index() != rhs->type_index()) return false;
+ auto it = equal_map_.find(lhs);
+ // If there is already a mapping
+ if (it != equal_map_.end()) return it->second.same_as(rhs);
+ equal_map_[lhs] = rhs;
+ return true;
+}
+
+bool TensorizeComparator::CompareAnnotation(const std::pair<String, ObjectRef>& lhs,
+ const std::pair<String, ObjectRef>& rhs) {
+ if (lhs.first != rhs.first) return false;
+ if (!lhs.second.same_as(rhs.second)) return false;
+ return VisitExpr(Downcast<PrimExpr>(lhs.second), Downcast<PrimExpr>(rhs.second));
+}
+
+bool TensorizeComparator::CompareAnnotationMap(const Map<String, ObjectRef>& lhs,
+ const Map<String, ObjectRef>& rhs) {
+ if (lhs.same_as(rhs)) return true;
+ if (lhs.size() != rhs.size()) return false;
+
+ auto sort_map =
+ [](const Map<String, ObjectRef>& map) -> std::vector<std::pair<String, ObjectRef>> {
+ std::vector<std::pair<String, ObjectRef>> ret;
+ ret.reserve(map.size());
+ for (const auto& pair : map) {
+ ret.emplace_back(pair);
+ }
+ sort(ret.begin(), ret.end());
Review comment:
Shall we do this?
```suggestion
std::vector<std::pair<String, ObjectRef>> ret(map.begin(), map.end());
sort(ret.begin(), ret.end());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780727564
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
Review comment:
Looks like it's a pair of zeros. What is this used for? Why are we using `Bool(true)` to indicate the extent BTW?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780726793
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
Review comment:
```suggestion
if (outer && !inner) {
res.push_back({/*outer_iter=*/iter_mark, /*inner_iter=*/unit_iter_mark});
} else if (inner && !outer) {
res.push_back({/*outer_iter=*/unit_iter_mark, /*inner_iter=*/iter_mark});
} else if (!outer && !inner) {
res.push_back({/*outer_iter=*/unit_iter_mark, /*inner_iter=*/unit_iter_mark});
} else {
return {};
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780605458
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
+using StmtComparator = StmtFunctor<bool(const Stmt& n, const Stmt& other)>;
+
+/*! \brief Deep comparison to check if two IR ASTs are equivalent */
+class TensorizeComparator : public ExprComparator, public StmtComparator {
+ public:
+ /*!
+ * \brief Constructor of TensorizeComparator
+ * \param assert_mode Whether to raise an error if the two IR ASTs do not match.
+ */
+ explicit TensorizeComparator(bool assert_mode = true) : assert_mode_(assert_mode) {}
+
+ // Map from rhs buffer to lhs buffer
+ std::unordered_map<Buffer, Buffer, ObjectHash, ObjectEqual> rhs_buffer_map_;
+ // Buffer indices mapping
+ std::unordered_map<Buffer, std::vector<PrimExpr>, ObjectPtrHash, ObjectPtrEqual> buffer_indices_;
+ std::vector<IterVar> extra_block_vars_;
+ // variable remap if any
+ std::unordered_map<ObjectRef, ObjectRef, ObjectPtrHash, ObjectPtrEqual> equal_map_;
+
+ bool VisitExpr(const PrimExpr& n, const PrimExpr& other) override;
+ bool VisitStmt(const Stmt& n, const Stmt& other) override;
+
+ bool VisitStmt_(const ForNode* op, const Stmt& other) override;
+ bool VisitStmt_(const SeqStmtNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BufferStoreNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockRealizeNode* op, const Stmt& other) override;
+ bool VisitStmt_(const BlockNode* op, const Stmt& other) override;
Review comment:
Just curious, does it mean that we only allow 5 types of statements?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780603282
##########
File path: src/tir/schedule/analysis.h
##########
@@ -442,6 +442,79 @@ bool CanComputeAt(const ScheduleState& self, const StmtSRef& block_sref, const S
bool CanReverseComputeAt(const ScheduleState& self, const StmtSRef& block_sref,
const StmtSRef& loop_sref, bool preserve_unit_loops);
+/******** Tensorization Related ********/
+
+using ExprComparator = ExprFunctor<bool(const PrimExpr& n, const PrimExpr& other)>;
Review comment:
Shall we move this into `src/tir/schedule/ir_comparator.h` and the impl to a corresponding cc file? The reason I'm considering this is that it's relatively independent but takes huge chunk of code
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780590829
##########
File path: python/tvm/tir/schedule/schedule.py
##########
@@ -1733,6 +1733,258 @@ def after_set_scope(
########## Schedule: Blockize & Tensorize ##########
+ @type_checked
+ def blockize(self, loop: LoopRV) -> BlockRV:
+ """Convert the subtree rooted at a specific loop into a block.
+
+ Parameters
+ ----------
+ loop : LoopRV
+ The root of the subtree.
+
+ Returns
+ -------
+ result : BlockRV
+ The new block.
+
+ Examples
+ --------
+
+ Before blockize, in TensorIR, the IR is:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def before_blockize(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"]
+ ) -> None:
+ for i_0, j_0, i_1, j_1 in T.grid(8, 8, 16, 16):
+ with T.block("B"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1)
+ T.reads(A[vi, vj])
+ T.writes(B[vi, vj])
+ B[vi, vj] = A[vi, vj] * T.float32(2)
+
+ Create the schedule and do set_scope:
+
+ .. code-block:: python
+
+ sch = tir.Schedule(before_blockize)
+ B = sch.get_block("B")
+ _, _, i1, _ = sch.get_loops(B)
+ sch.blockize(i1)
+ print(sch.mod["main"].script())
+
+ After applying blockize, the IR becomes:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def after_blockize(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"]
+ )-> None:
+ for i_0, j_0 in T.grid(8, 8):
+ with T.block("blockized_B"):
+ vio, vjo = T.axis.remap("SS", [i_0, j_0])
+ T.reads(A[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16])
+ T.writes(B[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16])
+ for i_1, j_1 in T.grid(16, 16):
+ with T.block("B"):
+ vi = T.axis.spatial(128, vio * 16 + i_1)
+ vj = T.axis.spatial(128, vjo * 16 + j_1)
+ T.reads(A[vi, vj])
+ T.writes(B[vi, vj])
+ B[vi, vj] = A[vi, vj] * T.float32(2)
+
+ Note
+ ----
+ blockize requires there is exactly one block under the given loop and the bindings of the
+ block are divisible by the subspace represented by the loops starting at the given loop.
+
+ """
+
+ return _ffi_api.ScheduleBlockize(self, loop) # type: ignore # pylint: disable=no-member
+
+ @type_checked
+ def tensorize(self, loop: LoopRV, tensor_intrin: Union[str, TensorIntrin]) -> None:
+ """Tensorize the computation enclosed by loop with the tensor intrinsic.
+
+ Parameters
+ ----------
+ loop : LoopRV
+ The loop to be tensorized.
+ tensor_intrin : Union[str, TensorIntrin]
+ The tensor intrin or the name of the tensor intrin.
+
+ Examples
+ --------
+
+ Before tensorize, in TensorIR, the IR is:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def before_tensorize(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"],
+ C: T.Buffer[(128, 128), "float32"],
+ ) -> None:
+ # body
+ # with T.block("root")
+ for i_0, j_0 in T.grid(8, 8):
+ for i_1_init, j_1_init in T.grid(16, 16):
+ with T.block("init"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1_init)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1_init)
+ T.reads()
+ T.writes(C[vi, vj])
+ C[vi, vj] = T.float32(0)
+ for k_0, i_1, j_1, k_1 in T.grid(8, 16, 16, 16):
+ with T.block("update"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1)
+ vk = T.axis.reduce(128, k_0 * 16 + k_1)
+ T.reads(C[vi, vj], A[vi, vk], B[vj, vk])
+ T.writes(C[vi, vj])
+ C[vi, vj] = C[vi, vj] + A[vi, vk] * B[vj, vk]
+
+ Declare and register the tensor intrinsic:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def mma_desc(a: T.handle, b: T.handle, c: T.handle) -> None:
+ A = T.match_buffer(a, (16, 16), align=128, offset_factor=1)
+ B = T.match_buffer(b, (16, 16), align=128, offset_factor=1)
+ C = T.match_buffer(c, (16, 16), align=128, offset_factor=1)
+
+ with T.block("root"):
+ vi = T.axis.S(16, 0)
+ vj = T.axis.S(16, 0)
+ vk = T.axis.R(16, 0)
+ for i, j, k in T.grid(16, 16, 16):
+ with T.block("update"):
+ vii = T.axis.S(16, vi + i)
+ vjj = T.axis.S(16, vj + j)
+ vkk = T.axis.R(16, vk + k)
+ C[vii, vjj] = C[vii, vjj] + A[vii, vkk] * B[vjj, vkk]
+
+
+ @T.prim_func
+ def mma_intrin(a: T.handle, b: T.handle, c: T.handle) -> None:
+ A = T.match_buffer(a, (16, 16), align=128, offset_factor=1)
+ B = T.match_buffer(b, (16, 16), align=128, offset_factor=1)
+ C = T.match_buffer(c, (16, 16), align=128, offset_factor=1)
+
+ with T.block("root"):
+ vi = T.axis.S(16, 0)
+ vj = T.axis.S(16, 0)
+ vk = T.axis.R(16, 0)
+ T.reads(
+ [
+ C[vi : vi + 16, vj : vj + 16],
+ A[vi : vi + 16, vk : vk + 16],
+ B[vj : vj + 16, vk : vk + 16],
+ ]
+ )
+ T.writes(C[vi : vi + 16, vj : vj + 16])
+ T.evaluate(
+ T.tvm_mma_sync(
+ C.data,
+ C.elem_offset // 256,
+ A.data,
+ A.elem_offset // 256,
+ B.data,
+ B.elem_offset // 256,
+ C.data,
+ C.elem_offset // 256,
+ dtype="handle",
+ )
+ )
+
+ tir.TensorIntrin.register("test_mma_intrin", mma_desc, mma_intrin)
+
+ Create the schedule and do tensorize:
+
+ .. code-block:: python
+
+ sch = tir.Schedule(before_tensorize)
+ update = s.get_block("update")
+ _, _, _, i1, _, _ = s.get_loops(update)
+ s.tensorize(ii, "test_mma_intrin")
+ print(sch.mod["main"].script())
+
+ After applying tensorize, the IR becomes:
+
+ .. code-block:: python
+
+ @T.prim_func
+ def after_tensoirze(
+ A: T.Buffer[(128, 128), "float32"],
+ B: T.Buffer[(128, 128), "float32"],
+ C: T.Buffer[(128, 128), "float32"],
+ ) -> None:
+ # body
+ # with T.block("root")
+ for i_0, j_0 in T.grid(8, 8):
+ for i_1_init, j_1_init in T.grid(16, 16):
+ with T.block("init"):
+ vi = T.axis.spatial(128, i_0 * 16 + i_1_init)
+ vj = T.axis.spatial(128, j_0 * 16 + j_1_init)
+ T.reads()
+ T.writes(C[vi, vj])
+ C[vi, vj] = T.float32(0)
+ for k_0 in T.serial(8):
+ with T.block("blockized_update"):
+ vio, vjo, vko = T.axis.remap("SSR", [i_0, j_0, k_0])
+ T.reads(
+ C[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16],
+ A[vio * 16 : vio * 16 + 16, vko * 16 : vko * 16 + 16],
+ B[vjo * 16 : vjo * 16 + 16, vko * 16 : vko * 16 + 16],
+ )
+ T.writes(C[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16])
+ A_1 = T.match_buffer(
+ A[vio * 16 : vio * 16 + 16, vko * 16 : vko * 16 + 16],
+ [16, 16],
+ dtype="float32",
+ offset_factor=1,
+ )
+ B_1 = T.match_buffer(
+ B[vjo * 16 : vjo * 16 + 16, vko * 16 : vko * 16 + 16],
+ [16, 16],
+ dtype="float32",
+ offset_factor=1,
+ )
+ C_1 = T.match_buffer(
+ C[vio * 16 : vio * 16 + 16, vjo * 16 : vjo * 16 + 16],
+ [16, 16],
+ dtype="float32",
+ offset_factor=1,
+ )
+ T.evaluate(
+ T.tvm_mma_sync(
+ C_1.data,
+ C_1.elem_offset // 256,
+ A_1.data,
+ A_1.elem_offset // 256,
+ B_1.data,
+ B_1.elem_offset // 256,
+ C_1.data,
+ C_1.elem_offset // 256,
+ dtype="handle",
+ )
+ )
+
+ """
+ if isinstance(tensor_intrin, str):
+ tensor_intrin = String(tensor_intrin)
Review comment:
oh i see. It's because our C++ registry are trying to decide between String and TensorIntrin - then it makes sense to me
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780592805
##########
File path: src/tir/schedule/schedule.cc
##########
@@ -183,6 +183,20 @@ TVM_REGISTER_GLOBAL("tir.schedule.ScheduleStorageAlign")
TVM_REGISTER_GLOBAL("tir.schedule.ScheduleSetScope")
.set_body_method<Schedule>(&ScheduleNode::SetScope);
/******** (FFI) Blockize & Tensorize ********/
+TVM_REGISTER_GLOBAL("tir.schedule.ScheduleBlockize")
+ .set_body_method<Schedule>(&ScheduleNode::Blockize);
+TVM_REGISTER_GLOBAL("tir.schedule.ScheduleTensorize")
+ .set_body_typed([](Schedule self, LoopRV loop_rv, ObjectRef intrin) {
+ if (const auto* str = intrin.as<runtime::StringObj>()) {
+ return self->Tensorize(loop_rv, GetRef<String>(str));
+ }
+ if (const auto* p_intrin = intrin.as<TensorIntrinNode>()) {
+ return self->Tensorize(loop_rv, GetRef<TensorIntrin>(p_intrin));
+ }
+ LOG(FATAL) << "TypeError: Cannot handle type: " << intrin->GetTypeKey();
+ throw;
Review comment:
I'm considering...is it reasonable to use only the `String` API and not expose the `TensorIntrin` one? The reason I'm asking this is that `TensorIntrin` isn't available for `TracedSchedule` anyway, which is the default go-to for python frontend users
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780592950
##########
File path: src/tir/schedule/state.cc
##########
@@ -201,9 +201,8 @@ class BlockInfoCollector : private StmtVisitor {
bool is_root_block = srefs_.empty();
// Calculate `BlockInfo::scope`
Array<StmtSRef> child_block_srefs = std::move(block_frames_.back());
- BlockInfo& info =
- self_->block_info.emplace(scope_root, BlockInfo(BlockScope(child_block_srefs)))
- .first->second;
+ self_->block_info[scope_root] = BlockInfo(BlockScope(child_block_srefs));
+ BlockInfo& info = self_->block_info[scope_root];
Review comment:
Why this change? I suppose they are logically equivalent and the original one saves one lookup?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780587280
##########
File path: include/tvm/tir/function.h
##########
@@ -187,6 +187,56 @@ class LinkedParam : public ObjectRef {
TVM_DEFINE_OBJECT_REF_COW_METHOD(LinkedParamNode);
};
+/*!
+ * \brief Tensor intrinsics for tensorization
+ */
+class TensorIntrinNode : public Object {
+ public:
+ /*! \brief The function to describe the computation. */
+ PrimFunc description;
+ /*! \brief The intrinsic function for lower-level implementation. */
+ PrimFunc implementation;
Review comment:
Which one do you prefer, shortened name or full name? Like, `desc` and `impl` vs `description` and `implementation`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780718904
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
+ }
+ };
+ std::tie(lhs_expr, lhs_iter) = detect(op->iter_values[i + offset]);
+ std::tie(rhs_expr, rhs_iter) = detect(rhs->iter_values[i]);
+ CHECK((lhs_iter && rhs_iter) || (!lhs_iter && !rhs_iter)) << "Incompatible binding";
+ if (lhs_iter) VisitExpr(lhs_iter.value(), rhs_iter.value());
+ if (is_zero(rhs_expr)) {
+ CHECK(is_zero(lhs_expr)) << "Incompatible binding";
+ } else {
+ const auto* bv = rhs_expr.as<VarNode>();
+ if (!bv) {
+ VisitExpr(lhs_expr, rhs_expr);
+ } else {
+ auto it = equal_map_.find(GetRef<Var>(bv));
+ if (it == equal_map_.end()) {
+ equal_map_[GetRef<Var>(bv)] = lhs_expr;
+ } else {
+ CHECK(it->second->IsInstance<PrimExprNode>());
+ VisitExpr(lhs_expr, Downcast<PrimExpr>(it->second));
+ }
+ }
+ }
+ }
+ } else {
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ if (!VisitExpr(op->iter_values[i + offset], rhs->iter_values[i])) return false;
+ }
+ const Block& block = op->block;
+ for (size_t i = 0; i < offset; ++i) {
+ Var block_var = Downcast<Var>(op->iter_values[i]);
+ auto it = equal_map_.find(block_var);
+ equal_map_[block->iter_vars[i]->var] = (it == equal_map_.end() ? block_var : it->second);
+ }
+ }
+ }
+
+ return VisitExpr(op->predicate, rhs->predicate) && VisitStmt(op->block, rhs->block);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockNode>();
+ // Check block equality.
+ // All iter vars and buffer regions including the order shoudl match.
+ // When checking iter vars, DefEqual is used to remap variables.
+ // Only the inner most several axis are compared. Other iter vars are added to extra_block_vars.
+ if (op->iter_vars.size() < rhs->iter_vars.size()) return false;
+
+ size_t offset = op->iter_vars.size() - rhs->iter_vars.size();
+ for (size_t i = 0; i < rhs->iter_vars.size(); ++i) {
+ auto lhs_var = op->iter_vars[i + offset], rhs_var = rhs->iter_vars[i];
+ // Skip iter dom
+ if (!DefEqual(lhs_var->var, rhs_var->var)) {
+ return false;
+ }
+ if (lhs_var->iter_type != rhs_var->iter_type) {
+ return false;
+ }
+ }
+
+ if (is_scope_block) {
+ for (size_t i = 0; i < offset; ++i) {
+ extra_block_vars_.push_back(op->iter_vars[i]);
+ }
+ }
+
+ if (!is_scope_block) {
Review comment:
```suggestion
else {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780718868
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
+ }
+ };
+ std::tie(lhs_expr, lhs_iter) = detect(op->iter_values[i + offset]);
+ std::tie(rhs_expr, rhs_iter) = detect(rhs->iter_values[i]);
+ CHECK((lhs_iter && rhs_iter) || (!lhs_iter && !rhs_iter)) << "Incompatible binding";
+ if (lhs_iter) VisitExpr(lhs_iter.value(), rhs_iter.value());
+ if (is_zero(rhs_expr)) {
+ CHECK(is_zero(lhs_expr)) << "Incompatible binding";
+ } else {
+ const auto* bv = rhs_expr.as<VarNode>();
+ if (!bv) {
+ VisitExpr(lhs_expr, rhs_expr);
+ } else {
+ auto it = equal_map_.find(GetRef<Var>(bv));
+ if (it == equal_map_.end()) {
+ equal_map_[GetRef<Var>(bv)] = lhs_expr;
+ } else {
+ CHECK(it->second->IsInstance<PrimExprNode>());
+ VisitExpr(lhs_expr, Downcast<PrimExpr>(it->second));
+ }
+ }
+ }
+ }
+ } else {
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ if (!VisitExpr(op->iter_values[i + offset], rhs->iter_values[i])) return false;
+ }
+ const Block& block = op->block;
+ for (size_t i = 0; i < offset; ++i) {
+ Var block_var = Downcast<Var>(op->iter_values[i]);
+ auto it = equal_map_.find(block_var);
+ equal_map_[block->iter_vars[i]->var] = (it == equal_map_.end() ? block_var : it->second);
+ }
+ }
+ }
+
+ return VisitExpr(op->predicate, rhs->predicate) && VisitStmt(op->block, rhs->block);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockNode>();
+ // Check block equality.
+ // All iter vars and buffer regions including the order shoudl match.
+ // When checking iter vars, DefEqual is used to remap variables.
+ // Only the inner most several axis are compared. Other iter vars are added to extra_block_vars.
+ if (op->iter_vars.size() < rhs->iter_vars.size()) return false;
+
+ size_t offset = op->iter_vars.size() - rhs->iter_vars.size();
+ for (size_t i = 0; i < rhs->iter_vars.size(); ++i) {
+ auto lhs_var = op->iter_vars[i + offset], rhs_var = rhs->iter_vars[i];
Review comment:
nit:
```suggestion
Var lhs_var = op->iter_vars[i + offset];
Var rhs_var = rhs->iter_vars[i];
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780718325
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
+ return equal;
+}
+
+bool TensorizeComparator::VisitStmt_(const ForNode* op, const Stmt& other) {
+ const auto* rhs = other.as<ForNode>();
+ if (!DefEqual(op->loop_var, rhs->loop_var)) return false;
+ if (!VisitExpr(op->min, rhs->min)) return false;
+ if (!VisitExpr(op->extent, rhs->extent)) return false;
+ if (!VisitStmt(op->body, rhs->body)) return false;
+ if (op->kind != rhs->kind) return false;
+ if (op->thread_binding.defined() ^ rhs->thread_binding.defined()) return false;
+ if (op->thread_binding.defined() &&
+ !VisitExpr(op->thread_binding.value(), rhs->thread_binding.value()))
+ return false;
+ return CompareAnnotationMap(op->annotations, rhs->annotations);
+}
+
+bool TensorizeComparator::VisitStmt_(const SeqStmtNode* op, const Stmt& other) {
+ const auto* rhs = other.as<SeqStmtNode>();
+ return CompareArray(op->seq, rhs->seq, &TensorizeComparator::VisitStmt);
+}
+
+bool TensorizeComparator::VisitStmt_(const BufferStoreNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BufferStoreNode>();
+ return CompareBufferAccess(op, rhs) && VisitExpr(op->value, rhs->value);
+}
+
+bool TensorizeComparator::VisitStmt_(const BlockRealizeNode* op, const Stmt& other) {
+ const auto* rhs = other.as<BlockRealizeNode>();
+ // Skip Compare binding values if the block is scope block (the outermost one).
+ if (!is_scope_block) {
+ size_t offset = op->iter_values.size() - rhs->iter_values.size();
+ if (rhs->iter_values.size() > op->iter_values.size()) return false;
+ if (is_inner_block) {
+ // weak pattern matching for the inner block (the son of the scope block)
+ // where the pattern is v + iter <=> expr + iter
+ for (size_t i = 0; i < rhs->iter_values.size(); ++i) {
+ PrimExpr lhs_expr, rhs_expr;
+ Optional<Var> lhs_iter, rhs_iter;
+ auto detect = [](const PrimExpr& binding) -> std::pair<PrimExpr, Optional<Var>> {
+ arith::PVar<PrimExpr> expr;
+ arith::PVar<Var> iter;
+ if (iter.Match(binding)) {
+ return std::make_pair(0, iter.Eval());
+ } else if ((expr + iter).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else if ((iter + expr).Match(binding)) {
+ return std::make_pair(expr.Eval(), iter.Eval());
+ } else {
+ return std::make_pair(expr.Eval(), NullOpt);
Review comment:
is expr.Eval() == nullptr?
```suggestion
return std::make_pair(PrimExpr{nullptr}, NullOpt);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780725406
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
Review comment:
no need to use const here
```suggestion
arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785350314
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
Review comment:
```suggestion
// realize and its surrounding loops.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785354820
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
Review comment:
```suggestion
/*!
* \brief Update the map from the buffers in the description to the implementation of the tensor
* intrinsic.
* \param intrinsic The tensor intrinsic.
* \param buffer_map The map to be updated.
*/
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785354766
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
Review comment:
```suggestion
bool scope_block_affine_binding = self->IsAffineBlockBinding(scope_root);
self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
self->block_info[scope_root].affine_binding = scope_block_affine_binding;
return outer_block_sref;
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785354410
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
Review comment:
No need to use reference, because it's returned by this method
```suggestion
StmtSRef outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] vinx13 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
vinx13 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780598226
##########
File path: src/tir/schedule/state.cc
##########
@@ -201,9 +201,8 @@ class BlockInfoCollector : private StmtVisitor {
bool is_root_block = srefs_.empty();
// Calculate `BlockInfo::scope`
Array<StmtSRef> child_block_srefs = std::move(block_frames_.back());
- BlockInfo& info =
- self_->block_info.emplace(scope_root, BlockInfo(BlockScope(child_block_srefs)))
- .first->second;
+ self_->block_info[scope_root] = BlockInfo(BlockScope(child_block_srefs));
+ BlockInfo& info = self_->block_info[scope_root];
Review comment:
`emplace` will not overwrite the old entry if the key exists. This function may be called in `UpdateScopeBlockInfo`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780589481
##########
File path: src/tir/ir/function.cc
##########
@@ -64,6 +64,67 @@ FuncType PrimFuncNode::func_type_annotation() const {
TVM_REGISTER_NODE_TYPE(PrimFuncNode);
+class TensorIntrinManager {
+ public:
+ Map<String, tir::TensorIntrin> reg;
+
+ static TensorIntrinManager* Global() {
+ static TensorIntrinManager* inst = new TensorIntrinManager();
+ return inst;
+ }
+};
+
+TensorIntrin::TensorIntrin(PrimFunc desc_func, PrimFunc intrin_func) {
+ // check the number of func var is equal
+ CHECK_EQ(desc_func->params.size(), intrin_func->params.size());
+ CHECK_EQ(desc_func->buffer_map.size(), intrin_func->buffer_map.size());
+
+ // check both functions' bodies are directly block
+ const auto* desc_realize =
+ Downcast<BlockRealize>(desc_func->body)->block->body.as<BlockRealizeNode>();
+ const auto* intrin_realize =
+ Downcast<BlockRealize>(intrin_func->body)->block->body.as<BlockRealizeNode>();
+ CHECK(desc_realize != nullptr) << "description function's body expect a directly block";
+ CHECK(intrin_realize != nullptr) << "intrinsic function's body expect a directly block";
+
+ const Block& desc_block = desc_realize->block;
+ const Block& intrin_block = intrin_realize->block;
+
+ // check block var number and iter type
+ CHECK_EQ(desc_block->iter_vars.size(), intrin_block->iter_vars.size())
+ << "Two blocks should have the same number of block vars";
+ for (size_t i = 0; i < desc_block->iter_vars.size(); i++) {
+ const IterVar& desc_var = desc_block->iter_vars[i];
+ const IterVar& intrin_var = intrin_block->iter_vars[i];
+ CHECK(desc_var->iter_type == intrin_var->iter_type)
+ << "Block iter_type mismatch between " << desc_var->iter_type << " and "
+ << intrin_var->iter_type;
Review comment:
Let's use `IterVarType2String`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780724523
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
Review comment:
```suggestion
* \param predicate The predicate constraint on the input iterators.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r780725356
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
Review comment:
To save a copy
```suggestion
Array<Array<arith::IterMark>> res;
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785350967
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
Review comment:
Let's be more consistent with naming :-)
```suggestion
/*name_hint=*/block->name_hint + "_o", //
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785346551
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
Review comment:
1. Why not we use `IterSumExprNode` instead to be more accurate, given the `source` is either `IterSumExpr` or `Var`?
2. `TVM_TYPE_AS` could help with type checking BTW.
3. Use some intermediate variables to make the logic clearer.
```suggestion
arith::IterMark outer_mark = division[i][0];
arith::IterMark inner_mark = division[i][1];
const auto* outer_binding = TVM_TYPE_AS(outer_binding, outer_mark->source, arith::IterMapExprNode);
const auto* inner_binding = TVM_TYPE_AS(inner_binding, inner_mark->source, arith::IterMapExprNode);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785359692
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
+ const Block& desc_block = desc_block_realize->block;
+ const auto* impl_block_realize =
+ Downcast<BlockRealize>(intrinsic->implementation->body)->block->body.as<BlockRealizeNode>();
+ Block impl_block = impl_block_realize->block;
+
+ // Step 1: Blockize the subtree rooted at the given loop
+ const StmtSRef& block_sref = Blockize(self, loop_sref);
+ const BlockRealize& block_realize = GetBlockRealize(self, block_sref);
+
+ // Step 2: Compare the block with the description of the tensor intrinsic, find the correspondence
+ // between buffers in the block and the description.
+ TensorizeComparator comparator(/*assert_mode=*/true);
+ comparator.VisitStmt(block_realize, GetRef<Stmt>(desc_block_realize));
+
+ // Step 3: Find the correspondence between buffers in the current AST and the implementation of
+ // the tensor intrinsic
+
+ // Step 3.1: Map from intrinsic func buffer to description func buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> intrin_buffer_map;
+ RemapTensorIntrinBuffers(intrinsic, &intrin_buffer_map);
+ // Step 3.2: Map form intrinsic func buffer to current AST buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> buffer_map;
+ for (const auto& pair : intrin_buffer_map) {
+ auto it = comparator.rhs_buffer_map_.find(pair.second);
+ ICHECK(it != comparator.rhs_buffer_map_.end());
+ buffer_map[pair.first] = it->second;
+ }
+
+ // Step 4: Create MatchBufferRegion for the params of the implementation function of the tensor
+ // intrin to make them subregions of the buffer in the original IR.
+ std::unordered_map<Buffer, Array<Range>, ObjectPtrHash, ObjectPtrEqual> buffer_region_map;
+ for (const auto& read : impl_block->reads) {
+ buffer_region_map.emplace(read->buffer, read->region);
+ }
+ for (const auto& write : impl_block->writes) {
+ buffer_region_map.emplace(write->buffer, write->region);
+ }
+ Array<MatchBufferRegion> match_buffer_regions;
+ for (size_t i = 0; i < intrinsic->implementation->params.size(); ++i) {
+ const auto& param = intrinsic->implementation->params[i];
+ const auto& buffer = intrinsic->implementation->buffer_map.at(param);
+ const auto& source = buffer_map.at(buffer);
+ Region region = buffer_region_map.at(buffer);
+ auto extra_indices = comparator.buffer_indices_.at(source);
Review comment:
nit: i found it a little bit confusing to use auto here, so...
```suggestion
const Var& param = intrinsic->implementation->params[i];
Buffer buffer = intrinsic->implementation->buffer_map.at(param);
Buffer source = buffer_map.at(buffer);
Array<Range> region = buffer_region_map.at(buffer);
const std::vector<PrimExpr>& extra_indices = comparator.buffer_indices_.at(source);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] junrushao1994 commented on a change in pull request #9871: [TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
junrushao1994 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r785360053
##########
File path: src/tir/schedule/analysis/analysis.cc
##########
@@ -1376,5 +1376,333 @@ void CheckStorageScope(const ScheduleState& self, String storage_scope) {
}
}
+/******** Tensorize Comparator ********/
+
+bool TensorizeComparator::VisitStmt(const Stmt& n, const Stmt& other) {
+ if (n.same_as(other)) return true;
+ if (n->type_index() != other->type_index()) return false;
+ bool equal = StmtComparator::VisitStmt(n, other);
+ if (!equal && assert_mode_)
+ LOG(FATAL) << "Stmts are not matching between:\n" << n << "\nand\n" << other;
Review comment:
looks like it's non-trivial for now to do so, let's leave it as future improvement
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] vinx13 commented on a change in pull request #9871: [WIP][TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
vinx13 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r788263073
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
+ const Block& desc_block = desc_block_realize->block;
+ const auto* impl_block_realize =
+ Downcast<BlockRealize>(intrinsic->implementation->body)->block->body.as<BlockRealizeNode>();
+ Block impl_block = impl_block_realize->block;
+
+ // Step 1: Blockize the subtree rooted at the given loop
+ const StmtSRef& block_sref = Blockize(self, loop_sref);
+ const BlockRealize& block_realize = GetBlockRealize(self, block_sref);
+
+ // Step 2: Compare the block with the description of the tensor intrinsic, find the correspondence
+ // between buffers in the block and the description.
+ TensorizeComparator comparator(/*assert_mode=*/true);
+ comparator.VisitStmt(block_realize, GetRef<Stmt>(desc_block_realize));
+
+ // Step 3: Find the correspondence between buffers in the current AST and the implementation of
+ // the tensor intrinsic
+
+ // Step 3.1: Map from intrinsic func buffer to description func buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> intrin_buffer_map;
+ RemapTensorIntrinBuffers(intrinsic, &intrin_buffer_map);
+ // Step 3.2: Map form intrinsic func buffer to current AST buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> buffer_map;
+ for (const auto& pair : intrin_buffer_map) {
+ auto it = comparator.rhs_buffer_map_.find(pair.second);
+ ICHECK(it != comparator.rhs_buffer_map_.end());
+ buffer_map[pair.first] = it->second;
+ }
+
+ // Step 4: Create MatchBufferRegion for the params of the implementation function of the tensor
+ // intrin to make them subregions of the buffer in the original IR.
+ std::unordered_map<Buffer, Array<Range>, ObjectPtrHash, ObjectPtrEqual> buffer_region_map;
+ for (const auto& read : impl_block->reads) {
+ buffer_region_map.emplace(read->buffer, read->region);
+ }
+ for (const auto& write : impl_block->writes) {
+ buffer_region_map.emplace(write->buffer, write->region);
+ }
+ Array<MatchBufferRegion> match_buffer_regions;
+ for (size_t i = 0; i < intrinsic->implementation->params.size(); ++i) {
+ const auto& param = intrinsic->implementation->params[i];
+ const auto& buffer = intrinsic->implementation->buffer_map.at(param);
+ const auto& source = buffer_map.at(buffer);
+ Region region = buffer_region_map.at(buffer);
+ auto extra_indices = comparator.buffer_indices_.at(source);
+ std::vector<Range> extra_buffer_ranges;
+ std::transform(extra_indices.begin(), extra_indices.end(),
+ std::back_inserter(extra_buffer_ranges),
+ [](const PrimExpr& index) { return Range::FromMinExtent(index, 1); });
+ region.insert(region.begin(), extra_buffer_ranges.begin(), extra_buffer_ranges.end());
+ match_buffer_regions.push_back(MatchBufferRegion(buffer, BufferRegion(source, region)));
+ }
+
+ // Step 5: Replace the subtree in the original IR with the tensor intrin implementation.
+ ObjectPtr<BlockNode> new_block_ptr = make_object<BlockNode>(*block_realize->block.get());
+ new_block_ptr->body = impl_block->body;
+ ICHECK(new_block_ptr->match_buffers.empty());
+ new_block_ptr->match_buffers = std::move(match_buffer_regions);
+ Block new_block(new_block_ptr);
+
+ // Substitution map for the intrin block to get the correct bindings.
+ Map<Var, PrimExpr> bv_map;
+ for (size_t i = 0; i < desc_block->iter_vars.size(); ++i) {
+ auto it = comparator.equal_map_.find(desc_block->iter_vars[i]->var);
+ if (it != comparator.equal_map_.end()) {
+ bv_map.Set(impl_block->iter_vars[i]->var, Downcast<PrimExpr>(it->second));
+ } else {
+ bv_map.Set(impl_block->iter_vars[i]->var, Integer(0));
+ }
+ }
+ new_block = Downcast<Block>(SubstituteInScope(new_block, bv_map));
+ self->Replace(block_sref, new_block, {{block_realize->block, new_block}});
Review comment:
`Tensorize` now allows to pass in a block instead of a loop. In that case, we don't need to do blockize.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] vinx13 commented on a change in pull request #9871: [WIP][TIR][Schedule] Blockize and Tensorize
Posted by GitBox <gi...@apache.org>.
vinx13 commented on a change in pull request #9871:
URL: https://github.com/apache/tvm/pull/9871#discussion_r789051829
##########
File path: src/tir/schedule/primitive/blockize_tensorize.cc
##########
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <functional>
+
+#include "../utils.h"
+
+namespace tvm {
+namespace tir {
+
+/*!
+ * \brief ScheduleError that the bindings of the inner block are not divisible by the subspace
+ * represented by the outer loops.
+ */
+class SubspaceNotDivisibleError : public ScheduleError {
+ public:
+ explicit SubspaceNotDivisibleError(IRModule mod, For scope_loop, Block inner_block)
+ : mod_(std::move(mod)),
+ scope_loop_(std::move(scope_loop)),
+ inner_block_(std::move(inner_block)) {}
+
+ String FastErrorString() const final {
+ return "ScheduleError: The bindings of the inner block can not be blockized.";
+ }
+
+ String DetailRenderTemplate() const final {
+ return "ScheduleError: The bindings of the inner block {0} can not be blockized by the loops "
+ "starting at {1}.";
+ }
+
+ IRModule mod() const final { return mod_; }
+
+ Array<ObjectRef> LocationsOfInterest() const final { return {inner_block_, scope_loop_}; }
+
+ private:
+ IRModule mod_;
+ For scope_loop_;
+ Block inner_block_;
+};
+
+/*!
+ * \brief Detect if bindings are a trivial case of the subspace division where we can divide the
+ * block iter bindings into two categories:
+ * 1. The binding covers no inner loop vars.
+ * 2. The binding covers only inner loop vars.
+ *
+ * The bindings are not required to be quasi-affine.
+ *
+ * \param iter_vars The input iterators
+ * \param bindings The values of iter_vars
+ * \param outer_loops Iterators outside the subspace.
+ * \param inner_loops Iterators of the subspace
+ * \param predicate The predicate constaints on the input iterators.
+ * \return The result of the subspace division.
+ */
+Array<Array<arith::IterMark>> TrivialSubspaceDivision(const Array<IterVar>& iter_vars,
+ const Array<PrimExpr>& bindings,
+ const Array<Var>& outer_iters,
+ const Array<Var>& inner_iters,
+ const PrimExpr& predicate) {
+ if (!is_one(predicate)) return {};
+ std::vector<Array<arith::IterMark>> res;
+ std::unordered_set<const VarNode*> outer_loop_vars;
+ std::unordered_set<const VarNode*> inner_loop_vars;
+ for (const Var& var : outer_iters) {
+ outer_loop_vars.insert(var.get());
+ }
+ for (const Var& var : inner_iters) {
+ inner_loop_vars.insert(var.get());
+ }
+ const arith::IterMark unit_iter_mark(arith::IterSumExpr({}, 0), 1);
+
+ for (size_t i = 0; i < bindings.size(); ++i) {
+ bool outer = UsesVar(
+ bindings[i], [&outer_loop_vars](const VarNode* var) { return outer_loop_vars.count(var); });
+ bool inner = UsesVar(
+ bindings[i], [&inner_loop_vars](const VarNode* var) { return inner_loop_vars.count(var); });
+ arith::IterMark iter_mark;
+ if (bindings[i]->IsInstance<VarNode>()) {
+ iter_mark = arith::IterMark(
+ arith::IterSplitExpr(arith::IterMark(bindings[i], iter_vars[i]->dom->extent)),
+ iter_vars[i]->dom->extent);
+ } else {
+ iter_mark = arith::IterMark(arith::IterSumExpr({}, bindings[i]), iter_vars[i]->dom->extent);
+ }
+ if (outer && !inner) {
+ arith::IterMark outer{nullptr};
+ const auto& outer_iter = iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (inner && !outer) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else if (!outer && !inner) {
+ const auto& outer_iter = unit_iter_mark;
+ const auto& inner_iter = unit_iter_mark;
+ res.push_back({outer_iter, inner_iter});
+ } else {
+ return {};
+ }
+ }
+ res.push_back({arith::IterMark(arith::IterSumExpr({}, 0), Bool(true)),
+ arith::IterMark(arith::IterSumExpr({}, 0), Bool(true))});
+ return res;
+}
+
+/*!
+ * \brief Generate the blockized init block.
+ * \param block The original block with init.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \return The subtree of the init block and its outer loops.
+ */
+Stmt GenerateBlockizedInit(const Block& block, const BlockRealize& inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops) {
+ Array<IterVar> init_block_iters;
+ Array<PrimExpr> init_bindings;
+ const Block& inner_block = inner_block_realize->block;
+
+ // Step 1: Collect data-parallel block iters
+ for (size_t i = 0; i < inner_block->iter_vars.size(); i++) {
+ const IterVar& iter_var = inner_block->iter_vars[i];
+ const PrimExpr& binding = inner_block_realize->iter_values[i];
+ if (iter_var->iter_type == IterVarType::kDataPar &&
+ UsesVar(block->init.value(),
+ [&iter_var](const VarNode* var) { return var == iter_var->var.get(); })) {
+ init_block_iters.push_back(iter_var);
+ init_bindings.push_back(binding);
+ }
+ }
+
+ // Step 2: Collect loops related to iters of the init block
+ std::vector<const ForNode*> init_loops;
+ for (const ForNode* inner_loop : inner_loops) {
+ for (const PrimExpr& init_binding : init_bindings) {
+ if (UsesVar(init_binding,
+ [inner_loop](const VarNode* var) { return var == inner_loop->loop_var.get(); })) {
+ init_loops.push_back(inner_loop);
+ }
+ }
+ }
+
+ // Step 3: Create new block iters for the init block
+ Map<Var, PrimExpr> subst_map;
+ for (size_t i = 0; i < init_block_iters.size(); i++) {
+ IterVar new_iter_var = init_block_iters[i];
+ auto* new_init_var_node = new_iter_var.CopyOnWrite();
+ Var old_var = new_iter_var->var;
+ new_init_var_node->var = old_var.copy_with_suffix("_init");
+ subst_map.Set(old_var, new_iter_var->var);
+ init_block_iters.Set(i, std::move(new_iter_var));
+ }
+
+ // Step 4: Generate loop nests and the init block
+ Block init_block{/*iter_vars=*/init_block_iters, //
+ /*reads=*/{}, //
+ /*writes=*/block->writes, //
+ /*name_hint=*/block->name_hint + "_init", //
+ /*body=*/block->init.value(), //
+ /*init=*/NullOpt};
+ Stmt new_init = BlockRealize(
+ /*iter_values=*/init_bindings,
+ /*predicate=*/inner_block_realize->predicate,
+ /*block=*/std::move(init_block));
+
+ // Step 5: Generate the parent loops for the init block
+ for (const ForNode* init_loop : init_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*init_loop);
+ new_loop->loop_var = init_loop->loop_var.copy_with_suffix("");
+ subst_map.Set(init_loop->loop_var, new_loop->loop_var);
+ new_loop->body = std::move(new_init);
+ new_init = For(new_loop);
+ }
+
+ // Step 6: Substitute with new loop variables and block iters to prevent duplication of
+ // variables in the outer block.
+ new_init = Substitute(new_init, subst_map);
+
+ return new_init;
+}
+
+/*!
+ * \brief A helper to collect the parent loops of the block. The loops are divided into two groups,
+ * 'outer_loops', and 'inner_loops', by a specified loop as the separator. 'outer_loops' are the
+ * ancestor loops of the separator loop. 'inner_loops' include the separator loop itself, and its
+ * successor loops. It is possible that 'outer_loops' is empty.
+ */
+class LoopSubspaceCollector {
+ public:
+ /*!
+ * \brief Collect the parent loops of the block and store the result in the corresponding fields.
+ * \param block_sref The sref to the target block.
+ * \param loop_sref The sref to the separator loop.
+ */
+ void Collect(const StmtSRef& block_sref, const StmtSRef& loop_sref) {
+ bool inner = true;
+ for (StmtSRefNode* current_sref = block_sref->parent;
+ current_sref && current_sref->stmt->IsInstance<ForNode>();
+ current_sref = current_sref->parent) {
+ const auto* current_loop = current_sref->StmtAs<ForNode>();
+ ICHECK(current_loop);
+ if (inner) {
+ inner_loops.push_back(current_loop);
+ inner_loop_vars.push_back(current_loop->loop_var);
+ } else {
+ outer_loops.push_back(current_loop);
+ outer_loop_vars.push_back(current_loop->loop_var);
+ }
+ loop_var_domain.Set(current_loop->loop_var,
+ Range::FromMinExtent(current_loop->min, current_loop->extent));
+ if (current_sref == loop_sref.get()) inner = false;
+ }
+ }
+ /*! \brief Outer loops which are ancestors of the separator. */
+ std::vector<const ForNode*> outer_loops;
+ /*! \brief Inner loops which are the separator itself or its successors. */
+ std::vector<const ForNode*> inner_loops;
+ /*! \brief Loop variables of the outer loops. */
+ Array<Var> outer_loop_vars;
+ /*! \brief Loop variables of the inner loops. */
+ Array<Var> inner_loop_vars;
+ /*! \brief Domain of the loop variables. */
+ Map<Var, Range> loop_var_domain;
+};
+
+/*!
+ * \brief Check the bindings of the block iters can be divided by a subspace collected by the
+ * collector.
+ * \param mod The current IR module.
+ * \param block_realize The block realize to be checked.
+ * \param collector The collector which has collected the loops of the block.
+ * \param analyzer The arithmetic analyzer.
+ * \return The result of the subspace division.
+ * \throws ScheduleError If the bindings are not divisible by the subspace.
+ */
+Array<Array<arith::IterMark>> CheckSubspaceDivisible(const IRModule& mod,
+ const BlockRealize& block_realize,
+ const LoopSubspaceCollector& collector,
+ arith::Analyzer* analyzer) {
+ const Block& block = block_realize->block;
+ DiagnosticContext diag_ctx(DiagnosticContext::Default(mod));
+
+ Array<Array<arith::IterMark>> division =
+ arith::SubspaceDivide(block_realize->iter_values, collector.loop_var_domain,
+ collector.inner_loop_vars, block_realize->predicate,
+ /*require_bijective=*/false, analyzer, diag_ctx);
+
+ if (division.empty()) {
+ // If we can't do perfect subspace division, check if it is a trivial case of subspace division.
+ // In this case, we can still blockize.
+ division = TrivialSubspaceDivision(block->iter_vars, block_realize->iter_values,
+ collector.outer_loop_vars, collector.inner_loop_vars,
+ block_realize->predicate);
+ }
+ if (division.empty()) {
+ throw SubspaceNotDivisibleError(mod, GetRef<For>(collector.inner_loops.back()), block);
+ }
+ return division;
+}
+
+/*!
+ * \brief The binding extractor to compute the bindings of the outer and the inner blocks after
+ * blockize.
+ */
+class BlockizedBindingExtractor {
+ public:
+ /*!
+ * \brief Extract bindings for blockize.
+ * \param iter_vars The iter vars of the original inner block.
+ * \param division The result of the subspace division.
+ */
+ void ExtractBindings(const Array<IterVar>& iter_vars,
+ const Array<Array<arith::IterMark>>& division) {
+ ICHECK(iter_vars.size() + 1 == division.size());
+ for (size_t i = 0; i < iter_vars.size(); ++i) {
+ const IterVar& iter_var = iter_vars[i];
+ const arith::IterMapExprNode* outer_binding =
+ division[i][0]->source.as<arith::IterMapExprNode>();
+ const arith::IterMapExprNode* inner_binding =
+ division[i][1]->source.as<arith::IterMapExprNode>();
+ ICHECK(outer_binding);
+ ICHECK(inner_binding);
+
+ // After computing the subspace division, bindings[i] can be written as
+ // outer_binding * inner_binding->extent + inner_binding
+ // The outer block will have binding: iter_outer -> outer_binding
+ // The inner block will have binding: iter_inner -> iter_outer * inner_binding->extent +
+ // inner_binding
+
+ if (is_one(division[i][1]->extent)) { // IsOuter
+ // extract this iter var to outer block directly
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(iter_var);
+ } else {
+ const IterVar outer_var(Range::FromMinExtent(0, division[i][0]->extent),
+ iter_var->var.copy_with_suffix("o"), iter_var->iter_type);
+ outer_bindings.push_back(
+ arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(outer_binding)));
+ outer_iter_vars.push_back(outer_var);
+ PrimExpr base = is_one(division[i][0]->extent) ? 0 : outer_var * division[i][1]->extent;
+ if (const auto* op = division[i][1]->source.as<arith::IterSumExprNode>()) {
+ base = base + op->base;
+ inner_bindings.push_back(base +
+ arith::NormalizeIterMapToExpr(arith::IterSumExpr(op->args, 0)));
+ } else {
+ inner_bindings.push_back(
+ base + arith::NormalizeIterMapToExpr(GetRef<arith::IterMapExpr>(inner_binding)));
+ }
+ inner_iter_vars.push_back(iter_var);
+ inner_iter_relaxed_range.Set(iter_var->var,
+ arith::IntSet::FromMinExtent(base, division[i][1]->extent));
+ }
+ }
+ }
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> outer_iter_vars;
+ /*! \brief Iters of the outer block. */
+ Array<IterVar> inner_iter_vars;
+ /*! \brief Binding values of the outer block. */
+ Array<PrimExpr> outer_bindings;
+ /*! \brief Binding values of the inner block. */
+ Array<PrimExpr> inner_bindings;
+
+ /*! \brief The range of the inner block iters Note that this is different from the domain of the
+ * inner block iters.
+ */
+ Map<Var, arith::IntSet> inner_iter_relaxed_range;
+};
+
+/*!
+ * \brief Compute the access region of the outer block by relaxing the inner loops.
+ * \param buffer_region The original buffer region.
+ * \param The range of the inner loops.
+ * \param analyzer The arithmetic analyzer.
+ * \return The new buffer region.
+ */
+BufferRegion RelaxBlockizedInnerIters(const BufferRegion& buffer_region,
+ const Map<Var, arith::IntSet>& inner_iter_relaxed_range,
+ arith::Analyzer* analyzer) {
+ Array<Range> new_region;
+ new_region.reserve(buffer_region->region.size());
+ Array<arith::IntSet> relaxed_int_set =
+ arith::EvalSet(buffer_region->region, inner_iter_relaxed_range);
+ ICHECK(buffer_region->region.size() == buffer_region->buffer->shape.size());
+ for (size_t i = 0; i < buffer_region->region.size(); i++) {
+ Range max_range = Range::FromMinExtent(0, buffer_region->buffer->shape[i]);
+ new_region.push_back(relaxed_int_set[i].CoverRange(max_range));
+ }
+ return BufferRegion(buffer_region->buffer, std::move(new_region));
+}
+
+/*!
+ * \brief Generate the outer block after blockize.
+ * \param extractor The binding extractor which has extracted the blockized bindings.
+ * \param block The original inner block.
+ * \param inner_block_realize The block realize of the inner block after blockize.
+ * \param inner_loops The inner loops after blockize.
+ * \param predicate The outer predicate of the subspace division.
+ * \param analyzer The arithmetic analyzer.
+ * \return The block realize of the outer block after blockize.
+ */
+BlockRealize GenerateBlockizedOuterBlock(const BlockizedBindingExtractor& extractor,
+ const Block& block, BlockRealize inner_block_realize,
+ const std::vector<const ForNode*>& inner_loops,
+ PrimExpr predicate, arith::Analyzer* analyzer) {
+ // Step 1: Generate the init block if needed
+ Optional<Stmt> new_init = NullOpt;
+ if (block->init.defined()) {
+ new_init = GenerateBlockizedInit(block, inner_block_realize, inner_loops);
+ }
+
+ // Step 2: Compute the access regions of the outer block by relaxing the inner loops
+ Array<BufferRegion> new_reads = block->reads;
+ Array<BufferRegion> new_writes = block->writes;
+
+ auto f_mutate = [&](const BufferRegion& buffer_region) {
+ return RelaxBlockizedInnerIters(buffer_region, extractor.inner_iter_relaxed_range, analyzer);
+ };
+ new_reads.MutateByApply(f_mutate);
+ new_writes.MutateByApply(f_mutate);
+
+ // Step 3: Generate the body of the outer block. The body of the outer block is the inner block
+ // realize and its surounding loops.
+ Stmt outer_block_body = inner_block_realize;
+ for (const ForNode* loop : inner_loops) {
+ ObjectPtr<ForNode> new_loop = make_object<ForNode>(*loop);
+ new_loop->body = std::move(outer_block_body);
+ outer_block_body = For(new_loop);
+ }
+
+ // Step 4: Generate the outer block and block realize.
+ Block outer_block{/*iter_vars=*/extractor.outer_iter_vars, //
+ /*reads=*/new_reads, //
+ /*writes=*/new_writes, //
+ /*name_hint=*/"blockized_" + block->name_hint, //
+ /*body=*/std::move(outer_block_body), //
+ /*init=*/new_init};
+ BlockRealize outer_block_realize{/*iter_values=*/extractor.outer_bindings,
+ /*predicate=*/std::move(predicate),
+ /*block=*/std::move(outer_block)};
+ return outer_block_realize;
+}
+
+StmtSRef Blockize(ScheduleState self, const StmtSRef& loop_sref) {
+ const ForNode* loop = TVM_SREF_TO_FOR(loop, loop_sref);
+ arith::Analyzer analyzer;
+
+ // Step 1: Check the loop has a single child BlockRealize on the sref tree.
+ BlockRealize block_realize = CheckGetSingleChildBlockRealizeOnSRefTree(self, loop_sref);
+ Block block = block_realize->block;
+ StmtSRef block_sref = self->stmt2ref.at(block.get());
+
+ // Step 2: Collect loops inside and outside loop_sref.
+ LoopSubspaceCollector collector;
+ collector.Collect(block_sref, loop_sref);
+
+ // Step 3: Calculate subspace division for the inner loops.
+ Array<Array<arith::IterMark>> division =
+ CheckSubspaceDivisible(self->mod, block_realize, collector, &analyzer);
+
+ // Step 4: Generate bindings for the outer block and the inner block based on the result of
+ // the subspace division.
+ BlockizedBindingExtractor extractor;
+ extractor.ExtractBindings(block->iter_vars, division);
+ const PrimExpr& outer_pred = division.back()[0]->extent;
+ const PrimExpr& inner_pred = division.back()[1]->extent;
+
+ // Step 5: Generate the inner block.
+ BlockRealizeNode* inner_block_realize = block_realize.CopyOnWrite();
+ BlockNode* inner_block = inner_block_realize->block.CopyOnWrite();
+ inner_block_realize->iter_values = extractor.inner_bindings;
+ inner_block_realize->predicate = inner_pred;
+ inner_block->iter_vars = extractor.inner_iter_vars;
+ inner_block->init = NullOpt;
+
+ // Step 6: Generate the outer block.
+ BlockRealize outer_realize =
+ GenerateBlockizedOuterBlock(extractor, block, GetRef<BlockRealize>(inner_block_realize),
+ collector.inner_loops, outer_pred, &analyzer);
+ // Step 7: Do the actual replacement
+ self->Replace(loop_sref, outer_realize, {{block, GetRef<Block>(inner_block)}});
+
+ // Step 8: Update the cached flags
+ const StmtSRef& outer_block_sref = self->stmt2ref.at(outer_realize->block.get());
+ StmtSRef scope_root = tir::GetScopeRoot(self, outer_block_sref, /*require_stage_pipeline=*/false,
+ /*require_subtree_compact_dataflow=*/false);
+ BlockInfo old_block_info = self->GetBlockInfo(scope_root);
+ self->UpdateScopeBlockInfo(tir::GetBlockRealize(self, scope_root));
+ // 'affine_binding' depends on the outer loops and are not changed.
+ self->block_info[scope_root].affine_binding = old_block_info.affine_binding;
+ return outer_block_sref;
+}
+
+/*!
+ * \brief Update the map from the buffers in the description to the implementation of the tensor
+ * intrinsic. \param intrinsic The tensor intrinsic. \param buffer_map The map to be updated.
+ */
+void RemapTensorIntrinBuffers(
+ const TensorIntrin& intrinsic,
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual>* buffer_map) {
+ ICHECK_EQ(intrinsic->description->params.size(), intrinsic->implementation->params.size());
+ for (size_t i = 0; i < intrinsic->description->params.size(); ++i) {
+ const auto& lhs_var = intrinsic->description->params[i];
+ const auto& lhs_buffer = intrinsic->description->buffer_map[lhs_var];
+ const auto& rhs_var = intrinsic->implementation->params[i];
+ const auto& rhs_buffer = intrinsic->implementation->buffer_map[rhs_var];
+ (*buffer_map)[rhs_buffer] = lhs_buffer;
+ }
+}
+
+void Tensorize(ScheduleState self, const StmtSRef& loop_sref, const TensorIntrin& intrinsic) {
+ /*!
+ * Check:
+ * - Check buffer binding, including type, alignment, shape and etc.
+ * - Check the sub AST is equal to the description function.
+ *
+ * Mutate:
+ * - Blockize the sub AST (please refer blockize for details)
+ * - Bind buffers
+ * - Mutate the implementation of the tensor intrinsic by replacing its buffers with new
+ * buffers created via match buffer region.
+ * - Replace the sub tree with the mutated function.
+ */
+ const auto* loop = loop_sref->StmtAs<ForNode>();
+ CHECK(loop) << "Only support tensorize a loop for now";
+
+ const auto* desc_block_realize =
+ Downcast<BlockRealize>(intrinsic->description->body)->block->body.as<BlockRealizeNode>();
+ const Block& desc_block = desc_block_realize->block;
+ const auto* impl_block_realize =
+ Downcast<BlockRealize>(intrinsic->implementation->body)->block->body.as<BlockRealizeNode>();
+ Block impl_block = impl_block_realize->block;
+
+ // Step 1: Blockize the subtree rooted at the given loop
+ const StmtSRef& block_sref = Blockize(self, loop_sref);
+ const BlockRealize& block_realize = GetBlockRealize(self, block_sref);
+
+ // Step 2: Compare the block with the description of the tensor intrinsic, find the correspondence
+ // between buffers in the block and the description.
+ TensorizeComparator comparator(/*assert_mode=*/true);
+ comparator.VisitStmt(block_realize, GetRef<Stmt>(desc_block_realize));
+
+ // Step 3: Find the correspondence between buffers in the current AST and the implementation of
+ // the tensor intrinsic
+
+ // Step 3.1: Map from intrinsic func buffer to description func buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> intrin_buffer_map;
+ RemapTensorIntrinBuffers(intrinsic, &intrin_buffer_map);
+ // Step 3.2: Map form intrinsic func buffer to current AST buffer
+ std::unordered_map<Buffer, Buffer, ObjectPtrHash, ObjectPtrEqual> buffer_map;
+ for (const auto& pair : intrin_buffer_map) {
+ auto it = comparator.rhs_buffer_map_.find(pair.second);
+ ICHECK(it != comparator.rhs_buffer_map_.end());
+ buffer_map[pair.first] = it->second;
+ }
+
+ // Step 4: Create MatchBufferRegion for the params of the implementation function of the tensor
+ // intrin to make them subregions of the buffer in the original IR.
+ std::unordered_map<Buffer, Array<Range>, ObjectPtrHash, ObjectPtrEqual> buffer_region_map;
+ for (const auto& read : impl_block->reads) {
+ buffer_region_map.emplace(read->buffer, read->region);
+ }
+ for (const auto& write : impl_block->writes) {
+ buffer_region_map.emplace(write->buffer, write->region);
+ }
+ Array<MatchBufferRegion> match_buffer_regions;
+ for (size_t i = 0; i < intrinsic->implementation->params.size(); ++i) {
+ const auto& param = intrinsic->implementation->params[i];
+ const auto& buffer = intrinsic->implementation->buffer_map.at(param);
+ const auto& source = buffer_map.at(buffer);
+ Region region = buffer_region_map.at(buffer);
+ auto extra_indices = comparator.buffer_indices_.at(source);
+ std::vector<Range> extra_buffer_ranges;
+ std::transform(extra_indices.begin(), extra_indices.end(),
+ std::back_inserter(extra_buffer_ranges),
+ [](const PrimExpr& index) { return Range::FromMinExtent(index, 1); });
+ region.insert(region.begin(), extra_buffer_ranges.begin(), extra_buffer_ranges.end());
+ match_buffer_regions.push_back(MatchBufferRegion(buffer, BufferRegion(source, region)));
+ }
+
+ // Step 5: Replace the subtree in the original IR with the tensor intrin implementation.
+ ObjectPtr<BlockNode> new_block_ptr = make_object<BlockNode>(*block_realize->block.get());
+ new_block_ptr->body = impl_block->body;
+ ICHECK(new_block_ptr->match_buffers.empty());
+ new_block_ptr->match_buffers = std::move(match_buffer_regions);
+ Block new_block(new_block_ptr);
+
+ // Substitution map for the intrin block to get the correct bindings.
+ Map<Var, PrimExpr> bv_map;
+ for (size_t i = 0; i < desc_block->iter_vars.size(); ++i) {
+ auto it = comparator.equal_map_.find(desc_block->iter_vars[i]->var);
+ if (it != comparator.equal_map_.end()) {
+ bv_map.Set(impl_block->iter_vars[i]->var, Downcast<PrimExpr>(it->second));
+ } else {
+ bv_map.Set(impl_block->iter_vars[i]->var, Integer(0));
+ }
+ }
+ new_block = Downcast<Block>(SubstituteInScope(new_block, bv_map));
+ self->Replace(block_sref, new_block, {{block_realize->block, new_block}});
Review comment:
it's possible, we need to modify `blockize` to have a dry-run version that returns the new block directly and doesn't call `Replace`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org