You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by ha...@apache.org on 2016/07/27 16:52:11 UTC
incubator-quickstep git commit: Additional changes for adding fuse
optimization: Changed HashJoinWorkOrder layout etc.
Repository: incubator-quickstep
Updated Branches:
refs/heads/selection-probe-fuse 590b6f012 -> 8f705ed54
Additional changes for adding fuse optimization: Changed HashJoinWorkOrder layout etc.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8f705ed5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8f705ed5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8f705ed5
Branch: refs/heads/selection-probe-fuse
Commit: 8f705ed54ff6f0e3eadb37e48506d74839f33eaf
Parents: 590b6f0
Author: Hakan Memisoglu <ha...@apache.org>
Authored: Wed Jul 27 11:50:56 2016 -0500
Committer: Hakan Memisoglu <ha...@apache.org>
Committed: Wed Jul 27 11:50:56 2016 -0500
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 1 +
query_optimizer/ExecutionGenerator.cpp | 9 ++++-
query_optimizer/PhysicalGenerator.cpp | 3 +-
query_optimizer/physical/HashJoin.cpp | 6 +++
query_optimizer/physical/HashJoin.hpp | 34 ++++++++++++++++-
query_optimizer/rules/FuseJoinSelect.cpp | 51 ++++++++++++++------------
query_optimizer/rules/FuseJoinSelect.hpp | 13 +++++--
relational_operators/HashJoinOperator.cpp | 6 ++-
relational_operators/HashJoinOperator.hpp | 20 ++++------
9 files changed, 99 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f705ed5/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 7e53b9d..65368c3 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -195,6 +195,7 @@ target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
quickstep_queryoptimizer_LogicalToPhysicalMapper
quickstep_queryoptimizer_logical_Logical
quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_rules_FuseJoinSelect
quickstep_queryoptimizer_rules_PruneColumns
quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
quickstep_queryoptimizer_strategy_Aggregate
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f705ed5/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index ac8dc62..9f6c5f2 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -695,6 +695,13 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
query_context_proto_->add_predicates()->CopyFrom(residual_predicate->getProto());
}
+ QueryContext::predicate_id filter_predicate_index = QueryContext::kInvalidPredicateId;
+ if (physical_plan->filter_predicate()) {
+ filter_predicate_index = query_context_proto_->predicates_size();
+ unique_ptr<const Predicate> filter_predicate(convertPredicate(physical_plan->filter_predicate()));
+ query_context_proto_->add_predicates()->CopyFrom(filter_predicate->getProto());
+ }
+
// Convert the project expressions proto.
const QueryContext::scalar_group_id project_expressions_group_index =
query_context_proto_->scalar_groups_size();
@@ -802,7 +809,7 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
project_expressions_group_index,
is_selection_on_build.get(),
join_type,
- nullptr /* filter predicate */));
+ filter_predicate_index));
insert_destination_proto->set_relational_op_index(join_operator_index);
const QueryPlan::DAGNodeIndex destroy_operator_index =
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f705ed5/query_optimizer/PhysicalGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/PhysicalGenerator.cpp b/query_optimizer/PhysicalGenerator.cpp
index 731c17d..41d33a9 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -26,6 +26,7 @@
#include "query_optimizer/Validator.hpp"
#include "query_optimizer/logical/Logical.hpp"
#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/rules/FuseJoinSelect.hpp"
#include "query_optimizer/rules/PruneColumns.hpp"
#include "query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp"
#include "query_optimizer/strategy/Aggregate.hpp"
@@ -99,7 +100,7 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
}
rules.emplace_back(new PruneColumns());
- rules.emplace_back(new Fuse());
+ rules.emplace_back(new FuseJoinSelect());
for (std::unique_ptr<Rule<P::Physical>> &rule : rules) {
physical_plan_ = rule->apply(physical_plan_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f705ed5/query_optimizer/physical/HashJoin.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/HashJoin.cpp b/query_optimizer/physical/HashJoin.cpp
index 71c3692..6420940 100644
--- a/query_optimizer/physical/HashJoin.cpp
+++ b/query_optimizer/physical/HashJoin.cpp
@@ -102,10 +102,16 @@ void HashJoin::getFieldStringItems(
non_container_child_field_names->push_back("residual_predicate");
non_container_child_fields->push_back(residual_predicate_);
}
+ if (filter_predicate_ != nullptr) {
+ non_container_child_field_names->push_back("right_filter_predicate");
+ non_container_child_fields->push_back(filter_predicate_);
+ }
+
container_child_field_names->push_back("left_join_attributes");
container_child_fields->push_back(CastSharedPtrVector<OptimizerTreeBase>(left_join_attributes_));
container_child_field_names->push_back("right_join_attributes");
container_child_fields->push_back(CastSharedPtrVector<OptimizerTreeBase>(right_join_attributes_));
+
}
} // namespace physical
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f705ed5/query_optimizer/physical/HashJoin.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/HashJoin.hpp b/query_optimizer/physical/HashJoin.hpp
index 6761ac6..7fb6e29 100644
--- a/query_optimizer/physical/HashJoin.hpp
+++ b/query_optimizer/physical/HashJoin.hpp
@@ -100,6 +100,13 @@ class HashJoin : public BinaryJoin {
}
/**
+ * @brief Fused filter predicate.
+ */
+ const expressions::PredicatePtr& filter_predicate() const {
+ return filter_predicate_;
+ }
+
+ /**
* @return Join type of this hash join.
*/
JoinType join_type() const {
@@ -155,6 +162,27 @@ class HashJoin : public BinaryJoin {
join_type));
}
+ static HashJoinPtr CreateWithFusedSelect(
+ const PhysicalPtr &left,
+ const PhysicalPtr &right,
+ const std::vector<expressions::AttributeReferencePtr> &left_join_attributes,
+ const std::vector<expressions::AttributeReferencePtr> &right_join_attributes,
+ const expressions::PredicatePtr &residual_predicate,
+ const std::vector<expressions::NamedExpressionPtr> &project_expressions,
+ const JoinType join_type,
+ const expressions::PredicatePtr &filter_predicate) {
+ return HashJoinPtr(
+ new HashJoin(left,
+ right,
+ left_join_attributes,
+ right_join_attributes,
+ residual_predicate,
+ project_expressions,
+ join_type,
+ filter_predicate));
+
+ }
+
protected:
void getFieldStringItems(
std::vector<std::string> *inline_field_names,
@@ -172,12 +200,14 @@ class HashJoin : public BinaryJoin {
const std::vector<expressions::AttributeReferencePtr> &right_join_attributes,
const expressions::PredicatePtr &residual_predicate,
const std::vector<expressions::NamedExpressionPtr> &project_expressions,
- const JoinType join_type)
+ const JoinType join_type,
+ const expressions::PredicatePtr &filter_predicate = nullptr)
: BinaryJoin(left, right, project_expressions),
left_join_attributes_(left_join_attributes),
right_join_attributes_(right_join_attributes),
residual_predicate_(residual_predicate),
- join_type_(join_type) {
+ join_type_(join_type),
+ filter_predicate_(filter_predicate) {
}
std::vector<expressions::AttributeReferencePtr> left_join_attributes_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f705ed5/query_optimizer/rules/FuseJoinSelect.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/FuseJoinSelect.cpp b/query_optimizer/rules/FuseJoinSelect.cpp
index a776578..5dfe015 100644
--- a/query_optimizer/rules/FuseJoinSelect.cpp
+++ b/query_optimizer/rules/FuseJoinSelect.cpp
@@ -1,40 +1,45 @@
+#include <iostream>
+
#include "query_optimizer/rules/FuseJoinSelect.hpp"
+#include "query_optimizer/expressions/Predicate.hpp"
#include "query_optimizer/physical/HashJoin.hpp"
#include "query_optimizer/physical/PatternMatcher.hpp"
#include "query_optimizer/physical/Selection.hpp"
+#include "query_optimizer/physical/TableReference.hpp"
+#include "query_optimizer/rules/Rule.hpp"
namespace quickstep {
namespace optimizer {
namespace P = ::quickstep::optimizer::physical;
+namespace E = ::quickstep::optimizer::expressions;
-P::PhysicalPtr FuseJoinSelect::apply(const P::PhysicalPtr &input) {
- DCHECK(input->getPhysicalType() == P::PhysicalType::kTopLevelPlan);
-
- return applyInternal(input);
-}
+P::PhysicalPtr FuseJoinSelect::applyToNode(const P::PhysicalPtr &input) {
-P::PhysicalPtr FuseJoinSelect::applyInternal(const P::PhysicalPtr &input) {
P::HashJoinPtr hash_join;
- const bool is_hash_inner_join =
- P::SomeHashJoin::MatchesWithConditionalCast(input, &hash_join)
- && hash_join->join_type() == P::HashJoin::JoinType::kInnerJoin;
-
- bool fuse_right = false;
-
- if (is_hash_inner_join) {
- //auto &left = hash_join->left();
- auto &right = hash_join->right();
- P::SelectionPtr selection;
- const bool is_right_child_selection =
- P::SomeSelection::MatchesWithConditionalCast(right, &selection);
- if (is_right_child_selection) {
- if (selection->input() != nullptr) {
-
- }
- }
+ P::SelectionPtr selection;
+ P::TableReferencePtr table_reference;
+
+ if (P::SomeHashJoin::MatchesWithConditionalCast(input, &hash_join)
+ && hash_join->join_type() == P::HashJoin::JoinType::kInnerJoin
+ && P::SomeSelection::MatchesWithConditionalCast(hash_join->right(), &selection)
+ && P::SomeTableReference::MatchesWithConditionalCast(selection->input(), &table_reference)) {
+ const E::PredicatePtr filter_predicate = selection->filter_predicate();
+ P::PhysicalPtr output = P::HashJoin::CreateWithFusedSelect(hash_join->left(),
+ table_reference,
+ hash_join->left_join_attributes(),
+ hash_join->right_join_attributes(),
+ hash_join->residual_predicate(),
+ hash_join->project_expressions(),
+ hash_join->join_type(),
+ filter_predicate);
+ LOG_APPLYING_RULE(input, output);
+ return output;
}
+
+ LOG_IGNORING_RULE(input);
+ return input;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f705ed5/query_optimizer/rules/FuseJoinSelect.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/FuseJoinSelect.hpp b/query_optimizer/rules/FuseJoinSelect.hpp
index d2ed5d8..4b8210e 100644
--- a/query_optimizer/rules/FuseJoinSelect.hpp
+++ b/query_optimizer/rules/FuseJoinSelect.hpp
@@ -1,23 +1,30 @@
#ifndef QUICKSTEP_QUERY_OPTIMIZER_RULES_FUSE_JOIN_SELECT_HPP_
#define QUICKSTEP_QUERY_OPTIMIZER_RULES_FUSE_JOIN_SELECT_HPP_
+#include <string>
+
#include "query_optimizer/physical/Physical.hpp"
#include "query_optimizer/rules/Rule.hpp"
+#include "query_optimizer/rules/BottomUpRule.hpp"
+#include "utility/Macros.hpp"
namespace quickstep {
namespace optimizer {
namespace P = ::quickstep::optimizer::physical;
-class FuseJoinSelect : public Rule<physical::Physical> {
+class FuseJoinSelect : public BottomUpRule<P::Physical> {
public:
FuseJoinSelect() {
}
- P::PhysicalPtr apply(const P::PhysicalPtr &input) override;
+ std::string getName() const override { return "FuseJoinSelect"; }
+
+ protected:
+ P::PhysicalPtr applyToNode(const P::PhysicalPtr &input) override;
private:
- P::PhysicalPtr applyInternal(const P::PhysicalPtr &input);
+ DISALLOW_COPY_AND_ASSIGN(FuseJoinSelect);
};
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f705ed5/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index 2dec2c5..6b70a33 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -201,6 +201,8 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders(
const Predicate *residual_predicate =
query_context->getPredicate(residual_predicate_index_);
+ const Predicate *filter_predicate =
+ query_context->getPredicate(filter_predicate_index_);
const vector<unique_ptr<const Scalar>> &selection =
query_context->getScalarGroup(selection_index_);
InsertDestination *output_destination =
@@ -223,7 +225,7 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders(
hash_table,
output_destination,
storage_manager,
- filter_predicate_),
+ filter_predicate),
op_index_);
}
started_ = true;
@@ -244,7 +246,7 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders(
hash_table,
output_destination,
storage_manager,
- filter_predicate_),
+ filter_predicate),
op_index_);
++num_workorders_generated_;
} // end while
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f705ed5/relational_operators/HashJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.hpp b/relational_operators/HashJoinOperator.hpp
index d67e11c..7e445c5 100644
--- a/relational_operators/HashJoinOperator.hpp
+++ b/relational_operators/HashJoinOperator.hpp
@@ -130,7 +130,7 @@ class HashJoinOperator : public RelationalOperator {
const QueryContext::scalar_group_id selection_index,
const std::vector<bool> *is_selection_on_build = nullptr,
const JoinType join_type = JoinType::kInnerJoin,
- const Predicate *filter_predicate = nullptr)
+ const QueryContext::predicate_id filter_predicate_index = QueryContext::kInvalidPredicateId)
: RelationalOperator(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -146,7 +146,7 @@ class HashJoinOperator : public RelationalOperator {
? std::vector<bool>()
: *is_selection_on_build),
join_type_(join_type),
- filter_predicate_(filter_predicate),
+ filter_predicate_index_(filter_predicate_index),
probe_relation_block_ids_(probe_relation_is_stored
? probe_relation.getBlocksSnapshot()
: std::vector<block_id>()),
@@ -237,7 +237,7 @@ class HashJoinOperator : public RelationalOperator {
const QueryContext::scalar_group_id selection_index_;
const std::vector<bool> is_selection_on_build_;
const JoinType join_type_;
- const Predicate *filter_predicate_;
+ const QueryContext::predicate_id filter_predicate_index_;
std::vector<block_id> probe_relation_block_ids_;
std::size_t num_workorders_generated_;
@@ -419,7 +419,7 @@ class HashSemiJoinWorkOrder : public WorkOrder {
const JoinHashTable &hash_table,
InsertDestination *output_destination,
StorageManager *storage_manager,
- const Predicate *filter_predicate = nullptr)
+ const Predicate *filter_predicate = nullptr /* Not used for now. */)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -430,8 +430,7 @@ class HashSemiJoinWorkOrder : public WorkOrder {
selection_(selection),
hash_table_(hash_table),
output_destination_(DCHECK_NOTNULL(output_destination)),
- storage_manager_(DCHECK_NOTNULL(storage_manager)),
- filter_predicate_(filter_predicate) {}
+ storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
/**
* @brief Constructor for the distributed version.
@@ -501,7 +500,7 @@ class HashSemiJoinWorkOrder : public WorkOrder {
InsertDestination *output_destination_;
StorageManager *storage_manager_;
- const Predicate *filter_predicate_;
+ // const Predicate *filter_predicate_;
DISALLOW_COPY_AND_ASSIGN(HashSemiJoinWorkOrder);
};
@@ -547,7 +546,7 @@ class HashAntiJoinWorkOrder : public WorkOrder {
const JoinHashTable &hash_table,
InsertDestination *output_destination,
StorageManager *storage_manager,
- const Predicate *filter_predicate)
+ const Predicate *filter_predicate = nullptr)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -558,8 +557,7 @@ class HashAntiJoinWorkOrder : public WorkOrder {
selection_(selection),
hash_table_(hash_table),
output_destination_(DCHECK_NOTNULL(output_destination)),
- storage_manager_(DCHECK_NOTNULL(storage_manager)),
- filter_predicate_(filter_predicate) {}
+ storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
/**
* @brief Constructor for the distributed version.
@@ -635,8 +633,6 @@ class HashAntiJoinWorkOrder : public WorkOrder {
InsertDestination *output_destination_;
StorageManager *storage_manager_;
- const Predicate *filter_predicate_;
-
DISALLOW_COPY_AND_ASSIGN(HashAntiJoinWorkOrder);
};