You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by zu...@apache.org on 2017/01/18 21:16:04 UTC
[1/3] incubator-quickstep git commit: Marked LIP as a non-default
argument. [Forced Update!]
Repository: incubator-quickstep
Updated Branches:
refs/heads/feedInputBlock-part-id d8dff98f6 -> ee3b7f0e9 (forced update)
Marked LIP as a non-default argument.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/bed1e261
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/bed1e261
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/bed1e261
Branch: refs/heads/feedInputBlock-part-id
Commit: bed1e2616061f1841fd106f4f54463f7ebfe888b
Parents: 60e3406
Author: Zuyu Zhang <zu...@apache.org>
Authored: Tue Jan 17 17:38:50 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Tue Jan 17 18:40:22 2017 -0800
----------------------------------------------------------------------
relational_operators/AggregationOperator.hpp | 2 +-
relational_operators/BuildHashOperator.hpp | 4 ++--
relational_operators/HashJoinOperator.hpp | 12 ++++++------
relational_operators/SelectOperator.hpp | 4 ++--
storage/AggregationOperationState.hpp | 2 +-
5 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bed1e261/relational_operators/AggregationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/AggregationOperator.hpp b/relational_operators/AggregationOperator.hpp
index ce6015d..c4e887d 100644
--- a/relational_operators/AggregationOperator.hpp
+++ b/relational_operators/AggregationOperator.hpp
@@ -137,7 +137,7 @@ class AggregationWorkOrder : public WorkOrder {
AggregationWorkOrder(const std::size_t query_id,
const block_id input_block_id,
AggregationOperationState *state,
- LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr)
+ LIPFilterAdaptiveProber *lip_filter_adaptive_prober)
: WorkOrder(query_id),
input_block_id_(input_block_id),
state_(DCHECK_NOTNULL(state)),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bed1e261/relational_operators/BuildHashOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/BuildHashOperator.hpp b/relational_operators/BuildHashOperator.hpp
index f0f42e3..074c1e1 100644
--- a/relational_operators/BuildHashOperator.hpp
+++ b/relational_operators/BuildHashOperator.hpp
@@ -166,7 +166,7 @@ class BuildHashWorkOrder : public WorkOrder {
const block_id build_block_id,
JoinHashTable *hash_table,
StorageManager *storage_manager,
- LIPFilterBuilder *lip_filter_builder = nullptr)
+ LIPFilterBuilder *lip_filter_builder)
: WorkOrder(query_id),
input_relation_(input_relation),
join_key_attributes_(join_key_attributes),
@@ -196,7 +196,7 @@ class BuildHashWorkOrder : public WorkOrder {
const block_id build_block_id,
JoinHashTable *hash_table,
StorageManager *storage_manager,
- LIPFilterBuilder *lip_filter_builder = nullptr)
+ LIPFilterBuilder *lip_filter_builder)
: WorkOrder(query_id),
input_relation_(input_relation),
join_key_attributes_(std::move(join_key_attributes)),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bed1e261/relational_operators/HashJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.hpp b/relational_operators/HashJoinOperator.hpp
index 8829d1f..190f9d2 100644
--- a/relational_operators/HashJoinOperator.hpp
+++ b/relational_operators/HashJoinOperator.hpp
@@ -302,7 +302,7 @@ class HashInnerJoinWorkOrder : public WorkOrder {
const JoinHashTable &hash_table,
InsertDestination *output_destination,
StorageManager *storage_manager,
- LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr)
+ LIPFilterAdaptiveProber *lip_filter_adaptive_prober)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -352,7 +352,7 @@ class HashInnerJoinWorkOrder : public WorkOrder {
const JoinHashTable &hash_table,
InsertDestination *output_destination,
StorageManager *storage_manager,
- LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr)
+ LIPFilterAdaptiveProber *lip_filter_adaptive_prober)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -438,7 +438,7 @@ class HashSemiJoinWorkOrder : public WorkOrder {
const JoinHashTable &hash_table,
InsertDestination *output_destination,
StorageManager *storage_manager,
- LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr)
+ LIPFilterAdaptiveProber *lip_filter_adaptive_prober)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -488,7 +488,7 @@ class HashSemiJoinWorkOrder : public WorkOrder {
const JoinHashTable &hash_table,
InsertDestination *output_destination,
StorageManager *storage_manager,
- LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr)
+ LIPFilterAdaptiveProber *lip_filter_adaptive_prober)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -570,7 +570,7 @@ class HashAntiJoinWorkOrder : public WorkOrder {
const JoinHashTable &hash_table,
InsertDestination *output_destination,
StorageManager *storage_manager,
- LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr)
+ LIPFilterAdaptiveProber *lip_filter_adaptive_prober)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -620,7 +620,7 @@ class HashAntiJoinWorkOrder : public WorkOrder {
const JoinHashTable &hash_table,
InsertDestination *output_destination,
StorageManager *storage_manager,
- LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr)
+ LIPFilterAdaptiveProber *lip_filter_adaptive_prober)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bed1e261/relational_operators/SelectOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.hpp b/relational_operators/SelectOperator.hpp
index 9c0767b..e58ff81 100644
--- a/relational_operators/SelectOperator.hpp
+++ b/relational_operators/SelectOperator.hpp
@@ -295,7 +295,7 @@ class SelectWorkOrder : public WorkOrder {
const std::vector<std::unique_ptr<const Scalar>> *selection,
InsertDestination *output_destination,
StorageManager *storage_manager,
- LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr,
+ LIPFilterAdaptiveProber *lip_filter_adaptive_prober,
const numa_node_id numa_node = 0)
: WorkOrder(query_id),
input_relation_(input_relation),
@@ -340,7 +340,7 @@ class SelectWorkOrder : public WorkOrder {
const std::vector<std::unique_ptr<const Scalar>> *selection,
InsertDestination *output_destination,
StorageManager *storage_manager,
- LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr,
+ LIPFilterAdaptiveProber *lip_filter_adaptive_prober,
const numa_node_id numa_node = 0)
: WorkOrder(query_id),
input_relation_(input_relation),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bed1e261/storage/AggregationOperationState.hpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.hpp b/storage/AggregationOperationState.hpp
index e0826b0..591e3a1 100644
--- a/storage/AggregationOperationState.hpp
+++ b/storage/AggregationOperationState.hpp
@@ -166,7 +166,7 @@ class AggregationOperationState {
* the block.
**/
void aggregateBlock(const block_id input_block,
- LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr);
+ LIPFilterAdaptiveProber *lip_filter_adaptive_prober);
/**
* @brief Generate the final results for the aggregates managed by this
[3/3] incubator-quickstep git commit: Added partition_id in
feedInputBlock.
Posted by zu...@apache.org.
Added partition_id in feedInputBlock.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ee3b7f0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ee3b7f0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ee3b7f0e
Branch: refs/heads/feedInputBlock-part-id
Commit: ee3b7f0e976523e0421d3003cd7e8cb972fccbbe
Parents: 6c10e99
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Jan 15 19:53:54 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Wed Jan 18 12:45:15 2017 -0800
----------------------------------------------------------------------
query_execution/PolicyEnforcerBase.cpp | 2 +-
query_execution/QueryExecutionMessages.proto | 3 +++
query_execution/QueryManagerBase.cpp | 5 +++--
query_execution/QueryManagerBase.hpp | 5 ++++-
query_execution/tests/QueryManagerSingleNode_unittest.cpp | 4 ++--
relational_operators/AggregationOperator.hpp | 3 ++-
relational_operators/BuildHashOperator.hpp | 4 ++--
relational_operators/DeleteOperator.hpp | 3 ++-
relational_operators/HashJoinOperator.hpp | 4 ++--
relational_operators/NestedLoopsJoinOperator.hpp | 3 ++-
relational_operators/RelationalOperator.hpp | 6 +++---
relational_operators/SampleOperator.hpp | 3 ++-
relational_operators/SaveBlocksOperator.hpp | 3 ++-
relational_operators/SelectOperator.hpp | 5 ++---
relational_operators/SortMergeRunOperator.hpp | 4 ++--
relational_operators/SortRunGenerationOperator.hpp | 3 ++-
relational_operators/tests/SortMergeRunOperator_unittest.cpp | 2 +-
storage/InsertDestination.cpp | 2 +-
storage/InsertDestination.hpp | 4 +++-
19 files changed, 41 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/query_execution/PolicyEnforcerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.cpp b/query_execution/PolicyEnforcerBase.cpp
index 1a2ab46..a26b84e 100644
--- a/query_execution/PolicyEnforcerBase.cpp
+++ b/query_execution/PolicyEnforcerBase.cpp
@@ -118,7 +118,7 @@ void PolicyEnforcerBase::processMessage(const TaggedMessage &tagged_message) {
op_index = proto.operator_index();
admitted_queries_[query_id]->processDataPipelineMessage(
- op_index, proto.block_id(), proto.relation_id());
+ op_index, proto.block_id(), proto.relation_id(), proto.partition_id());
break;
}
case kWorkOrderFeedbackMessage: {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 28b5ebd..115a9a3 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -65,6 +65,9 @@ message DataPipelineMessage {
required fixed64 block_id = 2;
required int32 relation_id = 3;
required uint64 query_id = 4;
+
+ // Used by PartitionAwareInsertDestination.
+ optional uint64 partition_id = 5 [default = 0];
}
// Distributed version related messages.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/query_execution/QueryManagerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.cpp b/query_execution/QueryManagerBase.cpp
index 8e37da8..5f8c6a3 100644
--- a/query_execution/QueryManagerBase.cpp
+++ b/query_execution/QueryManagerBase.cpp
@@ -192,13 +192,14 @@ void QueryManagerBase::processOperator(const dag_node_index index,
void QueryManagerBase::processDataPipelineMessage(const dag_node_index op_index,
const block_id block,
- const relation_id rel_id) {
+ const relation_id rel_id,
+ const partition_id part_id) {
for (const dag_node_index consumer_index :
output_consumers_[op_index]) {
// Feed the streamed block to the consumer. Note that 'output_consumers_'
// only contain those dependents of operator with index = op_index which are
// eligible to receive streamed input.
- query_dag_->getNodePayloadMutable(consumer_index)->feedInputBlock(block, rel_id);
+ query_dag_->getNodePayloadMutable(consumer_index)->feedInputBlock(block, rel_id, part_id);
// Because of the streamed input just fed, check if there are any new
// WorkOrders available and if so, fetch them.
fetchNormalWorkOrders(consumer_index);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/query_execution/QueryManagerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.hpp b/query_execution/QueryManagerBase.hpp
index ddb76d5..d0bb0ea 100644
--- a/query_execution/QueryManagerBase.hpp
+++ b/query_execution/QueryManagerBase.hpp
@@ -109,10 +109,13 @@ class QueryManagerBase {
* for the pipelining block.
* @param block The block id.
* @param rel_id The ID of the relation that produced 'block'.
+ * @param part_id The partition ID of 'block', if any. By default, a block
+ * blongs to the only partition (aka, no partition).
**/
void processDataPipelineMessage(const dag_node_index op_index,
const block_id block,
- const relation_id rel_id);
+ const relation_id rel_id,
+ const partition_id part_id = 0);
/**
* @brief Fetch all work orders currently available in relational operator and
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/query_execution/tests/QueryManagerSingleNode_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/QueryManagerSingleNode_unittest.cpp b/query_execution/tests/QueryManagerSingleNode_unittest.cpp
index 87b8934..28ab388 100644
--- a/query_execution/tests/QueryManagerSingleNode_unittest.cpp
+++ b/query_execution/tests/QueryManagerSingleNode_unittest.cpp
@@ -177,8 +177,8 @@ class MockOperator: public RelationalOperator {
return true;
}
- void feedInputBlock(const block_id input_block_id,
- const relation_id input_relation_id) override {
+ void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id,
+ const partition_id part_id) override {
++num_calls_feedblock_;
MOCK_OP_LOG(3) << "count(" << num_calls_feedblock_ << ")";
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/relational_operators/AggregationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/AggregationOperator.hpp b/relational_operators/AggregationOperator.hpp
index c4e887d..2bd69f3 100644
--- a/relational_operators/AggregationOperator.hpp
+++ b/relational_operators/AggregationOperator.hpp
@@ -98,7 +98,8 @@ class AggregationOperator : public RelationalOperator {
bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
- void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
+ void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id,
+ const partition_id part_id) override {
input_relation_block_ids_.push_back(input_block_id);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/relational_operators/BuildHashOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/BuildHashOperator.hpp b/relational_operators/BuildHashOperator.hpp
index 074c1e1..dec121c 100644
--- a/relational_operators/BuildHashOperator.hpp
+++ b/relational_operators/BuildHashOperator.hpp
@@ -114,8 +114,8 @@ class BuildHashOperator : public RelationalOperator {
bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
- void feedInputBlock(const block_id input_block_id,
- const relation_id input_relation_id) override {
+ void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id,
+ const partition_id part_id) override {
input_relation_block_ids_.push_back(input_block_id);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/relational_operators/DeleteOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/DeleteOperator.hpp b/relational_operators/DeleteOperator.hpp
index 9c3f357..7b69d9c 100644
--- a/relational_operators/DeleteOperator.hpp
+++ b/relational_operators/DeleteOperator.hpp
@@ -100,7 +100,8 @@ class DeleteOperator : public RelationalOperator {
return relation_.getID();
}
- void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
+ void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id,
+ const partition_id part_id) override {
DCHECK(!relation_is_stored_);
relation_block_ids_.push_back(input_block_id);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/relational_operators/HashJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.hpp b/relational_operators/HashJoinOperator.hpp
index 190f9d2..98c87bf 100644
--- a/relational_operators/HashJoinOperator.hpp
+++ b/relational_operators/HashJoinOperator.hpp
@@ -190,8 +190,8 @@ class HashJoinOperator : public RelationalOperator {
bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
- void feedInputBlock(const block_id input_block_id,
- const relation_id input_relation_id) override {
+ void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id,
+ const partition_id part_id) override {
DCHECK(input_relation_id == probe_relation_.getID());
probe_relation_block_ids_.push_back(input_block_id);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/relational_operators/NestedLoopsJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/NestedLoopsJoinOperator.hpp b/relational_operators/NestedLoopsJoinOperator.hpp
index 951851d..f8eb080 100644
--- a/relational_operators/NestedLoopsJoinOperator.hpp
+++ b/relational_operators/NestedLoopsJoinOperator.hpp
@@ -141,7 +141,8 @@ class NestedLoopsJoinOperator : public RelationalOperator {
}
}
- void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
+ void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id,
+ const partition_id part_id) override {
if (input_relation_id == left_input_relation_.getID()) {
left_relation_block_ids_.push_back(input_block_id);
} else if (input_relation_id == right_input_relation_.getID()) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/relational_operators/RelationalOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/RelationalOperator.hpp b/relational_operators/RelationalOperator.hpp
index a7f4177..fdea307 100644
--- a/relational_operators/RelationalOperator.hpp
+++ b/relational_operators/RelationalOperator.hpp
@@ -138,11 +138,11 @@ class RelationalOperator {
* @brief Receive input blocks for this RelationalOperator.
*
* @param input_block_id The ID of the input block.
- *
* @param relation_id The ID of the relation that produced this input_block.
+ * @param part_id The partition ID of 'input_block_id'.
**/
- virtual void feedInputBlock(const block_id input_block_id,
- const relation_id input_relation_id) {}
+ virtual void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id,
+ const partition_id part_id) {}
/**
* @brief Signal the end of feeding of input blocks for this
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/relational_operators/SampleOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SampleOperator.hpp b/relational_operators/SampleOperator.hpp
index ccf6595..e56201a 100644
--- a/relational_operators/SampleOperator.hpp
+++ b/relational_operators/SampleOperator.hpp
@@ -108,7 +108,8 @@ class SampleOperator : public RelationalOperator {
bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
- void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
+ void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id,
+ const partition_id part_id) override {
input_relation_block_ids_.push_back(input_block_id);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/relational_operators/SaveBlocksOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SaveBlocksOperator.hpp b/relational_operators/SaveBlocksOperator.hpp
index 27fd911..cd79733 100644
--- a/relational_operators/SaveBlocksOperator.hpp
+++ b/relational_operators/SaveBlocksOperator.hpp
@@ -83,7 +83,8 @@ class SaveBlocksOperator : public RelationalOperator {
bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
- void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
+ void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id,
+ const partition_id part_id) override {
destination_block_ids_.push_back(input_block_id);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/relational_operators/SelectOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.hpp b/relational_operators/SelectOperator.hpp
index e58ff81..79ab37f 100644
--- a/relational_operators/SelectOperator.hpp
+++ b/relational_operators/SelectOperator.hpp
@@ -204,10 +204,9 @@ class SelectOperator : public RelationalOperator {
bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
- void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
+ void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id,
+ const partition_id part_id) override {
if (input_relation_.hasPartitionScheme()) {
- const partition_id part_id =
- input_relation_.getPartitionScheme()->getPartitionForBlock(input_block_id);
input_relation_block_ids_in_partition_[part_id].push_back(input_block_id);
} else {
input_relation_block_ids_.push_back(input_block_id);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/relational_operators/SortMergeRunOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortMergeRunOperator.hpp b/relational_operators/SortMergeRunOperator.hpp
index aff7d8d..d2d9a2a 100644
--- a/relational_operators/SortMergeRunOperator.hpp
+++ b/relational_operators/SortMergeRunOperator.hpp
@@ -144,8 +144,8 @@ class SortMergeRunOperator : public RelationalOperator {
bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
- void feedInputBlock(const block_id input_block_id,
- const relation_id input_relation_id) override {
+ void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id,
+ const partition_id part_id) override {
input_relation_block_ids_.push_back(input_block_id);
if (started_) {
initializeInputRuns();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/relational_operators/SortRunGenerationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortRunGenerationOperator.hpp b/relational_operators/SortRunGenerationOperator.hpp
index a2ffb2b..25a1273 100644
--- a/relational_operators/SortRunGenerationOperator.hpp
+++ b/relational_operators/SortRunGenerationOperator.hpp
@@ -124,7 +124,8 @@ class SortRunGenerationOperator : public RelationalOperator {
bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
- void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
+ void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id,
+ const partition_id part_id) override {
DCHECK(input_relation_id == input_relation_.getID());
input_relation_block_ids_.push_back(input_block_id);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/relational_operators/tests/SortMergeRunOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/SortMergeRunOperator_unittest.cpp b/relational_operators/tests/SortMergeRunOperator_unittest.cpp
index 74fecec..7a46e6e 100644
--- a/relational_operators/tests/SortMergeRunOperator_unittest.cpp
+++ b/relational_operators/tests/SortMergeRunOperator_unittest.cpp
@@ -1602,7 +1602,7 @@ class SortMergeRunOperatorTest : public ::testing::Test {
// Feed blocks.
DVLOG(1) << "Feeding " << to_feed.size() << " blocks.";
for (const block_id block : to_feed) {
- merge_op_->feedInputBlock(block, input_table_->getID());
+ merge_op_->feedInputBlock(block, input_table_->getID(), 0 /* partition_id */);
}
// Remove fed blocks.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/storage/InsertDestination.cpp
----------------------------------------------------------------------
diff --git a/storage/InsertDestination.cpp b/storage/InsertDestination.cpp
index 19bb356..944998f 100644
--- a/storage/InsertDestination.cpp
+++ b/storage/InsertDestination.cpp
@@ -789,7 +789,7 @@ void PartitionAwareInsertDestination::returnBlockInPartition(MutableBlockReferen
<< "invalidated one or more IndexSubBlocks.");
}
// Note that the block will only be sent if it's full (true).
- sendBlockFilledMessage(block->getID());
+ sendBlockFilledMessage(block->getID(), part_id);
}
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee3b7f0e/storage/InsertDestination.hpp
----------------------------------------------------------------------
diff --git a/storage/InsertDestination.hpp b/storage/InsertDestination.hpp
index 3487638..c3c40bd 100644
--- a/storage/InsertDestination.hpp
+++ b/storage/InsertDestination.hpp
@@ -216,13 +216,15 @@ class InsertDestination : public InsertDestinationInterface {
* scheduler.
*
* @param id The id of the StorageBlock to be pipelined.
+ * @param part_id The partition id of Block <id>, if any.
**/
- void sendBlockFilledMessage(const block_id id) const {
+ void sendBlockFilledMessage(const block_id id, const partition_id part_id = 0) const {
serialization::DataPipelineMessage proto;
proto.set_operator_index(relational_op_index_);
proto.set_block_id(id);
proto.set_relation_id(relation_.getID());
proto.set_query_id(query_id_);
+ proto.set_partition_id(part_id);
// NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
const std::size_t proto_length = proto.ByteSize();
[2/3] incubator-quickstep git commit: Added optimizer support
regarding hash partitions.
Posted by zu...@apache.org.
Added optimizer support regarding hash partitions.
- CreateTable
- Insert / InsertSelection
- UpdateTable
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/6c10e99f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/6c10e99f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/6c10e99f
Branch: refs/heads/feedInputBlock-part-id
Commit: 6c10e99f11f424926ab26fff1d1fd0e5c76e4fa7
Parents: bed1e26
Author: Zuyu Zhang <zu...@apache.org>
Authored: Tue Jan 10 17:00:08 2017 -0800
Committer: jianqiao <ji...@node-2.jianqiao.quickstep-pg0.wisc.cloudlab.us>
Committed: Wed Jan 18 14:31:01 2017 -0600
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 2 +
query_optimizer/ExecutionGenerator.cpp | 46 ++++++--
query_optimizer/logical/CMakeLists.txt | 1 +
query_optimizer/logical/CreateTable.hpp | 27 ++++-
query_optimizer/physical/CMakeLists.txt | 1 +
query_optimizer/physical/CreateTable.hpp | 27 ++++-
query_optimizer/resolver/CMakeLists.txt | 3 +
query_optimizer/resolver/Resolver.cpp | 106 ++++++++++++++-----
query_optimizer/resolver/Resolver.hpp | 13 ++-
query_optimizer/strategy/OneToOne.cpp | 3 +-
.../tests/execution_generator/CMakeLists.txt | 6 ++
.../tests/execution_generator/Create.test | 9 ++
.../tests/execution_generator/Partition.test | 54 ++++++++++
.../tests/logical_generator/Create.test | 10 ++
.../tests/physical_generator/Create.test | 18 ++++
query_optimizer/tests/resolver/Create.test | 10 ++
utility/PtrList.hpp | 16 +++
17 files changed, 304 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 10c52a1..1a15271 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -63,6 +63,8 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogRelationSchema
quickstep_catalog_CatalogTypedefs
+ quickstep_catalog_PartitionScheme
+ quickstep_catalog_PartitionSchemeHeader
quickstep_expressions_Expressions_proto
quickstep_expressions_aggregation_AggregateFunction
quickstep_expressions_aggregation_AggregateFunction_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 29e67f7..e0bfb3b 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -44,6 +44,8 @@
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogRelationSchema.hpp"
#include "catalog/CatalogTypedefs.hpp"
+#include "catalog/PartitionScheme.hpp"
+#include "catalog/PartitionSchemeHeader.hpp"
#include "expressions/Expressions.pb.h"
#include "expressions/aggregation/AggregateFunction.hpp"
#include "expressions/aggregation/AggregateFunction.pb.h"
@@ -1017,6 +1019,11 @@ void ExecutionGenerator::convertCreateTable(
catalog_relation->setDefaultStorageBlockLayout(layout.release());
}
+ if (physical_plan->partition_scheme_header_proto()) {
+ catalog_relation->setPartitionScheme(new PartitionScheme(
+ PartitionSchemeHeader::ReconstructFromProto(*physical_plan->partition_scheme_header_proto())));
+ }
+
execution_plan_->addRelationalOperator(
new CreateTableOperator(query_handle_->query_id(),
catalog_relation.release(),
@@ -1139,14 +1146,21 @@ void ExecutionGenerator::convertInsertTuple(
query_context_proto_->insert_destinations_size();
S::InsertDestination *insert_destination_proto = query_context_proto_->add_insert_destinations();
- insert_destination_proto->set_insert_destination_type(S::InsertDestinationType::BLOCK_POOL);
insert_destination_proto->set_relation_id(input_relation.getID());
insert_destination_proto->mutable_layout()->MergeFrom(
input_relation.getDefaultStorageBlockLayout().getDescription());
- const vector<block_id> blocks(input_relation.getBlocksSnapshot());
- for (const block_id block : blocks) {
- insert_destination_proto->AddExtension(S::BlockPoolInsertDestination::blocks, block);
+ if (input_relation.hasPartitionScheme()) {
+ insert_destination_proto->set_insert_destination_type(S::InsertDestinationType::PARTITION_AWARE);
+ insert_destination_proto->MutableExtension(S::PartitionAwareInsertDestination::partition_scheme)
+ ->MergeFrom(input_relation.getPartitionScheme()->getProto());
+ } else {
+ insert_destination_proto->set_insert_destination_type(S::InsertDestinationType::BLOCK_POOL);
+
+ const vector<block_id> blocks(input_relation.getBlocksSnapshot());
+ for (const block_id block : blocks) {
+ insert_destination_proto->AddExtension(S::BlockPoolInsertDestination::blocks, block);
+ }
}
const QueryPlan::DAGNodeIndex insert_operator_index =
@@ -1197,15 +1211,21 @@ void ExecutionGenerator::convertInsertSelection(
const QueryContext::insert_destination_id insert_destination_index =
query_context_proto_->insert_destinations_size();
S::InsertDestination *insert_destination_proto = query_context_proto_->add_insert_destinations();
-
- insert_destination_proto->set_insert_destination_type(S::InsertDestinationType::BLOCK_POOL);
insert_destination_proto->set_relation_id(destination_relation.getID());
insert_destination_proto->mutable_layout()->MergeFrom(
destination_relation.getDefaultStorageBlockLayout().getDescription());
- const vector<block_id> blocks(destination_relation.getBlocksSnapshot());
- for (const block_id block : blocks) {
- insert_destination_proto->AddExtension(S::BlockPoolInsertDestination::blocks, block);
+ if (destination_relation.hasPartitionScheme()) {
+ insert_destination_proto->set_insert_destination_type(S::InsertDestinationType::PARTITION_AWARE);
+ insert_destination_proto->MutableExtension(S::PartitionAwareInsertDestination::partition_scheme)
+ ->MergeFrom(destination_relation.getPartitionScheme()->getProto());
+ } else {
+ insert_destination_proto->set_insert_destination_type(S::InsertDestinationType::BLOCK_POOL);
+
+ const vector<block_id> blocks(destination_relation.getBlocksSnapshot());
+ for (const block_id block : blocks) {
+ insert_destination_proto->AddExtension(S::BlockPoolInsertDestination::blocks, block);
+ }
}
const CatalogRelationInfo *selection_relation_info =
@@ -1274,7 +1294,13 @@ void ExecutionGenerator::convertUpdateTable(
query_context_proto_->insert_destinations_size();
S::InsertDestination *relocation_destination_proto = query_context_proto_->add_insert_destinations();
- relocation_destination_proto->set_insert_destination_type(S::InsertDestinationType::BLOCK_POOL);
+ if (input_relation->hasPartitionScheme()) {
+ relocation_destination_proto->set_insert_destination_type(S::InsertDestinationType::PARTITION_AWARE);
+ relocation_destination_proto->MutableExtension(S::PartitionAwareInsertDestination::partition_scheme)
+ ->MergeFrom(input_relation->getPartitionScheme()->getProto());
+ } else {
+ relocation_destination_proto->set_insert_destination_type(S::InsertDestinationType::BLOCK_POOL);
+ }
relocation_destination_proto->set_relation_id(input_rel_id);
// Convert the predicate proto.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/logical/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/logical/CMakeLists.txt b/query_optimizer/logical/CMakeLists.txt
index c67f96f..8aca550 100644
--- a/query_optimizer/logical/CMakeLists.txt
+++ b/query_optimizer/logical/CMakeLists.txt
@@ -86,6 +86,7 @@ target_link_libraries(quickstep_queryoptimizer_logical_CreateIndex
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_logical_CreateTable
glog
+ quickstep_catalog_Catalog_proto
quickstep_queryoptimizer_OptimizerTree
quickstep_queryoptimizer_expressions_AttributeReference
quickstep_queryoptimizer_logical_Logical
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/logical/CreateTable.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/logical/CreateTable.hpp b/query_optimizer/logical/CreateTable.hpp
index cc7c6d7..da4325d 100644
--- a/query_optimizer/logical/CreateTable.hpp
+++ b/query_optimizer/logical/CreateTable.hpp
@@ -24,6 +24,7 @@
#include <string>
#include <vector>
+#include "catalog/Catalog.pb.h"
#include "query_optimizer/OptimizerTree.hpp"
#include "query_optimizer/expressions/AttributeReference.hpp"
#include "query_optimizer/logical/Logical.hpp"
@@ -72,10 +73,17 @@ class CreateTable : public Logical {
return block_properties_;
}
+ /**
+ * @return Shared pointer to the serialized partition scheme header.
+ */
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto() const {
+ return partition_scheme_header_proto_;
+ }
+
LogicalPtr copyWithNewChildren(
const std::vector<LogicalPtr> &new_children) const override {
DCHECK_EQ(getNumChildren(), new_children.size());
- return Create(relation_name_, attributes_, block_properties_);
+ return Create(relation_name_, attributes_, block_properties_, partition_scheme_header_proto_);
}
std::vector<expressions::AttributeReferencePtr> getOutputAttributes() const override {
@@ -95,13 +103,19 @@ class CreateTable : public Logical {
* @param block_properties The physical layout description of this block.
* Note that the pointer's ownership is assumed and shared by
* Logical::CreateTable and Physical::CreateTable.
+ * @param partition_scheme_header_proto The serialized partition scheme
+ * header. It is 'nullptr' if no partitions specified. Note that the
+ * pointer's ownership is shared by Logical::CreateTable and
+ * Physical::CreateTable.
+ *
* @return An immutable CreateTable node.
*/
static CreateTablePtr Create(
const std::string &relation_name,
const std::vector<expressions::AttributeReferencePtr> &attributes,
- const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties) {
- return CreateTablePtr(new CreateTable(relation_name, attributes, block_properties));
+ const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties,
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> &partition_scheme_header_proto) {
+ return CreateTablePtr(new CreateTable(relation_name, attributes, block_properties, partition_scheme_header_proto));
}
protected:
@@ -117,17 +131,20 @@ class CreateTable : public Logical {
CreateTable(
const std::string &relation_name,
const std::vector<expressions::AttributeReferencePtr> &attributes,
- const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties)
+ const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties,
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> &partition_scheme_header_proto)
: relation_name_(relation_name),
attributes_(attributes),
block_properties_(block_properties),
block_properties_representation_(
- getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())) {}
+ getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())),
+ partition_scheme_header_proto_(partition_scheme_header_proto) {}
std::string relation_name_;
std::vector<expressions::AttributeReferencePtr> attributes_;
std::shared_ptr<const StorageBlockLayoutDescription> block_properties_;
std::shared_ptr<const OptimizerProtoRepresentation<OptimizerTreeBaseNodePtr> > block_properties_representation_;
+ std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto_;
DISALLOW_COPY_AND_ASSIGN(CreateTable);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/physical/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/CMakeLists.txt b/query_optimizer/physical/CMakeLists.txt
index 5c2cd0b..7f26943 100644
--- a/query_optimizer/physical/CMakeLists.txt
+++ b/query_optimizer/physical/CMakeLists.txt
@@ -85,6 +85,7 @@ target_link_libraries(quickstep_queryoptimizer_physical_CreateIndex
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_physical_CreateTable
glog
+ quickstep_catalog_Catalog_proto
quickstep_queryoptimizer_OptimizerTree
quickstep_queryoptimizer_expressions_AttributeReference
quickstep_queryoptimizer_expressions_ExpressionUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/physical/CreateTable.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/CreateTable.hpp b/query_optimizer/physical/CreateTable.hpp
index 8e3bbd4..05eab0d 100644
--- a/query_optimizer/physical/CreateTable.hpp
+++ b/query_optimizer/physical/CreateTable.hpp
@@ -24,6 +24,7 @@
#include <string>
#include <vector>
+#include "catalog/Catalog.pb.h"
#include "query_optimizer/OptimizerTree.hpp"
#include "query_optimizer/expressions/AttributeReference.hpp"
#include "query_optimizer/expressions/ExpressionUtil.hpp"
@@ -75,10 +76,17 @@ class CreateTable : public Physical {
return block_properties_;
}
+ /**
+ * @return Shared pointer to the serialized partition scheme header.
+ */
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto() const {
+ return partition_scheme_header_proto_;
+ }
+
PhysicalPtr copyWithNewChildren(
const std::vector<PhysicalPtr> &new_children) const override {
DCHECK_EQ(getNumChildren(), new_children.size());
- return Create(relation_name_, attributes_, block_properties_);
+ return Create(relation_name_, attributes_, block_properties_, partition_scheme_header_proto_);
}
std::vector<expressions::AttributeReferencePtr> getOutputAttributes() const override {
@@ -102,13 +110,19 @@ class CreateTable : public Physical {
* @param relation_name The name of the relation to be inserted.
* @param attributes Schema of the relation.
* @param block_properties The optional proto message describing the block.
+ * @param partition_scheme_header_proto The serialized partition scheme
+ * header. It is 'nullptr' if no partitions specified. Note that the
+ * pointer's ownership is shared by Logical::CreateTable and
+ * Physical::CreateTable.
+ *
* @return An immutable CreateTable node.
*/
static CreateTablePtr Create(
const std::string &relation_name,
const std::vector<expressions::AttributeReferencePtr> &attributes,
- const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties) {
- return CreateTablePtr(new CreateTable(relation_name, attributes, block_properties));
+ const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties,
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> &partition_scheme_header_proto) {
+ return CreateTablePtr(new CreateTable(relation_name, attributes, block_properties, partition_scheme_header_proto));
}
protected:
@@ -124,17 +138,20 @@ class CreateTable : public Physical {
CreateTable(
const std::string &relation_name,
const std::vector<expressions::AttributeReferencePtr> &attributes,
- const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties)
+ const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties,
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> &partition_scheme_header_proto)
: relation_name_(relation_name),
attributes_(attributes),
block_properties_(block_properties),
block_properties_representation_(
- getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())) {}
+ getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())),
+ partition_scheme_header_proto_(partition_scheme_header_proto) {}
std::string relation_name_;
std::vector<expressions::AttributeReferencePtr> attributes_;
std::shared_ptr<const StorageBlockLayoutDescription> block_properties_;
std::shared_ptr<const OptimizerProtoRepresentation<OptimizerTreeBaseNodePtr> > block_properties_representation_;
+ std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto_;
DISALLOW_COPY_AND_ASSIGN(CreateTable);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/resolver/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/CMakeLists.txt b/query_optimizer/resolver/CMakeLists.txt
index 5251ccc..a34273e 100644
--- a/query_optimizer/resolver/CMakeLists.txt
+++ b/query_optimizer/resolver/CMakeLists.txt
@@ -34,6 +34,8 @@ target_link_libraries(quickstep_queryoptimizer_resolver_NameResolver
target_link_libraries(quickstep_queryoptimizer_resolver_Resolver
glog
quickstep_catalog_CatalogDatabase
+ quickstep_catalog_CatalogTypedefs
+ quickstep_catalog_Catalog_proto
quickstep_expressions_aggregation_AggregateFunction
quickstep_expressions_aggregation_AggregateFunctionFactory
quickstep_expressions_tablegenerator_GeneratorFunction
@@ -53,6 +55,7 @@ target_link_libraries(quickstep_queryoptimizer_resolver_Resolver
quickstep_parser_ParseLimit
quickstep_parser_ParseLiteralValue
quickstep_parser_ParseOrderBy
+ quickstep_parser_ParsePartitionClause
quickstep_parser_ParsePredicate
quickstep_parser_ParsePredicateExists
quickstep_parser_ParsePredicateInTableQuery
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index 2580342..df589fd 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -29,7 +29,9 @@
#include <vector>
#include <utility>
+#include "catalog/Catalog.pb.h"
#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogTypedefs.hpp"
#include "expressions/aggregation/AggregateFunction.hpp"
#include "expressions/aggregation/AggregateFunctionFactory.hpp"
#include "expressions/table_generator/GeneratorFunction.hpp"
@@ -49,6 +51,7 @@
#include "parser/ParseLimit.hpp"
#include "parser/ParseLiteralValue.hpp"
#include "parser/ParseOrderBy.hpp"
+#include "parser/ParsePartitionClause.hpp"
#include "parser/ParsePredicate.hpp"
#include "parser/ParsePredicateExists.hpp"
#include "parser/ParsePredicateInTableQuery.hpp"
@@ -129,12 +132,15 @@
#include "glog/logging.h"
+using std::make_unique;
+
namespace quickstep {
namespace optimizer {
namespace resolver {
namespace E = ::quickstep::optimizer::expressions;
namespace L = ::quickstep::optimizer::logical;
+namespace S = ::quickstep::serialization;
struct Resolver::ExpressionResolutionInfo {
/**
@@ -478,9 +484,32 @@ L::LogicalPtr Resolver::resolveCreateTable(
std::shared_ptr<const StorageBlockLayoutDescription>
block_properties(resolveBlockProperties(create_table_statement));
- return L::CreateTable::Create(relation_name, attributes, block_properties);
+ std::shared_ptr<const S::PartitionSchemeHeader> partition_scheme_header_proto(
+ resolvePartitionClause(create_table_statement));
+
+ return L::CreateTable::Create(relation_name, attributes, block_properties, partition_scheme_header_proto);
+}
+
+namespace {
+
+attribute_id GetAttributeIdFromName(const PtrList<ParseAttributeDefinition> &attribute_definition_list,
+ const std::string &attribute_name) {
+ const std::string lower_attribute_name = ToLower(attribute_name);
+
+ attribute_id attr_id = 0;
+ for (const ParseAttributeDefinition &attribute_definition : attribute_definition_list) {
+ if (lower_attribute_name == ToLower(attribute_definition.name()->value())) {
+ return attr_id;
+ }
+
+ ++attr_id;
+ }
+
+ return kInvalidAttributeID;
}
+} // namespace
+
StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
const ParseStatementCreateTable &create_table_statement) {
const ParseBlockProperties *block_properties
@@ -540,25 +569,6 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
THROW_SQL_ERROR_AT(type_parse_string) << "Unrecognized storage type.";
}
- // Helper lambda function which will be used in COMPRESS and SORT resolution.
- // Returns the column id from the name of the given attribute. Returns -1 if
- // the attribute is not found.
- auto columnIdFromAttributeName = [&create_table_statement](
- const std::string& attribute_name) -> int {
- const std::string search_name = ToLower(attribute_name);
- int i = 0;
- for (const ParseAttributeDefinition &attribute_definition :
- create_table_statement.attribute_definition_list()) {
- const std::string lower_attribute_name =
- ToLower(attribute_definition.name()->value());
- if (lower_attribute_name.compare(search_name) == 0) {
- return i;
- }
- i++;
- }
- return -1;
- };
-
// Resolve the SORT property.
const ParseString *sort_parse_string = block_properties->getSort();
if (block_requires_sort) {
@@ -566,10 +576,10 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
THROW_SQL_ERROR_AT(type_parse_string)
<< "The SORT property must be specified as an attribute name.";
} else {
- const std::string &sort_name = sort_parse_string->value();
// Lookup the name and map to a column id.
- int sort_id = columnIdFromAttributeName(sort_name);
- if (sort_id == -1) {
+ const attribute_id sort_id = GetAttributeIdFromName(create_table_statement.attribute_definition_list(),
+ sort_parse_string->value());
+ if (sort_id == kInvalidAttributeID) {
THROW_SQL_ERROR_AT(sort_parse_string)
<< "The SORT property did not match any attribute name.";
} else {
@@ -609,8 +619,9 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
<< "The COMPRESS property must be specified as ALL or a list of attributes.";
}
for (const ParseString &compressed_attribute_name : *compress_parse_strings) {
- int column_id = columnIdFromAttributeName(compressed_attribute_name.value());
- if (column_id == -1) {
+ const attribute_id column_id = GetAttributeIdFromName(create_table_statement.attribute_definition_list(),
+ compressed_attribute_name.value());
+ if (column_id == kInvalidAttributeID) {
THROW_SQL_ERROR_AT(&compressed_attribute_name)
<< "The given attribute was not found.";
} else {
@@ -671,6 +682,51 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
return storage_block_description.release();
}
+const S::PartitionSchemeHeader* Resolver::resolvePartitionClause(
+ const ParseStatementCreateTable &create_table_statement) {
+ const ParsePartitionClause *partition_clause = create_table_statement.opt_partition_clause();
+ if (partition_clause == nullptr) {
+ return nullptr;
+ }
+
+ const ParseString *partition_type_string = partition_clause->partition_type();
+ if (partition_type_string == nullptr) {
+ THROW_SQL_ERROR_AT(partition_clause)
+ << "Partition type must be specified and be a string.";
+ }
+
+ const PtrList<ParseString> &attribute_name_list = partition_clause->attribute_name_list();
+ if (attribute_name_list.size() != 1) {
+ THROW_SQL_ERROR_AT(partition_clause)
+ << "Partition is supported on only one attribute.";
+ }
+
+ const ParseString &partition_attribute_name = attribute_name_list.front();
+ const attribute_id attr_id = GetAttributeIdFromName(create_table_statement.attribute_definition_list(),
+ partition_attribute_name.value());
+ if (attr_id == kInvalidAttributeID) {
+ THROW_SQL_ERROR_AT(&partition_attribute_name)
+ << "The given attribute was not found.";
+ }
+
+ auto proto = make_unique<S::PartitionSchemeHeader>();
+ proto->set_num_partitions(partition_clause->num_partitions()->long_value());
+ proto->set_partition_attribute_id(attr_id);
+
+ const std::string partition_type = ToLower(partition_type_string->value());
+ if (partition_type == kHashPartitionType) {
+ proto->set_partition_type(S::PartitionSchemeHeader::HASH);
+ } else if (partition_type == kRangePartitionType) {
+ proto->set_partition_type(S::PartitionSchemeHeader::RANGE);
+ THROW_SQL_ERROR_AT(partition_clause)
+ << "Range partition is not supported.";
+ } else {
+ THROW_SQL_ERROR_AT(partition_type_string) << "Unrecognized partition type: " << partition_type;
+ }
+
+ return proto.release();
+}
+
L::LogicalPtr Resolver::resolveCreateIndex(
const ParseStatementCreateIndex &create_index_statement) {
// Resolve relation reference.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/resolver/Resolver.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.hpp b/query_optimizer/resolver/Resolver.hpp
index 855e6ba..a3d0833 100644
--- a/query_optimizer/resolver/Resolver.hpp
+++ b/query_optimizer/resolver/Resolver.hpp
@@ -74,9 +74,8 @@ class PtrList;
class StorageBlockLayoutDescription;
class Type;
-} // namespace quickstep
+namespace serialization { class PartitionSchemeHeader; }
-namespace quickstep {
namespace optimizer {
class OptimizerContext;
@@ -213,6 +212,16 @@ class Resolver {
const ParseStatementCreateTable &create_table_statement);
/**
+ * @brief Resolves the PARTITION clause of a CREATE TABLE statement to a
+ * the serialized PartitionSchemeHeader describing the user input.
+ *
+ * @param create_table_statement The create table statement.
+ * @return A pointer to a user-owned serialized PartitionSchemeHeader.
+ */
+ const serialization::PartitionSchemeHeader* resolvePartitionClause(
+ const ParseStatementCreateTable &create_table_statement);
+
+ /**
* @brief Resolves a DELETE query and returns a logical plan.
*
* @param delete_statement The DELETE parse tree.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/strategy/OneToOne.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/strategy/OneToOne.cpp b/query_optimizer/strategy/OneToOne.cpp
index 78003f4..7d0c4cb 100644
--- a/query_optimizer/strategy/OneToOne.cpp
+++ b/query_optimizer/strategy/OneToOne.cpp
@@ -121,7 +121,8 @@ bool OneToOne::generatePlan(const L::LogicalPtr &logical_input,
std::static_pointer_cast<const L::CreateTable>(logical_input);
*physical_output = P::CreateTable::Create(create_table->relation_name(),
create_table->attributes(),
- create_table->block_properties());
+ create_table->block_properties(),
+ create_table->partition_scheme_header_proto());
return true;
}
case L::LogicalType::kDeleteTuples: {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/tests/execution_generator/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/CMakeLists.txt b/query_optimizer/tests/execution_generator/CMakeLists.txt
index 1ea6a17..2705130 100644
--- a/query_optimizer/tests/execution_generator/CMakeLists.txt
+++ b/query_optimizer/tests/execution_generator/CMakeLists.txt
@@ -107,6 +107,11 @@ add_test(quickstep_queryoptimizer_tests_executiongenerator_join
"${CMAKE_CURRENT_SOURCE_DIR}/Join.test"
"${CMAKE_CURRENT_BINARY_DIR}/Join.test"
"${CMAKE_CURRENT_BINARY_DIR}/Join/")
+add_test(quickstep_queryoptimizer_tests_executiongenerator_partition
+ "../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Partition.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Partition.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Partition/")
add_test(quickstep_queryoptimizer_tests_executiongenerator_select
"../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
"${CMAKE_CURRENT_SOURCE_DIR}/Select.test"
@@ -150,6 +155,7 @@ file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Drop)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Index)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Insert)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Join)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Partition)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Select)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/StringPatternMatching)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/TableGenerator)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/tests/execution_generator/Create.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Create.test b/query_optimizer/tests/execution_generator/Create.test
index 5bd0e76..4ffa665 100644
--- a/query_optimizer/tests/execution_generator/Create.test
+++ b/query_optimizer/tests/execution_generator/Create.test
@@ -40,3 +40,12 @@ CREATE TABLE foo2 (col1 INT, col2 VARCHAR(80))
WITH BLOCKPROPERTIES (TYPE columnstore, SORT col2);
--
ERROR: BLOCKPROPERTIES is invalid.
+==
+
+CREATE TABLE foo3 (attr INT) PARTITION BY HASH(attr) PARTITIONS 4;
+SELECT * FROM foo3;
+--
++-----------+
+|attr |
++-----------+
++-----------+
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/tests/execution_generator/Partition.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Partition.test b/query_optimizer/tests/execution_generator/Partition.test
new file mode 100644
index 0000000..ab05391
--- /dev/null
+++ b/query_optimizer/tests/execution_generator/Partition.test
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+CREATE TABLE foo (id INT NULL,
+ name CHAR(20))
+PARTITION BY HASH(id) PARTITIONS 4;
+
+INSERT INTO foo
+SELECT int_col, char_col
+FROM test
+WHERE int_col > 0 OR int_col < 0;
+
+SELECT * FROM foo;
+--
++-----------+--------------------+
+|id |name |
++-----------+--------------------+
+| 4| 4 2.000000|
+| 8| 8 2.828427|
+| 12| 12 3.464102|
+| 16| 16 4.000000|
+| 24| 24 4.898979|
+| -3| -3 1.732051|
+| -7| -7 2.645751|
+| -11| -11 3.316625|
+| -15| -15 3.872983|
+| -19| -19 4.358899|
+| -23| -23 4.795832|
+| 2| 2 1.414214|
+| 6| 6 2.449490|
+| 14| 14 3.741657|
+| 18| 18 4.242641|
+| 22| 22 4.690416|
+| -1| -1 1.000000|
+| -5| -5 2.236068|
+| -9| -9 3.000000|
+| -13| -13 3.605551|
+| -17| -17 4.123106|
+| -21| -21 4.582576|
++-----------+--------------------+
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/tests/logical_generator/Create.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/logical_generator/Create.test b/query_optimizer/tests/logical_generator/Create.test
index aeff9ec..04134f9 100644
--- a/query_optimizer/tests/logical_generator/Create.test
+++ b/query_optimizer/tests/logical_generator/Create.test
@@ -46,3 +46,13 @@ TopLevelPlan
+-output_attributes=
+-AttributeReference[id=0,name=attr,relation=foo,type=Int]
+-AttributeReference[id=1,name=attr2,relation=foo,type=Int]
+==
+
+CREATE TABLE foo (attr INT) PARTITION BY HASH(attr) PARTITIONS 4;
+--
+TopLevelPlan
++-plan=CreateTable[relation=foo]
+| +-attributes=
+| +-AttributeReference[id=0,name=attr,relation=foo,type=Int]
++-output_attributes=
+ +-AttributeReference[id=0,name=attr,relation=foo,type=Int]
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/tests/physical_generator/Create.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/physical_generator/Create.test b/query_optimizer/tests/physical_generator/Create.test
index 54af3fa..c555371 100644
--- a/query_optimizer/tests/physical_generator/Create.test
+++ b/query_optimizer/tests/physical_generator/Create.test
@@ -115,3 +115,21 @@ TopLevelPlan
| +-AttributeReference[id=0,name=col1,relation=foo,type=Int]
+-output_attributes=
+-AttributeReference[id=0,name=col1,relation=foo,type=Int]
+==
+
+CREATE TABLE foo (attr INT) PARTITION BY HASH(attr) PARTITIONS 4;
+--
+[Optimized Logical Plan]
+TopLevelPlan
++-plan=CreateTable[relation=foo]
+| +-attributes=
+| +-AttributeReference[id=0,name=attr,relation=foo,type=Int]
++-output_attributes=
+ +-AttributeReference[id=0,name=attr,relation=foo,type=Int]
+[Physical Plan]
+TopLevelPlan
++-plan=CreateTable[relation=foo]
+| +-attributes=
+| +-AttributeReference[id=0,name=attr,relation=foo,type=Int]
++-output_attributes=
+ +-AttributeReference[id=0,name=attr,relation=foo,type=Int]
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/query_optimizer/tests/resolver/Create.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/resolver/Create.test b/query_optimizer/tests/resolver/Create.test
index b04d785..28bd4f5 100644
--- a/query_optimizer/tests/resolver/Create.test
+++ b/query_optimizer/tests/resolver/Create.test
@@ -229,3 +229,13 @@ CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
ERROR: The BLOCKSIZEMB property must be between 2MB and 1024MB. (2 : 23)
(TYPE split_rowstore, BLOCKSIZEMB 2000);
^
+==
+
+CREATE TABLE foo (attr INT) PARTITION BY HASH(attr) PARTITIONS 4;
+--
+TopLevelPlan
++-plan=CreateTable[relation=foo]
+| +-attributes=
+| +-AttributeReference[id=0,name=attr,relation=foo,type=Int]
++-output_attributes=
+ +-AttributeReference[id=0,name=attr,relation=foo,type=Int]
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c10e99f/utility/PtrList.hpp
----------------------------------------------------------------------
diff --git a/utility/PtrList.hpp b/utility/PtrList.hpp
index 5e34ec8..4707c26 100644
--- a/utility/PtrList.hpp
+++ b/utility/PtrList.hpp
@@ -218,6 +218,14 @@ class PtrList {
return PtrListIterator(internal_list_.end());
}
+ T& front() {
+ return *(internal_list_.front());
+ }
+
+ T& back() {
+ return *(internal_list_.back());
+ }
+
const_iterator begin() const {
return PtrListConstIterator(internal_list_.begin());
}
@@ -226,6 +234,14 @@ class PtrList {
return PtrListConstIterator(internal_list_.end());
}
+ const T& front() const {
+ return *(internal_list_.front());
+ }
+
+ const T& back() const {
+ return *(internal_list_.back());
+ }
+
void splice(iterator position, PtrList<T> &source) { // NOLINT(runtime/references) - STL-style interface
internal_list_.splice(position.internal_iterator_, source.internal_list_);
}