You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by ji...@apache.org on 2017/01/06 23:25:11 UTC
incubator-quickstep git commit: tests
Repository: incubator-quickstep
Updated Branches:
refs/heads/output-attr-order c7fdc360e -> 65b92b47c
tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/65b92b47
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/65b92b47
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/65b92b47
Branch: refs/heads/output-attr-order
Commit: 65b92b47c79349b22616bd4f6c47323227e5abb6
Parents: c7fdc36
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Fri Jan 6 17:24:56 2017 -0600
Committer: Jianqiao Zhu <ji...@cs.wisc.edu>
Committed: Fri Jan 6 17:24:56 2017 -0600
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 1 +
query_optimizer/PhysicalGenerator.cpp | 2 +
query_optimizer/rules/CMakeLists.txt | 9 ++
query_optimizer/rules/ReorderColumns.cpp | 81 +++++++++++++
query_optimizer/rules/ReorderColumns.hpp | 63 +++++++++++
relational_operators/CMakeLists.txt | 1 +
relational_operators/HashJoinOperator.cpp | 1 +
storage/CMakeLists.txt | 3 +
storage/InsertContext.hpp | 150 +++++++++++++++++++------
storage/InsertDestination.cpp | 22 ++++
storage/InsertDestination.hpp | 3 +
storage/InsertDestinationInterface.hpp | 4 +
storage/StorageBlock.cpp | 65 ++++++++---
13 files changed, 354 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65b92b47/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 10c52a1..1b7a912 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -209,6 +209,7 @@ target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_rules_AttachLIPFilters
quickstep_queryoptimizer_rules_PruneColumns
+ quickstep_queryoptimizer_rules_ReorderColumns
quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
quickstep_queryoptimizer_rules_SwapProbeBuild
quickstep_queryoptimizer_strategy_Aggregate
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65b92b47/query_optimizer/PhysicalGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/PhysicalGenerator.cpp b/query_optimizer/PhysicalGenerator.cpp
index 7cb97dc..d1e4c65 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -28,6 +28,7 @@
#include "query_optimizer/physical/Physical.hpp"
#include "query_optimizer/rules/AttachLIPFilters.hpp"
#include "query_optimizer/rules/PruneColumns.hpp"
+#include "query_optimizer/rules/ReorderColumns.hpp"
#include "query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp"
#include "query_optimizer/rules/SwapProbeBuild.hpp"
#include "query_optimizer/strategy/Aggregate.hpp"
@@ -109,6 +110,7 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
} else {
rules.emplace_back(new SwapProbeBuild());
}
+ rules.emplace_back(new ReorderColumns());
if (FLAGS_use_lip_filters) {
rules.emplace_back(new AttachLIPFilters());
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65b92b47/query_optimizer/rules/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/CMakeLists.txt b/query_optimizer/rules/CMakeLists.txt
index 7fffadc..0b76725 100644
--- a/query_optimizer/rules/CMakeLists.txt
+++ b/query_optimizer/rules/CMakeLists.txt
@@ -25,6 +25,7 @@ add_library(quickstep_queryoptimizer_rules_GenerateJoins GenerateJoins.cpp Gener
add_library(quickstep_queryoptimizer_rules_PruneColumns PruneColumns.cpp PruneColumns.hpp)
add_library(quickstep_queryoptimizer_rules_PushDownFilter PushDownFilter.cpp PushDownFilter.hpp)
add_library(quickstep_queryoptimizer_rules_PushDownSemiAntiJoin PushDownSemiAntiJoin.cpp PushDownSemiAntiJoin.hpp)
+add_library(quickstep_queryoptimizer_rules_ReorderColumns ReorderColumns.cpp ReorderColumns.hpp)
add_library(quickstep_queryoptimizer_rules_Rule ../../empty_src.cpp Rule.hpp)
add_library(quickstep_queryoptimizer_rules_RuleHelper RuleHelper.cpp RuleHelper.hpp)
add_library(quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
@@ -118,6 +119,13 @@ target_link_libraries(quickstep_queryoptimizer_rules_PushDownSemiAntiJoin
quickstep_queryoptimizer_logical_PatternMatcher
quickstep_queryoptimizer_rules_TopDownRule
quickstep_utility_Macros)
+target_link_libraries(quickstep_queryoptimizer_rules_ReorderColumns
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_expressions_ExprId
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_queryoptimizer_rules_Rule
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_rules_Rule
glog
quickstep_utility_Macros)
@@ -213,6 +221,7 @@ target_link_libraries(quickstep_queryoptimizer_rules
quickstep_queryoptimizer_rules_PruneColumns
quickstep_queryoptimizer_rules_PushDownFilter
quickstep_queryoptimizer_rules_PushDownSemiAntiJoin
+ quickstep_queryoptimizer_rules_ReorderColumns
quickstep_queryoptimizer_rules_Rule
quickstep_queryoptimizer_rules_RuleHelper
quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65b92b47/query_optimizer/rules/ReorderColumns.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/ReorderColumns.cpp b/query_optimizer/rules/ReorderColumns.cpp
new file mode 100644
index 0000000..f289655
--- /dev/null
+++ b/query_optimizer/rules/ReorderColumns.cpp
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "query_optimizer/rules/ReorderColumns.hpp"
+
+#include <vector>
+
+#include "query_optimizer/expressions/ExprId.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/PhysicalType.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+
+namespace E = ::quickstep::optimizer::expressions;
+namespace P = ::quickstep::optimizer::physical;
+
+P::PhysicalPtr ReorderColumns::apply(const P::PhysicalPtr &input) {
+ DCHECK(input->getPhysicalType() == P::PhysicalType::kTopLevelPlan);
+
+ return applyInternal(input);
+}
+
+bool ReorderColumns::IsApplicable(const physical::PhysicalPtr &input) {
+ switch (input->getPhysicalType()) {
+ case P::PhysicalType::kHashJoin: // Fall through
+ case P::PhysicalType::kSelection:
+ return true;
+ default:
+ return false;
+ }
+}
+
+P::PhysicalPtr ReorderColumns::applyInternal(const P::PhysicalPtr &input) {
+ if (!IsApplicable(input)) {
+ std::vector<P::PhysicalPtr> new_children;
+ for (const P::PhysicalPtr &child : input->children()) {
+ new_children.emplace_back(applyInternal(child));
+ }
+
+ if (new_children != input->children()) {
+ return input->copyWithNewChildren(new_children);
+ } else {
+ return input;
+ }
+ }
+
+ std::vector<P::PhysicalPtr> nodes;
+ std::unordered_map<E::ExprId, std::size_t> gen;
+ std::unordered_map<E::ExprId, std::size_t> kill;
+
+ for (P::PhysicalPtr node = input; IsApplicable(node); node = node->children()[0]) {
+ nodes.emplace_back(node);
+ }
+
+ std::cout << nodes.size() << "\n";
+ exit(0);
+
+ return nullptr;
+}
+
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65b92b47/query_optimizer/rules/ReorderColumns.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/ReorderColumns.hpp b/query_optimizer/rules/ReorderColumns.hpp
new file mode 100644
index 0000000..dd0cf33
--- /dev/null
+++ b/query_optimizer/rules/ReorderColumns.hpp
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_OPTIMIZER_RULES_REORDER_COLUMNS_HPP_
+#define QUICKSTEP_QUERY_OPTIMIZER_RULES_REORDER_COLUMNS_HPP_
+
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/rules/Rule.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+namespace optimizer {
+
+/** \addtogroup OptimizerRules
+ * @{
+ */
+
+class ReorderColumns : public Rule<physical::Physical> {
+ public:
+ /**
+ * @brief Constructor.
+ */
+ ReorderColumns() {}
+
+ ~ReorderColumns() override {}
+
+ std::string getName() const override {
+ return "ReorderColumns";
+ }
+
+ physical::PhysicalPtr apply(const physical::PhysicalPtr &input) override;
+
+ private:
+ physical::PhysicalPtr applyInternal(const physical::PhysicalPtr &input);
+
+ inline static bool IsApplicable(const physical::PhysicalPtr &input);
+
+ DISALLOW_COPY_AND_ASSIGN(ReorderColumns);
+};
+
+/** @} */
+
+} // namespace optimizer
+} // namespace quickstep
+
+#endif /* QUICKSTEP_QUERY_OPTIMIZER_RULES_REORDER_COLUMNS_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65b92b47/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index b792a7b..5e173a4 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -206,6 +206,7 @@ target_link_libraries(quickstep_relationaloperators_HashJoinOperator
quickstep_catalog_CatalogTypedefs
quickstep_expressions_predicate_Predicate
quickstep_expressions_scalar_Scalar
+ quickstep_expressions_scalar_ScalarAttribute
quickstep_queryexecution_QueryContext
quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65b92b47/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index 1a34e32..dae2ba7 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -31,6 +31,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "expressions/predicate/Predicate.hpp"
#include "expressions/scalar/Scalar.hpp"
+#include "expressions/scalar/ScalarAttribute.hpp"
#include "query_execution/QueryContext.hpp"
#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65b92b47/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index f3869c9..cabd717 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -801,6 +801,7 @@ target_link_libraries(quickstep_storage_InsertDestination
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_InsertContext
quickstep_storage_InsertDestinationInterface
quickstep_storage_InsertDestination_proto
quickstep_storage_StorageBlock
@@ -984,6 +985,7 @@ target_link_libraries(quickstep_storage_StorageBlock
quickstep_expressions_aggregation_AggregationHandle
quickstep_expressions_predicate_Predicate
quickstep_expressions_scalar_Scalar
+ quickstep_expressions_scalar_ScalarAttribute
quickstep_storage_BasicColumnStoreTupleStorageSubBlock
quickstep_storage_BloomFilterIndexSubBlock
quickstep_storage_CSBTreeIndexSubBlock
@@ -992,6 +994,7 @@ target_link_libraries(quickstep_storage_StorageBlock
quickstep_storage_CountedReference
quickstep_storage_HashTableBase
quickstep_storage_IndexSubBlock
+ quickstep_storage_InsertContext
quickstep_storage_InsertDestinationInterface
quickstep_storage_PackedRowStoreTupleStorageSubBlock
quickstep_storage_SMAIndexSubBlock
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65b92b47/storage/InsertContext.hpp
----------------------------------------------------------------------
diff --git a/storage/InsertContext.hpp b/storage/InsertContext.hpp
index b321528..b8d7178 100644
--- a/storage/InsertContext.hpp
+++ b/storage/InsertContext.hpp
@@ -95,20 +95,68 @@ class CopyList {
const std::size_t bytes_to_copy,
ArgTypes &&...args) {
switch (bytes_to_copy) {
+ case 1:
+ return CreateStrideCopyFunctor<1>(std::forward<ArgTypes>(args)...);
+ case 2:
+ return CreateStrideCopyFunctor<2>(std::forward<ArgTypes>(args)...);
+ case 3:
+ return CreateStrideCopyFunctor<3>(std::forward<ArgTypes>(args)...);
case 4:
return CreateStrideCopyFunctor<4>(std::forward<ArgTypes>(args)...);
+ case 5:
+ return CreateStrideCopyFunctor<5>(std::forward<ArgTypes>(args)...);
+ case 6:
+ return CreateStrideCopyFunctor<6>(std::forward<ArgTypes>(args)...);
+ case 7:
+ return CreateStrideCopyFunctor<7>(std::forward<ArgTypes>(args)...);
case 8:
return CreateStrideCopyFunctor<8>(std::forward<ArgTypes>(args)...);
+ case 9:
+ return CreateStrideCopyFunctor<9>(std::forward<ArgTypes>(args)...);
+ case 10:
+ return CreateStrideCopyFunctor<10>(std::forward<ArgTypes>(args)...);
+ case 11:
+ return CreateStrideCopyFunctor<11>(std::forward<ArgTypes>(args)...);
case 12:
return CreateStrideCopyFunctor<12>(std::forward<ArgTypes>(args)...);
+ case 13:
+ return CreateStrideCopyFunctor<13>(std::forward<ArgTypes>(args)...);
+ case 14:
+ return CreateStrideCopyFunctor<14>(std::forward<ArgTypes>(args)...);
+ case 15:
+ return CreateStrideCopyFunctor<15>(std::forward<ArgTypes>(args)...);
case 16:
return CreateStrideCopyFunctor<16>(std::forward<ArgTypes>(args)...);
+ case 17:
+ return CreateStrideCopyFunctor<17>(std::forward<ArgTypes>(args)...);
+ case 18:
+ return CreateStrideCopyFunctor<18>(std::forward<ArgTypes>(args)...);
+ case 19:
+ return CreateStrideCopyFunctor<19>(std::forward<ArgTypes>(args)...);
case 20:
return CreateStrideCopyFunctor<20>(std::forward<ArgTypes>(args)...);
+ case 21:
+ return CreateStrideCopyFunctor<21>(std::forward<ArgTypes>(args)...);
+ case 22:
+ return CreateStrideCopyFunctor<22>(std::forward<ArgTypes>(args)...);
+ case 23:
+ return CreateStrideCopyFunctor<23>(std::forward<ArgTypes>(args)...);
case 24:
return CreateStrideCopyFunctor<24>(std::forward<ArgTypes>(args)...);
+ case 25:
+ return CreateStrideCopyFunctor<25>(std::forward<ArgTypes>(args)...);
+ case 26:
+ return CreateStrideCopyFunctor<26>(std::forward<ArgTypes>(args)...);
+ case 27:
+ return CreateStrideCopyFunctor<27>(std::forward<ArgTypes>(args)...);
case 28:
return CreateStrideCopyFunctor<28>(std::forward<ArgTypes>(args)...);
+ case 29:
+ return CreateStrideCopyFunctor<29>(std::forward<ArgTypes>(args)...);
+ case 30:
+ return CreateStrideCopyFunctor<30>(std::forward<ArgTypes>(args)...);
+ case 31:
+ return CreateStrideCopyFunctor<31>(std::forward<ArgTypes>(args)...);
case 32:
return CreateStrideCopyFunctor<32>(std::forward<ArgTypes>(args)...);
default:
@@ -180,6 +228,56 @@ class InsertContext {
: output_relation_(output_relation) {}
void addSource(const std::map<attribute_id, attribute_id> &attribute_map) {
+ if (!attribute_map.empty()) {
+ non_empty_copy_indices_.emplace_back(row_store_copy_lists_.size());
+ }
+
+ addSourceRowStore(attribute_map);
+ addSourceDirect(attribute_map);
+ }
+
+ std::size_t bulkInsertTuples(const std::vector<ValueAccessor *> &accessors,
+ const std::size_t stride_width,
+ const std::size_t num_tuples,
+ void *storage) {
+ DCHECK_EQ(row_store_copy_lists_.size(), accessors.size());
+ DCHECK_EQ(direct_copy_lists_.size(), accessors.size());
+ DCHECK(!non_empty_copy_indices_.empty());
+
+ std::size_t num_tuples_inserted;
+ for (const auto idx : non_empty_copy_indices_) {
+ ValueAccessor *accessor = accessors[idx];
+ const auto impl = accessor->getImplementationType();
+
+ if (impl == ValueAccessor::Implementation::kPackedRowStore ||
+ impl == ValueAccessor::Implementation::kSplitRowStore) {
+ num_tuples_inserted =
+ row_store_copy_lists_[idx]->bulkInsertTuples(
+ accessor, stride_width, num_tuples, storage);
+// std::cerr << "Use row_store\n";
+ } else {
+ num_tuples_inserted =
+ direct_copy_lists_[idx]->bulkInsertTuples(
+ accessor, stride_width, num_tuples, storage);
+// std::cerr << "Use direct\n";
+ }
+
+ iteration_finished_ = accessor->iterationFinishedVirtual();
+ }
+
+ return num_tuples_inserted;
+ }
+
+ void beginIteration() {
+ iteration_finished_ = false;
+ }
+
+ bool iterationFinished() const {
+ return iteration_finished_;
+ }
+
+ private:
+ void addSourceRowStore(const std::map<attribute_id, attribute_id> &attribute_map) {
std::vector<CopyGroup> copy_groups;
if (!attribute_map.empty()) {
@@ -221,52 +319,34 @@ class InsertContext {
output_relation_.getFixedLengthAttributeOffset(init_dst_attr_id),
accum_length);
- for (const auto &cg : copy_groups) {
- std::cout << cg.source_attr_id << ": " << cg.bytes_to_copy << " @" << cg.bytes_to_advance << "\n";
- }
- non_empty_copy_indices_.emplace_back(copy_lists_.size());
+// for (const auto &cg : copy_groups) {
+// std::cerr << cg.source_attr_id << ": " << cg.bytes_to_copy << " @" << cg.bytes_to_advance << "\n";
+// }
}
- copy_lists_.emplace_back(std::make_unique<CopyList>(copy_groups));
+ row_store_copy_lists_.emplace_back(std::make_unique<CopyList>(copy_groups));
}
- std::size_t bulkInsertTuples(const std::vector<ValueAccessor *> &accessors,
- const std::size_t stride_width,
- const std::size_t num_tuples,
- void *storage) {
- DCHECK_EQ(copy_lists_.size(), accessors.size());
- DCHECK(!non_empty_copy_indices_.empty());
-
- auto idx_it = non_empty_copy_indices_.begin();
- const std::size_t num_tuples_inserted =
- copy_lists_[*idx_it]->bulkInsertTuples(
- accessors[*idx_it], stride_width, num_tuples, storage);
- iteration_finished_ = accessors[*idx_it]->iterationFinishedVirtual();
+ void addSourceDirect(const std::map<attribute_id, attribute_id> &attribute_map) {
+ std::vector<CopyGroup> copy_groups;
- for (++idx_it; idx_it != non_empty_copy_indices_.end(); ++idx_it) {
- const std::size_t other_num_tuples_inserted =
- copy_lists_[*idx_it]->bulkInsertTuples(
- accessors[*idx_it], stride_width, num_tuples, storage);
+ for (const auto &attr_pair : attribute_map) {
+ const std::size_t bytes_to_advance =
+ output_relation_.getFixedLengthAttributeOffset(attr_pair.second);
+ const std::size_t bytes_to_copy =
+ output_relation_.getAttributeById(attr_pair.second)->getType().maximumByteLength();
- (void)other_num_tuples_inserted;
- DCHECK_EQ(num_tuples_inserted, other_num_tuples_inserted);
- DCHECK_EQ(iteration_finished_, accessors[*idx_it]->iterationFinishedVirtual());
+ copy_groups.emplace_back(attr_pair.first,
+ bytes_to_advance,
+ bytes_to_copy);
}
- return num_tuples_inserted;
- }
-
- void beginIteration() {
- iteration_finished_ = false;
- }
-
- bool iterationFinished() const {
- return iteration_finished_;
+ direct_copy_lists_.emplace_back(std::make_unique<CopyList>(copy_groups));
}
- private:
const CatalogRelationSchema &output_relation_;
- std::vector<std::unique_ptr<CopyList>> copy_lists_;
+ std::vector<std::unique_ptr<CopyList>> direct_copy_lists_;
+ std::vector<std::unique_ptr<CopyList>> row_store_copy_lists_;
std::vector<std::size_t> non_empty_copy_indices_;
bool iteration_finished_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65b92b47/storage/InsertDestination.cpp
----------------------------------------------------------------------
diff --git a/storage/InsertDestination.cpp b/storage/InsertDestination.cpp
index 5c7d430..1919ebf 100644
--- a/storage/InsertDestination.cpp
+++ b/storage/InsertDestination.cpp
@@ -222,6 +222,28 @@ void InsertDestination::bulkInsertTuples(ValueAccessor *accessor, bool always_ma
});
}
+
+
+void InsertDestination::bulkInsertTuples(const std::vector<ValueAccessor *> &accessors,
+ InsertContext *insert_context) {
+ DCHECK_GE(accessors.size(), 1u);
+
+ insert_context->beginIteration();
+ while (!insert_context->iterationFinished()) {
+ MutableBlockReference output_block = this->getBlockForInsertion();
+ // FIXME(chasseur): Deal with TupleTooLargeForBlock exception.
+ if (output_block->bulkInsertTuples(accessors, insert_context) == 0) {
+ // output_block is full.
+ this->returnBlock(std::move(output_block), true);
+ } else {
+ // Bulk insert into output_block was successful. output_block
+ // will be rebuilt when there won't be any more insertions to it.
+ this->returnBlock(std::move(output_block),
+ !insert_context->iterationFinished());
+ }
+ }
+}
+
void InsertDestination::bulkInsertTuples(const std::vector<ValueAccessor *> &accessors,
InsertContext *insert_context,
MutableBlockReference *output_block) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65b92b47/storage/InsertDestination.hpp
----------------------------------------------------------------------
diff --git a/storage/InsertDestination.hpp b/storage/InsertDestination.hpp
index ca2ed57..5dbd933 100644
--- a/storage/InsertDestination.hpp
+++ b/storage/InsertDestination.hpp
@@ -149,6 +149,9 @@ class InsertDestination : public InsertDestinationInterface {
void bulkInsertTuples(ValueAccessor *accessor, bool always_mark_full = false) override;
void bulkInsertTuples(const std::vector<ValueAccessor *> &accessors,
+ InsertContext *insert_context) override;
+
+ void bulkInsertTuples(const std::vector<ValueAccessor *> &accessors,
InsertContext *insert_context,
MutableBlockReference *output_block);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65b92b47/storage/InsertDestinationInterface.hpp
----------------------------------------------------------------------
diff --git a/storage/InsertDestinationInterface.hpp b/storage/InsertDestinationInterface.hpp
index 423dff1..227dae6 100644
--- a/storage/InsertDestinationInterface.hpp
+++ b/storage/InsertDestinationInterface.hpp
@@ -28,6 +28,7 @@
namespace quickstep {
class CatalogRelationSchema;
+class InsertContext;
class ValueAccessor;
/** \addtogroup Storage
@@ -104,6 +105,9 @@ class InsertDestinationInterface {
virtual void bulkInsertTuples(ValueAccessor *accessor,
bool always_mark_full = false) = 0;
+ virtual void bulkInsertTuples(const std::vector<ValueAccessor *> &accessors,
+ InsertContext *insert_context) = 0;
+
/**
* @brief Bulk-insert tuples from a ValueAccessor with differently-ordered
* attributes into blocks managed by this InsertDestination.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65b92b47/storage/StorageBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.cpp b/storage/StorageBlock.cpp
index 9029cd7..d7dffb8 100644
--- a/storage/StorageBlock.cpp
+++ b/storage/StorageBlock.cpp
@@ -20,6 +20,7 @@
#include "storage/StorageBlock.hpp"
#include <climits>
+#include <map>
#include <memory>
#include <type_traits>
#include <unordered_map>
@@ -31,6 +32,7 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/predicate/Predicate.hpp"
#include "expressions/scalar/Scalar.hpp"
+#include "expressions/scalar/ScalarAttribute.hpp"
#include "storage/BasicColumnStoreTupleStorageSubBlock.hpp"
#include "storage/BloomFilterIndexSubBlock.hpp"
#include "storage/CSBTreeIndexSubBlock.hpp"
@@ -39,6 +41,7 @@
#include "storage/CountedReference.hpp"
#include "storage/HashTableBase.hpp"
#include "storage/IndexSubBlock.hpp"
+#include "storage/InsertContext.hpp"
#include "storage/InsertDestinationInterface.hpp"
#include "storage/PackedRowStoreTupleStorageSubBlock.hpp"
#include "storage/SMAIndexSubBlock.hpp"
@@ -358,25 +361,46 @@ void StorageBlock::sample(const bool is_block_sample,
void StorageBlock::select(const vector<unique_ptr<const Scalar>> &selection,
const TupleIdSequence *filter,
InsertDestinationInterface *destination) const {
+ std::map<attribute_id, attribute_id> base_attribute_map;
+ std::map<attribute_id, attribute_id> non_trivial_attribute_map;
+ std::vector<const Scalar *> non_trivial_expressions;
+ for (std::size_t i = 0; i < selection.size(); ++i) {
+ const Scalar *scalar = selection[i].get();
+ if (scalar->getDataSource() == Scalar::ScalarDataSource::kAttribute) {
+ const ScalarAttribute *scalar_attr =
+ static_cast<const ScalarAttribute *>(scalar);
+ base_attribute_map.emplace(
+ scalar_attr->getAttributeIdForValueAccessor(), i);
+ } else {
+ non_trivial_attribute_map.emplace(non_trivial_expressions.size(), i);
+ non_trivial_expressions.emplace_back(scalar);
+ }
+ }
+
+ std::unique_ptr<InsertContext> insert_context(
+ new InsertContext(destination->getRelation()));
+ insert_context->addSource(base_attribute_map);
+ insert_context->addSource(non_trivial_attribute_map);
+
ColumnVectorsValueAccessor temp_result;
- {
- SubBlocksReference sub_blocks_ref(*tuple_store_,
- indices_,
- indices_consistent_);
+ SubBlocksReference sub_blocks_ref(*tuple_store_,
+ indices_,
+ indices_consistent_);
- std::unique_ptr<ValueAccessor> accessor(
- tuple_store_->createValueAccessor(filter));
+ std::unique_ptr<ValueAccessor> accessor(
+ tuple_store_->createValueAccessor(filter));
- for (vector<unique_ptr<const Scalar>>::const_iterator selection_cit = selection.begin();
- selection_cit != selection.end();
- ++selection_cit) {
- // TODO(chasseur): Can probably elide some copies for parts of the
- // selection that are ScalarAttribute or ScalarLiteral.
- temp_result.addColumn((*selection_cit)->getAllValues(accessor.get(), &sub_blocks_ref));
- }
+ for (auto expr_cit = non_trivial_expressions.begin();
+ expr_cit != non_trivial_expressions.end();
+ ++expr_cit) {
+ temp_result.addColumn((*expr_cit)->getAllValues(accessor.get(), &sub_blocks_ref));
}
- destination->bulkInsertTuples(&temp_result);
+ // Rewind the base accessor.
+ accessor->beginIterationVirtual();
+
+ destination->bulkInsertTuples({ accessor.get(), &temp_result },
+ insert_context.get());
}
void StorageBlock::selectSimple(const std::vector<attribute_id> &selection,
@@ -385,8 +409,17 @@ void StorageBlock::selectSimple(const std::vector<attribute_id> &selection,
std::unique_ptr<ValueAccessor> accessor(
tuple_store_->createValueAccessor(filter));
- destination->bulkInsertTuplesWithRemappedAttributes(selection,
- accessor.get());
+ std::map<attribute_id, attribute_id> attribute_map;
+ for (std::size_t i = 0; i < selection.size(); ++i) {
+ attribute_map.emplace(selection[i], i);
+ }
+
+ std::unique_ptr<InsertContext> insert_context(
+ new InsertContext(destination->getRelation()));
+ insert_context->addSource(attribute_map);
+
+ const std::vector<ValueAccessor *> accessors = { accessor.get() };
+ destination->bulkInsertTuples(accessors, insert_context.get());
}
AggregationState* StorageBlock::aggregate(