You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by ji...@apache.org on 2016/10/05 04:25:35 UTC
[01/17] incubator-quickstep git commit: QUICKSTEP-36 fixed. [Forced
Update!]
Repository: incubator-quickstep
Updated Branches:
refs/heads/lip-refactor 0cca12890 -> 396f8576a (forced update)
QUICKSTEP-36 fixed.
Added read access check in TextScanOperator file before opening a file
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/43c7a42d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/43c7a42d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/43c7a42d
Branch: refs/heads/lip-refactor
Commit: 43c7a42db736aa6e61e6f4db12721ded6e646e13
Parents: 590ba4d
Author: Tarun Bansal <ta...@gmail.com>
Authored: Tue Sep 6 23:40:28 2016 -0500
Committer: tarunbansal <ta...@gmail.com>
Committed: Wed Sep 14 11:18:25 2016 -0500
----------------------------------------------------------------------
relational_operators/CMakeLists.txt | 7 +++++++
.../RelationalOperatorsConfig.h.in | 20 ++++++++++++++++++++
relational_operators/TextScanOperator.cpp | 14 ++++++++++++++
3 files changed, 41 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/43c7a42d/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index cdfe309..a9645b4 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -15,6 +15,13 @@
# specific language governing permissions and limitations
# under the License.
+include(CheckIncludeFiles)
+check_include_files("unistd.h" QUICKSTEP_HAVE_UNISTD)
+configure_file (
+ "${CMAKE_CURRENT_SOURCE_DIR}/RelationalOperatorsConfig.h.in"
+ "${CMAKE_CURRENT_BINARY_DIR}/RelationalOperatorsConfig.h"
+)
+
QS_PROTOBUF_GENERATE_CPP(relationaloperators_SortMergeRunOperator_proto_srcs
relationaloperators_SortMergeRunOperator_proto_hdrs
SortMergeRunOperator.proto)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/43c7a42d/relational_operators/RelationalOperatorsConfig.h.in
----------------------------------------------------------------------
diff --git a/relational_operators/RelationalOperatorsConfig.h.in b/relational_operators/RelationalOperatorsConfig.h.in
new file mode 100644
index 0000000..879d5b3
--- /dev/null
+++ b/relational_operators/RelationalOperatorsConfig.h.in
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#cmakedefine QUICKSTEP_HAVE_UNISTD
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/43c7a42d/relational_operators/TextScanOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/TextScanOperator.cpp b/relational_operators/TextScanOperator.cpp
index 1a0b715..4151bac 100644
--- a/relational_operators/TextScanOperator.cpp
+++ b/relational_operators/TextScanOperator.cpp
@@ -19,6 +19,12 @@
#include "relational_operators/TextScanOperator.hpp"
+#include "relational_operators/RelationalOperatorsConfig.h" // For QUICKSTEP_HAVE_UNISTD.
+
+#ifdef QUICKSTEP_HAVE_UNISTD
+#include <unistd.h>
+#endif // QUICKSTEP_HAVE_UNISTD
+
#include <algorithm>
#include <cctype>
#include <cstddef>
@@ -91,6 +97,14 @@ bool TextScanOperator::getAllWorkOrders(
if (blocking_dependencies_met_ && !work_generated_) {
for (const std::string &file : files) {
// Use standard C libary to retrieve the file size.
+
+#ifdef QUICKSTEP_HAVE_UNISTD
+ // Check file permissions before trying to open it.
+ const int access_result = access(file.c_str(), R_OK);
+ CHECK_EQ(0, access_result)
+ << "File " << file << " is not readable due to permission issues.";
+#endif // QUICKSTEP_HAVE_UNISTD
+
FILE *fp = std::fopen(file.c_str(), "rb");
std::fseek(fp, 0, SEEK_END);
const std::size_t file_size = std::ftell(fp);
[05/17] incubator-quickstep git commit: Initial commit for
QUICKSTEP-28 and QUICKSTEP-29.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/storage/AggregationOperationState.cpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.cpp b/storage/AggregationOperationState.cpp
index 3f6e23a..073b813 100644
--- a/storage/AggregationOperationState.cpp
+++ b/storage/AggregationOperationState.cpp
@@ -59,7 +59,7 @@ namespace quickstep {
AggregationOperationState::AggregationOperationState(
const CatalogRelationSchema &input_relation,
- const std::vector<const AggregateFunction*> &aggregate_functions,
+ const std::vector<const AggregateFunction *> &aggregate_functions,
std::vector<std::vector<std::unique_ptr<const Scalar>>> &&arguments,
std::vector<bool> &&is_distinct,
std::vector<std::unique_ptr<const Scalar>> &&group_by,
@@ -78,11 +78,14 @@ AggregationOperationState::AggregationOperationState(
DCHECK(aggregate_functions.size() == arguments_.size());
// Get the types of GROUP BY expressions for creating HashTables below.
- std::vector<const Type*> group_by_types;
+ std::vector<const Type *> group_by_types;
for (const std::unique_ptr<const Scalar> &group_by_element : group_by_list_) {
group_by_types.emplace_back(&group_by_element->getType());
}
+ std::vector<AggregationHandle *> group_by_handles;
+ group_by_handles.clear();
+
if (aggregate_functions.size() == 0) {
// If there is no aggregation function, then it is a distinctify operation
// on the group-by expressions.
@@ -91,26 +94,28 @@ AggregationOperationState::AggregationOperationState(
handles_.emplace_back(new AggregationHandleDistinct());
arguments_.push_back({});
is_distinct_.emplace_back(false);
-
- group_by_hashtable_pools_.emplace_back(std::unique_ptr<HashTablePool>(
- new HashTablePool(estimated_num_entries,
- hash_table_impl_type,
- group_by_types,
- handles_.back().get(),
- storage_manager)));
+ group_by_hashtable_pool_.reset(new HashTablePool(estimated_num_entries,
+ hash_table_impl_type,
+ group_by_types,
+ {1},
+ handles_,
+ storage_manager));
} else {
// Set up each individual aggregate in this operation.
- std::vector<const AggregateFunction*>::const_iterator agg_func_it
- = aggregate_functions.begin();
- std::vector<std::vector<std::unique_ptr<const Scalar>>>::const_iterator args_it
- = arguments_.begin();
+ std::vector<const AggregateFunction *>::const_iterator agg_func_it =
+ aggregate_functions.begin();
+ std::vector<std::vector<std::unique_ptr<const Scalar>>>::const_iterator
+ args_it = arguments_.begin();
std::vector<bool>::const_iterator is_distinct_it = is_distinct_.begin();
- std::vector<HashTableImplType>::const_iterator distinctify_hash_table_impl_types_it
- = distinctify_hash_table_impl_types.begin();
- for (; agg_func_it != aggregate_functions.end(); ++agg_func_it, ++args_it, ++is_distinct_it) {
+ std::vector<HashTableImplType>::const_iterator
+ distinctify_hash_table_impl_types_it =
+ distinctify_hash_table_impl_types.begin();
+ std::vector<std::size_t> payload_sizes;
+ for (; agg_func_it != aggregate_functions.end();
+ ++agg_func_it, ++args_it, ++is_distinct_it) {
// Get the Types of this aggregate's arguments so that we can create an
// AggregationHandle.
- std::vector<const Type*> argument_types;
+ std::vector<const Type *> argument_types;
for (const std::unique_ptr<const Scalar> &argument : *args_it) {
argument_types.emplace_back(&argument->getType());
}
@@ -125,13 +130,13 @@ AggregationOperationState::AggregationOperationState(
handles_.emplace_back((*agg_func_it)->createHandle(argument_types));
if (!group_by_list_.empty()) {
- // Aggregation with GROUP BY: create a HashTable pool for per-group states.
- group_by_hashtable_pools_.emplace_back(std::unique_ptr<HashTablePool>(
- new HashTablePool(estimated_num_entries,
- hash_table_impl_type,
- group_by_types,
- handles_.back().get(),
- storage_manager)));
+ // Aggregation with GROUP BY: combined payload is partially updated in
+ // the presence of DISTINCT.
+ if (*is_distinct_it) {
+ handles_.back()->blockUpdate();
+ }
+ group_by_handles.emplace_back(handles_.back());
+ payload_sizes.emplace_back(group_by_handles.back()->getPayloadSize());
} else {
// Aggregation without GROUP BY: create a single global state.
single_states_.emplace_back(handles_.back()->createInitialState());
@@ -143,40 +148,60 @@ AggregationOperationState::AggregationOperationState(
std::vector<attribute_id> local_arguments_as_attributes;
local_arguments_as_attributes.reserve(args_it->size());
for (const std::unique_ptr<const Scalar> &argument : *args_it) {
- const attribute_id argument_id = argument->getAttributeIdForValueAccessor();
+ const attribute_id argument_id =
+ argument->getAttributeIdForValueAccessor();
if (argument_id == -1) {
local_arguments_as_attributes.clear();
break;
} else {
- DCHECK_EQ(input_relation_.getID(), argument->getRelationIdForValueAccessor());
+ DCHECK_EQ(input_relation_.getID(),
+ argument->getRelationIdForValueAccessor());
local_arguments_as_attributes.push_back(argument_id);
}
}
- arguments_as_attributes_.emplace_back(std::move(local_arguments_as_attributes));
+ arguments_as_attributes_.emplace_back(
+ std::move(local_arguments_as_attributes));
#endif
}
- // Initialize the corresponding distinctify hash table if this is a DISTINCT
+ // Initialize the corresponding distinctify hash table if this is a
+ // DISTINCT
// aggregation.
if (*is_distinct_it) {
- std::vector<const Type*> key_types(group_by_types);
- key_types.insert(key_types.end(), argument_types.begin(), argument_types.end());
- // TODO(jianqiao): estimated_num_entries is quite inaccurate for estimating
+ std::vector<const Type *> key_types(group_by_types);
+ key_types.insert(
+ key_types.end(), argument_types.begin(), argument_types.end());
+ // TODO(jianqiao): estimated_num_entries is quite inaccurate for
+ // estimating
// the number of entries in the distinctify hash table. We may estimate
- // for each distinct aggregation an estimated_num_distinct_keys value during
+ // for each distinct aggregation an estimated_num_distinct_keys value
+ // during
// query optimization, if it worths.
distinctify_hashtables_.emplace_back(
- handles_.back()->createDistinctifyHashTable(
+ AggregationStateFastHashTableFactory::CreateResizable(
*distinctify_hash_table_impl_types_it,
key_types,
estimated_num_entries,
+ {0},
+ {},
storage_manager));
++distinctify_hash_table_impl_types_it;
} else {
distinctify_hashtables_.emplace_back(nullptr);
}
}
+
+ if (!group_by_handles.empty()) {
+ // Aggregation with GROUP BY: create a HashTable pool for per-group
+ // states.
+ group_by_hashtable_pool_.reset(new HashTablePool(estimated_num_entries,
+ hash_table_impl_type,
+ group_by_types,
+ payload_sizes,
+ group_by_handles,
+ storage_manager));
+ }
}
}
@@ -187,7 +212,7 @@ AggregationOperationState* AggregationOperationState::ReconstructFromProto(
DCHECK(ProtoIsValid(proto, database));
// Rebuild contructor arguments from their representation in 'proto'.
- std::vector<const AggregateFunction*> aggregate_functions;
+ std::vector<const AggregateFunction *> aggregate_functions;
std::vector<std::vector<std::unique_ptr<const Scalar>>> arguments;
std::vector<bool> is_distinct;
std::vector<HashTableImplType> distinctify_hash_table_impl_types;
@@ -200,62 +225,63 @@ AggregationOperationState* AggregationOperationState::ReconstructFromProto(
arguments.emplace_back();
arguments.back().reserve(agg_proto.argument_size());
- for (int argument_idx = 0; argument_idx < agg_proto.argument_size(); ++argument_idx) {
+ for (int argument_idx = 0; argument_idx < agg_proto.argument_size();
+ ++argument_idx) {
arguments.back().emplace_back(ScalarFactory::ReconstructFromProto(
- agg_proto.argument(argument_idx),
- database));
+ agg_proto.argument(argument_idx), database));
}
is_distinct.emplace_back(agg_proto.is_distinct());
if (agg_proto.is_distinct()) {
distinctify_hash_table_impl_types.emplace_back(
- HashTableImplTypeFromProto(
- proto.distinctify_hash_table_impl_types(distinctify_hash_table_impl_type_index)));
+ HashTableImplTypeFromProto(proto.distinctify_hash_table_impl_types(
+ distinctify_hash_table_impl_type_index)));
++distinctify_hash_table_impl_type_index;
}
}
std::vector<std::unique_ptr<const Scalar>> group_by_expressions;
- for (int group_by_idx = 0;
- group_by_idx < proto.group_by_expressions_size();
+ for (int group_by_idx = 0; group_by_idx < proto.group_by_expressions_size();
++group_by_idx) {
group_by_expressions.emplace_back(ScalarFactory::ReconstructFromProto(
- proto.group_by_expressions(group_by_idx),
- database));
+ proto.group_by_expressions(group_by_idx), database));
}
unique_ptr<Predicate> predicate;
if (proto.has_predicate()) {
predicate.reset(
- PredicateFactory::ReconstructFromProto(proto.predicate(),
- database));
+ PredicateFactory::ReconstructFromProto(proto.predicate(), database));
}
- return new AggregationOperationState(database.getRelationSchemaById(proto.relation_id()),
- aggregate_functions,
- std::move(arguments),
- std::move(is_distinct),
- std::move(group_by_expressions),
- predicate.release(),
- proto.estimated_num_entries(),
- HashTableImplTypeFromProto(proto.hash_table_impl_type()),
- distinctify_hash_table_impl_types,
- storage_manager);
+ return new AggregationOperationState(
+ database.getRelationSchemaById(proto.relation_id()),
+ aggregate_functions,
+ std::move(arguments),
+ std::move(is_distinct),
+ std::move(group_by_expressions),
+ predicate.release(),
+ proto.estimated_num_entries(),
+ HashTableImplTypeFromProto(proto.hash_table_impl_type()),
+ distinctify_hash_table_impl_types,
+ storage_manager);
}
-bool AggregationOperationState::ProtoIsValid(const serialization::AggregationOperationState &proto,
- const CatalogDatabaseLite &database) {
+bool AggregationOperationState::ProtoIsValid(
+ const serialization::AggregationOperationState &proto,
+ const CatalogDatabaseLite &database) {
if (!proto.IsInitialized() ||
!database.hasRelationWithId(proto.relation_id()) ||
(proto.aggregates_size() < 0)) {
return false;
}
- std::size_t num_distinctify_hash_tables = proto.distinctify_hash_table_impl_types_size();
+ std::size_t num_distinctify_hash_tables =
+ proto.distinctify_hash_table_impl_types_size();
std::size_t distinctify_hash_table_impl_type_index = 0;
for (int i = 0; i < proto.aggregates_size(); ++i) {
- if (!AggregateFunctionFactory::ProtoIsValid(proto.aggregates(i).function())) {
+ if (!AggregateFunctionFactory::ProtoIsValid(
+ proto.aggregates(i).function())) {
return false;
}
@@ -266,16 +292,18 @@ bool AggregationOperationState::ProtoIsValid(const serialization::AggregationOpe
for (int argument_idx = 0;
argument_idx < proto.aggregates(i).argument_size();
++argument_idx) {
- if (!ScalarFactory::ProtoIsValid(proto.aggregates(i).argument(argument_idx),
- database)) {
+ if (!ScalarFactory::ProtoIsValid(
+ proto.aggregates(i).argument(argument_idx), database)) {
return false;
}
}
if (proto.aggregates(i).is_distinct()) {
- if (distinctify_hash_table_impl_type_index >= num_distinctify_hash_tables ||
+ if (distinctify_hash_table_impl_type_index >=
+ num_distinctify_hash_tables ||
!serialization::HashTableImplType_IsValid(
- proto.distinctify_hash_table_impl_types(distinctify_hash_table_impl_type_index))) {
+ proto.distinctify_hash_table_impl_types(
+ distinctify_hash_table_impl_type_index))) {
return false;
}
}
@@ -288,8 +316,9 @@ bool AggregationOperationState::ProtoIsValid(const serialization::AggregationOpe
}
if (proto.group_by_expressions_size() > 0) {
- if (!proto.has_hash_table_impl_type()
- || !serialization::HashTableImplType_IsValid(proto.hash_table_impl_type())) {
+ if (!proto.has_hash_table_impl_type() ||
+ !serialization::HashTableImplType_IsValid(
+ proto.hash_table_impl_type())) {
return false;
}
}
@@ -311,7 +340,8 @@ void AggregationOperationState::aggregateBlock(const block_id input_block) {
}
}
-void AggregationOperationState::finalizeAggregate(InsertDestination *output_destination) {
+void AggregationOperationState::finalizeAggregate(
+ InsertDestination *output_destination) {
if (group_by_list_.empty()) {
finalizeSingleState(output_destination);
} else {
@@ -330,19 +360,19 @@ void AggregationOperationState::mergeSingleState(
}
}
-void AggregationOperationState::aggregateBlockSingleState(const block_id input_block) {
+void AggregationOperationState::aggregateBlockSingleState(
+ const block_id input_block) {
// Aggregate per-block state for each aggregate.
std::vector<std::unique_ptr<AggregationState>> local_state;
- BlockReference block(storage_manager_->getBlock(input_block, input_relation_));
+ BlockReference block(
+ storage_manager_->getBlock(input_block, input_relation_));
// If there is a filter predicate, 'reuse_matches' holds the set of matching
// tuples so that it can be reused across multiple aggregates (i.e. we only
// pay the cost of evaluating the predicate once).
std::unique_ptr<TupleIdSequence> reuse_matches;
- for (std::size_t agg_idx = 0;
- agg_idx < handles_.size();
- ++agg_idx) {
+ for (std::size_t agg_idx = 0; agg_idx < handles_.size(); ++agg_idx) {
const std::vector<attribute_id> *local_arguments_as_attributes = nullptr;
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
// If all arguments are attributes of the input relation, elide a copy.
@@ -365,12 +395,11 @@ void AggregationOperationState::aggregateBlockSingleState(const block_id input_b
local_state.emplace_back(nullptr);
} else {
// Call StorageBlock::aggregate() to actually do the aggregation.
- local_state.emplace_back(
- block->aggregate(*handles_[agg_idx],
- arguments_[agg_idx],
- local_arguments_as_attributes,
- predicate_.get(),
- &reuse_matches));
+ local_state.emplace_back(block->aggregate(*handles_[agg_idx],
+ arguments_[agg_idx],
+ local_arguments_as_attributes,
+ predicate_.get(),
+ &reuse_matches));
}
}
@@ -378,8 +407,10 @@ void AggregationOperationState::aggregateBlockSingleState(const block_id input_b
mergeSingleState(local_state);
}
-void AggregationOperationState::aggregateBlockHashTable(const block_id input_block) {
- BlockReference block(storage_manager_->getBlock(input_block, input_relation_));
+void AggregationOperationState::aggregateBlockHashTable(
+ const block_id input_block) {
+ BlockReference block(
+ storage_manager_->getBlock(input_block, input_relation_));
// If there is a filter predicate, 'reuse_matches' holds the set of matching
// tuples so that it can be reused across multiple aggregates (i.e. we only
@@ -391,11 +422,10 @@ void AggregationOperationState::aggregateBlockHashTable(const block_id input_blo
// GROUP BY expressions once).
std::vector<std::unique_ptr<ColumnVector>> reuse_group_by_vectors;
- for (std::size_t agg_idx = 0;
- agg_idx < handles_.size();
- ++agg_idx) {
+ for (std::size_t agg_idx = 0; agg_idx < handles_.size(); ++agg_idx) {
if (is_distinct_[agg_idx]) {
- // Call StorageBlock::aggregateDistinct() to insert the GROUP BY expression
+ // Call StorageBlock::aggregateDistinct() to insert the GROUP BY
+ // expression
// values and the aggregation arguments together as keys directly into the
// (threadsafe) shared global distinctify HashTable for this aggregate.
block->aggregateDistinct(*handles_[agg_idx],
@@ -406,45 +436,54 @@ void AggregationOperationState::aggregateBlockHashTable(const block_id input_blo
distinctify_hashtables_[agg_idx].get(),
&reuse_matches,
&reuse_group_by_vectors);
- } else {
- // Call StorageBlock::aggregateGroupBy() to aggregate this block's values
- // directly into the (threadsafe) shared global HashTable for this
- // aggregate.
- DCHECK(group_by_hashtable_pools_[agg_idx] != nullptr);
- AggregationStateHashTableBase *agg_hash_table = group_by_hashtable_pools_[agg_idx]->getHashTable();
- DCHECK(agg_hash_table != nullptr);
- block->aggregateGroupBy(*handles_[agg_idx],
- arguments_[agg_idx],
- group_by_list_,
- predicate_.get(),
- agg_hash_table,
- &reuse_matches,
- &reuse_group_by_vectors);
- group_by_hashtable_pools_[agg_idx]->returnHashTable(agg_hash_table);
}
}
+
+ // Call StorageBlock::aggregateGroupBy() to aggregate this block's values
+ // directly into the (threadsafe) shared global HashTable for this
+ // aggregate.
+ DCHECK(group_by_hashtable_pool_ != nullptr);
+ AggregationStateHashTableBase *agg_hash_table =
+ group_by_hashtable_pool_->getHashTableFast();
+ DCHECK(agg_hash_table != nullptr);
+ block->aggregateGroupBy(arguments_,
+ group_by_list_,
+ predicate_.get(),
+ agg_hash_table,
+ &reuse_matches,
+ &reuse_group_by_vectors);
+ group_by_hashtable_pool_->returnHashTable(agg_hash_table);
}
-void AggregationOperationState::finalizeSingleState(InsertDestination *output_destination) {
+void AggregationOperationState::finalizeSingleState(
+ InsertDestination *output_destination) {
// Simply build up a Tuple from the finalized values for each aggregate and
// insert it in '*output_destination'.
std::vector<TypedValue> attribute_values;
- for (std::size_t agg_idx = 0;
- agg_idx < handles_.size();
- ++agg_idx) {
+ for (std::size_t agg_idx = 0; agg_idx < handles_.size(); ++agg_idx) {
if (is_distinct_[agg_idx]) {
single_states_[agg_idx].reset(
- handles_[agg_idx]->aggregateOnDistinctifyHashTableForSingle(*distinctify_hashtables_[agg_idx]));
+ handles_[agg_idx]->aggregateOnDistinctifyHashTableForSingle(
+ *distinctify_hashtables_[agg_idx]));
}
- attribute_values.emplace_back(handles_[agg_idx]->finalize(*single_states_[agg_idx]));
+ attribute_values.emplace_back(
+ handles_[agg_idx]->finalize(*single_states_[agg_idx]));
}
output_destination->insertTuple(Tuple(std::move(attribute_values)));
}
-void AggregationOperationState::finalizeHashTable(InsertDestination *output_destination) {
+void AggregationOperationState::mergeGroupByHashTables(
+ AggregationStateHashTableBase *src, AggregationStateHashTableBase *dst) {
+ HashTableMergerFast merger(dst);
+ (static_cast<FastHashTable<true, false, true, false> *>(src))
+ ->forEachCompositeKeyFast(&merger);
+}
+
+void AggregationOperationState::finalizeHashTable(
+ InsertDestination *output_destination) {
// Each element of 'group_by_keys' is a vector of values for a particular
// group (which is also the prefix of the finalized Tuple for that group).
std::vector<std::vector<TypedValue>> group_by_keys;
@@ -455,60 +494,57 @@ void AggregationOperationState::finalizeHashTable(InsertDestination *output_dest
// TODO(harshad) - Find heuristics for faster merge, even in a single thread.
// e.g. Keep merging entries from smaller hash tables to larger.
- for (std::size_t agg_idx = 0; agg_idx < handles_.size(); ++agg_idx) {
- auto *hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
- if (hash_tables->size() > 1) {
- for (int hash_table_index = 0;
- hash_table_index < static_cast<int>(hash_tables->size() - 1);
- ++hash_table_index) {
- // Merge each hash table to the last hash table.
- handles_[agg_idx]->mergeGroupByHashTables(
- (*(*hash_tables)[hash_table_index]),
- hash_tables->back().get());
- }
+
+ auto *hash_tables = group_by_hashtable_pool_->getAllHashTables();
+ if (hash_tables->size() > 1) {
+ for (int hash_table_index = 0;
+ hash_table_index < static_cast<int>(hash_tables->size() - 1);
+ ++hash_table_index) {
+ // Merge each hash table to the last hash table.
+ mergeGroupByHashTables((*hash_tables)[hash_table_index].get(),
+ hash_tables->back().get());
}
}
// Collect per-aggregate finalized values.
std::vector<std::unique_ptr<ColumnVector>> final_values;
- for (std::size_t agg_idx = 0;
- agg_idx < handles_.size();
- ++agg_idx) {
+ for (std::size_t agg_idx = 0; agg_idx < handles_.size(); ++agg_idx) {
if (is_distinct_[agg_idx]) {
- DCHECK(group_by_hashtable_pools_[agg_idx] != nullptr);
- auto *hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ DCHECK(group_by_hashtable_pool_ != nullptr);
+ auto *hash_tables = group_by_hashtable_pool_->getAllHashTables();
DCHECK(hash_tables != nullptr);
if (hash_tables->empty()) {
// We may have a case where hash_tables is empty, e.g. no input blocks.
// However for aggregateOnDistinctifyHashTableForGroupBy to work
// correctly, we should create an empty group by hash table.
- AggregationStateHashTableBase *new_hash_table = group_by_hashtable_pools_[agg_idx]->getHashTable();
- group_by_hashtable_pools_[agg_idx]->returnHashTable(new_hash_table);
- hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ AggregationStateHashTableBase *new_hash_table =
+ group_by_hashtable_pool_->getHashTableFast();
+ group_by_hashtable_pool_->returnHashTable(new_hash_table);
+ hash_tables = group_by_hashtable_pool_->getAllHashTables();
}
DCHECK(hash_tables->back() != nullptr);
AggregationStateHashTableBase *agg_hash_table = hash_tables->back().get();
DCHECK(agg_hash_table != nullptr);
+ handles_[agg_idx]->allowUpdate();
handles_[agg_idx]->aggregateOnDistinctifyHashTableForGroupBy(
- *distinctify_hashtables_[agg_idx],
- agg_hash_table);
+ *distinctify_hashtables_[agg_idx], agg_hash_table, agg_idx);
}
- auto *hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ auto *hash_tables = group_by_hashtable_pool_->getAllHashTables();
DCHECK(hash_tables != nullptr);
if (hash_tables->empty()) {
// We may have a case where hash_tables is empty, e.g. no input blocks.
// However for aggregateOnDistinctifyHashTableForGroupBy to work
// correctly, we should create an empty group by hash table.
- AggregationStateHashTableBase *new_hash_table = group_by_hashtable_pools_[agg_idx]->getHashTable();
- group_by_hashtable_pools_[agg_idx]->returnHashTable(new_hash_table);
- hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ AggregationStateHashTableBase *new_hash_table =
+ group_by_hashtable_pool_->getHashTable();
+ group_by_hashtable_pool_->returnHashTable(new_hash_table);
+ hash_tables = group_by_hashtable_pool_->getAllHashTables();
}
AggregationStateHashTableBase *agg_hash_table = hash_tables->back().get();
DCHECK(agg_hash_table != nullptr);
- ColumnVector* agg_result_col =
- handles_[agg_idx]->finalizeHashTable(*agg_hash_table,
- &group_by_keys);
+ ColumnVector *agg_result_col = handles_[agg_idx]->finalizeHashTable(
+ *agg_hash_table, &group_by_keys, agg_idx);
if (agg_result_col != nullptr) {
final_values.emplace_back(agg_result_col);
}
@@ -526,16 +562,20 @@ void AggregationOperationState::finalizeHashTable(InsertDestination *output_dest
for (const std::unique_ptr<const Scalar> &group_by_element : group_by_list_) {
const Type &group_by_type = group_by_element->getType();
if (NativeColumnVector::UsableForType(group_by_type)) {
- NativeColumnVector *element_cv = new NativeColumnVector(group_by_type, group_by_keys.size());
+ NativeColumnVector *element_cv =
+ new NativeColumnVector(group_by_type, group_by_keys.size());
group_by_cvs.emplace_back(element_cv);
for (std::vector<TypedValue> &group_key : group_by_keys) {
- element_cv->appendTypedValue(std::move(group_key[group_by_element_idx]));
+ element_cv->appendTypedValue(
+ std::move(group_key[group_by_element_idx]));
}
} else {
- IndirectColumnVector *element_cv = new IndirectColumnVector(group_by_type, group_by_keys.size());
+ IndirectColumnVector *element_cv =
+ new IndirectColumnVector(group_by_type, group_by_keys.size());
group_by_cvs.emplace_back(element_cv);
for (std::vector<TypedValue> &group_key : group_by_keys) {
- element_cv->appendTypedValue(std::move(group_key[group_by_element_idx]));
+ element_cv->appendTypedValue(
+ std::move(group_key[group_by_element_idx]));
}
}
++group_by_element_idx;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/storage/AggregationOperationState.hpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.hpp b/storage/AggregationOperationState.hpp
index ecd116b..cbbfc22 100644
--- a/storage/AggregationOperationState.hpp
+++ b/storage/AggregationOperationState.hpp
@@ -102,16 +102,17 @@ class AggregationOperationState {
* tables. Single aggregation state (when GROUP BY list is not
* specified) is not allocated using memory from storage manager.
*/
- AggregationOperationState(const CatalogRelationSchema &input_relation,
- const std::vector<const AggregateFunction*> &aggregate_functions,
- std::vector<std::vector<std::unique_ptr<const Scalar>>> &&arguments,
- std::vector<bool> &&is_distinct,
- std::vector<std::unique_ptr<const Scalar>> &&group_by,
- const Predicate *predicate,
- const std::size_t estimated_num_entries,
- const HashTableImplType hash_table_impl_type,
- const std::vector<HashTableImplType> &distinctify_hash_table_impl_types,
- StorageManager *storage_manager);
+ AggregationOperationState(
+ const CatalogRelationSchema &input_relation,
+ const std::vector<const AggregateFunction *> &aggregate_functions,
+ std::vector<std::vector<std::unique_ptr<const Scalar>>> &&arguments,
+ std::vector<bool> &&is_distinct,
+ std::vector<std::unique_ptr<const Scalar>> &&group_by,
+ const Predicate *predicate,
+ const std::size_t estimated_num_entries,
+ const HashTableImplType hash_table_impl_type,
+ const std::vector<HashTableImplType> &distinctify_hash_table_impl_types,
+ StorageManager *storage_manager);
~AggregationOperationState() {}
@@ -143,8 +144,9 @@ class AggregationOperationState {
* in.
* @return Whether proto is fully-formed and valid.
**/
- static bool ProtoIsValid(const serialization::AggregationOperationState &proto,
- const CatalogDatabaseLite &database);
+ static bool ProtoIsValid(
+ const serialization::AggregationOperationState &proto,
+ const CatalogDatabaseLite &database);
/**
* @brief Compute aggregates on the tuples of the given storage block,
@@ -165,10 +167,16 @@ class AggregationOperationState {
**/
void finalizeAggregate(InsertDestination *output_destination);
+ static void mergeGroupByHashTables(AggregationStateHashTableBase *src,
+ AggregationStateHashTableBase *dst);
+
+ int dflag;
+
private:
// Merge locally (per storage block) aggregated states with global aggregation
// states.
- void mergeSingleState(const std::vector<std::unique_ptr<AggregationState>> &local_state);
+ void mergeSingleState(
+ const std::vector<std::unique_ptr<AggregationState>> &local_state);
// Aggregate on input block.
void aggregateBlockSingleState(const block_id input_block);
@@ -185,7 +193,8 @@ class AggregationOperationState {
// Each individual aggregate in this operation has an AggregationHandle and
// some number of Scalar arguments.
- std::vector<std::unique_ptr<AggregationHandle>> handles_;
+ // std::vector<std::unique_ptr<AggregationHandle>> handles_;
+ std::vector<AggregationHandle *> handles_;
std::vector<std::vector<std::unique_ptr<const Scalar>>> arguments_;
// For each aggregate, whether DISTINCT should be applied to the aggregate's
@@ -193,7 +202,8 @@ class AggregationOperationState {
std::vector<bool> is_distinct_;
// Hash table for obtaining distinct (i.e. unique) arguments.
- std::vector<std::unique_ptr<AggregationStateHashTableBase>> distinctify_hashtables_;
+ std::vector<std::unique_ptr<AggregationStateHashTableBase>>
+ distinctify_hashtables_;
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
// If all an aggregate's argument expressions are simply attributes in
@@ -208,10 +218,11 @@ class AggregationOperationState {
//
// TODO(shoban): We should ideally store the aggregation state together in one
// hash table to prevent multiple lookups.
- std::vector<std::unique_ptr<AggregationStateHashTableBase>> group_by_hashtables_;
+ std::vector<std::unique_ptr<AggregationStateHashTableBase>>
+ group_by_hashtables_;
- // A vector of group by hash table pools, one for each group by clause.
- std::vector<std::unique_ptr<HashTablePool>> group_by_hashtable_pools_;
+ // A vector of group by hash table pools.
+ std::unique_ptr<HashTablePool> group_by_hashtable_pool_;
StorageManager *storage_manager_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 65a7975..f05cc46 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -198,6 +198,9 @@ if (ENABLE_DISTRIBUTED)
endif()
add_library(quickstep_storage_EvictionPolicy EvictionPolicy.cpp EvictionPolicy.hpp)
+add_library(quickstep_storage_FastHashTable ../empty_src.cpp FastHashTable.hpp)
+add_library(quickstep_storage_FastHashTableFactory ../empty_src.cpp FastHashTableFactory.hpp)
+add_library(quickstep_storage_FastSeparateChainingHashTable ../empty_src.cpp FastSeparateChainingHashTable.hpp)
add_library(quickstep_storage_FileManager ../empty_src.cpp FileManager.hpp)
if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
add_library(quickstep_storage_FileManagerHdfs FileManagerHdfs.cpp FileManagerHdfs.hpp)
@@ -626,6 +629,53 @@ target_link_libraries(quickstep_storage_EvictionPolicy
quickstep_threading_SpinMutex
quickstep_threading_SpinSharedMutex
quickstep_utility_Macros)
+target_link_libraries(quickstep_storage_FastHashTable
+ quickstep_catalog_CatalogTypedefs
+ quickstep_storage_HashTableBase
+ quickstep_storage_StorageBlob
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_storage_StorageManager
+ quickstep_storage_TupleReference
+ quickstep_storage_ValueAccessor
+ quickstep_storage_ValueAccessorUtil
+ quickstep_threading_SpinMutex
+ quickstep_threading_SpinSharedMutex
+ quickstep_types_Type
+ quickstep_types_TypedValue
+ quickstep_utility_BloomFilter
+ quickstep_utility_HashPair
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_storage_FastHashTableFactory
+ glog
+ quickstep_storage_FastHashTable
+ quickstep_storage_FastSeparateChainingHashTable
+ quickstep_storage_HashTable
+ quickstep_storage_HashTable_proto
+ quickstep_storage_HashTableBase
+ quickstep_storage_HashTableFactory
+ quickstep_storage_LinearOpenAddressingHashTable
+ quickstep_storage_SeparateChainingHashTable
+ quickstep_storage_SimpleScalarSeparateChainingHashTable
+ quickstep_storage_TupleReference
+ quickstep_types_TypeFactory
+ quickstep_utility_BloomFilter
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_storage_FastSeparateChainingHashTable
+ quickstep_storage_FastHashTable
+ quickstep_storage_HashTable
+ quickstep_storage_HashTableBase
+ quickstep_storage_HashTableKeyManager
+ quickstep_storage_StorageBlob
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_storage_StorageManager
+ quickstep_threading_SpinSharedMutex
+ quickstep_types_Type
+ quickstep_types_TypedValue
+ quickstep_utility_Alignment
+ quickstep_utility_Macros
+ quickstep_utility_PrimeNumber)
target_link_libraries(quickstep_storage_FileManager
quickstep_storage_StorageBlockInfo
quickstep_utility_Macros
@@ -711,6 +761,8 @@ target_link_libraries(quickstep_storage_HashTableKeyManager
target_link_libraries(quickstep_storage_HashTablePool
glog
quickstep_expressions_aggregation_AggregationHandle
+ quickstep_storage_FastHashTable
+ quickstep_storage_FastHashTableFactory
quickstep_storage_HashTableBase
quickstep_threading_SpinMutex
quickstep_utility_Macros
@@ -1098,6 +1150,9 @@ target_link_libraries(quickstep_storage
quickstep_storage_EvictionPolicy
quickstep_storage_FileManager
quickstep_storage_FileManagerLocal
+ quickstep_storage_FastHashTable
+ quickstep_storage_FastHashTableFactory
+ quickstep_storage_FastSeparateChainingHashTable
quickstep_storage_HashTable
quickstep_storage_HashTable_proto
quickstep_storage_HashTableBase
[04/17] incubator-quickstep git commit: Initial commit for
QUICKSTEP-28 and QUICKSTEP-29.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/storage/FastHashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/FastHashTable.hpp b/storage/FastHashTable.hpp
new file mode 100644
index 0000000..4a95cd9
--- /dev/null
+++ b/storage/FastHashTable.hpp
@@ -0,0 +1,2515 @@
+/**
+ * Copyright 2011-2015 Quickstep Technologies LLC.
+ * Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_FAST_HASH_TABLE_HPP_
+#define QUICKSTEP_STORAGE_FAST_HASH_TABLE_HPP_
+
+#include <atomic>
+#include <cstddef>
+#include <cstdlib>
+#include <type_traits>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "storage/HashTableBase.hpp"
+#include "storage/StorageBlob.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageConstants.hpp"
+#include "storage/StorageManager.hpp"
+#include "storage/TupleReference.hpp"
+#include "storage/ValueAccessor.hpp"
+#include "storage/ValueAccessorUtil.hpp"
+#include "threading/SpinMutex.hpp"
+#include "threading/SpinSharedMutex.hpp"
+#include "types/Type.hpp"
+#include "types/TypedValue.hpp"
+#include "utility/BloomFilter.hpp"
+#include "utility/HashPair.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+/** \addtogroup Storage
+ * @{
+ */
+
+/**
+ * @brief Base class for the hash table implementation in which the payload can
+ * be just a bunch of bytes. This implementation is suitable for
+ * aggregation hash table with multiple aggregation handles (e.g. SUM,
+ * MAX, MIN etc).
+ *
+ * At present there is one implementation for this base class.
+ * 1. SeparateChainingHashTable - Keys/values are stored in a separate
+ * region of memory from the base hash table slot array. Every bucket
+ * has a "next" pointer so that entries that collide (i.e. map to the
+ * same base slot) form chains of pointers with each other. Although
+ * this implementation has some extra indirection compared to
+ * LinearOpenAddressingHashTable, it does not have the same
+ * vulnerabilities to key skew, and it additionally supports a very
+ * efficient bucket-preallocation mechanism that minimizes cache
+ * coherency overhead when multiple threads are building a HashTable.
+ *
+ * @note If you need to create a HashTable and not just use it as a client, see
+ * HashTableFactory, which simplifies the process of creating a
+ * HashTable.
+ *
+ * @param resizable Whether this hash table is resizable (using memory from a
+ * StorageManager) or not (using a private, fixed memory allocation).
+ * @param serializable If true, this hash table can safely be saved to and
+ * loaded from disk. If false, some out of band memory may be used (e.g.
+ * to store variable length keys).
+ * @param force_key_copy If true, inserted keys are always copied into this
+ * HashTable's memory. If false, pointers to external values may be
+ * stored instead. force_key_copy should be true if the hash table will
+ * outlive the external key values which are inserted into it. Note that
+ * if serializable is true and force_key_copy is false, then relative
+ * offsets will be used instead of absolute pointers to keys, meaning
+ * that the pointed-to keys must be serialized and deserialized in
+ * exactly the same relative byte order (e.g. as part of the same
+ * StorageBlock), and keys must not change position relative to this
+ * HashTable (beware TupleStorageSubBlocks that may self-reorganize when
+ * modified). If serializable and resizable are both true, then
+ * force_key_copy must also be true.
+ * @param allow_duplicate_keys If true, multiple values can be mapped to the
+ * same key. If false, one and only one value may be mapped.
+ **/
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+class FastHashTable : public HashTableBase<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys> {
+ static_assert(!(serializable && resizable && !force_key_copy),
+ "A HashTable must have force_key_copy=true when serializable "
+ "and resizable are both true.");
+
+ public:
+ // Shadow template parameters. This is useful for shared test harnesses.
+ static constexpr bool template_resizable = resizable;
+ static constexpr bool template_serializable = serializable;
+ static constexpr bool template_force_key_copy = force_key_copy;
+ static constexpr bool template_allow_duplicate_keys = allow_duplicate_keys;
+
+ // Some HashTable implementations (notably LinearOpenAddressingHashTable)
+ // use a special hash code to represent an empty bucket, and another special
+ // code to indicate that a bucket is currently being inserted into. For those
+ // HashTables, this is a surrogate hash value for empty buckets. Keys which
+ // actually hash to this value should have their hashes mutated (e.g. by
+ // adding 1). We use zero, since we will often be using memory which is
+ // already zeroed-out and this saves us the trouble of a memset. This has
+ // some downside, as the hash function we use is the identity hash for
+ // integers, and the integer 0 is common in many data sets and must be
+ // adjusted (and will then spuriously collide with 1). Nevertheless, this
+ // expense is outweighed by no longer having to memset large regions of
+ // memory when initializing a HashTable.
+ static constexpr unsigned char kEmptyHashByte = 0x0;
+ static constexpr std::size_t kEmptyHash = 0x0;
+
+ // A surrogate hash value for a bucket which is currently being inserted
+ // into. As with kEmptyHash, keys which actually hash to this value should
+ // have their hashes adjusted.
+ static constexpr std::size_t kPendingHash = ~kEmptyHash;
+
+ /**
+ * @brief Virtual destructor.
+ **/
+ virtual ~FastHashTable() {
+ if (resizable) {
+ if (blob_.valid()) {
+ if (serializable) {
+ DEV_WARNING(
+ "Destroying a resizable serializable HashTable's underlying "
+ "StorageBlob.");
+ }
+ const block_id blob_id = blob_->getID();
+ blob_.release();
+ storage_manager_->deleteBlockOrBlobFile(blob_id);
+ }
+ }
+ }
+
+ /**
+ * @brief Get the ID of the StorageBlob used to store a resizable HashTable.
+ *
+ * @warning This method must not be used for a non-resizable HashTable.
+ *
+ * @return The ID of the StorageBlob used to store this HashTable.
+ **/
+ inline block_id getBlobId() const {
+ DEBUG_ASSERT(resizable);
+ return blob_->getID();
+ }
+
+ /**
+ * @brief Erase all entries in this hash table.
+ *
+ * @warning This method is not guaranteed to be threadsafe.
+ **/
+ virtual void clear() = 0;
+
+ /**
+ * @brief Add a new entry into the hash table.
+ *
+ * @warning The key must not be null.
+ * @warning This method is threadsafe with regard to other calls to put(),
+ * putCompositeKey(), putValueAccessor(), and
+ * putValueAccessorCompositeKey(), but should not be used
+ * simultaneously with upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey().
+ * @note This version is for single scalar keys, see also putCompositeKey().
+ * @note If the hash table is (close to) full and resizable is true, this
+ * routine might result in rebuilding the entire hash table.
+ *
+ * @param key The key.
+ * @param value The value payload.
+ * @return HashTablePutResult::kOK if an entry was successfully inserted,
+ * HashTablePutResult::kDuplicateKey if allow_duplicate_keys is false
+ * and key was a duplicate, or HashTablePutResult::kOutOfSpace if
+ * resizable is false and storage space for the hash table has been
+ * exhausted.
+ **/
+ HashTablePutResult put(const TypedValue &key, const std::uint8_t &value);
+
+ /**
+ * @brief Add a new entry into the hash table (composite key version).
+ *
+ * @warning No component of the key may be null.
+ * @warning This method is threadsafe with regard to other calls to put(),
+ * putCompositeKey(), putValueAccessor(), and
+ * putValueAccessorCompositeKey(), but should not be used
+ * simultaneously with upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey().
+ * @note This version is for composite keys, see also put().
+ * @note If the hash table is (close to) full and resizable is true, this
+ * routine might result in rebuilding the entire hash table.
+ *
+ * @param key The components of the key.
+ * @param value The value payload.
+ * @return HashTablePutResult::kOK if an entry was successfully inserted,
+ * HashTablePutResult::kDuplicateKey if allow_duplicate_keys is false
+ * and key was a duplicate, or HashTablePutResult::kOutOfSpace if
+ * resizable is false and storage space for the hash table has been
+ * exhausted.
+ **/
+
+ HashTablePutResult putCompositeKey(const std::vector<TypedValue> &key,
+ const std::uint8_t *value_ptr);
+
+ /**
+ * @brief Add (multiple) new entries into the hash table from a
+ * ValueAccessor.
+ *
+ * @warning This method is threadsafe with regard to other calls to put(),
+ * putCompositeKey(), putValueAccessor(), and
+ * putValueAccessorCompositeKey(), but should not be used
+ * simultaneously with upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey().
+ * @note This version is for scalar keys, see also
+ * putValueAccessorCompositeKey().
+ * @note If the hash table fills up while this call is in progress and
+ * resizable is true, this might result in rebuilding the entire hash
+ * table.
+ *
+ * @param accessor A ValueAccessor which will be used to access keys.
+ * beginIteration() should be called on accessor before calling this
+ * method.
+ * @param key_attr_id The attribute ID of the keys to be read from accessor.
+ * @param check_for_null_keys If true, each key will be checked to see if it
+ * is null before inserting it (null keys are skipped). This must be
+ * set to true if some of the keys that will be read from accessor may
+ * be null.
+ * @param functor A pointer to a functor, which should provide a call
+ * operator that takes const ValueAccessor& as an argument (or better
+ * yet, a templated call operator which takes a const reference to
+ * some subclass of ValueAccessor as an argument) and returns either
+ * a ValueT or a reference to a ValueT. The functor should generate
+ * the appropriate mapped value for the current tuple the accessor is
+ * iterating on.
+ * @return HashTablePutResult::kOK if all keys and generated values from
+ * accessor were successfully inserted.
+ * HashTablePutResult::kOutOfSpace is returned if this hash-table is
+ * non-resizable and ran out of space (note that some entries may
+ * still have been inserted, and accessor's iteration will be left on
+ * the first tuple which could not be inserted).
+ * HashTablePutResult::kDuplicateKey is returned if
+ * allow_duplicate_keys is false and a duplicate key is encountered
+ * (as with HashTablePutResult::kOutOfSpace, some entries may have
+ * been inserted, and accessor will be left on the tuple with a
+ * duplicate key).
+ **/
+ template <typename FunctorT>
+ HashTablePutResult putValueAccessor(ValueAccessor *accessor,
+ const attribute_id key_attr_id,
+ const bool check_for_null_keys,
+ FunctorT *functor);
+
+ /**
+ * @brief Add (multiple) new entries into the hash table from a
+ * ValueAccessor (composite key version).
+ *
+ * @warning This method is threadsafe with regard to other calls to put(),
+ * putCompositeKey(), putValueAccessor(), and
+ * putValueAccessorCompositeKey(), but should not be used
+ * simultaneously with upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey().
+ * @note This version is for composite keys, see also putValueAccessor().
+ * @note If the hash table fills up while this call is in progress and
+ * resizable is true, this might result in rebuilding the entire hash
+ * table.
+ *
+ * @param accessor A ValueAccessor which will be used to access keys.
+ * beginIteration() should be called on accessor before calling this
+ * method.
+ * @param key_attr_ids The attribute IDs of each key component to be read
+ * from accessor.
+ * @param check_for_null_keys If true, each key will be checked to see if it
+ * has a null component before inserting it (null keys are skipped).
+ * This must be set to true if some of the keys that will be read from
+ * accessor may be null.
+ * @param functor A pointer to a functor, which should provide a call
+ * operator that takes const ValueAccessor& as an argument (or better
+ * yet, a templated call operator which takes a const reference to
+ * some subclass of ValueAccessor as an argument) and returns either
+ * a ValueT or a reference to a ValueT. The functor should generate
+ * the appropriate mapped value for the current tuple the accessor is
+ * iterating on.
+ * @return HashTablePutResult::kOK if all keys and generated values from
+ * accessor were successfully inserted.
+ * HashTablePutResult::kOutOfSpace is returned if this hash-table is
+ * non-resizable and ran out of space (note that some entries may
+ * still have been inserted, and accessor's iteration will be left on
+ * the first tuple which could not be inserted).
+ * HashTablePutResult::kDuplicateKey is returned if
+ * allow_duplicate_keys is false and a duplicate key is encountered
+ * (as with HashTablePutResult::kOutOfSpace, some entries may have
+ * been inserted, and accessor will be left on the tuple with a
+ * duplicate key).
+ **/
+ template <typename FunctorT>
+ HashTablePutResult putValueAccessorCompositeKey(
+ ValueAccessor *accessor,
+ const std::vector<attribute_id> &key_attr_ids,
+ const bool check_for_null_keys,
+ FunctorT *functor);
+
+ /**
+ * @brief Apply a functor to the value mapped to a key, first inserting a new
+ * value if one is not already present.
+ *
+ * @warning The key must not be null.
+ * @warning This method is only usable if allow_duplicate_keys is false.
+ * @warning This method is threadsafe with regard to other calls to upsert(),
+ * upsertCompositeKey(), upsertValueAccessor(), and
+ * upsertValueAccessorCompositeKey(), but should not be used
+ * simultaneously with put(), putCompositeKey(), putValueAccessor(),
+ * or putValueAccessorCompositeKey().
+ * @warning The ValueT* pointer passed to functor's call operator is only
+ * guaranteed to be valid for the duration of the call. The functor
+ * should not store a copy of the pointer and assume that it remains
+ * valid.
+ * @warning Although this method itself is threadsafe, the ValueT object
+ * accessed by functor is not guaranteed to be (although it is
+ * guaranteed that its initial insertion will be atomic). If it is
+ * possible for multiple threads to call upsert() with the same key
+ * at the same time, then their access to ValueT should be made
+ * threadsafe (e.g. with the use of atomic types, mutexes, or some
+ * other external synchronization).
+ * @note This version is for single scalar keys, see also
+ * upsertCompositeKey().
+ * @note If the hash table is (close to) full and resizable is true, this
+ * routine might result in rebuilding the entire hash table.
+ *
+ * @param key The key.
+ * @param initial_value If there was not already a preexisting entry in this
+ * HashTable for the specified key, then the value will be initialized
+ * with a copy of initial_value. This parameter is ignored if a value
+ * is already present for key.
+ * @param functor A pointer to a functor, which should provide a call
+ * operator which takes ValueT* as an argument. The call operator will
+ * be invoked once on the value corresponding to key (which may be
+ * newly inserted and default-constructed).
+ * @return True on success, false if upsert failed because there was not
+ * enough space to insert a new entry in this HashTable.
+ **/
+ template <typename FunctorT>
+ bool upsert(const TypedValue &key,
+ const std::uint8_t *initial_value_ptr,
+ FunctorT *functor);
+
+ /**
+ * @brief Apply a functor to the value mapped to a key, first inserting a new
+ * value if one is not already present.
+ *
+ * @warning The key must not be null.
+ * @warning This method is only usable if allow_duplicate_keys is false.
+ * @warning This method is threadsafe with regard to other calls to upsert(),
+ * upsertCompositeKey(), upsertValueAccessor(), and
+ * upsertValueAccessorCompositeKey(), but should not be used
+ * simultaneously with put(), putCompositeKey(), putValueAccessor(),
+ * or putValueAccessorCompositeKey().
+ * @warning The ValueT* pointer passed to functor's call operator is only
+ * guaranteed to be valid for the duration of the call. The functor
+ * should not store a copy of the pointer and assume that it remains
+ * valid.
+ * @warning Although this method itself is threadsafe, the ValueT object
+ * accessed by functor is not guaranteed to be (although it is
+ * guaranteed that its initial insertion will be atomic). If it is
+ * possible for multiple threads to call upsertCompositeKey() with
+ * the same key at the same time, then their access to ValueT should
+ * be made threadsafe (e.g. with the use of atomic types, mutexes,
+ * or some other external synchronization).
+ * @note This version is for composite keys, see also upsert().
+ * @note If the hash table is (close to) full and resizable is true, this
+ * routine might result in rebuilding the entire hash table.
+ *
+ * @param key The key.
+ * @param initial_value If there was not already a preexisting entry in this
+ * HashTable for the specified key, then the value will be initialized
+ * with a copy of initial_value. This parameter is ignored if a value
+ * is already present for key.
+ * @param functor A pointer to a functor, which should provide a call
+ * operator which takes ValueT* as an argument. The call operator will
+ * be invoked once on the value corresponding to key (which may be
+ * newly inserted and default-constructed).
+ * @return True on success, false if upsert failed because there was not
+ * enough space to insert a new entry in this HashTable.
+ **/
+ template <typename FunctorT>
+ bool upsertCompositeKeyFast(const std::vector<TypedValue> &key,
+ const std::uint8_t *init_value_ptr,
+ FunctorT *functor);
+
+ template <typename FunctorT>
+ bool upsertCompositeKeyFast(const std::vector<TypedValue> &key,
+ const std::uint8_t *init_value_ptr,
+ FunctorT *functor,
+ int index);
+
+ bool upsertCompositeKeyFast(const std::vector<TypedValue> &key,
+ const std::uint8_t *init_value_ptr,
+ const std::uint8_t *source_state);
+
+ /**
+ * @brief Apply a functor to (multiple) entries in this hash table, with keys
+ * drawn from a ValueAccessor. New values are first inserted if not
+ * already present.
+ *
+ * @warning This method is only usable if allow_duplicate_keys is false.
+ * @warning This method is threadsafe with regard to other calls to upsert(),
+ * upsertCompositeKey(), upsertValueAccessor(), and
+ * upsertValueAccessorCompositeKey(), but should not be used
+ * simultaneously with put(), putCompositeKey(), putValueAccessor(),
+ * or putValueAccessorCompositeKey().
+ * @warning The ValueAccessor reference and ValueT* pointer passed to
+ * functor's call operator are only guaranteed to be valid for the
+ * duration of the call. The functor should not store a copy of
+ * these pointers and assume that they remain valid.
+ * @warning Although this method itself is threadsafe, the ValueT object
+ * accessed by functor is not guaranteed to be (although it is
+ * guaranteed that its initial insertion will be atomic). If it is
+ * possible for multiple threads to call upsertValueAccessor() with
+ * the same key at the same time, then their access to ValueT should
+ * be made threadsafe (e.g. with the use of atomic types, mutexes,
+ * or some other external synchronization).
+ * @note This version is for single scalar keys, see also
+ * upsertValueAccessorCompositeKey().
+ * @note If the hash table is (close to) full and resizable is true, this
+ * routine might result in rebuilding the entire hash table.
+ *
+ * @param accessor A ValueAccessor which will be used to access keys.
+ * beginIteration() should be called on accessor before calling this
+ * method.
+ * @param key_attr_id The attribute ID of the keys to be read from accessor.
+ * @param check_for_null_keys If true, each key will be checked to see if it
+ * is null before upserting it (null keys are skipped). This must be
+ * set to true if some of the keys that will be read from accessor may
+ * be null.
+ * @param functor A pointer to a functor, which should provide a call
+ * operator that takes two arguments: const ValueAccessor& (or better
+ * yet, a templated call operator which takes a const reference to
+ * some subclass of ValueAccessor as its first argument) and ValueT*.
+ * The call operator will be invoked once for every tuple with a
+ * non-null key in accessor.
+ * @return True on success, false if upsert failed because there was not
+ * enough space to insert new entries for all the keys in accessor
+ * (note that some entries may still have been upserted, and
+ * accessor's iteration will be left on the first tuple which could
+ * not be inserted).
+ **/
+ bool upsertValueAccessorFast(
+ const std::vector<attribute_id> &argument_ids,
+ ValueAccessor *accessor,
+ const attribute_id key_attr_id,
+ const bool check_for_null_keys);
+
+ /**
+ * @brief Apply a functor to (multiple) entries in this hash table, with keys
+ * drawn from a ValueAccessor. New values are first inserted if not
+ * already present. Composite key version.
+ *
+ * @warning This method is only usable if allow_duplicate_keys is false.
+ * @warning This method is threadsafe with regard to other calls to upsert(),
+ * upsertCompositeKey(), upsertValueAccessor(), and
+ * upsertValueAccessorCompositeKey(), but should not be used
+ * simultaneously with put(), putCompositeKey(), putValueAccessor(),
+ * or putValueAccessorCompositeKey().
+ * @warning The ValueAccessor reference and ValueT* pointer passed to
+ * functor's call operator are only guaranteed to be valid for the
+ * duration of the call. The functor should not store a copy of
+ * these pointers and assume that they remain valid.
+ * @warning Although this method itself is threadsafe, the ValueT object
+ * accessed by functor is not guaranteed to be (although it is
+ * guaranteed that its initial insertion will be atomic). If it is
+ * possible for multiple threads to call upsertValueAccessor() with
+ * the same key at the same time, then their access to ValueT should
+ * be made threadsafe (e.g. with the use of atomic types, mutexes,
+ * or some other external synchronization).
+ * @note This version is for composite keys, see also upsertValueAccessor().
+ * @note If the hash table is (close to) full and resizable is true, this
+ * routine might result in rebuilding the entire hash table.
+ *
+ * @param accessor A ValueAccessor which will be used to access keys.
+ * beginIteration() should be called on accessor before calling this
+ * method.
+ * @param key_attr_ids The attribute IDs of each key component to be read
+ * from accessor.
+ * @param check_for_null_keys If true, each key will be checked to see if it
+ * is null before upserting it (null keys are skipped). This must be
+ * set to true if some of the keys that will be read from accessor may
+ * be null.
+ * @param functor A pointer to a functor, which should provide a call
+ * operator that takes two arguments: const ValueAccessor& (or better
+ * yet, a templated call operator which takes a const reference to
+ * some subclass of ValueAccessor as its first argument) and ValueT*.
+ * The call operator will be invoked once for every tuple with a
+ * non-null key in accessor.
+ * @return True on success, false if upsert failed because there was not
+ * enough space to insert new entries for all the keys in accessor
+ * (note that some entries may still have been upserted, and
+ * accessor's iteration will be left on the first tuple which could
+ * not be inserted).
+ **/
+ bool upsertValueAccessorCompositeKeyFast(
+ const std::vector<attribute_id> &argument,
+ ValueAccessor *accessor,
+ const std::vector<attribute_id> &key_attr_ids,
+ const bool check_for_null_keys) override;
+
+ /**
+ * @brief Determine the number of entries (key-value pairs) contained in this
+ * HashTable.
+ * @note For some HashTable implementations, this is O(1), but for others it
+ * may be O(n) where n is the number of buckets.
+ *
+ * @warning This method assumes that no concurrent calls to put(),
+ * putCompositeKey(), putValueAccessor(),
+ * putValueAccessorCompositeKey(), upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey() are
+ * taking place (i.e. that this HashTable is immutable for the
+ * duration of the call). Concurrent calls to getSingle(),
+ * getSingleCompositeKey(), getAll(), getAllCompositeKey(),
+ * getAllFromValueAccessor(), getAllFromValueAccessorCompositeKey(),
+ * forEach(), and forEachCompositeKey() are safe.
+ *
+ * @return The number of entries in this HashTable.
+ **/
+ virtual std::size_t numEntries() const = 0;
+
+ /**
+ * @brief Lookup a key against this hash table to find a matching entry.
+ *
+ * @warning Only usable with the hash table that does not allow duplicate
+ * keys.
+ * @warning The key must not be null.
+ * @warning This method assumes that no concurrent calls to put(),
+ * putCompositeKey(), putValueAccessor(),
+ * putValueAccessorCompositeKey(), upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey() are
+ * taking place (i.e. that this HashTable is immutable for the
+ * duration of the call and as long as the returned pointer may be
+ * dereferenced). Concurrent calls to getSingle(),
+ * getSingleCompositeKey(), getAll(), getAllCompositeKey(),
+ * getAllFromValueAccessor(), getAllFromValueAccessorCompositeKey(),
+ * forEach(), and forEachCompositeKey() are safe.
+ * @note This version is for single scalar keys. See also
+ * getSingleCompositeKey().
+ *
+ * @param key The key to look up.
+ * @return The value of a matched entry if a matching key is found.
+ * Otherwise, return NULL.
+ **/
+ virtual const std::uint8_t* getSingle(const TypedValue &key) const = 0;
+
+ /**
+ * @brief Lookup a composite key against this hash table to find a matching
+ * entry.
+ *
+ * @warning Only usable with the hash table that does not allow duplicate
+ * keys.
+ * @warning The key must not be null.
+ * @warning This method assumes that no concurrent calls to put(),
+ * putCompositeKey(), putValueAccessor(),
+ * putValueAccessorCompositeKey(), upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey() are
+ * taking place (i.e. that this HashTable is immutable for the
+ * duration of the call and as long as the returned pointer may be
+ * dereferenced). Concurrent calls to getSingle(),
+ * getSingleCompositeKey(), getAll(), getAllCompositeKey(),
+ * getAllFromValueAccessor(), getAllFromValueAccessorCompositeKey(),
+ * forEach(), and forEachCompositeKey() are safe.
+ * @note This version is for composite keys. See also getSingle().
+ *
+ * @param key The key to look up.
+ * @return The value of a matched entry if a matching key is found.
+ * Otherwise, return NULL.
+ **/
+ virtual const std::uint8_t* getSingleCompositeKey(
+ const std::vector<TypedValue> &key) const = 0;
+ virtual const std::uint8_t *getSingleCompositeKey(
+ const std::vector<TypedValue> &key, int index) const = 0;
+
+ /**
+ * @brief Lookup a key against this hash table to find matching entries.
+ *
+ * @warning The key must not be null.
+ * @warning This method assumes that no concurrent calls to put(),
+ * putCompositeKey(), putValueAccessor(),
+ * putValueAccessorCompositeKey(), upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey() are
+ * taking place (i.e. that this HashTable is immutable for the
+ * duration of the call and as long as the returned pointer may be
+ * dereferenced). Concurrent calls to getSingle(),
+ * getSingleCompositeKey(), getAll(), getAllCompositeKey(),
+ * getAllFromValueAccessor(), getAllFromValueAccessorCompositeKey(),
+ * forEach(), and forEachCompositeKey() are safe.
+ * @note It is more efficient to call getSingle() if the hash table does not
+ * allow duplicate keys.
+ * @note This version is for single scalar keys. See also
+ * getAllCompositeKey().
+ *
+ * @param key The key to look up.
+ * @param values A vector to hold values of all matching entries. Matches
+ * will be appended to the vector.
+ **/
+ virtual void getAll(const TypedValue &key,
+ std::vector<const std::uint8_t *> *values) const = 0;
+
+ /**
+ * @brief Lookup a composite key against this hash table to find matching
+ * entries.
+ *
+ * @warning The key must not be null.
+ * @warning This method assumes that no concurrent calls to put(),
+ * putCompositeKey(), putValueAccessor(),
+ * putValueAccessorCompositeKey(), upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey() are
+ * taking place (i.e. that this HashTable is immutable for the
+ * duration of the call and as long as the returned pointer may be
+ * dereferenced). Concurrent calls to getSingle(),
+ * getSingleCompositeKey(), getAll(), getAllCompositeKey(),
+ * getAllFromValueAccessor(), getAllFromValueAccessorCompositeKey(),
+ * forEach(), and forEachCompositeKey() are safe.
+ * @note It is more efficient to call getSingleCompositeKey() if the hash
+ * table does not allow duplicate keys.
+ * @note This version is for composite keys. See also getAll().
+ *
+ * @param key The key to look up.
+ * @param values A vector to hold values of all matching entries. Matches
+ * will be appended to the vector.
+ **/
+ virtual void getAllCompositeKey(
+ const std::vector<TypedValue> &key,
+ std::vector<const std::uint8_t *> *values) const = 0;
+
+ /**
+ * @brief Lookup (multiple) keys from a ValueAccessor and apply a functor to
+ * the matching values.
+ *
+ * @warning This method assumes that no concurrent calls to put(),
+ * putCompositeKey(), putValueAccessor(),
+ * putValueAccessorCompositeKey(), upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey() are
+ * taking place (i.e. that this HashTable is immutable for the
+ * duration of the call and as long as the returned pointer may be
+ * dereferenced). Concurrent calls to getSingle(),
+ * getSingleCompositeKey(), getAll(), getAllCompositeKey(),
+ * getAllFromValueAccessor(), getAllFromValueAccessorCompositeKey(),
+ * forEach(), and forEachCompositeKey() are safe.
+ * @note This version is for single scalar keys. See also
+ * getAllFromValueAccessorCompositeKey().
+ *
+ * @param accessor A ValueAccessor which will be used to access keys.
+ * beginIteration() should be called on accessor before calling this
+ * method.
+ * @param key_attr_id The attribute ID of the keys to be read from accessor.
+ * @param check_for_null_keys If true, each key will be checked to see if it
+ * is null before looking it up (null keys are skipped). This must be
+ * set to true if some of the keys that will be read from accessor may
+ * be null.
+ * @param functor A pointer to a functor, which should provide a call
+ * operator that takes 2 arguments: const ValueAccessor& (or better
+ * yet, a templated call operator which takes a const reference to
+ * some subclass of ValueAccessor as its first argument) and
+ * const ValueT&. The functor will be invoked once for each pair of a
+ * key taken from accessor and matching value.
+ **/
+ template <typename FunctorT>
+ void getAllFromValueAccessor(ValueAccessor *accessor,
+ const attribute_id key_attr_id,
+ const bool check_for_null_keys,
+ FunctorT *functor) const;
+
+ /**
+ * @brief Lookup (multiple) keys from a ValueAccessor, apply a functor to the
+ * matching values and additionally call a recordMatch() function of
+ * the functor when the first match for a key is found.
+ * @warning This method assumes that no concurrent calls to put(),
+ * putCompositeKey(), putValueAccessor(),
+ * putValueAccessorCompositeKey(), upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey() are
+ * taking place (i.e. that this HashTable is immutable for the
+ * duration of the call and as long as the returned pointer may be
+ * dereferenced). Concurrent calls to getSingle(),
+ * getSingleCompositeKey(), getAll(), getAllCompositeKey(),
+ * getAllFromValueAccessor(), getAllFromValueAccessorCompositeKey(),
+ * forEach(), and forEachCompositeKey() are safe.
+ * @note This version is for single scalar keys. See also
+ * getAllFromValueAccessorCompositeKeyWithExtraWorkForFirstMatch().
+ *
+ * @param accessor A ValueAccessor which will be used to access keys.
+ * beginIteration() should be called on accessor before calling this
+ * method.
+ * @param key_attr_id The attribute ID of the keys to be read from accessor.
+ * @param check_for_null_keys If true, each key will be checked to see if it
+ * is null before looking it up (null keys are skipped). This must be
+ * set to true if some of the keys that will be read from accessor may
+ * be null.
+ * @param functor A pointer to a functor, which should provide two functions:
+ * 1) An operator that takes 2 arguments: const ValueAccessor& (or
+ * better
+ * yet, a templated call operator which takes a const reference to
+ * some subclass of ValueAccessor as its first argument) and
+ * const ValueT&. The operator will be invoked once for each pair of a
+ * key taken from accessor and matching value.
+ * 2) A function hasMatch that takes 1 argument: const ValueAccessor&.
+ * The function will be called only once for a key from accessor when
+ * the first match is found.
+ */
+ template <typename FunctorT>
+ void getAllFromValueAccessorWithExtraWorkForFirstMatch(
+ ValueAccessor *accessor,
+ const attribute_id key_attr_id,
+ const bool check_for_null_keys,
+ FunctorT *functor) const;
+
+ /**
+ * @brief Lookup (multiple) keys from a ValueAccessor, apply a functor to the
+ * matching values and additionally call a recordMatch() function of
+ * the functor when the first match for a key is found. Composite key
+ * version.
+ * @warning This method assumes that no concurrent calls to put(),
+ * putCompositeKey(), putValueAccessor(),
+ * putValueAccessorCompositeKey(), upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey() are
+ * taking place (i.e. that this HashTable is immutable for the
+ * duration of the call and as long as the returned pointer may be
+ * dereferenced). Concurrent calls to getSingle(),
+ * getSingleCompositeKey(), getAll(), getAllCompositeKey(),
+ * getAllFromValueAccessor(), getAllFromValueAccessorCompositeKey(),
+ * forEach(), and forEachCompositeKey() are safe.
+ *
+ * @param accessor A ValueAccessor which will be used to access keys.
+ * beginIteration() should be called on accessor before calling this
+ * method.
+ * @param key_attr_id The attribute ID of the keys to be read from accessor.
+ * @param check_for_null_keys If true, each key will be checked to see if it
+ * is null before looking it up (null keys are skipped). This must be
+ * set to true if some of the keys that will be read from accessor may
+ * be null.
+ * @param functor A pointer to a functor, which should provide two functions:
+ * 1) An operator that takes 2 arguments: const ValueAccessor& (or
+ * better
+ * yet, a templated call operator which takes a const reference to
+ * some subclass of ValueAccessor as its first argument) and
+ * const ValueT&. The operator will be invoked once for each pair of a
+ * key taken from accessor and matching value.
+ * 2) A function hasMatch that takes 1 argument: const ValueAccessor&.
+ * The function will be called only once for a key from accessor when
+ * the first match is found.
+ */
+ template <typename FunctorT>
+ void getAllFromValueAccessorCompositeKeyWithExtraWorkForFirstMatch(
+ ValueAccessor *accessor,
+ const std::vector<attribute_id> &key_attr_ids,
+ const bool check_for_null_keys,
+ FunctorT *functor) const;
+
+ /**
+ * @brief Lookup (multiple) keys from a ValueAccessor and apply a functor to
+ * the matching values. Composite key version.
+ *
+ * @warning This method assumes that no concurrent calls to put(),
+ * putCompositeKey(), putValueAccessor(),
+ * putValueAccessorCompositeKey(), upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey() are
+ * taking place (i.e. that this HashTable is immutable for the
+ * duration of the call and as long as the returned pointer may be
+ * dereferenced). Concurrent calls to getSingle(),
+ * getSingleCompositeKey(), getAll(), getAllCompositeKey(),
+ * getAllFromValueAccessor(), getAllFromValueAccessorCompositeKey(),
+ * forEach(), and forEachCompositeKey() are safe.
+ * @note This version is for composite keys. See also
+ * getAllFromValueAccessor().
+ *
+ * @param accessor A ValueAccessor which will be used to access keys.
+ * beginIteration() should be called on accessor before calling this
+ * method.
+ * @param key_attr_ids The attribute IDs of each key component to be read
+ * from accessor.
+ * @param check_for_null_keys If true, each key will be checked to see if it
+ * has a null component before inserting it (null keys are skipped).
+ * This must be set to true if some of the keys that will be read from
+ * accessor may be null.
+ * @param functor A pointer to a functor, which should provide a call
+ * operator that takes 2 arguments: const ValueAccessor& (or better
+ * yet, a templated call operator which takes a const reference to
+ * some subclass of ValueAccessor as its first argument) and
+ * const ValueT&. The functor will be invoked once for each pair of a
+ * key taken from accessor and matching value.
+ **/
+ template <typename FunctorT>
+ void getAllFromValueAccessorCompositeKey(
+ ValueAccessor *accessor,
+ const std::vector<attribute_id> &key_attr_ids,
+ const bool check_for_null_keys,
+ FunctorT *functor) const;
+
+ /**
+ * @brief Apply the functor to each key with a match in the hash table.
+ *
+ * @param accessor A ValueAccessor which will be used to access keys.
+ * beginIteration() should be called on accessor before calling this
+ * method.
+ * @param key_attr_id The attribute ID of the keys to be read from accessor.
+ * @param check_for_null_keys If true, each key will be checked to see if it
+ * is null before looking it up (null keys are skipped). This must be
+ * set to true if some of the keys that will be read from accessor may
+ * be null.
+ * @param functor A pointer to a functor which should provide an operator that
+ * takes 1 argument: const ValueAccessor&. The operator will be called
+ * only once for a key from accessor if there is a match.
+ */
+ template <typename FunctorT>
+ void runOverKeysFromValueAccessorIfMatchFound(ValueAccessor *accessor,
+ const attribute_id key_attr_id,
+ const bool check_for_null_keys,
+ FunctorT *functor) const {
+ return runOverKeysFromValueAccessor<true>(
+ accessor, key_attr_id, check_for_null_keys, functor);
+ }
+
+ /**
+ * @brief Apply the functor to each key with a match in the hash table.
+ *
+ * @param accessor A ValueAccessor which will be used to access keys.
+ * beginIteration() should be called on accessor before calling this
+ * method.
+ * @param key_attr_id The attribute ID of the keys to be read from accessor.
+ * @param check_for_null_keys If true, each key will be checked to see if it
+ * is null before looking it up (null keys are skipped). This must be
+ * set to true if some of the keys that will be read from accessor may
+ * be null.
+ * @param functor A pointer to a functor which should provide an operator that
+ * takes 1 argument: const ValueAccessor&. The operator will be called
+ * only once for a key from accessor if there is a match.
+ */
+ template <typename FunctorT>
+ void runOverKeysFromValueAccessorIfMatchFoundCompositeKey(
+ ValueAccessor *accessor,
+ const std::vector<attribute_id> &key_attr_ids,
+ const bool check_for_null_keys,
+ FunctorT *functor) const {
+ return runOverKeysFromValueAccessorCompositeKey<true>(
+ accessor, key_attr_ids, check_for_null_keys, functor);
+ }
+
+ /**
+ * @brief Apply the functor to each key without a match in the hash table.
+ *
+ * @param accessor A ValueAccessor which will be used to access keys.
+ * beginIteration() should be called on accessor before calling this
+ * method.
+ * @param key_attr_id The attribute ID of the keys to be read from accessor.
+ * @param check_for_null_keys If true, each key will be checked to see if it
+ * is null before looking it up (null keys are skipped). This must be
+ * set to true if some of the keys that will be read from accessor may
+ * be null.
+ * @param functor A pointer to a functor which should provide an operator that
+ * takes 1 argument: const ValueAccessor&. The operator will be called
+ * only once for a key from accessor if there is no match.
+ */
+ template <typename FunctorT>
+ void runOverKeysFromValueAccessorIfMatchNotFound(
+ ValueAccessor *accessor,
+ const attribute_id key_attr_id,
+ const bool check_for_null_keys,
+ FunctorT *functor) const {
+ return runOverKeysFromValueAccessor<false>(
+ accessor, key_attr_id, check_for_null_keys, functor);
+ }
+
+ /**
+ * @brief Apply the functor to each key without a match in the hash table.
+ *
+ * @param accessor A ValueAccessor which will be used to access keys.
+ * beginIteration() should be called on accessor before calling this
+ * method.
+ * @param key_attr_id The attribute ID of the keys to be read from accessor.
+ * @param check_for_null_keys If true, each key will be checked to see if it
+ * is null before looking it up (null keys are skipped). This must be
+ * set to true if some of the keys that will be read from accessor may
+ * be null.
+ * @param functor A pointer to a functor which should provide an operator that
+ * takes 1 argument: const ValueAccessor&. The operator will be called
+ * only once for a key from accessor if there is no match.
+ */
+ template <typename FunctorT>
+ void runOverKeysFromValueAccessorIfMatchNotFoundCompositeKey(
+ ValueAccessor *accessor,
+ const std::vector<attribute_id> &key_attr_ids,
+ const bool check_for_null_keys,
+ FunctorT *functor) const {
+ return runOverKeysFromValueAccessorCompositeKey<false>(
+ accessor, key_attr_ids, check_for_null_keys, functor);
+ }
+
+ /**
+ * @brief Apply a functor to each key, value pair in this hash table.
+ *
+ * @warning This method assumes that no concurrent calls to put(),
+ * putCompositeKey(), putValueAccessor(),
+ * putValueAccessorCompositeKey(), upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey() are
+ * taking place (i.e. that this HashTable is immutable for the
+ * duration of the call and as long as the returned pointer may be
+ * dereferenced). Concurrent calls to getSingle(),
+ * getSingleCompositeKey(), getAll(), getAllCompositeKey(),
+ * getAllFromValueAccessor(), getAllFromValueAccessorCompositeKey(),
+ * forEach(), and forEachCompositeKey() are safe.
+ * @note This version is for single scalar keys. See also
+ * forEachCompositeKey().
+ *
+ * @param functor A pointer to a functor, which should provide a call
+ * operator which takes 2 arguments: const TypedValue&, const ValueT&.
+ * The call operator will be invoked once on each key, value pair in
+ * this hash table (note that if allow_duplicate_keys is true,
+ * the call may occur multiple times for the same key with different
+ * values).
+ * @return The number of key-value pairs visited.
+ **/
+ template <typename FunctorT>
+ std::size_t forEach(FunctorT *functor) const;
+
+ /**
+ * @brief Apply a functor to each key, value pair in this hash table.
+ *
+ * @warning This method assumes that no concurrent calls to put(),
+ * putCompositeKey(), putValueAccessor(),
+ * putValueAccessorCompositeKey(), upsert(), upsertCompositeKey(),
+ * upsertValueAccessor(), or upsertValueAccessorCompositeKey() are
+ * taking place (i.e. that this HashTable is immutable for the
+ * duration of the call and as long as the returned pointer may be
+ * dereferenced). Concurrent calls to getSingle(),
+ * getSingleCompositeKey(), getAll(), getAllCompositeKey(),
+ * getAllFromValueAccessor(), getAllFromValueAccessorCompositeKey(),
+ * forEach(), and forEachCompositeKey() are safe.
+ * @note This version is for composite keys. See also forEach().
+ *
+ * @param functor A pointer to a functor, which should provide a call
+ * operator which takes 2 arguments: const std::vector<TypedValue>&,
+ * const ValueT&. The call operator will be invoked once on each key,
+ * value pair in this hash table (note that if allow_duplicate_keys is
+ * true, the call may occur multiple times for the same key with
+ * different values).
+ * @return The number of key-value pairs visited.
+ **/
+ template <typename FunctorT>
+ std::size_t forEachCompositeKeyFast(FunctorT *functor) const;
+
+ template <typename FunctorT>
+ std::size_t forEachCompositeKeyFast(FunctorT *functor, int index) const;
+
+ /**
+ * @brief A call to this function will cause a bloom filter to be built
+ * during the build phase of this hash table.
+ **/
+ inline void enableBuildSideBloomFilter() {
+ has_build_side_bloom_filter_ = true;
+ }
+
+ /**
+ * @brief A call to this function will cause a set of bloom filters to be
+ * probed during the probe phase of this hash table.
+ **/
+ inline void enableProbeSideBloomFilter() {
+ has_probe_side_bloom_filter_ = true;
+ }
+
+ /**
+ * @brief This function sets the pointer to the bloom filter to be
+ * used during the build phase of this hash table.
+ * @warning Should call enable_build_side_bloom_filter() first to enable
+ * bloom filter usage during build phase.
+ * @note The ownership of the bloom filter lies with the caller.
+ *
+ * @param bloom_filter The pointer to the bloom filter.
+ **/
+ inline void setBuildSideBloomFilter(BloomFilter *bloom_filter) {
+ build_bloom_filter_ = bloom_filter;
+ }
+
+ /**
+ * @brief This function adds a pointer to the list of bloom filters to be
+ * used during the probe phase of this hash table.
+ * @warning Should call enable_probe_side_bloom_filter() first to enable
+ * bloom filter usage during probe phase.
+ * @note The ownership of the bloom filter lies with the caller.
+ *
+ * @param bloom_filter The pointer to the bloom filter.
+ **/
+ inline void addProbeSideBloomFilter(const BloomFilter *bloom_filter) {
+ probe_bloom_filters_.emplace_back(bloom_filter);
+ }
+
+ /**
+ * @brief This function adds a vector of attribute ids corresponding to a
+ * bloom filter used during the probe phase of this hash table.
+ * @warning Should call enable_probe_side_bloom_filter() first to enable
+ * bloom filter usage during probe phase.
+ *
+ * @param probe_attribute_ids The vector of attribute ids to use for probing
+ * the bloom filter.
+ **/
+ inline void addProbeSideAttributeIds(
+ std::vector<attribute_id> &&probe_attribute_ids) {
+ probe_attribute_ids_.push_back(probe_attribute_ids);
+ }
+
+ protected:
+ /**
+ * @brief Constructor for new resizable hash table.
+ *
+ * @param key_types A vector of one or more types (>1 indicates a composite
+ * key).
+ * @param num_entries The estimated number of entries this hash table will
+ * hold.
+ * @param storage_manager The StorageManager to use (a StorageBlob will be
+ * allocated to hold this hash table's contents).
+ * @param adjust_hashes If true, the hash of a key should be modified by
+ * applying AdjustHash() so that it does not collide with one of the
+ * special values kEmptyHash or kPendingHash. If false, the hash is
+ * used as-is.
+ * @param use_scalar_literal_hash If true, the key is a single scalar literal
+ * (non-composite) that it is safe to use the simplified hash function
+ * TypedValue::getHashScalarLiteral() on. If false, the generic
+ * TypedValue::getHash() method will be used.
+ * @param preallocate_supported If true, this HashTable overrides
+ * preallocateForBulkInsert() to allow bulk-allocation of resources
+ * (i.e. buckets and variable-length key storage) in a single up-front
+ * pass when bulk-inserting entries. If false, resources are allocated
+ * on the fly for each entry.
+ **/
+ FastHashTable(const std::vector<const Type *> &key_types,
+ const std::size_t num_entries,
+ const std::vector<AggregationHandle *> &handles,
+ const std::vector<std::size_t> &payload_sizes,
+ StorageManager *storage_manager,
+ const bool adjust_hashes,
+ const bool use_scalar_literal_hash,
+ const bool preallocate_supported)
+ : key_types_(key_types),
+ scalar_key_inline_(true),
+ key_inline_(nullptr),
+ adjust_hashes_(adjust_hashes),
+ use_scalar_literal_hash_(use_scalar_literal_hash),
+ preallocate_supported_(preallocate_supported),
+ handles_(handles),
+ num_handles_(handles.size()),
+ total_payload_size_(std::accumulate(
+ payload_sizes.begin(), payload_sizes.end(), sizeof(SpinMutex))),
+ storage_manager_(storage_manager),
+ hash_table_memory_(nullptr),
+ hash_table_memory_size_(0) {
+ DEBUG_ASSERT(resizable);
+ std::size_t running_sum = sizeof(SpinMutex);
+ for (auto size : payload_sizes) {
+ payload_offsets_.emplace_back(running_sum);
+ running_sum += size;
+ }
+ }
+
+ /**
+ * @brief Constructor for non-resizable hash table.
+ *
+ * @param key_types A vector of one or more types (>1 indicates a composite
+ * key).
+ * @param hash_table_memory A pointer to memory to use for this hash table.
+ * @param hash_table_memory_size The size of hash_table_memory in bytes.
+ * @param new_hash_table If true, this hash table is being constructed for
+ * the first time and hash_table_memory will be cleared. If false,
+ * reload a pre-existing hash table.
+ * @param hash_table_memory_zeroed If new_hash_table is true, setting this to
+ * true means that this HashTable will assume that hash_table_memory
+ * has already been zeroed-out (any newly-allocated block or blob
+ * memory from StorageManager is zeroed-out). If false, this HashTable
+ * will explicitly zero-fill its memory as neccessary. This parameter
+ * has no effect when new_hash_table is false.
+ * @param adjust_hashes If true, the hash of a key should be modified by
+ * applying AdjustHash() so that it does not collide with one of the
+ * special values kEmptyHash or kPendingHash. If false, the hash is
+ * used as-is.
+ * @param use_scalar_literal_hash If true, the key is a single scalar literal
+ * (non-composite) that it is safe to use the simplified hash function
+ * TypedValue::getHashScalarLiteral() on. If false, the generic
+ * TypedValue::getHash() method will be used.
+ * @param preallocate_supported If true, this HashTable overrides
+ * preallocateForBulkInsert() to allow bulk-allocation of resources
+ * (i.e. buckets and variable-length key storage) in a single up-front
+ * pass when bulk-inserting entries. If false, resources are allocated
+ * on the fly for each entry.
+ **/
+ FastHashTable(const std::vector<const Type *> &key_types,
+ void *hash_table_memory,
+ const std::size_t hash_table_memory_size,
+ const bool new_hash_table,
+ const bool hash_table_memory_zeroed,
+ const bool adjust_hashes,
+ const bool use_scalar_literal_hash,
+ const bool preallocate_supported)
+ : key_types_(key_types),
+ scalar_key_inline_(true),
+ key_inline_(nullptr),
+ adjust_hashes_(adjust_hashes),
+ use_scalar_literal_hash_(use_scalar_literal_hash),
+ preallocate_supported_(preallocate_supported),
+ storage_manager_(nullptr),
+ hash_table_memory_(hash_table_memory),
+ hash_table_memory_size_(hash_table_memory_size) {
+ DEBUG_ASSERT(!resizable);
+ }
+
+ // Adjust 'hash' so that it is not exactly equal to either of the special
+ // values kEmptyHash or kPendingHash.
+ inline constexpr static std::size_t AdjustHash(const std::size_t hash) {
+ return hash + (hash == kEmptyHash) - (hash == kPendingHash);
+ }
+
+ // Set information about which key components are stored inline. This usually
+ // comes from a HashTableKeyManager, and is set by the constructor of a
+ // subclass of HashTable.
+ inline void setKeyInline(const std::vector<bool> *key_inline) {
+ scalar_key_inline_ = key_inline->front();
+ key_inline_ = key_inline;
+ }
+
+ // Generate a hash for a composite key by hashing each component of 'key' and
+ // mixing their bits with CombineHashes().
+ inline std::size_t hashCompositeKey(const std::vector<TypedValue> &key) const;
+
+ // If 'force_key_copy' is true and some part of a composite key is
+ // variable-length, calculate the total number of bytes for variable-length
+ // key components that need to be copied. Otherwise, return 0 to indicate
+ // that no variable-length copy is required.
+ inline std::size_t calculateVariableLengthCompositeKeyCopySize(
+ const std::vector<TypedValue> &key) const;
+
+ // Helpers for put. If this HashTable is resizable, 'resize_shared_mutex_'
+ // should be locked in shared mode before calling either of these methods.
+ virtual HashTablePutResult putInternal(
+ const TypedValue &key,
+ const std::size_t variable_key_size,
+ const std::uint8_t &value,
+ HashTablePreallocationState *prealloc_state) = 0;
+
+ virtual HashTablePutResult putCompositeKeyInternalFast(
+ const std::vector<TypedValue> &key,
+ const std::size_t variable_key_size,
+ const std::uint8_t *init_value_ptr,
+ HashTablePreallocationState *prealloc_state) = 0;
+
+ // Helpers for upsert. Both return a pointer to the value corresponding to
+ // 'key'. If this HashTable is resizable, 'resize_shared_mutex_' should be
+ // locked in shared mode while calling and using the returned pointer. May
+ // return NULL if there is not enough space to insert a new key, in which
+ // case a resizable HashTable should release the 'resize_shared_mutex_' and
+ // call resize(), then try again.
+ virtual std::uint8_t *upsertInternalFast(
+ const TypedValue &key,
+ const std::size_t variable_key_size,
+ const std::uint8_t *init_value_ptr) = 0;
+
+ virtual std::uint8_t *upsertCompositeKeyInternalFast(
+ const std::vector<TypedValue> &key,
+ const std::uint8_t *init_value_ptr,
+ const std::size_t variable_key_size) = 0;
+
+ // Helpers for forEach. Each return true on success, false if no more entries
+ // exist to iterate over. After a successful call, '*key' is overwritten with
+ // the key of the next entry, '*value' points to the associated value, and
+ // '*entry_num' is incremented to the next (implementation defined) entry to
+ // check ('*entry_num' should initially be set to zero).
+ virtual bool getNextEntry(TypedValue *key,
+ const std::uint8_t **value,
+ std::size_t *entry_num) const = 0;
+ virtual bool getNextEntryCompositeKey(std::vector<TypedValue> *key,
+ const std::uint8_t **value,
+ std::size_t *entry_num) const = 0;
+
+ // Helpers for getAllFromValueAccessor. Each return true on success, false if
+ // no more entries exist for the specified key. After a successful call,
+ // '*value' points to the associated value, and '*entry_num' is incremented
+ // to the next (implementation defined) entry to check ('*entry_num' should
+ // initially be set to zero).
+ virtual bool getNextEntryForKey(const TypedValue &key,
+ const std::size_t hash_code,
+ const std::uint8_t **value,
+ std::size_t *entry_num) const = 0;
+ virtual bool getNextEntryForCompositeKey(const std::vector<TypedValue> &key,
+ const std::size_t hash_code,
+ const std::uint8_t **value,
+ std::size_t *entry_num) const = 0;
+
+ // Return true if key exists in the hash table.
+ virtual bool hasKey(const TypedValue &key) const = 0;
+ virtual bool hasCompositeKey(const std::vector<TypedValue> &key) const = 0;
+
+ // For a resizable HashTable, grow to accomodate more entries. If
+ // 'extra_buckets' is not zero, it may serve as a "hint" to implementations
+ // that at least the requested number of extra buckets are required when
+ // resizing (mainly used in putValueAccessor() and
+ // putValueAccessorCompositeKey() when 'preallocate_supported_' is true).
+ // Implementations are free to ignore 'extra_buckets'. If
+ // 'extra_variable_storage' is not zero, implementations will attempt to
+ // allocate at least enough additional variable-key storage space to
+ // accomodate the number of bytes specified. 'retry_num' is intended ONLY for
+ // when resize() recursively calls itself and should not be set to nonzero by
+ // any other caller.
+ virtual void resize(const std::size_t extra_buckets,
+ const std::size_t extra_variable_storage,
+ const std::size_t retry_num = 0) = 0;
+
+ // In the case where 'allow_duplicate_keys' is true, it is possible to
+ // pre-calculate the number of key-value entries and the amount of
+ // variable-length key storage that will be needed to insert all the
+ // entries from a ValueAccessor in putValueAccessor() or
+ // putValueAccessorCompositeKey() before actually inserting anything. Some
+ // HashTable implemetations (notably SeparateChainingHashTable) can achieve
+ // better performance by ammortizing the cost of allocating certain resources
+ // (buckets and variable-length key storage) in one up-front allocation. This
+ // method is intended to support that. Returns true and fills in
+ // '*prealloc_state' if pre-allocation was successful. Returns false if a
+ // resize() is needed.
+ virtual bool preallocateForBulkInsert(
+ const std::size_t total_entries,
+ const std::size_t total_variable_key_size,
+ HashTablePreallocationState *prealloc_state) {
+ FATAL_ERROR(
+ "Called HashTable::preallocateForBulkInsert() on a HashTable "
+ "implementation that does not support preallocation.");
+ }
+
+ // Type(s) of keys.
+ const std::vector<const Type *> key_types_;
+
+ // Information about whether key components are stored inline or in a
+ // separate variable-length storage region. This is usually determined by a
+ // HashTableKeyManager and set by calling setKeyInline().
+ bool scalar_key_inline_;
+ const std::vector<bool> *key_inline_;
+
+ // Whether hashes should be adjusted by AdjustHash() before being used.
+ const bool adjust_hashes_;
+ // Whether it is safe to use the simplified TypedValue::getHashScalarLiteral()
+ // method instead of the generic TypedValue::getHash() method.
+ const bool use_scalar_literal_hash_;
+ // Whether preallocateForBulkInsert() is supported by this HashTable.
+ const bool preallocate_supported_;
+
+ const std::vector<AggregationHandle *> handles_;
+ const unsigned int num_handles_;
+ const std::size_t total_payload_size_;
+ std::vector<std::size_t> payload_offsets_;
+
+ // Used only when resizable is true:
+ StorageManager *storage_manager_;
+ MutableBlobReference blob_;
+ // Locked in shared mode for most operations, exclusive mode during resize.
+ // Not locked at all for non-resizable HashTables.
+ alignas(kCacheLineBytes) SpinSharedMutex<true> resize_shared_mutex_;
+
+ // Used only when resizable is false:
+ void *hash_table_memory_;
+ const std::size_t hash_table_memory_size_;
+
+ private:
+ // Assign '*key_vector' with the attribute values specified by 'key_attr_ids'
+ // at the current position of 'accessor'. If 'check_for_null_keys' is true,
+ // stops and returns true if any of the values is null, otherwise returns
+ // false.
+ template <typename ValueAccessorT>
+ inline static bool GetCompositeKeyFromValueAccessor(
+ const ValueAccessorT &accessor,
+ const std::vector<attribute_id> &key_attr_ids,
+ const bool check_for_null_keys,
+ std::vector<TypedValue> *key_vector) {
+ for (std::vector<attribute_id>::size_type key_idx = 0;
+ key_idx < key_attr_ids.size();
+ ++key_idx) {
+ (*key_vector)[key_idx] = accessor.getTypedValue(key_attr_ids[key_idx]);
+ if (check_for_null_keys && (*key_vector)[key_idx].isNull()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // If run_if_match_found is true, apply the functor to each key if a match is
+ // found; otherwise, apply the functor if no match is found.
+ template <bool run_if_match_found, typename FunctorT>
+ void runOverKeysFromValueAccessor(ValueAccessor *accessor,
+ const attribute_id key_attr_id,
+ const bool check_for_null_keys,
+ FunctorT *functor) const;
+
+ template <bool run_if_match_found, typename FunctorT>
+ void runOverKeysFromValueAccessorCompositeKey(
+ ValueAccessor *accessor,
+ const std::vector<attribute_id> &key_attr_ids,
+ const bool check_for_null_keys,
+ FunctorT *functor) const;
+
+ // Method containing the actual logic implementing getAllFromValueAccessor().
+ // Has extra template parameters that control behavior to avoid some
+ // inner-loop branching.
+ template <typename FunctorT,
+ bool check_for_null_keys,
+ bool adjust_hashes_template,
+ bool use_scalar_literal_hash_template>
+ void getAllFromValueAccessorImpl(ValueAccessor *accessor,
+ const attribute_id key_attr_id,
+ FunctorT *functor) const;
+
+ // Data structures used for bloom filter optimized semi-joins.
+ bool has_build_side_bloom_filter_ = false;
+ bool has_probe_side_bloom_filter_ = false;
+ BloomFilter *build_bloom_filter_;
+ std::vector<const BloomFilter *> probe_bloom_filters_;
+ std::vector<std::vector<attribute_id>> probe_attribute_ids_;
+ DISALLOW_COPY_AND_ASSIGN(FastHashTable);
+};
+
+/**
+ * @brief An instantiation of the HashTable template for use in aggregations.
+ * @note This has force_key_copy = true, so that we don't have dangling pointers
+ * to blocks that are evicted.
+ **/
+using AggregationStateFastHashTable = FastHashTable<true, false, true, false>;
+
+/** @} */
+
+// ----------------------------------------------------------------------------
+// Implementations of template class methods follow.
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+HashTablePutResult
+FastHashTable<resizable, serializable, force_key_copy, allow_duplicate_keys>::
+ put(const TypedValue &key, const std::uint8_t &value) {
+ const std::size_t variable_size =
+ (force_key_copy && !scalar_key_inline_) ? key.getDataSize() : 0;
+ if (resizable) {
+ HashTablePutResult result = HashTablePutResult::kOutOfSpace;
+ while (result == HashTablePutResult::kOutOfSpace) {
+ {
+ SpinSharedMutexSharedLock<true> lock(resize_shared_mutex_);
+ result = putInternal(key, variable_size, value, nullptr);
+ }
+ if (result == HashTablePutResult::kOutOfSpace) {
+ resize(0, variable_size);
+ }
+ }
+ return result;
+ } else {
+ return putInternal(key, variable_size, value, nullptr);
+ }
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+HashTablePutResult
+FastHashTable<resizable, serializable, force_key_copy, allow_duplicate_keys>::
+ putCompositeKey(const std::vector<TypedValue> &key,
+ const std::uint8_t *init_value_ptr) {
+ const std::size_t variable_size =
+ calculateVariableLengthCompositeKeyCopySize(key);
+ if (resizable) {
+ HashTablePutResult result = HashTablePutResult::kOutOfSpace;
+ while (result == HashTablePutResult::kOutOfSpace) {
+ {
+ SpinSharedMutexSharedLock<true> lock(resize_shared_mutex_);
+ result = putCompositeKeyInternalFast(
+ key, variable_size, init_value_ptr, nullptr);
+ }
+ if (result == HashTablePutResult::kOutOfSpace) {
+ resize(0, variable_size);
+ }
+ }
+ return result;
+ } else {
+ return putCompositeKeyInternalFast(
+ key, variable_size, init_value_ptr, nullptr);
+ }
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+template <typename FunctorT>
+HashTablePutResult
+FastHashTable<resizable, serializable, force_key_copy, allow_duplicate_keys>::
+ putValueAccessor(ValueAccessor *accessor,
+ const attribute_id key_attr_id,
+ const bool check_for_null_keys,
+ FunctorT *functor) {
+ HashTablePutResult result = HashTablePutResult::kOutOfSpace;
+ std::size_t variable_size;
+ HashTablePreallocationState prealloc_state;
+ bool using_prealloc = allow_duplicate_keys && preallocate_supported_;
+ return InvokeOnAnyValueAccessor(
+ accessor,
+ [&](auto *accessor) -> HashTablePutResult { // NOLINT(build/c++11)
+ if (using_prealloc) {
+ std::size_t total_entries = 0;
+ std::size_t total_variable_key_size = 0;
+ if (check_for_null_keys || (force_key_copy && !scalar_key_inline_)) {
+ // If we need to filter out nulls OR make variable copies, make a
+ // prepass over the ValueAccessor.
+ while (accessor->next()) {
+ TypedValue key = accessor->getTypedValue(key_attr_id);
+ if (check_for_null_keys && key.isNull()) {
+ continue;
+ }
+ ++total_entries;
+ total_variable_key_size += (force_key_copy && !scalar_key_inline_)
+ ? key.getDataSize()
+ : 0;
+ }
+ accessor->beginIteration();
+ } else {
+ total_entries = accessor->getNumTuples();
+ }
+ if (resizable) {
+ bool prealloc_succeeded = false;
+ while (!prealloc_succeeded) {
+ {
+ SpinSharedMutexSharedLock<true> lock(resize_shared_mutex_);
+ prealloc_succeeded = this->preallocateForBulkInsert(
+ total_entries, total_variable_key_size, &prealloc_state);
+ }
+ if (!prealloc_succeeded) {
+ this->resize(total_entries, total_variable_key_size);
+ }
+ }
+ } else {
+ using_prealloc = this->preallocateForBulkInsert(
+ total_entries, total_variable_key_size, &prealloc_state);
+ }
+ }
+ std::unique_ptr<BloomFilter> thread_local_bloom_filter;
+ if (has_build_side_bloom_filter_) {
+ thread_local_bloom_filter.reset(
+ new BloomFilter(build_bloom_filter_->getRandomSeed(),
+ build_bloom_filter_->getNumberOfHashes(),
+ build_bloom_filter_->getBitArraySize()));
+ }
+ if (resizable) {
+ while (result == HashTablePutResult::kOutOfSpace) {
+ {
+ result = HashTablePutResult::kOK;
+ SpinSharedMutexSharedLock<true> lock(resize_shared_mutex_);
+ while (accessor->next()) {
+ TypedValue key = accessor->getTypedValue(key_attr_id);
+ if (check_for_null_keys && key.isNull()) {
+ continue;
+ }
+ variable_size = (force_key_copy && !scalar_key_inline_)
+ ? key.getDataSize()
+ : 0;
+ result = this->putInternal(
+ key,
+ variable_size,
+ (*functor)(*accessor),
+ using_prealloc ? &prealloc_state : nullptr);
+ // Insert into bloom filter, if enabled.
+ if (has_build_side_bloom_filter_) {
+ thread_local_bloom_filter->insertUnSafe(
+ static_cast<const std::uint8_t *>(key.getDataPtr()),
+ key.getDataSize());
+ }
+ if (result == HashTablePutResult::kDuplicateKey) {
+ DEBUG_ASSERT(!using_prealloc);
+ return result;
+ } else if (result == HashTablePutResult::kOutOfSpace) {
+ DEBUG_ASSERT(!using_prealloc);
+ break;
+ }
+ }
+ }
+ if (result == HashTablePutResult::kOutOfSpace) {
+ this->resize(0, variable_size);
+ accessor->previous();
+ }
+ }
+ } else {
+ while (accessor->next()) {
+ TypedValue key = accessor->getTypedValue(key_attr_id);
+ if (check_for_null_keys && key.isNull()) {
+ continue;
+ }
+ variable_size =
+ (force_key_copy && !scalar_key_inline_) ? key.getDataSize() : 0;
+ result =
+ this->putInternal(key,
+ variable_size,
+ (*functor)(*accessor),
+ using_prealloc ? &prealloc_state : nullptr);
+ // Insert into bloom filter, if enabled.
+ if (has_build_side_bloom_filter_) {
+ thread_local_bloom_filter->insertUnSafe(
+ static_cast<const std::uint8_t *>(key.getDataPtr()),
+ key.getDataSize());
+ }
+ if (result != HashTablePutResult::kOK) {
+ return result;
+ }
+ }
+ }
+ // Update the build side bloom filter with thread local copy, if
+ // available.
+ if (has_build_side_bloom_filter_) {
+ build_bloom_filter_->bitwiseOr(thread_local_bloom_filter.get());
+ }
+
+ return HashTablePutResult::kOK;
+ });
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+template <typename FunctorT>
+HashTablePutResult
+FastHashTable<resizable, serializable, force_key_copy, allow_duplicate_keys>::
+ putValueAccessorCompositeKey(ValueAccessor *accessor,
+ const std::vector<attribute_id> &key_attr_ids,
+ const bool check_for_null_keys,
+ FunctorT *functor) {
+ DEBUG_ASSERT(key_types_.size() == key_attr_ids.size());
+ HashTablePutResult result = HashTablePutResult::kOutOfSpace;
+ std::size_t variable_size;
+ HashTablePreallocationState prealloc_state;
+ bool using_prealloc = allow_duplicate_keys && preallocate_supported_;
+ std::vector<TypedValue> key_vector;
+ key_vector.resize(key_attr_ids.size());
+ return InvokeOnAnyValueAccessor(
+ accessor,
+ [&](auto *accessor) -> HashTablePutResult { // NOLINT(build/c++11)
+ if (using_prealloc) {
+ std::size_t total_entries = 0;
+ std::size_t total_variable_key_size = 0;
+ if (check_for_null_keys || force_key_copy) {
+ // If we need to filter out nulls OR make variable copies, make a
+ // prepass over the ValueAccessor.
+ while (accessor->next()) {
+ if (this->GetCompositeKeyFromValueAccessor(*accessor,
+ key_attr_ids,
+ check_for_null_keys,
+ &key_vector)) {
+ continue;
+ }
+ ++total_entries;
+ total_variable_key_size +=
+ this->calculateVariableLengthCompositeKeyCopySize(key_vector);
+ }
+ accessor->beginIteration();
+ } else {
+ total_entries = accessor->getNumTuples();
+ }
+ if (resizable) {
+ bool prealloc_succeeded = false;
+ while (!prealloc_succeeded) {
+ {
+ SpinSharedMutexSharedLock<true> lock(resize_shared_mutex_);
+ prealloc_succeeded = this->preallocateForBulkInsert(
+ total_entries, total_variable_key_size, &prealloc_state);
+ }
+ if (!prealloc_succeeded) {
+ this->resize(total_entries, total_variable_key_size);
+ }
+ }
+ } else {
+ using_prealloc = this->preallocateForBulkInsert(
+ total_entries, total_variable_key_size, &prealloc_state);
+ }
+ }
+ if (resizable) {
+ while (result == HashTablePutResult::kOutOfSpace) {
+ {
+ result = HashTablePutResult::kOK;
+ SpinSharedMutexSharedLock<true> lock(resize_shared_mutex_);
+ while (accessor->next()) {
+ if (this->GetCompositeKeyFromValueAccessor(*accessor,
+ key_attr_ids,
+ check_for_null_keys,
+ &key_vector)) {
+ continue;
+ }
+ variable_size =
+ this->calculateVariableLengthCompositeKeyCopySize(
+ key_vector);
+ result = this->putCompositeKeyInternal(
+ key_vector,
+ variable_size,
+ (*functor)(*accessor),
+ using_prealloc ? &prealloc_state : nullptr);
+ if (result == HashTablePutResult::kDuplicateKey) {
+ DEBUG_ASSERT(!using_prealloc);
+ return result;
+ } else if (result == HashTablePutResult::kOutOfSpace) {
+ DEBUG_ASSERT(!using_prealloc);
+ break;
+ }
+ }
+ }
+ if (result == HashTablePutResult::kOutOfSpace) {
+ this->resize(0, variable_size);
+ accessor->previous();
+ }
+ }
+ } else {
+ while (accessor->next()) {
+ if (this->GetCompositeKeyFromValueAccessor(*accessor,
+ key_attr_ids,
+ check_for_null_keys,
+ &key_vector)) {
+ continue;
+ }
+ variable_size =
+ this->calculateVariableLengthCompositeKeyCopySize(key_vector);
+ result = this->putCompositeKeyInternal(
+ key_vector,
+ variable_size,
+ (*functor)(*accessor),
+ using_prealloc ? &prealloc_state : nullptr);
+ if (result != HashTablePutResult::kOK) {
+ return result;
+ }
+ }
+ }
+
+ return HashTablePutResult::kOK;
+ });
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+template <typename FunctorT>
+bool FastHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::upsert(const TypedValue &key,
+ const std::uint8_t
+ *initial_value_ptr,
+ FunctorT *functor) {
+ DEBUG_ASSERT(!allow_duplicate_keys);
+ const std::size_t variable_size =
+ (force_key_copy && !scalar_key_inline_) ? key.getDataSize() : 0;
+ if (resizable) {
+ for (;;) {
+ {
+ SpinSharedMutexSharedLock<true> resize_lock(resize_shared_mutex_);
+ std::uint8_t *value =
+ upsertInternalFast(key, variable_size, initial_value_ptr);
+ if (value != nullptr) {
+ (*functor)(value);
+ return true;
+ }
+ }
+ resize(0, force_key_copy && !scalar_key_inline_ ? key.getDataSize() : 0);
+ }
+ } else {
+ std::uint8_t *value =
+ upsertInternalFast(key, variable_size, initial_value_ptr);
+ if (value == nullptr) {
+ return false;
+ } else {
+ (*functor)(value);
+ return true;
+ }
+ }
+}
+
+class HashTableMergerFast {
+ public:
+ /**
+ * @brief Constructor
+ *
+ * @param handle The Aggregation handle being used.
+ * @param destination_hash_table The destination hash table to which other
+ * hash tables will be merged.
+ **/
+ explicit HashTableMergerFast(
+ AggregationStateHashTableBase *destination_hash_table)
+ : destination_hash_table_(
+ static_cast<FastHashTable<true, false, true, false> *>(
+ destination_hash_table)) {}
+
+ /**
+ * @brief The operator for the functor.
+ *
+ * @param group_by_key The group by key being merged.
+ * @param source_state The aggregation state for the given key in the source
+ * aggregation hash table.
+ **/
+ inline void operator()(const std::vector<TypedValue> &group_by_key,
+ const std::uint8_t *source_state) {
+ const std::uint8_t *original_state =
+ destination_hash_table_->getSingleCompositeKey(group_by_key);
+ if (original_state != nullptr) {
+ // The CHECK is required as upsertCompositeKey can return false if the
+ // hash table runs out of space during the upsert process. The ideal
+ // solution will be to retry again if the upsert fails.
+ CHECK(destination_hash_table_->upsertCompositeKeyFast(
+ group_by_key, original_state, source_state));
+ } else {
+ destination_hash_table_->putCompositeKey(group_by_key, source_state);
+ }
+ }
+
+ private:
+ FastHashTable<true, false, true, false> *destination_hash_table_;
+
+ DISALLOW_COPY_AND_ASSIGN(HashTableMergerFast);
+};
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+template <typename FunctorT>
+bool FastHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ upsertCompositeKeyFast(const std::vector<TypedValue> &key,
+ const std::uint8_t *init_value_ptr,
+ FunctorT *functor) {
+ DEBUG_ASSERT(!allow_duplicate_keys);
+ const std::size_t variable_size =
+ calculateVariableLengthCompositeKeyCopySize(key);
+ if (resizable) {
+ for (;;) {
+ {
+ SpinSharedMutexSharedLock<true> resize_lock(resize_shared_mutex_);
+ std::uint8_t *value =
+ upsertCompositeKeyInternalFast(key, init_value_ptr, variable_size);
+ if (value != nullptr) {
+ (*functor)(value);
+ return true;
+ }
+ }
+ resize(0, variable_size);
+ }
+ } else {
+ std::uint8_t *value =
+ upsertCompositeKeyInternalFast(key, init_value_ptr, variable_size);
+ if (value == nullptr) {
+ return false;
+ } else {
+ (*functor)(value);
+ return true;
+ }
+ }
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+template <typename FunctorT>
+bool FastHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ upsertCompositeKeyFast(const std::vector<TypedValue> &key,
+ const std::uint8_t *init_value_ptr,
+ FunctorT *functor,
+ int index) {
+ DEBUG_ASSERT(!allow_duplicate_keys);
+ const std::size_t variable_size =
+ calculateVariableLengthCompositeKeyCopySize(key);
+ if (resizable) {
+ for (;;) {
+ {
+ SpinSharedMutexSharedLock<true> resize_lock(resize_shared_mutex_);
+ std::uint8_t *value =
+ upsertCompositeKeyInternalFast(key, init_value_ptr, variable_size);
+ if (value != nullptr) {
+ (*functor)(value + payload_offsets_[index]);
+ return true;
+ }
+ }
+ resize(0, variable_size);
+ }
+ } else {
+ std::uint8_t *value =
+ upsertCompositeKeyInternalFast(key, init_value_ptr, variable_size);
+ if (value == nullptr) {
+ return false;
+ } else {
+ (*functor)(value + payload_offsets_[index]);
+ return true;
+ }
+ }
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+bool FastHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ upsertCompositeKeyFast(const std::vector<TypedValue> &key,
+ const std::uint8_t *init_value_ptr,
+ const std::uint8_t *source_state) {
+ DEBUG_ASSERT(!allow_duplicate_keys);
+ const std::size_t variable_size =
+ calculateVariableLengthCompositeKeyCopySize(key);
+ if (resizable) {
+ for (;;) {
+ {
+ SpinSharedMutexSharedLock<true> resize_lock(resize_shared_mutex_);
+ std::uint8_t *value =
+ upsertCompositeKeyInternalFast(key, init_value_ptr, variable_size);
+ if (value != nullptr) {
+ SpinMutexLock lock(*(reinterpret_cast<SpinMutex *>(value)));
+ for (unsigned int k = 0; k < num_handles_; ++k) {
+ handles_[k]->mergeStatesFast(source_state + payload_offsets_[k],
+ value + payload_offsets_[k]);
+ }
+ return true;
+ }
+ }
+ resize(0, variable_size);
+ }
+ } else {
+ std::uint8_t *value =
+ upsertCompositeKeyInternalFast(key, init_value_ptr, variable_size);
+ i
<TRUNCATED>
[09/17] incubator-quickstep git commit: Initial commit for
QUICKSTEP-28 and QUICKSTEP-29.
Posted by ji...@apache.org.
Initial commit for QUICKSTEP-28 and QUICKSTEP-29.
- This PR creates a specialized hash table implementation for
aggregation.
- A hash table's bucket can store multiple AggregationStates,
corresponding to various AggregationHandles in the query. Earlier we
created one hash table for each AggregationHandle.
- All Aggregation states in a single hash table bucket are protected
using a single mutex. Earlier AggregationStates provided internal
concurrency protection (mutex).
- Single aggregationGroupBy method in StorageBlock.
- New methods for separating unary and nullary updation of AggregationState.
- Added TODO to move method from HashTableBase class.
- Added doxygen for the AggregationHandle new functions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ac3512ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ac3512ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ac3512ce
Branch: refs/heads/lip-refactor
Commit: ac3512ceb16109702116aa2a51db382dc99ba23e
Parents: 43c7a42
Author: rathijit <ra...@node-2.hashtable.quickstep-pg0.wisc.cloudlab.us>
Authored: Mon Jul 4 02:44:48 2016 -0500
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Tue Sep 20 15:26:52 2016 -0500
----------------------------------------------------------------------
catalog/CatalogTypedefs.hpp | 2 +
.../aggregation/AggregationConcreteHandle.cpp | 21 +-
.../aggregation/AggregationConcreteHandle.hpp | 372 +--
expressions/aggregation/AggregationHandle.hpp | 118 +-
.../aggregation/AggregationHandleAvg.cpp | 118 +-
.../aggregation/AggregationHandleAvg.hpp | 118 +-
.../aggregation/AggregationHandleCount.cpp | 180 +-
.../aggregation/AggregationHandleCount.hpp | 111 +-
.../aggregation/AggregationHandleDistinct.cpp | 5 +-
.../aggregation/AggregationHandleDistinct.hpp | 39 +-
.../aggregation/AggregationHandleMax.cpp | 98 +-
.../aggregation/AggregationHandleMax.hpp | 112 +-
.../aggregation/AggregationHandleMin.cpp | 99 +-
.../aggregation/AggregationHandleMin.hpp | 106 +-
.../aggregation/AggregationHandleSum.cpp | 112 +-
.../aggregation/AggregationHandleSum.hpp | 108 +-
expressions/aggregation/CMakeLists.txt | 8 +
.../tests/AggregationHandleAvg_unittest.cpp | 255 +-
.../tests/AggregationHandleCount_unittest.cpp | 311 ++-
.../tests/AggregationHandleMax_unittest.cpp | 382 +--
.../tests/AggregationHandleMin_unittest.cpp | 378 +--
.../tests/AggregationHandleSum_unittest.cpp | 291 +-
query_optimizer/ExecutionGenerator.cpp | 20 +-
.../tests/AggregationOperator_unittest.cpp | 3 +-
storage/AggregationOperationState.cpp | 314 ++-
storage/AggregationOperationState.hpp | 47 +-
storage/CMakeLists.txt | 55 +
storage/FastHashTable.hpp | 2515 ++++++++++++++++++
storage/FastHashTableFactory.hpp | 257 ++
storage/FastSeparateChainingHashTable.hpp | 1750 ++++++++++++
storage/HashTable.hpp | 10 -
storage/HashTableBase.hpp | 37 +-
storage/HashTablePool.hpp | 65 +
storage/StorageBlock.cpp | 27 +-
storage/StorageBlock.hpp | 16 +-
35 files changed, 6803 insertions(+), 1657 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/catalog/CatalogTypedefs.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogTypedefs.hpp b/catalog/CatalogTypedefs.hpp
index f7a2d53..70bac84 100644
--- a/catalog/CatalogTypedefs.hpp
+++ b/catalog/CatalogTypedefs.hpp
@@ -49,6 +49,8 @@ constexpr int kInvalidCatalogId = -1;
// Used to indicate no preference for a NUMA Node ID.
constexpr numa_node_id kAnyNUMANodeID = -1;
+constexpr attribute_id kInvalidAttributeID = -1;
+
/** @} */
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationConcreteHandle.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationConcreteHandle.cpp b/expressions/aggregation/AggregationConcreteHandle.cpp
index 719920f..e3fb520 100644
--- a/expressions/aggregation/AggregationConcreteHandle.cpp
+++ b/expressions/aggregation/AggregationConcreteHandle.cpp
@@ -23,6 +23,7 @@
#include <vector>
#include "catalog/CatalogTypedefs.hpp"
+#include "storage/FastHashTable.hpp"
#include "storage/HashTable.hpp"
#include "storage/HashTableFactory.hpp"
@@ -51,22 +52,16 @@ void AggregationConcreteHandle::insertValueAccessorIntoDistinctifyHashTable(
AggregationStateHashTableBase *distinctify_hash_table) const {
// If the key-value pair is already there, we don't need to update the value,
// which should always be "true". I.e. the value is just a placeholder.
- const auto noop_upserter = [](const auto &accessor, const bool *value) -> void {};
- AggregationStateHashTable<bool> *hash_table =
- static_cast<AggregationStateHashTable<bool>*>(distinctify_hash_table);
+ AggregationStateFastHashTable *hash_table =
+ static_cast<AggregationStateFastHashTable *>(distinctify_hash_table);
if (key_ids.size() == 1) {
- hash_table->upsertValueAccessor(accessor,
- key_ids[0],
- true /* check_for_null_keys */,
- true /* initial_value */,
- &noop_upserter);
+ hash_table->upsertValueAccessorFast(
+ key_ids, accessor, key_ids[0], true /* check_for_null_keys */);
} else {
- hash_table->upsertValueAccessorCompositeKey(accessor,
- key_ids,
- true /* check_for_null_keys */,
- true /* initial_value */,
- &noop_upserter);
+ std::vector<attribute_id> empty_args {kInvalidAttributeID};
+ hash_table->upsertValueAccessorCompositeKeyFast(
+ empty_args, accessor, key_ids, true /* check_for_null_keys */);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationConcreteHandle.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationConcreteHandle.hpp b/expressions/aggregation/AggregationConcreteHandle.hpp
index c5ca061..398a032 100644
--- a/expressions/aggregation/AggregationConcreteHandle.hpp
+++ b/expressions/aggregation/AggregationConcreteHandle.hpp
@@ -21,13 +21,15 @@
#define QUICKSTEP_EXPRESSIONS_AGGREGATION_AGGREGATION_CONCRETE_HANDLE_HPP_
#include <cstddef>
-#include <vector>
#include <utility>
+#include <vector>
#include "catalog/CatalogTypedefs.hpp"
#include "expressions/aggregation/AggregationHandle.hpp"
+#include "storage/FastHashTable.hpp"
#include "storage/HashTable.hpp"
#include "storage/HashTableBase.hpp"
+#include "threading/SpinMutex.hpp"
#include "types/TypedValue.hpp"
#include "types/containers/ColumnVector.hpp"
#include "utility/Macros.hpp"
@@ -48,8 +50,8 @@ class ValueAccessor;
* @brief An upserter class for modifying the destination hash table while
* merging two group by hash tables.
**/
-template <typename HandleT, typename StateT>
-class HashTableStateUpserter {
+template <typename HandleT>
+class HashTableStateUpserterFast {
public:
/**
* @brief Constructor.
@@ -59,7 +61,8 @@ class HashTableStateUpserter {
* table. The corresponding state (for the same key) in the destination
* hash table will be upserted.
**/
- HashTableStateUpserter(const HandleT &handle, const StateT &source_state)
+ HashTableStateUpserterFast(const HandleT &handle,
+ const std::uint8_t *source_state)
: handle_(handle), source_state_(source_state) {}
/**
@@ -68,65 +71,15 @@ class HashTableStateUpserter {
* @param destination_state The aggregation state in the aggregation hash
* table that is being upserted.
**/
- void operator()(StateT *destination_state) {
- handle_.mergeStates(source_state_, destination_state);
+ void operator()(std::uint8_t *destination_state) {
+ handle_.mergeStatesFast(source_state_, destination_state);
}
private:
const HandleT &handle_;
- const StateT &source_state_;
+ const std::uint8_t *source_state_;
- DISALLOW_COPY_AND_ASSIGN(HashTableStateUpserter);
-};
-
-/**
- * @brief A class to support the functor for merging group by hash tables.
- **/
-template <typename HandleT, typename StateT, typename HashTableT>
-class HashTableMerger {
- public:
- /**
- * @brief Constructor
- *
- * @param handle The Aggregation handle being used.
- * @param destination_hash_table The destination hash table to which other
- * hash tables will be merged.
- **/
- HashTableMerger(const HandleT &handle,
- AggregationStateHashTableBase *destination_hash_table)
- : handle_(handle),
- destination_hash_table_(
- static_cast<HashTableT *>(destination_hash_table)) {}
-
- /**
- * @brief The operator for the functor.
- *
- * @param group_by_key The group by key being merged.
- * @param source_state The aggregation state for the given key in the source
- * aggregation hash table.
- **/
- inline void operator()(const std::vector<TypedValue> &group_by_key,
- const StateT &source_state) {
- const StateT *original_state =
- destination_hash_table_->getSingleCompositeKey(group_by_key);
- if (original_state != nullptr) {
- HashTableStateUpserter<HandleT, StateT> upserter(
- handle_, source_state);
- // The CHECK is required as upsertCompositeKey can return false if the
- // hash table runs out of space during the upsert process. The ideal
- // solution will be to retry again if the upsert fails.
- CHECK(destination_hash_table_->upsertCompositeKey(
- group_by_key, *original_state, &upserter));
- } else {
- destination_hash_table_->putCompositeKey(group_by_key, source_state);
- }
- }
-
- private:
- const HandleT &handle_;
- HashTableT *destination_hash_table_;
-
- DISALLOW_COPY_AND_ASSIGN(HashTableMerger);
+ DISALLOW_COPY_AND_ASSIGN(HashTableStateUpserterFast);
};
/**
@@ -156,14 +109,15 @@ class AggregationConcreteHandle : public AggregationHandle {
*/
AggregationStateHashTableBase* createDistinctifyHashTable(
const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &key_types,
+ const std::vector<const Type *> &key_types,
const std::size_t estimated_num_distinct_keys,
StorageManager *storage_manager) const override;
/**
- * @brief Implementaion for AggregationHandle::insertValueAccessorIntoDistinctifyHashTable()
- * that inserts the GROUP BY expressions and aggregation arguments together
- * as keys into the distinctify hash table.
+ * @brief Implementaion for
+ * AggregationHandle::insertValueAccessorIntoDistinctifyHashTable()
+ * that inserts the GROUP BY expressions and aggregation arguments together
+ * as keys into the distinctify hash table.
*/
void insertValueAccessorIntoDistinctifyHashTable(
ValueAccessor *accessor,
@@ -171,61 +125,40 @@ class AggregationConcreteHandle : public AggregationHandle {
AggregationStateHashTableBase *distinctify_hash_table) const override;
protected:
- AggregationConcreteHandle() {
- }
+ AggregationConcreteHandle() {}
- template <typename HandleT,
- typename StateT,
- typename HashTableT>
- void aggregateValueAccessorIntoHashTableNullaryHelper(
- ValueAccessor *accessor,
- const std::vector<attribute_id> &group_by_key_ids,
- const StateT &default_state,
- AggregationStateHashTableBase *hash_table) const;
-
- template <typename HandleT,
- typename StateT,
- typename HashTableT>
- void aggregateValueAccessorIntoHashTableUnaryHelper(
- ValueAccessor *accessor,
- const attribute_id argument_id,
- const std::vector<attribute_id> &group_by_key_ids,
- const StateT &default_state,
- AggregationStateHashTableBase *hash_table) const;
-
- template <typename HandleT,
- typename StateT>
- StateT* aggregateOnDistinctifyHashTableForSingleUnaryHelper(
+ template <typename HandleT, typename StateT>
+ StateT* aggregateOnDistinctifyHashTableForSingleUnaryHelperFast(
const AggregationStateHashTableBase &distinctify_hash_table) const;
- template <typename HandleT,
- typename StateT,
- typename HashTableT>
- void aggregateOnDistinctifyHashTableForGroupByUnaryHelper(
+ template <typename HandleT, typename HashTableT>
+ void aggregateOnDistinctifyHashTableForGroupByUnaryHelperFast(
const AggregationStateHashTableBase &distinctify_hash_table,
- const StateT &default_state,
- AggregationStateHashTableBase *hash_table) const;
+ AggregationStateHashTableBase *hash_table,
+ std::size_t index) const;
- template <typename HandleT,
- typename HashTableT>
- ColumnVector* finalizeHashTableHelper(
+ template <typename HandleT, typename HashTableT>
+ ColumnVector* finalizeHashTableHelperFast(
const Type &result_type,
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const;
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const;
template <typename HandleT, typename HashTableT>
- inline TypedValue finalizeGroupInHashTable(
+ inline TypedValue finalizeGroupInHashTableFast(
const AggregationStateHashTableBase &hash_table,
- const std::vector<TypedValue> &group_key) const {
- const AggregationState *group_state
- = static_cast<const HashTableT&>(hash_table).getSingleCompositeKey(group_key);
+ const std::vector<TypedValue> &group_key,
+ int index) const {
+ const std::uint8_t *group_state =
+ static_cast<const HashTableT &>(hash_table).getSingleCompositeKey(group_key, index);
DCHECK(group_state != nullptr)
<< "Could not find entry for specified group_key in HashTable";
- return static_cast<const HandleT*>(this)->finalizeHashTableEntry(*group_state);
+ return static_cast<const HandleT *>(this)->finalizeHashTableEntryFast(
+ group_state);
}
- template <typename HandleT, typename StateT, typename HashTableT>
- void mergeGroupByHashTablesHelper(
+ template <typename HandleT, typename HashTableT>
+ void mergeGroupByHashTablesHelperFast(
const AggregationStateHashTableBase &source_hash_table,
AggregationStateHashTableBase *destination_hash_table) const;
@@ -234,51 +167,6 @@ class AggregationConcreteHandle : public AggregationHandle {
};
/**
- * @brief Templated class to implement value-accessor-based upserter for each
- * aggregation state payload type. This version is for nullary
- * aggregates (those that take no arguments).
- **/
-template <typename HandleT, typename StateT>
-class NullaryAggregationStateValueAccessorUpserter {
- public:
- explicit NullaryAggregationStateValueAccessorUpserter(const HandleT &handle)
- : handle_(handle) {
- }
-
- template <typename ValueAccessorT>
- inline void operator()(const ValueAccessorT &accessor, StateT *state) {
- handle_.iterateNullaryInl(state);
- }
-
- private:
- const HandleT &handle_;
-};
-
-/**
- * @brief Templated class to implement value-accessor-based upserter for each
- * aggregation state payload type. This version is for unary aggregates
- * (those that take a single argument).
- **/
-template <typename HandleT, typename StateT>
-class UnaryAggregationStateValueAccessorUpserter {
- public:
- UnaryAggregationStateValueAccessorUpserter(const HandleT &handle,
- attribute_id value_id)
- : handle_(handle),
- value_id_(value_id) {
- }
-
- template <typename ValueAccessorT>
- inline void operator()(const ValueAccessorT &accessor, StateT *state) {
- handle_.iterateUnaryInl(state, accessor.getTypedValue(value_id_));
- }
-
- private:
- const HandleT &handle_;
- const attribute_id value_id_;
-};
-
-/**
* @brief Templated helper class used to implement
* AggregationHandle::finalizeHashTable() by visiting each entry (i.e.
* GROUP) in a HashTable, finalizing the aggregation for the GROUP, and
@@ -288,18 +176,26 @@ class UnaryAggregationStateValueAccessorUpserter {
template <typename HandleT, typename ColumnVectorT>
class HashTableAggregateFinalizer {
public:
- HashTableAggregateFinalizer(const HandleT &handle,
- std::vector<std::vector<TypedValue>> *group_by_keys,
- ColumnVectorT *output_column_vector)
+ HashTableAggregateFinalizer(
+ const HandleT &handle,
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ ColumnVectorT *output_column_vector)
: handle_(handle),
group_by_keys_(group_by_keys),
- output_column_vector_(output_column_vector) {
- }
+ output_column_vector_(output_column_vector) {}
inline void operator()(const std::vector<TypedValue> &group_by_key,
const AggregationState &group_state) {
group_by_keys_->emplace_back(group_by_key);
- output_column_vector_->appendTypedValue(handle_.finalizeHashTableEntry(group_state));
+ output_column_vector_->appendTypedValue(
+ handle_.finalizeHashTableEntry(group_state));
+ }
+
+ inline void operator()(const std::vector<TypedValue> &group_by_key,
+ const unsigned char *byte_ptr) {
+ group_by_keys_->emplace_back(group_by_key);
+ output_column_vector_->appendTypedValue(
+ handle_.finalizeHashTableEntryFast(byte_ptr));
}
private:
@@ -313,171 +209,117 @@ class HashTableAggregateFinalizer {
// ----------------------------------------------------------------------------
// Implementations of templated methods follow:
-template <typename HandleT,
- typename StateT,
- typename HashTableT>
-void AggregationConcreteHandle::aggregateValueAccessorIntoHashTableNullaryHelper(
- ValueAccessor *accessor,
- const std::vector<attribute_id> &group_by_key_ids,
- const StateT &default_state,
- AggregationStateHashTableBase *hash_table) const {
- NullaryAggregationStateValueAccessorUpserter<HandleT, StateT>
- upserter(static_cast<const HandleT&>(*this));
- static_cast<HashTableT*>(hash_table)->upsertValueAccessorCompositeKey(
- accessor,
- group_by_key_ids,
- true,
- default_state,
- &upserter);
-}
-
-template <typename HandleT,
- typename StateT,
- typename HashTableT>
-void AggregationConcreteHandle::aggregateValueAccessorIntoHashTableUnaryHelper(
- ValueAccessor *accessor,
- const attribute_id argument_id,
- const std::vector<attribute_id> &group_by_key_ids,
- const StateT &default_state,
- AggregationStateHashTableBase *hash_table) const {
- UnaryAggregationStateValueAccessorUpserter<HandleT, StateT>
- upserter(static_cast<const HandleT&>(*this), argument_id);
- static_cast<HashTableT*>(hash_table)->upsertValueAccessorCompositeKey(
- accessor,
- group_by_key_ids,
- true,
- default_state,
- &upserter);
-}
-
-template <typename HandleT,
- typename StateT>
-StateT* AggregationConcreteHandle::aggregateOnDistinctifyHashTableForSingleUnaryHelper(
- const AggregationStateHashTableBase &distinctify_hash_table) const {
- const HandleT& handle = static_cast<const HandleT&>(*this);
- StateT *state = static_cast<StateT*>(createInitialState());
+template <typename HandleT, typename StateT>
+StateT* AggregationConcreteHandle::
+ aggregateOnDistinctifyHashTableForSingleUnaryHelperFast(
+ const AggregationStateHashTableBase &distinctify_hash_table) const {
+ const HandleT &handle = static_cast<const HandleT &>(*this);
+ StateT *state = static_cast<StateT *>(createInitialState());
// A lambda function which will be called on each key from the distinctify
// hash table.
- const auto aggregate_functor = [&handle, &state](const TypedValue &key,
- const bool &dumb_placeholder) {
+ const auto aggregate_functor = [&handle, &state](
+ const TypedValue &key, const std::uint8_t &dumb_placeholder) {
// For each (unary) key in the distinctify hash table, aggregate the key
// into "state".
handle.iterateUnaryInl(state, key);
};
- const AggregationStateHashTable<bool> &hash_table =
- static_cast<const AggregationStateHashTable<bool>&>(distinctify_hash_table);
- // Invoke the lambda function "aggregate_functor" on each key from the distinctify
- // hash table.
+ const AggregationStateFastHashTable &hash_table =
+ static_cast<const AggregationStateFastHashTable &>(
+ distinctify_hash_table);
+ // Invoke the lambda function "aggregate_functor" on each key from the
+ // distinctify hash table.
hash_table.forEach(&aggregate_functor);
return state;
}
-template <typename HandleT,
- typename StateT,
- typename HashTableT>
-void AggregationConcreteHandle::aggregateOnDistinctifyHashTableForGroupByUnaryHelper(
- const AggregationStateHashTableBase &distinctify_hash_table,
- const StateT &default_state,
- AggregationStateHashTableBase *aggregation_hash_table) const {
- const HandleT& handle = static_cast<const HandleT&>(*this);
- HashTableT *target_hash_table = static_cast<HashTableT*>(aggregation_hash_table);
+template <typename HandleT, typename HashTableT>
+void AggregationConcreteHandle::
+ aggregateOnDistinctifyHashTableForGroupByUnaryHelperFast(
+ const AggregationStateHashTableBase &distinctify_hash_table,
+ AggregationStateHashTableBase *aggregation_hash_table,
+ std::size_t index) const {
+ const HandleT &handle = static_cast<const HandleT &>(*this);
+ HashTableT *target_hash_table =
+ static_cast<HashTableT *>(aggregation_hash_table);
// A lambda function which will be called on each key-value pair from the
// distinctify hash table.
- const auto aggregate_functor = [&handle, &target_hash_table, &default_state](
- std::vector<TypedValue> &key,
- const bool &dumb_placeholder) {
+ const auto aggregate_functor = [&handle, &target_hash_table, &index](
+ std::vector<TypedValue> &key, const bool &dumb_placeholder) {
// For each (composite) key vector in the distinctify hash table with size N.
- // The first N-1 entries are GROUP BY columns and the last entry is the argument
- // to be aggregated on.
+ // The first N-1 entries are GROUP BY columns and the last entry is the
+ // argument to be aggregated on.
const TypedValue argument(std::move(key.back()));
key.pop_back();
// An upserter as lambda function for aggregating the argument into its
// GROUP BY group's entry inside aggregation_hash_table.
- const auto upserter = [&handle, &argument](StateT *state) {
- handle.iterateUnaryInl(state, argument);
+ const auto upserter = [&handle, &argument](std::uint8_t *state) {
+ handle.iterateUnaryInlFast(argument, state);
};
- target_hash_table->upsertCompositeKey(key, default_state, &upserter);
+ target_hash_table->upsertCompositeKeyFast(key, nullptr, &upserter, index);
};
- const AggregationStateHashTable<bool> &source_hash_table =
- static_cast<const AggregationStateHashTable<bool>&>(distinctify_hash_table);
+ const HashTableT &source_hash_table =
+ static_cast<const HashTableT &>(distinctify_hash_table);
// Invoke the lambda function "aggregate_functor" on each composite key vector
// from the distinctify hash table.
- source_hash_table.forEachCompositeKey(&aggregate_functor);
+ source_hash_table.forEachCompositeKeyFast(&aggregate_functor);
}
-template <typename HandleT,
- typename HashTableT>
-ColumnVector* AggregationConcreteHandle::finalizeHashTableHelper(
+template <typename HandleT, typename HashTableT>
+ColumnVector* AggregationConcreteHandle::finalizeHashTableHelperFast(
const Type &result_type,
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const {
- const HandleT &handle = static_cast<const HandleT&>(*this);
- const HashTableT &hash_table_concrete = static_cast<const HashTableT&>(hash_table);
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const {
+ const HandleT &handle = static_cast<const HandleT &>(*this);
+ const HashTableT &hash_table_concrete =
+ static_cast<const HashTableT &>(hash_table);
if (group_by_keys->empty()) {
if (NativeColumnVector::UsableForType(result_type)) {
- NativeColumnVector *result = new NativeColumnVector(result_type,
- hash_table_concrete.numEntries());
+ NativeColumnVector *result =
+ new NativeColumnVector(result_type, hash_table_concrete.numEntries());
HashTableAggregateFinalizer<HandleT, NativeColumnVector> finalizer(
- handle,
- group_by_keys,
- result);
- hash_table_concrete.forEachCompositeKey(&finalizer);
+ handle, group_by_keys, result);
+ hash_table_concrete.forEachCompositeKeyFast(&finalizer, index);
return result;
} else {
- IndirectColumnVector *result = new IndirectColumnVector(result_type,
- hash_table_concrete.numEntries());
+ IndirectColumnVector *result = new IndirectColumnVector(
+ result_type, hash_table_concrete.numEntries());
HashTableAggregateFinalizer<HandleT, IndirectColumnVector> finalizer(
- handle,
- group_by_keys,
- result);
- hash_table_concrete.forEachCompositeKey(&finalizer);
+ handle, group_by_keys, result);
+ hash_table_concrete.forEachCompositeKeyFast(&finalizer, index);
return result;
}
} else {
if (NativeColumnVector::UsableForType(result_type)) {
- NativeColumnVector *result = new NativeColumnVector(result_type,
- group_by_keys->size());
+ NativeColumnVector *result =
+ new NativeColumnVector(result_type, group_by_keys->size());
for (const std::vector<TypedValue> &group_by_key : *group_by_keys) {
- result->appendTypedValue(finalizeGroupInHashTable<HandleT, HashTableT>(hash_table,
- group_by_key));
+ result->appendTypedValue(
+ finalizeGroupInHashTableFast<HandleT, HashTableT>(
+ hash_table, group_by_key, index));
}
return result;
} else {
- IndirectColumnVector *result = new IndirectColumnVector(result_type,
- hash_table_concrete.numEntries());
+ IndirectColumnVector *result = new IndirectColumnVector(
+ result_type, hash_table_concrete.numEntries());
for (const std::vector<TypedValue> &group_by_key : *group_by_keys) {
- result->appendTypedValue(finalizeGroupInHashTable<HandleT, HashTableT>(hash_table,
- group_by_key));
+ result->appendTypedValue(
+ finalizeGroupInHashTableFast<HandleT, HashTableT>(
+ hash_table, group_by_key, index));
}
return result;
}
}
}
-template <typename HandleT,
- typename StateT,
- typename HashTableT>
-void AggregationConcreteHandle::mergeGroupByHashTablesHelper(
- const AggregationStateHashTableBase &source_hash_table,
- AggregationStateHashTableBase *destination_hash_table) const {
- const HandleT &handle = static_cast<const HandleT &>(*this);
- const HashTableT &source_hash_table_concrete =
- static_cast<const HashTableT &>(source_hash_table);
-
- HashTableMerger<HandleT, StateT, HashTableT> merger(handle,
- destination_hash_table);
-
- source_hash_table_concrete.forEachCompositeKey(&merger);
-}
-
} // namespace quickstep
#endif // QUICKSTEP_EXPRESSIONS_AGGREGATION_AGGREGATION_CONCRETE_HANDLE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationHandle.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandle.hpp b/expressions/aggregation/AggregationHandle.hpp
index 3d6e872..48ce7fe 100644
--- a/expressions/aggregation/AggregationHandle.hpp
+++ b/expressions/aggregation/AggregationHandle.hpp
@@ -40,7 +40,6 @@ class ValueAccessor;
* @{
*/
-
/**
* @brief Abstract base class for aggregation state.
**/
@@ -107,8 +106,7 @@ class AggregationHandle {
* @brief Virtual destructor.
*
**/
- virtual ~AggregationHandle() {
- }
+ virtual ~AggregationHandle() {}
/**
* @brief Create an initial "blank" state for this aggregation.
@@ -132,11 +130,11 @@ class AggregationHandle {
* A StorageBlob will be allocated to serve as the HashTable's
* in-memory storage.
* @return A new HashTable instance with the appropriate state type for this
- * aggregate as the ValueT.
+ * aggregate.
**/
virtual AggregationStateHashTableBase* createGroupByHashTable(
const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &group_by_types,
+ const std::vector<const Type *> &group_by_types,
const std::size_t estimated_num_groups,
StorageManager *storage_manager) const = 0;
@@ -261,14 +259,18 @@ class AggregationHandle {
* values returned in the ColumnVector. If this is already filled in,
* then this method will visit the GROUP BY keys in the exact order
* specified.
+ * @param index The index of the AggregationHandle to be finalized.
+ *
* @return A ColumnVector containing each group's finalized aggregate value.
**/
virtual ColumnVector* finalizeHashTable(
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const = 0;
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const = 0;
/**
- * @brief Create a new HashTable for the distinctify step for DISTINCT aggregation.
+ * @brief Create a new HashTable for the distinctify step for DISTINCT
+ * aggregation.
*
* Distinctify is the first step for DISTINCT aggregation. This step inserts
* the GROUP BY expression values and aggregation arguments together as keys
@@ -281,8 +283,8 @@ class AggregationHandle {
* we simply treat it as a special GROUP BY case that the GROUP BY expression
* vector is empty.
*
- * @param hash_table_impl The choice of which concrete HashTable implementation
- * to use.
+ * @param hash_table_impl The choice of which concrete HashTable
+ * implementation to use.
* @param key_types The types of the GROUP BY expressions together with the
* types of the aggregation arguments.
* @param estimated_num_distinct_keys The estimated number of distinct keys
@@ -291,14 +293,15 @@ class AggregationHandle {
* This is an estimate only, and the HashTable will be resized if it
* becomes over-full.
* @param storage_manager The StorageManager to use to create the HashTable.
- * A StorageBlob will be allocated to serve as the HashTable's in-memory
- * storage.
+ * A StorageBlob will be allocated to serve as the HashTable's
+ * in-memory storage.
+ *
* @return A new HashTable instance with the appropriate state type for this
- * aggregate as the ValueT.
+ * aggregate.
*/
virtual AggregationStateHashTableBase* createDistinctifyHashTable(
const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &key_types,
+ const std::vector<const Type *> &key_types,
const std::size_t estimated_num_distinct_keys,
StorageManager *storage_manager) const = 0;
@@ -306,15 +309,16 @@ class AggregationHandle {
* @brief Inserts the GROUP BY expressions and aggregation arguments together
* as keys into the distinctify hash table.
*
- * @param accessor The ValueAccessor that will be iterated over to read tuples.
+ * @param accessor The ValueAccessor that will be iterated over to read
+ * tuples.
* @param key_ids The attribute_ids of the GROUP BY expressions in accessor
* together with the attribute_ids of the arguments to this aggregate
* in accessor, in order.
- * @param distinctify_hash_table The HashTable to store the GROUP BY expressions
- * and the aggregation arguments together as hash table keys and a bool
- * constant \c true as hash table value (So the hash table actually
- * serves as a hash set). This should have been created by calling
- * createDistinctifyHashTable();
+ * @param distinctify_hash_table The HashTable to store the GROUP BY
+ * expressions and the aggregation arguments together as hash table
+ * keys and a bool constant \c true as hash table value (So the hash
+ * table actually serves as a hash set). This should have been created
+ * by calling createDistinctifyHashTable();
*/
virtual void insertValueAccessorIntoDistinctifyHashTable(
ValueAccessor *accessor,
@@ -339,32 +343,82 @@ class AggregationHandle {
* @brief Perform GROUP BY aggregation on the keys from the distinctify hash
* table and upserts states into the aggregation hash table.
*
- * @param distinctify_hash_table Hash table which stores the GROUP BY expression
- * values and aggregation arguments together as hash table keys.
+ * @param distinctify_hash_table Hash table which stores the GROUP BY
+ * expression values and aggregation arguments together as hash table
+ * keys.
* @param aggregation_hash_table The HashTable to upsert AggregationStates in.
* This should have been created by calling createGroupByHashTable() on
* this same AggregationHandle.
+ * @param index The index of the distinctify hash table for which we perform
+ * the DISTINCT aggregation.
*/
virtual void aggregateOnDistinctifyHashTableForGroupBy(
const AggregationStateHashTableBase &distinctify_hash_table,
- AggregationStateHashTableBase *aggregation_hash_table) const = 0;
+ AggregationStateHashTableBase *aggregation_hash_table,
+ std::size_t index) const = 0;
+
+ /**
+ * @brief Get the number of bytes needed to store the aggregation handle's
+ * state.
+ **/
+ virtual std::size_t getPayloadSize() const { return 1; }
/**
- * @brief Merge two GROUP BY hash tables in one.
+ * @brief Update the aggregation state for nullary aggregation function e.g.
+ * COUNT(*).
*
- * @note Both the hash tables should have the same structure.
+ * @note This function should be overloaded by those aggregation function
+ * which can perform nullary operations, e.g. COUNT.
+ *
+ * @param byte_ptr The pointer where the aggregation state is stored.
+ **/
+ virtual void updateStateNullary(std::uint8_t *byte_ptr) const {}
+
+ /**
+ * @brief Update the aggregation state for unary aggregation function e.g.
+ * SUM(a).
*
- * @param source_hash_table The hash table which will get merged.
- * @param destination_hash_table The hash table to which we will merge the
- * other hash table.
+ * @param argument The argument which will be used to update the state of the
+ * aggregation function.
+ * @param byte_ptr The pointer where the aggregation state is stored.
+ **/
+ virtual void updateStateUnary(const TypedValue &argument,
+ std::uint8_t *byte_ptr) const {}
+
+ /**
+ * @brief Merge two aggregation states for this aggregation handle.
+ *
+ * @note This function should be used with the hash table specifically meant
+ * for aggregations only.
+ *
+ * @param src A pointer to the source aggregation state.
+ * @param dst A pointer to the destination aggregation state.
+ **/
+ virtual void mergeStatesFast(const std::uint8_t *src,
+ std::uint8_t *dst) const {}
+
+ /**
+ * @brief Initialize the payload (in the aggregation hash table) for the given
+ * aggregation handle.
+ *
+ * @param byte_ptr The pointer to the aggregation state in the hash table.
+ **/
+ virtual void initPayload(std::uint8_t *byte_ptr) const {}
+
+ /**
+ * @brief Inform the aggregation handle to block (prohibit) updates on the
+ * aggregation state.
+ **/
+ virtual void blockUpdate() {}
+
+ /**
+ * @brief Inform the aggregation handle to allow updates on the
+ * aggregation state.
**/
- virtual void mergeGroupByHashTables(
- const AggregationStateHashTableBase &source_hash_table,
- AggregationStateHashTableBase *destination_hash_table) const = 0;
+ virtual void allowUpdate() {}
protected:
- AggregationHandle() {
- }
+ AggregationHandle() {}
private:
DISALLOW_COPY_AND_ASSIGN(AggregationHandle);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationHandleAvg.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleAvg.cpp b/expressions/aggregation/AggregationHandleAvg.cpp
index 4bd43d6..2481092 100644
--- a/expressions/aggregation/AggregationHandleAvg.cpp
+++ b/expressions/aggregation/AggregationHandleAvg.cpp
@@ -42,7 +42,7 @@ namespace quickstep {
class StorageManager;
AggregationHandleAvg::AggregationHandleAvg(const Type &type)
- : argument_type_(type) {
+ : argument_type_(type), block_update_(false) {
// We sum Int as Long and Float as Double so that we have more headroom when
// adding many values.
TypeID type_precision_id;
@@ -76,26 +76,24 @@ AggregationHandleAvg::AggregationHandleAvg(const Type &type)
// Divide operator for dividing sum by count to get final average.
divide_operator_.reset(
BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kDivide)
- .makeUncheckedBinaryOperatorForTypes(sum_type, TypeFactory::GetType(kDouble)));
+ .makeUncheckedBinaryOperatorForTypes(sum_type,
+ TypeFactory::GetType(kDouble)));
// Result is nullable, because AVG() over 0 values (or all NULL values) is
// NULL.
- result_type_
- = &(BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kDivide)
- .resultTypeForArgumentTypes(sum_type, TypeFactory::GetType(kDouble))
- ->getNullableVersion());
+ result_type_ =
+ &(BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kDivide)
+ .resultTypeForArgumentTypes(sum_type, TypeFactory::GetType(kDouble))
+ ->getNullableVersion());
}
AggregationStateHashTableBase* AggregationHandleAvg::createGroupByHashTable(
const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &group_by_types,
+ const std::vector<const Type *> &group_by_types,
const std::size_t estimated_num_groups,
StorageManager *storage_manager) const {
return AggregationStateHashTableFactory<AggregationStateAvg>::CreateResizable(
- hash_table_impl,
- group_by_types,
- estimated_num_groups,
- storage_manager);
+ hash_table_impl, group_by_types, estimated_num_groups, storage_manager);
}
AggregationState* AggregationHandleAvg::accumulateColumnVectors(
@@ -105,9 +103,8 @@ AggregationState* AggregationHandleAvg::accumulateColumnVectors(
AggregationStateAvg *state = new AggregationStateAvg(blank_state_);
std::size_t count = 0;
- state->sum_ = fast_add_operator_->accumulateColumnVector(state->sum_,
- *column_vectors.front(),
- &count);
+ state->sum_ = fast_add_operator_->accumulateColumnVector(
+ state->sum_, *column_vectors.front(), &count);
state->count_ = count;
return state;
}
@@ -121,10 +118,8 @@ AggregationState* AggregationHandleAvg::accumulateValueAccessor(
AggregationStateAvg *state = new AggregationStateAvg(blank_state_);
std::size_t count = 0;
- state->sum_ = fast_add_operator_->accumulateValueAccessor(state->sum_,
- accessor,
- accessor_ids.front(),
- &count);
+ state->sum_ = fast_add_operator_->accumulateValueAccessor(
+ state->sum_, accessor, accessor_ids.front(), &count);
state->count_ = count;
return state;
}
@@ -137,79 +132,74 @@ void AggregationHandleAvg::aggregateValueAccessorIntoHashTable(
AggregationStateHashTableBase *hash_table) const {
DCHECK_EQ(1u, argument_ids.size())
<< "Got wrong number of arguments for AVG: " << argument_ids.size();
-
- aggregateValueAccessorIntoHashTableUnaryHelper<
- AggregationHandleAvg,
- AggregationStateAvg,
- AggregationStateHashTable<AggregationStateAvg>>(
- accessor,
- argument_ids.front(),
- group_by_key_ids,
- blank_state_,
- hash_table);
}
-void AggregationHandleAvg::mergeStates(
- const AggregationState &source,
- AggregationState *destination) const {
- const AggregationStateAvg &avg_source = static_cast<const AggregationStateAvg&>(source);
- AggregationStateAvg *avg_destination = static_cast<AggregationStateAvg*>(destination);
+void AggregationHandleAvg::mergeStates(const AggregationState &source,
+ AggregationState *destination) const {
+ const AggregationStateAvg &avg_source =
+ static_cast<const AggregationStateAvg &>(source);
+ AggregationStateAvg *avg_destination =
+ static_cast<AggregationStateAvg *>(destination);
SpinMutexLock lock(avg_destination->mutex_);
avg_destination->count_ += avg_source.count_;
- avg_destination->sum_ = merge_add_operator_->applyToTypedValues(avg_destination->sum_,
- avg_source.sum_);
+ avg_destination->sum_ = merge_add_operator_->applyToTypedValues(
+ avg_destination->sum_, avg_source.sum_);
+}
+
+void AggregationHandleAvg::mergeStatesFast(const std::uint8_t *source,
+ std::uint8_t *destination) const {
+ const TypedValue *src_sum_ptr =
+ reinterpret_cast<const TypedValue *>(source + blank_state_.sum_offset_);
+ const std::int64_t *src_count_ptr = reinterpret_cast<const std::int64_t *>(
+ source + blank_state_.count_offset_);
+ TypedValue *dst_sum_ptr =
+ reinterpret_cast<TypedValue *>(destination + blank_state_.sum_offset_);
+ std::int64_t *dst_count_ptr = reinterpret_cast<std::int64_t *>(
+ destination + blank_state_.count_offset_);
+ (*dst_count_ptr) += (*src_count_ptr);
+ *dst_sum_ptr =
+ merge_add_operator_->applyToTypedValues(*dst_sum_ptr, *src_sum_ptr);
}
TypedValue AggregationHandleAvg::finalize(const AggregationState &state) const {
- const AggregationStateAvg &agg_state = static_cast<const AggregationStateAvg&>(state);
+ const AggregationStateAvg &agg_state =
+ static_cast<const AggregationStateAvg &>(state);
if (agg_state.count_ == 0) {
// AVG() over no values is NULL.
return result_type_->makeNullValue();
} else {
// Divide sum by count to get final average.
- return divide_operator_->applyToTypedValues(agg_state.sum_,
- TypedValue(static_cast<double>(agg_state.count_)));
+ return divide_operator_->applyToTypedValues(
+ agg_state.sum_, TypedValue(static_cast<double>(agg_state.count_)));
}
}
ColumnVector* AggregationHandleAvg::finalizeHashTable(
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const {
- return finalizeHashTableHelper<AggregationHandleAvg,
- AggregationStateHashTable<AggregationStateAvg>>(
- *result_type_,
- hash_table,
- group_by_keys);
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const {
+ return finalizeHashTableHelperFast<AggregationHandleAvg,
+ AggregationStateFastHashTable>(
+ *result_type_, hash_table, group_by_keys, index);
}
-AggregationState* AggregationHandleAvg::aggregateOnDistinctifyHashTableForSingle(
+AggregationState*
+AggregationHandleAvg::aggregateOnDistinctifyHashTableForSingle(
const AggregationStateHashTableBase &distinctify_hash_table) const {
- return aggregateOnDistinctifyHashTableForSingleUnaryHelper<
+ return aggregateOnDistinctifyHashTableForSingleUnaryHelperFast<
AggregationHandleAvg,
- AggregationStateAvg>(
- distinctify_hash_table);
+ AggregationStateAvg>(distinctify_hash_table);
}
void AggregationHandleAvg::aggregateOnDistinctifyHashTableForGroupBy(
const AggregationStateHashTableBase &distinctify_hash_table,
- AggregationStateHashTableBase *aggregation_hash_table) const {
- aggregateOnDistinctifyHashTableForGroupByUnaryHelper<
+ AggregationStateHashTableBase *aggregation_hash_table,
+ std::size_t index) const {
+ aggregateOnDistinctifyHashTableForGroupByUnaryHelperFast<
AggregationHandleAvg,
- AggregationStateAvg,
- AggregationStateHashTable<AggregationStateAvg>>(
- distinctify_hash_table,
- blank_state_,
- aggregation_hash_table);
-}
-
-void AggregationHandleAvg::mergeGroupByHashTables(
- const AggregationStateHashTableBase &source_hash_table,
- AggregationStateHashTableBase *destination_hash_table) const {
- mergeGroupByHashTablesHelper<AggregationHandleAvg,
- AggregationStateAvg,
- AggregationStateHashTable<AggregationStateAvg>>(
- source_hash_table, destination_hash_table);
+ AggregationStateFastHashTable>(
+ distinctify_hash_table, aggregation_hash_table, index);
}
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationHandleAvg.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleAvg.hpp b/expressions/aggregation/AggregationHandleAvg.hpp
index 31997b1..3c6e0c2 100644
--- a/expressions/aggregation/AggregationHandleAvg.hpp
+++ b/expressions/aggregation/AggregationHandleAvg.hpp
@@ -28,6 +28,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "expressions/aggregation/AggregationConcreteHandle.hpp"
#include "expressions/aggregation/AggregationHandle.hpp"
+#include "storage/FastHashTable.hpp"
#include "storage/HashTableBase.hpp"
#include "threading/SpinMutex.hpp"
#include "types/Type.hpp"
@@ -57,26 +58,45 @@ class AggregationStateAvg : public AggregationState {
*/
AggregationStateAvg(const AggregationStateAvg &orig)
: sum_(orig.sum_),
- count_(orig.count_) {
- }
+ count_(orig.count_),
+ sum_offset_(orig.sum_offset_),
+ count_offset_(orig.count_offset_),
+ mutex_offset_(orig.mutex_offset_) {}
/**
* @brief Destructor.
*/
~AggregationStateAvg() override {}
+ std::size_t getPayloadSize() const {
+ std::size_t p1 = reinterpret_cast<std::size_t>(&sum_);
+ std::size_t p2 = reinterpret_cast<std::size_t>(&mutex_);
+ return (p2 - p1);
+ }
+
+ const std::uint8_t *getPayloadAddress() const {
+ return reinterpret_cast<const uint8_t *>(&sum_);
+ }
+
private:
friend class AggregationHandleAvg;
AggregationStateAvg()
- : sum_(0), count_(0) {
- }
+ : sum_(0),
+ count_(0),
+ sum_offset_(0),
+ count_offset_(reinterpret_cast<std::uint8_t *>(&count_) -
+ reinterpret_cast<std::uint8_t *>(&sum_)),
+ mutex_offset_(reinterpret_cast<std::uint8_t *>(&mutex_) -
+ reinterpret_cast<std::uint8_t *>(&sum_)) {}
// TODO(shoban): We might want to specialize sum_ and count_ to use atomics
// for int types similar to in AggregationStateCount.
TypedValue sum_;
std::int64_t count_;
SpinMutex mutex_;
+
+ int sum_offset_, count_offset_, mutex_offset_;
};
/**
@@ -84,8 +104,7 @@ class AggregationStateAvg : public AggregationState {
**/
class AggregationHandleAvg : public AggregationConcreteHandle {
public:
- ~AggregationHandleAvg() override {
- }
+ ~AggregationHandleAvg() override {}
AggregationState* createInitialState() const override {
return new AggregationStateAvg(blank_state_);
@@ -93,14 +112,15 @@ class AggregationHandleAvg : public AggregationConcreteHandle {
AggregationStateHashTableBase* createGroupByHashTable(
const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &group_by_types,
+ const std::vector<const Type *> &group_by_types,
const std::size_t estimated_num_groups,
StorageManager *storage_manager) const override;
/**
* @brief Iterate method with average aggregation state.
**/
- inline void iterateUnaryInl(AggregationStateAvg *state, const TypedValue &value) const {
+ inline void iterateUnaryInl(AggregationStateAvg *state,
+ const TypedValue &value) const {
DCHECK(value.isPlausibleInstanceOf(argument_type_.getSignature()));
if (value.isNull()) return;
@@ -109,8 +129,41 @@ class AggregationHandleAvg : public AggregationConcreteHandle {
++state->count_;
}
+ inline void iterateUnaryInlFast(const TypedValue &value,
+ std::uint8_t *byte_ptr) const {
+ DCHECK(value.isPlausibleInstanceOf(argument_type_.getSignature()));
+ if (value.isNull()) return;
+ TypedValue *sum_ptr =
+ reinterpret_cast<TypedValue *>(byte_ptr + blank_state_.sum_offset_);
+ std::int64_t *count_ptr =
+ reinterpret_cast<std::int64_t *>(byte_ptr + blank_state_.count_offset_);
+ *sum_ptr = fast_add_operator_->applyToTypedValues(*sum_ptr, value);
+ ++(*count_ptr);
+ }
+
+ inline void updateStateUnary(const TypedValue &argument,
+ std::uint8_t *byte_ptr) const override {
+ if (!block_update_) {
+ iterateUnaryInlFast(argument, byte_ptr);
+ }
+ }
+
+ void blockUpdate() override { block_update_ = true; }
+
+ void allowUpdate() override { block_update_ = false; }
+
+ void initPayload(std::uint8_t *byte_ptr) const override {
+ TypedValue *sum_ptr =
+ reinterpret_cast<TypedValue *>(byte_ptr + blank_state_.sum_offset_);
+ std::int64_t *count_ptr =
+ reinterpret_cast<std::int64_t *>(byte_ptr + blank_state_.count_offset_);
+ *sum_ptr = blank_state_.sum_;
+ *count_ptr = blank_state_.count_;
+ }
+
AggregationState* accumulateColumnVectors(
- const std::vector<std::unique_ptr<ColumnVector>> &column_vectors) const override;
+ const std::vector<std::unique_ptr<ColumnVector>> &column_vectors)
+ const override;
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
AggregationState* accumulateValueAccessor(
@@ -127,40 +180,61 @@ class AggregationHandleAvg : public AggregationConcreteHandle {
void mergeStates(const AggregationState &source,
AggregationState *destination) const override;
+ void mergeStatesFast(const std::uint8_t *source,
+ std::uint8_t *destination) const override;
+
TypedValue finalize(const AggregationState &state) const override;
- inline TypedValue finalizeHashTableEntry(const AggregationState &state) const {
- const AggregationStateAvg &agg_state = static_cast<const AggregationStateAvg&>(state);
+ inline TypedValue finalizeHashTableEntry(
+ const AggregationState &state) const {
+ const AggregationStateAvg &agg_state =
+ static_cast<const AggregationStateAvg &>(state);
// TODO(chasseur): Could improve performance further if we made a special
// version of finalizeHashTable() that collects all the sums into one
// ColumnVector and all the counts into another and then applies
// '*divide_operator_' to them in bulk.
- return divide_operator_->applyToTypedValues(agg_state.sum_,
- TypedValue(static_cast<double>(agg_state.count_)));
+ return divide_operator_->applyToTypedValues(
+ agg_state.sum_, TypedValue(static_cast<double>(agg_state.count_)));
+ }
+
+ inline TypedValue finalizeHashTableEntryFast(
+ const std::uint8_t *byte_ptr) const {
+ std::uint8_t *value_ptr = const_cast<std::uint8_t *>(byte_ptr);
+ TypedValue *sum_ptr =
+ reinterpret_cast<TypedValue *>(value_ptr + blank_state_.sum_offset_);
+ std::int64_t *count_ptr = reinterpret_cast<std::int64_t *>(
+ value_ptr + blank_state_.count_offset_);
+ return divide_operator_->applyToTypedValues(
+ *sum_ptr, TypedValue(static_cast<double>(*count_ptr)));
}
ColumnVector* finalizeHashTable(
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const override;
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const override;
/**
- * @brief Implementation of AggregationHandle::aggregateOnDistinctifyHashTableForSingle()
+ * @brief Implementation of
+ * AggregationHandle::aggregateOnDistinctifyHashTableForSingle()
* for AVG aggregation.
*/
AggregationState* aggregateOnDistinctifyHashTableForSingle(
- const AggregationStateHashTableBase &distinctify_hash_table) const override;
+ const AggregationStateHashTableBase &distinctify_hash_table)
+ const override;
/**
- * @brief Implementation of AggregationHandle::aggregateOnDistinctifyHashTableForGroupBy()
+ * @brief Implementation of
+ * AggregationHandle::aggregateOnDistinctifyHashTableForGroupBy()
* for AVG aggregation.
*/
void aggregateOnDistinctifyHashTableForGroupBy(
const AggregationStateHashTableBase &distinctify_hash_table,
- AggregationStateHashTableBase *aggregation_hash_table) const override;
+ AggregationStateHashTableBase *aggregation_hash_table,
+ std::size_t index) const override;
- void mergeGroupByHashTables(
- const AggregationStateHashTableBase &source_hash_table,
- AggregationStateHashTableBase *destination_hash_table) const override;
+ std::size_t getPayloadSize() const override {
+ return blank_state_.getPayloadSize();
+ }
private:
friend class AggregateFunctionAvg;
@@ -179,6 +253,8 @@ class AggregationHandleAvg : public AggregationConcreteHandle {
std::unique_ptr<UncheckedBinaryOperator> merge_add_operator_;
std::unique_ptr<UncheckedBinaryOperator> divide_operator_;
+ bool block_update_;
+
DISALLOW_COPY_AND_ASSIGN(AggregationHandleAvg);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationHandleCount.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleCount.cpp b/expressions/aggregation/AggregationHandleCount.cpp
index dfcf131..034c942 100644
--- a/expressions/aggregation/AggregationHandleCount.cpp
+++ b/expressions/aggregation/AggregationHandleCount.cpp
@@ -49,48 +49,47 @@ class ValueAccessor;
template <bool count_star, bool nullable_type>
AggregationStateHashTableBase*
- AggregationHandleCount<count_star, nullable_type>::createGroupByHashTable(
- const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &group_by_types,
- const std::size_t estimated_num_groups,
- StorageManager *storage_manager) const {
- return AggregationStateHashTableFactory<AggregationStateCount>::CreateResizable(
- hash_table_impl,
- group_by_types,
- estimated_num_groups,
- storage_manager);
+AggregationHandleCount<count_star, nullable_type>::createGroupByHashTable(
+ const HashTableImplType hash_table_impl,
+ const std::vector<const Type *> &group_by_types,
+ const std::size_t estimated_num_groups,
+ StorageManager *storage_manager) const {
+ return AggregationStateHashTableFactory<
+ AggregationStateCount>::CreateResizable(hash_table_impl,
+ group_by_types,
+ estimated_num_groups,
+ storage_manager);
}
template <bool count_star, bool nullable_type>
AggregationState*
- AggregationHandleCount<count_star, nullable_type>::accumulateColumnVectors(
- const std::vector<std::unique_ptr<ColumnVector>> &column_vectors) const {
+AggregationHandleCount<count_star, nullable_type>::accumulateColumnVectors(
+ const std::vector<std::unique_ptr<ColumnVector>> &column_vectors) const {
DCHECK(!count_star)
<< "Called non-nullary accumulation method on an AggregationHandleCount "
<< "set up for nullary COUNT(*)";
DCHECK_EQ(1u, column_vectors.size())
- << "Got wrong number of ColumnVectors for COUNT: " << column_vectors.size();
+ << "Got wrong number of ColumnVectors for COUNT: "
+ << column_vectors.size();
std::size_t count = 0;
InvokeOnColumnVector(
*column_vectors.front(),
[&](const auto &column_vector) -> void { // NOLINT(build/c++11)
- if (nullable_type) {
- // TODO(shoban): Iterating over the ColumnVector is a rather slow way to
- // do this. We should look at extending the ColumnVector interface to do
- // a quick count of the non-null values (i.e. the length minus the
- // population count of the null bitmap). We should do something similar
- // for ValueAccessor too.
- for (std::size_t pos = 0;
- pos < column_vector.size();
- ++pos) {
- count += !column_vector.getTypedValue(pos).isNull();
- }
- } else {
- count = column_vector.size();
- }
- });
+ if (nullable_type) {
+ // TODO(shoban): Iterating over the ColumnVector is a rather slow way
+ // to do this. We should look at extending the ColumnVector interface
+ // to do a quick count of the non-null values (i.e. the length minus
+ // the population count of the null bitmap). We should do something
+ // similar for ValueAccessor too.
+ for (std::size_t pos = 0; pos < column_vector.size(); ++pos) {
+ count += !column_vector.getTypedValue(pos).isNull();
+ }
+ } else {
+ count = column_vector.size();
+ }
+ });
return new AggregationStateCount(count);
}
@@ -98,9 +97,9 @@ AggregationState*
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
template <bool count_star, bool nullable_type>
AggregationState*
- AggregationHandleCount<count_star, nullable_type>::accumulateValueAccessor(
- ValueAccessor *accessor,
- const std::vector<attribute_id> &accessor_ids) const {
+AggregationHandleCount<count_star, nullable_type>::accumulateValueAccessor(
+ ValueAccessor *accessor,
+ const std::vector<attribute_id> &accessor_ids) const {
DCHECK(!count_star)
<< "Called non-nullary accumulation method on an AggregationHandleCount "
<< "set up for nullary COUNT(*)";
@@ -113,108 +112,91 @@ AggregationState*
InvokeOnValueAccessorMaybeTupleIdSequenceAdapter(
accessor,
[&accessor_id, &count](auto *accessor) -> void { // NOLINT(build/c++11)
- if (nullable_type) {
- while (accessor->next()) {
- count += !accessor->getTypedValue(accessor_id).isNull();
- }
- } else {
- count = accessor->getNumTuples();
- }
- });
+ if (nullable_type) {
+ while (accessor->next()) {
+ count += !accessor->getTypedValue(accessor_id).isNull();
+ }
+ } else {
+ count = accessor->getNumTuples();
+ }
+ });
return new AggregationStateCount(count);
}
#endif
template <bool count_star, bool nullable_type>
- void AggregationHandleCount<count_star, nullable_type>::aggregateValueAccessorIntoHashTable(
+void AggregationHandleCount<count_star, nullable_type>::
+ aggregateValueAccessorIntoHashTable(
ValueAccessor *accessor,
const std::vector<attribute_id> &argument_ids,
const std::vector<attribute_id> &group_by_key_ids,
AggregationStateHashTableBase *hash_table) const {
if (count_star) {
DCHECK_EQ(0u, argument_ids.size())
- << "Got wrong number of arguments for COUNT(*): " << argument_ids.size();
- aggregateValueAccessorIntoHashTableNullaryHelper<
- AggregationHandleCount<count_star, nullable_type>,
- AggregationStateCount,
- AggregationStateHashTable<AggregationStateCount>>(
- accessor,
- group_by_key_ids,
- AggregationStateCount(),
- hash_table);
+ << "Got wrong number of arguments for COUNT(*): "
+ << argument_ids.size();
} else {
DCHECK_EQ(1u, argument_ids.size())
<< "Got wrong number of arguments for COUNT: " << argument_ids.size();
- aggregateValueAccessorIntoHashTableUnaryHelper<
- AggregationHandleCount<count_star, nullable_type>,
- AggregationStateCount,
- AggregationStateHashTable<AggregationStateCount>>(
- accessor,
- argument_ids.front(),
- group_by_key_ids,
- AggregationStateCount(),
- hash_table);
}
}
template <bool count_star, bool nullable_type>
- void AggregationHandleCount<count_star, nullable_type>::mergeStates(
- const AggregationState &source,
- AggregationState *destination) const {
- const AggregationStateCount &count_source = static_cast<const AggregationStateCount&>(source);
- AggregationStateCount *count_destination = static_cast<AggregationStateCount*>(destination);
-
- count_destination->count_.fetch_add(count_source.count_.load(std::memory_order_relaxed),
- std::memory_order_relaxed);
+void AggregationHandleCount<count_star, nullable_type>::mergeStates(
+ const AggregationState &source, AggregationState *destination) const {
+ const AggregationStateCount &count_source =
+ static_cast<const AggregationStateCount &>(source);
+ AggregationStateCount *count_destination =
+ static_cast<AggregationStateCount *>(destination);
+
+ count_destination->count_.fetch_add(
+ count_source.count_.load(std::memory_order_relaxed),
+ std::memory_order_relaxed);
}
template <bool count_star, bool nullable_type>
- ColumnVector* AggregationHandleCount<count_star, nullable_type>::finalizeHashTable(
- const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const {
- return finalizeHashTableHelper<AggregationHandleCount<count_star, nullable_type>,
- AggregationStateHashTable<AggregationStateCount>>(
- TypeFactory::GetType(kLong),
- hash_table,
- group_by_keys);
+void AggregationHandleCount<count_star, nullable_type>::mergeStatesFast(
+ const std::uint8_t *source, std::uint8_t *destination) const {
+ const std::int64_t *src_count_ptr =
+ reinterpret_cast<const std::int64_t *>(source);
+ std::int64_t *dst_count_ptr = reinterpret_cast<std::int64_t *>(destination);
+ (*dst_count_ptr) += (*src_count_ptr);
}
template <bool count_star, bool nullable_type>
-AggregationState* AggregationHandleCount<count_star, nullable_type>
- ::aggregateOnDistinctifyHashTableForSingle(
- const AggregationStateHashTableBase &distinctify_hash_table) const {
- DCHECK_EQ(count_star, false);
- return aggregateOnDistinctifyHashTableForSingleUnaryHelper<
+ColumnVector*
+AggregationHandleCount<count_star, nullable_type>::finalizeHashTable(
+ const AggregationStateHashTableBase &hash_table,
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const {
+ return finalizeHashTableHelperFast<
AggregationHandleCount<count_star, nullable_type>,
- AggregationStateCount>(
- distinctify_hash_table);
+ AggregationStateFastHashTable>(
+ TypeFactory::GetType(kLong), hash_table, group_by_keys, index);
}
template <bool count_star, bool nullable_type>
-void AggregationHandleCount<count_star, nullable_type>
- ::aggregateOnDistinctifyHashTableForGroupBy(
- const AggregationStateHashTableBase &distinctify_hash_table,
- AggregationStateHashTableBase *aggregation_hash_table) const {
+AggregationState* AggregationHandleCount<count_star, nullable_type>::
+ aggregateOnDistinctifyHashTableForSingle(
+ const AggregationStateHashTableBase &distinctify_hash_table) const {
DCHECK_EQ(count_star, false);
- aggregateOnDistinctifyHashTableForGroupByUnaryHelper<
+ return aggregateOnDistinctifyHashTableForSingleUnaryHelperFast<
AggregationHandleCount<count_star, nullable_type>,
- AggregationStateCount,
- AggregationStateHashTable<AggregationStateCount>>(
- distinctify_hash_table,
- AggregationStateCount(),
- aggregation_hash_table);
+ AggregationStateCount>(distinctify_hash_table);
}
template <bool count_star, bool nullable_type>
-void AggregationHandleCount<count_star, nullable_type>::mergeGroupByHashTables(
- const AggregationStateHashTableBase &source_hash_table,
- AggregationStateHashTableBase *destination_hash_table) const {
- mergeGroupByHashTablesHelper<
- AggregationHandleCount,
- AggregationStateCount,
- AggregationStateHashTable<AggregationStateCount>>(source_hash_table,
- destination_hash_table);
+void AggregationHandleCount<count_star, nullable_type>::
+ aggregateOnDistinctifyHashTableForGroupBy(
+ const AggregationStateHashTableBase &distinctify_hash_table,
+ AggregationStateHashTableBase *aggregation_hash_table,
+ std::size_t index) const {
+ DCHECK_EQ(count_star, false);
+ aggregateOnDistinctifyHashTableForGroupByUnaryHelperFast<
+ AggregationHandleCount<count_star, nullable_type>,
+ AggregationStateFastHashTable>(
+ distinctify_hash_table, aggregation_hash_table, index);
}
// Explicitly instantiate and compile in the different versions of
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationHandleCount.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleCount.hpp b/expressions/aggregation/AggregationHandleCount.hpp
index 1cd5bda..6aab0cd 100644
--- a/expressions/aggregation/AggregationHandleCount.hpp
+++ b/expressions/aggregation/AggregationHandleCount.hpp
@@ -29,6 +29,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "expressions/aggregation/AggregationConcreteHandle.hpp"
#include "expressions/aggregation/AggregationHandle.hpp"
+#include "storage/FastHashTable.hpp"
#include "storage/HashTableBase.hpp"
#include "types/TypedValue.hpp"
#include "utility/Macros.hpp"
@@ -40,7 +41,8 @@ class StorageManager;
class Type;
class ValueAccessor;
-template <bool, bool> class AggregationHandleCount;
+template <bool, bool>
+class AggregationHandleCount;
/** \addtogroup Expressions
* @{
@@ -62,19 +64,22 @@ class AggregationStateCount : public AggregationState {
*/
~AggregationStateCount() override {}
+ std::size_t getPayloadSize() const { return sizeof(count_); }
+
+ const std::uint8_t* getPayloadAddress() const {
+ return reinterpret_cast<const uint8_t *>(&count_);
+ }
+
private:
friend class AggregationHandleCount<false, false>;
friend class AggregationHandleCount<false, true>;
friend class AggregationHandleCount<true, false>;
friend class AggregationHandleCount<true, true>;
- AggregationStateCount()
- : count_(0) {
- }
+ AggregationStateCount() : count_(0) {}
explicit AggregationStateCount(const std::int64_t initial_count)
- : count_(initial_count) {
- }
+ : count_(initial_count) {}
std::atomic<std::int64_t> count_;
};
@@ -91,8 +96,7 @@ class AggregationStateCount : public AggregationState {
template <bool count_star, bool nullable_type>
class AggregationHandleCount : public AggregationConcreteHandle {
public:
- ~AggregationHandleCount() override {
- }
+ ~AggregationHandleCount() override {}
AggregationState* createInitialState() const override {
return new AggregationStateCount();
@@ -100,7 +104,7 @@ class AggregationHandleCount : public AggregationConcreteHandle {
AggregationStateHashTableBase* createGroupByHashTable(
const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &group_by_types,
+ const std::vector<const Type *> &group_by_types,
const std::size_t estimated_num_groups,
StorageManager *storage_manager) const override;
@@ -108,21 +112,59 @@ class AggregationHandleCount : public AggregationConcreteHandle {
state->count_.fetch_add(1, std::memory_order_relaxed);
}
+ inline void iterateNullaryInlFast(std::uint8_t *byte_ptr) const {
+ std::int64_t *count_ptr = reinterpret_cast<std::int64_t *>(byte_ptr);
+ (*count_ptr)++;
+ }
+
/**
* @brief Iterate with count aggregation state.
*/
- inline void iterateUnaryInl(AggregationStateCount *state, const TypedValue &value) const {
+ inline void iterateUnaryInl(AggregationStateCount *state,
+ const TypedValue &value) const {
if ((!nullable_type) || (!value.isNull())) {
state->count_.fetch_add(1, std::memory_order_relaxed);
}
}
- AggregationState* accumulateNullary(const std::size_t num_tuples) const override {
+ inline void iterateUnaryInlFast(const TypedValue &value,
+ std::uint8_t *byte_ptr) const {
+ if ((!nullable_type) || (!value.isNull())) {
+ std::int64_t *count_ptr = reinterpret_cast<std::int64_t *>(byte_ptr);
+ (*count_ptr)++;
+ }
+ }
+
+ inline void updateStateUnary(const TypedValue &argument,
+ std::uint8_t *byte_ptr) const override {
+ if (!block_update_) {
+ iterateUnaryInlFast(argument, byte_ptr);
+ }
+ }
+
+ inline void updateStateNullary(std::uint8_t *byte_ptr) const override {
+ if (!block_update_) {
+ iterateNullaryInlFast(byte_ptr);
+ }
+ }
+
+ void blockUpdate() override { block_update_ = true; }
+
+ void allowUpdate() override { block_update_ = false; }
+
+ void initPayload(std::uint8_t *byte_ptr) const override {
+ std::int64_t *count_ptr = reinterpret_cast<std::int64_t *>(byte_ptr);
+ *count_ptr = 0;
+ }
+
+ AggregationState* accumulateNullary(
+ const std::size_t num_tuples) const override {
return new AggregationStateCount(num_tuples);
}
AggregationState* accumulateColumnVectors(
- const std::vector<std::unique_ptr<ColumnVector>> &column_vectors) const override;
+ const std::vector<std::unique_ptr<ColumnVector>> &column_vectors)
+ const override;
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
AggregationState* accumulateValueAccessor(
@@ -139,36 +181,54 @@ class AggregationHandleCount : public AggregationConcreteHandle {
void mergeStates(const AggregationState &source,
AggregationState *destination) const override;
+ void mergeStatesFast(const std::uint8_t *source,
+ std::uint8_t *destination) const override;
+
TypedValue finalize(const AggregationState &state) const override {
- return TypedValue(static_cast<const AggregationStateCount&>(state).count_.load(std::memory_order_relaxed));
+ return TypedValue(
+ static_cast<const AggregationStateCount &>(state).count_.load(
+ std::memory_order_relaxed));
+ }
+
+ inline TypedValue finalizeHashTableEntry(
+ const AggregationState &state) const {
+ return TypedValue(
+ static_cast<const AggregationStateCount &>(state).count_.load(
+ std::memory_order_relaxed));
}
- inline TypedValue finalizeHashTableEntry(const AggregationState &state) const {
- return TypedValue(static_cast<const AggregationStateCount&>(state).count_.load(std::memory_order_relaxed));
+ inline TypedValue finalizeHashTableEntryFast(
+ const std::uint8_t *byte_ptr) const {
+ const std::int64_t *count_ptr =
+ reinterpret_cast<const std::int64_t *>(byte_ptr);
+ return TypedValue(*count_ptr);
}
ColumnVector* finalizeHashTable(
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const override;
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const override;
/**
- * @brief Implementation of AggregationHandle::aggregateOnDistinctifyHashTableForSingle()
+ * @brief Implementation of
+ * AggregationHandle::aggregateOnDistinctifyHashTableForSingle()
* for SUM aggregation.
*/
AggregationState* aggregateOnDistinctifyHashTableForSingle(
- const AggregationStateHashTableBase &distinctify_hash_table) const override;
+ const AggregationStateHashTableBase &distinctify_hash_table)
+ const override;
/**
- * @brief Implementation of AggregationHandle::aggregateOnDistinctifyHashTableForGroupBy()
+ * @brief Implementation of
+ * AggregationHandle::aggregateOnDistinctifyHashTableForGroupBy()
* for SUM aggregation.
*/
void aggregateOnDistinctifyHashTableForGroupBy(
const AggregationStateHashTableBase &distinctify_hash_table,
- AggregationStateHashTableBase *aggregation_hash_table) const override;
+ AggregationStateHashTableBase *aggregation_hash_table,
+ std::size_t index) const override;
- void mergeGroupByHashTables(
- const AggregationStateHashTableBase &source_hash_table,
- AggregationStateHashTableBase *destination_hash_table) const override;
+ std::size_t getPayloadSize() const override { return sizeof(std::int64_t); }
private:
friend class AggregateFunctionCount;
@@ -176,8 +236,9 @@ class AggregationHandleCount : public AggregationConcreteHandle {
/**
* @brief Constructor.
**/
- AggregationHandleCount() {
- }
+ AggregationHandleCount() : block_update_(false) {}
+
+ bool block_update_;
DISALLOW_COPY_AND_ASSIGN(AggregationHandleCount);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationHandleDistinct.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleDistinct.cpp b/expressions/aggregation/AggregationHandleDistinct.cpp
index 68fcd4c..0dc8b56 100644
--- a/expressions/aggregation/AggregationHandleDistinct.cpp
+++ b/expressions/aggregation/AggregationHandleDistinct.cpp
@@ -65,14 +65,15 @@ void AggregationHandleDistinct::aggregateValueAccessorIntoHashTable(
ColumnVector* AggregationHandleDistinct::finalizeHashTable(
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const {
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const {
DCHECK(group_by_keys->empty());
const auto keys_retriever = [&group_by_keys](std::vector<TypedValue> &group_by_key,
const bool &dumb_placeholder) -> void {
group_by_keys->emplace_back(std::move(group_by_key));
};
- static_cast<const AggregationStateHashTable<bool>&>(hash_table).forEachCompositeKey(&keys_retriever);
+ static_cast<const AggregationStateFastHashTable&>(hash_table).forEachCompositeKeyFast(&keys_retriever);
return nullptr;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationHandleDistinct.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleDistinct.hpp b/expressions/aggregation/AggregationHandleDistinct.hpp
index 8524fcc..838bfdd 100644
--- a/expressions/aggregation/AggregationHandleDistinct.hpp
+++ b/expressions/aggregation/AggregationHandleDistinct.hpp
@@ -49,27 +49,32 @@ class AggregationHandleDistinct : public AggregationConcreteHandle {
/**
* @brief Constructor.
**/
- AggregationHandleDistinct() {
- }
+ AggregationHandleDistinct() {}
AggregationState* createInitialState() const override {
- LOG(FATAL) << "AggregationHandleDistinct does not support createInitialState().";
+ LOG(FATAL)
+ << "AggregationHandleDistinct does not support createInitialState().";
}
- AggregationState* accumulateNullary(const std::size_t num_tuples) const override {
- LOG(FATAL) << "AggregationHandleDistinct does not support accumulateNullary().";
+ AggregationState* accumulateNullary(
+ const std::size_t num_tuples) const override {
+ LOG(FATAL)
+ << "AggregationHandleDistinct does not support accumulateNullary().";
}
AggregationState* accumulateColumnVectors(
- const std::vector<std::unique_ptr<ColumnVector>> &column_vectors) const override {
- LOG(FATAL) << "AggregationHandleDistinct does not support accumulateColumnVectors().";
+ const std::vector<std::unique_ptr<ColumnVector>> &column_vectors)
+ const override {
+ LOG(FATAL) << "AggregationHandleDistinct does not support "
+ "accumulateColumnVectors().";
}
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
AggregationState* accumulateValueAccessor(
ValueAccessor *accessor,
const std::vector<attribute_id> &accessor_ids) const override {
- LOG(FATAL) << "AggregationHandleDistinct does not support accumulateValueAccessor().";
+ LOG(FATAL) << "AggregationHandleDistinct does not support "
+ "accumulateValueAccessor().";
}
#endif
@@ -83,21 +88,23 @@ class AggregationHandleDistinct : public AggregationConcreteHandle {
}
AggregationState* aggregateOnDistinctifyHashTableForSingle(
- const AggregationStateHashTableBase &distinctify_hash_table) const override {
+ const AggregationStateHashTableBase &distinctify_hash_table)
+ const override {
LOG(FATAL) << "AggregationHandleDistinct does not support "
<< "aggregateOnDistinctifyHashTableForSingle().";
}
void aggregateOnDistinctifyHashTableForGroupBy(
const AggregationStateHashTableBase &distinctify_hash_table,
- AggregationStateHashTableBase *groupby_hash_table) const override {
+ AggregationStateHashTableBase *groupby_hash_table,
+ std::size_t index) const override {
LOG(FATAL) << "AggregationHandleDistinct does not support "
<< "aggregateOnDistinctifyHashTableForGroupBy().";
}
AggregationStateHashTableBase* createGroupByHashTable(
const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &group_by_types,
+ const std::vector<const Type *> &group_by_types,
const std::size_t estimated_num_groups,
StorageManager *storage_manager) const override;
@@ -109,14 +116,8 @@ class AggregationHandleDistinct : public AggregationConcreteHandle {
ColumnVector* finalizeHashTable(
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const override;
-
- void mergeGroupByHashTables(
- const AggregationStateHashTableBase &source_hash_table,
- AggregationStateHashTableBase *destination_hash_table) const override {
- LOG(FATAL)
- << "AggregationHandleDistinct does not support mergeGroupByHashTables";
- }
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const override;
private:
DISALLOW_COPY_AND_ASSIGN(AggregationHandleDistinct);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationHandleMax.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMax.cpp b/expressions/aggregation/AggregationHandleMax.cpp
index 435f5f2..c2d571b 100644
--- a/expressions/aggregation/AggregationHandleMax.cpp
+++ b/expressions/aggregation/AggregationHandleMax.cpp
@@ -39,22 +39,19 @@ namespace quickstep {
class StorageManager;
AggregationHandleMax::AggregationHandleMax(const Type &type)
- : type_(type) {
- fast_comparator_.reset(ComparisonFactory::GetComparison(ComparisonID::kGreater)
- .makeUncheckedComparatorForTypes(type,
- type.getNonNullableVersion()));
+ : type_(type), block_update_(false) {
+ fast_comparator_.reset(
+ ComparisonFactory::GetComparison(ComparisonID::kGreater)
+ .makeUncheckedComparatorForTypes(type, type.getNonNullableVersion()));
}
AggregationStateHashTableBase* AggregationHandleMax::createGroupByHashTable(
const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &group_by_types,
+ const std::vector<const Type *> &group_by_types,
const std::size_t estimated_num_groups,
StorageManager *storage_manager) const {
return AggregationStateHashTableFactory<AggregationStateMax>::CreateResizable(
- hash_table_impl,
- group_by_types,
- estimated_num_groups,
- storage_manager);
+ hash_table_impl, group_by_types, estimated_num_groups, storage_manager);
}
AggregationState* AggregationHandleMax::accumulateColumnVectors(
@@ -62,9 +59,8 @@ AggregationState* AggregationHandleMax::accumulateColumnVectors(
DCHECK_EQ(1u, column_vectors.size())
<< "Got wrong number of ColumnVectors for MAX: " << column_vectors.size();
- return new AggregationStateMax(
- fast_comparator_->accumulateColumnVector(type_.getNullableVersion().makeNullValue(),
- *column_vectors.front()));
+ return new AggregationStateMax(fast_comparator_->accumulateColumnVector(
+ type_.getNullableVersion().makeNullValue(), *column_vectors.front()));
}
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -74,10 +70,10 @@ AggregationState* AggregationHandleMax::accumulateValueAccessor(
DCHECK_EQ(1u, accessor_ids.size())
<< "Got wrong number of attributes for MAX: " << accessor_ids.size();
- return new AggregationStateMax(
- fast_comparator_->accumulateValueAccessor(type_.getNullableVersion().makeNullValue(),
- accessor,
- accessor_ids.front()));
+ return new AggregationStateMax(fast_comparator_->accumulateValueAccessor(
+ type_.getNullableVersion().makeNullValue(),
+ accessor,
+ accessor_ids.front()));
}
#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -88,66 +84,54 @@ void AggregationHandleMax::aggregateValueAccessorIntoHashTable(
AggregationStateHashTableBase *hash_table) const {
DCHECK_EQ(1u, argument_ids.size())
<< "Got wrong number of arguments for MAX: " << argument_ids.size();
-
- aggregateValueAccessorIntoHashTableUnaryHelper<
- AggregationHandleMax,
- AggregationStateMax,
- AggregationStateHashTable<AggregationStateMax>>(
- accessor,
- argument_ids.front(),
- group_by_key_ids,
- AggregationStateMax(type_),
- hash_table);
}
-void AggregationHandleMax::mergeStates(
- const AggregationState &source,
- AggregationState *destination) const {
- const AggregationStateMax &max_source = static_cast<const AggregationStateMax&>(source);
- AggregationStateMax *max_destination = static_cast<AggregationStateMax*>(destination);
+void AggregationHandleMax::mergeStates(const AggregationState &source,
+ AggregationState *destination) const {
+ const AggregationStateMax &max_source =
+ static_cast<const AggregationStateMax &>(source);
+ AggregationStateMax *max_destination =
+ static_cast<AggregationStateMax *>(destination);
if (!max_source.max_.isNull()) {
compareAndUpdate(max_destination, max_source.max_);
}
}
+void AggregationHandleMax::mergeStatesFast(const std::uint8_t *source,
+ std::uint8_t *destination) const {
+ const TypedValue *src_max_ptr = reinterpret_cast<const TypedValue *>(source);
+ TypedValue *dst_max_ptr = reinterpret_cast<TypedValue *>(destination);
+ if (!(src_max_ptr->isNull())) {
+ compareAndUpdateFast(dst_max_ptr, *src_max_ptr);
+ }
+}
+
ColumnVector* AggregationHandleMax::finalizeHashTable(
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const {
- return finalizeHashTableHelper<AggregationHandleMax,
- AggregationStateHashTable<AggregationStateMax>>(
- type_.getNullableVersion(),
- hash_table,
- group_by_keys);
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const {
+ return finalizeHashTableHelperFast<AggregationHandleMax,
+ AggregationStateFastHashTable>(
+ type_.getNullableVersion(), hash_table, group_by_keys, index);
}
-AggregationState* AggregationHandleMax::aggregateOnDistinctifyHashTableForSingle(
+AggregationState*
+AggregationHandleMax::aggregateOnDistinctifyHashTableForSingle(
const AggregationStateHashTableBase &distinctify_hash_table) const {
- return aggregateOnDistinctifyHashTableForSingleUnaryHelper<
+ return aggregateOnDistinctifyHashTableForSingleUnaryHelperFast<
AggregationHandleMax,
- AggregationStateMax>(
- distinctify_hash_table);
+ AggregationStateMax>(distinctify_hash_table);
}
void AggregationHandleMax::aggregateOnDistinctifyHashTableForGroupBy(
const AggregationStateHashTableBase &distinctify_hash_table,
- AggregationStateHashTableBase *aggregation_hash_table) const {
- aggregateOnDistinctifyHashTableForGroupByUnaryHelper<
+ AggregationStateHashTableBase *aggregation_hash_table,
+ std::size_t index) const {
+ aggregateOnDistinctifyHashTableForGroupByUnaryHelperFast<
AggregationHandleMax,
- AggregationStateMax,
- AggregationStateHashTable<AggregationStateMax>>(
- distinctify_hash_table,
- AggregationStateMax(type_),
- aggregation_hash_table);
-}
-
-void AggregationHandleMax::mergeGroupByHashTables(
- const AggregationStateHashTableBase &source_hash_table,
- AggregationStateHashTableBase *destination_hash_table) const {
- mergeGroupByHashTablesHelper<AggregationHandleMax,
- AggregationStateMax,
- AggregationStateHashTable<AggregationStateMax>>(
- source_hash_table, destination_hash_table);
+ AggregationStateFastHashTable>(
+ distinctify_hash_table, aggregation_hash_table, index);
}
} // namespace quickstep
[17/17] incubator-quickstep git commit: Generator updates
Posted by ji...@apache.org.
Generator updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/396f8576
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/396f8576
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/396f8576
Branch: refs/heads/lip-refactor
Commit: 396f8576a48e5ad913c3df011bf15abe4d6ad001
Parents: 18e7209
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Tue Sep 20 16:19:30 2016 -0500
Committer: Jianqiao Zhu <ji...@cs.wisc.edu>
Committed: Tue Oct 4 23:25:23 2016 -0500
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 7 +-
query_execution/QueryContext.cpp | 15 +-
query_execution/QueryContext.hpp | 148 ++++++++++++-------
query_execution/QueryContext.proto | 25 ++--
query_optimizer/CMakeLists.txt | 20 ++-
query_optimizer/ExecutionGenerator.cpp | 20 ++-
query_optimizer/ExecutionGenerator.hpp | 4 +
query_optimizer/LIPFilterGenerator.cpp | 130 ++++++++++++++++
query_optimizer/LIPFilterGenerator.hpp | 95 +++++++++++-
query_optimizer/PhysicalGenerator.cpp | 1 -
query_optimizer/physical/CMakeLists.txt | 9 +-
.../physical/LIPFilterConfiguration.hpp | 21 ++-
query_optimizer/rules/AttachLIPFilters.cpp | 6 +-
query_optimizer/rules/CMakeLists.txt | 4 +-
storage/HashTableFactory.hpp | 5 +-
utility/CMakeLists.txt | 11 +-
utility/DAG.hpp | 6 +-
utility/LIPFilter.cpp | 24 ---
utility/LIPFilter.hpp | 44 ------
utility/LIPFilterAdapter.hpp | 0
utility/LIPFilterBuilder.hpp | 0
utility/PlanVisualizer.cpp | 4 +-
utility/lip_filter/CMakeLists.txt | 49 ++++++
utility/lip_filter/LIPFilter.cpp | 24 +++
utility/lip_filter/LIPFilter.hpp | 57 +++++++
utility/lip_filter/LIPFilter.proto | 53 +++++++
utility/lip_filter/LIPFilterAdaptiveProber.hpp | 87 +++++++++++
utility/lip_filter/LIPFilterBuilder.hpp | 72 +++++++++
utility/lip_filter/LIPFilterDeploymentInfo.hpp | 69 +++++++++
utility/lip_filter/LIPFilterFactory.cpp | 0
utility/lip_filter/LIPFilterFactory.hpp | 47 ++++++
31 files changed, 869 insertions(+), 188 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 1b27194..58e5761 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -189,11 +189,11 @@ target_link_libraries(quickstep_queryexecution_QueryContext
quickstep_storage_WindowAggregationOperationState
quickstep_types_TypedValue
quickstep_types_containers_Tuple
- quickstep_utility_BloomFilter
quickstep_utility_Macros
- quickstep_utility_SortConfiguration)
+ quickstep_utility_SortConfiguration
+ quickstep_utility_lipfilter_LIPFilter
+ quickstep_utility_lipfilter_LIPFilterDeploymentInfo)
target_link_libraries(quickstep_queryexecution_QueryContext_proto
- quickstep_utility_BloomFilter_proto
quickstep_expressions_Expressions_proto
quickstep_expressions_tablegenerator_GeneratorFunction_proto
quickstep_storage_AggregationOperationState_proto
@@ -202,6 +202,7 @@ target_link_libraries(quickstep_queryexecution_QueryContext_proto
quickstep_storage_WindowAggregationOperationState_proto
quickstep_types_containers_Tuple_proto
quickstep_utility_SortConfiguration_proto
+ quickstep_utility_lipfilter_LIPFilter_proto
${PROTOBUF_LIBRARY})
target_link_libraries(quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_catalog_Catalog_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_execution/QueryContext.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.cpp b/query_execution/QueryContext.cpp
index 2572e18..47408d4 100644
--- a/query_execution/QueryContext.cpp
+++ b/query_execution/QueryContext.cpp
@@ -39,7 +39,7 @@
#include "storage/InsertDestination.pb.h"
#include "types/TypedValue.hpp"
#include "types/containers/Tuple.hpp"
-#include "utility/BloomFilter.hpp"
+#include "utility/lip_filter/LIPFilter.hpp"
#include "utility/SortConfiguration.hpp"
#include "glog/logging.h"
@@ -68,10 +68,6 @@ QueryContext::QueryContext(const serialization::QueryContext &proto,
storage_manager));
}
- for (int i = 0; i < proto.bloom_filters_size(); ++i) {
- bloom_filters_.emplace_back(new BloomFilter(proto.bloom_filters(i)));
- }
-
for (int i = 0; i < proto.generator_functions_size(); ++i) {
const GeneratorFunctionHandle *func_handle =
GeneratorFunctionFactory::Instance().reconstructFromProto(proto.generator_functions(i));
@@ -83,8 +79,7 @@ QueryContext::QueryContext(const serialization::QueryContext &proto,
for (int i = 0; i < proto.join_hash_tables_size(); ++i) {
join_hash_tables_.emplace_back(
JoinHashTableFactory::CreateResizableFromProto(proto.join_hash_tables(i),
- storage_manager,
- bloom_filters_));
+ storage_manager));
}
for (int i = 0; i < proto.insert_destinations_size(); ++i) {
@@ -157,12 +152,6 @@ bool QueryContext::ProtoIsValid(const serialization::QueryContext &proto,
}
}
- for (int i = 0; i < proto.bloom_filters_size(); ++i) {
- if (!BloomFilter::ProtoIsValid(proto.bloom_filters(i))) {
- return false;
- }
- }
-
// Each GeneratorFunctionHandle object is serialized as a function name with
// a list of arguments. Here checks that the arguments are valid TypedValue's.
for (int i = 0; i < proto.generator_functions_size(); ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_execution/QueryContext.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp
index 393b55e..3e287c0 100644
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@ -35,7 +35,8 @@
#include "storage/InsertDestination.hpp"
#include "storage/WindowAggregationOperationState.hpp"
#include "types/containers/Tuple.hpp"
-#include "utility/BloomFilter.hpp"
+#include "utility/lip_filter/LIPFilter.hpp"
+#include "utility/lip_filter/LIPFilterDeploymentInfo.hpp"
#include "utility/Macros.hpp"
#include "utility/SortConfiguration.hpp"
@@ -67,11 +68,6 @@ class QueryContext {
typedef std::uint32_t aggregation_state_id;
/**
- * @brief A unique identifier for a BloomFilter per query.
- **/
- typedef std::uint32_t bloom_filter_id;
-
- /**
* @brief A unique identifier for a GeneratorFunctionHandle per query.
**/
typedef std::uint32_t generator_function_id;
@@ -90,6 +86,16 @@ class QueryContext {
typedef std::uint32_t join_hash_table_id;
/**
+ * @brief A unique identifier for a LIPFilter per query.
+ **/
+ typedef std::uint32_t lip_filter_id;
+
+ /**
+ * @brief A unique identifier for a LIPFilterDeploymentInfo per query.
+ **/
+ typedef std::uint32_t lip_filter_deployment_info_id;
+
+ /**
* @brief A unique identifier for a Predicate per query.
*
* @note A negative value indicates a null Predicate.
@@ -193,52 +199,6 @@ class QueryContext {
}
/**
- * @brief Whether the given BloomFilter id is valid.
- *
- * @param id The BloomFilter id.
- *
- * @return True if valid, otherwise false.
- **/
- bool isValidBloomFilterId(const bloom_filter_id id) const {
- return id < bloom_filters_.size();
- }
-
- /**
- * @brief Get a mutable reference to the BloomFilter.
- *
- * @param id The BloomFilter id.
- *
- * @return The BloomFilter, already created in the constructor.
- **/
- inline BloomFilter* getBloomFilterMutable(const bloom_filter_id id) {
- DCHECK_LT(id, bloom_filters_.size());
- return bloom_filters_[id].get();
- }
-
- /**
- * @brief Get a constant pointer to the BloomFilter.
- *
- * @param id The BloomFilter id.
- *
- * @return The constant pointer to BloomFilter that is
- * already created in the constructor.
- **/
- inline const BloomFilter* getBloomFilter(const bloom_filter_id id) const {
- DCHECK_LT(id, bloom_filters_.size());
- return bloom_filters_[id].get();
- }
-
- /**
- * @brief Destory the given BloomFilter.
- *
- * @param id The id of the BloomFilter to destroy.
- **/
- inline void destroyBloomFilter(const bloom_filter_id id) {
- DCHECK_LT(id, bloom_filters_.size());
- bloom_filters_[id].reset();
- }
-
- /**
* @brief Whether the given GeneratorFunctionHandle id is valid.
*
* @param id The GeneratorFunctionHandle id.
@@ -333,6 +293,87 @@ class QueryContext {
}
/**
+ * @brief Whether the given LIPFilter id is valid.
+ *
+ * @param id The LIPFilter id.
+ *
+ * @return True if valid, otherwise false.
+ **/
+ bool isValidLIPFilterId(const lip_filter_id id) const {
+ return id < lip_filters_.size();
+ }
+
+ /**
+ * @brief Get a mutable reference to the LIPFilter.
+ *
+ * @param id The LIPFilter id.
+ *
+ * @return The LIPFilter, already created in the constructor.
+ **/
+ inline LIPFilter* getLIPFilterMutable(const lip_filter_id id) {
+ DCHECK_LT(id, lip_filters_.size());
+ return lip_filters_[id].get();
+ }
+
+ /**
+ * @brief Get a constant pointer to the LIPFilter.
+ *
+ * @param id The LIPFilter id.
+ *
+ * @return The constant pointer to LIPFilter that is
+ * already created in the constructor.
+ **/
+ inline const LIPFilter* getLIPFilter(const lip_filter_id id) const {
+ DCHECK_LT(id, lip_filters_.size());
+ return lip_filters_[id].get();
+ }
+
+ /**
+ * @brief Destory the given LIPFilter.
+ *
+ * @param id The id of the LIPFilter to destroy.
+ **/
+ inline void destroyLIPFilter(const lip_filter_id id) {
+ DCHECK_LT(id, lip_filters_.size());
+ lip_filters_[id].reset();
+ }
+
+ /**
+ * @brief Whether the given LIPFilterDeploymentInfo id is valid.
+ *
+ * @param id The LIPFilterDeploymentInfo id.
+ *
+ * @return True if valid, otherwise false.
+ **/
+ bool isValidLIPFilterDeploymentInfoId(const lip_filter_deployment_info_id id) const {
+ return id < lip_filter_deployment_infos_.size();
+ }
+
+ /**
+ * @brief Get a constant pointer to the LIPFilterDeploymentInfo.
+ *
+ * @param id The LIPFilterDeploymentInfo id.
+ *
+ * @return The constant pointer to LIPFilterDeploymentInfo that is
+ * already created in the constructor.
+ **/
+ inline const LIPFilterDeploymentInfo* getLIPFilterDeploymentInfo(
+ const lip_filter_deployment_info_id id) const {
+ DCHECK_LT(id, lip_filter_deployment_infos_.size());
+ return lip_filter_deployment_infos_[id].get();
+ }
+
+ /**
+ * @brief Destory the given LIPFilterDeploymentInfo.
+ *
+ * @param id The id of the LIPFilterDeploymentInfo to destroy.
+ **/
+ inline void destroyLIPFilterDeploymentInfo(const lip_filter_deployment_info_id id) {
+ DCHECK_LT(id, lip_filter_deployment_infos_.size());
+ lip_filter_deployment_infos_[id].reset();
+ }
+
+ /**
* @brief Whether the given Predicate id is valid or no predicate.
*
* @param id The Predicate id.
@@ -507,10 +548,11 @@ class QueryContext {
private:
std::vector<std::unique_ptr<AggregationOperationState>> aggregation_states_;
- std::vector<std::unique_ptr<BloomFilter>> bloom_filters_;
std::vector<std::unique_ptr<const GeneratorFunctionHandle>> generator_functions_;
std::vector<std::unique_ptr<InsertDestination>> insert_destinations_;
std::vector<std::unique_ptr<JoinHashTable>> join_hash_tables_;
+ std::vector<std::unique_ptr<LIPFilter>> lip_filters_;
+ std::vector<std::unique_ptr<LIPFilterDeploymentInfo>> lip_filter_deployment_infos_;
std::vector<std::unique_ptr<const Predicate>> predicates_;
std::vector<std::vector<std::unique_ptr<const Scalar>>> scalar_groups_;
std::vector<std::unique_ptr<const SortConfiguration>> sort_configs_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_execution/QueryContext.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.proto b/query_execution/QueryContext.proto
index 1a586a4..d79b990 100644
--- a/query_execution/QueryContext.proto
+++ b/query_execution/QueryContext.proto
@@ -26,8 +26,8 @@ import "storage/HashTable.proto";
import "storage/InsertDestination.proto";
import "storage/WindowAggregationOperationState.proto";
import "types/containers/Tuple.proto";
-import "utility/BloomFilter.proto";
import "utility/SortConfiguration.proto";
+import "utility/lip_filter/LIPFilter.proto";
message QueryContext {
message ScalarGroup {
@@ -46,19 +46,20 @@ message QueryContext {
}
repeated AggregationOperationState aggregation_states = 1;
- repeated BloomFilter bloom_filters = 2;
- repeated GeneratorFunctionHandle generator_functions = 3;
- repeated HashTable join_hash_tables = 4;
- repeated InsertDestination insert_destinations = 5;
- repeated Predicate predicates = 6;
- repeated ScalarGroup scalar_groups = 7;
- repeated SortConfiguration sort_configs = 8;
- repeated Tuple tuples = 9;
+ repeated GeneratorFunctionHandle generator_functions = 2;
+ repeated HashTable join_hash_tables = 3;
+ repeated InsertDestination insert_destinations = 4;
+ repeated LIPFilter lip_filters = 5;
+ repeated LIPFilterDeploymentInfo lip_filter_deployment_infos = 6;
+ repeated Predicate predicates = 7;
+ repeated ScalarGroup scalar_groups = 8;
+ repeated SortConfiguration sort_configs = 9;
+ repeated Tuple tuples = 10;
// NOTE(zuyu): For UpdateWorkOrder only.
- repeated UpdateGroup update_groups = 10;
+ repeated UpdateGroup update_groups = 11;
- repeated WindowAggregationOperationState window_aggregation_states = 11;
+ repeated WindowAggregationOperationState window_aggregation_states = 12;
- required uint64 query_id = 12;
+ required uint64 query_id = 13;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 32eaea3..4013e3e 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -41,7 +41,7 @@ add_subdirectory(tests)
# Declare micro-libs:
add_library(quickstep_queryoptimizer_ExecutionGenerator ExecutionGenerator.cpp ExecutionGenerator.hpp)
-add_library(quickstep_queryoptimizer_LIPFilterGenerator ../empty_src.cpp LIPFilterGenerator.hpp)
+add_library(quickstep_queryoptimizer_LIPFilterGenerator LIPFilterGenerator.cpp LIPFilterGenerator.hpp)
add_library(quickstep_queryoptimizer_LogicalGenerator LogicalGenerator.cpp LogicalGenerator.hpp)
add_library(quickstep_queryoptimizer_LogicalToPhysicalMapper
../empty_src.cpp
@@ -73,6 +73,7 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_expressions_windowaggregation_WindowAggregateFunction_proto
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryContext_proto
+ quickstep_queryoptimizer_LIPFilterGenerator
quickstep_queryoptimizer_OptimizerContext
quickstep_queryoptimizer_QueryHandle
quickstep_queryoptimizer_QueryPlan
@@ -152,8 +153,20 @@ if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_catalog_Catalog_proto)
endif()
-target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
- glog)
+target_link_libraries(quickstep_queryoptimizer_LIPFilterGenerator
+ glog
+ quickstep_catalog_CatalogAttribute
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_QueryContext_proto
+ quickstep_queryoptimizer_QueryPlan
+ quickstep_queryoptimizer_physical_Aggregate
+ quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_LIPFilterConfiguration
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_Selection
+ quickstep_utility_lipfilter_LIPFilter
+ quickstep_utility_lipfilter_LIPFilter_proto)
target_link_libraries(quickstep_queryoptimizer_LogicalGenerator
glog
quickstep_parser_ParseStatement
@@ -227,6 +240,7 @@ target_link_libraries(quickstep_queryoptimizer_Validator
add_library(quickstep_queryoptimizer ../empty_src.cpp QueryOptimizerModule.hpp)
target_link_libraries(quickstep_queryoptimizer
quickstep_queryoptimizer_ExecutionGenerator
+ quickstep_queryoptimizer_LIPFilterGenerator
quickstep_queryoptimizer_LogicalGenerator
quickstep_queryoptimizer_LogicalToPhysicalMapper
quickstep_queryoptimizer_Optimizer
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index efebe92..559de57 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -54,6 +54,7 @@
#include "expressions/window_aggregation/WindowAggregateFunction.pb.h"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/LIPFilterGenerator.hpp"
#include "query_optimizer/OptimizerContext.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryPlan.hpp"
@@ -164,6 +165,8 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) {
cost_model_.reset(
new cost::SimpleCostModel(top_level_physical_plan_->shared_subplans()));
+ lip_filter_generator_.reset(
+ new LIPFilterGenerator(top_level_physical_plan_->lip_filter_configuration()));
const CatalogRelation *result_relation = nullptr;
@@ -173,6 +176,8 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) {
}
generatePlanInternal(top_level_physical_plan_->plan());
+ lip_filter_generator_->deployLIPFilters(execution_plan_, query_context_proto_);
+
// Set the query result relation if the input plan exists in physical_to_execution_map_,
// which indicates the plan is the result of a SELECT query.
const std::unordered_map<P::PhysicalPtr, CatalogRelationInfo>::const_iterator it =
@@ -229,6 +234,8 @@ void ExecutionGenerator::generatePlanInternal(
generatePlanInternal(child);
}
+ lip_filter_generator_->registerAttributeMap(physical_plan, attribute_substitution_map_);
+
switch (physical_plan->getPhysicalType()) {
case P::PhysicalType::kAggregate:
return convertAggregate(
@@ -560,6 +567,8 @@ void ExecutionGenerator::convertSelection(
std::forward_as_tuple(select_index,
output_relation));
temporary_relation_info_vec_.emplace_back(select_index, output_relation);
+
+ lip_filter_generator_->addSelectionInfo(physical_selection, select_index);
}
void ExecutionGenerator::convertSharedSubplanReference(const physical::SharedSubplanReferencePtr &physical_plan) {
@@ -819,6 +828,10 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::forward_as_tuple(join_operator_index,
output_relation));
temporary_relation_info_vec_.emplace_back(join_operator_index, output_relation);
+
+ lip_filter_generator_->addHashJoinInfo(physical_plan,
+ build_operator_index,
+ join_operator_index);
}
void ExecutionGenerator::convertNestedLoopsJoin(
@@ -1391,7 +1404,8 @@ void ExecutionGenerator::convertAggregate(
aggr_state_proto->mutable_predicate()->CopyFrom(predicate->getProto());
}
- aggr_state_proto->set_estimated_num_entries(cost_model_->estimateCardinality(physical_plan));
+// aggr_state_proto->set_estimated_num_entries(cost_model_->estimateCardinality(physical_plan));
+ aggr_state_proto->set_estimated_num_entries(64uL);
const QueryPlan::DAGNodeIndex aggregation_operator_index =
execution_plan_->addRelationalOperator(
@@ -1444,6 +1458,10 @@ void ExecutionGenerator::convertAggregate(
execution_plan_->addDirectDependency(destroy_aggregation_state_operator_index,
finalize_aggregation_operator_index,
true);
+
+ lip_filter_generator_->addAggregateInfo(physical_plan,
+ aggregation_operator_index,
+ aggr_state_index);
}
void ExecutionGenerator::convertSort(const P::SortPtr &physical_sort) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_optimizer/ExecutionGenerator.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.hpp b/query_optimizer/ExecutionGenerator.hpp
index a830e65..a666f6d 100644
--- a/query_optimizer/ExecutionGenerator.hpp
+++ b/query_optimizer/ExecutionGenerator.hpp
@@ -33,6 +33,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/LIPFilterGenerator.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryPlan.hpp"
#include "query_optimizer/cost_model/CostModel.hpp"
@@ -423,6 +424,9 @@ class ExecutionGenerator {
physical::TopLevelPlanPtr top_level_physical_plan_;
+ // Sub-generator for deploying LIP (lookahead information passing) filters.
+ std::unique_ptr<LIPFilterGenerator> lip_filter_generator_;
+
DISALLOW_COPY_AND_ASSIGN(ExecutionGenerator);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_optimizer/LIPFilterGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/LIPFilterGenerator.cpp b/query_optimizer/LIPFilterGenerator.cpp
new file mode 100644
index 0000000..4e24740
--- /dev/null
+++ b/query_optimizer/LIPFilterGenerator.cpp
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "query_optimizer/LIPFilterGenerator.hpp"
+
+#include <map>
+#include <utility>
+
+#include "catalog/CatalogAttribute.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.pb.h"
+#include "utility/lip_filter/LIPFilter.hpp"
+#include "utility/lip_filter/LIPFilter.pb.h"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+
+namespace E = ::quickstep::optimizer::expressions;
+namespace P = ::quickstep::optimizer::physical;
+
+void LIPFilterGenerator::registerAttributeMap(
+ const P::PhysicalPtr &node,
+ const std::unordered_map<E::ExprId, const CatalogAttribute *> &attribute_substitution_map) {
+ const auto &build_info_map = lip_filter_configuration_->getBuildInfoMap();
+ const auto build_it = build_info_map.find(node);
+ if (build_it != build_info_map.end()) {
+ auto &map_entry = attribute_map_[node];
+ for (const auto &info : build_it->second) {
+ E::ExprId attr_id = info.build_attribute->id();
+ map_entry.emplace(attr_id, attribute_substitution_map.at(attr_id));
+ }
+ }
+ const auto &probe_info_map = lip_filter_configuration_->getProbeInfoMap();
+ const auto probe_it = probe_info_map.find(node);
+ if (probe_it != probe_info_map.end()) {
+ auto &map_entry = attribute_map_[node];
+ for (const auto &info : probe_it->second) {
+ E::ExprId attr_id = info.probe_attribute->id();
+ map_entry.emplace(attr_id, attribute_substitution_map.at(attr_id));
+ }
+ }
+}
+
+void LIPFilterGenerator::deployLIPFilters(QueryPlan *execution_plan,
+ serialization::QueryContext *query_context_proto) const {
+ LIPFilterBuilderMap lip_filter_builder_map;
+
+ // Deploy builders
+ const auto &build_info_map = lip_filter_configuration_->getBuildInfoMap();
+ for (const auto &hash_join_info : hash_join_infos_) {
+ const P::PhysicalPtr &builder_node = hash_join_info.hash_join;
+ const auto build_it = build_info_map.find(builder_node);
+ if (build_it != build_info_map.end()) {
+ deployBuilderInternal(execution_plan,
+ query_context_proto,
+ builder_node,
+ hash_join_info.build_operator_index,
+ build_it->second,
+ &lip_filter_builder_map);
+ }
+ }
+
+ // Deploy probers
+ // const auto &probe_info_map = lip_filter_configuration_->getProbeInfoMap();
+}
+
+void LIPFilterGenerator::deployBuilderInternal(
+ QueryPlan *execution_plan,
+ serialization::QueryContext *query_context_proto,
+ const physical::PhysicalPtr &builder_node,
+ const QueryPlan::DAGNodeIndex builder_operator_index,
+ const std::vector<physical::LIPFilterBuildInfo> &build_info_vec,
+ LIPFilterBuilderMap *lip_filter_builder_map) const {
+ const auto &builder_attribute_map = attribute_map_.at(builder_node);
+ for (const auto &info : build_info_vec) {
+ const QueryContext::lip_filter_id lip_filter_id = query_context_proto->lip_filters_size();
+ serialization::LIPFilter *lip_filter_proto = query_context_proto->add_lip_filters();
+
+ switch (info.filter_type) {
+ case LIPFilterType::kSingleIdentityHashFilter:
+ lip_filter_proto->set_lip_filter_type(
+ serialization::LIPFilterType::SINGLE_IDENTITY_HASH_FILTER);
+ lip_filter_proto->SetExtension(
+ serialization::SingleIdentityHashFilter::num_bits, info.filter_size);
+ break;
+ default:
+ LOG(FATAL) << "Unsupported LIPFilter type";
+ break;
+ }
+
+ lip_filter_builder_map->emplace(
+ std::make_pair(info.build_attribute->id(), builder_node),
+ std::make_pair(lip_filter_id, builder_operator_index));
+
+ auto *lip_filter_deployment_info_proto =
+ query_context_proto->add_lip_filter_deployment_infos();
+ lip_filter_deployment_info_proto->set_action_type(serialization::LIPFilterActionType::BUILD);
+ lip_filter_deployment_info_proto->set_lip_filter_id(lip_filter_id);
+
+ const CatalogAttribute *target_attr = builder_attribute_map.at(info.build_attribute->id());
+ lip_filter_deployment_info_proto->set_attribute_id(target_attr->getID());
+ lip_filter_deployment_info_proto->mutable_attribute_type()->CopyFrom(
+ target_attr->getType().getProto());
+
+ std::cerr << "Build " << info.build_attribute->toString()
+ << " @" << builder_node << "\n";
+ }
+}
+
+
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_optimizer/LIPFilterGenerator.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/LIPFilterGenerator.hpp b/query_optimizer/LIPFilterGenerator.hpp
index 4137b98..05270c9 100644
--- a/query_optimizer/LIPFilterGenerator.hpp
+++ b/query_optimizer/LIPFilterGenerator.hpp
@@ -20,7 +20,24 @@
#ifndef QUICKSTEP_QUERY_OPTIMIZER_LIP_FILTER_GENERATOR_HPP_
#define QUICKSTEP_QUERY_OPTIMIZER_LIP_FILTER_GENERATOR_HPP_
+#include <map>
+#include <unordered_map>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/QueryPlan.hpp"
+#include "query_optimizer/physical/LIPFilterConfiguration.hpp"
+#include "query_optimizer/physical/Aggregate.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/Selection.hpp"
+
+
namespace quickstep {
+
+class CatalogAttribute;
+
namespace optimizer {
/** \addtogroup QueryOptimizer
@@ -29,12 +46,88 @@ namespace optimizer {
class LIPFilterGenerator {
public:
-
+ LIPFilterGenerator(const physical::LIPFilterConfigurationPtr &lip_filter_configuration)
+ : lip_filter_configuration_(lip_filter_configuration) {
+ }
+
+ void registerAttributeMap(
+ const physical::PhysicalPtr &node,
+ const std::unordered_map<expressions::ExprId, const CatalogAttribute *> &attribute_substitution_map);
+
+ void addAggregateInfo(const physical::AggregatePtr &aggregate,
+ const QueryPlan::DAGNodeIndex aggregate_operator_index,
+ const QueryContext::aggregation_state_id aggregation_state_id) {
+ aggregate_infos_.emplace_back(aggregate, aggregate_operator_index, aggregation_state_id);
+ }
+ void addHashJoinInfo(const physical::HashJoinPtr &hash_join,
+ const QueryPlan::DAGNodeIndex build_operator_index,
+ const QueryPlan::DAGNodeIndex join_operator_index) {
+ hash_join_infos_.emplace_back(hash_join, build_operator_index, join_operator_index);
+ }
+
+ void addSelectionInfo(const physical::SelectionPtr &selection,
+ const QueryPlan::DAGNodeIndex select_operator_index) {
+ selection_infos_.emplace_back(selection, select_operator_index);
+ }
+
+ void deployLIPFilters(QueryPlan *execution_plan,
+ serialization::QueryContext *query_context_proto) const;
private:
+ struct AggregateInfo {
+ AggregateInfo(const physical::AggregatePtr &aggregate_in,
+ const QueryPlan::DAGNodeIndex aggregate_operator_index_in,
+ const QueryContext::aggregation_state_id aggregation_state_id_in)
+ : aggregate(aggregate_in),
+ aggregate_operator_index(aggregate_operator_index_in),
+ aggregation_state_id(aggregation_state_id_in) {
+ }
+ const physical::AggregatePtr aggregate;
+ const QueryPlan::DAGNodeIndex aggregate_operator_index;
+ const QueryContext::aggregation_state_id aggregation_state_id;
+ };
+
+ struct HashJoinInfo {
+ HashJoinInfo(const physical::HashJoinPtr &hash_join_in,
+ const QueryPlan::DAGNodeIndex build_operator_index_in,
+ const QueryPlan::DAGNodeIndex join_operator_index_in)
+ : hash_join(hash_join_in),
+ build_operator_index(build_operator_index_in),
+ join_operator_index(join_operator_index_in) {
+ }
+ const physical::HashJoinPtr hash_join;
+ const QueryPlan::DAGNodeIndex build_operator_index;
+ const QueryPlan::DAGNodeIndex join_operator_index;
+ };
+
+ struct SelectionInfo {
+ SelectionInfo(const physical::SelectionPtr &selection_in,
+ const QueryPlan::DAGNodeIndex select_operator_index_in)
+ : selection(selection_in),
+ select_operator_index(select_operator_index_in) {
+ }
+ const physical::SelectionPtr selection;
+ const QueryPlan::DAGNodeIndex select_operator_index;
+ };
+
+ typedef std::map<std::pair<expressions::ExprId, physical::PhysicalPtr>,
+ std::pair<QueryContext::lip_filter_id, QueryPlan::DAGNodeIndex>> LIPFilterBuilderMap;
+
+ void deployBuilderInternal(QueryPlan *execution_plan,
+ serialization::QueryContext *query_context_proto,
+ const physical::PhysicalPtr &builder_node,
+ const QueryPlan::DAGNodeIndex builder_operator_index,
+ const std::vector<physical::LIPFilterBuildInfo> &build_info_vec,
+ LIPFilterBuilderMap *lip_filter_builder_map) const;
+ void deployProberInteral();
+ const physical::LIPFilterConfigurationPtr lip_filter_configuration_;
+ std::map<physical::PhysicalPtr, std::map<expressions::ExprId, const CatalogAttribute *>> attribute_map_;
+ std::vector<AggregateInfo> aggregate_infos_;
+ std::vector<HashJoinInfo> hash_join_infos_;
+ std::vector<SelectionInfo> selection_infos_;
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_optimizer/PhysicalGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/PhysicalGenerator.cpp b/query_optimizer/PhysicalGenerator.cpp
index a3fedc9..eb1ec08 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -115,7 +115,6 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
quickstep::PlanVisualizer plan_visualizer;
std::cerr << "\n" << plan_visualizer.visualize(physical_plan_) << "\n";
}
- exit(0);
#ifdef QUICKSTEP_DEBUG
Validate(physical_plan_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_optimizer/physical/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/CMakeLists.txt b/query_optimizer/physical/CMakeLists.txt
index 83682b3..5c2cd0b 100644
--- a/query_optimizer/physical/CMakeLists.txt
+++ b/query_optimizer/physical/CMakeLists.txt
@@ -52,7 +52,6 @@ target_link_libraries(quickstep_queryoptimizer_physical_Aggregate
quickstep_queryoptimizer_expressions_ExpressionUtil
quickstep_queryoptimizer_expressions_NamedExpression
quickstep_queryoptimizer_expressions_Predicate
- quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_physical_PhysicalType
quickstep_utility_Cast
@@ -123,7 +122,6 @@ target_link_libraries(quickstep_queryoptimizer_physical_HashJoin
quickstep_queryoptimizer_expressions_NamedExpression
quickstep_queryoptimizer_expressions_Predicate
quickstep_queryoptimizer_physical_BinaryJoin
- quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_physical_PhysicalType
quickstep_utility_Cast
@@ -155,8 +153,8 @@ target_link_libraries(quickstep_queryoptimizer_physical_Join
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_expressions_AttributeReference
- quickstep_utility_LIPFilter
- quickstep_utility_Macros)
+ quickstep_utility_Macros
+ quickstep_utility_lipfilter_LIPFilter)
target_link_libraries(quickstep_queryoptimizer_physical_NestedLoopsJoin
glog
quickstep_queryoptimizer_OptimizerTree
@@ -174,7 +172,6 @@ target_link_libraries(quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_OptimizerTree
quickstep_queryoptimizer_expressions_AttributeReference
quickstep_queryoptimizer_expressions_ExpressionUtil
- quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_physical_PhysicalType
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_physical_Sample
@@ -195,7 +192,6 @@ target_link_libraries(quickstep_queryoptimizer_physical_Selection
quickstep_queryoptimizer_expressions_LogicalAnd
quickstep_queryoptimizer_expressions_NamedExpression
quickstep_queryoptimizer_expressions_Predicate
- quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_physical_PhysicalType
quickstep_utility_Cast
@@ -246,6 +242,7 @@ target_link_libraries(quickstep_queryoptimizer_physical_TopLevelPlan
quickstep_queryoptimizer_expressions_AttributeReference
quickstep_queryoptimizer_expressions_ExprId
quickstep_queryoptimizer_expressions_ExpressionUtil
+ quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_physical_PhysicalType
quickstep_utility_Cast
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_optimizer/physical/LIPFilterConfiguration.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/LIPFilterConfiguration.hpp b/query_optimizer/physical/LIPFilterConfiguration.hpp
index 59db2e6..9b028ad 100644
--- a/query_optimizer/physical/LIPFilterConfiguration.hpp
+++ b/query_optimizer/physical/LIPFilterConfiguration.hpp
@@ -26,7 +26,7 @@
#include <vector>
#include "query_optimizer/expressions/AttributeReference.hpp"
-#include "utility/LIPFilter.hpp"
+#include "utility/lip_filter/LIPFilter.hpp"
#include "utility/Macros.hpp"
#include "glog/logging.h"
@@ -42,7 +42,7 @@ namespace physical {
class Physical;
typedef std::shared_ptr<const Physical> PhysicalPtr;
-struct LIPFilterBuildInfo{
+struct LIPFilterBuildInfo {
LIPFilterBuildInfo(const expressions::AttributeReferencePtr &build_attribute_in,
const std::size_t filter_size_in,
const LIPFilterType &filter_type_in)
@@ -64,7 +64,6 @@ struct LIPFilterProbeInfo {
builder(builder_in) {
}
expressions::AttributeReferencePtr probe_attribute;
- PhysicalPtr target;
expressions::AttributeReferencePtr build_attribute;
PhysicalPtr builder;
};
@@ -82,7 +81,7 @@ class LIPFilterConfiguration {
const PhysicalPtr &builder,
const std::size_t filter_size,
const LIPFilterType &filter_type) {
- build_info_[builder].emplace_back(
+ build_info_map_[builder].emplace_back(
build_attribute, filter_size, filter_type);
}
@@ -90,21 +89,21 @@ class LIPFilterConfiguration {
const PhysicalPtr &prober,
const expressions::AttributeReferencePtr &build_attribute,
const PhysicalPtr &builder) {
- probe_info_[prober].emplace_back(
+ probe_info_map_[prober].emplace_back(
probe_attribute, build_attribute, builder);
}
- const std::map<PhysicalPtr, std::vector<LIPFilterBuildInfo>>& getBuildInfo() const {
- return build_info_;
+ const std::map<PhysicalPtr, std::vector<LIPFilterBuildInfo>>& getBuildInfoMap() const {
+ return build_info_map_;
}
- const std::map<PhysicalPtr, std::vector<LIPFilterProbeInfo>>& getProbeInfo() const {
- return probe_info_;
+ const std::map<PhysicalPtr, std::vector<LIPFilterProbeInfo>>& getProbeInfoMap() const {
+ return probe_info_map_;
}
private:
- std::map<PhysicalPtr, std::vector<LIPFilterBuildInfo>> build_info_;
- std::map<PhysicalPtr, std::vector<LIPFilterProbeInfo>> probe_info_;
+ std::map<PhysicalPtr, std::vector<LIPFilterBuildInfo>> build_info_map_;
+ std::map<PhysicalPtr, std::vector<LIPFilterProbeInfo>> probe_info_map_;
DISALLOW_COPY_AND_ASSIGN(LIPFilterConfiguration);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_optimizer/rules/AttachLIPFilters.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/AttachLIPFilters.cpp b/query_optimizer/rules/AttachLIPFilters.cpp
index 5493e00..d67eacd 100644
--- a/query_optimizer/rules/AttachLIPFilters.cpp
+++ b/query_optimizer/rules/AttachLIPFilters.cpp
@@ -39,7 +39,7 @@
#include "query_optimizer/physical/Physical.hpp"
#include "query_optimizer/physical/PhysicalType.hpp"
#include "query_optimizer/physical/TopLevelPlan.hpp"
-#include "utility/LIPFilter.hpp"
+#include "utility/lip_filter/LIPFilter.hpp"
#include "glog/logging.h"
@@ -71,8 +71,8 @@ P::PhysicalPtr AttachLIPFilters::apply(const P::PhysicalPtr &input) {
// }
P::PhysicalPtr output;
- if (!lip_filter_configuration_->getBuildInfo().empty() ||
- !lip_filter_configuration_->getProbeInfo().empty()) {
+ if (!lip_filter_configuration_->getBuildInfoMap().empty() ||
+ !lip_filter_configuration_->getProbeInfoMap().empty()) {
output = top_level_plan->copyWithLIPFilterConfiguration(
P::LIPFilterConfigurationPtr(lip_filter_configuration_.release()));
} else {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/query_optimizer/rules/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/CMakeLists.txt b/query_optimizer/rules/CMakeLists.txt
index 9dc6118..f042ba8 100644
--- a/query_optimizer/rules/CMakeLists.txt
+++ b/query_optimizer/rules/CMakeLists.txt
@@ -45,12 +45,14 @@ target_link_libraries(quickstep_queryoptimizer_rules_AttachLIPFilters
quickstep_queryoptimizer_expressions_PatternMatcher
quickstep_queryoptimizer_expressions_Predicate
quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_physical_PatternMatcher
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_physical_PhysicalType
quickstep_queryoptimizer_physical_TopLevelPlan
quickstep_queryoptimizer_rules_Rule
- quickstep_utility_Macros)
+ quickstep_utility_Macros
+ quickstep_utility_lipfilter_LIPFilter)
target_link_libraries(quickstep_queryoptimizer_rules_BottomUpRule
glog
quickstep_queryoptimizer_rules_Rule
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/storage/HashTableFactory.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTableFactory.hpp b/storage/HashTableFactory.hpp
index cd79723..d690557 100644
--- a/storage/HashTableFactory.hpp
+++ b/storage/HashTableFactory.hpp
@@ -295,14 +295,11 @@ class HashTableFactory {
* @param proto A protobuf description of a resizable HashTable.
* @param storage_manager The StorageManager to use (a StorageBlob will be
* allocated to hold the HashTable's contents).
- * @param bloom_filters A vector of pointers to bloom filters that may be used
- * during hash table construction in build/probe phase.
* @return A new resizable HashTable with parameters specified by proto.
**/
static HashTable<ValueT, resizable, serializable, force_key_copy, allow_duplicate_keys>*
CreateResizableFromProto(const serialization::HashTable &proto,
- StorageManager *storage_manager,
- const std::vector<std::unique_ptr<BloomFilter>> &bloom_filters) {
+ StorageManager *storage_manager) {
DCHECK(ProtoIsValid(proto))
<< "Attempted to create HashTable from invalid proto description:\n"
<< proto.DebugString();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index 8a2c7ae..9025015 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -156,6 +156,8 @@ QS_PROTOBUF_GENERATE_CPP(quickstep_utility_SortConfiguration_proto_srcs
quickstep_utility_SortConfiguration_proto_hdrs
SortConfiguration.proto)
+add_subdirectory(lip_filter)
+
# Declare micro-libs:
add_library(quickstep_utility_Alignment ../empty_src.cpp Alignment.hpp)
add_library(quickstep_utility_BitManipulation ../empty_src.cpp BitManipulation.hpp)
@@ -174,7 +176,6 @@ add_library(quickstep_utility_ExecutionDAGVisualizer
ExecutionDAGVisualizer.hpp)
add_library(quickstep_utility_Glob Glob.cpp Glob.hpp)
add_library(quickstep_utility_HashPair ../empty_src.cpp HashPair.hpp)
-add_library(quickstep_utility_LIPFilter LIPFilter.cpp LIPFilter.hpp)
add_library(quickstep_utility_Macros ../empty_src.cpp Macros.hpp)
add_library(quickstep_utility_MemStream ../empty_src.cpp MemStream.hpp)
add_library(quickstep_utility_PlanVisualizer PlanVisualizer.cpp PlanVisualizer.hpp)
@@ -245,8 +246,6 @@ target_link_libraries(quickstep_utility_ExecutionDAGVisualizer
quickstep_utility_StringUtil)
target_link_libraries(quickstep_utility_Glob
glog)
-target_link_libraries(quickstep_utility_LIPFilter
- glog)
target_link_libraries(quickstep_utility_MemStream
glog
quickstep_utility_Macros)
@@ -254,11 +253,16 @@ target_link_libraries(quickstep_utility_PrimeNumber
glog)
target_link_libraries(quickstep_utility_PlanVisualizer
quickstep_catalog_CatalogRelation
+ quickstep_catalog_CatalogRelationStatistics
+ quickstep_catalog_CatalogTypedefs
quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_physical_Aggregate
quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_queryoptimizer_physical_Selection
quickstep_queryoptimizer_physical_TableReference
quickstep_queryoptimizer_physical_TopLevelPlan
quickstep_utility_Macros
@@ -326,7 +330,6 @@ target_link_libraries(quickstep_utility
quickstep_utility_ExecutionDAGVisualizer
quickstep_utility_Glob
quickstep_utility_HashPair
- quickstep_utility_LIPFilter
quickstep_utility_Macros
quickstep_utility_MemStream
quickstep_utility_PlanVisualizer
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/DAG.hpp
----------------------------------------------------------------------
diff --git a/utility/DAG.hpp b/utility/DAG.hpp
index a1f2619..b35f2b5 100644
--- a/utility/DAG.hpp
+++ b/utility/DAG.hpp
@@ -293,8 +293,10 @@ class DAG {
* node at node_index.
**/
inline void addDependent(const size_type_nodes node_index, const LinkMetadataT &link_metadata) {
- DCHECK(dependents_with_metadata_.find(node_index) == dependents_with_metadata_.end());
- dependents_with_metadata_.emplace(node_index, link_metadata);
+// DCHECK(dependents_with_metadata_.find(node_index) == dependents_with_metadata_.end());
+// dependents_with_metadata_.emplace(node_index, link_metadata);
+ // TODO(jianqiao): implement upsert
+ dependents_with_metadata_[node_index] = link_metadata;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/LIPFilter.cpp
----------------------------------------------------------------------
diff --git a/utility/LIPFilter.cpp b/utility/LIPFilter.cpp
deleted file mode 100644
index f503f4f..0000000
--- a/utility/LIPFilter.cpp
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- **/
-
-#include "utility/LIPFilter.hpp"
-
-namespace quickstep {
-
-} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/LIPFilter.hpp
----------------------------------------------------------------------
diff --git a/utility/LIPFilter.hpp b/utility/LIPFilter.hpp
deleted file mode 100644
index 12a19d7..0000000
--- a/utility/LIPFilter.hpp
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- **/
-
-#ifndef QUICKSTEP_UTILITY_LIP_FILTER_HPP_
-#define QUICKSTEP_UTILITY_LIP_FILTER_HPP_
-
-#include <vector>
-
-#include "glog/logging.h"
-
-namespace quickstep {
-
-/** \addtogroup Utility
- * @{
- */
-
-enum class LIPFilterType {
- kBloomFilter,
- kExactFilter,
- kSingleIdentityHashFilter
-};
-
-
-/** @} */
-
-} // namespace quickstep
-
-#endif // QUICKSTEP_UTILITY_LIP_FILTER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/LIPFilterAdapter.hpp
----------------------------------------------------------------------
diff --git a/utility/LIPFilterAdapter.hpp b/utility/LIPFilterAdapter.hpp
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/LIPFilterBuilder.hpp
----------------------------------------------------------------------
diff --git a/utility/LIPFilterBuilder.hpp b/utility/LIPFilterBuilder.hpp
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/PlanVisualizer.cpp
----------------------------------------------------------------------
diff --git a/utility/PlanVisualizer.cpp b/utility/PlanVisualizer.cpp
index fdb5812..5156c1e 100644
--- a/utility/PlanVisualizer.cpp
+++ b/utility/PlanVisualizer.cpp
@@ -168,7 +168,7 @@ void PlanVisualizer::visit(const P::PhysicalPtr &input) {
}
if (lip_filter_conf_ != nullptr) {
- const auto &build_filters = lip_filter_conf_->getBuildInfo();
+ const auto &build_filters = lip_filter_conf_->getBuildInfoMap();
const auto build_it = build_filters.find(input);
if (build_it != build_filters.end()) {
for (const auto &build_info : build_it->second) {
@@ -176,7 +176,7 @@ void PlanVisualizer::visit(const P::PhysicalPtr &input) {
std::string("[LIP build] ") + build_info.build_attribute->attribute_alias());
}
}
- const auto &probe_filters = lip_filter_conf_->getProbeInfo();
+ const auto &probe_filters = lip_filter_conf_->getProbeInfoMap();
const auto probe_it = probe_filters.find(input);
if (probe_it != probe_filters.end()) {
for (const auto &probe_info : probe_it->second) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/lip_filter/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/lip_filter/CMakeLists.txt b/utility/lip_filter/CMakeLists.txt
new file mode 100644
index 0000000..df6a5ec
--- /dev/null
+++ b/utility/lip_filter/CMakeLists.txt
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+QS_PROTOBUF_GENERATE_CPP(utility_lipfilter_LIPFilter_proto_srcs
+ utility_lipfilter_LIPFilter_proto_hdrs
+ LIPFilter.proto)
+
+# Declare micro-libs:
+add_library(quickstep_utility_lipfilter_LIPFilter LIPFilter.cpp LIPFilter.hpp)
+add_library(quickstep_utility_lipfilter_LIPFilterAdaptiveProber ../../empty_src.cpp LIPFilterAdaptiveProber.hpp)
+add_library(quickstep_utility_lipfilter_LIPFilterBuilder ../../empty_src.cpp LIPFilterBuilder.hpp)
+add_library(quickstep_utility_lipfilter_LIPFilterDeploymentInfo ../../empty_src.cpp LIPFilterDeploymentInfo.hpp)
+add_library(quickstep_utility_lipfilter_LIPFilterFactory LIPFilterFactory.cpp LIPFilterFactory.hpp)
+add_library(quickstep_utility_lipfilter_LIPFilter_proto
+ ${utility_lipfilter_LIPFilter_proto_srcs})
+
+# Link dependencies:
+target_link_libraries(quickstep_utility_lipfilter_LIPFilter
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_utility_lipfilter_LIPFilterAdaptiveProber
+ quickstep_catalog_CatalogTypedefs
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_utility_lipfilter_LIPFilterBuilder
+ quickstep_catalog_CatalogTypedefs
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_utility_lipfilter_LIPFilterDeploymentInfo
+ quickstep_catalog_CatalogTypedefs
+ quickstep_utility_Macros
+ quickstep_utility_lipfilter_LIPFilter)
+target_link_libraries(quickstep_utility_lipfilter_LIPFilterFactory
+ quickstep_utility_lipfilter_LIPFilter_proto
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_utility_lipfilter_LIPFilter_proto
+ ${PROTOBUF_LIBRARY}
+ quickstep_types_Type_proto)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/lip_filter/LIPFilter.cpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilter.cpp b/utility/lip_filter/LIPFilter.cpp
new file mode 100644
index 0000000..92bfab1
--- /dev/null
+++ b/utility/lip_filter/LIPFilter.cpp
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "utility/lip_filter/LIPFilter.hpp"
+
+namespace quickstep {
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/lip_filter/LIPFilter.hpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilter.hpp b/utility/lip_filter/LIPFilter.hpp
new file mode 100644
index 0000000..c14b526
--- /dev/null
+++ b/utility/lip_filter/LIPFilter.hpp
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_HPP_
+#define QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_HPP_
+
+#include <vector>
+
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+/** \addtogroup Utility
+ * @{
+ */
+
+enum class LIPFilterType {
+ kBloomFilter,
+ kExactFilter,
+ kSingleIdentityHashFilter
+};
+
+class LIPFilter {
+ public:
+ LIPFilterType getType() const {
+ return type_;
+ }
+
+ private:
+ LIPFilterType type_;
+
+ DISALLOW_COPY_AND_ASSIGN(LIPFilter);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/lip_filter/LIPFilter.proto
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilter.proto b/utility/lip_filter/LIPFilter.proto
new file mode 100644
index 0000000..897a86e
--- /dev/null
+++ b/utility/lip_filter/LIPFilter.proto
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax = "proto2";
+
+package quickstep.serialization;
+
+import "types/Type.proto";
+
+enum LIPFilterType {
+ BLOOM_FILTER = 1;
+ EXACT_FILTER = 2;
+ SINGLE_IDENTITY_HASH_FILTER = 3;
+}
+
+message LIPFilter {
+ required LIPFilterType lip_filter_type = 1;
+
+ extensions 16 to max;
+}
+
+message SingleIdentityHashFilter {
+ extend LIPFilter {
+ // All required
+ optional uint64 num_bits = 16;
+ }
+}
+
+enum LIPFilterActionType {
+ BUILD = 1;
+ PROBE = 2;
+}
+
+message LIPFilterDeploymentInfo {
+ required LIPFilterActionType action_type = 1;
+ required uint32 lip_filter_id = 2;
+ required int32 attribute_id = 3;
+ required Type attribute_type = 4;
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/lip_filter/LIPFilterAdaptiveProber.hpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilterAdaptiveProber.hpp b/utility/lip_filter/LIPFilterAdaptiveProber.hpp
new file mode 100644
index 0000000..6005690
--- /dev/null
+++ b/utility/lip_filter/LIPFilterAdaptiveProber.hpp
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_ADAPTIVE_PROBER_HPP_
+#define QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_ADAPTIVE_PROBER_HPP_
+
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+/** \addtogroup Utility
+ * @{
+ */
+
+class LIPFilterAdaptiveProber {
+ public:
+ LIPFilterAdaptiveProber(const std::vector<const LIPFilter *> &lip_filters,
+ const std::vector<attribute_id> &attr_ids,
+ const std::vector<std::size_t> &attr_sizes) {
+ DCHECK_EQ(lip_filters.size(), attr_ids.size());
+ DCHECK_EQ(lip_filters.size(), attr_sizes.size());
+
+ probe_entries_.reserve(lip_filters.size());
+ for (std::size_t i = 0; i < lip_filters.size(); ++i) {
+ probe_entries_.emplace_back(
+ new ProbeEntry(lip_filters[i], attr_ids[i], attr_sizes[i]));
+ }
+ }
+
+ ~LIPFilterAdaptiveProber() {
+ for (ProbeEntry *entry : probe_entries_) {
+ delete entry;
+ }
+ }
+
+ private:
+ struct ProbeEntry {
+ ProbeEntry(const LIPFilter *lip_filter_in,
+ const attribute_id attr_id_in,
+ const std::size_t attr_size_in)
+ : lip_filter(lip_filter_in),
+ attr_id(attr_id_in),
+ attr_size(attr_size_in),
+ miss(0),
+ cnt(0) {
+ }
+ static bool isBetterThan(const ProbeEntry *a,
+ const ProbeEntry *b) {
+ return a->miss_rate > b->miss_rate;
+ }
+ const LIPFilter *lip_filter;
+ const attribute_id attr_id;
+ const std::size_t attr_size;
+ std::uint32_t miss;
+ std::uint32_t cnt;
+ float miss_rate;
+ };
+
+ std::vector<ProbeEntry *> probe_entries_;
+
+ DISALLOW_COPY_AND_ASSIGN(LIPFilterAdaptiveProber);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_ADAPTIVE_PROBER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/lip_filter/LIPFilterBuilder.hpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilterBuilder.hpp b/utility/lip_filter/LIPFilterBuilder.hpp
new file mode 100644
index 0000000..07b26da
--- /dev/null
+++ b/utility/lip_filter/LIPFilterBuilder.hpp
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_BUILDER_HPP_
+#define QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_BUILDER_HPP_
+
+#include <vector>
+
+#include "utility/Macros.hpp"
+
+#include "catalog/CatalogTypedefs.hpp"
+
+namespace quickstep {
+
+/** \addtogroup Utility
+ * @{
+ */
+
+class LIPFilterBuilder {
+ public:
+ LIPFilterBuilder(const std::vector<LIPFilter *> &lip_filters,
+ const std::vector<attribute_id> &attr_ids,
+ const std::vector<std::size_t> &attr_sizes) {
+ DCHECK_EQ(lip_filters.size(), attr_ids.size());
+ DCHECK_EQ(lip_filters.size(), attr_sizes.size());
+
+ build_entries_.reserve(lip_filters.size());
+ for (std::size_t i = 0; i < lip_filters.size(); ++i) {
+ build_entries_.emplace_back(lip_filters[i], attr_ids[i], attr_sizes[i]);
+ }
+ }
+
+ private:
+ struct BuildEntry {
+ BuildEntry(LIPFilter *lip_filter_in,
+ const attribute_id attr_id_in,
+ const std::size_t attr_size_in)
+ : lip_filter(lip_filter_in),
+ attr_id(attr_id_in),
+ attr_size(attr_size_in) {
+ }
+ LIPFilter *lip_filter;
+ const attribute_id attr_id;
+ const std::size_t attr_size;
+ };
+
+ std::vector<BuildEntry> build_entries_;
+
+ DISALLOW_COPY_AND_ASSIGN(LIPFilterBuilder);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/lip_filter/LIPFilterDeploymentInfo.hpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilterDeploymentInfo.hpp b/utility/lip_filter/LIPFilterDeploymentInfo.hpp
new file mode 100644
index 0000000..db75021
--- /dev/null
+++ b/utility/lip_filter/LIPFilterDeploymentInfo.hpp
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_DEPLOYMENT_INFO_HPP_
+#define QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_DEPLOYMENT_INFO_HPP_
+
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "utility/Macros.hpp"
+#include "utility/lip_filter/LIPFilter.hpp"
+
+namespace quickstep {
+
+/** \addtogroup Utility
+ * @{
+ */
+
+enum class LIPFilterActionType {
+ kBuild = 0,
+ kProbe
+};
+
+class LIPFilterDeploymentInfo {
+ public:
+ const LIPFilterActionType getActionType() const {
+ return action_type_;
+ }
+
+ const std::vector<LIPFilter*>& lip_filters() const {
+ return lip_filters_;
+ }
+
+ const std::vector<attribute_id>& attr_ids() const {
+ return attr_ids_;
+ }
+
+ const std::vector<const Type*>& attr_types() const {
+ return attr_types_;
+ }
+
+ private:
+ LIPFilterActionType action_type_;
+ std::vector<LIPFilter*> lip_filters_;
+ std::vector<attribute_id> attr_ids_;
+ std::vector<const Type*> attr_types_;
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_DEPLOYMENT_INFO_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/lip_filter/LIPFilterFactory.cpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilterFactory.cpp b/utility/lip_filter/LIPFilterFactory.cpp
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/396f8576/utility/lip_filter/LIPFilterFactory.hpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilterFactory.hpp b/utility/lip_filter/LIPFilterFactory.hpp
new file mode 100644
index 0000000..0567093
--- /dev/null
+++ b/utility/lip_filter/LIPFilterFactory.hpp
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_FACTORY_HPP_
+#define QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_FACTORY_HPP_
+
+#include <vector>
+
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+/** \addtogroup Utility
+ * @{
+ */
+
+class LIPFilterFactory {
+ public:
+
+ private:
+
+ DISALLOW_COPY_AND_ASSIGN(LIPFilterFactory);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_UTILITY_LIP_FILTER_LIP_FILTER_FACTORY_HPP_
[15/17] incubator-quickstep git commit: Undo old design
Posted by ji...@apache.org.
Undo old design
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/112d480e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/112d480e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/112d480e
Branch: refs/heads/lip-refactor
Commit: 112d480e4880b8f418ebcb0c4c2ad28f5bad0aba
Parents: 4126c4f
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Wed Sep 7 13:20:43 2016 -0500
Committer: Jianqiao Zhu <ji...@cs.wisc.edu>
Committed: Tue Oct 4 23:25:06 2016 -0500
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 11 -
query_optimizer/ExecutionGenerator.cpp | 20 --
query_optimizer/ExecutionGenerator.hpp | 5 +-
query_optimizer/ExecutionHeuristics.cpp | 129 --------
query_optimizer/ExecutionHeuristics.hpp | 157 ----------
query_optimizer/tests/CMakeLists.txt | 16 -
.../tests/ExecutionHeuristics_unittest.cpp | 311 -------------------
storage/HashTable.hpp | 103 ------
storage/HashTable.proto | 6 -
storage/HashTableFactory.hpp | 29 --
10 files changed, 1 insertion(+), 786 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/112d480e/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 32f7885..2b521ec 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -41,7 +41,6 @@ add_subdirectory(tests)
# Declare micro-libs:
add_library(quickstep_queryoptimizer_ExecutionGenerator ExecutionGenerator.cpp ExecutionGenerator.hpp)
-add_library(quickstep_queryoptimizer_ExecutionHeuristics ExecutionHeuristics.cpp ExecutionHeuristics.hpp)
add_library(quickstep_queryoptimizer_LogicalGenerator LogicalGenerator.cpp LogicalGenerator.hpp)
add_library(quickstep_queryoptimizer_LogicalToPhysicalMapper
../empty_src.cpp
@@ -73,7 +72,6 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_expressions_windowaggregation_WindowAggregateFunction_proto
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryContext_proto
- quickstep_queryoptimizer_ExecutionHeuristics
quickstep_queryoptimizer_OptimizerContext
quickstep_queryoptimizer_QueryHandle
quickstep_queryoptimizer_QueryPlan
@@ -153,14 +151,6 @@ if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_catalog_Catalog_proto)
endif()
-target_link_libraries(quickstep_queryoptimizer_ExecutionHeuristics
- glog
- quickstep_catalog_CatalogRelation
- quickstep_catalog_CatalogTypedefs
- quickstep_queryexecution_QueryContext
- quickstep_queryexecution_QueryContext_proto
- quickstep_queryoptimizer_QueryPlan
- quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_LogicalGenerator
glog
quickstep_parser_ParseStatement
@@ -233,7 +223,6 @@ target_link_libraries(quickstep_queryoptimizer_Validator
add_library(quickstep_queryoptimizer ../empty_src.cpp QueryOptimizerModule.hpp)
target_link_libraries(quickstep_queryoptimizer
quickstep_queryoptimizer_ExecutionGenerator
- quickstep_queryoptimizer_ExecutionHeuristics
quickstep_queryoptimizer_LogicalGenerator
quickstep_queryoptimizer_LogicalToPhysicalMapper
quickstep_queryoptimizer_Optimizer
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/112d480e/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 968314e..efebe92 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -54,7 +54,6 @@
#include "expressions/window_aggregation/WindowAggregateFunction.pb.h"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryContext.pb.h"
-#include "query_optimizer/ExecutionHeuristics.hpp"
#include "query_optimizer/OptimizerContext.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryPlan.hpp"
@@ -153,9 +152,6 @@ static const volatile bool aggregate_hashtable_type_dummy
DEFINE_bool(parallelize_load, true, "Parallelize loading data files.");
-DEFINE_bool(optimize_joins, false,
- "Enable post execution plan generation optimizations for joins.");
-
namespace E = ::quickstep::optimizer::expressions;
namespace P = ::quickstep::optimizer::physical;
namespace S = ::quickstep::serialization;
@@ -211,11 +207,6 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) {
temporary_relation_info.producer_operator_index);
}
- // Optimize execution plan based on heuristics captured during execution plan generation, if enabled.
- if (FLAGS_optimize_joins) {
- execution_heuristics_->optimizeExecutionPlan(execution_plan_, query_context_proto_);
- }
-
#ifdef QUICKSTEP_DISTRIBUTED
catalog_database_cache_proto_->set_name(catalog_database_->getName());
@@ -828,17 +819,6 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::forward_as_tuple(join_operator_index,
output_relation));
temporary_relation_info_vec_.emplace_back(join_operator_index, output_relation);
-
- // Add heuristics for the Hash Join, if enabled.
- if (FLAGS_optimize_joins && !skip_hash_join_optimization) {
- execution_heuristics_->addHashJoinInfo(build_operator_index,
- join_operator_index,
- referenced_stored_build_relation,
- referenced_stored_probe_relation,
- std::move(build_original_attribute_ids),
- std::move(probe_original_attribute_ids),
- join_hash_table_index);
- }
}
void ExecutionGenerator::convertNestedLoopsJoin(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/112d480e/query_optimizer/ExecutionGenerator.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.hpp b/query_optimizer/ExecutionGenerator.hpp
index 6017aa5..a830e65 100644
--- a/query_optimizer/ExecutionGenerator.hpp
+++ b/query_optimizer/ExecutionGenerator.hpp
@@ -33,7 +33,6 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryContext.pb.h"
-#include "query_optimizer/ExecutionHeuristics.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryPlan.hpp"
#include "query_optimizer/cost_model/CostModel.hpp"
@@ -103,8 +102,7 @@ class ExecutionGenerator {
: catalog_database_(DCHECK_NOTNULL(catalog_database)),
query_handle_(DCHECK_NOTNULL(query_handle)),
execution_plan_(DCHECK_NOTNULL(query_handle->getQueryPlanMutable())),
- query_context_proto_(DCHECK_NOTNULL(query_handle->getQueryContextProtoMutable())),
- execution_heuristics_(new ExecutionHeuristics()) {
+ query_context_proto_(DCHECK_NOTNULL(query_handle->getQueryContextProtoMutable())) {
query_context_proto_->set_query_id(query_handle_->query_id());
#ifdef QUICKSTEP_DISTRIBUTED
catalog_database_cache_proto_ = DCHECK_NOTNULL(query_handle->getCatalogDatabaseCacheProtoMutable());
@@ -387,7 +385,6 @@ class ExecutionGenerator {
QueryHandle *query_handle_;
QueryPlan *execution_plan_; // A part of QueryHandle.
serialization::QueryContext *query_context_proto_; // A part of QueryHandle.
- std::unique_ptr<ExecutionHeuristics> execution_heuristics_;
#ifdef QUICKSTEP_DISTRIBUTED
serialization::CatalogDatabase *catalog_database_cache_proto_; // A part of QueryHandle.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/112d480e/query_optimizer/ExecutionHeuristics.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionHeuristics.cpp b/query_optimizer/ExecutionHeuristics.cpp
deleted file mode 100644
index 4fd7320..0000000
--- a/query_optimizer/ExecutionHeuristics.cpp
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- **/
-
-#include "query_optimizer/ExecutionHeuristics.hpp"
-
-#include <cstddef>
-#include <utility>
-#include <unordered_map>
-#include <vector>
-
-#include "catalog/CatalogTypedefs.hpp"
-#include "query_execution/QueryContext.pb.h"
-#include "query_optimizer/QueryPlan.hpp"
-#include "utility/Macros.hpp"
-
-#include "glog/logging.h"
-
-namespace quickstep {
-namespace optimizer {
-
-void ExecutionHeuristics::optimizeExecutionPlan(QueryPlan *query_plan,
- serialization::QueryContext *query_context_proto) {
- // Currently this only optimizes left deep joins using bloom filters.
- // It uses a simple algorithm to discover the left deep joins.
- // It starts with the first hash join in the plan and keeps on iterating
- // over the next hash joins, till a probe on a different relation id is found.
- // The set of hash joins found in this way forms a chain and can be recognized
- // as a left deep join. It becomes a candidate for optimization.
-
- // The optimization is done by modifying each of the build operators in the chain
- // to generate a bloom filter on the build key during their hash table creation.
- // The leaf-level probe operator is then modified to query all the bloom
- // filters generated from all the build operators in the chain. These
- // bloom filters are queried to test the membership of the probe key
- // just prior to probing the hash table.
-
- QueryPlan::DAGNodeIndex origin_node = 0;
- while (origin_node < hash_joins_.size() - 1) {
- std::vector<std::size_t> chained_nodes;
- chained_nodes.push_back(origin_node);
- for (std::size_t i = origin_node + 1; i < hash_joins_.size(); ++i) {
- const relation_id checked_relation_id = hash_joins_[origin_node].referenced_stored_probe_relation_->getID();
- const relation_id expected_relation_id = hash_joins_[i].referenced_stored_probe_relation_->getID();
- if (checked_relation_id == expected_relation_id) {
- chained_nodes.push_back(i);
- } else {
- break;
- }
- }
-
- // Only chains of length greater than one are suitable candidates for semi-join optimization.
- if (chained_nodes.size() > 1) {
- std::unordered_map<QueryContext::bloom_filter_id, std::vector<attribute_id>> probe_bloom_filter_info;
- for (const std::size_t node : chained_nodes) {
- // Provision for a new bloom filter to be used by the build operator.
- const QueryContext::bloom_filter_id bloom_filter_id = query_context_proto->bloom_filters_size();
- serialization::BloomFilter *bloom_filter_proto = query_context_proto->add_bloom_filters();
-
- // Modify the bloom filter properties based on the statistics of the relation.
- setBloomFilterProperties(bloom_filter_proto, hash_joins_[node].referenced_stored_build_relation_);
-
- // Add build-side bloom filter information to the corresponding hash table proto.
- query_context_proto->mutable_join_hash_tables(hash_joins_[node].join_hash_table_id_)
- ->add_build_side_bloom_filter_id(bloom_filter_id);
-
- probe_bloom_filter_info.insert(std::make_pair(bloom_filter_id, hash_joins_[node].probe_attributes_));
- }
-
- // Add probe-side bloom filter information to the corresponding hash table proto for each build-side bloom filter.
- for (const std::pair<QueryContext::bloom_filter_id, std::vector<attribute_id>>
- &bloom_filter_info : probe_bloom_filter_info) {
- auto *probe_side_bloom_filter =
- query_context_proto->mutable_join_hash_tables(hash_joins_[origin_node].join_hash_table_id_)
- ->add_probe_side_bloom_filters();
- probe_side_bloom_filter->set_probe_side_bloom_filter_id(bloom_filter_info.first);
- for (const attribute_id &probe_attribute_id : bloom_filter_info.second) {
- probe_side_bloom_filter->add_probe_side_attr_ids(probe_attribute_id);
- }
- }
-
- // Add node dependencies from chained build nodes to origin node probe.
- for (std::size_t i = 1; i < chained_nodes.size(); ++i) { // Note: It starts from index 1.
- query_plan->addDirectDependency(hash_joins_[origin_node].join_operator_index_,
- hash_joins_[origin_node + i].build_operator_index_,
- true /* is_pipeline_breaker */);
- }
- }
-
- // Update the origin node.
- origin_node = chained_nodes.back() + 1;
- }
-}
-
-void ExecutionHeuristics::setBloomFilterProperties(serialization::BloomFilter *bloom_filter_proto,
- const CatalogRelation *relation) {
- const std::size_t cardinality = relation->estimateTupleCardinality();
- if (cardinality < kOneThousand) {
- bloom_filter_proto->set_bloom_filter_size(kOneThousand / kCompressionFactor);
- bloom_filter_proto->set_number_of_hashes(kVeryLowSparsityHash);
- } else if (cardinality < kTenThousand) {
- bloom_filter_proto->set_bloom_filter_size(kTenThousand / kCompressionFactor);
- bloom_filter_proto->set_number_of_hashes(kLowSparsityHash);
- } else if (cardinality < kHundredThousand) {
- bloom_filter_proto->set_bloom_filter_size(kHundredThousand / kCompressionFactor);
- bloom_filter_proto->set_number_of_hashes(kMediumSparsityHash);
- } else {
- bloom_filter_proto->set_bloom_filter_size(kMillion / kCompressionFactor);
- bloom_filter_proto->set_number_of_hashes(kHighSparsityHash);
- }
-}
-
-} // namespace optimizer
-} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/112d480e/query_optimizer/ExecutionHeuristics.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionHeuristics.hpp b/query_optimizer/ExecutionHeuristics.hpp
deleted file mode 100644
index 8ad3b7a..0000000
--- a/query_optimizer/ExecutionHeuristics.hpp
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- **/
-
-#ifndef QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_HEURISTICS_HPP_
-#define QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_HEURISTICS_HPP_
-
-#include <vector>
-
-#include "catalog/CatalogRelation.hpp"
-#include "catalog/CatalogTypedefs.hpp"
-#include "query_execution/QueryContext.hpp"
-#include "query_execution/QueryContext.pb.h"
-#include "query_optimizer/QueryPlan.hpp"
-#include "utility/Macros.hpp"
-
-#include "glog/logging.h"
-
-namespace quickstep {
-namespace optimizer {
-
-/** \addtogroup QueryOptimizer
- * @{
- */
-
-/**
- * @brief The ExecutionHeuristics compiles certain heuristics for an execution plan
- * as it is being converted to a physical plan. These heuristics can then be
- * used to optimize the execution plan after it has been generated.
- **/
-class ExecutionHeuristics {
- public:
- static const std::size_t kOneHundred = 100;
- static const std::size_t kOneThousand = 1000;
- static const std::size_t kTenThousand = 10000;
- static const std::size_t kHundredThousand = 100000;
- static const std::size_t kMillion = 1000000;
-
- static const std::size_t kCompressionFactor = 10;
-
- static const std::size_t kVeryLowSparsityHash = 1;
- static const std::size_t kLowSparsityHash = 2;
- static const std::size_t kMediumSparsityHash = 5;
- static const std::size_t kHighSparsityHash = 10;
-
- /**
- * @brief A simple internal class that holds information about various
- * hash joins within the execution plan for a query.
- **/
- struct HashJoinInfo {
- HashJoinInfo(const QueryPlan::DAGNodeIndex build_operator_index,
- const QueryPlan::DAGNodeIndex join_operator_index,
- const CatalogRelation *referenced_stored_build_relation,
- const CatalogRelation *referenced_stored_probe_relation,
- std::vector<attribute_id> &&build_attributes,
- std::vector<attribute_id> &&probe_attributes,
- const QueryContext::join_hash_table_id join_hash_table_id)
- : build_operator_index_(build_operator_index),
- join_operator_index_(join_operator_index),
- referenced_stored_build_relation_(referenced_stored_build_relation),
- referenced_stored_probe_relation_(referenced_stored_probe_relation),
- build_attributes_(std::move(build_attributes)),
- probe_attributes_(std::move(probe_attributes)),
- join_hash_table_id_(join_hash_table_id) {
- }
-
- const QueryPlan::DAGNodeIndex build_operator_index_;
- const QueryPlan::DAGNodeIndex join_operator_index_;
- const CatalogRelation *referenced_stored_build_relation_;
- const CatalogRelation *referenced_stored_probe_relation_;
- const std::vector<attribute_id> build_attributes_;
- const std::vector<attribute_id> probe_attributes_;
- const QueryContext::join_hash_table_id join_hash_table_id_;
- };
-
-
- /**
- * @brief Constructor.
- **/
- ExecutionHeuristics() {}
-
- /**
- * @brief Saves information about a hash join used within the execution plan
- * for a query.
- *
- * @param build_operator_index Index of the build operator of the hash join.
- * @param join_operator_index Index of the join operator of the hash join.
- * @param build_relation_id Id of the relation on which hash table is being built.
- * @param probe_relation_id Id of the relation on which hash table is being probed.
- * @param build_attributes List of attributes on which hash table is being built.
- * @param probe_attributes List of attributes on which hash table is being probed.
- * @param join_hash_table_id Id of the hash table which refers to the actual hash
- * table within the query context.
- **/
- inline void addHashJoinInfo(const QueryPlan::DAGNodeIndex build_operator_index,
- const QueryPlan::DAGNodeIndex join_operator_index,
- const CatalogRelation *referenced_stored_build_relation,
- const CatalogRelation *referenced_stored_probe_relation,
- std::vector<attribute_id> &&build_attributes,
- std::vector<attribute_id> &&probe_attributes,
- const QueryContext::join_hash_table_id join_hash_table_id) {
- hash_joins_.push_back(HashJoinInfo(build_operator_index,
- join_operator_index,
- referenced_stored_build_relation,
- referenced_stored_probe_relation,
- std::move(build_attributes),
- std::move(probe_attributes),
- join_hash_table_id));
- }
-
- /**
- * @brief Optimize the execution plan based on heuristics generated
- * during physical plan to execution plan conversion.
- *
- * @param query_plan A mutable reference to the query execution plan.
- * @param query_context_proto A mutable reference to the protobuf representation
- * of the query context.
- **/
- void optimizeExecutionPlan(QueryPlan *query_plan, serialization::QueryContext *query_context_proto);
-
- /**
- * @brief Set the properties of the bloom filter proto based on the statistics
- * of the given relation.
- *
- * @param bloom_filter_proto A mutable reference to the bloom filter protobuf representation.
- * @param relation The catalog relation on which bloom filter is being built.
- **/
- void setBloomFilterProperties(serialization::BloomFilter *bloom_filter_proto,
- const CatalogRelation *relation);
-
- private:
- std::vector<HashJoinInfo> hash_joins_;
-
- DISALLOW_COPY_AND_ASSIGN(ExecutionHeuristics);
-};
-
-/** @} */
-
-} // namespace optimizer
-} // namespace quickstep
-
-#endif /* QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_HEURISTICS_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/112d480e/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/CMakeLists.txt b/query_optimizer/tests/CMakeLists.txt
index 597dbe0..ac4548a 100644
--- a/query_optimizer/tests/CMakeLists.txt
+++ b/query_optimizer/tests/CMakeLists.txt
@@ -94,22 +94,6 @@ add_executable(quickstep_queryoptimizer_tests_ExecutionGeneratorTest
ExecutionGeneratorTestRunner.hpp
"${PROJECT_SOURCE_DIR}/utility/textbased_test/TextBasedTest.cpp"
"${PROJECT_SOURCE_DIR}/utility/textbased_test/TextBasedTest.hpp")
-add_executable(ExecutionHeuristics_unittest ExecutionHeuristics_unittest.cpp)
-target_link_libraries(ExecutionHeuristics_unittest
- gtest
- gtest_main
- quickstep_catalog_Catalog
- quickstep_catalog_CatalogDatabase
- quickstep_catalog_CatalogTypedefs
- quickstep_queryexecution_QueryContext
- quickstep_queryexecution_QueryContext_proto
- quickstep_queryoptimizer_ExecutionHeuristics
- quickstep_queryoptimizer_QueryPlan
- quickstep_relationaloperators_BuildHashOperator
- quickstep_relationaloperators_HashJoinOperator
- quickstep_utility_Macros)
-add_test(ExecutionHeuristics_unittest ExecutionHeuristics_unittest)
-
add_executable(quickstep_queryoptimizer_tests_OptimizerTextTest
OptimizerTextTest.cpp
OptimizerTextTestRunner.cpp
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/112d480e/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/ExecutionHeuristics_unittest.cpp b/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
deleted file mode 100644
index 73b3e84..0000000
--- a/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
+++ /dev/null
@@ -1,311 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- **/
-
-#include <cstddef>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "catalog/Catalog.hpp"
-#include "catalog/CatalogDatabase.hpp"
-#include "catalog/CatalogTypedefs.hpp"
-#include "query_execution/QueryContext.hpp"
-#include "query_execution/QueryContext.pb.h"
-#include "query_optimizer/ExecutionHeuristics.hpp"
-#include "query_optimizer/QueryPlan.hpp"
-#include "relational_operators/BuildHashOperator.hpp"
-#include "relational_operators/HashJoinOperator.hpp"
-#include "utility/Macros.hpp"
-
-#include "glog/logging.h"
-#include "gtest/gtest.h"
-
-namespace quickstep {
-namespace optimizer {
-
-namespace {
-constexpr std::size_t kQueryId = 0;
-}
-
-class ExecutionHeuristicsTest : public ::testing::Test {
- protected:
- virtual void SetUp() {
- db_ = cat_.getDatabaseByIdMutable(cat_.addDatabase(new CatalogDatabase(nullptr, "db")));
- execution_heuristics_.reset(new ExecutionHeuristics());
- query_plan_.reset(new QueryPlan());
- query_context_proto_.reset(new serialization::QueryContext());
- }
-
- CatalogRelation* createCatalogRelation(const std::string &name, bool temporary = false) {
- return db_->getRelationByIdMutable(db_->addRelation(new CatalogRelation(nullptr, name, -1, temporary)));
- }
-
- void addDummyHashJoinInfo(ExecutionHeuristics *execution_heuristics,
- const QueryPlan::DAGNodeIndex build_operator_index,
- const QueryPlan::DAGNodeIndex join_operator_index,
- const CatalogRelation *build_relation,
- const CatalogRelation *probe_relation,
- const attribute_id build_attribute_id,
- const attribute_id probe_attribute_id,
- const QueryContext::join_hash_table_id join_hash_table_id) {
- std::vector<attribute_id> build_attribute_ids(1, build_attribute_id);
- std::vector<attribute_id> probe_attribute_ids(1, probe_attribute_id);
- execution_heuristics->addHashJoinInfo(build_operator_index,
- join_operator_index,
- build_relation,
- probe_relation,
- std::move(build_attribute_ids),
- std::move(probe_attribute_ids),
- join_hash_table_id);
- }
-
- QueryPlan::DAGNodeIndex createDummyBuildHashOperator(QueryPlan *query_plan,
- const CatalogRelation *build_relation,
- const attribute_id build_attribute_id,
- const QueryContext::join_hash_table_id join_hash_table_index) {
- std::vector<attribute_id> build_attribute_ids;
- build_attribute_ids.push_back(build_attribute_id);
- QueryPlan::DAGNodeIndex build_operator_index =
- query_plan->addRelationalOperator(new BuildHashOperator(kQueryId,
- *build_relation,
- true,
- build_attribute_ids,
- false,
- join_hash_table_index));
- return build_operator_index;
- }
-
- QueryPlan::DAGNodeIndex createDummyHashJoinOperator(QueryPlan *query_plan,
- const CatalogRelation *build_relation,
- const CatalogRelation *probe_relation,
- const attribute_id probe_attribute_id,
- const QueryContext::join_hash_table_id join_hash_table_index) {
- std::vector<attribute_id> probe_attribute_ids;
- probe_attribute_ids.push_back(probe_attribute_id);
- QueryPlan::DAGNodeIndex join_operator_index =
- query_plan->addRelationalOperator(
- new HashJoinOperator(kQueryId,
- *build_relation,
- *probe_relation,
- true,
- probe_attribute_ids,
- false,
- *probe_relation,
- 0,
- join_hash_table_index,
- 0,
- 0));
- return join_operator_index;
- }
-
- Catalog cat_;
- CatalogDatabase *db_; // db_ is owned by cat_.
- std::unique_ptr<QueryPlan> query_plan_;
- std::unique_ptr<serialization::QueryContext> query_context_proto_;
- std::unique_ptr<ExecutionHeuristics> execution_heuristics_;
-};
-
-TEST_F(ExecutionHeuristicsTest, HashJoinOptimizedTest) {
- // This test case creates three hash joins, all of which are being probed on the same relation.
- // Since the probe are being made on the same relation, ExecutionHeuristics should optimize
- // these hash joins using bloom filters.
-
- const CatalogRelation *build_relation_1 = createCatalogRelation("build_relation_1");
- const CatalogRelation *build_relation_2 = createCatalogRelation("build_relation_2");
- const CatalogRelation *build_relation_3 = createCatalogRelation("build_relation_3");
- const CatalogRelation *probe_relation_1 = createCatalogRelation("probe_relation_1");
-
- const attribute_id build_attribute_id_1 = 0;
- const attribute_id build_attribute_id_2 = 0;
- const attribute_id build_attribute_id_3 = 0;
- const attribute_id probe_attribute_id_1 = 1;
- const attribute_id probe_attribute_id_2 = 2;
- const attribute_id probe_attribute_id_3 = 3;
-
- const QueryContext::join_hash_table_id join_hash_table_index_1 = 0;
- const QueryContext::join_hash_table_id join_hash_table_index_2 = 1;
- const QueryContext::join_hash_table_id join_hash_table_index_3 = 2;
- query_context_proto_->add_join_hash_tables();
- query_context_proto_->add_join_hash_tables();
- query_context_proto_->add_join_hash_tables();
-
- const QueryPlan::DAGNodeIndex build_operator_index_1 = createDummyBuildHashOperator(query_plan_.get(),
- build_relation_1,
- build_attribute_id_1,
- join_hash_table_index_1);
- const QueryPlan::DAGNodeIndex probe_operator_index_1 = createDummyHashJoinOperator(query_plan_.get(),
- build_relation_1,
- probe_relation_1,
- probe_attribute_id_1,
- join_hash_table_index_1);
- const QueryPlan::DAGNodeIndex build_operator_index_2 = createDummyBuildHashOperator(query_plan_.get(),
- build_relation_2,
- build_attribute_id_2,
- join_hash_table_index_2);
- const QueryPlan::DAGNodeIndex probe_operator_index_2 = createDummyHashJoinOperator(query_plan_.get(),
- build_relation_2,
- probe_relation_1,
- probe_attribute_id_2,
- join_hash_table_index_2);
- const QueryPlan::DAGNodeIndex build_operator_index_3 = createDummyBuildHashOperator(query_plan_.get(),
- build_relation_3,
- build_attribute_id_3,
- join_hash_table_index_3);
- const QueryPlan::DAGNodeIndex probe_operator_index_3 = createDummyHashJoinOperator(query_plan_.get(),
- build_relation_3,
- probe_relation_1,
- probe_attribute_id_3,
- join_hash_table_index_3);
-
- addDummyHashJoinInfo(execution_heuristics_.get(),
- build_operator_index_1,
- probe_operator_index_1,
- build_relation_1,
- probe_relation_1,
- build_attribute_id_1,
- probe_attribute_id_1,
- join_hash_table_index_1);
- addDummyHashJoinInfo(execution_heuristics_.get(),
- build_operator_index_2,
- probe_operator_index_2,
- build_relation_2,
- probe_relation_1,
- build_attribute_id_2,
- probe_attribute_id_2,
- join_hash_table_index_2);
- addDummyHashJoinInfo(execution_heuristics_.get(),
- build_operator_index_3,
- probe_operator_index_3,
- build_relation_3,
- probe_relation_1,
- build_attribute_id_3,
- probe_attribute_id_3,
- join_hash_table_index_3);
-
- execution_heuristics_->optimizeExecutionPlan(query_plan_.get(), query_context_proto_.get());
-
- // Test whether correct number of bloom filters were added.
- EXPECT_EQ(1, query_context_proto_->join_hash_tables(0).build_side_bloom_filter_id_size());
- EXPECT_EQ(1, query_context_proto_->join_hash_tables(1).build_side_bloom_filter_id_size());
- EXPECT_EQ(1, query_context_proto_->join_hash_tables(2).build_side_bloom_filter_id_size());
- EXPECT_EQ(3, query_context_proto_->join_hash_tables(0).probe_side_bloom_filters_size());
-
- // Test that the DAG was modified correctly or not.
- // Probe operator 1 should have now build operator 1 and build operator 2 added as dependencies.
- auto const probe_node_dependencies = query_plan_->getQueryPlanDAG().getDependencies(probe_operator_index_1);
- EXPECT_EQ(1u, probe_node_dependencies.count(build_operator_index_2));
- EXPECT_EQ(1u, probe_node_dependencies.count(build_operator_index_3));
-}
-
-TEST_F(ExecutionHeuristicsTest, HashJoinNotOptimizedTest) {
- // This test case creates three hash joins, all of which are being probed on different relations.
- // Since the probe are being made on the different relations, ExecutionHeuristics should optimize
- // these hash joins using bloom filters.
-
- const CatalogRelation *build_relation_1 = createCatalogRelation("build_relation_1");
- const CatalogRelation *build_relation_2 = createCatalogRelation("build_relation_2");
- const CatalogRelation *build_relation_3 = createCatalogRelation("build_relation_3");
- const CatalogRelation *probe_relation_1 = createCatalogRelation("probe_relation_1");
- const CatalogRelation *probe_relation_2 = createCatalogRelation("probe_relation_2");
- const CatalogRelation *probe_relation_3 = createCatalogRelation("probe_relation_3");
-
- const attribute_id build_attribute_id_1 = 0;
- const attribute_id build_attribute_id_2 = 0;
- const attribute_id build_attribute_id_3 = 0;
- const attribute_id probe_attribute_id_1 = 1;
- const attribute_id probe_attribute_id_2 = 2;
- const attribute_id probe_attribute_id_3 = 3;
-
- const QueryContext::join_hash_table_id join_hash_table_index_1 = 0;
- const QueryContext::join_hash_table_id join_hash_table_index_2 = 1;
- const QueryContext::join_hash_table_id join_hash_table_index_3 = 2;
- query_context_proto_->add_join_hash_tables();
- query_context_proto_->add_join_hash_tables();
- query_context_proto_->add_join_hash_tables();
-
- const QueryPlan::DAGNodeIndex build_operator_index_1 = createDummyBuildHashOperator(query_plan_.get(),
- build_relation_1,
- build_attribute_id_1,
- join_hash_table_index_1);
- const QueryPlan::DAGNodeIndex probe_operator_index_1 = createDummyHashJoinOperator(query_plan_.get(),
- build_relation_1,
- probe_relation_1,
- probe_attribute_id_1,
- join_hash_table_index_1);
- const QueryPlan::DAGNodeIndex build_operator_index_2 = createDummyBuildHashOperator(query_plan_.get(),
- build_relation_2,
- build_attribute_id_2,
- join_hash_table_index_2);
- const QueryPlan::DAGNodeIndex probe_operator_index_2 = createDummyHashJoinOperator(query_plan_.get(),
- build_relation_2,
- probe_relation_2,
- probe_attribute_id_2,
- join_hash_table_index_2);
- const QueryPlan::DAGNodeIndex build_operator_index_3 = createDummyBuildHashOperator(query_plan_.get(),
- build_relation_3,
- build_attribute_id_3,
- join_hash_table_index_3);
- const QueryPlan::DAGNodeIndex probe_operator_index_3 = createDummyHashJoinOperator(query_plan_.get(),
- build_relation_3,
- probe_relation_3,
- probe_attribute_id_3,
- join_hash_table_index_3);
-
- addDummyHashJoinInfo(execution_heuristics_.get(),
- build_operator_index_1,
- probe_operator_index_1,
- build_relation_1,
- probe_relation_1,
- build_attribute_id_1,
- probe_attribute_id_1,
- join_hash_table_index_1);
- addDummyHashJoinInfo(execution_heuristics_.get(),
- build_operator_index_2,
- probe_operator_index_2,
- build_relation_2,
- probe_relation_2,
- build_attribute_id_2,
- probe_attribute_id_2,
- join_hash_table_index_2);
- addDummyHashJoinInfo(execution_heuristics_.get(),
- build_operator_index_3,
- probe_operator_index_3,
- build_relation_3,
- probe_relation_3,
- build_attribute_id_3,
- probe_attribute_id_3,
- join_hash_table_index_3);
-
- execution_heuristics_->optimizeExecutionPlan(query_plan_.get(), query_context_proto_.get());
-
- // Test that no bloom filters were added.
- EXPECT_EQ(0, query_context_proto_->join_hash_tables(0).build_side_bloom_filter_id_size());
- EXPECT_EQ(0, query_context_proto_->join_hash_tables(1).build_side_bloom_filter_id_size());
- EXPECT_EQ(0, query_context_proto_->join_hash_tables(2).build_side_bloom_filter_id_size());
- EXPECT_EQ(0, query_context_proto_->join_hash_tables(0).probe_side_bloom_filters_size());
-
- // Test that the DAG was not modified at all.
- // Probe operator 1 should not have build operator 1 and build operator 2 added as dependencies.
- auto probe_node_dependencies = query_plan_->getQueryPlanDAG().getDependencies(probe_operator_index_1);
- EXPECT_EQ(0u, probe_node_dependencies.count(build_operator_index_2));
- EXPECT_EQ(0u, probe_node_dependencies.count(build_operator_index_3));
-}
-
-} // namespace optimizer
-} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/112d480e/storage/HashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTable.hpp b/storage/HashTable.hpp
index f2dcb03..786a9bb 100644
--- a/storage/HashTable.hpp
+++ b/storage/HashTable.hpp
@@ -981,61 +981,6 @@ class HashTable : public HashTableBase<resizable,
template <typename FunctorT>
std::size_t forEachCompositeKey(FunctorT *functor) const;
- /**
- * @brief A call to this function will cause a bloom filter to be built
- * during the build phase of this hash table.
- **/
- inline void enableBuildSideBloomFilter() {
- has_build_side_bloom_filter_ = true;
- }
-
- /**
- * @brief A call to this function will cause a set of bloom filters to be
- * probed during the probe phase of this hash table.
- **/
- inline void enableProbeSideBloomFilter() {
- has_probe_side_bloom_filter_ = true;
- }
-
- /**
- * @brief This function sets the pointer to the bloom filter to be
- * used during the build phase of this hash table.
- * @warning Should call enable_build_side_bloom_filter() first to enable
- * bloom filter usage during build phase.
- * @note The ownership of the bloom filter lies with the caller.
- *
- * @param bloom_filter The pointer to the bloom filter.
- **/
- inline void setBuildSideBloomFilter(BloomFilter *bloom_filter) {
- build_bloom_filter_ = bloom_filter;
- }
-
- /**
- * @brief This function adds a pointer to the list of bloom filters to be
- * used during the probe phase of this hash table.
- * @warning Should call enable_probe_side_bloom_filter() first to enable
- * bloom filter usage during probe phase.
- * @note The ownership of the bloom filter lies with the caller.
- *
- * @param bloom_filter The pointer to the bloom filter.
- **/
- inline void addProbeSideBloomFilter(const BloomFilter *bloom_filter) {
- probe_bloom_filters_.emplace_back(bloom_filter);
- }
-
- /**
- * @brief This function adds a vector of attribute ids corresponding to a
- * bloom filter used during the probe phase of this hash table.
- * @warning Should call enable_probe_side_bloom_filter() first to enable
- * bloom filter usage during probe phase.
- *
- * @param probe_attribute_ids The vector of attribute ids to use for probing
- * the bloom filter.
- **/
- inline void addProbeSideAttributeIds(std::vector<attribute_id> &&probe_attribute_ids) {
- probe_attribute_ids_.push_back(probe_attribute_ids);
- }
-
protected:
/**
* @brief Constructor for new resizable hash table.
@@ -1316,13 +1261,6 @@ class HashTable : public HashTableBase<resizable,
const attribute_id key_attr_id,
FunctorT *functor) const;
- // Data structures used for bloom filter optimized semi-joins.
- bool has_build_side_bloom_filter_ = false;
- bool has_probe_side_bloom_filter_ = false;
- BloomFilter *build_bloom_filter_;
- std::vector<const BloomFilter*> probe_bloom_filters_;
- std::vector<std::vector<attribute_id>> probe_attribute_ids_;
-
DISALLOW_COPY_AND_ASSIGN(HashTable);
};
@@ -1467,12 +1405,6 @@ HashTablePutResult HashTable<ValueT, resizable, serializable, force_key_copy, al
&prealloc_state);
}
}
- std::unique_ptr<BloomFilter> thread_local_bloom_filter;
- if (has_build_side_bloom_filter_) {
- thread_local_bloom_filter.reset(new BloomFilter(build_bloom_filter_->getRandomSeed(),
- build_bloom_filter_->getNumberOfHashes(),
- build_bloom_filter_->getBitArraySize()));
- }
if (resizable) {
while (result == HashTablePutResult::kOutOfSpace) {
{
@@ -1488,11 +1420,6 @@ HashTablePutResult HashTable<ValueT, resizable, serializable, force_key_copy, al
variable_size,
(*functor)(*accessor),
using_prealloc ? &prealloc_state : nullptr);
- // Insert into bloom filter, if enabled.
- if (has_build_side_bloom_filter_) {
- thread_local_bloom_filter->insertUnSafe(static_cast<const std::uint8_t *>(key.getDataPtr()),
- key.getDataSize());
- }
if (result == HashTablePutResult::kDuplicateKey) {
DEBUG_ASSERT(!using_prealloc);
return result;
@@ -1518,20 +1445,11 @@ HashTablePutResult HashTable<ValueT, resizable, serializable, force_key_copy, al
variable_size,
(*functor)(*accessor),
using_prealloc ? &prealloc_state : nullptr);
- // Insert into bloom filter, if enabled.
- if (has_build_side_bloom_filter_) {
- thread_local_bloom_filter->insertUnSafe(static_cast<const std::uint8_t *>(key.getDataPtr()),
- key.getDataSize());
- }
if (result != HashTablePutResult::kOK) {
return result;
}
}
}
- // Update the build side bloom filter with thread local copy, if available.
- if (has_build_side_bloom_filter_) {
- build_bloom_filter_->bitwiseOr(thread_local_bloom_filter.get());
- }
return HashTablePutResult::kOK;
});
@@ -2237,27 +2155,6 @@ void HashTable<ValueT, resizable, serializable, force_key_copy, allow_duplicate_
accessor,
[&](auto *accessor) -> void { // NOLINT(build/c++11)
while (accessor->next()) {
- // Probe any bloom filters, if enabled.
- if (has_probe_side_bloom_filter_) {
- DCHECK_EQ(probe_bloom_filters_.size(), probe_attribute_ids_.size());
- // Check if the key is contained in the BloomFilters or not.
- bool bloom_miss = false;
- for (std::size_t i = 0; i < probe_bloom_filters_.size() && !bloom_miss; ++i) {
- const BloomFilter *bloom_filter = probe_bloom_filters_[i];
- for (const attribute_id &attr_id : probe_attribute_ids_[i]) {
- TypedValue bloom_key = accessor->getTypedValue(attr_id);
- if (!bloom_filter->contains(static_cast<const std::uint8_t*>(bloom_key.getDataPtr()),
- bloom_key.getDataSize())) {
- bloom_miss = true;
- break;
- }
- }
- }
- if (bloom_miss) {
- continue; // On a bloom filter miss, probing the hash table can be skipped.
- }
- }
-
TypedValue key = accessor->getTypedValue(key_attr_id);
if (check_for_null_keys && key.isNull()) {
continue;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/112d480e/storage/HashTable.proto
----------------------------------------------------------------------
diff --git a/storage/HashTable.proto b/storage/HashTable.proto
index ade30d8..1d4ccb0 100644
--- a/storage/HashTable.proto
+++ b/storage/HashTable.proto
@@ -34,10 +34,4 @@ message HashTable {
required HashTableImplType hash_table_impl_type = 1;
repeated Type key_types = 2;
required uint64 estimated_num_entries = 3;
- repeated uint32 build_side_bloom_filter_id = 4;
- message ProbeSideBloomFilter {
- required uint32 probe_side_bloom_filter_id = 1;
- repeated uint32 probe_side_attr_ids = 2;
- }
- repeated ProbeSideBloomFilter probe_side_bloom_filters = 6;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/112d480e/storage/HashTableFactory.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTableFactory.hpp b/storage/HashTableFactory.hpp
index 40b39de..cd79723 100644
--- a/storage/HashTableFactory.hpp
+++ b/storage/HashTableFactory.hpp
@@ -316,35 +316,6 @@ class HashTableFactory {
key_types,
proto.estimated_num_entries(),
storage_manager);
-
- // TODO(ssaurabh): These lazy initializations can be moved from here and pushed to the
- // individual implementations of the hash table constructors.
-
- // Check if there are any build side bloom filter defined on the hash table.
- if (proto.build_side_bloom_filter_id_size() > 0) {
- hash_table->enableBuildSideBloomFilter();
- hash_table->setBuildSideBloomFilter(bloom_filters[proto.build_side_bloom_filter_id(0)].get());
- }
-
- // Check if there are any probe side bloom filters defined on the hash table.
- if (proto.probe_side_bloom_filters_size() > 0) {
- hash_table->enableProbeSideBloomFilter();
- // Add as many probe bloom filters as defined by the proto.
- for (int j = 0; j < proto.probe_side_bloom_filters_size(); ++j) {
- // Add the pointer to the probe bloom filter within the list of probe bloom filters to use.
- const auto probe_side_bloom_filter = proto.probe_side_bloom_filters(j);
- hash_table->addProbeSideBloomFilter(bloom_filters[probe_side_bloom_filter.probe_side_bloom_filter_id()].get());
-
- // Add the attribute ids corresponding to this probe bloom filter.
- std::vector<attribute_id> probe_attribute_ids;
- for (int k = 0; k < probe_side_bloom_filter.probe_side_attr_ids_size(); ++k) {
- const attribute_id probe_attribute_id = probe_side_bloom_filter.probe_side_attr_ids(k);
- probe_attribute_ids.push_back(probe_attribute_id);
- }
- hash_table->addProbeSideAttributeIds(std::move(probe_attribute_ids));
- }
- }
-
return hash_table;
}
[10/17] incubator-quickstep git commit: Refactored RebuildStatus.
Posted by ji...@apache.org.
Refactored RebuildStatus.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/6b377d5d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/6b377d5d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/6b377d5d
Branch: refs/heads/lip-refactor
Commit: 6b377d5d6c6f2e05ef74a4c1a022590af2db6571
Parents: ac3512c
Author: Zuyu Zhang <zu...@twitter.com>
Authored: Sat Sep 24 15:05:16 2016 -0700
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Sat Sep 24 15:05:16 2016 -0700
----------------------------------------------------------------------
query_execution/QueryExecutionState.hpp | 72 ++++++++++++++++++++--------
1 file changed, 53 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b377d5d/query_execution/QueryExecutionState.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionState.hpp b/query_execution/QueryExecutionState.hpp
index 9ae9563..f5281d5 100644
--- a/query_execution/QueryExecutionState.hpp
+++ b/query_execution/QueryExecutionState.hpp
@@ -92,8 +92,15 @@ class QueryExecutionState {
const std::size_t num_rebuild_workorders,
const bool rebuild_initiated) {
DCHECK(operator_index < num_operators_);
- rebuild_status_[operator_index].second = num_rebuild_workorders;
- rebuild_status_[operator_index].first = rebuild_initiated;
+ auto search_res = rebuild_status_.find(operator_index);
+ if (search_res != rebuild_status_.end()) {
+ search_res->second.has_initiated = rebuild_initiated;
+ search_res->second.num_pending_workorders = num_rebuild_workorders;
+ } else {
+ RebuildStatus rebuild_status(rebuild_initiated, num_rebuild_workorders);
+
+ rebuild_status_.emplace(operator_index, std::move(rebuild_status));
+ }
}
/**
@@ -107,7 +114,7 @@ class QueryExecutionState {
DCHECK(operator_index < num_operators_);
const auto search_res = rebuild_status_.find(operator_index);
if (search_res != rebuild_status_.end()) {
- return search_res->second.first;
+ return search_res->second.has_initiated;
}
return false;
}
@@ -124,7 +131,7 @@ class QueryExecutionState {
DCHECK(operator_index < num_operators_);
const auto search_res = rebuild_status_.find(operator_index);
if (search_res != rebuild_status_.end()) {
- return search_res->second.second;
+ return search_res->second.num_pending_workorders;
}
LOG(WARNING) << "Called QueryExecutionState::getNumRebuildWorkOrders() "
"for an operator whose rebuild entry doesn't exist.";
@@ -132,22 +139,39 @@ class QueryExecutionState {
}
/**
+ * @brief Increment the number of rebuild WorkOrders for the given operator.
+ *
+ * @param operator_index The index of the given operator.
+ * @param num_rebuild_workorders The number of rebuild workorders of the given
+ * operator.
+ **/
+ inline void incrementNumRebuildWorkOrders(const std::size_t operator_index,
+ const std::size_t num_rebuild_workorders) {
+ DCHECK_LT(operator_index, num_operators_);
+ auto search_res = rebuild_status_.find(operator_index);
+ DCHECK(search_res != rebuild_status_.end())
+ << "Called for an operator whose rebuild status does not exist.";
+ DCHECK(search_res->second.has_initiated);
+
+ search_res->second.num_pending_workorders += num_rebuild_workorders;
+ }
+
+ /**
* @brief Decrement the number of rebuild WorkOrders for the given operator.
*
* @param operator_index The index of the given operator.
**/
inline void decrementNumRebuildWorkOrders(const std::size_t operator_index) {
DCHECK(operator_index < num_operators_);
- const auto search_res = rebuild_status_.find(operator_index);
- if (search_res != rebuild_status_.end()) {
- DCHECK(search_res->second.first);
- DCHECK_GE(search_res->second.second, 1u);
- --rebuild_status_[operator_index].second;
- } else {
- LOG(FATAL) <<
- "Called QueryExecutionState::decrementNumRebuildWorkOrders() for an "
- "operator whose rebuild entry doesn't exist.";
- }
+ auto search_res = rebuild_status_.find(operator_index);
+ CHECK(search_res != rebuild_status_.end())
+ << "Called QueryExecutionState::decrementNumRebuildWorkOrders() for an "
+ "operator whose rebuild entry doesn't exist.";
+
+ DCHECK(search_res->second.has_initiated);
+ DCHECK_GE(search_res->second.num_pending_workorders, 1u);
+
+ --(search_res->second.num_pending_workorders);
}
/**
@@ -279,11 +303,21 @@ class QueryExecutionState {
// The ith bit denotes if the operator with ID = i has finished its execution.
std::vector<bool> execution_finished_;
- // Key is dag_node_index for which rebuild is required. Value is a pair -
- // first element is a bool (whether rebuild for operator at index i has been
- // initiated) and if the boolean is true, the second element denotes the
- // number of pending rebuild workorders for the operator.
- std::unordered_map<std::size_t, std::pair<bool, std::size_t>> rebuild_status_;
+ struct RebuildStatus {
+ RebuildStatus(const bool initiated,
+ const std::size_t num_workorders)
+ : has_initiated(initiated),
+ num_pending_workorders(num_workorders) {}
+
+ // Whether rebuild for operator at index i has been initiated.
+ bool has_initiated;
+ // The number of pending rebuild workorders for the operator.
+ // Valid if and only if 'has_initiated' is true.
+ std::size_t num_pending_workorders;
+ };
+
+ // Key is dag_node_index for which rebuild is required.
+ std::unordered_map<std::size_t, RebuildStatus> rebuild_status_;
DISALLOW_COPY_AND_ASSIGN(QueryExecutionState);
};
[08/17] incubator-quickstep git commit: Initial commit for
QUICKSTEP-28 and QUICKSTEP-29.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationHandleMax.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMax.hpp b/expressions/aggregation/AggregationHandleMax.hpp
index 7e38473..5fb9f44 100644
--- a/expressions/aggregation/AggregationHandleMax.hpp
+++ b/expressions/aggregation/AggregationHandleMax.hpp
@@ -28,6 +28,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "expressions/aggregation/AggregationConcreteHandle.hpp"
#include "expressions/aggregation/AggregationHandle.hpp"
+#include "storage/FastHashTable.hpp"
#include "storage/HashTableBase.hpp"
#include "threading/SpinMutex.hpp"
#include "types/Type.hpp"
@@ -55,25 +56,24 @@ class AggregationStateMax : public AggregationState {
/**
* @brief Copy constructor (ignores mutex).
*/
- AggregationStateMax(const AggregationStateMax &orig)
- : max_(orig.max_) {
- }
+ AggregationStateMax(const AggregationStateMax &orig) : max_(orig.max_) {}
/**
* @brief Destructor.
*/
- ~AggregationStateMax() override {};
+ ~AggregationStateMax() override{};
+
+ const std::uint8_t* getPayloadAddress() const {
+ return reinterpret_cast<const uint8_t *>(&max_);
+ }
private:
friend class AggregationHandleMax;
explicit AggregationStateMax(const Type &type)
- : max_(type.getNullableVersion().makeNullValue()) {
- }
+ : max_(type.getNullableVersion().makeNullValue()) {}
- explicit AggregationStateMax(TypedValue &&value)
- : max_(std::move(value)) {
- }
+ explicit AggregationStateMax(TypedValue &&value) : max_(std::move(value)) {}
TypedValue max_;
SpinMutex mutex_;
@@ -84,8 +84,7 @@ class AggregationStateMax : public AggregationState {
**/
class AggregationHandleMax : public AggregationConcreteHandle {
public:
- ~AggregationHandleMax() override {
- }
+ ~AggregationHandleMax() override {}
AggregationState* createInitialState() const override {
return new AggregationStateMax(type_);
@@ -93,20 +92,46 @@ class AggregationHandleMax : public AggregationConcreteHandle {
AggregationStateHashTableBase* createGroupByHashTable(
const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &group_by_types,
+ const std::vector<const Type *> &group_by_types,
const std::size_t estimated_num_groups,
StorageManager *storage_manager) const override;
/**
* @brief Iterate with max aggregation state.
*/
- inline void iterateUnaryInl(AggregationStateMax *state, const TypedValue &value) const {
+ inline void iterateUnaryInl(AggregationStateMax *state,
+ const TypedValue &value) const {
DCHECK(value.isPlausibleInstanceOf(type_.getSignature()));
- compareAndUpdate(static_cast<AggregationStateMax*>(state), value);
+ compareAndUpdate(static_cast<AggregationStateMax *>(state), value);
+ }
+
+ inline void iterateUnaryInlFast(const TypedValue &value,
+ std::uint8_t *byte_ptr) const {
+ DCHECK(value.isPlausibleInstanceOf(type_.getSignature()));
+ TypedValue *max_ptr = reinterpret_cast<TypedValue *>(byte_ptr);
+ compareAndUpdateFast(max_ptr, value);
+ }
+
+ inline void updateStateUnary(const TypedValue &argument,
+ std::uint8_t *byte_ptr) const override {
+ if (!block_update_) {
+ iterateUnaryInlFast(argument, byte_ptr);
+ }
+ }
+
+ void blockUpdate() override { block_update_ = true; }
+
+ void allowUpdate() override { block_update_ = false; }
+
+ void initPayload(std::uint8_t *byte_ptr) const override {
+ TypedValue *max_ptr = reinterpret_cast<TypedValue *>(byte_ptr);
+ TypedValue t1 = (type_.getNullableVersion().makeNullValue());
+ *max_ptr = t1;
}
AggregationState* accumulateColumnVectors(
- const std::vector<std::unique_ptr<ColumnVector>> &column_vectors) const override;
+ const std::vector<std::unique_ptr<ColumnVector>> &column_vectors)
+ const override;
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
AggregationState* accumulateValueAccessor(
@@ -123,37 +148,49 @@ class AggregationHandleMax : public AggregationConcreteHandle {
void mergeStates(const AggregationState &source,
AggregationState *destination) const override;
+ void mergeStatesFast(const std::uint8_t *source,
+ std::uint8_t *destination) const override;
+
TypedValue finalize(const AggregationState &state) const override {
- return TypedValue(static_cast<const AggregationStateMax&>(state).max_);
+ return TypedValue(static_cast<const AggregationStateMax &>(state).max_);
+ }
+
+ inline TypedValue finalizeHashTableEntry(
+ const AggregationState &state) const {
+ return TypedValue(static_cast<const AggregationStateMax &>(state).max_);
}
- inline TypedValue finalizeHashTableEntry(const AggregationState &state) const {
- return TypedValue(static_cast<const AggregationStateMax&>(state).max_);
+ inline TypedValue finalizeHashTableEntryFast(
+ const std::uint8_t *byte_ptr) const {
+ const TypedValue *max_ptr = reinterpret_cast<const TypedValue *>(byte_ptr);
+ return TypedValue(*max_ptr);
}
ColumnVector* finalizeHashTable(
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const override;
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const override;
/**
- * @brief Implementation of AggregationHandle::aggregateOnDistinctifyHashTableForSingle()
+ * @brief Implementation of
+ * AggregationHandle::aggregateOnDistinctifyHashTableForSingle()
* for MAX aggregation.
*/
AggregationState* aggregateOnDistinctifyHashTableForSingle(
- const AggregationStateHashTableBase &distinctify_hash_table) const override;
-
+ const AggregationStateHashTableBase &distinctify_hash_table)
+ const override;
/**
- * @brief Implementation of AggregationHandle::aggregateOnDistinctifyHashTableForGroupBy()
+ * @brief Implementation of
+ * AggregationHandle::aggregateOnDistinctifyHashTableForGroupBy()
* for MAX aggregation.
*/
void aggregateOnDistinctifyHashTableForGroupBy(
const AggregationStateHashTableBase &distinctify_hash_table,
- AggregationStateHashTableBase *aggregation_hash_table) const override;
+ AggregationStateHashTableBase *aggregation_hash_table,
+ std::size_t index) const override;
- void mergeGroupByHashTables(
- const AggregationStateHashTableBase &source_hash_table,
- AggregationStateHashTableBase *destination_hash_table) const override;
+ std::size_t getPayloadSize() const override { return sizeof(TypedValue); }
private:
friend class AggregateFunctionMax;
@@ -166,24 +203,37 @@ class AggregationHandleMax : public AggregationConcreteHandle {
explicit AggregationHandleMax(const Type &type);
/**
- * @brief compare the value with max_ and update it if the value is larger than
- * current maximum. NULLs are ignored.
+ * @brief compare the value with max_ and update it if the value is larger
+ * than current maximum. NULLs are ignored.
*
* @param value A TypedValue to compare
**/
- inline void compareAndUpdate(AggregationStateMax *state, const TypedValue &value) const {
+ inline void compareAndUpdate(AggregationStateMax *state,
+ const TypedValue &value) const {
// TODO(chasseur): Avoid null-checks when aggregating a non-nullable Type.
if (value.isNull()) return;
SpinMutexLock lock(state->mutex_);
- if (state->max_.isNull() || fast_comparator_->compareTypedValues(value, state->max_)) {
+ if (state->max_.isNull() ||
+ fast_comparator_->compareTypedValues(value, state->max_)) {
state->max_ = value;
}
}
+ inline void compareAndUpdateFast(TypedValue *max_ptr,
+ const TypedValue &value) const {
+ if (value.isNull()) return;
+ if (max_ptr->isNull() ||
+ fast_comparator_->compareTypedValues(value, *max_ptr)) {
+ *max_ptr = value;
+ }
+ }
+
const Type &type_;
std::unique_ptr<UncheckedComparator> fast_comparator_;
+ bool block_update_;
+
DISALLOW_COPY_AND_ASSIGN(AggregationHandleMax);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationHandleMin.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMin.cpp b/expressions/aggregation/AggregationHandleMin.cpp
index e860d8d..a07f299 100644
--- a/expressions/aggregation/AggregationHandleMin.cpp
+++ b/expressions/aggregation/AggregationHandleMin.cpp
@@ -39,22 +39,19 @@ namespace quickstep {
class StorageManager;
AggregationHandleMin::AggregationHandleMin(const Type &type)
- : type_(type) {
- fast_comparator_.reset(ComparisonFactory::GetComparison(ComparisonID::kLess)
- .makeUncheckedComparatorForTypes(type,
- type.getNonNullableVersion()));
+ : type_(type), block_update_(false) {
+ fast_comparator_.reset(
+ ComparisonFactory::GetComparison(ComparisonID::kLess)
+ .makeUncheckedComparatorForTypes(type, type.getNonNullableVersion()));
}
AggregationStateHashTableBase* AggregationHandleMin::createGroupByHashTable(
const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &group_by_types,
+ const std::vector<const Type *> &group_by_types,
const std::size_t estimated_num_groups,
StorageManager *storage_manager) const {
return AggregationStateHashTableFactory<AggregationStateMin>::CreateResizable(
- hash_table_impl,
- group_by_types,
- estimated_num_groups,
- storage_manager);
+ hash_table_impl, group_by_types, estimated_num_groups, storage_manager);
}
AggregationState* AggregationHandleMin::accumulateColumnVectors(
@@ -62,9 +59,8 @@ AggregationState* AggregationHandleMin::accumulateColumnVectors(
DCHECK_EQ(1u, column_vectors.size())
<< "Got wrong number of ColumnVectors for MIN: " << column_vectors.size();
- return new AggregationStateMin(
- fast_comparator_->accumulateColumnVector(type_.getNullableVersion().makeNullValue(),
- *column_vectors.front()));
+ return new AggregationStateMin(fast_comparator_->accumulateColumnVector(
+ type_.getNullableVersion().makeNullValue(), *column_vectors.front()));
}
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -74,10 +70,10 @@ AggregationState* AggregationHandleMin::accumulateValueAccessor(
DCHECK_EQ(1u, accessor_ids.size())
<< "Got wrong number of attributes for MIN: " << accessor_ids.size();
- return new AggregationStateMin(
- fast_comparator_->accumulateValueAccessor(type_.getNullableVersion().makeNullValue(),
- accessor,
- accessor_ids.front()));
+ return new AggregationStateMin(fast_comparator_->accumulateValueAccessor(
+ type_.getNullableVersion().makeNullValue(),
+ accessor,
+ accessor_ids.front()));
}
#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -88,66 +84,55 @@ void AggregationHandleMin::aggregateValueAccessorIntoHashTable(
AggregationStateHashTableBase *hash_table) const {
DCHECK_EQ(1u, argument_ids.size())
<< "Got wrong number of arguments for MIN: " << argument_ids.size();
-
- aggregateValueAccessorIntoHashTableUnaryHelper<
- AggregationHandleMin,
- AggregationStateMin,
- AggregationStateHashTable<AggregationStateMin>>(
- accessor,
- argument_ids.front(),
- group_by_key_ids,
- AggregationStateMin(type_),
- hash_table);
}
-void AggregationHandleMin::mergeStates(
- const AggregationState &source,
- AggregationState *destination) const {
- const AggregationStateMin &min_source = static_cast<const AggregationStateMin&>(source);
- AggregationStateMin *min_destination = static_cast<AggregationStateMin*>(destination);
+void AggregationHandleMin::mergeStates(const AggregationState &source,
+ AggregationState *destination) const {
+ const AggregationStateMin &min_source =
+ static_cast<const AggregationStateMin &>(source);
+ AggregationStateMin *min_destination =
+ static_cast<AggregationStateMin *>(destination);
if (!min_source.min_.isNull()) {
compareAndUpdate(min_destination, min_source.min_);
}
}
+void AggregationHandleMin::mergeStatesFast(const std::uint8_t *source,
+ std::uint8_t *destination) const {
+ const TypedValue *src_min_ptr = reinterpret_cast<const TypedValue *>(source);
+ TypedValue *dst_min_ptr = reinterpret_cast<TypedValue *>(destination);
+
+ if (!(src_min_ptr->isNull())) {
+ compareAndUpdateFast(dst_min_ptr, *src_min_ptr);
+ }
+}
+
ColumnVector* AggregationHandleMin::finalizeHashTable(
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const {
- return finalizeHashTableHelper<AggregationHandleMin,
- AggregationStateHashTable<AggregationStateMin>>(
- type_.getNonNullableVersion(),
- hash_table,
- group_by_keys);
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const {
+ return finalizeHashTableHelperFast<AggregationHandleMin,
+ AggregationStateFastHashTable>(
+ type_.getNonNullableVersion(), hash_table, group_by_keys, index);
}
-AggregationState* AggregationHandleMin::aggregateOnDistinctifyHashTableForSingle(
+AggregationState*
+AggregationHandleMin::aggregateOnDistinctifyHashTableForSingle(
const AggregationStateHashTableBase &distinctify_hash_table) const {
- return aggregateOnDistinctifyHashTableForSingleUnaryHelper<
+ return aggregateOnDistinctifyHashTableForSingleUnaryHelperFast<
AggregationHandleMin,
- AggregationStateMin>(
- distinctify_hash_table);
+ AggregationStateMin>(distinctify_hash_table);
}
void AggregationHandleMin::aggregateOnDistinctifyHashTableForGroupBy(
const AggregationStateHashTableBase &distinctify_hash_table,
- AggregationStateHashTableBase *aggregation_hash_table) const {
- aggregateOnDistinctifyHashTableForGroupByUnaryHelper<
+ AggregationStateHashTableBase *aggregation_hash_table,
+ std::size_t index) const {
+ aggregateOnDistinctifyHashTableForGroupByUnaryHelperFast<
AggregationHandleMin,
- AggregationStateMin,
- AggregationStateHashTable<AggregationStateMin>>(
- distinctify_hash_table,
- AggregationStateMin(type_),
- aggregation_hash_table);
-}
-
-void AggregationHandleMin::mergeGroupByHashTables(
- const AggregationStateHashTableBase &source_hash_table,
- AggregationStateHashTableBase *destination_hash_table) const {
- mergeGroupByHashTablesHelper<AggregationHandleMin,
- AggregationStateMin,
- AggregationStateHashTable<AggregationStateMin>>(
- source_hash_table, destination_hash_table);
+ AggregationStateFastHashTable>(
+ distinctify_hash_table, aggregation_hash_table, index);
}
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationHandleMin.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMin.hpp b/expressions/aggregation/AggregationHandleMin.hpp
index 924698c..173911d 100644
--- a/expressions/aggregation/AggregationHandleMin.hpp
+++ b/expressions/aggregation/AggregationHandleMin.hpp
@@ -28,6 +28,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "expressions/aggregation/AggregationConcreteHandle.hpp"
#include "expressions/aggregation/AggregationHandle.hpp"
+#include "storage/FastHashTable.hpp"
#include "storage/HashTableBase.hpp"
#include "threading/SpinMutex.hpp"
#include "types/Type.hpp"
@@ -55,24 +56,26 @@ class AggregationStateMin : public AggregationState {
/**
* @brief Copy constructor (ignores mutex).
*/
- AggregationStateMin(const AggregationStateMin &orig)
- : min_(orig.min_) {
- }
+ AggregationStateMin(const AggregationStateMin &orig) : min_(orig.min_) {}
/**
* @brief Destructor.
*/
~AggregationStateMin() override {}
+ std::size_t getPayloadSize() const { return sizeof(TypedValue); }
+
+ const std::uint8_t *getPayloadAddress() const {
+ return reinterpret_cast<const uint8_t *>(&min_);
+ }
+
private:
friend class AggregationHandleMin;
explicit AggregationStateMin(const Type &type)
: min_(type.getNullableVersion().makeNullValue()) {}
- explicit AggregationStateMin(TypedValue &&value)
- : min_(std::move(value)) {
- }
+ explicit AggregationStateMin(TypedValue &&value) : min_(std::move(value)) {}
TypedValue min_;
SpinMutex mutex_;
@@ -83,8 +86,7 @@ class AggregationStateMin : public AggregationState {
**/
class AggregationHandleMin : public AggregationConcreteHandle {
public:
- ~AggregationHandleMin() override {
- }
+ ~AggregationHandleMin() override {}
AggregationState* createInitialState() const override {
return new AggregationStateMin(type_);
@@ -92,20 +94,46 @@ class AggregationHandleMin : public AggregationConcreteHandle {
AggregationStateHashTableBase* createGroupByHashTable(
const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &group_by_types,
+ const std::vector<const Type *> &group_by_types,
const std::size_t estimated_num_groups,
StorageManager *storage_manager) const override;
/**
* @brief Iterate with min aggregation state.
*/
- inline void iterateUnaryInl(AggregationStateMin *state, const TypedValue &value) const {
+ inline void iterateUnaryInl(AggregationStateMin *state,
+ const TypedValue &value) const {
DCHECK(value.isPlausibleInstanceOf(type_.getSignature()));
compareAndUpdate(state, value);
}
+ inline void iterateUnaryInlFast(const TypedValue &value,
+ std::uint8_t *byte_ptr) const {
+ DCHECK(value.isPlausibleInstanceOf(type_.getSignature()));
+ TypedValue *min_ptr = reinterpret_cast<TypedValue *>(byte_ptr);
+ compareAndUpdateFast(min_ptr, value);
+ }
+
+ inline void updateStateUnary(const TypedValue &argument,
+ std::uint8_t *byte_ptr) const override {
+ if (!block_update_) {
+ iterateUnaryInlFast(argument, byte_ptr);
+ }
+ }
+
+ void blockUpdate() override { block_update_ = true; }
+
+ void allowUpdate() override { block_update_ = false; }
+
+ void initPayload(std::uint8_t *byte_ptr) const override {
+ TypedValue *min_ptr = reinterpret_cast<TypedValue *>(byte_ptr);
+ TypedValue t1 = (type_.getNullableVersion().makeNullValue());
+ *min_ptr = t1;
+ }
+
AggregationState* accumulateColumnVectors(
- const std::vector<std::unique_ptr<ColumnVector>> &column_vectors) const override;
+ const std::vector<std::unique_ptr<ColumnVector>> &column_vectors)
+ const override;
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
AggregationState* accumulateValueAccessor(
@@ -122,36 +150,49 @@ class AggregationHandleMin : public AggregationConcreteHandle {
void mergeStates(const AggregationState &source,
AggregationState *destination) const override;
+ void mergeStatesFast(const std::uint8_t *source,
+ std::uint8_t *destination) const override;
+
TypedValue finalize(const AggregationState &state) const override {
- return static_cast<const AggregationStateMin&>(state).min_;
+ return static_cast<const AggregationStateMin &>(state).min_;
+ }
+
+ inline TypedValue finalizeHashTableEntry(
+ const AggregationState &state) const {
+ return static_cast<const AggregationStateMin &>(state).min_;
}
- inline TypedValue finalizeHashTableEntry(const AggregationState &state) const {
- return static_cast<const AggregationStateMin&>(state).min_;
+ inline TypedValue finalizeHashTableEntryFast(
+ const std::uint8_t *byte_ptr) const {
+ const TypedValue *min_ptr = reinterpret_cast<const TypedValue *>(byte_ptr);
+ return TypedValue(*min_ptr);
}
ColumnVector* finalizeHashTable(
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const override;
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const override;
/**
- * @brief Implementation of AggregationHandle::aggregateOnDistinctifyHashTableForSingle()
+ * @brief Implementation of
+ * AggregationHandle::aggregateOnDistinctifyHashTableForSingle()
* for MIN aggregation.
*/
AggregationState* aggregateOnDistinctifyHashTableForSingle(
- const AggregationStateHashTableBase &distinctify_hash_table) const override;
+ const AggregationStateHashTableBase &distinctify_hash_table)
+ const override;
/**
- * @brief Implementation of AggregationHandle::aggregateOnDistinctifyHashTableForGroupBy()
+ * @brief Implementation of
+ * AggregationHandle::aggregateOnDistinctifyHashTableForGroupBy()
* for MIN aggregation.
*/
void aggregateOnDistinctifyHashTableForGroupBy(
const AggregationStateHashTableBase &distinctify_hash_table,
- AggregationStateHashTableBase *aggregation_hash_table) const override;
+ AggregationStateHashTableBase *aggregation_hash_table,
+ std::size_t index) const override;
- void mergeGroupByHashTables(
- const AggregationStateHashTableBase &source_hash_table,
- AggregationStateHashTableBase *destination_hash_table) const override;
+ std::size_t getPayloadSize() const override { return sizeof(TypedValue); }
private:
friend class AggregateFunctionMin;
@@ -164,23 +205,36 @@ class AggregationHandleMin : public AggregationConcreteHandle {
explicit AggregationHandleMin(const Type &type);
/**
- * @brief compare the value with min_ and update it if the value is smaller than
- * current minimum. NULLs are ignored.
+ * @brief compare the value with min_ and update it if the value is smaller
+ * than current minimum. NULLs are ignored.
*
* @param value A TypedValue to compare.
**/
- inline void compareAndUpdate(AggregationStateMin *state, const TypedValue &value) const {
+ inline void compareAndUpdate(AggregationStateMin *state,
+ const TypedValue &value) const {
if (value.isNull()) return;
SpinMutexLock lock(state->mutex_);
- if (state->min_.isNull() || fast_comparator_->compareTypedValues(value, state->min_)) {
+ if (state->min_.isNull() ||
+ fast_comparator_->compareTypedValues(value, state->min_)) {
state->min_ = value;
}
}
+ inline void compareAndUpdateFast(TypedValue *min_ptr,
+ const TypedValue &value) const {
+ if (value.isNull()) return;
+ if (min_ptr->isNull() ||
+ fast_comparator_->compareTypedValues(value, *min_ptr)) {
+ *min_ptr = value;
+ }
+ }
+
const Type &type_;
std::unique_ptr<UncheckedComparator> fast_comparator_;
+ bool block_update_;
+
DISALLOW_COPY_AND_ASSIGN(AggregationHandleMin);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationHandleSum.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleSum.cpp b/expressions/aggregation/AggregationHandleSum.cpp
index b5036a8..642d88d 100644
--- a/expressions/aggregation/AggregationHandleSum.cpp
+++ b/expressions/aggregation/AggregationHandleSum.cpp
@@ -43,7 +43,7 @@ namespace quickstep {
class StorageManager;
AggregationHandleSum::AggregationHandleSum(const Type &type)
- : argument_type_(type) {
+ : argument_type_(type), block_update_(false) {
// We sum Int as Long and Float as Double so that we have more headroom when
// adding many values.
TypeID type_precision_id;
@@ -66,11 +66,13 @@ AggregationHandleSum::AggregationHandleSum(const Type &type)
// Make operators to do arithmetic:
// Add operator for summing argument values.
- fast_operator_.reset(BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kAdd)
- .makeUncheckedBinaryOperatorForTypes(sum_type, argument_type_));
+ fast_operator_.reset(
+ BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kAdd)
+ .makeUncheckedBinaryOperatorForTypes(sum_type, argument_type_));
// Add operator for merging states.
- merge_operator_.reset(BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kAdd)
- .makeUncheckedBinaryOperatorForTypes(sum_type, sum_type));
+ merge_operator_.reset(
+ BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kAdd)
+ .makeUncheckedBinaryOperatorForTypes(sum_type, sum_type));
// Result is nullable, because SUM() over 0 values (or all NULL values) is
// NULL.
@@ -79,26 +81,20 @@ AggregationHandleSum::AggregationHandleSum(const Type &type)
AggregationStateHashTableBase* AggregationHandleSum::createGroupByHashTable(
const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &group_by_types,
+ const std::vector<const Type *> &group_by_types,
const std::size_t estimated_num_groups,
StorageManager *storage_manager) const {
return AggregationStateHashTableFactory<AggregationStateSum>::CreateResizable(
- hash_table_impl,
- group_by_types,
- estimated_num_groups,
- storage_manager);
+ hash_table_impl, group_by_types, estimated_num_groups, storage_manager);
}
AggregationState* AggregationHandleSum::accumulateColumnVectors(
const std::vector<std::unique_ptr<ColumnVector>> &column_vectors) const {
DCHECK_EQ(1u, column_vectors.size())
<< "Got wrong number of ColumnVectors for SUM: " << column_vectors.size();
-
std::size_t num_tuples = 0;
TypedValue cv_sum = fast_operator_->accumulateColumnVector(
- blank_state_.sum_,
- *column_vectors.front(),
- &num_tuples);
+ blank_state_.sum_, *column_vectors.front(), &num_tuples);
return new AggregationStateSum(std::move(cv_sum), num_tuples == 0);
}
@@ -111,10 +107,7 @@ AggregationState* AggregationHandleSum::accumulateValueAccessor(
std::size_t num_tuples = 0;
TypedValue va_sum = fast_operator_->accumulateValueAccessor(
- blank_state_.sum_,
- accessor,
- accessor_ids.front(),
- &num_tuples);
+ blank_state_.sum_, accessor, accessor_ids.front(), &num_tuples);
return new AggregationStateSum(std::move(va_sum), num_tuples == 0);
}
#endif
@@ -126,32 +119,39 @@ void AggregationHandleSum::aggregateValueAccessorIntoHashTable(
AggregationStateHashTableBase *hash_table) const {
DCHECK_EQ(1u, argument_ids.size())
<< "Got wrong number of arguments for SUM: " << argument_ids.size();
-
- aggregateValueAccessorIntoHashTableUnaryHelper<
- AggregationHandleSum,
- AggregationStateSum,
- AggregationStateHashTable<AggregationStateSum>>(
- accessor,
- argument_ids.front(),
- group_by_key_ids,
- blank_state_,
- hash_table);
}
-void AggregationHandleSum::mergeStates(
- const AggregationState &source,
- AggregationState *destination) const {
- const AggregationStateSum &sum_source = static_cast<const AggregationStateSum&>(source);
- AggregationStateSum *sum_destination = static_cast<AggregationStateSum*>(destination);
+void AggregationHandleSum::mergeStates(const AggregationState &source,
+ AggregationState *destination) const {
+ const AggregationStateSum &sum_source =
+ static_cast<const AggregationStateSum &>(source);
+ AggregationStateSum *sum_destination =
+ static_cast<AggregationStateSum *>(destination);
SpinMutexLock lock(sum_destination->mutex_);
- sum_destination->sum_ = merge_operator_->applyToTypedValues(sum_destination->sum_,
- sum_source.sum_);
+ sum_destination->sum_ = merge_operator_->applyToTypedValues(
+ sum_destination->sum_, sum_source.sum_);
sum_destination->null_ = sum_destination->null_ && sum_source.null_;
}
+void AggregationHandleSum::mergeStatesFast(const std::uint8_t *source,
+ std::uint8_t *destination) const {
+ const TypedValue *src_sum_ptr =
+ reinterpret_cast<const TypedValue *>(source + blank_state_.sum_offset_);
+ const bool *src_null_ptr =
+ reinterpret_cast<const bool *>(source + blank_state_.null_offset_);
+ TypedValue *dst_sum_ptr =
+ reinterpret_cast<TypedValue *>(destination + blank_state_.sum_offset_);
+ bool *dst_null_ptr =
+ reinterpret_cast<bool *>(destination + blank_state_.null_offset_);
+ *dst_sum_ptr =
+ merge_operator_->applyToTypedValues(*dst_sum_ptr, *src_sum_ptr);
+ *dst_null_ptr = (*dst_null_ptr) && (*src_null_ptr);
+}
+
TypedValue AggregationHandleSum::finalize(const AggregationState &state) const {
- const AggregationStateSum &agg_state = static_cast<const AggregationStateSum&>(state);
+ const AggregationStateSum &agg_state =
+ static_cast<const AggregationStateSum &>(state);
if (agg_state.null_) {
// SUM() over no values is NULL.
return result_type_->makeNullValue();
@@ -162,41 +162,29 @@ TypedValue AggregationHandleSum::finalize(const AggregationState &state) const {
ColumnVector* AggregationHandleSum::finalizeHashTable(
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const {
- return finalizeHashTableHelper<AggregationHandleSum,
- AggregationStateHashTable<AggregationStateSum>>(
- *result_type_,
- hash_table,
- group_by_keys);
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const {
+ return finalizeHashTableHelperFast<AggregationHandleSum,
+ AggregationStateFastHashTable>(
+ *result_type_, hash_table, group_by_keys, index);
}
-AggregationState* AggregationHandleSum::aggregateOnDistinctifyHashTableForSingle(
+AggregationState*
+AggregationHandleSum::aggregateOnDistinctifyHashTableForSingle(
const AggregationStateHashTableBase &distinctify_hash_table) const {
- return aggregateOnDistinctifyHashTableForSingleUnaryHelper<
+ return aggregateOnDistinctifyHashTableForSingleUnaryHelperFast<
AggregationHandleSum,
- AggregationStateSum>(
- distinctify_hash_table);
+ AggregationStateSum>(distinctify_hash_table);
}
void AggregationHandleSum::aggregateOnDistinctifyHashTableForGroupBy(
const AggregationStateHashTableBase &distinctify_hash_table,
- AggregationStateHashTableBase *aggregation_hash_table) const {
- aggregateOnDistinctifyHashTableForGroupByUnaryHelper<
+ AggregationStateHashTableBase *aggregation_hash_table,
+ std::size_t index) const {
+ aggregateOnDistinctifyHashTableForGroupByUnaryHelperFast<
AggregationHandleSum,
- AggregationStateSum,
- AggregationStateHashTable<AggregationStateSum>>(
- distinctify_hash_table,
- blank_state_,
- aggregation_hash_table);
-}
-
-void AggregationHandleSum::mergeGroupByHashTables(
- const AggregationStateHashTableBase &source_hash_table,
- AggregationStateHashTableBase *destination_hash_table) const {
- mergeGroupByHashTablesHelper<AggregationHandleSum,
- AggregationStateSum,
- AggregationStateHashTable<AggregationStateSum>>(
- source_hash_table, destination_hash_table);
+ AggregationStateFastHashTable>(
+ distinctify_hash_table, aggregation_hash_table, index);
}
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/AggregationHandleSum.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleSum.hpp b/expressions/aggregation/AggregationHandleSum.hpp
index 3382646..6c334a6 100644
--- a/expressions/aggregation/AggregationHandleSum.hpp
+++ b/expressions/aggregation/AggregationHandleSum.hpp
@@ -28,6 +28,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "expressions/aggregation/AggregationConcreteHandle.hpp"
#include "expressions/aggregation/AggregationHandle.hpp"
+#include "storage/FastHashTable.hpp"
#include "storage/HashTableBase.hpp"
#include "threading/SpinMutex.hpp"
#include "types/Type.hpp"
@@ -57,25 +58,40 @@ class AggregationStateSum : public AggregationState {
*/
AggregationStateSum(const AggregationStateSum &orig)
: sum_(orig.sum_),
- null_(orig.null_) {
+ null_(orig.null_),
+ sum_offset_(orig.sum_offset_),
+ null_offset_(orig.null_offset_) {}
+
+ std::size_t getPayloadSize() const {
+ std::size_t p1 = reinterpret_cast<std::size_t>(&sum_);
+ std::size_t p2 = reinterpret_cast<std::size_t>(&mutex_);
+ return (p2 - p1);
+ }
+
+ const std::uint8_t* getPayloadAddress() const {
+ return reinterpret_cast<const uint8_t *>(&sum_);
}
private:
friend class AggregationHandleSum;
AggregationStateSum()
- : sum_(0), null_(true) {
- }
+ : sum_(0),
+ null_(true),
+ sum_offset_(0),
+ null_offset_(reinterpret_cast<std::uint8_t *>(&null_) -
+ reinterpret_cast<std::uint8_t *>(&sum_)) {}
AggregationStateSum(TypedValue &&sum, const bool is_null)
- : sum_(std::move(sum)), null_(is_null) {
- }
+ : sum_(std::move(sum)), null_(is_null) {}
// TODO(shoban): We might want to specialize sum_ to use atomics for int types
// similar to in AggregationStateCount.
TypedValue sum_;
bool null_;
SpinMutex mutex_;
+
+ int sum_offset_, null_offset_;
};
/**
@@ -83,8 +99,7 @@ class AggregationStateSum : public AggregationState {
**/
class AggregationHandleSum : public AggregationConcreteHandle {
public:
- ~AggregationHandleSum() override {
- }
+ ~AggregationHandleSum() override {}
AggregationState* createInitialState() const override {
return new AggregationStateSum(blank_state_);
@@ -92,11 +107,12 @@ class AggregationHandleSum : public AggregationConcreteHandle {
AggregationStateHashTableBase* createGroupByHashTable(
const HashTableImplType hash_table_impl,
- const std::vector<const Type*> &group_by_types,
+ const std::vector<const Type *> &group_by_types,
const std::size_t estimated_num_groups,
StorageManager *storage_manager) const override;
- inline void iterateUnaryInl(AggregationStateSum *state, const TypedValue &value) const {
+ inline void iterateUnaryInl(AggregationStateSum *state,
+ const TypedValue &value) const {
DCHECK(value.isPlausibleInstanceOf(argument_type_.getSignature()));
if (value.isNull()) return;
@@ -105,8 +121,41 @@ class AggregationHandleSum : public AggregationConcreteHandle {
state->null_ = false;
}
+ inline void iterateUnaryInlFast(const TypedValue &value,
+ std::uint8_t *byte_ptr) const {
+ DCHECK(value.isPlausibleInstanceOf(argument_type_.getSignature()));
+ if (value.isNull()) return;
+ TypedValue *sum_ptr =
+ reinterpret_cast<TypedValue *>(byte_ptr + blank_state_.sum_offset_);
+ bool *null_ptr =
+ reinterpret_cast<bool *>(byte_ptr + blank_state_.null_offset_);
+ *sum_ptr = fast_operator_->applyToTypedValues(*sum_ptr, value);
+ *null_ptr = false;
+ }
+
+ inline void updateStateUnary(const TypedValue &argument,
+ std::uint8_t *byte_ptr) const override {
+ if (!block_update_) {
+ iterateUnaryInlFast(argument, byte_ptr);
+ }
+ }
+
+ void blockUpdate() override { block_update_ = true; }
+
+ void allowUpdate() override { block_update_ = false; }
+
+ void initPayload(std::uint8_t *byte_ptr) const override {
+ TypedValue *sum_ptr =
+ reinterpret_cast<TypedValue *>(byte_ptr + blank_state_.sum_offset_);
+ bool *null_ptr =
+ reinterpret_cast<bool *>(byte_ptr + blank_state_.null_offset_);
+ *sum_ptr = blank_state_.sum_;
+ *null_ptr = true;
+ }
+
AggregationState* accumulateColumnVectors(
- const std::vector<std::unique_ptr<ColumnVector>> &column_vectors) const override;
+ const std::vector<std::unique_ptr<ColumnVector>> &column_vectors)
+ const override;
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
AggregationState* accumulateValueAccessor(
@@ -123,34 +172,51 @@ class AggregationHandleSum : public AggregationConcreteHandle {
void mergeStates(const AggregationState &source,
AggregationState *destination) const override;
+ void mergeStatesFast(const std::uint8_t *source,
+ std::uint8_t *destination) const override;
+
TypedValue finalize(const AggregationState &state) const override;
- inline TypedValue finalizeHashTableEntry(const AggregationState &state) const {
- return static_cast<const AggregationStateSum&>(state).sum_;
+ inline TypedValue finalizeHashTableEntry(
+ const AggregationState &state) const {
+ return static_cast<const AggregationStateSum &>(state).sum_;
+ }
+
+ inline TypedValue finalizeHashTableEntryFast(
+ const std::uint8_t *byte_ptr) const {
+ std::uint8_t *value_ptr = const_cast<std::uint8_t *>(byte_ptr);
+ TypedValue *sum_ptr =
+ reinterpret_cast<TypedValue *>(value_ptr + blank_state_.sum_offset_);
+ return *sum_ptr;
}
ColumnVector* finalizeHashTable(
const AggregationStateHashTableBase &hash_table,
- std::vector<std::vector<TypedValue>> *group_by_keys) const override;
+ std::vector<std::vector<TypedValue>> *group_by_keys,
+ int index) const override;
/**
- * @brief Implementation of AggregationHandle::aggregateOnDistinctifyHashTableForSingle()
+ * @brief Implementation of
+ * AggregationHandle::aggregateOnDistinctifyHashTableForSingle()
* for SUM aggregation.
*/
AggregationState* aggregateOnDistinctifyHashTableForSingle(
- const AggregationStateHashTableBase &distinctify_hash_table) const override;
+ const AggregationStateHashTableBase &distinctify_hash_table)
+ const override;
/**
- * @brief Implementation of AggregationHandle::aggregateOnDistinctifyHashTableForGroupBy()
+ * @brief Implementation of
+ * AggregationHandle::aggregateOnDistinctifyHashTableForGroupBy()
* for SUM aggregation.
*/
void aggregateOnDistinctifyHashTableForGroupBy(
const AggregationStateHashTableBase &distinctify_hash_table,
- AggregationStateHashTableBase *aggregation_hash_table) const override;
+ AggregationStateHashTableBase *aggregation_hash_table,
+ std::size_t index) const override;
- void mergeGroupByHashTables(
- const AggregationStateHashTableBase &source_hash_table,
- AggregationStateHashTableBase *destination_hash_table) const override;
+ std::size_t getPayloadSize() const override {
+ return blank_state_.getPayloadSize();
+ }
private:
friend class AggregateFunctionSum;
@@ -168,6 +234,8 @@ class AggregationHandleSum : public AggregationConcreteHandle {
std::unique_ptr<UncheckedBinaryOperator> fast_operator_;
std::unique_ptr<UncheckedBinaryOperator> merge_operator_;
+ bool block_update_;
+
DISALLOW_COPY_AND_ASSIGN(AggregationHandleSum);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/expressions/aggregation/CMakeLists.txt b/expressions/aggregation/CMakeLists.txt
index 888d95c..e9503f7 100644
--- a/expressions/aggregation/CMakeLists.txt
+++ b/expressions/aggregation/CMakeLists.txt
@@ -146,9 +146,11 @@ target_link_libraries(quickstep_expressions_aggregation_AggregationConcreteHandl
glog
quickstep_catalog_CatalogTypedefs
quickstep_expressions_aggregation_AggregationHandle
+ quickstep_storage_FastHashTable
quickstep_storage_HashTable
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
+ quickstep_threading_SpinMutex
quickstep_types_TypedValue
quickstep_types_containers_ColumnVector
quickstep_utility_Macros)
@@ -163,6 +165,7 @@ target_link_libraries(quickstep_expressions_aggregation_AggregationHandleAvg
quickstep_catalog_CatalogTypedefs
quickstep_expressions_aggregation_AggregationConcreteHandle
quickstep_expressions_aggregation_AggregationHandle
+ quickstep_storage_FastHashTable
quickstep_storage_HashTable
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
@@ -180,6 +183,7 @@ target_link_libraries(quickstep_expressions_aggregation_AggregationHandleCount
quickstep_catalog_CatalogTypedefs
quickstep_expressions_aggregation_AggregationConcreteHandle
quickstep_expressions_aggregation_AggregationHandle
+ quickstep_storage_FastHashTable
quickstep_storage_HashTable
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
@@ -204,6 +208,7 @@ target_link_libraries(quickstep_expressions_aggregation_AggregationHandleMax
quickstep_catalog_CatalogTypedefs
quickstep_expressions_aggregation_AggregationConcreteHandle
quickstep_expressions_aggregation_AggregationHandle
+ quickstep_storage_FastHashTable
quickstep_storage_HashTable
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
@@ -220,6 +225,7 @@ target_link_libraries(quickstep_expressions_aggregation_AggregationHandleMin
quickstep_catalog_CatalogTypedefs
quickstep_expressions_aggregation_AggregationConcreteHandle
quickstep_expressions_aggregation_AggregationHandle
+ quickstep_storage_FastHashTable
quickstep_storage_HashTable
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
@@ -236,6 +242,7 @@ target_link_libraries(quickstep_expressions_aggregation_AggregationHandleSum
quickstep_catalog_CatalogTypedefs
quickstep_expressions_aggregation_AggregationConcreteHandle
quickstep_expressions_aggregation_AggregationHandle
+ quickstep_storage_FastHashTable
quickstep_storage_HashTable
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
@@ -292,6 +299,7 @@ target_link_libraries(AggregationHandle_tests
quickstep_expressions_aggregation_AggregationHandleMin
quickstep_expressions_aggregation_AggregationHandleSum
quickstep_expressions_aggregation_AggregationID
+ quickstep_storage_AggregationOperationState
quickstep_storage_HashTableBase
quickstep_storage_StorageManager
quickstep_types_CharType
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp b/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
index afc02ec..79d4448 100644
--- a/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
@@ -28,6 +28,8 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleAvg.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/AggregationOperationState.hpp"
+#include "storage/FastHashTableFactory.hpp"
#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DateOperatorOverloads.hpp"
@@ -53,51 +55,56 @@
namespace quickstep {
-class AggregationHandleAvgTest : public::testing::Test {
+class AggregationHandleAvgTest : public ::testing::Test {
protected:
static const int kNumSamples = 100;
// Helper method that calls AggregationHandleAvg::iterateUnaryInl() to
// aggregate 'value' into '*state'.
void iterateHandle(AggregationState *state, const TypedValue &value) {
- static_cast<const AggregationHandleAvg&>(*aggregation_handle_avg_).iterateUnaryInl(
- static_cast<AggregationStateAvg*>(state),
- value);
+ static_cast<const AggregationHandleAvg &>(*aggregation_handle_avg_)
+ .iterateUnaryInl(static_cast<AggregationStateAvg *>(state), value);
}
void initializeHandle(const Type &type) {
aggregation_handle_avg_.reset(
- AggregateFunctionFactory::Get(AggregationID::kAvg).createHandle(
- std::vector<const Type*>(1, &type)));
+ AggregateFunctionFactory::Get(AggregationID::kAvg)
+ .createHandle(std::vector<const Type *>(1, &type)));
aggregation_handle_avg_state_.reset(
aggregation_handle_avg_->createInitialState());
}
static bool ApplyToTypesTest(TypeID typeID) {
- const Type &type = (typeID == kChar || typeID == kVarChar) ?
- TypeFactory::GetType(typeID, static_cast<std::size_t>(10)) :
- TypeFactory::GetType(typeID);
+ const Type &type =
+ (typeID == kChar || typeID == kVarChar)
+ ? TypeFactory::GetType(typeID, static_cast<std::size_t>(10))
+ : TypeFactory::GetType(typeID);
- return AggregateFunctionFactory::Get(AggregationID::kAvg).canApplyToTypes(
- std::vector<const Type*>(1, &type));
+ return AggregateFunctionFactory::Get(AggregationID::kAvg)
+ .canApplyToTypes(std::vector<const Type *>(1, &type));
}
static bool ResultTypeForArgumentTypeTest(TypeID input_type_id,
TypeID output_type_id) {
- const Type *result_type
- = AggregateFunctionFactory::Get(AggregationID::kAvg).resultTypeForArgumentTypes(
- std::vector<const Type*>(1, &TypeFactory::GetType(input_type_id)));
+ const Type *result_type =
+ AggregateFunctionFactory::Get(AggregationID::kAvg)
+ .resultTypeForArgumentTypes(std::vector<const Type *>(
+ 1, &TypeFactory::GetType(input_type_id)));
return (result_type->getTypeID() == output_type_id);
}
template <typename CppType>
- static void CheckAvgValue(
- CppType expected,
- const AggregationHandle &handle,
- const AggregationState &state) {
+ static void CheckAvgValue(CppType expected,
+ const AggregationHandle &handle,
+ const AggregationState &state) {
EXPECT_EQ(expected, handle.finalize(state).getLiteral<CppType>());
}
+ template <typename CppType>
+ static void CheckAvgValue(CppType expected, const TypedValue &value) {
+ EXPECT_EQ(expected, value.getLiteral<CppType>());
+ }
+
// Static templated method for set a meaningful value to data types.
template <typename CppType>
static void SetDataType(int value, CppType *data) {
@@ -108,7 +115,9 @@ class AggregationHandleAvgTest : public::testing::Test {
void checkAggregationAvgGeneric() {
const GenericType &type = GenericType::Instance(true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_avg_->finalize(*aggregation_handle_avg_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_avg_->finalize(*aggregation_handle_avg_state_)
+ .isNull());
typename GenericType::cpptype val;
typename GenericType::cpptype sum;
@@ -119,15 +128,16 @@ class AggregationHandleAvgTest : public::testing::Test {
if (type.getTypeID() == kInt || type.getTypeID() == kLong) {
SetDataType(i - 10, &val);
} else {
- SetDataType(static_cast<float>(i - 10)/10, &val);
+ SetDataType(static_cast<float>(i - 10) / 10, &val);
}
iterateHandle(aggregation_handle_avg_state_.get(), type.makeValue(&val));
sum += val;
}
iterateHandle(aggregation_handle_avg_state_.get(), type.makeNullValue());
- CheckAvgValue<typename OutputType::cpptype>(static_cast<typename OutputType::cpptype>(sum) / kNumSamples,
- *aggregation_handle_avg_,
- *aggregation_handle_avg_state_);
+ CheckAvgValue<typename OutputType::cpptype>(
+ static_cast<typename OutputType::cpptype>(sum) / kNumSamples,
+ *aggregation_handle_avg_,
+ *aggregation_handle_avg_state_);
// Test mergeStates().
std::unique_ptr<AggregationState> merge_state(
@@ -140,7 +150,7 @@ class AggregationHandleAvgTest : public::testing::Test {
if (type.getTypeID() == kInt || type.getTypeID() == kLong) {
SetDataType(i - 10, &val);
} else {
- SetDataType(static_cast<float>(i - 10)/10, &val);
+ SetDataType(static_cast<float>(i - 10) / 10, &val);
}
iterateHandle(merge_state.get(), type.makeValue(&val));
sum += val;
@@ -155,7 +165,8 @@ class AggregationHandleAvgTest : public::testing::Test {
}
template <typename GenericType>
- ColumnVector *createColumnVectorGeneric(const Type &type, typename GenericType::cpptype *sum) {
+ ColumnVector* createColumnVectorGeneric(const Type &type,
+ typename GenericType::cpptype *sum) {
NativeColumnVector *column = new NativeColumnVector(type, kNumSamples + 3);
typename GenericType::cpptype val;
@@ -166,12 +177,12 @@ class AggregationHandleAvgTest : public::testing::Test {
if (type.getTypeID() == kInt || type.getTypeID() == kLong) {
SetDataType(i - 10, &val);
} else {
- SetDataType(static_cast<float>(i - 10)/10, &val);
+ SetDataType(static_cast<float>(i - 10) / 10, &val);
}
column->appendTypedValue(type.makeValue(&val));
*sum += val;
// One NULL in the middle.
- if (i == kNumSamples/2) {
+ if (i == kNumSamples / 2) {
column->appendTypedValue(type.makeNullValue());
}
}
@@ -184,12 +195,15 @@ class AggregationHandleAvgTest : public::testing::Test {
void checkAggregationAvgGenericColumnVector() {
const GenericType &type = GenericType::Instance(true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_avg_->finalize(*aggregation_handle_avg_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_avg_->finalize(*aggregation_handle_avg_state_)
+ .isNull());
typename GenericType::cpptype sum;
SetDataType(0, &sum);
std::vector<std::unique_ptr<ColumnVector>> column_vectors;
- column_vectors.emplace_back(createColumnVectorGeneric<GenericType>(type, &sum));
+ column_vectors.emplace_back(
+ createColumnVectorGeneric<GenericType>(type, &sum));
std::unique_ptr<AggregationState> cv_state(
aggregation_handle_avg_->accumulateColumnVectors(column_vectors));
@@ -201,7 +215,8 @@ class AggregationHandleAvgTest : public::testing::Test {
*aggregation_handle_avg_,
*cv_state);
- aggregation_handle_avg_->mergeStates(*cv_state, aggregation_handle_avg_state_.get());
+ aggregation_handle_avg_->mergeStates(*cv_state,
+ aggregation_handle_avg_state_.get());
CheckAvgValue<typename OutputType::cpptype>(
static_cast<typename OutputType::cpptype>(sum) / kNumSamples,
*aggregation_handle_avg_,
@@ -213,16 +228,19 @@ class AggregationHandleAvgTest : public::testing::Test {
void checkAggregationAvgGenericValueAccessor() {
const GenericType &type = GenericType::Instance(true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_avg_->finalize(*aggregation_handle_avg_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_avg_->finalize(*aggregation_handle_avg_state_)
+ .isNull());
typename GenericType::cpptype sum;
SetDataType(0, &sum);
- std::unique_ptr<ColumnVectorsValueAccessor> accessor(new ColumnVectorsValueAccessor());
+ std::unique_ptr<ColumnVectorsValueAccessor> accessor(
+ new ColumnVectorsValueAccessor());
accessor->addColumn(createColumnVectorGeneric<GenericType>(type, &sum));
std::unique_ptr<AggregationState> va_state(
- aggregation_handle_avg_->accumulateValueAccessor(accessor.get(),
- std::vector<attribute_id>(1, 0)));
+ aggregation_handle_avg_->accumulateValueAccessor(
+ accessor.get(), std::vector<attribute_id>(1, 0)));
// Test the state generated directly by accumulateValueAccessor(), and also
// test after merging back.
@@ -231,7 +249,8 @@ class AggregationHandleAvgTest : public::testing::Test {
*aggregation_handle_avg_,
*va_state);
- aggregation_handle_avg_->mergeStates(*va_state, aggregation_handle_avg_state_.get());
+ aggregation_handle_avg_->mergeStates(*va_state,
+ aggregation_handle_avg_state_.get());
CheckAvgValue<typename OutputType::cpptype>(
static_cast<typename OutputType::cpptype>(sum) / kNumSamples,
*aggregation_handle_avg_,
@@ -255,12 +274,14 @@ void AggregationHandleAvgTest::CheckAvgValue<double>(
}
template <>
-void AggregationHandleAvgTest::SetDataType<DatetimeIntervalLit>(int value, DatetimeIntervalLit *data) {
+void AggregationHandleAvgTest::SetDataType<DatetimeIntervalLit>(
+ int value, DatetimeIntervalLit *data) {
data->interval_ticks = value;
}
template <>
-void AggregationHandleAvgTest::SetDataType<YearMonthIntervalLit>(int value, YearMonthIntervalLit *data) {
+void AggregationHandleAvgTest::SetDataType<YearMonthIntervalLit>(
+ int value, YearMonthIntervalLit *data) {
data->months = value;
}
@@ -307,11 +328,13 @@ TEST_F(AggregationHandleAvgTest, DoubleTypeColumnVectorTest) {
}
TEST_F(AggregationHandleAvgTest, DatetimeIntervalTypeColumnVectorTest) {
- checkAggregationAvgGenericColumnVector<DatetimeIntervalType, DatetimeIntervalType>();
+ checkAggregationAvgGenericColumnVector<DatetimeIntervalType,
+ DatetimeIntervalType>();
}
TEST_F(AggregationHandleAvgTest, YearMonthIntervalTypeColumnVectorTest) {
- checkAggregationAvgGenericColumnVector<YearMonthIntervalType, YearMonthIntervalType>();
+ checkAggregationAvgGenericColumnVector<YearMonthIntervalType,
+ YearMonthIntervalType>();
}
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -332,11 +355,13 @@ TEST_F(AggregationHandleAvgTest, DoubleTypeValueAccessorTest) {
}
TEST_F(AggregationHandleAvgTest, DatetimeIntervalTypeValueAccessorTest) {
- checkAggregationAvgGenericValueAccessor<DatetimeIntervalType, DatetimeIntervalType>();
+ checkAggregationAvgGenericValueAccessor<DatetimeIntervalType,
+ DatetimeIntervalType>();
}
TEST_F(AggregationHandleAvgTest, YearMonthIntervalTypeValueAccessorTest) {
- checkAggregationAvgGenericValueAccessor<YearMonthIntervalType, YearMonthIntervalType>();
+ checkAggregationAvgGenericValueAccessor<YearMonthIntervalType,
+ YearMonthIntervalType>();
}
#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -365,38 +390,53 @@ TEST_F(AggregationHandleAvgDeathTest, WrongTypeTest) {
double double_val = 0;
float float_val = 0;
- iterateHandle(aggregation_handle_avg_state_.get(), int_non_null_type.makeValue(&int_val));
+ iterateHandle(aggregation_handle_avg_state_.get(),
+ int_non_null_type.makeValue(&int_val));
- EXPECT_DEATH(iterateHandle(aggregation_handle_avg_state_.get(), long_type.makeValue(&long_val)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_avg_state_.get(), double_type.makeValue(&double_val)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_avg_state_.get(), float_type.makeValue(&float_val)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_avg_state_.get(), char_type.makeValue("asdf", 5)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_avg_state_.get(), varchar_type.makeValue("asdf", 5)), "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_avg_state_.get(),
+ long_type.makeValue(&long_val)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_avg_state_.get(),
+ double_type.makeValue(&double_val)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_avg_state_.get(),
+ float_type.makeValue(&float_val)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_avg_state_.get(),
+ char_type.makeValue("asdf", 5)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_avg_state_.get(),
+ varchar_type.makeValue("asdf", 5)),
+ "");
// Test mergeStates() with incorrectly typed handles.
std::unique_ptr<AggregationHandle> aggregation_handle_avg_double(
- AggregateFunctionFactory::Get(AggregationID::kAvg).createHandle(
- std::vector<const Type*>(1, &double_type)));
+ AggregateFunctionFactory::Get(AggregationID::kAvg)
+ .createHandle(std::vector<const Type *>(1, &double_type)));
std::unique_ptr<AggregationState> aggregation_state_avg_merge_double(
aggregation_handle_avg_double->createInitialState());
- static_cast<const AggregationHandleAvg&>(*aggregation_handle_avg_double).iterateUnaryInl(
- static_cast<AggregationStateAvg*>(aggregation_state_avg_merge_double.get()),
- double_type.makeValue(&double_val));
- EXPECT_DEATH(aggregation_handle_avg_->mergeStates(*aggregation_state_avg_merge_double,
- aggregation_handle_avg_state_.get()),
- "");
+ static_cast<const AggregationHandleAvg &>(*aggregation_handle_avg_double)
+ .iterateUnaryInl(static_cast<AggregationStateAvg *>(
+ aggregation_state_avg_merge_double.get()),
+ double_type.makeValue(&double_val));
+ EXPECT_DEATH(
+ aggregation_handle_avg_->mergeStates(*aggregation_state_avg_merge_double,
+ aggregation_handle_avg_state_.get()),
+ "");
std::unique_ptr<AggregationHandle> aggregation_handle_avg_float(
- AggregateFunctionFactory::Get(AggregationID::kAvg).createHandle(
- std::vector<const Type*>(1, &float_type)));
+ AggregateFunctionFactory::Get(AggregationID::kAvg)
+ .createHandle(std::vector<const Type *>(1, &float_type)));
std::unique_ptr<AggregationState> aggregation_state_avg_merge_float(
aggregation_handle_avg_float->createInitialState());
- static_cast<const AggregationHandleAvg&>(*aggregation_handle_avg_float).iterateUnaryInl(
- static_cast<AggregationStateAvg*>(aggregation_state_avg_merge_float.get()),
- float_type.makeValue(&float_val));
- EXPECT_DEATH(aggregation_handle_avg_->mergeStates(*aggregation_state_avg_merge_float,
- aggregation_handle_avg_state_.get()),
- "");
+ static_cast<const AggregationHandleAvg &>(*aggregation_handle_avg_float)
+ .iterateUnaryInl(static_cast<AggregationStateAvg *>(
+ aggregation_state_avg_merge_float.get()),
+ float_type.makeValue(&float_val));
+ EXPECT_DEATH(
+ aggregation_handle_avg_->mergeStates(*aggregation_state_avg_merge_float,
+ aggregation_handle_avg_state_.get()),
+ "");
}
#endif
@@ -417,8 +457,10 @@ TEST_F(AggregationHandleAvgTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kLong, kDouble));
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kFloat, kDouble));
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kDouble, kDouble));
- EXPECT_TRUE(ResultTypeForArgumentTypeTest(kDatetimeInterval, kDatetimeInterval));
- EXPECT_TRUE(ResultTypeForArgumentTypeTest(kYearMonthInterval, kYearMonthInterval));
+ EXPECT_TRUE(
+ ResultTypeForArgumentTypeTest(kDatetimeInterval, kDatetimeInterval));
+ EXPECT_TRUE(
+ ResultTypeForArgumentTypeTest(kYearMonthInterval, kYearMonthInterval));
}
TEST_F(AggregationHandleAvgTest, GroupByTableMergeTestAvg) {
@@ -426,25 +468,28 @@ TEST_F(AggregationHandleAvgTest, GroupByTableMergeTestAvg) {
initializeHandle(long_non_null_type);
storage_manager_.reset(new StorageManager("./test_avg_data"));
std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
- aggregation_handle_avg_->createGroupByHashTable(
- HashTableImplType::kSimpleScalarSeparateChaining,
+ AggregationStateFastHashTableFactory::CreateResizable(
+ HashTableImplType::kSeparateChaining,
std::vector<const Type *>(1, &long_non_null_type),
10,
+ {aggregation_handle_avg_.get()->getPayloadSize()},
+ {aggregation_handle_avg_.get()},
storage_manager_.get()));
std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
- aggregation_handle_avg_->createGroupByHashTable(
- HashTableImplType::kSimpleScalarSeparateChaining,
+ AggregationStateFastHashTableFactory::CreateResizable(
+ HashTableImplType::kSeparateChaining,
std::vector<const Type *>(1, &long_non_null_type),
10,
+ {aggregation_handle_avg_.get()->getPayloadSize()},
+ {aggregation_handle_avg_.get()},
storage_manager_.get()));
- AggregationStateHashTable<AggregationStateAvg> *destination_hash_table_derived =
- static_cast<AggregationStateHashTable<AggregationStateAvg> *>(
+ AggregationStateFastHashTable *destination_hash_table_derived =
+ static_cast<AggregationStateFastHashTable *>(
destination_hash_table.get());
- AggregationStateHashTable<AggregationStateAvg> *source_hash_table_derived =
- static_cast<AggregationStateHashTable<AggregationStateAvg> *>(
- source_hash_table.get());
+ AggregationStateFastHashTable *source_hash_table_derived =
+ static_cast<AggregationStateFastHashTable *>(source_hash_table.get());
AggregationHandleAvg *aggregation_handle_avg_derived =
static_cast<AggregationHandleAvg *>(aggregation_handle_avg_.get());
@@ -496,36 +541,56 @@ TEST_F(AggregationHandleAvgTest, GroupByTableMergeTestAvg) {
exclusive_key_source_state.get(), exclusive_key_source_avg_val);
// Add the key-state pairs to the hash tables.
- source_hash_table_derived->putCompositeKey(common_key,
- *common_key_source_state);
- destination_hash_table_derived->putCompositeKey(
- common_key, *common_key_destination_state);
- source_hash_table_derived->putCompositeKey(exclusive_source_key,
- *exclusive_key_source_state);
- destination_hash_table_derived->putCompositeKey(
- exclusive_destination_key, *exclusive_key_destination_state);
+ unsigned char buffer[100];
+ buffer[0] = '\0';
+ memcpy(buffer + 1,
+ common_key_source_state.get()->getPayloadAddress(),
+ aggregation_handle_avg_.get()->getPayloadSize());
+ source_hash_table_derived->putCompositeKey(common_key, buffer);
+
+ memcpy(buffer + 1,
+ common_key_destination_state.get()->getPayloadAddress(),
+ aggregation_handle_avg_.get()->getPayloadSize());
+ destination_hash_table_derived->putCompositeKey(common_key, buffer);
+
+ memcpy(buffer + 1,
+ exclusive_key_source_state.get()->getPayloadAddress(),
+ aggregation_handle_avg_.get()->getPayloadSize());
+ source_hash_table_derived->putCompositeKey(exclusive_source_key, buffer);
+
+ memcpy(buffer + 1,
+ exclusive_key_destination_state.get()->getPayloadAddress(),
+ aggregation_handle_avg_.get()->getPayloadSize());
+ destination_hash_table_derived->putCompositeKey(exclusive_destination_key,
+ buffer);
EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
EXPECT_EQ(2u, source_hash_table_derived->numEntries());
- aggregation_handle_avg_->mergeGroupByHashTables(*source_hash_table,
- destination_hash_table.get());
+ AggregationOperationState::mergeGroupByHashTables(
+ source_hash_table.get(), destination_hash_table.get());
EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
CheckAvgValue<double>(
(common_key_destination_avg_val.getLiteral<std::int64_t>() +
- common_key_source_avg_val.getLiteral<std::int64_t>()) / static_cast<double>(2),
- *aggregation_handle_avg_derived,
- *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
- CheckAvgValue<double>(exclusive_key_destination_avg_val.getLiteral<std::int64_t>(),
- *aggregation_handle_avg_derived,
- *(destination_hash_table_derived->getSingleCompositeKey(
- exclusive_destination_key)));
- CheckAvgValue<double>(exclusive_key_source_avg_val.getLiteral<std::int64_t>(),
- *aggregation_handle_avg_derived,
- *(source_hash_table_derived->getSingleCompositeKey(
- exclusive_source_key)));
+ common_key_source_avg_val.getLiteral<std::int64_t>()) /
+ static_cast<double>(2),
+ aggregation_handle_avg_derived->finalizeHashTableEntryFast(
+ destination_hash_table_derived->getSingleCompositeKey(common_key) +
+ 1));
+ CheckAvgValue<double>(
+ exclusive_key_destination_avg_val.getLiteral<std::int64_t>(),
+ aggregation_handle_avg_derived->finalizeHashTableEntryFast(
+ destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key) +
+ 1));
+ CheckAvgValue<double>(
+ exclusive_key_source_avg_val.getLiteral<std::int64_t>(),
+ aggregation_handle_avg_derived->finalizeHashTableEntryFast(
+ source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key) +
+ 1));
}
} // namespace quickstep
[03/17] incubator-quickstep git commit: Initial commit for
QUICKSTEP-28 and QUICKSTEP-29.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/storage/FastHashTableFactory.hpp
----------------------------------------------------------------------
diff --git a/storage/FastHashTableFactory.hpp b/storage/FastHashTableFactory.hpp
new file mode 100644
index 0000000..6d0b693
--- /dev/null
+++ b/storage/FastHashTableFactory.hpp
@@ -0,0 +1,257 @@
+/**
+ * Copyright 2015-2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_FAST_HASH_TABLE_FACTORY_HPP_
+#define QUICKSTEP_STORAGE_FAST_HASH_TABLE_FACTORY_HPP_
+
+#include <cstddef>
+#include <string>
+#include <vector>
+
+#include "storage/HashTable.hpp"
+#include "storage/FastHashTable.hpp"
+#include "storage/HashTableBase.hpp"
+#include "storage/HashTableFactory.hpp"
+#include "storage/HashTable.pb.h"
+#include "storage/LinearOpenAddressingHashTable.hpp"
+#include "storage/SeparateChainingHashTable.hpp"
+#include "storage/FastSeparateChainingHashTable.hpp"
+#include "storage/SimpleScalarSeparateChainingHashTable.hpp"
+#include "storage/TupleReference.hpp"
+#include "types/TypeFactory.hpp"
+#include "utility/BloomFilter.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class StorageManager;
+class Type;
+
+/** \addtogroup Storage
+ * @{
+ */
+
+/**
+ * @brief Templated all-static factory class that makes it easier to
+ * instantiate HashTables with the particular HashTable implementation
+ * chosen at runtime. All template parameters are exactly the same as
+ * those of HashTable.
+ **/
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+class FastHashTableFactory {
+ public:
+ /**
+ * @brief Create a new resizable HashTable, with the type selected by
+ * hash_table_type. Other parameters are forwarded to the HashTable's
+ * constructor.
+ *
+ * @param hash_table_type The specific HashTable implementation that should
+ * be used.
+ * @param key_types A vector of one or more types (>1 indicates a composite
+ * key). Forwarded as-is to the HashTable's constructor.
+ * @param num_entries The estimated number of entries the HashTable will
+ * hold. Forwarded as-is to the HashTable's constructor.
+ * @param payload_sizes The sizes in bytes for the AggregationStates for the
+ * respective AggregationHandles.
+ * @param handles The AggregationHandles used in this HashTable.
+ * @param storage_manager The StorageManager to use (a StorageBlob will be
+ * allocated to hold the HashTable's contents). Forwarded as-is to the
+ * HashTable's constructor.
+ * @return A new resizable HashTable.
+ **/
+ static FastHashTable<resizable, serializable, force_key_copy, allow_duplicate_keys>*
+ CreateResizable(const HashTableImplType hash_table_type,
+ const std::vector<const Type*> &key_types,
+ const std::size_t num_entries,
+ const std::vector<std::size_t> &payload_sizes,
+ const std::vector<AggregationHandle *> &handles,
+ StorageManager *storage_manager) {
+ DCHECK(resizable);
+
+ switch (hash_table_type) {
+ case HashTableImplType::kSeparateChaining:
+ return new FastSeparateChainingHashTable<
+ resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>(key_types, num_entries, payload_sizes, handles, storage_manager);
+ default: {
+ LOG(FATAL) << "Unrecognized HashTableImplType in HashTableFactory::createResizable()\n";
+ }
+ }
+ }
+
+ /**
+ * @brief Create a new fixed-sized HashTable, with the type selected by
+ * hash_table_type. Other parameters are forwarded to the HashTables's
+ * constructor.
+ *
+ * @param hash_table_type The specific HashTable implementation that should
+ * be used.
+ * @param key_types A vector of one or more types (>1 indicates a composite
+ * key). Forwarded as-is to the HashTable's constructor.
+ * @param hash_table_memory A pointer to memory to use for the HashTable.
+ * Forwarded as-is to the HashTable's constructor.
+ * @param hash_table_memory_size The size of hash_table_memory in bytes.
+ * Forwarded as-is to the HashTable's constructor.
+ * @param new_hash_table If true, the HashTable is being constructed for the
+ * first time and hash_table_memory will be cleared. If false, reload
+ * a pre-existing HashTable. Forwarded as-is to the HashTable's
+ * constructor.
+ * @param hash_table_memory_zeroed If new_hash_table is true, setting this to
+ * true means that the HashTable will assume that hash_table_memory
+ * has already been zeroed-out (any newly-allocated block or blob
+ * memory from StorageManager is zeroed-out). If false, the HashTable
+ * will explicitly zero-fill its memory as neccessary. This parameter
+ * has no effect when new_hash_table is false. Forwarded as-is to the
+ * HashTable's constructor.
+ * @return A new (or reloaded) fixed-size HashTable.
+ **/
+ static FastHashTable<resizable, serializable, force_key_copy, allow_duplicate_keys>*
+ CreateFixedSize(const HashTableImplType hash_table_type,
+ const std::vector<const Type*> &key_types,
+ void *hash_table_memory,
+ const std::size_t hash_table_memory_size,
+ const bool new_hash_table,
+ const bool hash_table_memory_zeroed) {
+ DCHECK(!resizable);
+
+ switch (hash_table_type) {
+ case HashTableImplType::kSeparateChaining:
+ return new SeparateChainingHashTable<
+ int,
+ resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>(key_types,
+ hash_table_memory,
+ hash_table_memory_size,
+ new_hash_table,
+ hash_table_memory_zeroed);
+ default: {
+ LOG(FATAL) << "Unrecognized HashTableImplType\n";
+ }
+ }
+ }
+
+ /**
+ * @brief Check whether a serialization::HashTable describing a resizable
+ * HashTable is fully-formed and all parts are valid.
+ *
+ * @param proto A serialized Protocol Buffer description of a HashTable,
+ * originally generated by the optimizer.
+ * @return Whether proto is fully-formed and valid.
+ **/
+ static bool ProtoIsValid(const serialization::HashTable &proto) {
+ if (!proto.IsInitialized() ||
+ !serialization::HashTableImplType_IsValid(
+ proto.hash_table_impl_type())) {
+ return false;
+ }
+
+ for (int i = 0; i < proto.key_types_size(); ++i) {
+ if (!TypeFactory::ProtoIsValid(proto.key_types(i))) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * @brief Create a new resizable HashTable according to a protobuf
+ * description.
+ *
+ * @param proto A protobuf description of a resizable HashTable.
+ * @param storage_manager The StorageManager to use (a StorageBlob will be
+ * allocated to hold the HashTable's contents).
+ * @param bloom_filters A vector of pointers to bloom filters that may be used
+ * during hash table construction in build/probe phase.
+ * @return A new resizable HashTable with parameters specified by proto.
+ **/
+ static FastHashTable<resizable, serializable, force_key_copy, allow_duplicate_keys>*
+ CreateResizableFromProto(const serialization::HashTable &proto,
+ StorageManager *storage_manager,
+ const std::vector<std::unique_ptr<BloomFilter>> &bloom_filters) {
+ DCHECK(ProtoIsValid(proto))
+ << "Attempted to create HashTable from invalid proto description:\n"
+ << proto.DebugString();
+
+ std::vector<const Type*> key_types;
+ for (int i = 0; i < proto.key_types_size(); ++i) {
+ key_types.emplace_back(&TypeFactory::ReconstructFromProto(proto.key_types(i)));
+ }
+
+ auto hash_table = CreateResizable(HashTableImplTypeFromProto(proto.hash_table_impl_type()),
+ key_types,
+ proto.estimated_num_entries(),
+ storage_manager);
+
+ // TODO(ssaurabh): These lazy initializations can be moved from here and pushed to the
+ // individual implementations of the hash table constructors.
+
+ // Check if there are any build side bloom filter defined on the hash table.
+ if (proto.build_side_bloom_filter_id_size() > 0) {
+ hash_table->enableBuildSideBloomFilter();
+ hash_table->setBuildSideBloomFilter(bloom_filters[proto.build_side_bloom_filter_id(0)].get());
+ }
+
+ // Check if there are any probe side bloom filters defined on the hash table.
+ if (proto.probe_side_bloom_filters_size() > 0) {
+ hash_table->enableProbeSideBloomFilter();
+ // Add as many probe bloom filters as defined by the proto.
+ for (int j = 0; j < proto.probe_side_bloom_filters_size(); ++j) {
+ // Add the pointer to the probe bloom filter within the list of probe bloom filters to use.
+ const auto probe_side_bloom_filter = proto.probe_side_bloom_filters(j);
+ hash_table->addProbeSideBloomFilter(bloom_filters[probe_side_bloom_filter.probe_side_bloom_filter_id()].get());
+
+ // Add the attribute ids corresponding to this probe bloom filter.
+ std::vector<attribute_id> probe_attribute_ids;
+ for (int k = 0; k < probe_side_bloom_filter.probe_side_attr_ids_size(); ++k) {
+ const attribute_id probe_attribute_id = probe_side_bloom_filter.probe_side_attr_ids(k);
+ probe_attribute_ids.push_back(probe_attribute_id);
+ }
+ hash_table->addProbeSideAttributeIds(std::move(probe_attribute_ids));
+ }
+ }
+
+ return hash_table;
+ }
+
+ private:
+ // Class is all-static and should not be instantiated.
+ FastHashTableFactory();
+
+ DISALLOW_COPY_AND_ASSIGN(FastHashTableFactory);
+};
+
+/**
+ * @brief Convenient alias that provides a HashTableFactory whose only template
+ * parameter is the aggregate state type.
+ **/
+using AggregationStateFastHashTableFactory
+ = FastHashTableFactory<true, false, true, false>;
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_STORAGE_HASH_TABLE_FACTORY_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/storage/FastSeparateChainingHashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/FastSeparateChainingHashTable.hpp b/storage/FastSeparateChainingHashTable.hpp
new file mode 100644
index 0000000..886a8ca
--- /dev/null
+++ b/storage/FastSeparateChainingHashTable.hpp
@@ -0,0 +1,1750 @@
+/**
+ * Copyright 2011-2015 Quickstep Technologies LLC.
+ * Copyright 2015-2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_FAST_SEPARATE_CHAINING_HASH_TABLE_HPP_
+#define QUICKSTEP_STORAGE_FAST_SEPARATE_CHAINING_HASH_TABLE_HPP_
+
+#include <algorithm>
+#include <atomic>
+#include <cstddef>
+#include <cstring>
+#include <limits>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "storage/FastHashTable.hpp"
+#include "storage/HashTable.hpp"
+#include "storage/HashTableBase.hpp"
+#include "storage/HashTableKeyManager.hpp"
+#include "storage/StorageBlob.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageConstants.hpp"
+#include "storage/StorageManager.hpp"
+#include "threading/SpinSharedMutex.hpp"
+#include "types/Type.hpp"
+#include "types/TypedValue.hpp"
+#include "utility/Alignment.hpp"
+#include "utility/Macros.hpp"
+#include "utility/PrimeNumber.hpp"
+
+namespace quickstep {
+
+/** \addtogroup Storage
+ * @{
+ */
+
+/**
+ * @brief A hash table implementation which uses separate chaining for buckets.
+ **/
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+class FastSeparateChainingHashTable
+ : public FastHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys> {
+ public:
+ FastSeparateChainingHashTable(const std::vector<const Type *> &key_types,
+ const std::size_t num_entries,
+ const std::vector<std::size_t> &payload_sizes,
+ const std::vector<AggregationHandle *> &handles,
+ StorageManager *storage_manager);
+
+ FastSeparateChainingHashTable(const std::vector<const Type *> &key_types,
+ void *hash_table_memory,
+ const std::size_t hash_table_memory_size,
+ const bool new_hash_table,
+ const bool hash_table_memory_zeroed);
+
+ // Delegating constructors for single scalar keys.
+ FastSeparateChainingHashTable(const Type &key_type,
+ const std::size_t num_entries,
+ StorageManager *storage_manager)
+ : FastSeparateChainingHashTable(std::vector<const Type *>(1, &key_type),
+ num_entries,
+ storage_manager) {}
+
+ FastSeparateChainingHashTable(const Type &key_type,
+ void *hash_table_memory,
+ const std::size_t hash_table_memory_size,
+ const bool new_hash_table,
+ const bool hash_table_memory_zeroed)
+ : FastSeparateChainingHashTable(std::vector<const Type *>(1, &key_type),
+ hash_table_memory,
+ hash_table_memory_size,
+ new_hash_table,
+ hash_table_memory_zeroed) {}
+
+ ~FastSeparateChainingHashTable() override {
+ DestroyValues(buckets_,
+ header_->buckets_allocated.load(std::memory_order_relaxed),
+ bucket_size_);
+ std::free(init_payload_);
+ }
+
+ void clear() override;
+
+ std::size_t numEntries() const override {
+ return header_->buckets_allocated.load(std::memory_order_relaxed);
+ }
+
+ const std::uint8_t* getSingle(const TypedValue &key) const override;
+ const std::uint8_t* getSingleCompositeKey(
+ const std::vector<TypedValue> &key) const override;
+ const std::uint8_t* getSingleCompositeKey(const std::vector<TypedValue> &key,
+ int index) const override;
+
+ void getAll(const TypedValue &key,
+ std::vector<const std::uint8_t *> *values) const override;
+ void getAllCompositeKey(
+ const std::vector<TypedValue> &key,
+ std::vector<const std::uint8_t *> *values) const override;
+
+ protected:
+ HashTablePutResult putInternal(
+ const TypedValue &key,
+ const std::size_t variable_key_size,
+ const std::uint8_t &value,
+ HashTablePreallocationState *prealloc_state) override;
+
+ HashTablePutResult putCompositeKeyInternalFast(
+ const std::vector<TypedValue> &key,
+ const std::size_t variable_key_size,
+ const std::uint8_t *init_value_ptr,
+ HashTablePreallocationState *prealloc_state) override;
+
+ std::uint8_t* upsertInternalFast(const TypedValue &key,
+ const std::size_t variable_key_size,
+ const std::uint8_t *init_value_ptr) override;
+
+ std::uint8_t* upsertCompositeKeyInternalFast(
+ const std::vector<TypedValue> &key,
+ const std::uint8_t *init_value_ptr,
+ const std::size_t variable_key_size) override;
+
+ bool getNextEntry(TypedValue *key,
+ const std::uint8_t **value,
+ std::size_t *entry_num) const override;
+ bool getNextEntryCompositeKey(std::vector<TypedValue> *key,
+ const std::uint8_t **value,
+ std::size_t *entry_num) const override;
+
+ bool getNextEntryForKey(const TypedValue &key,
+ const std::size_t hash_code,
+ const std::uint8_t **value,
+ std::size_t *entry_num) const override;
+ bool getNextEntryForCompositeKey(const std::vector<TypedValue> &key,
+ const std::size_t hash_code,
+ const std::uint8_t **value,
+ std::size_t *entry_num) const override;
+
+ bool hasKey(const TypedValue &key) const override;
+ bool hasCompositeKey(const std::vector<TypedValue> &key) const override;
+
+ void resize(const std::size_t extra_buckets,
+ const std::size_t extra_variable_storage,
+ const std::size_t retry_num = 0) override;
+
+ bool preallocateForBulkInsert(
+ const std::size_t total_entries,
+ const std::size_t total_variable_key_size,
+ HashTablePreallocationState *prealloc_state) override;
+
+ private:
+ struct Header {
+ std::size_t num_slots;
+ std::size_t num_buckets;
+ alignas(kCacheLineBytes) std::atomic<std::size_t> buckets_allocated;
+ alignas(kCacheLineBytes)
+ std::atomic<std::size_t> variable_length_bytes_allocated;
+ };
+
+ std::uint8_t *init_payload_;
+ std::size_t kBucketAlignment;
+
+ // Value's offset in a bucket is the first alignof(ValueT) boundary after the
+ // next pointer and hash code.
+ std::size_t kValueOffset;
+
+ // Round bucket size up to a multiple of kBucketAlignment.
+ constexpr std::size_t ComputeBucketSize(const std::size_t fixed_key_size) {
+ return (((kValueOffset + this->total_payload_size_ + fixed_key_size - 1) /
+ kBucketAlignment) +
+ 1) *
+ kBucketAlignment;
+ }
+ // If ValueT is not trivially destructible, invoke its destructor for all
+ // values held in the specified buckets (including those in "empty" buckets
+ // that were default constructed). If ValueT is trivially destructible, this
+ // is a no-op.
+ void DestroyValues(void *buckets,
+ const std::size_t num_buckets,
+ const std::size_t bucket_size);
+
+ // Attempt to find an empty bucket to insert 'hash_code' into, starting after
+ // '*bucket' in the chain (or, if '*bucket' is NULL, starting from the slot
+ // array). Returns true and stores SIZE_T_MAX in '*pending_chain_ptr' if an
+ // empty bucket is found. Returns false if 'allow_duplicate_keys' is false
+ // and a hash collision is found (caller should then check whether there is a
+ // genuine key collision or the hash collision is spurious). Returns false
+ // and sets '*bucket' to NULL if there are no more empty buckets in the hash
+ // table. If 'variable_key_allocation_required' is nonzero, this method will
+ // attempt to allocate storage for a variable-length key BEFORE allocating a
+ // bucket, so that no bucket number below 'header_->num_buckets' is ever
+ // deallocated after being allocated.
+ inline bool locateBucketForInsertion(
+ const std::size_t hash_code,
+ const std::size_t variable_key_allocation_required,
+ void **bucket,
+ std::atomic<std::size_t> **pending_chain_ptr,
+ std::size_t *pending_chain_ptr_finish_value,
+ HashTablePreallocationState *prealloc_state);
+
+ // Write a scalar 'key' and its 'hash_code' into the '*bucket', which was
+ // found by locateBucketForInsertion(). Assumes that storage for a
+ // variable-length key copy (if any) was already allocated by a successful
+ // call to allocateVariableLengthKeyStorage().
+ inline void writeScalarKeyToBucket(
+ const TypedValue &key,
+ const std::size_t hash_code,
+ void *bucket,
+ HashTablePreallocationState *prealloc_state);
+
+ // Write a composite 'key' and its 'hash_code' into the '*bucket', which was
+ // found by locateBucketForInsertion(). Assumes that storage for
+ // variable-length key copies (if any) was already allocated by a successful
+ // call to allocateVariableLengthKeyStorage().
+ inline void writeCompositeKeyToBucket(
+ const std::vector<TypedValue> &key,
+ const std::size_t hash_code,
+ void *bucket,
+ HashTablePreallocationState *prealloc_state);
+
+ // Determine whether it is actually necessary to resize this hash table.
+ // Checks that there is at least one unallocated bucket, and that there is
+ // at least 'extra_variable_storage' bytes of variable-length storage free.
+ bool isFull(const std::size_t extra_variable_storage) const;
+
+ // Helper object to manage key storage.
+ HashTableKeyManager<serializable, force_key_copy> key_manager_;
+
+ // In-memory structure is as follows:
+ // - SeparateChainingHashTable::Header
+ // - Array of slots, interpreted as follows:
+ // - 0 = Points to nothing (empty)
+ // - SIZE_T_MAX = Pending (some thread is starting a chain from this
+ // slot and will overwrite it soon)
+ // - Anything else = The number of the first bucket in the chain for
+ // this slot PLUS ONE (i.e. subtract one to get the actual bucket
+ // number).
+ // - Array of buckets, each of which is:
+ // - atomic size_t "next" pointer, interpreted the same as slots above.
+ // - size_t hash value
+ // - possibly some unused bytes as needed so that ValueT's alignment
+ // requirement is met
+ // - ValueT value slot
+ // - fixed-length key storage (which may include pointers to external
+ // memory or offsets of variable length keys stored within this hash
+ // table)
+ // - possibly some additional unused bytes so that bucket size is a
+ // multiple of both alignof(std::atomic<std::size_t>) and
+ // alignof(ValueT)
+ // - Variable-length key storage region (referenced by offsets stored in
+ // fixed-length keys).
+ Header *header_;
+
+ std::atomic<std::size_t> *slots_;
+ void *buckets_;
+ const std::size_t bucket_size_;
+
+ DISALLOW_COPY_AND_ASSIGN(FastSeparateChainingHashTable);
+};
+
+/** @} */
+
+// ----------------------------------------------------------------------------
+// Implementations of template class methods follow.
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ FastSeparateChainingHashTable(
+ const std::vector<const Type *> &key_types,
+ const std::size_t num_entries,
+ const std::vector<std::size_t> &payload_sizes,
+ const std::vector<AggregationHandle *> &handles,
+ StorageManager *storage_manager)
+ : FastHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>(key_types,
+ num_entries,
+ handles,
+ payload_sizes,
+ storage_manager,
+ false,
+ false,
+ true),
+ kBucketAlignment(alignof(std::atomic<std::size_t>)),
+ kValueOffset(sizeof(std::atomic<std::size_t>) + sizeof(std::size_t)),
+ key_manager_(this->key_types_, kValueOffset + this->total_payload_size_),
+ bucket_size_(ComputeBucketSize(key_manager_.getFixedKeySize())) {
+ init_payload_ =
+ static_cast<std::uint8_t *>(calloc(this->total_payload_size_, 1));
+ int k = 0;
+ for (auto handle : handles) {
+ handle->initPayload(init_payload_ + this->payload_offsets_[k]);
+ k++;
+ }
+ // Bucket size always rounds up to the alignment requirement of the atomic
+ // size_t "next" pointer at the front or a ValueT, whichever is larger.
+ //
+ // Give base HashTable information about what key components are stored
+ // inline from 'key_manager_'.
+ this->setKeyInline(key_manager_.getKeyInline());
+
+ // Pick out a prime number of slots and calculate storage requirements.
+ std::size_t num_slots_tmp =
+ get_next_prime_number(num_entries * kHashTableLoadFactor);
+ std::size_t required_memory =
+ sizeof(Header) + num_slots_tmp * sizeof(std::atomic<std::size_t>) +
+ (num_slots_tmp / kHashTableLoadFactor) *
+ (bucket_size_ + key_manager_.getEstimatedVariableKeySize());
+ std::size_t num_storage_slots =
+ this->storage_manager_->SlotsNeededForBytes(required_memory);
+ if (num_storage_slots == 0) {
+ FATAL_ERROR(
+ "Storage requirement for SeparateChainingHashTable "
+ "exceeds maximum allocation size.");
+ }
+
+ // Get a StorageBlob to hold the hash table.
+ const block_id blob_id =
+ this->storage_manager_->createBlob(num_storage_slots);
+ this->blob_ = this->storage_manager_->getBlobMutable(blob_id);
+
+ void *aligned_memory_start = this->blob_->getMemoryMutable();
+ std::size_t available_memory = num_storage_slots * kSlotSizeBytes;
+ if (align(alignof(Header),
+ sizeof(Header),
+ aligned_memory_start,
+ available_memory) == nullptr) {
+ // With current values from StorageConstants.hpp, this should be
+ // impossible. A blob is at least 1 MB, while a Header has alignment
+ // requirement of just kCacheLineBytes (64 bytes).
+ FATAL_ERROR(
+ "StorageBlob used to hold resizable "
+ "SeparateChainingHashTable is too small to meet alignment "
+ "requirements of SeparateChainingHashTable::Header.");
+ } else if (aligned_memory_start != this->blob_->getMemoryMutable()) {
+ // This should also be impossible, since the StorageManager allocates slots
+ // aligned to kCacheLineBytes.
+ DEV_WARNING("StorageBlob memory adjusted by "
+ << (num_storage_slots * kSlotSizeBytes - available_memory)
+ << " bytes to meet alignment requirement for "
+ << "SeparateChainingHashTable::Header.");
+ }
+
+ // Locate the header.
+ header_ = static_cast<Header *>(aligned_memory_start);
+ aligned_memory_start =
+ static_cast<char *>(aligned_memory_start) + sizeof(Header);
+ available_memory -= sizeof(Header);
+
+ // Recompute the number of slots & buckets using the actual available memory.
+ // Most likely, we got some extra free bucket space due to "rounding up" to
+ // the storage blob's size. It's also possible (though very unlikely) that we
+ // will wind up with fewer buckets than we initially wanted because of screwy
+ // alignment requirements for ValueT.
+ std::size_t num_buckets_tmp =
+ available_memory /
+ (kHashTableLoadFactor * sizeof(std::atomic<std::size_t>) + bucket_size_ +
+ key_manager_.getEstimatedVariableKeySize());
+ num_slots_tmp =
+ get_previous_prime_number(num_buckets_tmp * kHashTableLoadFactor);
+ num_buckets_tmp = num_slots_tmp / kHashTableLoadFactor;
+ DEBUG_ASSERT(num_slots_tmp > 0);
+ DEBUG_ASSERT(num_buckets_tmp > 0);
+
+ // Locate the slot array.
+ slots_ = static_cast<std::atomic<std::size_t> *>(aligned_memory_start);
+ aligned_memory_start = static_cast<char *>(aligned_memory_start) +
+ sizeof(std::atomic<std::size_t>) * num_slots_tmp;
+ available_memory -= sizeof(std::atomic<std::size_t>) * num_slots_tmp;
+
+ // Locate the buckets.
+ buckets_ = aligned_memory_start;
+ // Extra-paranoid: If ValueT has an alignment requirement greater than that
+ // of std::atomic<std::size_t>, we may need to adjust the start of the bucket
+ // array.
+ if (align(kBucketAlignment, bucket_size_, buckets_, available_memory) ==
+ nullptr) {
+ FATAL_ERROR(
+ "StorageBlob used to hold resizable "
+ "SeparateChainingHashTable is too small to meet "
+ "alignment requirements of buckets.");
+ } else if (buckets_ != aligned_memory_start) {
+ DEV_WARNING(
+ "Bucket array start position adjusted to meet alignment "
+ "requirement for SeparateChainingHashTable's value type.");
+ if (num_buckets_tmp * bucket_size_ > available_memory) {
+ --num_buckets_tmp;
+ }
+ }
+
+ // Fill in the header.
+ header_->num_slots = num_slots_tmp;
+ header_->num_buckets = num_buckets_tmp;
+ header_->buckets_allocated.store(0, std::memory_order_relaxed);
+ header_->variable_length_bytes_allocated.store(0, std::memory_order_relaxed);
+ available_memory -= bucket_size_ * (header_->num_buckets);
+
+ // Locate variable-length key storage region, and give it all the remaining
+ // bytes in the blob.
+ key_manager_.setVariableLengthStorageInfo(
+ static_cast<char *>(buckets_) + header_->num_buckets * bucket_size_,
+ available_memory,
+ &(header_->variable_length_bytes_allocated));
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ FastSeparateChainingHashTable(const std::vector<const Type *> &key_types,
+ void *hash_table_memory,
+ const std::size_t hash_table_memory_size,
+ const bool new_hash_table,
+ const bool hash_table_memory_zeroed)
+ : FastHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>(key_types,
+ hash_table_memory,
+ hash_table_memory_size,
+ new_hash_table,
+ hash_table_memory_zeroed,
+ false,
+ false,
+ true),
+ kBucketAlignment(alignof(std::atomic<std::size_t>) < alignof(std::uint8_t)
+ ? alignof(std::uint8_t)
+ : alignof(std::atomic<std::size_t>)),
+ kValueOffset(sizeof(std::atomic<std::size_t>) + sizeof(std::size_t)),
+ key_manager_(this->key_types_, kValueOffset + sizeof(std::uint8_t)),
+ bucket_size_(ComputeBucketSize(key_manager_.getFixedKeySize())) {
+ // Bucket size always rounds up to the alignment requirement of the atomic
+ // size_t "next" pointer at the front or a ValueT, whichever is larger.
+ //
+ // Make sure that the larger of the two alignment requirements also satisfies
+ // the smaller.
+ static_assert(
+ alignof(std::atomic<std::size_t>) < alignof(std::uint8_t)
+ ? alignof(std::uint8_t) % alignof(std::atomic<std::size_t>) == 0
+ : alignof(std::atomic<std::size_t>) % alignof(std::uint8_t) == 0,
+ "Alignment requirement of std::atomic<std::size_t> does not "
+ "evenly divide with alignment requirement of ValueT.");
+
+ // Give base HashTable information about what key components are stored
+ // inline from 'key_manager_'.
+ this->setKeyInline(key_manager_.getKeyInline());
+
+ // FIXME(chasseur): If we are reconstituting a HashTable using a block of
+ // memory whose start was aligned differently than the memory block that was
+ // originally used (modulo alignof(Header)), we could wind up with all of our
+ // data structures misaligned. If memory is inside a
+ // StorageBlock/StorageBlob, this will never occur, since the StorageManager
+ // always allocates slots aligned to kCacheLineBytes. Similarly, this isn't
+ // a problem for memory inside any other allocation aligned to at least
+ // alignof(Header) == kCacheLineBytes.
+
+ void *aligned_memory_start = this->hash_table_memory_;
+ std::size_t available_memory = this->hash_table_memory_size_;
+
+ if (align(alignof(Header),
+ sizeof(Header),
+ aligned_memory_start,
+ available_memory) == nullptr) {
+ FATAL_ERROR("Attempted to create a non-resizable "
+ << "SeparateChainingHashTable with "
+ << available_memory
+ << " bytes of memory at "
+ << aligned_memory_start
+ << " which either can not fit a "
+ << "SeparateChainingHashTable::Header or meet its alignement "
+ << "requirement.");
+ } else if (aligned_memory_start != this->hash_table_memory_) {
+ // In general, we could get memory of any alignment, although at least
+ // cache-line aligned would be nice.
+ DEV_WARNING("StorageBlob memory adjusted by "
+ << (this->hash_table_memory_size_ - available_memory)
+ << " bytes to meet alignment requirement for "
+ << "SeparateChainingHashTable::Header.");
+ }
+
+ header_ = static_cast<Header *>(aligned_memory_start);
+ aligned_memory_start =
+ static_cast<char *>(aligned_memory_start) + sizeof(Header);
+ available_memory -= sizeof(Header);
+
+ if (new_hash_table) {
+ std::size_t estimated_bucket_capacity =
+ available_memory /
+ (kHashTableLoadFactor * sizeof(std::atomic<std::size_t>) +
+ bucket_size_ + key_manager_.getEstimatedVariableKeySize());
+ std::size_t num_slots = get_previous_prime_number(
+ estimated_bucket_capacity * kHashTableLoadFactor);
+
+ // Fill in the header.
+ header_->num_slots = num_slots;
+ header_->num_buckets = num_slots / kHashTableLoadFactor;
+ header_->buckets_allocated.store(0, std::memory_order_relaxed);
+ header_->variable_length_bytes_allocated.store(0,
+ std::memory_order_relaxed);
+ }
+
+ // Locate the slot array.
+ slots_ = static_cast<std::atomic<std::size_t> *>(aligned_memory_start);
+ aligned_memory_start = static_cast<char *>(aligned_memory_start) +
+ sizeof(std::atomic<std::size_t>) * header_->num_slots;
+ available_memory -= sizeof(std::atomic<std::size_t>) * header_->num_slots;
+
+ if (new_hash_table && !hash_table_memory_zeroed) {
+ std::memset(
+ slots_, 0x0, sizeof(std::atomic<std::size_t>) * header_->num_slots);
+ }
+
+ // Locate the buckets.
+ buckets_ = aligned_memory_start;
+ // Extra-paranoid: sizeof(Header) should almost certainly be a multiple of
+ // kBucketAlignment, unless ValueT has some members with seriously big
+ // (> kCacheLineBytes) alignment requirements specified using alignas().
+ if (align(kBucketAlignment, bucket_size_, buckets_, available_memory) ==
+ nullptr) {
+ FATAL_ERROR("Attempted to create a non-resizable "
+ << "SeparateChainingHashTable with "
+ << this->hash_table_memory_size_
+ << " bytes of memory at "
+ << this->hash_table_memory_
+ << ", which can hold an aligned "
+ << "SeparateChainingHashTable::Header but does not have "
+ << "enough remaining space for even a single hash bucket.");
+ } else if (buckets_ != aligned_memory_start) {
+ DEV_WARNING(
+ "Bucket array start position adjusted to meet alignment "
+ "requirement for SeparateChainingHashTable's value type.");
+ if (header_->num_buckets * bucket_size_ > available_memory) {
+ DEBUG_ASSERT(new_hash_table);
+ --(header_->num_buckets);
+ }
+ }
+ available_memory -= bucket_size_ * header_->num_buckets;
+
+ // Make sure "next" pointers in buckets are zeroed-out.
+ if (new_hash_table && !hash_table_memory_zeroed) {
+ std::memset(buckets_, 0x0, header_->num_buckets * bucket_size_);
+ }
+
+ // Locate variable-length key storage region.
+ key_manager_.setVariableLengthStorageInfo(
+ static_cast<char *>(buckets_) + header_->num_buckets * bucket_size_,
+ available_memory,
+ &(header_->variable_length_bytes_allocated));
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+void FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::clear() {
+ const std::size_t used_buckets =
+ header_->buckets_allocated.load(std::memory_order_relaxed);
+ // Destroy existing values, if necessary.
+ DestroyValues(buckets_, used_buckets, bucket_size_);
+
+ // Zero-out slot array.
+ std::memset(
+ slots_, 0x0, sizeof(std::atomic<std::size_t>) * header_->num_slots);
+
+ // Zero-out used buckets.
+ std::memset(buckets_, 0x0, used_buckets * bucket_size_);
+
+ header_->buckets_allocated.store(0, std::memory_order_relaxed);
+ header_->variable_length_bytes_allocated.store(0, std::memory_order_relaxed);
+ key_manager_.zeroNextVariableLengthKeyOffset();
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+const std::uint8_t* FastSeparateChainingHashTable<
+ resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::getSingle(const TypedValue &key) const {
+ DEBUG_ASSERT(!allow_duplicate_keys);
+ DEBUG_ASSERT(this->key_types_.size() == 1);
+ DEBUG_ASSERT(
+ key.isPlausibleInstanceOf(this->key_types_.front()->getSignature()));
+
+ const std::size_t hash_code = key.getHash();
+ std::size_t bucket_ref =
+ slots_[hash_code % header_->num_slots].load(std::memory_order_relaxed);
+ while (bucket_ref != 0) {
+ DEBUG_ASSERT(bucket_ref != std::numeric_limits<std::size_t>::max());
+ const char *bucket =
+ static_cast<const char *>(buckets_) + (bucket_ref - 1) * bucket_size_;
+ const std::size_t bucket_hash = *reinterpret_cast<const std::size_t *>(
+ bucket + sizeof(std::atomic<std::size_t>));
+ if ((bucket_hash == hash_code) &&
+ key_manager_.scalarKeyCollisionCheck(key, bucket)) {
+ // Match located.
+ return reinterpret_cast<const std::uint8_t *>(bucket + kValueOffset);
+ }
+ bucket_ref =
+ reinterpret_cast<const std::atomic<std::size_t> *>(bucket)->load(
+ std::memory_order_relaxed);
+ }
+
+ // Reached the end of the chain and didn't find a match.
+ return nullptr;
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+const std::uint8_t* FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ getSingleCompositeKey(const std::vector<TypedValue> &key) const {
+ DEBUG_ASSERT(!allow_duplicate_keys);
+ DEBUG_ASSERT(this->key_types_.size() == key.size());
+
+ const std::size_t hash_code = this->hashCompositeKey(key);
+ std::size_t bucket_ref =
+ slots_[hash_code % header_->num_slots].load(std::memory_order_relaxed);
+ while (bucket_ref != 0) {
+ DEBUG_ASSERT(bucket_ref != std::numeric_limits<std::size_t>::max());
+ const char *bucket =
+ static_cast<const char *>(buckets_) + (bucket_ref - 1) * bucket_size_;
+ const std::size_t bucket_hash = *reinterpret_cast<const std::size_t *>(
+ bucket + sizeof(std::atomic<std::size_t>));
+ if ((bucket_hash == hash_code) &&
+ key_manager_.compositeKeyCollisionCheck(key, bucket)) {
+ // Match located.
+ return reinterpret_cast<const std::uint8_t *>(bucket + kValueOffset);
+ }
+ bucket_ref =
+ reinterpret_cast<const std::atomic<std::size_t> *>(bucket)->load(
+ std::memory_order_relaxed);
+ }
+
+ // Reached the end of the chain and didn't find a match.
+ return nullptr;
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+const std::uint8_t* FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ getSingleCompositeKey(const std::vector<TypedValue> &key, int index) const {
+ DEBUG_ASSERT(!allow_duplicate_keys);
+ DEBUG_ASSERT(this->key_types_.size() == key.size());
+
+ const std::size_t hash_code = this->hashCompositeKey(key);
+ std::size_t bucket_ref =
+ slots_[hash_code % header_->num_slots].load(std::memory_order_relaxed);
+ while (bucket_ref != 0) {
+ DEBUG_ASSERT(bucket_ref != std::numeric_limits<std::size_t>::max());
+ const char *bucket =
+ static_cast<const char *>(buckets_) + (bucket_ref - 1) * bucket_size_;
+ const std::size_t bucket_hash = *reinterpret_cast<const std::size_t *>(
+ bucket + sizeof(std::atomic<std::size_t>));
+ if ((bucket_hash == hash_code) &&
+ key_manager_.compositeKeyCollisionCheck(key, bucket)) {
+ // Match located.
+ return reinterpret_cast<const std::uint8_t *>(bucket + kValueOffset) +
+ this->payload_offsets_[index];
+ }
+ bucket_ref =
+ reinterpret_cast<const std::atomic<std::size_t> *>(bucket)->load(
+ std::memory_order_relaxed);
+ }
+
+ // Reached the end of the chain and didn't find a match.
+ return nullptr;
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+void FastSeparateChainingHashTable<
+ resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::getAll(const TypedValue &key,
+ std::vector<const std::uint8_t *> *values)
+ const {
+ DEBUG_ASSERT(this->key_types_.size() == 1);
+ DEBUG_ASSERT(
+ key.isPlausibleInstanceOf(this->key_types_.front()->getSignature()));
+
+ const std::size_t hash_code = key.getHash();
+ std::size_t bucket_ref =
+ slots_[hash_code % header_->num_slots].load(std::memory_order_relaxed);
+ while (bucket_ref != 0) {
+ DEBUG_ASSERT(bucket_ref != std::numeric_limits<std::size_t>::max());
+ const char *bucket =
+ static_cast<const char *>(buckets_) + (bucket_ref - 1) * bucket_size_;
+ const std::size_t bucket_hash = *reinterpret_cast<const std::size_t *>(
+ bucket + sizeof(std::atomic<std::size_t>));
+ if ((bucket_hash == hash_code) &&
+ key_manager_.scalarKeyCollisionCheck(key, bucket)) {
+ // Match located.
+ values->push_back(
+ reinterpret_cast<const std::uint8_t *>(bucket + kValueOffset));
+ if (!allow_duplicate_keys) {
+ return;
+ }
+ }
+ bucket_ref =
+ reinterpret_cast<const std::atomic<std::size_t> *>(bucket)->load(
+ std::memory_order_relaxed);
+ }
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+void FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ getAllCompositeKey(const std::vector<TypedValue> &key,
+ std::vector<const std::uint8_t *> *values) const {
+ DEBUG_ASSERT(this->key_types_.size() == key.size());
+
+ const std::size_t hash_code = this->hashCompositeKey(key);
+ std::size_t bucket_ref =
+ slots_[hash_code % header_->num_slots].load(std::memory_order_relaxed);
+ while (bucket_ref != 0) {
+ DEBUG_ASSERT(bucket_ref != std::numeric_limits<std::size_t>::max());
+ const char *bucket =
+ static_cast<const char *>(buckets_) + (bucket_ref - 1) * bucket_size_;
+ const std::size_t bucket_hash = *reinterpret_cast<const std::size_t *>(
+ bucket + sizeof(std::atomic<std::size_t>));
+ if ((bucket_hash == hash_code) &&
+ key_manager_.compositeKeyCollisionCheck(key, bucket)) {
+ // Match located.
+ values->push_back(
+ reinterpret_cast<const std::uint8_t *>(bucket + kValueOffset));
+ if (!allow_duplicate_keys) {
+ return;
+ }
+ }
+ bucket_ref =
+ reinterpret_cast<const std::atomic<std::size_t> *>(bucket)->load(
+ std::memory_order_relaxed);
+ }
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+HashTablePutResult FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ putInternal(const TypedValue &key,
+ const std::size_t variable_key_size,
+ const std::uint8_t &value,
+ HashTablePreallocationState *prealloc_state) {
+ DEBUG_ASSERT(this->key_types_.size() == 1);
+ DEBUG_ASSERT(
+ key.isPlausibleInstanceOf(this->key_types_.front()->getSignature()));
+
+ if (prealloc_state == nullptr) {
+ // Early check for a free bucket.
+ if (header_->buckets_allocated.load(std::memory_order_relaxed) >=
+ header_->num_buckets) {
+ return HashTablePutResult::kOutOfSpace;
+ }
+
+ // TODO(chasseur): If allow_duplicate_keys is true, avoid storing more than
+ // one copy of the same variable-length key.
+ if (!key_manager_.allocateVariableLengthKeyStorage(variable_key_size)) {
+ // Ran out of variable-length key storage space.
+ return HashTablePutResult::kOutOfSpace;
+ }
+ }
+
+ const std::size_t hash_code = key.getHash();
+ void *bucket = nullptr;
+ std::atomic<std::size_t> *pending_chain_ptr;
+ std::size_t pending_chain_ptr_finish_value;
+ for (;;) {
+ if (locateBucketForInsertion(hash_code,
+ 0,
+ &bucket,
+ &pending_chain_ptr,
+ &pending_chain_ptr_finish_value,
+ prealloc_state)) {
+ // Found an empty bucket.
+ break;
+ } else if (bucket == nullptr) {
+ // Ran out of buckets. Deallocate any variable space that we were unable
+ // to use.
+ DEBUG_ASSERT(prealloc_state == nullptr);
+ key_manager_.deallocateVariableLengthKeyStorage(variable_key_size);
+ return HashTablePutResult::kOutOfSpace;
+ } else {
+ // Hash collision found, and duplicates aren't allowed.
+ DEBUG_ASSERT(!allow_duplicate_keys);
+ DEBUG_ASSERT(prealloc_state == nullptr);
+ if (key_manager_.scalarKeyCollisionCheck(key, bucket)) {
+ // Duplicate key. Deallocate any variable storage space and return.
+ key_manager_.deallocateVariableLengthKeyStorage(variable_key_size);
+ return HashTablePutResult::kDuplicateKey;
+ }
+ }
+ }
+
+ // Write the key and hash.
+ writeScalarKeyToBucket(key, hash_code, bucket, prealloc_state);
+
+ // Store the value by using placement new with ValueT's copy constructor.
+ new (static_cast<char *>(bucket) + kValueOffset) std::uint8_t(value);
+
+ // Update the previous chain pointer to point to the new bucket.
+ pending_chain_ptr->store(pending_chain_ptr_finish_value,
+ std::memory_order_release);
+
+ // We're all done.
+ return HashTablePutResult::kOK;
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+HashTablePutResult FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ putCompositeKeyInternalFast(const std::vector<TypedValue> &key,
+ const std::size_t variable_key_size,
+ const std::uint8_t *init_value_ptr,
+ HashTablePreallocationState *prealloc_state) {
+ DEBUG_ASSERT(this->key_types_.size() == key.size());
+
+ if (prealloc_state == nullptr) {
+ // Early check for a free bucket.
+ if (header_->buckets_allocated.load(std::memory_order_relaxed) >=
+ header_->num_buckets) {
+ return HashTablePutResult::kOutOfSpace;
+ }
+
+ // TODO(chasseur): If allow_duplicate_keys is true, avoid storing more than
+ // one copy of the same variable-length key.
+ if (!key_manager_.allocateVariableLengthKeyStorage(variable_key_size)) {
+ // Ran out of variable-length key storage space.
+ return HashTablePutResult::kOutOfSpace;
+ }
+ }
+
+ const std::size_t hash_code = this->hashCompositeKey(key);
+ void *bucket = nullptr;
+ std::atomic<std::size_t> *pending_chain_ptr;
+ std::size_t pending_chain_ptr_finish_value;
+ for (;;) {
+ if (locateBucketForInsertion(hash_code,
+ 0,
+ &bucket,
+ &pending_chain_ptr,
+ &pending_chain_ptr_finish_value,
+ prealloc_state)) {
+ // Found an empty bucket.
+ break;
+ } else if (bucket == nullptr) {
+ // Ran out of buckets. Deallocate any variable space that we were unable
+ // to use.
+ DEBUG_ASSERT(prealloc_state == nullptr);
+ key_manager_.deallocateVariableLengthKeyStorage(variable_key_size);
+ return HashTablePutResult::kOutOfSpace;
+ } else {
+ // Hash collision found, and duplicates aren't allowed.
+ DEBUG_ASSERT(!allow_duplicate_keys);
+ DEBUG_ASSERT(prealloc_state == nullptr);
+ if (key_manager_.compositeKeyCollisionCheck(key, bucket)) {
+ // Duplicate key. Deallocate any variable storage space and return.
+ key_manager_.deallocateVariableLengthKeyStorage(variable_key_size);
+ return HashTablePutResult::kDuplicateKey;
+ }
+ }
+ }
+
+ // Write the key and hash.
+ writeCompositeKeyToBucket(key, hash_code, bucket, prealloc_state);
+
+ std::uint8_t *value = static_cast<std::uint8_t *>(bucket) + kValueOffset;
+ memcpy(value, init_value_ptr, this->total_payload_size_);
+ // Update the previous chain pointer to point to the new bucket.
+ pending_chain_ptr->store(pending_chain_ptr_finish_value,
+ std::memory_order_release);
+
+ // We're all done.
+ return HashTablePutResult::kOK;
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+std::uint8_t* FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ upsertInternalFast(const TypedValue &key,
+ const std::size_t variable_key_size,
+ const std::uint8_t *init_value_ptr) {
+ DEBUG_ASSERT(!allow_duplicate_keys);
+ DEBUG_ASSERT(this->key_types_.size() == 1);
+ DEBUG_ASSERT(
+ key.isPlausibleInstanceOf(this->key_types_.front()->getSignature()));
+
+ if (variable_key_size > 0) {
+ // Don't allocate yet, since the key may already be present. However, we
+ // do check if either the allocated variable storage space OR the free
+ // space is big enough to hold the key (at least one must be true: either
+ // the key is already present and allocated, or we need to be able to
+ // allocate enough space for it).
+ std::size_t allocated_bytes = header_->variable_length_bytes_allocated.load(
+ std::memory_order_relaxed);
+ if ((allocated_bytes < variable_key_size) &&
+ (allocated_bytes + variable_key_size >
+ key_manager_.getVariableLengthKeyStorageSize())) {
+ return nullptr;
+ }
+ }
+
+ const std::size_t hash_code = key.getHash();
+ void *bucket = nullptr;
+ std::atomic<std::size_t> *pending_chain_ptr;
+ std::size_t pending_chain_ptr_finish_value;
+ for (;;) {
+ if (locateBucketForInsertion(hash_code,
+ variable_key_size,
+ &bucket,
+ &pending_chain_ptr,
+ &pending_chain_ptr_finish_value,
+ nullptr)) {
+ // Found an empty bucket.
+ break;
+ } else if (bucket == nullptr) {
+ // Ran out of buckets or variable-key space.
+ return nullptr;
+ } else if (key_manager_.scalarKeyCollisionCheck(key, bucket)) {
+ // Found an already-existing entry for this key.
+ return reinterpret_cast<std::uint8_t *>(static_cast<char *>(bucket) +
+ kValueOffset);
+ }
+ }
+
+ // We are now writing to an empty bucket.
+ // Write the key and hash.
+ writeScalarKeyToBucket(key, hash_code, bucket, nullptr);
+
+ // Copy the supplied 'initial_value' into place.
+ std::uint8_t *value = static_cast<unsigned char *>(bucket) + kValueOffset;
+ if (init_value_ptr == nullptr)
+ memcpy(value, init_payload_, this->total_payload_size_);
+ else
+ memcpy(value, init_value_ptr, this->total_payload_size_);
+
+ // Update the previous chain pointer to point to the new bucket.
+ pending_chain_ptr->store(pending_chain_ptr_finish_value,
+ std::memory_order_release);
+
+ // Return the value.
+ return value;
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+std::uint8_t* FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ upsertCompositeKeyInternalFast(const std::vector<TypedValue> &key,
+ const std::uint8_t *init_value_ptr,
+ const std::size_t variable_key_size) {
+ DEBUG_ASSERT(!allow_duplicate_keys);
+ DEBUG_ASSERT(this->key_types_.size() == key.size());
+
+ if (variable_key_size > 0) {
+ // Don't allocate yet, since the key may already be present. However, we
+ // do check if either the allocated variable storage space OR the free
+ // space is big enough to hold the key (at least one must be true: either
+ // the key is already present and allocated, or we need to be able to
+ // allocate enough space for it).
+ std::size_t allocated_bytes = header_->variable_length_bytes_allocated.load(
+ std::memory_order_relaxed);
+ if ((allocated_bytes < variable_key_size) &&
+ (allocated_bytes + variable_key_size >
+ key_manager_.getVariableLengthKeyStorageSize())) {
+ return nullptr;
+ }
+ }
+
+ const std::size_t hash_code = this->hashCompositeKey(key);
+ void *bucket = nullptr;
+ std::atomic<std::size_t> *pending_chain_ptr;
+ std::size_t pending_chain_ptr_finish_value;
+ for (;;) {
+ if (locateBucketForInsertion(hash_code,
+ variable_key_size,
+ &bucket,
+ &pending_chain_ptr,
+ &pending_chain_ptr_finish_value,
+ nullptr)) {
+ // Found an empty bucket.
+ break;
+ } else if (bucket == nullptr) {
+ // Ran out of buckets or variable-key space.
+ return nullptr;
+ } else if (key_manager_.compositeKeyCollisionCheck(key, bucket)) {
+ // Found an already-existing entry for this key.
+ return reinterpret_cast<std::uint8_t *>(static_cast<char *>(bucket) +
+ kValueOffset);
+ }
+ }
+
+ // We are now writing to an empty bucket.
+ // Write the key and hash.
+ writeCompositeKeyToBucket(key, hash_code, bucket, nullptr);
+
+ std::uint8_t *value = static_cast<unsigned char *>(bucket) + kValueOffset;
+ if (init_value_ptr == nullptr) {
+ memcpy(value, init_payload_, this->total_payload_size_);
+ } else {
+ memcpy(value, init_value_ptr, this->total_payload_size_);
+ }
+
+ // Update the previous chaing pointer to point to the new bucket.
+ pending_chain_ptr->store(pending_chain_ptr_finish_value,
+ std::memory_order_release);
+
+ // Return the value.
+ return value;
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+bool FastSeparateChainingHashTable<
+ resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::getNextEntry(TypedValue *key,
+ const std::uint8_t **value,
+ std::size_t *entry_num) const {
+ DEBUG_ASSERT(this->key_types_.size() == 1);
+ if (*entry_num < header_->buckets_allocated.load(std::memory_order_relaxed)) {
+ const char *bucket =
+ static_cast<const char *>(buckets_) + (*entry_num) * bucket_size_;
+ *key = key_manager_.getKeyComponentTyped(bucket, 0);
+ *value = reinterpret_cast<const std::uint8_t *>(bucket + kValueOffset);
+ ++(*entry_num);
+ return true;
+ } else {
+ return false;
+ }
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+bool FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ getNextEntryCompositeKey(std::vector<TypedValue> *key,
+ const std::uint8_t **value,
+ std::size_t *entry_num) const {
+ if (*entry_num < header_->buckets_allocated.load(std::memory_order_relaxed)) {
+ const char *bucket =
+ static_cast<const char *>(buckets_) + (*entry_num) * bucket_size_;
+ for (std::vector<const Type *>::size_type key_idx = 0;
+ key_idx < this->key_types_.size();
+ ++key_idx) {
+ key->emplace_back(key_manager_.getKeyComponentTyped(bucket, key_idx));
+ }
+ *value = reinterpret_cast<const std::uint8_t *>(bucket + kValueOffset);
+ ++(*entry_num);
+ return true;
+ } else {
+ return false;
+ }
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+bool FastSeparateChainingHashTable<
+ resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::getNextEntryForKey(const TypedValue &key,
+ const std::size_t hash_code,
+ const std::uint8_t **value,
+ std::size_t *entry_num) const {
+ DEBUG_ASSERT(this->key_types_.size() == 1);
+ DEBUG_ASSERT(
+ key.isPlausibleInstanceOf(this->key_types_.front()->getSignature()));
+
+ if (*entry_num == 0) {
+ *entry_num =
+ slots_[hash_code % header_->num_slots].load(std::memory_order_relaxed);
+ } else if (*entry_num == std::numeric_limits<std::size_t>::max()) {
+ return false;
+ }
+
+ while (*entry_num != 0) {
+ DEBUG_ASSERT(*entry_num != std::numeric_limits<std::size_t>::max());
+ const char *bucket =
+ static_cast<const char *>(buckets_) + (*entry_num - 1) * bucket_size_;
+ *entry_num =
+ reinterpret_cast<const std::atomic<std::size_t> *>(bucket)->load(
+ std::memory_order_relaxed);
+ const std::size_t bucket_hash = *reinterpret_cast<const std::size_t *>(
+ bucket + sizeof(std::atomic<std::size_t>));
+ if ((bucket_hash == hash_code) &&
+ key_manager_.scalarKeyCollisionCheck(key, bucket)) {
+ // Match located.
+ *value = reinterpret_cast<const std::uint8_t *>(bucket + kValueOffset);
+ if (*entry_num == 0) {
+ // If this is the last bucket in the chain, prevent the next call from
+ // starting over again.
+ *entry_num = std::numeric_limits<std::size_t>::max();
+ }
+ return true;
+ }
+ }
+
+ // Reached the end of the chain.
+ return false;
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+bool FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ getNextEntryForCompositeKey(const std::vector<TypedValue> &key,
+ const std::size_t hash_code,
+ const std::uint8_t **value,
+ std::size_t *entry_num) const {
+ DEBUG_ASSERT(this->key_types_.size() == key.size());
+
+ if (*entry_num == 0) {
+ *entry_num =
+ slots_[hash_code % header_->num_slots].load(std::memory_order_relaxed);
+ } else if (*entry_num == std::numeric_limits<std::size_t>::max()) {
+ return false;
+ }
+
+ while (*entry_num != 0) {
+ DEBUG_ASSERT(*entry_num != std::numeric_limits<std::size_t>::max());
+ const char *bucket =
+ static_cast<const char *>(buckets_) + (*entry_num - 1) * bucket_size_;
+ *entry_num =
+ reinterpret_cast<const std::atomic<std::size_t> *>(bucket)->load(
+ std::memory_order_relaxed);
+ const std::size_t bucket_hash = *reinterpret_cast<const std::size_t *>(
+ bucket + sizeof(std::atomic<std::size_t>));
+ if ((bucket_hash == hash_code) &&
+ key_manager_.compositeKeyCollisionCheck(key, bucket)) {
+ // Match located.
+ *value = reinterpret_cast<const std::uint8_t *>(bucket + kValueOffset);
+ if (*entry_num == 0) {
+ // If this is the last bucket in the chain, prevent the next call from
+ // starting over again.
+ *entry_num = std::numeric_limits<std::size_t>::max();
+ }
+ return true;
+ }
+ }
+
+ // Reached the end of the chain.
+ return false;
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+bool FastSeparateChainingHashTable<
+ resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::hasKey(const TypedValue &key) const {
+ DEBUG_ASSERT(this->key_types_.size() == 1);
+ DEBUG_ASSERT(
+ key.isPlausibleInstanceOf(this->key_types_.front()->getSignature()));
+
+ const std::size_t hash_code = key.getHash();
+ std::size_t bucket_ref =
+ slots_[hash_code % header_->num_slots].load(std::memory_order_relaxed);
+ while (bucket_ref != 0) {
+ DEBUG_ASSERT(bucket_ref != std::numeric_limits<std::size_t>::max());
+ const char *bucket =
+ static_cast<const char *>(buckets_) + (bucket_ref - 1) * bucket_size_;
+ const std::size_t bucket_hash = *reinterpret_cast<const std::size_t *>(
+ bucket + sizeof(std::atomic<std::size_t>));
+ if ((bucket_hash == hash_code) &&
+ key_manager_.scalarKeyCollisionCheck(key, bucket)) {
+ // Find a match.
+ return true;
+ }
+ bucket_ref =
+ reinterpret_cast<const std::atomic<std::size_t> *>(bucket)->load(
+ std::memory_order_relaxed);
+ }
+ return false;
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+bool FastSeparateChainingHashTable<
+ resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::hasCompositeKey(const std::vector<TypedValue> &key)
+ const {
+ DEBUG_ASSERT(this->key_types_.size() == key.size());
+
+ const std::size_t hash_code = this->hashCompositeKey(key);
+ std::size_t bucket_ref =
+ slots_[hash_code % header_->num_slots].load(std::memory_order_relaxed);
+ while (bucket_ref != 0) {
+ DEBUG_ASSERT(bucket_ref != std::numeric_limits<std::size_t>::max());
+ const char *bucket =
+ static_cast<const char *>(buckets_) + (bucket_ref - 1) * bucket_size_;
+ const std::size_t bucket_hash = *reinterpret_cast<const std::size_t *>(
+ bucket + sizeof(std::atomic<std::size_t>));
+ if ((bucket_hash == hash_code) &&
+ key_manager_.compositeKeyCollisionCheck(key, bucket)) {
+ // Find a match.
+ return true;
+ }
+ bucket_ref =
+ reinterpret_cast<const std::atomic<std::size_t> *>(bucket)->load(
+ std::memory_order_relaxed);
+ }
+ return false;
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+void FastSeparateChainingHashTable<
+ resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::resize(const std::size_t extra_buckets,
+ const std::size_t extra_variable_storage,
+ const std::size_t retry_num) {
+ DEBUG_ASSERT(resizable);
+
+ // A retry should never be necessary with this implementation of HashTable.
+ // Separate chaining ensures that any resized hash table with more buckets
+ // than the original table will be able to hold more entries than the
+ // original.
+ DEBUG_ASSERT(retry_num == 0);
+
+ SpinSharedMutexExclusiveLock<true> write_lock(this->resize_shared_mutex_);
+
+ // Recheck whether the hash table is still full. Note that multiple threads
+ // might wait to rebuild this hash table simultaneously. Only the first one
+ // should do the rebuild.
+ if (!isFull(extra_variable_storage)) {
+ return;
+ }
+
+ // Approximately double the number of buckets and slots.
+ //
+ // TODO(chasseur): It may be worth it to more than double the number of
+ // buckets here so that we can maintain a good, sparse fill factor for a
+ // longer time as more values are inserted. Such behavior should take into
+ // account kHashTableLoadFactor.
+ std::size_t resized_num_slots = get_next_prime_number(
+ (header_->num_buckets + extra_buckets / 2) * kHashTableLoadFactor * 2);
+ std::size_t variable_storage_required =
+ (resized_num_slots / kHashTableLoadFactor) *
+ key_manager_.getEstimatedVariableKeySize();
+ const std::size_t original_variable_storage_used =
+ header_->variable_length_bytes_allocated.load(std::memory_order_relaxed);
+ // If this resize was triggered by a too-large variable-length key, bump up
+ // the variable-length storage requirement.
+ if ((extra_variable_storage > 0) &&
+ (extra_variable_storage + original_variable_storage_used >
+ key_manager_.getVariableLengthKeyStorageSize())) {
+ variable_storage_required += extra_variable_storage;
+ }
+
+ const std::size_t resized_memory_required =
+ sizeof(Header) + resized_num_slots * sizeof(std::atomic<std::size_t>) +
+ (resized_num_slots / kHashTableLoadFactor) * bucket_size_ +
+ variable_storage_required;
+ const std::size_t resized_storage_slots =
+ this->storage_manager_->SlotsNeededForBytes(resized_memory_required);
+ if (resized_storage_slots == 0) {
+ FATAL_ERROR(
+ "Storage requirement for resized SeparateChainingHashTable "
+ "exceeds maximum allocation size.");
+ }
+
+ // Get a new StorageBlob to hold the resized hash table.
+ const block_id resized_blob_id =
+ this->storage_manager_->createBlob(resized_storage_slots);
+ MutableBlobReference resized_blob =
+ this->storage_manager_->getBlobMutable(resized_blob_id);
+
+ // Locate data structures inside the new StorageBlob.
+ void *aligned_memory_start = resized_blob->getMemoryMutable();
+ std::size_t available_memory = resized_storage_slots * kSlotSizeBytes;
+ if (align(alignof(Header),
+ sizeof(Header),
+ aligned_memory_start,
+ available_memory) == nullptr) {
+ // Should be impossible, as noted in constructor.
+ FATAL_ERROR(
+ "StorageBlob used to hold resized SeparateChainingHashTable "
+ "is too small to meet alignment requirements of "
+ "LinearOpenAddressingHashTable::Header.");
+ } else if (aligned_memory_start != resized_blob->getMemoryMutable()) {
+ // Again, should be impossible.
+ DEV_WARNING("In SeparateChainingHashTable::resize(), StorageBlob "
+ << "memory adjusted by "
+ << (resized_num_slots * kSlotSizeBytes - available_memory)
+ << " bytes to meet alignment requirement for "
+ << "LinearOpenAddressingHashTable::Header.");
+ }
+
+ Header *resized_header = static_cast<Header *>(aligned_memory_start);
+ aligned_memory_start =
+ static_cast<char *>(aligned_memory_start) + sizeof(Header);
+ available_memory -= sizeof(Header);
+
+ // As in constructor, recompute the number of slots and buckets using the
+ // actual available memory.
+ std::size_t resized_num_buckets =
+ (available_memory - extra_variable_storage) /
+ (kHashTableLoadFactor * sizeof(std::atomic<std::size_t>) + bucket_size_ +
+ key_manager_.getEstimatedVariableKeySize());
+ resized_num_slots =
+ get_previous_prime_number(resized_num_buckets * kHashTableLoadFactor);
+ resized_num_buckets = resized_num_slots / kHashTableLoadFactor;
+
+ // Locate slot array.
+ std::atomic<std::size_t> *resized_slots =
+ static_cast<std::atomic<std::size_t> *>(aligned_memory_start);
+ aligned_memory_start = static_cast<char *>(aligned_memory_start) +
+ sizeof(std::atomic<std::size_t>) * resized_num_slots;
+ available_memory -= sizeof(std::atomic<std::size_t>) * resized_num_slots;
+
+ // As in constructor, we will be extra paranoid and use align() to locate the
+ // start of the array of buckets, as well.
+ void *resized_buckets = aligned_memory_start;
+ if (align(
+ kBucketAlignment, bucket_size_, resized_buckets, available_memory) ==
+ nullptr) {
+ FATAL_ERROR(
+ "StorageBlob used to hold resized SeparateChainingHashTable "
+ "is too small to meet alignment requirements of buckets.");
+ } else if (resized_buckets != aligned_memory_start) {
+ DEV_WARNING(
+ "Bucket array start position adjusted to meet alignment "
+ "requirement for SeparateChainingHashTable's value type.");
+ if (resized_num_buckets * bucket_size_ + variable_storage_required >
+ available_memory) {
+ --resized_num_buckets;
+ }
+ }
+ aligned_memory_start = static_cast<char *>(aligned_memory_start) +
+ resized_num_buckets * bucket_size_;
+ available_memory -= resized_num_buckets * bucket_size_;
+
+ void *resized_variable_length_key_storage = aligned_memory_start;
+ const std::size_t resized_variable_length_key_storage_size = available_memory;
+
+ const std::size_t original_buckets_used =
+ header_->buckets_allocated.load(std::memory_order_relaxed);
+
+ // Initialize the header.
+ resized_header->num_slots = resized_num_slots;
+ resized_header->num_buckets = resized_num_buckets;
+ resized_header->buckets_allocated.store(original_buckets_used,
+ std::memory_order_relaxed);
+ resized_header->variable_length_bytes_allocated.store(
+ original_variable_storage_used, std::memory_order_relaxed);
+
+ // Bulk-copy buckets. This is safe because:
+ // 1. The "next" pointers will be adjusted when rebuilding chains below.
+ // 2. The hash codes will stay the same.
+ // 3. For key components:
+ // a. Inline keys will stay exactly the same.
+ // b. Offsets into variable-length storage will remain valid, because
+ // we also do a byte-for-byte copy of variable-length storage below.
+ // c. Absolute external pointers will still point to the same address.
+ // d. Relative pointers are not used with resizable hash tables.
+ // 4. If values are not trivially copyable, then we invoke ValueT's copy
+ // or move constructor with placement new.
+ std::memcpy(resized_buckets, buckets_, original_buckets_used * bucket_size_);
+
+ // TODO(chasseur): std::is_trivially_copyable is not yet implemented in
+ // GCC 4.8.3, so we assume we need to invoke ValueT's copy or move
+ // constructor, even though the plain memcpy above could suffice for many
+ // possible ValueTs.
+ void *current_value_original = static_cast<char *>(buckets_) + kValueOffset;
+ void *current_value_resized =
+ static_cast<char *>(resized_buckets) + kValueOffset;
+ for (std::size_t bucket_num = 0; bucket_num < original_buckets_used;
+ ++bucket_num) {
+ // Use a move constructor if available to avoid a deep-copy, since resizes
+ // always succeed.
+ new (current_value_resized) std::uint8_t(
+ std::move(*static_cast<std::uint8_t *>(current_value_original)));
+ current_value_original =
+ static_cast<char *>(current_value_original) + bucket_size_;
+ current_value_resized =
+ static_cast<char *>(current_value_resized) + bucket_size_;
+ }
+
+ // Copy over variable-length key components, if any.
+ if (original_variable_storage_used > 0) {
+ DEBUG_ASSERT(original_variable_storage_used ==
+ key_manager_.getNextVariableLengthKeyOffset());
+ DEBUG_ASSERT(original_variable_storage_used <=
+ resized_variable_length_key_storage_size);
+ std::memcpy(resized_variable_length_key_storage,
+ key_manager_.getVariableLengthKeyStorage(),
+ original_variable_storage_used);
+ }
+
+ // Destroy values in the original hash table, if neccesary,
+ DestroyValues(buckets_, original_buckets_used, bucket_size_);
+
+ // Make resized structures active.
+ std::swap(this->blob_, resized_blob);
+ header_ = resized_header;
+ slots_ = resized_slots;
+ buckets_ = resized_buckets;
+ key_manager_.setVariableLengthStorageInfo(
+ resized_variable_length_key_storage,
+ resized_variable_length_key_storage_size,
+ &(resized_header->variable_length_bytes_allocated));
+
+ // Drop the old blob.
+ const block_id old_blob_id = resized_blob->getID();
+ resized_blob.release();
+ this->storage_manager_->deleteBlockOrBlobFile(old_blob_id);
+
+ // Rebuild chains.
+ void *current_bucket = buckets_;
+ for (std::size_t bucket_num = 0; bucket_num < original_buckets_used;
+ ++bucket_num) {
+ std::atomic<std::size_t> *next_ptr =
+ static_cast<std::atomic<std::size_t> *>(current_bucket);
+ const std::size_t hash_code = *reinterpret_cast<const std::size_t *>(
+ static_cast<const char *>(current_bucket) +
+ sizeof(std::atomic<std::size_t>));
+
+ const std::size_t slot_number = hash_code % header_->num_slots;
+ std::size_t slot_ptr_value = 0;
+ if (slots_[slot_number].compare_exchange_strong(
+ slot_ptr_value, bucket_num + 1, std::memory_order_relaxed)) {
+ // This bucket is the first in the chain for this block, so reset its
+ // next pointer to 0.
+ next_ptr->store(0, std::memory_order_relaxed);
+ } else {
+ // A chain already exists starting from this slot, so put this bucket at
+ // the head.
+ next_ptr->store(slot_ptr_value, std::memory_order_relaxed);
+ slots_[slot_number].store(bucket_num + 1, std::memory_order_relaxed);
+ }
+ current_bucket = static_cast<char *>(current_bucket) + bucket_size_;
+ }
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+bool FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ preallocateForBulkInsert(const std::size_t total_entries,
+ const std::size_t total_variable_key_size,
+ HashTablePreallocationState *prealloc_state) {
+ DEBUG_ASSERT(allow_duplicate_keys);
+ if (!key_manager_.allocateVariableLengthKeyStorage(total_variable_key_size)) {
+ return false;
+ }
+
+ // We use load then compare-exchange here instead of simply fetch-add,
+ // because if multiple threads are simultaneously trying to allocate more
+ // than one bucket and exceed 'header_->num_buckets', their respective
+ // rollbacks might happen in such an order that some bucket ranges get
+ // skipped, while others might get double-allocated later.
+ std::size_t original_buckets_allocated =
+ header_->buckets_allocated.load(std::memory_order_relaxed);
+ std::size_t buckets_post_allocation =
+ original_buckets_allocated + total_entries;
+ while ((buckets_post_allocation <= header_->num_buckets) &&
+ !header_->buckets_allocated.compare_exchange_weak(
+ original_buckets_allocated,
+ buckets_post_allocation,
+ std::memory_order_relaxed)) {
+ buckets_post_allocation = original_buckets_allocated + total_entries;
+ }
+
+ if (buckets_post_allocation > header_->num_buckets) {
+ key_manager_.deallocateVariableLengthKeyStorage(total_variable_key_size);
+ return false;
+ }
+
+ prealloc_state->bucket_position = original_buckets_allocated;
+ if (total_variable_key_size != 0) {
+ prealloc_state->variable_length_key_position =
+ key_manager_.incrementNextVariableLengthKeyOffset(
+ total_variable_key_size);
+ }
+ return true;
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+void FastSeparateChainingHashTable<
+ resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::DestroyValues(void *hash_buckets,
+ const std::size_t num_buckets,
+ const std::size_t bucket_size) {
+ if (!std::is_trivially_destructible<std::uint8_t>::value) {
+ void *value_ptr = static_cast<char *>(hash_buckets) + kValueOffset;
+ for (std::size_t bucket_num = 0; bucket_num < num_buckets; ++bucket_num) {
+ static_cast<std::uint8_t *>(value_ptr)->~uint8_t();
+ value_ptr = static_cast<char *>(value_ptr) + bucket_size;
+ }
+ }
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+inline bool FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ locateBucketForInsertion(const std::size_t hash_code,
+ const std::size_t variable_key_allocation_required,
+ void **bucket,
+ std::atomic<std::size_t> **pending_chain_ptr,
+ std::size_t *pending_chain_ptr_finish_value,
+ HashTablePreallocationState *prealloc_state) {
+ DEBUG_ASSERT((prealloc_state == nullptr) || allow_duplicate_keys);
+ if (*bucket == nullptr) {
+ *pending_chain_ptr = &(slots_[hash_code % header_->num_slots]);
+ } else {
+ *pending_chain_ptr = static_cast<std::atomic<std::size_t> *>(*bucket);
+ }
+ for (;;) {
+ std::size_t existing_chain_ptr = 0;
+ if ((*pending_chain_ptr)
+ ->compare_exchange_strong(existing_chain_ptr,
+ std::numeric_limits<std::size_t>::max(),
+ std::memory_order_acq_rel)) {
+ // Got to the end of the chain. Allocate a new bucket.
+
+ // First, allocate variable-length key storage, if needed (i.e. if this
+ // is an upsert and we didn't allocate up-front).
+ if ((prealloc_state == nullptr) &&
+ !key_manager_.allocateVariableLengthKeyStorage(
+ variable_key_allocation_required)) {
+ // Ran out of variable-length storage.
+ (*pending_chain_ptr)->store(0, std::memory_order_release);
+ *bucket = nullptr;
+ return false;
+ }
+
+ const std::size_t allocated_bucket_num =
+ (prealloc_state == nullptr)
+ ? header_->buckets_allocated.fetch_add(1,
+ std::memory_order_relaxed)
+ : (prealloc_state->bucket_position)++;
+ if (allocated_bucket_num >= header_->num_buckets) {
+ // Ran out of buckets.
+ DEBUG_ASSERT(prealloc_state == nullptr);
+ header_->buckets_allocated.fetch_sub(1, std::memory_order_relaxed);
+ (*pending_chain_ptr)->store(0, std::memory_order_release);
+ *bucket = nullptr;
+ return false;
+ } else {
+ *bucket =
+ static_cast<char *>(buckets_) + allocated_bucket_num * bucket_size_;
+ *pending_chain_ptr_finish_value = allocated_bucket_num + 1;
+ return true;
+ }
+ }
+ // Spin until the real "next" pointer is available.
+ while (existing_chain_ptr == std::numeric_limits<std::size_t>::max()) {
+ existing_chain_ptr =
+ (*pending_chain_ptr)->load(std::memory_order_acquire);
+ }
+ if (existing_chain_ptr == 0) {
+ // Other thread had to roll back, so try again.
+ continue;
+ }
+ // Chase the next pointer.
+ *bucket =
+ static_cast<char *>(buckets_) + (existing_chain_ptr - 1) * bucket_size_;
+ *pending_chain_ptr = static_cast<std::atomic<std::size_t> *>(*bucket);
+ if (!allow_duplicate_keys) {
+ const std::size_t hash_in_bucket = *reinterpret_cast<const std::size_t *>(
+ static_cast<const char *>(*bucket) +
+ sizeof(std::atomic<std::size_t>));
+ if (hash_in_bucket == hash_code) {
+ return false;
+ }
+ }
+ }
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+inline void FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ writeScalarKeyToBucket(const TypedValue &key,
+ const std::size_t hash_code,
+ void *bucket,
+ HashTablePreallocationState *prealloc_state) {
+ *reinterpret_cast<std::size_t *>(static_cast<char *>(bucket) +
+ sizeof(std::atomic<std::size_t>)) =
+ hash_code;
+ key_manager_.writeKeyComponentToBucket(key, 0, bucket, prealloc_state);
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+inline void FastSeparateChainingHashTable<resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::
+ writeCompositeKeyToBucket(const std::vector<TypedValue> &key,
+ const std::size_t hash_code,
+ void *bucket,
+ HashTablePreallocationState *prealloc_state) {
+ DEBUG_ASSERT(key.size() == this->key_types_.size());
+ *reinterpret_cast<std::size_t *>(static_cast<char *>(bucket) +
+ sizeof(std::atomic<std::size_t>)) =
+ hash_code;
+ for (std::size_t idx = 0; idx < this->key_types_.size(); ++idx) {
+ key_manager_.writeKeyComponentToBucket(
+ key[idx], idx, bucket, prealloc_state);
+ }
+}
+
+template <bool resizable,
+ bool serializable,
+ bool force_key_copy,
+ bool allow_duplicate_keys>
+bool FastSeparateChainingHashTable<
+ resizable,
+ serializable,
+ force_key_copy,
+ allow_duplicate_keys>::isFull(const std::size_t extra_variable_storage)
+ const {
+ if (header_->buckets_allocated.load(std::memory_order_relaxed) >=
+ header_->num_buckets) {
+ // All buckets are allocated.
+ return true;
+ }
+
+ if (extra_variable_storage > 0) {
+ if (extra_variable_storage +
+ header_->variable_length_bytes_allocated.load(
+ std::memory_order_relaxed) >
+ key_manager_.getVariableLengthKeyStorageSize()) {
+ // Not enough variable-length key storage space.
+ return true;
+ }
+ }
+
+ return false;
+}
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_STORAGE_SEPARATE_CHAINING_HASH_TABLE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/storage/HashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTable.hpp b/storage/HashTable.hpp
index 9fa41a2..f2dcb03 100644
--- a/storage/HashTable.hpp
+++ b/storage/HashTable.hpp
@@ -49,16 +49,6 @@ namespace quickstep {
*/
/**
- * @brief Codes which indicate the result of a call to put() or
- * putCompositeKey().
- **/
-enum class HashTablePutResult {
- kOK = 0,
- kDuplicateKey,
- kOutOfSpace
-};
-
-/**
* @brief Base class for hash table.
*
* This class is templated so that the core hash-table logic can be reused in
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/storage/HashTableBase.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTableBase.hpp b/storage/HashTableBase.hpp
index f1594e3..cd0a141 100644
--- a/storage/HashTableBase.hpp
+++ b/storage/HashTableBase.hpp
@@ -21,7 +21,9 @@
#define QUICKSTEP_STORAGE_HASH_TABLE_BASE_HPP_
#include <cstddef>
+#include <vector>
+#include "ValueAccessor.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
@@ -52,6 +54,12 @@ struct HashTablePreallocationState {
};
/**
+ * @brief Codes which indicate the result of a call to put() or
+ * putCompositeKey().
+ **/
+enum class HashTablePutResult { kOK = 0, kDuplicateKey, kOutOfSpace };
+
+/**
* @brief An ultra-minimal base class that HashTables with different ValueT
* parameters inherit from. This allows for a bit more type-safety than
* just passing around void* pointers (although casting will still be
@@ -64,12 +72,35 @@ template <bool resizable,
bool allow_duplicate_keys>
class HashTableBase {
public:
- virtual ~HashTableBase() {
+ virtual ~HashTableBase() {}
+
+ /**
+ * TODO(harshad) We should get rid of this function from here. We are
+ * postponing it because of the amount of work to be done is significant.
+ * The steps are as follows:
+ * 1. Replace AggregationStateHashTableBase occurence in HashTablePool to
+ * the FastHashTable implementation (i.e. an implementation specialized for
+ * aggregation).
+ * 2. Remove createGroupByHashTable from the AggregationHandle* classes.
+ * 3. Replace AggregationStateHashTableBase occurences in AggregationHandle*
+ * clases to the FastHashTable implementation (i.e. an implementation
+ * specialized for aggregation).
+ * 4. Move this method to the FastHashTable class from here, so that it can
+ * be called from the AggregationHandle* classes.
+ *
+ * Optionally, we can also remove the AggregationStateHashTableBase
+ * specialization from this file.
+ **/
+ virtual bool upsertValueAccessorCompositeKeyFast(
+ const std::vector<attribute_id> &argument,
+ ValueAccessor *accessor,
+ const std::vector<attribute_id> &key_attr_ids,
+ const bool check_for_null_keys) {
+ return false;
}
protected:
- HashTableBase() {
- }
+ HashTableBase() {}
private:
DISALLOW_COPY_AND_ASSIGN(HashTableBase);
[06/17] incubator-quickstep git commit: Initial commit for
QUICKSTEP-28 and QUICKSTEP-29.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp b/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
index 85c3bf3..6e6d188 100644
--- a/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
@@ -31,6 +31,8 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleMin.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/AggregationOperationState.hpp"
+#include "storage/FastHashTableFactory.hpp"
#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DatetimeIntervalType.hpp"
@@ -69,54 +71,59 @@ class AggregationHandleMinTest : public ::testing::Test {
// Helper method that calls AggregationHandleMin::iterateUnaryInl() to
// aggregate 'value' into '*state'.
void iterateHandle(AggregationState *state, const TypedValue &value) {
- static_cast<const AggregationHandleMin&>(*aggregation_handle_min_).iterateUnaryInl(
- static_cast<AggregationStateMin*>(state),
- value);
+ static_cast<const AggregationHandleMin &>(*aggregation_handle_min_)
+ .iterateUnaryInl(static_cast<AggregationStateMin *>(state), value);
}
void initializeHandle(const Type &type) {
aggregation_handle_min_.reset(
- AggregateFunctionFactory::Get(AggregationID::kMin).createHandle(
- std::vector<const Type*>(1, &type)));
+ AggregateFunctionFactory::Get(AggregationID::kMin)
+ .createHandle(std::vector<const Type *>(1, &type)));
aggregation_handle_min_state_.reset(
aggregation_handle_min_->createInitialState());
}
static bool ApplyToTypesTest(TypeID typeID) {
- const Type &type = (typeID == kChar || typeID == kVarChar) ?
- TypeFactory::GetType(typeID, static_cast<std::size_t>(10)) :
- TypeFactory::GetType(typeID);
+ const Type &type =
+ (typeID == kChar || typeID == kVarChar)
+ ? TypeFactory::GetType(typeID, static_cast<std::size_t>(10))
+ : TypeFactory::GetType(typeID);
- return AggregateFunctionFactory::Get(AggregationID::kMin).canApplyToTypes(
- std::vector<const Type*>(1, &type));
+ return AggregateFunctionFactory::Get(AggregationID::kMin)
+ .canApplyToTypes(std::vector<const Type *>(1, &type));
}
static bool ResultTypeForArgumentTypeTest(TypeID input_type_id,
TypeID output_type_id) {
- const Type *result_type
- = AggregateFunctionFactory::Get(AggregationID::kMin).resultTypeForArgumentTypes(
- std::vector<const Type*>(1, &TypeFactory::GetType(input_type_id)));
+ const Type *result_type =
+ AggregateFunctionFactory::Get(AggregationID::kMin)
+ .resultTypeForArgumentTypes(std::vector<const Type *>(
+ 1, &TypeFactory::GetType(input_type_id)));
return (result_type->getTypeID() == output_type_id);
}
template <typename CppType>
- static void CheckMinValue(
- CppType expected,
- const AggregationHandle &handle,
- const AggregationState &state) {
+ static void CheckMinValue(CppType expected,
+ const AggregationHandle &handle,
+ const AggregationState &state) {
EXPECT_EQ(expected, handle.finalize(state).getLiteral<CppType>());
}
- static void CheckMinString(
- const std::string &expected,
- const AggregationHandle &handle,
- const AggregationState &state) {
+ template <typename CppType>
+ static void CheckMinValue(CppType expected, const TypedValue &value) {
+ EXPECT_EQ(expected, value.getLiteral<CppType>());
+ }
+
+ static void CheckMinString(const std::string &expected,
+ const AggregationHandle &handle,
+ const AggregationState &state) {
TypedValue value = handle.finalize(state);
ASSERT_EQ(expected.length(), value.getAsciiStringLength());
- EXPECT_EQ(0, std::strncmp(expected.c_str(),
- static_cast <const char *>(value.getDataPtr()),
- value.getAsciiStringLength()));
+ EXPECT_EQ(0,
+ std::strncmp(expected.c_str(),
+ static_cast<const char *>(value.getDataPtr()),
+ value.getAsciiStringLength()));
}
// Static templated method to initialize data types.
@@ -129,7 +136,9 @@ class AggregationHandleMinTest : public ::testing::Test {
void checkAggregationMinGeneric() {
const GenericType &type = GenericType::Instance(true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_min_->finalize(*aggregation_handle_min_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_min_->finalize(*aggregation_handle_min_state_)
+ .isNull());
typename GenericType::cpptype val;
typename GenericType::cpptype min;
@@ -141,16 +150,18 @@ class AggregationHandleMinTest : public ::testing::Test {
if (type.getTypeID() == kInt || type.getTypeID() == kLong) {
SetDataType(i * kNumSamples + j - 10, &val);
} else {
- SetDataType(static_cast<float>(i * kNumSamples + j - 10)/10, &val);
+ SetDataType(static_cast<float>(i * kNumSamples + j - 10) / 10, &val);
}
- iterateHandle(aggregation_handle_min_state_.get(), type.makeValue(&val));
+ iterateHandle(aggregation_handle_min_state_.get(),
+ type.makeValue(&val));
if (min > val) {
min = val;
}
}
}
iterateHandle(aggregation_handle_min_state_.get(), type.makeNullValue());
- CheckMinValue<typename GenericType::cpptype>(min, *aggregation_handle_min_, *aggregation_handle_min_state_);
+ CheckMinValue<typename GenericType::cpptype>(
+ min, *aggregation_handle_min_, *aggregation_handle_min_state_);
// Test mergeStates().
std::unique_ptr<AggregationState> merge_state(
@@ -164,7 +175,7 @@ class AggregationHandleMinTest : public ::testing::Test {
if (type.getTypeID() == kInt || type.getTypeID() == kLong) {
SetDataType(i * kNumSamples + j - 20, &val);
} else {
- SetDataType(static_cast<float>(i * kNumSamples + j - 20)/10, &val);
+ SetDataType(static_cast<float>(i * kNumSamples + j - 20) / 10, &val);
}
iterateHandle(merge_state.get(), type.makeValue(&val));
if (min > val) {
@@ -175,14 +186,14 @@ class AggregationHandleMinTest : public ::testing::Test {
aggregation_handle_min_->mergeStates(*merge_state,
aggregation_handle_min_state_.get());
CheckMinValue<typename GenericType::cpptype>(
- min,
- *aggregation_handle_min_,
- *aggregation_handle_min_state_);
+ min, *aggregation_handle_min_, *aggregation_handle_min_state_);
}
template <typename GenericType>
- ColumnVector *createColumnVectorGeneric(const Type &type, typename GenericType::cpptype *min) {
- NativeColumnVector *column = new NativeColumnVector(type, kIterations * kNumSamples + 3);
+ ColumnVector* createColumnVectorGeneric(const Type &type,
+ typename GenericType::cpptype *min) {
+ NativeColumnVector *column =
+ new NativeColumnVector(type, kIterations * kNumSamples + 3);
typename GenericType::cpptype val;
SetDataType(1000, min);
@@ -193,7 +204,7 @@ class AggregationHandleMinTest : public ::testing::Test {
if (type.getTypeID() == kInt || type.getTypeID() == kLong) {
SetDataType(i * kNumSamples + j - 10, &val);
} else {
- SetDataType(static_cast<float>(i * kNumSamples + j - 10)/10, &val);
+ SetDataType(static_cast<float>(i * kNumSamples + j - 10) / 10, &val);
}
column->appendTypedValue(type.makeValue(&val));
if (*min > val) {
@@ -201,7 +212,7 @@ class AggregationHandleMinTest : public ::testing::Test {
}
}
// One NULL in the middle.
- if (i == kIterations/2) {
+ if (i == kIterations / 2) {
column->appendTypedValue(type.makeNullValue());
}
}
@@ -214,11 +225,14 @@ class AggregationHandleMinTest : public ::testing::Test {
void checkAggregationMinGenericColumnVector() {
const GenericType &type = GenericType::Instance(true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_min_->finalize(*aggregation_handle_min_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_min_->finalize(*aggregation_handle_min_state_)
+ .isNull());
typename GenericType::cpptype min;
std::vector<std::unique_ptr<ColumnVector>> column_vectors;
- column_vectors.emplace_back(createColumnVectorGeneric<GenericType>(type, &min));
+ column_vectors.emplace_back(
+ createColumnVectorGeneric<GenericType>(type, &min));
std::unique_ptr<AggregationState> cv_state(
aggregation_handle_min_->accumulateColumnVectors(column_vectors));
@@ -226,15 +240,12 @@ class AggregationHandleMinTest : public ::testing::Test {
// Test the state generated directly by accumulateColumnVectors(), and also
// test after merging back.
CheckMinValue<typename GenericType::cpptype>(
- min,
- *aggregation_handle_min_,
- *cv_state);
+ min, *aggregation_handle_min_, *cv_state);
- aggregation_handle_min_->mergeStates(*cv_state, aggregation_handle_min_state_.get());
+ aggregation_handle_min_->mergeStates(*cv_state,
+ aggregation_handle_min_state_.get());
CheckMinValue<typename GenericType::cpptype>(
- min,
- *aggregation_handle_min_,
- *aggregation_handle_min_state_);
+ min, *aggregation_handle_min_, *aggregation_handle_min_state_);
}
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -242,29 +253,29 @@ class AggregationHandleMinTest : public ::testing::Test {
void checkAggregationMinGenericValueAccessor() {
const GenericType &type = GenericType::Instance(true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_min_->finalize(*aggregation_handle_min_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_min_->finalize(*aggregation_handle_min_state_)
+ .isNull());
- std::unique_ptr<ColumnVectorsValueAccessor> accessor(new ColumnVectorsValueAccessor());
+ std::unique_ptr<ColumnVectorsValueAccessor> accessor(
+ new ColumnVectorsValueAccessor());
typename GenericType::cpptype min;
accessor->addColumn(createColumnVectorGeneric<GenericType>(type, &min));
std::unique_ptr<AggregationState> va_state(
- aggregation_handle_min_->accumulateValueAccessor(accessor.get(),
- std::vector<attribute_id>(1, 0)));
+ aggregation_handle_min_->accumulateValueAccessor(
+ accessor.get(), std::vector<attribute_id>(1, 0)));
// Test the state generated directly by accumulateValueAccessor(), and also
// test after merging back.
CheckMinValue<typename GenericType::cpptype>(
- min,
- *aggregation_handle_min_,
- *va_state);
+ min, *aggregation_handle_min_, *va_state);
- aggregation_handle_min_->mergeStates(*va_state, aggregation_handle_min_state_.get());
+ aggregation_handle_min_->mergeStates(*va_state,
+ aggregation_handle_min_state_.get());
CheckMinValue<typename GenericType::cpptype>(
- min,
- *aggregation_handle_min_,
- *aggregation_handle_min_state_);
+ min, *aggregation_handle_min_, *aggregation_handle_min_state_);
}
#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -272,11 +283,13 @@ class AggregationHandleMinTest : public ::testing::Test {
void checkAggregationMinString() {
const StringType &type = StringType::Instance(10, true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_min_->finalize(*aggregation_handle_min_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_min_->finalize(*aggregation_handle_min_state_)
+ .isNull());
std::unique_ptr<UncheckedComparator> fast_comparator_;
fast_comparator_.reset(ComparisonFactory::GetComparison(ComparisonID::kLess)
- .makeUncheckedComparatorForTypes(type, type));
+ .makeUncheckedComparatorForTypes(type, type));
std::string string_literal;
std::string min = "z";
int val;
@@ -288,15 +301,19 @@ class AggregationHandleMinTest : public ::testing::Test {
oss << "test" << val;
string_literal = oss.str();
- iterateHandle(aggregation_handle_min_state_.get(), type.makeValue(string_literal.c_str(),
- string_literal.length() + 1).ensureNotReference());
- if (fast_comparator_->compareDataPtrs(string_literal.c_str(), min.c_str())) {
+ iterateHandle(
+ aggregation_handle_min_state_.get(),
+ type.makeValue(string_literal.c_str(), string_literal.length() + 1)
+ .ensureNotReference());
+ if (fast_comparator_->compareDataPtrs(string_literal.c_str(),
+ min.c_str())) {
min = string_literal;
}
}
}
iterateHandle(aggregation_handle_min_state_.get(), type.makeNullValue());
- CheckMinString(min, *aggregation_handle_min_, *aggregation_handle_min_state_);
+ CheckMinString(
+ min, *aggregation_handle_min_, *aggregation_handle_min_state_);
// Test mergeStates().
std::unique_ptr<AggregationState> merge_state(
@@ -314,24 +331,27 @@ class AggregationHandleMinTest : public ::testing::Test {
iterateHandle(
merge_state.get(),
- type.makeValue(string_literal.c_str(),
- string_literal.length() + 1).ensureNotReference());
- if (fast_comparator_->compareDataPtrs(string_literal.c_str(), min.c_str())) {
+ type.makeValue(string_literal.c_str(), string_literal.length() + 1)
+ .ensureNotReference());
+ if (fast_comparator_->compareDataPtrs(string_literal.c_str(),
+ min.c_str())) {
min = string_literal;
}
}
}
aggregation_handle_min_->mergeStates(*merge_state,
aggregation_handle_min_state_.get());
- CheckMinString(min, *aggregation_handle_min_, *aggregation_handle_min_state_);
+ CheckMinString(
+ min, *aggregation_handle_min_, *aggregation_handle_min_state_);
}
template <typename ColumnVectorType>
- ColumnVector *createColumnVectorString(const Type &type, std::string *min) {
- ColumnVectorType *column = new ColumnVectorType(type, kIterations * kNumSamples + 3);
+ ColumnVector* createColumnVectorString(const Type &type, std::string *min) {
+ ColumnVectorType *column =
+ new ColumnVectorType(type, kIterations * kNumSamples + 3);
std::unique_ptr<UncheckedComparator> fast_comparator_;
fast_comparator_.reset(ComparisonFactory::GetComparison(ComparisonID::kLess)
- .makeUncheckedComparatorForTypes(type, type));
+ .makeUncheckedComparatorForTypes(type, type));
std::string string_literal;
*min = "z";
int val;
@@ -343,14 +363,16 @@ class AggregationHandleMinTest : public ::testing::Test {
oss << "test" << val;
string_literal = oss.str();
- column->appendTypedValue(type.makeValue(string_literal.c_str(), string_literal.length() + 1)
- .ensureNotReference());
- if (fast_comparator_->compareDataPtrs(string_literal.c_str(), min->c_str())) {
+ column->appendTypedValue(
+ type.makeValue(string_literal.c_str(), string_literal.length() + 1)
+ .ensureNotReference());
+ if (fast_comparator_->compareDataPtrs(string_literal.c_str(),
+ min->c_str())) {
*min = string_literal;
}
}
// One NULL in the middle.
- if (i == kIterations/2) {
+ if (i == kIterations / 2) {
column->appendTypedValue(type.makeNullValue());
}
}
@@ -363,25 +385,26 @@ class AggregationHandleMinTest : public ::testing::Test {
void checkAggregationMinStringColumnVector() {
const StringType &type = StringType::Instance(10, true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_min_->finalize(*aggregation_handle_min_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_min_->finalize(*aggregation_handle_min_state_)
+ .isNull());
std::string min;
std::vector<std::unique_ptr<ColumnVector>> column_vectors;
- column_vectors.emplace_back(createColumnVectorString<ColumnVectorType>(type, &min));
+ column_vectors.emplace_back(
+ createColumnVectorString<ColumnVectorType>(type, &min));
std::unique_ptr<AggregationState> cv_state(
aggregation_handle_min_->accumulateColumnVectors(column_vectors));
// Test the state generated directly by accumulateColumnVectors(), and also
// test after merging back.
- CheckMinString(min,
- *aggregation_handle_min_,
- *cv_state);
-
- aggregation_handle_min_->mergeStates(*cv_state, aggregation_handle_min_state_.get());
- CheckMinString(min,
- *aggregation_handle_min_,
- *aggregation_handle_min_state_);
+ CheckMinString(min, *aggregation_handle_min_, *cv_state);
+
+ aggregation_handle_min_->mergeStates(*cv_state,
+ aggregation_handle_min_state_.get());
+ CheckMinString(
+ min, *aggregation_handle_min_, *aggregation_handle_min_state_);
}
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -389,26 +412,27 @@ class AggregationHandleMinTest : public ::testing::Test {
void checkAggregationMinStringValueAccessor() {
const StringType &type = StringType::Instance(10, true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_min_->finalize(*aggregation_handle_min_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_min_->finalize(*aggregation_handle_min_state_)
+ .isNull());
std::string min;
- std::unique_ptr<ColumnVectorsValueAccessor> accessor(new ColumnVectorsValueAccessor());
+ std::unique_ptr<ColumnVectorsValueAccessor> accessor(
+ new ColumnVectorsValueAccessor());
accessor->addColumn(createColumnVectorString<ColumnVectorType>(type, &min));
std::unique_ptr<AggregationState> va_state(
- aggregation_handle_min_->accumulateValueAccessor(accessor.get(),
- std::vector<attribute_id>(1, 0)));
+ aggregation_handle_min_->accumulateValueAccessor(
+ accessor.get(), std::vector<attribute_id>(1, 0)));
// Test the state generated directly by accumulateValueAccessor(), and also
// test after merging back.
- CheckMinString(min,
- *aggregation_handle_min_,
- *va_state);
-
- aggregation_handle_min_->mergeStates(*va_state, aggregation_handle_min_state_.get());
- CheckMinString(min,
- *aggregation_handle_min_,
- *aggregation_handle_min_state_);
+ CheckMinString(min, *aggregation_handle_min_, *va_state);
+
+ aggregation_handle_min_->mergeStates(*va_state,
+ aggregation_handle_min_state_.get());
+ CheckMinString(
+ min, *aggregation_handle_min_, *aggregation_handle_min_state_);
}
#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -419,9 +443,7 @@ class AggregationHandleMinTest : public ::testing::Test {
template <>
void AggregationHandleMinTest::CheckMinValue<float>(
- float val,
- const AggregationHandle &handle,
- const AggregationState &state) {
+ float val, const AggregationHandle &handle, const AggregationState &state) {
EXPECT_FLOAT_EQ(val, handle.finalize(state).getLiteral<float>());
}
@@ -434,17 +456,20 @@ void AggregationHandleMinTest::CheckMinValue<double>(
}
template <>
-void AggregationHandleMinTest::SetDataType<DatetimeLit>(int value, DatetimeLit *data) {
+void AggregationHandleMinTest::SetDataType<DatetimeLit>(int value,
+ DatetimeLit *data) {
data->ticks = value;
}
template <>
-void AggregationHandleMinTest::SetDataType<DatetimeIntervalLit>(int value, DatetimeIntervalLit *data) {
+void AggregationHandleMinTest::SetDataType<DatetimeIntervalLit>(
+ int value, DatetimeIntervalLit *data) {
data->interval_ticks = value;
}
template <>
-void AggregationHandleMinTest::SetDataType<YearMonthIntervalLit>(int value, YearMonthIntervalLit *data) {
+void AggregationHandleMinTest::SetDataType<YearMonthIntervalLit>(
+ int value, YearMonthIntervalLit *data) {
data->months = value;
}
@@ -575,50 +600,67 @@ TEST_F(AggregationHandleMinDeathTest, WrongTypeTest) {
double double_val = 0;
float float_val = 0;
- iterateHandle(aggregation_handle_min_state_.get(), int_non_null_type.makeValue(&int_val));
+ iterateHandle(aggregation_handle_min_state_.get(),
+ int_non_null_type.makeValue(&int_val));
- EXPECT_DEATH(iterateHandle(aggregation_handle_min_state_.get(), long_type.makeValue(&long_val)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_min_state_.get(), double_type.makeValue(&double_val)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_min_state_.get(), float_type.makeValue(&float_val)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_min_state_.get(), char_type.makeValue("asdf", 5)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_min_state_.get(), varchar_type.makeValue("asdf", 5)), "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_min_state_.get(),
+ long_type.makeValue(&long_val)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_min_state_.get(),
+ double_type.makeValue(&double_val)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_min_state_.get(),
+ float_type.makeValue(&float_val)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_min_state_.get(),
+ char_type.makeValue("asdf", 5)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_min_state_.get(),
+ varchar_type.makeValue("asdf", 5)),
+ "");
// Test mergeStates() with incorrectly typed handles.
std::unique_ptr<AggregationHandle> aggregation_handle_min_long(
- AggregateFunctionFactory::Get(AggregationID::kMin).createHandle(
- std::vector<const Type*>(1, &long_type)));
+ AggregateFunctionFactory::Get(AggregationID::kMin)
+ .createHandle(std::vector<const Type *>(1, &long_type)));
std::unique_ptr<AggregationState> aggregation_state_min_merge_long(
aggregation_handle_min_long->createInitialState());
- static_cast<const AggregationHandleMin&>(*aggregation_handle_min_long).iterateUnaryInl(
- static_cast<AggregationStateMin*>(aggregation_state_min_merge_long.get()),
- long_type.makeValue(&long_val));
- EXPECT_DEATH(aggregation_handle_min_->mergeStates(*aggregation_state_min_merge_long,
- aggregation_handle_min_state_.get()),
- "");
+ static_cast<const AggregationHandleMin &>(*aggregation_handle_min_long)
+ .iterateUnaryInl(static_cast<AggregationStateMin *>(
+ aggregation_state_min_merge_long.get()),
+ long_type.makeValue(&long_val));
+ EXPECT_DEATH(
+ aggregation_handle_min_->mergeStates(*aggregation_state_min_merge_long,
+ aggregation_handle_min_state_.get()),
+ "");
std::unique_ptr<AggregationHandle> aggregation_handle_min_double(
- AggregateFunctionFactory::Get(AggregationID::kMin).createHandle(
- std::vector<const Type*>(1, &double_type)));
+ AggregateFunctionFactory::Get(AggregationID::kMin)
+ .createHandle(std::vector<const Type *>(1, &double_type)));
std::unique_ptr<AggregationState> aggregation_state_min_merge_double(
aggregation_handle_min_double->createInitialState());
- static_cast<const AggregationHandleMin&>(*aggregation_handle_min_double).iterateUnaryInl(
- static_cast<AggregationStateMin*>(aggregation_state_min_merge_double.get()),
- double_type.makeValue(&double_val));
- EXPECT_DEATH(aggregation_handle_min_->mergeStates(*aggregation_state_min_merge_double,
- aggregation_handle_min_state_.get()),
- "");
+ static_cast<const AggregationHandleMin &>(*aggregation_handle_min_double)
+ .iterateUnaryInl(static_cast<AggregationStateMin *>(
+ aggregation_state_min_merge_double.get()),
+ double_type.makeValue(&double_val));
+ EXPECT_DEATH(
+ aggregation_handle_min_->mergeStates(*aggregation_state_min_merge_double,
+ aggregation_handle_min_state_.get()),
+ "");
std::unique_ptr<AggregationHandle> aggregation_handle_min_float(
- AggregateFunctionFactory::Get(AggregationID::kMin).createHandle(
- std::vector<const Type*>(1, &float_type)));
+ AggregateFunctionFactory::Get(AggregationID::kMin)
+ .createHandle(std::vector<const Type *>(1, &float_type)));
std::unique_ptr<AggregationState> aggregation_state_min_merge_float(
aggregation_handle_min_float->createInitialState());
- static_cast<const AggregationHandleMin&>(*aggregation_handle_min_float).iterateUnaryInl(
- static_cast<AggregationStateMin*>(aggregation_state_min_merge_float.get()),
- float_type.makeValue(&float_val));
- EXPECT_DEATH(aggregation_handle_min_->mergeStates(*aggregation_state_min_merge_float,
- aggregation_handle_min_state_.get()),
- "");
+ static_cast<const AggregationHandleMin &>(*aggregation_handle_min_float)
+ .iterateUnaryInl(static_cast<AggregationStateMin *>(
+ aggregation_state_min_merge_float.get()),
+ float_type.makeValue(&float_val));
+ EXPECT_DEATH(
+ aggregation_handle_min_->mergeStates(*aggregation_state_min_merge_float,
+ aggregation_handle_min_state_.get()),
+ "");
}
#endif
@@ -643,25 +685,28 @@ TEST_F(AggregationHandleMinTest, GroupByTableMergeTest) {
initializeHandle(int_non_null_type);
storage_manager_.reset(new StorageManager("./test_min_data"));
std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
- aggregation_handle_min_->createGroupByHashTable(
- HashTableImplType::kSimpleScalarSeparateChaining,
+ AggregationStateFastHashTableFactory::CreateResizable(
+ HashTableImplType::kSeparateChaining,
std::vector<const Type *>(1, &int_non_null_type),
10,
+ {aggregation_handle_min_.get()->getPayloadSize()},
+ {aggregation_handle_min_.get()},
storage_manager_.get()));
std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
- aggregation_handle_min_->createGroupByHashTable(
- HashTableImplType::kSimpleScalarSeparateChaining,
+ AggregationStateFastHashTableFactory::CreateResizable(
+ HashTableImplType::kSeparateChaining,
std::vector<const Type *>(1, &int_non_null_type),
10,
+ {aggregation_handle_min_.get()->getPayloadSize()},
+ {aggregation_handle_min_.get()},
storage_manager_.get()));
- AggregationStateHashTable<AggregationStateMin> *destination_hash_table_derived =
- static_cast<AggregationStateHashTable<AggregationStateMin> *>(
+ AggregationStateFastHashTable *destination_hash_table_derived =
+ static_cast<AggregationStateFastHashTable *>(
destination_hash_table.get());
- AggregationStateHashTable<AggregationStateMin> *source_hash_table_derived =
- static_cast<AggregationStateHashTable<AggregationStateMin> *>(
- source_hash_table.get());
+ AggregationStateFastHashTable *source_hash_table_derived =
+ static_cast<AggregationStateFastHashTable *>(source_hash_table.get());
AggregationHandleMin *aggregation_handle_min_derived =
static_cast<AggregationHandleMin *>(aggregation_handle_min_.get());
@@ -726,35 +771,52 @@ TEST_F(AggregationHandleMinTest, GroupByTableMergeTest) {
EXPECT_EQ(exclusive_key_source_min_val.getLiteral<int>(), actual_val);
// Add the key-state pairs to the hash tables.
- source_hash_table_derived->putCompositeKey(common_key,
- *common_key_source_state);
- destination_hash_table_derived->putCompositeKey(
- common_key, *common_key_destination_state);
- source_hash_table_derived->putCompositeKey(exclusive_source_key,
- *exclusive_key_source_state);
- destination_hash_table_derived->putCompositeKey(
- exclusive_destination_key, *exclusive_key_destination_state);
+ unsigned char buffer[100];
+ buffer[0] = '\0';
+ memcpy(buffer + 1,
+ common_key_source_state.get()->getPayloadAddress(),
+ aggregation_handle_min_.get()->getPayloadSize());
+ source_hash_table_derived->putCompositeKey(common_key, buffer);
+
+ memcpy(buffer + 1,
+ common_key_destination_state.get()->getPayloadAddress(),
+ aggregation_handle_min_.get()->getPayloadSize());
+ destination_hash_table_derived->putCompositeKey(common_key, buffer);
+
+ memcpy(buffer + 1,
+ exclusive_key_source_state.get()->getPayloadAddress(),
+ aggregation_handle_min_.get()->getPayloadSize());
+ source_hash_table_derived->putCompositeKey(exclusive_source_key, buffer);
+
+ memcpy(buffer + 1,
+ exclusive_key_destination_state.get()->getPayloadAddress(),
+ aggregation_handle_min_.get()->getPayloadSize());
+ destination_hash_table_derived->putCompositeKey(exclusive_destination_key,
+ buffer);
EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
EXPECT_EQ(2u, source_hash_table_derived->numEntries());
- aggregation_handle_min_->mergeGroupByHashTables(*source_hash_table,
- destination_hash_table.get());
+ AggregationOperationState::mergeGroupByHashTables(
+ source_hash_table.get(), destination_hash_table.get());
EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
CheckMinValue<int>(
common_key_source_min_val.getLiteral<int>(),
- *aggregation_handle_min_derived,
- *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ aggregation_handle_min_derived->finalizeHashTableEntryFast(
+ destination_hash_table_derived->getSingleCompositeKey(common_key) +
+ 1));
CheckMinValue<int>(exclusive_key_destination_min_val.getLiteral<int>(),
- *aggregation_handle_min_derived,
- *(destination_hash_table_derived->getSingleCompositeKey(
- exclusive_destination_key)));
+ aggregation_handle_min_derived->finalizeHashTableEntryFast(
+ destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key) +
+ 1));
CheckMinValue<int>(exclusive_key_source_min_val.getLiteral<int>(),
- *aggregation_handle_min_derived,
- *(source_hash_table_derived->getSingleCompositeKey(
- exclusive_source_key)));
+ aggregation_handle_min_derived->finalizeHashTableEntryFast(
+ source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key) +
+ 1));
}
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp b/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
index 0e35151..1d1c084 100644
--- a/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
@@ -28,6 +28,8 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleSum.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/AggregationOperationState.hpp"
+#include "storage/FastHashTableFactory.hpp"
#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DatetimeIntervalType.hpp"
@@ -52,51 +54,56 @@
namespace quickstep {
-class AggregationHandleSumTest : public::testing::Test {
+class AggregationHandleSumTest : public ::testing::Test {
protected:
static const int kNumSamples = 1000;
// Helper method that calls AggregationHandleSum::iterateUnaryInl() to
// aggregate 'value' into '*state'.
void iterateHandle(AggregationState *state, const TypedValue &value) {
- static_cast<const AggregationHandleSum&>(*aggregation_handle_sum_).iterateUnaryInl(
- static_cast<AggregationStateSum*>(state),
- value);
+ static_cast<const AggregationHandleSum &>(*aggregation_handle_sum_)
+ .iterateUnaryInl(static_cast<AggregationStateSum *>(state), value);
}
void initializeHandle(const Type &type) {
aggregation_handle_sum_.reset(
- AggregateFunctionFactory::Get(AggregationID::kSum).createHandle(
- std::vector<const Type*>(1, &type)));
+ AggregateFunctionFactory::Get(AggregationID::kSum)
+ .createHandle(std::vector<const Type *>(1, &type)));
aggregation_handle_sum_state_.reset(
aggregation_handle_sum_->createInitialState());
}
static bool ApplyToTypesTest(TypeID typeID) {
- const Type &type = (typeID == kChar || typeID == kVarChar) ?
- TypeFactory::GetType(typeID, static_cast<std::size_t>(10)) :
- TypeFactory::GetType(typeID);
+ const Type &type =
+ (typeID == kChar || typeID == kVarChar)
+ ? TypeFactory::GetType(typeID, static_cast<std::size_t>(10))
+ : TypeFactory::GetType(typeID);
- return AggregateFunctionFactory::Get(AggregationID::kSum).canApplyToTypes(
- std::vector<const Type*>(1, &type));
+ return AggregateFunctionFactory::Get(AggregationID::kSum)
+ .canApplyToTypes(std::vector<const Type *>(1, &type));
}
static bool ResultTypeForArgumentTypeTest(TypeID input_type_id,
TypeID output_type_id) {
- const Type *result_type
- = AggregateFunctionFactory::Get(AggregationID::kSum).resultTypeForArgumentTypes(
- std::vector<const Type*>(1, &TypeFactory::GetType(input_type_id)));
+ const Type *result_type =
+ AggregateFunctionFactory::Get(AggregationID::kSum)
+ .resultTypeForArgumentTypes(std::vector<const Type *>(
+ 1, &TypeFactory::GetType(input_type_id)));
return (result_type->getTypeID() == output_type_id);
}
template <typename CppType>
- static void CheckSumValue(
- CppType expected,
- const AggregationHandle &target,
- const AggregationState &state) {
+ static void CheckSumValue(CppType expected,
+ const AggregationHandle &target,
+ const AggregationState &state) {
EXPECT_EQ(expected, target.finalize(state).getLiteral<CppType>());
}
+ template <typename CppType>
+ static void CheckSumValue(CppType expected, const TypedValue &value) {
+ EXPECT_EQ(expected, value.getLiteral<CppType>());
+ }
+
// Static templated method to set a meaningful to data types.
template <typename CppType>
static void SetDataType(int value, CppType *data) {
@@ -108,7 +115,9 @@ class AggregationHandleSumTest : public::testing::Test {
const GenericType &type = GenericType::Instance(true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_sum_->finalize(*aggregation_handle_sum_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_sum_->finalize(*aggregation_handle_sum_state_)
+ .isNull());
typename GenericType::cpptype val;
typename PrecisionType::cpptype sum;
@@ -119,13 +128,14 @@ class AggregationHandleSumTest : public::testing::Test {
if (type.getTypeID() == kInt || type.getTypeID() == kLong) {
SetDataType(i - 10, &val);
} else {
- SetDataType(static_cast<float>(i - 10)/10, &val);
+ SetDataType(static_cast<float>(i - 10) / 10, &val);
}
iterateHandle(aggregation_handle_sum_state_.get(), type.makeValue(&val));
sum += val;
}
iterateHandle(aggregation_handle_sum_state_.get(), type.makeNullValue());
- CheckSumValue<typename PrecisionType::cpptype>(sum, *aggregation_handle_sum_, *aggregation_handle_sum_state_);
+ CheckSumValue<typename PrecisionType::cpptype>(
+ sum, *aggregation_handle_sum_, *aggregation_handle_sum_state_);
// Test mergeStates().
std::unique_ptr<AggregationState> merge_state(
@@ -138,7 +148,7 @@ class AggregationHandleSumTest : public::testing::Test {
if (type.getTypeID() == kInt || type.getTypeID() == kLong) {
SetDataType(i - 10, &val);
} else {
- SetDataType(static_cast<float>(i - 10)/10, &val);
+ SetDataType(static_cast<float>(i - 10) / 10, &val);
}
iterateHandle(merge_state.get(), type.makeValue(&val));
sum += val;
@@ -146,13 +156,11 @@ class AggregationHandleSumTest : public::testing::Test {
aggregation_handle_sum_->mergeStates(*merge_state,
aggregation_handle_sum_state_.get());
CheckSumValue<typename PrecisionType::cpptype>(
- sum,
- *aggregation_handle_sum_,
- *aggregation_handle_sum_state_);
+ sum, *aggregation_handle_sum_, *aggregation_handle_sum_state_);
}
template <typename GenericType, typename Output>
- ColumnVector *createColumnVectorGeneric(const Type &type, Output *sum) {
+ ColumnVector* createColumnVectorGeneric(const Type &type, Output *sum) {
NativeColumnVector *column = new NativeColumnVector(type, kNumSamples + 3);
typename GenericType::cpptype val;
@@ -163,12 +171,12 @@ class AggregationHandleSumTest : public::testing::Test {
if (type.getTypeID() == kInt || type.getTypeID() == kLong) {
SetDataType(i - 10, &val);
} else {
- SetDataType(static_cast<float>(i - 10)/10, &val);
+ SetDataType(static_cast<float>(i - 10) / 10, &val);
}
column->appendTypedValue(type.makeValue(&val));
*sum += val;
// One NULL in the middle.
- if (i == kNumSamples/2) {
+ if (i == kNumSamples / 2) {
column->appendTypedValue(type.makeNullValue());
}
}
@@ -182,12 +190,15 @@ class AggregationHandleSumTest : public::testing::Test {
const GenericType &type = GenericType::Instance(true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_sum_->finalize(*aggregation_handle_sum_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_sum_->finalize(*aggregation_handle_sum_state_)
+ .isNull());
typename PrecisionType::cpptype sum;
std::vector<std::unique_ptr<ColumnVector>> column_vectors;
column_vectors.emplace_back(
- createColumnVectorGeneric<GenericType, typename PrecisionType::cpptype>(type, &sum));
+ createColumnVectorGeneric<GenericType, typename PrecisionType::cpptype>(
+ type, &sum));
std::unique_ptr<AggregationState> cv_state(
aggregation_handle_sum_->accumulateColumnVectors(column_vectors));
@@ -195,15 +206,12 @@ class AggregationHandleSumTest : public::testing::Test {
// Test the state generated directly by accumulateColumnVectors(), and also
// test after merging back.
CheckSumValue<typename PrecisionType::cpptype>(
- sum,
- *aggregation_handle_sum_,
- *cv_state);
+ sum, *aggregation_handle_sum_, *cv_state);
- aggregation_handle_sum_->mergeStates(*cv_state, aggregation_handle_sum_state_.get());
+ aggregation_handle_sum_->mergeStates(*cv_state,
+ aggregation_handle_sum_state_.get());
CheckSumValue<typename PrecisionType::cpptype>(
- sum,
- *aggregation_handle_sum_,
- *aggregation_handle_sum_state_);
+ sum, *aggregation_handle_sum_, *aggregation_handle_sum_state_);
}
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -212,29 +220,30 @@ class AggregationHandleSumTest : public::testing::Test {
const GenericType &type = GenericType::Instance(true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_sum_->finalize(*aggregation_handle_sum_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_sum_->finalize(*aggregation_handle_sum_state_)
+ .isNull());
typename PrecisionType::cpptype sum;
- std::unique_ptr<ColumnVectorsValueAccessor> accessor(new ColumnVectorsValueAccessor());
+ std::unique_ptr<ColumnVectorsValueAccessor> accessor(
+ new ColumnVectorsValueAccessor());
accessor->addColumn(
- createColumnVectorGeneric<GenericType, typename PrecisionType::cpptype>(type, &sum));
+ createColumnVectorGeneric<GenericType, typename PrecisionType::cpptype>(
+ type, &sum));
std::unique_ptr<AggregationState> va_state(
- aggregation_handle_sum_->accumulateValueAccessor(accessor.get(),
- std::vector<attribute_id>(1, 0)));
+ aggregation_handle_sum_->accumulateValueAccessor(
+ accessor.get(), std::vector<attribute_id>(1, 0)));
// Test the state generated directly by accumulateValueAccessor(), and also
// test after merging back.
CheckSumValue<typename PrecisionType::cpptype>(
- sum,
- *aggregation_handle_sum_,
- *va_state);
+ sum, *aggregation_handle_sum_, *va_state);
- aggregation_handle_sum_->mergeStates(*va_state, aggregation_handle_sum_state_.get());
+ aggregation_handle_sum_->mergeStates(*va_state,
+ aggregation_handle_sum_state_.get());
CheckSumValue<typename PrecisionType::cpptype>(
- sum,
- *aggregation_handle_sum_,
- *aggregation_handle_sum_state_);
+ sum, *aggregation_handle_sum_, *aggregation_handle_sum_state_);
}
#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -247,9 +256,7 @@ const int AggregationHandleSumTest::kNumSamples;
template <>
void AggregationHandleSumTest::CheckSumValue<float>(
- float val,
- const AggregationHandle &handle,
- const AggregationState &state) {
+ float val, const AggregationHandle &handle, const AggregationState &state) {
EXPECT_FLOAT_EQ(val, handle.finalize(state).getLiteral<float>());
}
@@ -262,12 +269,14 @@ void AggregationHandleSumTest::CheckSumValue<double>(
}
template <>
-void AggregationHandleSumTest::SetDataType<DatetimeIntervalLit>(int value, DatetimeIntervalLit *data) {
+void AggregationHandleSumTest::SetDataType<DatetimeIntervalLit>(
+ int value, DatetimeIntervalLit *data) {
data->interval_ticks = value;
}
template <>
-void AggregationHandleSumTest::SetDataType<YearMonthIntervalLit>(int value, YearMonthIntervalLit *data) {
+void AggregationHandleSumTest::SetDataType<YearMonthIntervalLit>(
+ int value, YearMonthIntervalLit *data) {
data->months = value;
}
@@ -314,11 +323,13 @@ TEST_F(AggregationHandleSumTest, DoubleTypeColumnVectorTest) {
}
TEST_F(AggregationHandleSumTest, DatetimeIntervalTypeColumnVectorTest) {
- checkAggregationSumGenericColumnVector<DatetimeIntervalType, DatetimeIntervalType>();
+ checkAggregationSumGenericColumnVector<DatetimeIntervalType,
+ DatetimeIntervalType>();
}
TEST_F(AggregationHandleSumTest, YearMonthIntervalTypeColumnVectorTest) {
- checkAggregationSumGenericColumnVector<YearMonthIntervalType, YearMonthIntervalType>();
+ checkAggregationSumGenericColumnVector<YearMonthIntervalType,
+ YearMonthIntervalType>();
}
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -339,11 +350,13 @@ TEST_F(AggregationHandleSumTest, DoubleTypeValueAccessorTest) {
}
TEST_F(AggregationHandleSumTest, DatetimeIntervalTypeValueAccessorTest) {
- checkAggregationSumGenericValueAccessor<DatetimeIntervalType, DatetimeIntervalType>();
+ checkAggregationSumGenericValueAccessor<DatetimeIntervalType,
+ DatetimeIntervalType>();
}
TEST_F(AggregationHandleSumTest, YearMonthIntervalTypeValueAccessorTest) {
- checkAggregationSumGenericValueAccessor<YearMonthIntervalType, YearMonthIntervalType>();
+ checkAggregationSumGenericValueAccessor<YearMonthIntervalType,
+ YearMonthIntervalType>();
}
#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -373,38 +386,53 @@ TEST_F(AggregationHandleSumDeathTest, WrongTypeTest) {
float float_val = 0;
// Passes.
- iterateHandle(aggregation_handle_sum_state_.get(), int_non_null_type.makeValue(&int_val));
+ iterateHandle(aggregation_handle_sum_state_.get(),
+ int_non_null_type.makeValue(&int_val));
- EXPECT_DEATH(iterateHandle(aggregation_handle_sum_state_.get(), long_type.makeValue(&long_val)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_sum_state_.get(), double_type.makeValue(&double_val)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_sum_state_.get(), float_type.makeValue(&float_val)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_sum_state_.get(), char_type.makeValue("asdf", 5)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_sum_state_.get(), varchar_type.makeValue("asdf", 5)), "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_sum_state_.get(),
+ long_type.makeValue(&long_val)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_sum_state_.get(),
+ double_type.makeValue(&double_val)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_sum_state_.get(),
+ float_type.makeValue(&float_val)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_sum_state_.get(),
+ char_type.makeValue("asdf", 5)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_sum_state_.get(),
+ varchar_type.makeValue("asdf", 5)),
+ "");
// Test mergeStates() with incorrectly typed handles.
std::unique_ptr<AggregationHandle> aggregation_handle_sum_double(
- AggregateFunctionFactory::Get(AggregationID::kSum).createHandle(
- std::vector<const Type*>(1, &double_type)));
+ AggregateFunctionFactory::Get(AggregationID::kSum)
+ .createHandle(std::vector<const Type *>(1, &double_type)));
std::unique_ptr<AggregationState> aggregation_state_sum_merge_double(
aggregation_handle_sum_double->createInitialState());
- static_cast<const AggregationHandleSum&>(*aggregation_handle_sum_double).iterateUnaryInl(
- static_cast<AggregationStateSum*>(aggregation_state_sum_merge_double.get()),
- double_type.makeValue(&double_val));
- EXPECT_DEATH(aggregation_handle_sum_->mergeStates(*aggregation_state_sum_merge_double,
- aggregation_handle_sum_state_.get()),
- "");
+ static_cast<const AggregationHandleSum &>(*aggregation_handle_sum_double)
+ .iterateUnaryInl(static_cast<AggregationStateSum *>(
+ aggregation_state_sum_merge_double.get()),
+ double_type.makeValue(&double_val));
+ EXPECT_DEATH(
+ aggregation_handle_sum_->mergeStates(*aggregation_state_sum_merge_double,
+ aggregation_handle_sum_state_.get()),
+ "");
std::unique_ptr<AggregationHandle> aggregation_handle_sum_float(
- AggregateFunctionFactory::Get(AggregationID::kSum).createHandle(
- std::vector<const Type*>(1, &float_type)));
+ AggregateFunctionFactory::Get(AggregationID::kSum)
+ .createHandle(std::vector<const Type *>(1, &float_type)));
std::unique_ptr<AggregationState> aggregation_state_sum_merge_float(
aggregation_handle_sum_float->createInitialState());
- static_cast<const AggregationHandleSum&>(*aggregation_handle_sum_float).iterateUnaryInl(
- static_cast<AggregationStateSum*>(aggregation_state_sum_merge_float.get()),
- float_type.makeValue(&float_val));
- EXPECT_DEATH(aggregation_handle_sum_->mergeStates(*aggregation_state_sum_merge_float,
- aggregation_handle_sum_state_.get()),
- "");
+ static_cast<const AggregationHandleSum &>(*aggregation_handle_sum_float)
+ .iterateUnaryInl(static_cast<AggregationStateSum *>(
+ aggregation_state_sum_merge_float.get()),
+ float_type.makeValue(&float_val));
+ EXPECT_DEATH(
+ aggregation_handle_sum_->mergeStates(*aggregation_state_sum_merge_float,
+ aggregation_handle_sum_state_.get()),
+ "");
}
#endif
@@ -425,8 +453,10 @@ TEST_F(AggregationHandleSumTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kLong, kLong));
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kFloat, kDouble));
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kDouble, kDouble));
- EXPECT_TRUE(ResultTypeForArgumentTypeTest(kDatetimeInterval, kDatetimeInterval));
- EXPECT_TRUE(ResultTypeForArgumentTypeTest(kYearMonthInterval, kYearMonthInterval));
+ EXPECT_TRUE(
+ ResultTypeForArgumentTypeTest(kDatetimeInterval, kDatetimeInterval));
+ EXPECT_TRUE(
+ ResultTypeForArgumentTypeTest(kYearMonthInterval, kYearMonthInterval));
}
TEST_F(AggregationHandleSumTest, GroupByTableMergeTest) {
@@ -434,25 +464,28 @@ TEST_F(AggregationHandleSumTest, GroupByTableMergeTest) {
initializeHandle(long_non_null_type);
storage_manager_.reset(new StorageManager("./test_sum_data"));
std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
- aggregation_handle_sum_->createGroupByHashTable(
- HashTableImplType::kSimpleScalarSeparateChaining,
+ AggregationStateFastHashTableFactory::CreateResizable(
+ HashTableImplType::kSeparateChaining,
std::vector<const Type *>(1, &long_non_null_type),
10,
+ {aggregation_handle_sum_.get()->getPayloadSize()},
+ {aggregation_handle_sum_.get()},
storage_manager_.get()));
std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
- aggregation_handle_sum_->createGroupByHashTable(
- HashTableImplType::kSimpleScalarSeparateChaining,
+ AggregationStateFastHashTableFactory::CreateResizable(
+ HashTableImplType::kSeparateChaining,
std::vector<const Type *>(1, &long_non_null_type),
10,
+ {aggregation_handle_sum_.get()->getPayloadSize()},
+ {aggregation_handle_sum_.get()},
storage_manager_.get()));
- AggregationStateHashTable<AggregationStateSum> *destination_hash_table_derived =
- static_cast<AggregationStateHashTable<AggregationStateSum> *>(
+ AggregationStateFastHashTable *destination_hash_table_derived =
+ static_cast<AggregationStateFastHashTable *>(
destination_hash_table.get());
- AggregationStateHashTable<AggregationStateSum> *source_hash_table_derived =
- static_cast<AggregationStateHashTable<AggregationStateSum> *>(
- source_hash_table.get());
+ AggregationStateFastHashTable *source_hash_table_derived =
+ static_cast<AggregationStateFastHashTable *>(source_hash_table.get());
AggregationHandleSum *aggregation_handle_sum_derived =
static_cast<AggregationHandleSum *>(aggregation_handle_sum_.get());
@@ -471,7 +504,8 @@ TEST_F(AggregationHandleSumTest, GroupByTableMergeTest) {
const std::int64_t common_key_destination_sum = 4000;
TypedValue common_key_destination_sum_val(common_key_destination_sum);
- const std::int64_t merged_common_key = common_key_source_sum + common_key_destination_sum;
+ const std::int64_t merged_common_key =
+ common_key_source_sum + common_key_destination_sum;
TypedValue common_key_merged_val(merged_common_key);
const std::int64_t exclusive_key_source_sum = 100;
@@ -496,59 +530,82 @@ TEST_F(AggregationHandleSumTest, GroupByTableMergeTest) {
// Create sum value states for keys.
aggregation_handle_sum_derived->iterateUnaryInl(common_key_source_state.get(),
common_key_source_sum_val);
- std::int64_t actual_val = aggregation_handle_sum_->finalize(*common_key_source_state)
- .getLiteral<std::int64_t>();
+ std::int64_t actual_val =
+ aggregation_handle_sum_->finalize(*common_key_source_state)
+ .getLiteral<std::int64_t>();
EXPECT_EQ(common_key_source_sum_val.getLiteral<std::int64_t>(), actual_val);
aggregation_handle_sum_derived->iterateUnaryInl(
common_key_destination_state.get(), common_key_destination_sum_val);
actual_val = aggregation_handle_sum_->finalize(*common_key_destination_state)
.getLiteral<std::int64_t>();
- EXPECT_EQ(common_key_destination_sum_val.getLiteral<std::int64_t>(), actual_val);
+ EXPECT_EQ(common_key_destination_sum_val.getLiteral<std::int64_t>(),
+ actual_val);
aggregation_handle_sum_derived->iterateUnaryInl(
exclusive_key_destination_state.get(), exclusive_key_destination_sum_val);
actual_val =
aggregation_handle_sum_->finalize(*exclusive_key_destination_state)
.getLiteral<std::int64_t>();
- EXPECT_EQ(exclusive_key_destination_sum_val.getLiteral<std::int64_t>(), actual_val);
+ EXPECT_EQ(exclusive_key_destination_sum_val.getLiteral<std::int64_t>(),
+ actual_val);
aggregation_handle_sum_derived->iterateUnaryInl(
exclusive_key_source_state.get(), exclusive_key_source_sum_val);
actual_val = aggregation_handle_sum_->finalize(*exclusive_key_source_state)
.getLiteral<std::int64_t>();
- EXPECT_EQ(exclusive_key_source_sum_val.getLiteral<std::int64_t>(), actual_val);
+ EXPECT_EQ(exclusive_key_source_sum_val.getLiteral<std::int64_t>(),
+ actual_val);
// Add the key-state pairs to the hash tables.
- source_hash_table_derived->putCompositeKey(common_key,
- *common_key_source_state);
- destination_hash_table_derived->putCompositeKey(
- common_key, *common_key_destination_state);
- source_hash_table_derived->putCompositeKey(exclusive_source_key,
- *exclusive_key_source_state);
- destination_hash_table_derived->putCompositeKey(
- exclusive_destination_key, *exclusive_key_destination_state);
+ unsigned char buffer[100];
+ buffer[0] = '\0';
+ memcpy(buffer + 1,
+ common_key_source_state.get()->getPayloadAddress(),
+ aggregation_handle_sum_.get()->getPayloadSize());
+ source_hash_table_derived->putCompositeKey(common_key, buffer);
+
+ memcpy(buffer + 1,
+ common_key_destination_state.get()->getPayloadAddress(),
+ aggregation_handle_sum_.get()->getPayloadSize());
+ destination_hash_table_derived->putCompositeKey(common_key, buffer);
+
+ memcpy(buffer + 1,
+ exclusive_key_source_state.get()->getPayloadAddress(),
+ aggregation_handle_sum_.get()->getPayloadSize());
+ source_hash_table_derived->putCompositeKey(exclusive_source_key, buffer);
+
+ memcpy(buffer + 1,
+ exclusive_key_destination_state.get()->getPayloadAddress(),
+ aggregation_handle_sum_.get()->getPayloadSize());
+ destination_hash_table_derived->putCompositeKey(exclusive_destination_key,
+ buffer);
EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
EXPECT_EQ(2u, source_hash_table_derived->numEntries());
- aggregation_handle_sum_->mergeGroupByHashTables(*source_hash_table,
- destination_hash_table.get());
+ AggregationOperationState::mergeGroupByHashTables(
+ source_hash_table.get(), destination_hash_table.get());
EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
CheckSumValue<std::int64_t>(
common_key_merged_val.getLiteral<std::int64_t>(),
- *aggregation_handle_sum_derived,
- *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
- CheckSumValue<std::int64_t>(exclusive_key_destination_sum_val.getLiteral<std::int64_t>(),
- *aggregation_handle_sum_derived,
- *(destination_hash_table_derived->getSingleCompositeKey(
- exclusive_destination_key)));
- CheckSumValue<std::int64_t>(exclusive_key_source_sum_val.getLiteral<std::int64_t>(),
- *aggregation_handle_sum_derived,
- *(source_hash_table_derived->getSingleCompositeKey(
- exclusive_source_key)));
+ aggregation_handle_sum_derived->finalizeHashTableEntryFast(
+ destination_hash_table_derived->getSingleCompositeKey(common_key) +
+ 1));
+ CheckSumValue<std::int64_t>(
+ exclusive_key_destination_sum_val.getLiteral<std::int64_t>(),
+ aggregation_handle_sum_derived->finalizeHashTableEntryFast(
+ destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key) +
+ 1));
+ CheckSumValue<std::int64_t>(
+ exclusive_key_source_sum_val.getLiteral<std::int64_t>(),
+ aggregation_handle_sum_derived->finalizeHashTableEntryFast(
+ source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key) +
+ 1));
}
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 130134c..968314e 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1371,13 +1371,9 @@ void ExecutionGenerator::convertAggregate(
}
if (!group_by_types.empty()) {
- // SimplifyHashTableImplTypeProto() switches the hash table implementation
- // from SeparateChaining to SimpleScalarSeparateChaining when there is a
- // single scalar key type with a reversible hash function.
+ // Right now, only SeparateChaining is supported.
aggr_state_proto->set_hash_table_impl_type(
- SimplifyHashTableImplTypeProto(
- HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
- group_by_types));
+ serialization::HashTableImplType::SEPARATE_CHAINING);
}
for (const E::AliasPtr &named_aggregate_expression : physical_plan->aggregate_expressions()) {
@@ -1404,15 +1400,9 @@ void ExecutionGenerator::convertAggregate(
if (unnamed_aggregate_expression->is_distinct()) {
const std::vector<E::ScalarPtr> &arguments = unnamed_aggregate_expression->getArguments();
DCHECK_GE(arguments.size(), 1u);
- if (group_by_types.empty() && arguments.size() == 1) {
- aggr_state_proto->add_distinctify_hash_table_impl_types(
- SimplifyHashTableImplTypeProto(
- HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
- {&arguments[0]->getValueType()}));
- } else {
- aggr_state_proto->add_distinctify_hash_table_impl_types(
- HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type));
- }
+ // Right now only SeparateChaining implementation is supported.
+ aggr_state_proto->add_distinctify_hash_table_impl_types(
+ serialization::HashTableImplType::SEPARATE_CHAINING);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/relational_operators/tests/AggregationOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/AggregationOperator_unittest.cpp b/relational_operators/tests/AggregationOperator_unittest.cpp
index 0138362..6881dea 100644
--- a/relational_operators/tests/AggregationOperator_unittest.cpp
+++ b/relational_operators/tests/AggregationOperator_unittest.cpp
@@ -363,8 +363,9 @@ class AggregationOperatorTest : public ::testing::Test {
aggr_state_proto->set_estimated_num_entries(estimated_entries);
// Also need to set the HashTable implementation for GROUP BY.
+ // Right now, only SeparateChaining is supported.
aggr_state_proto->set_hash_table_impl_type(
- serialization::HashTableImplType::LINEAR_OPEN_ADDRESSING);
+ serialization::HashTableImplType::SEPARATE_CHAINING);
// Create Operators.
op_.reset(new AggregationOperator(0, *table_, true, aggr_state_index));
[02/17] incubator-quickstep git commit: Initial commit for
QUICKSTEP-28 and QUICKSTEP-29.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/storage/HashTablePool.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTablePool.hpp b/storage/HashTablePool.hpp
index 53fe514..3cdfcb3 100644
--- a/storage/HashTablePool.hpp
+++ b/storage/HashTablePool.hpp
@@ -27,6 +27,8 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "storage/HashTableBase.hpp"
+#include "storage/FastHashTable.hpp"
+#include "storage/FastHashTableFactory.hpp"
#include "threading/SpinMutex.hpp"
#include "utility/Macros.hpp"
#include "utility/StringUtil.hpp"
@@ -82,6 +84,34 @@ class HashTablePool {
storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
/**
+ * @brief Constructor.
+ *
+ * @note This constructor is relevant for HashTables specialized for
+ * aggregation.
+ *
+ * @param estimated_num_entries The maximum number of entries in a hash table.
+ * @param hash_table_impl_type The type of hash table implementation.
+ * @param group_by_types A vector of pointer of types which form the group by
+ * key.
+ * @param payload_sizes The sizes in bytes for the AggregationStates for the
+ * respective AggregationHandles.
+ * @param handles The AggregationHandles in this query.
+ * @param storage_manager A pointer to the storage manager.
+ **/
+ HashTablePool(const std::size_t estimated_num_entries,
+ const HashTableImplType hash_table_impl_type,
+ const std::vector<const Type *> &group_by_types,
+ const std::vector<std::size_t> &payload_sizes,
+ const std::vector<AggregationHandle *> &handles,
+ StorageManager *storage_manager)
+ : estimated_num_entries_(reduceEstimatedCardinality(estimated_num_entries)),
+ hash_table_impl_type_(hash_table_impl_type),
+ group_by_types_(group_by_types),
+ payload_sizes_(payload_sizes),
+ handles_(handles),
+ storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
+
+ /**
* @brief Check out a hash table for insertion.
*
* @return A hash table pointer.
@@ -101,6 +131,28 @@ class HashTablePool {
}
/**
+ * @brief Check out a hash table for insertion.
+ *
+ * @note This method is relevant for specialized (for aggregation)
+ * hash table implementation.
+ *
+ * @return A hash table pointer.
+ **/
+ AggregationStateHashTableBase* getHashTableFast() {
+ {
+ SpinMutexLock lock(mutex_);
+ if (!hash_tables_.empty()) {
+ std::unique_ptr<AggregationStateHashTableBase> ret_hash_table(
+ std::move(hash_tables_.back()));
+ hash_tables_.pop_back();
+ DCHECK(ret_hash_table != nullptr);
+ return ret_hash_table.release();
+ }
+ }
+ return createNewHashTableFast();
+ }
+
+ /**
* @brief Return a previously checked out hash table.
*
* @param hash_table A pointer to the checked out hash table.
@@ -134,6 +186,16 @@ class HashTablePool {
storage_manager_);
}
+ AggregationStateHashTableBase* createNewHashTableFast() {
+ return AggregationStateFastHashTableFactory::CreateResizable(
+ hash_table_impl_type_,
+ group_by_types_,
+ estimated_num_entries_,
+ payload_sizes_,
+ handles_,
+ storage_manager_);
+ }
+
inline std::size_t reduceEstimatedCardinality(
const std::size_t original_estimate) const {
if (original_estimate < kEstimateReductionFactor) {
@@ -153,7 +215,10 @@ class HashTablePool {
const std::vector<const Type *> group_by_types_;
+ std::vector<std::size_t> payload_sizes_;
+
AggregationHandle *agg_handle_;
+ const std::vector<AggregationHandle *> handles_;
StorageManager *storage_manager_;
SpinMutex mutex_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/storage/StorageBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.cpp b/storage/StorageBlock.cpp
index 21aa12c..ec5990f 100644
--- a/storage/StorageBlock.cpp
+++ b/storage/StorageBlock.cpp
@@ -415,8 +415,7 @@ AggregationState* StorageBlock::aggregate(
}
void StorageBlock::aggregateGroupBy(
- const AggregationHandle &handle,
- const std::vector<std::unique_ptr<const Scalar>> &arguments,
+ const std::vector<std::vector<std::unique_ptr<const Scalar>>> &arguments,
const std::vector<std::unique_ptr<const Scalar>> &group_by,
const Predicate *predicate,
AggregationStateHashTableBase *hash_table,
@@ -481,19 +480,24 @@ void StorageBlock::aggregateGroupBy(
}
// Compute argument vectors and add them to 'temp_result'.
- for (const std::unique_ptr<const Scalar> &argument : arguments) {
- temp_result.addColumn(argument->getAllValues(accessor.get(), &sub_blocks_ref));
- argument_ids.push_back(attr_id++);
- }
+ for (const std::vector<std::unique_ptr<const Scalar>> &argument : arguments) {
+ for (const std::unique_ptr<const Scalar> &args : argument) {
+ temp_result.addColumn(args->getAllValues(accessor.get(), &sub_blocks_ref));
+ argument_ids.push_back(attr_id++);
+ }
+ if (argument.empty()) {
+ argument_ids.push_back(kInvalidAttributeID);
+ }
+ }
}
- // Actually do aggregation into '*hash_table'.
- handle.aggregateValueAccessorIntoHashTable(&temp_result,
- argument_ids,
- key_ids,
- hash_table);
+ hash_table->upsertValueAccessorCompositeKeyFast(argument_ids,
+ &temp_result,
+ key_ids,
+ true);
}
+
void StorageBlock::aggregateDistinct(
const AggregationHandle &handle,
const std::vector<std::unique_ptr<const Scalar>> &arguments,
@@ -582,7 +586,6 @@ void StorageBlock::aggregateDistinct(
&temp_result, key_ids, distinctify_hash_table);
}
-
// TODO(chasseur): Vectorization for updates.
StorageBlock::UpdateResult StorageBlock::update(
const unordered_map<attribute_id, unique_ptr<const Scalar>> &assignments,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/storage/StorageBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.hpp b/storage/StorageBlock.hpp
index 97b4773..bab5bab 100644
--- a/storage/StorageBlock.hpp
+++ b/storage/StorageBlock.hpp
@@ -419,7 +419,6 @@ class StorageBlock : public StorageBlockBase {
* @brief Perform GROUP BY aggregation on the tuples in the this storage
* block.
*
- * @param handle Aggregation handle to compute aggregates with.
* @param arguments The arguments to the aggregation function as Scalars.
* @param group_by The list of GROUP BY attributes/expressions. The tuples in
* this storage block are grouped by these attributes before
@@ -459,14 +458,13 @@ class StorageBlock : public StorageBlockBase {
* attributes as std::vector<attribute_id> (like in selectSimple()) for fast
* path when there are no expressions specified in the query.
*/
- void aggregateGroupBy(const AggregationHandle &handle,
- const std::vector<std::unique_ptr<const Scalar>> &arguments,
- const std::vector<std::unique_ptr<const Scalar>> &group_by,
- const Predicate *predicate,
- AggregationStateHashTableBase *hash_table,
- std::unique_ptr<TupleIdSequence> *reuse_matches,
- std::vector<std::unique_ptr<ColumnVector>>
- *reuse_group_by_vectors) const;
+ void aggregateGroupBy(
+ const std::vector<std::vector<std::unique_ptr<const Scalar>>> &arguments,
+ const std::vector<std::unique_ptr<const Scalar>> &group_by,
+ const Predicate *predicate,
+ AggregationStateHashTableBase *hash_table,
+ std::unique_ptr<TupleIdSequence> *reuse_matches,
+ std::vector<std::unique_ptr<ColumnVector>> *reuse_group_by_vectors) const;
/**
* @brief Inserts the GROUP BY expressions and aggregation arguments together
[13/17] incubator-quickstep git commit: Checking in preprocessed
files for datetime change.
Posted by ji...@apache.org.
Checking in preprocessed files for datetime change.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0820e3d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0820e3d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0820e3d9
Branch: refs/heads/lip-refactor
Commit: 0820e3d92001b4f15caff0a5291beb134bd16a6c
Parents: 6b377d5
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Sun Oct 2 07:59:55 2016 -0500
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Sun Oct 2 07:59:55 2016 -0500
----------------------------------------------------------------------
parser/preprocessed/SqlParser_gen.cpp | 4602 +++++++++++++---------------
parser/preprocessed/SqlParser_gen.hpp | 322 +-
2 files changed, 2261 insertions(+), 2663 deletions(-)
----------------------------------------------------------------------
[16/17] incubator-quickstep git commit: Optimizer updates
Posted by ji...@apache.org.
Optimizer updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/18e72095
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/18e72095
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/18e72095
Branch: refs/heads/lip-refactor
Commit: 18e72095e8054782b773d52690edb43140b6bf21
Parents: 112d480
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu Sep 8 23:10:22 2016 -0500
Committer: Jianqiao Zhu <ji...@cs.wisc.edu>
Committed: Tue Oct 4 23:25:06 2016 -0500
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 4 +
query_optimizer/LIPFilterGenerator.hpp | 46 ++++
query_optimizer/PhysicalGenerator.cpp | 5 +-
query_optimizer/physical/CMakeLists.txt | 10 +
.../physical/LIPFilterConfiguration.hpp | 118 +++++++++
query_optimizer/physical/TopLevelPlan.hpp | 26 +-
query_optimizer/rules/AttachLIPFilters.cpp | 259 +++++++++++++++++++
query_optimizer/rules/AttachLIPFilters.hpp | 141 ++++++++++
query_optimizer/rules/CMakeLists.txt | 16 ++
utility/CMakeLists.txt | 4 +
utility/LIPFilter.cpp | 24 ++
utility/LIPFilter.hpp | 44 ++++
utility/LIPFilterAdapter.hpp | 0
utility/LIPFilterBuilder.hpp | 0
utility/PlanVisualizer.cpp | 48 +++-
utility/PlanVisualizer.hpp | 3 +
16 files changed, 735 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 2b521ec..32eaea3 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -41,6 +41,7 @@ add_subdirectory(tests)
# Declare micro-libs:
add_library(quickstep_queryoptimizer_ExecutionGenerator ExecutionGenerator.cpp ExecutionGenerator.hpp)
+add_library(quickstep_queryoptimizer_LIPFilterGenerator ../empty_src.cpp LIPFilterGenerator.hpp)
add_library(quickstep_queryoptimizer_LogicalGenerator LogicalGenerator.cpp LogicalGenerator.hpp)
add_library(quickstep_queryoptimizer_LogicalToPhysicalMapper
../empty_src.cpp
@@ -151,6 +152,8 @@ if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_catalog_Catalog_proto)
endif()
+target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
+ glog)
target_link_libraries(quickstep_queryoptimizer_LogicalGenerator
glog
quickstep_parser_ParseStatement
@@ -186,6 +189,7 @@ target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
quickstep_queryoptimizer_LogicalToPhysicalMapper
quickstep_queryoptimizer_logical_Logical
quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_rules_AttachLIPFilters
quickstep_queryoptimizer_rules_PruneColumns
quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
quickstep_queryoptimizer_rules_SwapProbeBuild
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/query_optimizer/LIPFilterGenerator.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/LIPFilterGenerator.hpp b/query_optimizer/LIPFilterGenerator.hpp
new file mode 100644
index 0000000..4137b98
--- /dev/null
+++ b/query_optimizer/LIPFilterGenerator.hpp
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_OPTIMIZER_LIP_FILTER_GENERATOR_HPP_
+#define QUICKSTEP_QUERY_OPTIMIZER_LIP_FILTER_GENERATOR_HPP_
+
+namespace quickstep {
+namespace optimizer {
+
+/** \addtogroup QueryOptimizer
+ * @{
+ */
+
+class LIPFilterGenerator {
+ public:
+
+
+
+ private:
+
+
+};
+
+
+/** @} */
+
+} // namespace optimizer
+} // namespace quickstep
+
+#endif /* QUICKSTEP_QUERY_OPTIMIZER_LIP_FILTER_GENERATOR_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/query_optimizer/PhysicalGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/PhysicalGenerator.cpp b/query_optimizer/PhysicalGenerator.cpp
index 8f19702..a3fedc9 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -26,6 +26,7 @@
#include "query_optimizer/Validator.hpp"
#include "query_optimizer/logical/Logical.hpp"
#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/rules/AttachLIPFilters.hpp"
#include "query_optimizer/rules/PruneColumns.hpp"
#include "query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp"
#include "query_optimizer/rules/SwapProbeBuild.hpp"
@@ -100,6 +101,7 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
}
rules.emplace_back(new PruneColumns());
rules.emplace_back(new SwapProbeBuild());
+ rules.emplace_back(new AttachLIPFilters());
for (std::unique_ptr<Rule<P::Physical>> &rule : rules) {
physical_plan_ = rule->apply(physical_plan_);
@@ -110,9 +112,10 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
DVLOG(4) << "Optimized physical plan:\n" << physical_plan_->toString();
if (FLAGS_visualize_plan) {
- quickstep::PlanVisualizer plan_visualizer;
+ quickstep::PlanVisualizer plan_visualizer;
std::cerr << "\n" << plan_visualizer.visualize(physical_plan_) << "\n";
}
+ exit(0);
#ifdef QUICKSTEP_DEBUG
Validate(physical_plan_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/query_optimizer/physical/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/CMakeLists.txt b/query_optimizer/physical/CMakeLists.txt
index 3b7d3f0..83682b3 100644
--- a/query_optimizer/physical/CMakeLists.txt
+++ b/query_optimizer/physical/CMakeLists.txt
@@ -27,6 +27,7 @@ add_library(quickstep_queryoptimizer_physical_HashJoin HashJoin.cpp HashJoin.hpp
add_library(quickstep_queryoptimizer_physical_InsertSelection InsertSelection.cpp InsertSelection.hpp)
add_library(quickstep_queryoptimizer_physical_InsertTuple InsertTuple.cpp InsertTuple.hpp)
add_library(quickstep_queryoptimizer_physical_Join ../../empty_src.cpp Join.hpp)
+add_library(quickstep_queryoptimizer_physical_LIPFilterConfiguration ../../empty_src.cpp LIPFilterConfiguration.hpp)
add_library(quickstep_queryoptimizer_physical_NestedLoopsJoin NestedLoopsJoin.cpp NestedLoopsJoin.hpp)
add_library(quickstep_queryoptimizer_physical_PatternMatcher ../../empty_src.cpp PatternMatcher.hpp)
add_library(quickstep_queryoptimizer_physical_Physical ../../empty_src.cpp Physical.hpp)
@@ -51,6 +52,7 @@ target_link_libraries(quickstep_queryoptimizer_physical_Aggregate
quickstep_queryoptimizer_expressions_ExpressionUtil
quickstep_queryoptimizer_expressions_NamedExpression
quickstep_queryoptimizer_expressions_Predicate
+ quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_physical_PhysicalType
quickstep_utility_Cast
@@ -121,6 +123,7 @@ target_link_libraries(quickstep_queryoptimizer_physical_HashJoin
quickstep_queryoptimizer_expressions_NamedExpression
quickstep_queryoptimizer_expressions_Predicate
quickstep_queryoptimizer_physical_BinaryJoin
+ quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_physical_PhysicalType
quickstep_utility_Cast
@@ -150,6 +153,10 @@ target_link_libraries(quickstep_queryoptimizer_physical_Join
quickstep_queryoptimizer_expressions_NamedExpression
quickstep_queryoptimizer_physical_Physical
quickstep_utility_Macros)
+target_link_libraries(quickstep_queryoptimizer_physical_LIPFilterConfiguration
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_utility_LIPFilter
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_physical_NestedLoopsJoin
glog
quickstep_queryoptimizer_OptimizerTree
@@ -167,6 +174,7 @@ target_link_libraries(quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_OptimizerTree
quickstep_queryoptimizer_expressions_AttributeReference
quickstep_queryoptimizer_expressions_ExpressionUtil
+ quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_physical_PhysicalType
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_physical_Sample
@@ -187,6 +195,7 @@ target_link_libraries(quickstep_queryoptimizer_physical_Selection
quickstep_queryoptimizer_expressions_LogicalAnd
quickstep_queryoptimizer_expressions_NamedExpression
quickstep_queryoptimizer_expressions_Predicate
+ quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_physical_PhysicalType
quickstep_utility_Cast
@@ -279,6 +288,7 @@ target_link_libraries(quickstep_queryoptimizer_physical
quickstep_queryoptimizer_physical_InsertSelection
quickstep_queryoptimizer_physical_InsertTuple
quickstep_queryoptimizer_physical_Join
+ quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_physical_NestedLoopsJoin
quickstep_queryoptimizer_physical_PatternMatcher
quickstep_queryoptimizer_physical_Physical
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/query_optimizer/physical/LIPFilterConfiguration.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/LIPFilterConfiguration.hpp b/query_optimizer/physical/LIPFilterConfiguration.hpp
new file mode 100644
index 0000000..59db2e6
--- /dev/null
+++ b/query_optimizer/physical/LIPFilterConfiguration.hpp
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_OPTIMIZER_PHYSICAL_LIP_FILTER_CONFIGURATION_HPP_
+#define QUICKSTEP_QUERY_OPTIMIZER_PHYSICAL_LIP_FILTER_CONFIGURATION_HPP_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "utility/LIPFilter.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+namespace physical {
+
+/** \addtogroup OptimizerPhysical
+ * @{
+ */
+
+class Physical;
+typedef std::shared_ptr<const Physical> PhysicalPtr;
+
+struct LIPFilterBuildInfo{
+ LIPFilterBuildInfo(const expressions::AttributeReferencePtr &build_attribute_in,
+ const std::size_t filter_size_in,
+ const LIPFilterType &filter_type_in)
+ : build_attribute(build_attribute_in),
+ filter_size(filter_size_in),
+ filter_type(filter_type_in) {
+ }
+ expressions::AttributeReferencePtr build_attribute;
+ std::size_t filter_size;
+ LIPFilterType filter_type;
+};
+
+struct LIPFilterProbeInfo {
+ LIPFilterProbeInfo(const expressions::AttributeReferencePtr &probe_attribute_in,
+ const expressions::AttributeReferencePtr &build_attribute_in,
+ const PhysicalPtr &builder_in)
+ : probe_attribute(probe_attribute_in),
+ build_attribute(build_attribute_in),
+ builder(builder_in) {
+ }
+ expressions::AttributeReferencePtr probe_attribute;
+ PhysicalPtr target;
+ expressions::AttributeReferencePtr build_attribute;
+ PhysicalPtr builder;
+};
+
+
+class LIPFilterConfiguration;
+typedef std::shared_ptr<const LIPFilterConfiguration> LIPFilterConfigurationPtr;
+
+class LIPFilterConfiguration {
+ public:
+ LIPFilterConfiguration() {
+ }
+
+ void addBuildInfo(const expressions::AttributeReferencePtr &build_attribute,
+ const PhysicalPtr &builder,
+ const std::size_t filter_size,
+ const LIPFilterType &filter_type) {
+ build_info_[builder].emplace_back(
+ build_attribute, filter_size, filter_type);
+ }
+
+ void addProbeInfo(const expressions::AttributeReferencePtr &probe_attribute,
+ const PhysicalPtr &prober,
+ const expressions::AttributeReferencePtr &build_attribute,
+ const PhysicalPtr &builder) {
+ probe_info_[prober].emplace_back(
+ probe_attribute, build_attribute, builder);
+ }
+
+ const std::map<PhysicalPtr, std::vector<LIPFilterBuildInfo>>& getBuildInfo() const {
+ return build_info_;
+ }
+
+ const std::map<PhysicalPtr, std::vector<LIPFilterProbeInfo>>& getProbeInfo() const {
+ return probe_info_;
+ }
+
+ private:
+ std::map<PhysicalPtr, std::vector<LIPFilterBuildInfo>> build_info_;
+ std::map<PhysicalPtr, std::vector<LIPFilterProbeInfo>> probe_info_;
+
+ DISALLOW_COPY_AND_ASSIGN(LIPFilterConfiguration);
+};
+
+/** @} */
+
+} // namespace physical
+} // namespace optimizer
+} // namespace quickstep
+
+#endif /* QUICKSTEP_QUERY_OPTIMIZER_PHYSICAL_LIP_FILTER_CONFIGURATION_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/query_optimizer/physical/TopLevelPlan.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/TopLevelPlan.hpp b/query_optimizer/physical/TopLevelPlan.hpp
index 8f07dec..c6ea940 100644
--- a/query_optimizer/physical/TopLevelPlan.hpp
+++ b/query_optimizer/physical/TopLevelPlan.hpp
@@ -29,6 +29,7 @@
#include "query_optimizer/expressions/AttributeReference.hpp"
#include "query_optimizer/expressions/ExprId.hpp"
#include "query_optimizer/expressions/ExpressionUtil.hpp"
+#include "query_optimizer/physical/LIPFilterConfiguration.hpp"
#include "query_optimizer/physical/Physical.hpp"
#include "query_optimizer/physical/PhysicalType.hpp"
#include "utility/Macros.hpp"
@@ -120,6 +121,18 @@ class TopLevelPlan : public Physical {
return false;
}
+ TopLevelPlanPtr copyWithLIPFilterConfiguration(
+ const LIPFilterConfigurationPtr &new_lip_filter_configuration) const {
+ return TopLevelPlan::Create(plan_,
+ shared_subplans_,
+ uncorrelated_subquery_map_,
+ new_lip_filter_configuration);
+ }
+
+ const LIPFilterConfigurationPtr& lip_filter_configuration() const {
+ return lip_filter_configuration_;
+ }
+
/**
* @brief Creates a TopLevelPlan.
*
@@ -133,10 +146,12 @@ class TopLevelPlan : public Physical {
const PhysicalPtr &plan,
const std::vector<PhysicalPtr> &shared_subplans = {},
const std::unordered_map<expressions::ExprId, int> &uncorrelated_subquery_map
- = std::unordered_map<expressions::ExprId, int>()) {
+ = std::unordered_map<expressions::ExprId, int>(),
+ const LIPFilterConfigurationPtr &lip_filter_configuration = nullptr) {
return TopLevelPlanPtr(new TopLevelPlan(plan,
shared_subplans,
- uncorrelated_subquery_map));
+ uncorrelated_subquery_map,
+ lip_filter_configuration));
}
protected:
@@ -151,10 +166,12 @@ class TopLevelPlan : public Physical {
private:
TopLevelPlan(const PhysicalPtr &plan,
const std::vector<PhysicalPtr> &shared_subplans,
- const std::unordered_map<expressions::ExprId, int> &uncorrelated_subquery_map)
+ const std::unordered_map<expressions::ExprId, int> &uncorrelated_subquery_map,
+ const LIPFilterConfigurationPtr &lip_filter_configuration)
: plan_(plan),
shared_subplans_(shared_subplans),
- uncorrelated_subquery_map_(uncorrelated_subquery_map) {
+ uncorrelated_subquery_map_(uncorrelated_subquery_map),
+ lip_filter_configuration_(lip_filter_configuration) {
addChild(plan);
for (const PhysicalPtr &shared_subplan : shared_subplans) {
addChild(shared_subplan);
@@ -165,6 +182,7 @@ class TopLevelPlan : public Physical {
// Stored in the topological ordering based on dependencies.
std::vector<PhysicalPtr> shared_subplans_;
std::unordered_map<expressions::ExprId, int> uncorrelated_subquery_map_;
+ LIPFilterConfigurationPtr lip_filter_configuration_;
DISALLOW_COPY_AND_ASSIGN(TopLevelPlan);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/query_optimizer/rules/AttachLIPFilters.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/AttachLIPFilters.cpp b/query_optimizer/rules/AttachLIPFilters.cpp
new file mode 100644
index 0000000..5493e00
--- /dev/null
+++ b/query_optimizer/rules/AttachLIPFilters.cpp
@@ -0,0 +1,259 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "query_optimizer/rules/AttachLIPFilters.hpp"
+
+#include <algorithm>
+#include <iterator>
+#include <memory>
+#include <set>
+#include <stack>
+#include <unordered_set>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/expressions/NamedExpression.hpp"
+#include "query_optimizer/expressions/PatternMatcher.hpp"
+#include "query_optimizer/physical/LIPFilterConfiguration.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/PatternMatcher.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/PhysicalType.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+#include "utility/LIPFilter.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+
+namespace E = ::quickstep::optimizer::expressions;
+namespace P = ::quickstep::optimizer::physical;
+
+P::PhysicalPtr AttachLIPFilters::apply(const P::PhysicalPtr &input) {
+ DCHECK(input->getPhysicalType() == P::PhysicalType::kTopLevelPlan);
+ const P::TopLevelPlanPtr top_level_plan =
+ std::static_pointer_cast<const P::TopLevelPlan>(input);
+ cost_model_.reset(
+ new cost::StarSchemaSimpleCostModel(
+ top_level_plan->shared_subplans()));
+ lip_filter_configuration_.reset(new P::LIPFilterConfiguration());
+
+ std::set<E::ExprId> already_filtered_attributes;
+ attachLIPFilters(NodeList(input), &already_filtered_attributes);
+
+// for (const auto &pair : attaches_) {
+// for (const auto &build_info : pair.second->getBuildInfo()) {
+// std::cout << "Build: " << build_info.build_attribute->attribute_alias() << "\n";
+// }
+// for (const auto &probe_info : pair.second->getProbeInfo()) {
+// std::cout << "Probe: " << probe_info.probe_attribute->attribute_alias() << "\n";
+// }
+// }
+
+ P::PhysicalPtr output;
+ if (!lip_filter_configuration_->getBuildInfo().empty() ||
+ !lip_filter_configuration_->getProbeInfo().empty()) {
+ output = top_level_plan->copyWithLIPFilterConfiguration(
+ P::LIPFilterConfigurationPtr(lip_filter_configuration_.release()));
+ } else {
+ output = input;
+ }
+ return output;
+}
+
+void AttachLIPFilters::attachLIPFilters(
+ const NodeList &path,
+ std::set<expressions::ExprId> *already_filtered_attributes) {
+ const P::PhysicalPtr &node = path.node;
+
+ // First process child nodes
+ for (const auto &child : node->children()) {
+ std::set<E::ExprId> child_filtered_attributes;
+ attachLIPFilters(path.cons(child), &child_filtered_attributes);
+ already_filtered_attributes->insert(child_filtered_attributes.begin(),
+ child_filtered_attributes.end());
+ }
+
+ // Attach LIP filters to HashJoin/Selection/Aggregate nodes
+ P::PhysicalPtr probe_child = nullptr;
+ switch (node->getPhysicalType()) {
+ case P::PhysicalType::kHashJoin:
+ probe_child = std::static_pointer_cast<const P::HashJoin>(node)->left();
+ break;
+ case P::PhysicalType::kSelection:
+ probe_child = std::static_pointer_cast<const P::Selection>(node)->input();
+ break;
+ case P::PhysicalType::kAggregate:
+ probe_child = std::static_pointer_cast<const P::Aggregate>(node)->input();
+ break;
+ default:
+ break;
+ }
+
+ if (probe_child != nullptr &&
+ cost_model_->estimateCardinality(probe_child) > 10000000) {
+ const auto &candidate_lip_filters = getProbeSideInfo(path.cons(probe_child));
+ if (!candidate_lip_filters.empty()) {
+ std::map<E::AttributeReferencePtr, LIPFilterInfoPtr> selected_filters;
+ for (const auto &info : candidate_lip_filters) {
+ auto it = selected_filters.find(info->attribute);
+ if (it == selected_filters.end()) {
+ selected_filters.emplace(info->attribute, info);
+ } else if (LIPFilterInfo::isBetterThan(*info, *it->second)) {
+ it->second = info;
+ }
+ }
+
+ for (const auto &pair : selected_filters) {
+ const E::ExprId source_attr_id = pair.second->source_attribute->id();
+ if (already_filtered_attributes->find(source_attr_id)
+ == already_filtered_attributes->end()) {
+ lip_filter_configuration_->addBuildInfo(
+ pair.second->source_attribute,
+ pair.second->source,
+ pair.second->estimated_cardinality * 8,
+ LIPFilterType::kSingleIdentityHashFilter);
+ lip_filter_configuration_->addProbeInfo(
+ pair.first,
+ node,
+ pair.second->source_attribute,
+ pair.second->source);
+ already_filtered_attributes->emplace(source_attr_id);
+ }
+ }
+ }
+ }
+}
+
+const std::vector<AttachLIPFilters::LIPFilterInfoPtr>& AttachLIPFilters
+ ::getBuildSideInfo(const NodeList &path) {
+ const P::PhysicalPtr &node = path.node;
+ if (build_side_info_.find(node) == build_side_info_.end()) {
+ std::vector<LIPFilterInfoPtr> lip_filters;
+
+ // 1. Gather candidate LIP filters propagated from descendant nodes.
+ std::unordered_set<E::ExprId> output_attribute_ids;
+ for (const auto &attr : node->getOutputAttributes()) {
+ output_attribute_ids.emplace(attr->id());
+ }
+ switch (node->getPhysicalType()) {
+ case P::PhysicalType::kAggregate:
+ case P::PhysicalType::kSelection:
+ case P::PhysicalType::kHashJoin: {
+ for (const P::PhysicalPtr &child : node->children()) {
+ for (const LIPFilterInfoPtr &info : getBuildSideInfo(path.cons(child))) {
+ lip_filters.emplace_back(info);
+ }
+ }
+ break;
+ }
+ default:
+ break;
+ }
+
+ // 2. Consider the parent physical node. If it is a HashJoin,
+ // then each build-side join attribute is a candidate LIP filter
+ // which can be built by the BuildHashOperator that corresponds
+ // to the parent HashJoin node.
+ P::HashJoinPtr hash_join;
+ if (path.cdr() != nullptr &&
+ P::SomeHashJoin::MatchesWithConditionalCast(path.cdr()->node, &hash_join)) {
+ const P::PhysicalPtr &build_node = hash_join->right();
+ // TODO(jianqiao): consider probe-side info to allow cascading propagation.
+ double selectivity = cost_model_->estimateSelectivity(build_node);
+ // Only consider attributes that are selective.
+ if (selectivity < 1.0) {
+ std::size_t cardinality = cost_model_->estimateCardinality(build_node);
+ for (const auto &attr : hash_join->right_join_attributes()) {
+ lip_filters.emplace_back(
+ std::make_shared<LIPFilterInfo>(attr,
+ path.cdr()->node,
+ path.depth,
+ selectivity,
+ cardinality));
+ }
+ }
+ }
+ build_side_info_.emplace(node, std::move(lip_filters));
+ }
+ return build_side_info_.at(node);
+}
+
+const std::vector<AttachLIPFilters::LIPFilterInfoPtr>& AttachLIPFilters
+ ::getProbeSideInfo(const NodeList &path) {
+ const P::PhysicalPtr &node = path.node;
+ if (probe_side_info_.find(node) == probe_side_info_.end()) {
+ std::vector<LIPFilterInfoPtr> lip_filters;
+ if (path.cdr() != nullptr) {
+ // 1. Gather candidate LIP filters propagated from ancestor nodes.
+ const auto &parent_lip_filters = getProbeSideInfo(*path.cdr());
+ if (!parent_lip_filters.empty()) {
+ std::unordered_set<E::ExprId> output_attribute_ids;
+ for (const auto &attr : node->getOutputAttributes()) {
+ output_attribute_ids.emplace(attr->id());
+ }
+ for (const auto &info : parent_lip_filters) {
+ if (output_attribute_ids.find(info->attribute->id()) != output_attribute_ids.end()) {
+ lip_filters.emplace_back(info);
+ }
+ }
+ }
+
+ // 2. Consider the parent physical node. If it is an InnerHashJoin or
+ // LeftSemiHashJoin, then we can propagate the build-side LIP filters
+ // to the probe-side.
+ P::HashJoinPtr hash_join;
+ if (P::SomeHashJoin::MatchesWithConditionalCast(path.cdr()->node, &hash_join) &&
+ (hash_join->join_type() == P::HashJoin::JoinType::kInnerJoin ||
+ hash_join->join_type() == P::HashJoin::JoinType::kLeftSemiJoin)) {
+ const P::PhysicalPtr &build_side_child = hash_join->right();
+ std::unordered_map<E::ExprId, E::AttributeReferencePtr> join_attribute_pairs;
+ for (std::size_t i = 0; i < hash_join->left_join_attributes().size(); ++i) {
+ const E::AttributeReferencePtr probe_side_join_attribute =
+ hash_join->left_join_attributes()[i];
+ const E::AttributeReferencePtr build_side_join_attribute =
+ hash_join->right_join_attributes()[i];
+ join_attribute_pairs.emplace(build_side_join_attribute->id(),
+ probe_side_join_attribute);
+ }
+ for (const auto &info : getBuildSideInfo(path.cdr()->cons(build_side_child))) {
+ const auto pair_it = join_attribute_pairs.find(info->attribute->id());
+ if (pair_it != join_attribute_pairs.end()) {
+ lip_filters.emplace_back(
+ std::make_shared<LIPFilterInfo>(pair_it->second,
+ info->source,
+ info->depth,
+ info->estimated_selectivity,
+ info->estimated_cardinality,
+ info->attribute));
+ }
+ }
+ }
+ }
+ probe_side_info_.emplace(node, std::move(lip_filters));
+ }
+ return probe_side_info_.at(node);
+}
+
+} // namespace optimizer
+} // namespace quickstep
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/query_optimizer/rules/AttachLIPFilters.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/AttachLIPFilters.hpp b/query_optimizer/rules/AttachLIPFilters.hpp
new file mode 100644
index 0000000..d0a6fa5
--- /dev/null
+++ b/query_optimizer/rules/AttachLIPFilters.hpp
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_OPTIMIZER_RULES_ATTACH_LIP_FILTERS_HPP_
+#define QUICKSTEP_QUERY_OPTIMIZER_RULES_ATTACH_LIP_FILTERS_HPP_
+
+#include <cstddef>
+#include <map>
+#include <memory>
+#include <set>
+#include <stack>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/expressions/ExprId.hpp"
+#include "query_optimizer/expressions/NamedExpression.hpp"
+#include "query_optimizer/expressions/Predicate.hpp"
+#include "query_optimizer/physical/LIPFilterConfiguration.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/rules/Rule.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+namespace optimizer {
+
+/** \addtogroup OptimizerRules
+ * @{
+ */
+
+class AttachLIPFilters : public Rule<physical::Physical> {
+ public:
+ AttachLIPFilters() {}
+
+ ~AttachLIPFilters() override {}
+
+ std::string getName() const override {
+ return "AttachLIPFilters";
+ }
+
+ physical::PhysicalPtr apply(const physical::PhysicalPtr &input) override;
+
+ private:
+ struct LIPFilterInfo {
+ LIPFilterInfo(const expressions::AttributeReferencePtr &attribute_in,
+ const physical::PhysicalPtr &source_in,
+ const int depth_in,
+ const double estimated_selectivity_in,
+ const std::size_t estimated_cardinality_in,
+ const expressions::AttributeReferencePtr &source_attribute_in = nullptr)
+ : attribute(attribute_in),
+ source(source_in),
+ depth(depth_in),
+ estimated_selectivity(estimated_selectivity_in),
+ estimated_cardinality(estimated_cardinality_in),
+ source_attribute(
+ source_attribute_in == nullptr
+ ? attribute_in
+ : source_attribute_in) {
+
+ }
+ static bool isBetterThan(const LIPFilterInfo &a, const LIPFilterInfo &b) {
+ if (a.estimated_selectivity == b.estimated_selectivity) {
+ return a.depth > b.depth;
+ } else {
+ return a.estimated_selectivity < b.estimated_selectivity;
+ }
+ }
+ expressions::AttributeReferencePtr attribute;
+ physical::PhysicalPtr source;
+ int depth;
+ double estimated_selectivity;
+ std::size_t estimated_cardinality;
+ expressions::AttributeReferencePtr source_attribute;
+ };
+
+ typedef std::shared_ptr<const LIPFilterInfo> LIPFilterInfoPtr;
+
+ struct NodeList {
+ NodeList(const physical::PhysicalPtr &node_in)
+ : node(node_in),
+ next(nullptr),
+ depth(0) {
+ }
+ NodeList(const physical::PhysicalPtr &node_in,
+ const NodeList *next_in,
+ const int depth_in)
+ : node(node_in),
+ next(next_in),
+ depth(depth_in) {
+ }
+ inline const NodeList *cdr() const {
+ return next;
+ }
+ inline const NodeList cons(const physical::PhysicalPtr &new_node) const {
+ return NodeList(new_node, this, depth+1);
+ }
+ const physical::PhysicalPtr node;
+ const NodeList *next;
+ const int depth;
+ };
+
+ void attachLIPFilters(const NodeList &path,
+ std::set<expressions::ExprId> *already_filtered_attributes);
+
+ const std::vector<LIPFilterInfoPtr>& getBuildSideInfo(const NodeList &path);
+
+ const std::vector<LIPFilterInfoPtr>& getProbeSideInfo(const NodeList &path);
+
+ std::unique_ptr<cost::StarSchemaSimpleCostModel> cost_model_;
+ std::map<physical::PhysicalPtr, std::vector<LIPFilterInfoPtr>> build_side_info_;
+ std::map<physical::PhysicalPtr, std::vector<LIPFilterInfoPtr>> probe_side_info_;
+ std::unique_ptr<physical::LIPFilterConfiguration> lip_filter_configuration_;
+
+ DISALLOW_COPY_AND_ASSIGN(AttachLIPFilters);
+};
+
+/** @} */
+
+} // namespace optimizer
+} // namespace quickstep
+
+#endif /* QUICKSTEP_QUERY_OPTIMIZER_RULES_ATTACH_LIP_FILTERS_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/query_optimizer/rules/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/CMakeLists.txt b/query_optimizer/rules/CMakeLists.txt
index d9709ce..9dc6118 100644
--- a/query_optimizer/rules/CMakeLists.txt
+++ b/query_optimizer/rules/CMakeLists.txt
@@ -18,6 +18,7 @@
add_subdirectory(tests)
# Declare micro-libs:
+add_library(quickstep_queryoptimizer_rules_AttachLIPFilters AttachLIPFilters.cpp AttachLIPFilters.hpp)
add_library(quickstep_queryoptimizer_rules_BottomUpRule ../../empty_src.cpp BottomUpRule.hpp)
add_library(quickstep_queryoptimizer_rules_CollapseProject CollapseProject.cpp CollapseProject.hpp)
add_library(quickstep_queryoptimizer_rules_GenerateJoins GenerateJoins.cpp GenerateJoins.hpp)
@@ -36,6 +37,20 @@ add_library(quickstep_queryoptimizer_rules_UnnestSubqueries UnnestSubqueries.cpp
# Link dependencies:
+target_link_libraries(quickstep_queryoptimizer_rules_AttachLIPFilters
+ quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_expressions_ExprId
+ quickstep_queryoptimizer_expressions_NamedExpression
+ quickstep_queryoptimizer_expressions_PatternMatcher
+ quickstep_queryoptimizer_expressions_Predicate
+ quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_PatternMatcher
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_queryoptimizer_physical_TopLevelPlan
+ quickstep_queryoptimizer_rules_Rule
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_rules_BottomUpRule
glog
quickstep_queryoptimizer_rules_Rule
@@ -187,6 +202,7 @@ target_link_libraries(quickstep_queryoptimizer_rules_UpdateExpression
# Module all-in-one library:
add_library(quickstep_queryoptimizer_rules ../../empty_src.cpp OptimizerRulesModule.hpp)
target_link_libraries(quickstep_queryoptimizer_rules
+ quickstep_queryoptimizer_rules_AttachLIPFilters
quickstep_queryoptimizer_rules_BottomUpRule
quickstep_queryoptimizer_rules_CollapseProject
quickstep_queryoptimizer_rules_GenerateJoins
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index ddaae45..8a2c7ae 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -174,6 +174,7 @@ add_library(quickstep_utility_ExecutionDAGVisualizer
ExecutionDAGVisualizer.hpp)
add_library(quickstep_utility_Glob Glob.cpp Glob.hpp)
add_library(quickstep_utility_HashPair ../empty_src.cpp HashPair.hpp)
+add_library(quickstep_utility_LIPFilter LIPFilter.cpp LIPFilter.hpp)
add_library(quickstep_utility_Macros ../empty_src.cpp Macros.hpp)
add_library(quickstep_utility_MemStream ../empty_src.cpp MemStream.hpp)
add_library(quickstep_utility_PlanVisualizer PlanVisualizer.cpp PlanVisualizer.hpp)
@@ -244,6 +245,8 @@ target_link_libraries(quickstep_utility_ExecutionDAGVisualizer
quickstep_utility_StringUtil)
target_link_libraries(quickstep_utility_Glob
glog)
+target_link_libraries(quickstep_utility_LIPFilter
+ glog)
target_link_libraries(quickstep_utility_MemStream
glog
quickstep_utility_Macros)
@@ -323,6 +326,7 @@ target_link_libraries(quickstep_utility
quickstep_utility_ExecutionDAGVisualizer
quickstep_utility_Glob
quickstep_utility_HashPair
+ quickstep_utility_LIPFilter
quickstep_utility_Macros
quickstep_utility_MemStream
quickstep_utility_PlanVisualizer
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/utility/LIPFilter.cpp
----------------------------------------------------------------------
diff --git a/utility/LIPFilter.cpp b/utility/LIPFilter.cpp
new file mode 100644
index 0000000..f503f4f
--- /dev/null
+++ b/utility/LIPFilter.cpp
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "utility/LIPFilter.hpp"
+
+namespace quickstep {
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/utility/LIPFilter.hpp
----------------------------------------------------------------------
diff --git a/utility/LIPFilter.hpp b/utility/LIPFilter.hpp
new file mode 100644
index 0000000..12a19d7
--- /dev/null
+++ b/utility/LIPFilter.hpp
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_UTILITY_LIP_FILTER_HPP_
+#define QUICKSTEP_UTILITY_LIP_FILTER_HPP_
+
+#include <vector>
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+/** \addtogroup Utility
+ * @{
+ */
+
+enum class LIPFilterType {
+ kBloomFilter,
+ kExactFilter,
+ kSingleIdentityHashFilter
+};
+
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_UTILITY_LIP_FILTER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/utility/LIPFilterAdapter.hpp
----------------------------------------------------------------------
diff --git a/utility/LIPFilterAdapter.hpp b/utility/LIPFilterAdapter.hpp
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/utility/LIPFilterBuilder.hpp
----------------------------------------------------------------------
diff --git a/utility/LIPFilterBuilder.hpp b/utility/LIPFilterBuilder.hpp
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/utility/PlanVisualizer.cpp
----------------------------------------------------------------------
diff --git a/utility/PlanVisualizer.cpp b/utility/PlanVisualizer.cpp
index 50cf7f0..fdb5812 100644
--- a/utility/PlanVisualizer.cpp
+++ b/utility/PlanVisualizer.cpp
@@ -21,18 +21,22 @@
#include <cstddef>
#include <memory>
+#include <set>
#include <sstream>
#include <string>
#include <unordered_map>
#include <vector>
#include "catalog/CatalogRelation.hpp"
-
+#include "catalog/CatalogRelationStatistics.hpp"
+#include "catalog/CatalogTypedefs.hpp"
#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/physical/Aggregate.hpp"
#include "query_optimizer/physical/HashJoin.hpp"
#include "query_optimizer/physical/Physical.hpp"
#include "query_optimizer/physical/PhysicalType.hpp"
+#include "query_optimizer/physical/Selection.hpp"
#include "query_optimizer/physical/TableReference.hpp"
#include "query_optimizer/physical/TopLevelPlan.hpp"
#include "utility/StringUtil.hpp"
@@ -47,9 +51,12 @@ namespace C = ::quickstep::optimizer::cost;
std::string PlanVisualizer::visualize(const P::PhysicalPtr &input) {
DCHECK(input->getPhysicalType() == P::PhysicalType::kTopLevelPlan);
+ const P::TopLevelPlanPtr top_level_plan =
+ std::static_pointer_cast<const P::TopLevelPlan>(input);
cost_model_.reset(
new C::StarSchemaSimpleCostModel(
- std::static_pointer_cast<const P::TopLevelPlan>(input)->shared_subplans()));
+ top_level_plan->shared_subplans()));
+ lip_filter_conf_ = top_level_plan->lip_filter_configuration();
color_map_["TableReference"] = "skyblue";
color_map_["Selection"] = "#90EE90";
@@ -86,6 +93,9 @@ std::string PlanVisualizer::visualize(const P::PhysicalPtr &input) {
for (const EdgeInfo &edge_info : edges_) {
graph_oss << " " << edge_info.src_node_id << " -> "
<< edge_info.dst_node_id << " [";
+ if (edge_info.dashed) {
+ graph_oss << "style=dashed ";
+ }
if (!edge_info.labels.empty()) {
graph_oss << "label=\""
<< EscapeSpecialChars(JoinToString(edge_info.labels, " "))
@@ -103,6 +113,10 @@ void PlanVisualizer::visit(const P::PhysicalPtr &input) {
int node_id = ++id_counter_;
node_id_map_.emplace(input, node_id);
+ std::set<E::ExprId> referenced_ids;
+ for (const auto &attr : input->getReferencedAttributes()) {
+ referenced_ids.emplace(attr->id());
+ }
for (const auto &child : input->children()) {
visit(child);
@@ -112,13 +126,11 @@ void PlanVisualizer::visit(const P::PhysicalPtr &input) {
EdgeInfo &edge_info = edges_.back();
edge_info.src_node_id = child_id;
edge_info.dst_node_id = node_id;
+ edge_info.dashed = false;
- // Print output attributes except for TableReference -- there are just too many
- // attributes out of TableReference.
- if (child->getPhysicalType() != P::PhysicalType::kTableReference) {
- for (const auto &attr : child->getOutputAttributes()) {
- edge_info.labels.emplace_back(attr->attribute_alias());
- }
+ if (input->getPhysicalType() == P::PhysicalType::kHashJoin &&
+ child == input->children()[1]) {
+ edge_info.dashed = true;
}
}
@@ -154,6 +166,26 @@ void PlanVisualizer::visit(const P::PhysicalPtr &input) {
break;
}
}
+
+ if (lip_filter_conf_ != nullptr) {
+ const auto &build_filters = lip_filter_conf_->getBuildInfo();
+ const auto build_it = build_filters.find(input);
+ if (build_it != build_filters.end()) {
+ for (const auto &build_info : build_it->second) {
+ node_info.labels.emplace_back(
+ std::string("[LIP build] ") + build_info.build_attribute->attribute_alias());
+ }
+ }
+ const auto &probe_filters = lip_filter_conf_->getProbeInfo();
+ const auto probe_it = probe_filters.find(input);
+ if (probe_it != probe_filters.end()) {
+ for (const auto &probe_info : probe_it->second) {
+ node_info.labels.emplace_back(
+ std::string("[LIP probe] ") + probe_info.probe_attribute->attribute_alias());
+ }
+ }
+ }
+
node_info.labels.emplace_back(
"est. # = " + std::to_string(cost_model_->estimateCardinality(input)));
node_info.labels.emplace_back(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18e72095/utility/PlanVisualizer.hpp
----------------------------------------------------------------------
diff --git a/utility/PlanVisualizer.hpp b/utility/PlanVisualizer.hpp
index 1c0df77..9b8b0db 100644
--- a/utility/PlanVisualizer.hpp
+++ b/utility/PlanVisualizer.hpp
@@ -26,6 +26,7 @@
#include <vector>
#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/physical/LIPFilterConfiguration.hpp"
#include "query_optimizer/physical/Physical.hpp"
#include "utility/Macros.hpp"
@@ -73,6 +74,7 @@ class PlanVisualizer {
int src_node_id;
int dst_node_id;
std::vector<std::string> labels;
+ bool dashed;
};
void visit(const optimizer::physical::PhysicalPtr &input);
@@ -85,6 +87,7 @@ class PlanVisualizer {
std::vector<EdgeInfo> edges_;
std::unique_ptr<optimizer::cost::StarSchemaSimpleCostModel> cost_model_;
+ optimizer::physical::LIPFilterConfigurationPtr lip_filter_conf_;
DISALLOW_COPY_AND_ASSIGN(PlanVisualizer);
};
[11/17] incubator-quickstep git commit: Checking in preprocessed
files for datetime change.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0820e3d9/parser/preprocessed/SqlParser_gen.hpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlParser_gen.hpp b/parser/preprocessed/SqlParser_gen.hpp
index 0f66d1d..fe963be 100644
--- a/parser/preprocessed/SqlParser_gen.hpp
+++ b/parser/preprocessed/SqlParser_gen.hpp
@@ -1,19 +1,19 @@
-/* A Bison parser, made by GNU Bison 2.7. */
+/* A Bison parser, made by GNU Bison 3.0.2. */
/* Bison interface for Yacc-like parsers in C
-
- Copyright (C) 1984, 1989-1990, 2000-2012 Free Software Foundation, Inc.
-
+
+ Copyright (C) 1984, 1989-1990, 2000-2013 Free Software Foundation, Inc.
+
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
-
+
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
-
+
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>. */
@@ -26,13 +26,13 @@
special exception, which will cause the skeleton and the resulting
Bison output files to be licensed under the GNU General Public
License without this special exception.
-
+
This special exception was added by the Free Software Foundation in
version 2.2 of Bison. */
#ifndef YY_QUICKSTEP_YY_SQLPARSER_GEN_HPP_INCLUDED
# define YY_QUICKSTEP_YY_SQLPARSER_GEN_HPP_INCLUDED
-/* Enabling traces. */
+/* Debug traces. */
#ifndef YYDEBUG
# define YYDEBUG 0
#endif
@@ -40,152 +40,151 @@
extern int quickstep_yydebug;
#endif
-/* Tokens. */
+/* Token type. */
#ifndef YYTOKENTYPE
# define YYTOKENTYPE
- /* Put the tokens into the symbol table, so that GDB and other debuggers
- know about them. */
- enum yytokentype {
- TOKEN_COMMAND = 258,
- TOKEN_NAME = 259,
- TOKEN_STRING_SINGLE_QUOTED = 260,
- TOKEN_STRING_DOUBLE_QUOTED = 261,
- TOKEN_UNSIGNED_NUMVAL = 262,
- TOKEN_OR = 263,
- TOKEN_AND = 264,
- TOKEN_NOT = 265,
- TOKEN_EQ = 266,
- TOKEN_NEQ = 267,
- TOKEN_GEQ = 268,
- TOKEN_GT = 269,
- TOKEN_LEQ = 270,
- TOKEN_LT = 271,
- TOKEN_REGEXP = 272,
- TOKEN_LIKE = 273,
- TOKEN_BETWEEN = 274,
- TOKEN_IS = 275,
- UNARY_MINUS = 276,
- UNARY_PLUS = 277,
- TOKEN_ADD = 278,
- TOKEN_ALL = 279,
- TOKEN_ALTER = 280,
- TOKEN_AS = 281,
- TOKEN_ASC = 282,
- TOKEN_BIGINT = 283,
- TOKEN_BIT = 284,
- TOKEN_BITWEAVING = 285,
- TOKEN_BLOCKPROPERTIES = 286,
- TOKEN_BLOCKSAMPLE = 287,
- TOKEN_BLOOM_FILTER = 288,
- TOKEN_CSB_TREE = 289,
- TOKEN_BY = 290,
- TOKEN_CASE = 291,
- TOKEN_CHARACTER = 292,
- TOKEN_CHECK = 293,
- TOKEN_COLUMN = 294,
- TOKEN_CONSTRAINT = 295,
- TOKEN_COPY = 296,
- TOKEN_CREATE = 297,
- TOKEN_CURRENT = 298,
- TOKEN_DATE = 299,
- TOKEN_DATETIME = 300,
- TOKEN_DAY = 301,
- TOKEN_DECIMAL = 302,
- TOKEN_DEFAULT = 303,
- TOKEN_DELETE = 304,
- TOKEN_DELIMITER = 305,
- TOKEN_DESC = 306,
- TOKEN_DISTINCT = 307,
- TOKEN_DOUBLE = 308,
- TOKEN_DROP = 309,
- TOKEN_ELSE = 310,
- TOKEN_END = 311,
- TOKEN_ESCAPE_STRINGS = 312,
- TOKEN_EXISTS = 313,
- TOKEN_EXTRACT = 314,
- TOKEN_FALSE = 315,
- TOKEN_FIRST = 316,
- TOKEN_FLOAT = 317,
- TOKEN_FOLLOWING = 318,
- TOKEN_FOR = 319,
- TOKEN_FOREIGN = 320,
- TOKEN_FROM = 321,
- TOKEN_FULL = 322,
- TOKEN_GROUP = 323,
- TOKEN_HASH = 324,
- TOKEN_HAVING = 325,
- TOKEN_HOUR = 326,
- TOKEN_IN = 327,
- TOKEN_INDEX = 328,
- TOKEN_INNER = 329,
- TOKEN_INSERT = 330,
- TOKEN_INTEGER = 331,
- TOKEN_INTERVAL = 332,
- TOKEN_INTO = 333,
- TOKEN_JOIN = 334,
- TOKEN_KEY = 335,
- TOKEN_LAST = 336,
- TOKEN_LEFT = 337,
- TOKEN_LIMIT = 338,
- TOKEN_LONG = 339,
- TOKEN_MINUTE = 340,
- TOKEN_MONTH = 341,
- TOKEN_NULL = 342,
- TOKEN_NULLS = 343,
- TOKEN_OFF = 344,
- TOKEN_ON = 345,
- TOKEN_ORDER = 346,
- TOKEN_OUTER = 347,
- TOKEN_OVER = 348,
- TOKEN_PARTITION = 349,
- TOKEN_PARTITIONS = 350,
- TOKEN_PERCENT = 351,
- TOKEN_PRECEDING = 352,
- TOKEN_PRIMARY = 353,
- TOKEN_PRIORITY = 354,
- TOKEN_QUIT = 355,
- TOKEN_RANGE = 356,
- TOKEN_REAL = 357,
- TOKEN_REFERENCES = 358,
- TOKEN_RIGHT = 359,
- TOKEN_ROW = 360,
- TOKEN_ROW_DELIMITER = 361,
- TOKEN_ROWS = 362,
- TOKEN_SECOND = 363,
- TOKEN_SELECT = 364,
- TOKEN_SET = 365,
- TOKEN_SMA = 366,
- TOKEN_SMALLINT = 367,
- TOKEN_SUBSTRING = 368,
- TOKEN_TABLE = 369,
- TOKEN_THEN = 370,
- TOKEN_TIME = 371,
- TOKEN_TIMESTAMP = 372,
- TOKEN_TRUE = 373,
- TOKEN_TUPLESAMPLE = 374,
- TOKEN_UNBOUNDED = 375,
- TOKEN_UNIQUE = 376,
- TOKEN_UPDATE = 377,
- TOKEN_USING = 378,
- TOKEN_VALUES = 379,
- TOKEN_VARCHAR = 380,
- TOKEN_WHEN = 381,
- TOKEN_WHERE = 382,
- TOKEN_WINDOW = 383,
- TOKEN_WITH = 384,
- TOKEN_YEAR = 385,
- TOKEN_YEARMONTH = 386,
- TOKEN_EOF = 387,
- TOKEN_LEX_ERROR = 388
- };
+ enum yytokentype
+ {
+ TOKEN_COMMAND = 258,
+ TOKEN_NAME = 259,
+ TOKEN_STRING_SINGLE_QUOTED = 260,
+ TOKEN_STRING_DOUBLE_QUOTED = 261,
+ TOKEN_UNSIGNED_NUMVAL = 262,
+ TOKEN_OR = 263,
+ TOKEN_AND = 264,
+ TOKEN_NOT = 265,
+ TOKEN_EQ = 266,
+ TOKEN_LT = 267,
+ TOKEN_LEQ = 268,
+ TOKEN_GT = 269,
+ TOKEN_GEQ = 270,
+ TOKEN_NEQ = 271,
+ TOKEN_LIKE = 272,
+ TOKEN_REGEXP = 273,
+ TOKEN_BETWEEN = 274,
+ TOKEN_IS = 275,
+ UNARY_PLUS = 276,
+ UNARY_MINUS = 277,
+ TOKEN_ADD = 278,
+ TOKEN_ALL = 279,
+ TOKEN_ALTER = 280,
+ TOKEN_AS = 281,
+ TOKEN_ASC = 282,
+ TOKEN_BIGINT = 283,
+ TOKEN_BIT = 284,
+ TOKEN_BITWEAVING = 285,
+ TOKEN_BLOCKPROPERTIES = 286,
+ TOKEN_BLOCKSAMPLE = 287,
+ TOKEN_BLOOM_FILTER = 288,
+ TOKEN_CSB_TREE = 289,
+ TOKEN_BY = 290,
+ TOKEN_CASE = 291,
+ TOKEN_CHARACTER = 292,
+ TOKEN_CHECK = 293,
+ TOKEN_COLUMN = 294,
+ TOKEN_CONSTRAINT = 295,
+ TOKEN_COPY = 296,
+ TOKEN_CREATE = 297,
+ TOKEN_CURRENT = 298,
+ TOKEN_DATE = 299,
+ TOKEN_DATETIME = 300,
+ TOKEN_DAY = 301,
+ TOKEN_DECIMAL = 302,
+ TOKEN_DEFAULT = 303,
+ TOKEN_DELETE = 304,
+ TOKEN_DELIMITER = 305,
+ TOKEN_DESC = 306,
+ TOKEN_DISTINCT = 307,
+ TOKEN_DOUBLE = 308,
+ TOKEN_DROP = 309,
+ TOKEN_ELSE = 310,
+ TOKEN_END = 311,
+ TOKEN_ESCAPE_STRINGS = 312,
+ TOKEN_EXISTS = 313,
+ TOKEN_EXTRACT = 314,
+ TOKEN_FALSE = 315,
+ TOKEN_FIRST = 316,
+ TOKEN_FLOAT = 317,
+ TOKEN_FOLLOWING = 318,
+ TOKEN_FOR = 319,
+ TOKEN_FOREIGN = 320,
+ TOKEN_FROM = 321,
+ TOKEN_FULL = 322,
+ TOKEN_GROUP = 323,
+ TOKEN_HASH = 324,
+ TOKEN_HAVING = 325,
+ TOKEN_HOUR = 326,
+ TOKEN_IN = 327,
+ TOKEN_INDEX = 328,
+ TOKEN_INNER = 329,
+ TOKEN_INSERT = 330,
+ TOKEN_INTEGER = 331,
+ TOKEN_INTERVAL = 332,
+ TOKEN_INTO = 333,
+ TOKEN_JOIN = 334,
+ TOKEN_KEY = 335,
+ TOKEN_LAST = 336,
+ TOKEN_LEFT = 337,
+ TOKEN_LIMIT = 338,
+ TOKEN_LONG = 339,
+ TOKEN_MINUTE = 340,
+ TOKEN_MONTH = 341,
+ TOKEN_NULL = 342,
+ TOKEN_NULLS = 343,
+ TOKEN_OFF = 344,
+ TOKEN_ON = 345,
+ TOKEN_ORDER = 346,
+ TOKEN_OUTER = 347,
+ TOKEN_OVER = 348,
+ TOKEN_PARTITION = 349,
+ TOKEN_PARTITIONS = 350,
+ TOKEN_PERCENT = 351,
+ TOKEN_PRECEDING = 352,
+ TOKEN_PRIMARY = 353,
+ TOKEN_PRIORITY = 354,
+ TOKEN_QUIT = 355,
+ TOKEN_RANGE = 356,
+ TOKEN_REAL = 357,
+ TOKEN_REFERENCES = 358,
+ TOKEN_RIGHT = 359,
+ TOKEN_ROW = 360,
+ TOKEN_ROW_DELIMITER = 361,
+ TOKEN_ROWS = 362,
+ TOKEN_SECOND = 363,
+ TOKEN_SELECT = 364,
+ TOKEN_SET = 365,
+ TOKEN_SMA = 366,
+ TOKEN_SMALLINT = 367,
+ TOKEN_SUBSTRING = 368,
+ TOKEN_TABLE = 369,
+ TOKEN_THEN = 370,
+ TOKEN_TIME = 371,
+ TOKEN_TIMESTAMP = 372,
+ TOKEN_TRUE = 373,
+ TOKEN_TUPLESAMPLE = 374,
+ TOKEN_UNBOUNDED = 375,
+ TOKEN_UNIQUE = 376,
+ TOKEN_UPDATE = 377,
+ TOKEN_USING = 378,
+ TOKEN_VALUES = 379,
+ TOKEN_VARCHAR = 380,
+ TOKEN_WHEN = 381,
+ TOKEN_WHERE = 382,
+ TOKEN_WINDOW = 383,
+ TOKEN_WITH = 384,
+ TOKEN_YEAR = 385,
+ TOKEN_YEARMONTH = 386,
+ TOKEN_EOF = 387,
+ TOKEN_LEX_ERROR = 388
+ };
#endif
-
+/* Value type. */
#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
-typedef union YYSTYPE
+typedef union YYSTYPE YYSTYPE;
+union YYSTYPE
{
-/* Line 2058 of yacc.c */
-#line 120 "../SqlParser.ypp"
+#line 120 "../SqlParser.ypp" /* yacc.c:1909 */
quickstep::ParseString *string_value_;
@@ -285,41 +284,28 @@ typedef union YYSTYPE
quickstep::ParsePriority *opt_priority_clause_;
-
-/* Line 2058 of yacc.c */
-#line 291 "SqlParser_gen.hpp"
-} YYSTYPE;
+#line 288 "SqlParser_gen.hpp" /* yacc.c:1909 */
+};
# define YYSTYPE_IS_TRIVIAL 1
-# define yystype YYSTYPE /* obsolescent; will be withdrawn */
# define YYSTYPE_IS_DECLARED 1
#endif
+/* Location type. */
#if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED
-typedef struct YYLTYPE
+typedef struct YYLTYPE YYLTYPE;
+struct YYLTYPE
{
int first_line;
int first_column;
int last_line;
int last_column;
-} YYLTYPE;
-# define yyltype YYLTYPE /* obsolescent; will be withdrawn */
+};
# define YYLTYPE_IS_DECLARED 1
# define YYLTYPE_IS_TRIVIAL 1
#endif
-#ifdef YYPARSE_PARAM
-#if defined __STDC__ || defined __cplusplus
-int quickstep_yyparse (void *YYPARSE_PARAM);
-#else
-int quickstep_yyparse ();
-#endif
-#else /* ! YYPARSE_PARAM */
-#if defined __STDC__ || defined __cplusplus
+
int quickstep_yyparse (yyscan_t yyscanner, quickstep::ParseStatement **parsedStatement);
-#else
-int quickstep_yyparse ();
-#endif
-#endif /* ! YYPARSE_PARAM */
#endif /* !YY_QUICKSTEP_YY_SQLPARSER_GEN_HPP_INCLUDED */
[12/17] incubator-quickstep git commit: Checking in preprocessed
files for datetime change.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0820e3d9/parser/preprocessed/SqlParser_gen.cpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlParser_gen.cpp b/parser/preprocessed/SqlParser_gen.cpp
index 8fdf490..283bcbe 100644
--- a/parser/preprocessed/SqlParser_gen.cpp
+++ b/parser/preprocessed/SqlParser_gen.cpp
@@ -1,19 +1,19 @@
-/* A Bison parser, made by GNU Bison 2.7. */
+/* A Bison parser, made by GNU Bison 3.0.2. */
/* Bison implementation for Yacc-like parsers in C
-
- Copyright (C) 1984, 1989-1990, 2000-2012 Free Software Foundation, Inc.
-
+
+ Copyright (C) 1984, 1989-1990, 2000-2013 Free Software Foundation, Inc.
+
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
-
+
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
-
+
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>. */
@@ -26,7 +26,7 @@
special exception, which will cause the skeleton and the resulting
Bison output files to be licensed under the GNU General Public
License without this special exception.
-
+
This special exception was added by the Free Software Foundation in
version 2.2 of Bison. */
@@ -44,7 +44,7 @@
#define YYBISON 1
/* Bison version. */
-#define YYBISON_VERSION "2.7"
+#define YYBISON_VERSION "3.0.2"
/* Skeleton name. */
#define YYSKELETON_NAME "yacc.c"
@@ -63,15 +63,12 @@
#define yyparse quickstep_yyparse
#define yylex quickstep_yylex
#define yyerror quickstep_yyerror
-#define yylval quickstep_yylval
-#define yychar quickstep_yychar
#define yydebug quickstep_yydebug
#define yynerrs quickstep_yynerrs
-#define yylloc quickstep_yylloc
+
/* Copy the first part of user declarations. */
-/* Line 371 of yacc.c */
-#line 35 "../SqlParser.ypp"
+#line 35 "../SqlParser.ypp" /* yacc.c:339 */
/* Override the default definition, as we only need <first_line> and <first_column>. */
@@ -99,8 +96,7 @@ typedef struct YYLTYPE {
} \
} while (0)
-/* Line 371 of yacc.c */
-#line 64 "../SqlParser.ypp"
+#line 64 "../SqlParser.ypp" /* yacc.c:339 */
#include <cstdlib>
#include <string>
@@ -156,14 +152,13 @@ typedef struct YYLTYPE {
// Needed for Bison 2.6 and higher, which do not automatically provide this typedef.
typedef void* yyscan_t;
-/* Line 371 of yacc.c */
-#line 161 "SqlParser_gen.cpp"
+#line 156 "SqlParser_gen.cpp" /* yacc.c:339 */
-# ifndef YY_NULL
+# ifndef YY_NULLPTR
# if defined __cplusplus && 201103L <= __cplusplus
-# define YY_NULL nullptr
+# define YY_NULLPTR nullptr
# else
-# define YY_NULL 0
+# define YY_NULLPTR 0
# endif
# endif
@@ -179,7 +174,7 @@ typedef void* yyscan_t;
by #include "SqlParser_gen.hpp". */
#ifndef YY_QUICKSTEP_YY_SQLPARSER_GEN_HPP_INCLUDED
# define YY_QUICKSTEP_YY_SQLPARSER_GEN_HPP_INCLUDED
-/* Enabling traces. */
+/* Debug traces. */
#ifndef YYDEBUG
# define YYDEBUG 0
#endif
@@ -187,152 +182,151 @@ typedef void* yyscan_t;
extern int quickstep_yydebug;
#endif
-/* Tokens. */
+/* Token type. */
#ifndef YYTOKENTYPE
# define YYTOKENTYPE
- /* Put the tokens into the symbol table, so that GDB and other debuggers
- know about them. */
- enum yytokentype {
- TOKEN_COMMAND = 258,
- TOKEN_NAME = 259,
- TOKEN_STRING_SINGLE_QUOTED = 260,
- TOKEN_STRING_DOUBLE_QUOTED = 261,
- TOKEN_UNSIGNED_NUMVAL = 262,
- TOKEN_OR = 263,
- TOKEN_AND = 264,
- TOKEN_NOT = 265,
- TOKEN_EQ = 266,
- TOKEN_NEQ = 267,
- TOKEN_GEQ = 268,
- TOKEN_GT = 269,
- TOKEN_LEQ = 270,
- TOKEN_LT = 271,
- TOKEN_REGEXP = 272,
- TOKEN_LIKE = 273,
- TOKEN_BETWEEN = 274,
- TOKEN_IS = 275,
- UNARY_MINUS = 276,
- UNARY_PLUS = 277,
- TOKEN_ADD = 278,
- TOKEN_ALL = 279,
- TOKEN_ALTER = 280,
- TOKEN_AS = 281,
- TOKEN_ASC = 282,
- TOKEN_BIGINT = 283,
- TOKEN_BIT = 284,
- TOKEN_BITWEAVING = 285,
- TOKEN_BLOCKPROPERTIES = 286,
- TOKEN_BLOCKSAMPLE = 287,
- TOKEN_BLOOM_FILTER = 288,
- TOKEN_CSB_TREE = 289,
- TOKEN_BY = 290,
- TOKEN_CASE = 291,
- TOKEN_CHARACTER = 292,
- TOKEN_CHECK = 293,
- TOKEN_COLUMN = 294,
- TOKEN_CONSTRAINT = 295,
- TOKEN_COPY = 296,
- TOKEN_CREATE = 297,
- TOKEN_CURRENT = 298,
- TOKEN_DATE = 299,
- TOKEN_DATETIME = 300,
- TOKEN_DAY = 301,
- TOKEN_DECIMAL = 302,
- TOKEN_DEFAULT = 303,
- TOKEN_DELETE = 304,
- TOKEN_DELIMITER = 305,
- TOKEN_DESC = 306,
- TOKEN_DISTINCT = 307,
- TOKEN_DOUBLE = 308,
- TOKEN_DROP = 309,
- TOKEN_ELSE = 310,
- TOKEN_END = 311,
- TOKEN_ESCAPE_STRINGS = 312,
- TOKEN_EXISTS = 313,
- TOKEN_EXTRACT = 314,
- TOKEN_FALSE = 315,
- TOKEN_FIRST = 316,
- TOKEN_FLOAT = 317,
- TOKEN_FOLLOWING = 318,
- TOKEN_FOR = 319,
- TOKEN_FOREIGN = 320,
- TOKEN_FROM = 321,
- TOKEN_FULL = 322,
- TOKEN_GROUP = 323,
- TOKEN_HASH = 324,
- TOKEN_HAVING = 325,
- TOKEN_HOUR = 326,
- TOKEN_IN = 327,
- TOKEN_INDEX = 328,
- TOKEN_INNER = 329,
- TOKEN_INSERT = 330,
- TOKEN_INTEGER = 331,
- TOKEN_INTERVAL = 332,
- TOKEN_INTO = 333,
- TOKEN_JOIN = 334,
- TOKEN_KEY = 335,
- TOKEN_LAST = 336,
- TOKEN_LEFT = 337,
- TOKEN_LIMIT = 338,
- TOKEN_LONG = 339,
- TOKEN_MINUTE = 340,
- TOKEN_MONTH = 341,
- TOKEN_NULL = 342,
- TOKEN_NULLS = 343,
- TOKEN_OFF = 344,
- TOKEN_ON = 345,
- TOKEN_ORDER = 346,
- TOKEN_OUTER = 347,
- TOKEN_OVER = 348,
- TOKEN_PARTITION = 349,
- TOKEN_PARTITIONS = 350,
- TOKEN_PERCENT = 351,
- TOKEN_PRECEDING = 352,
- TOKEN_PRIMARY = 353,
- TOKEN_PRIORITY = 354,
- TOKEN_QUIT = 355,
- TOKEN_RANGE = 356,
- TOKEN_REAL = 357,
- TOKEN_REFERENCES = 358,
- TOKEN_RIGHT = 359,
- TOKEN_ROW = 360,
- TOKEN_ROW_DELIMITER = 361,
- TOKEN_ROWS = 362,
- TOKEN_SECOND = 363,
- TOKEN_SELECT = 364,
- TOKEN_SET = 365,
- TOKEN_SMA = 366,
- TOKEN_SMALLINT = 367,
- TOKEN_SUBSTRING = 368,
- TOKEN_TABLE = 369,
- TOKEN_THEN = 370,
- TOKEN_TIME = 371,
- TOKEN_TIMESTAMP = 372,
- TOKEN_TRUE = 373,
- TOKEN_TUPLESAMPLE = 374,
- TOKEN_UNBOUNDED = 375,
- TOKEN_UNIQUE = 376,
- TOKEN_UPDATE = 377,
- TOKEN_USING = 378,
- TOKEN_VALUES = 379,
- TOKEN_VARCHAR = 380,
- TOKEN_WHEN = 381,
- TOKEN_WHERE = 382,
- TOKEN_WINDOW = 383,
- TOKEN_WITH = 384,
- TOKEN_YEAR = 385,
- TOKEN_YEARMONTH = 386,
- TOKEN_EOF = 387,
- TOKEN_LEX_ERROR = 388
- };
+ enum yytokentype
+ {
+ TOKEN_COMMAND = 258,
+ TOKEN_NAME = 259,
+ TOKEN_STRING_SINGLE_QUOTED = 260,
+ TOKEN_STRING_DOUBLE_QUOTED = 261,
+ TOKEN_UNSIGNED_NUMVAL = 262,
+ TOKEN_OR = 263,
+ TOKEN_AND = 264,
+ TOKEN_NOT = 265,
+ TOKEN_EQ = 266,
+ TOKEN_LT = 267,
+ TOKEN_LEQ = 268,
+ TOKEN_GT = 269,
+ TOKEN_GEQ = 270,
+ TOKEN_NEQ = 271,
+ TOKEN_LIKE = 272,
+ TOKEN_REGEXP = 273,
+ TOKEN_BETWEEN = 274,
+ TOKEN_IS = 275,
+ UNARY_PLUS = 276,
+ UNARY_MINUS = 277,
+ TOKEN_ADD = 278,
+ TOKEN_ALL = 279,
+ TOKEN_ALTER = 280,
+ TOKEN_AS = 281,
+ TOKEN_ASC = 282,
+ TOKEN_BIGINT = 283,
+ TOKEN_BIT = 284,
+ TOKEN_BITWEAVING = 285,
+ TOKEN_BLOCKPROPERTIES = 286,
+ TOKEN_BLOCKSAMPLE = 287,
+ TOKEN_BLOOM_FILTER = 288,
+ TOKEN_CSB_TREE = 289,
+ TOKEN_BY = 290,
+ TOKEN_CASE = 291,
+ TOKEN_CHARACTER = 292,
+ TOKEN_CHECK = 293,
+ TOKEN_COLUMN = 294,
+ TOKEN_CONSTRAINT = 295,
+ TOKEN_COPY = 296,
+ TOKEN_CREATE = 297,
+ TOKEN_CURRENT = 298,
+ TOKEN_DATE = 299,
+ TOKEN_DATETIME = 300,
+ TOKEN_DAY = 301,
+ TOKEN_DECIMAL = 302,
+ TOKEN_DEFAULT = 303,
+ TOKEN_DELETE = 304,
+ TOKEN_DELIMITER = 305,
+ TOKEN_DESC = 306,
+ TOKEN_DISTINCT = 307,
+ TOKEN_DOUBLE = 308,
+ TOKEN_DROP = 309,
+ TOKEN_ELSE = 310,
+ TOKEN_END = 311,
+ TOKEN_ESCAPE_STRINGS = 312,
+ TOKEN_EXISTS = 313,
+ TOKEN_EXTRACT = 314,
+ TOKEN_FALSE = 315,
+ TOKEN_FIRST = 316,
+ TOKEN_FLOAT = 317,
+ TOKEN_FOLLOWING = 318,
+ TOKEN_FOR = 319,
+ TOKEN_FOREIGN = 320,
+ TOKEN_FROM = 321,
+ TOKEN_FULL = 322,
+ TOKEN_GROUP = 323,
+ TOKEN_HASH = 324,
+ TOKEN_HAVING = 325,
+ TOKEN_HOUR = 326,
+ TOKEN_IN = 327,
+ TOKEN_INDEX = 328,
+ TOKEN_INNER = 329,
+ TOKEN_INSERT = 330,
+ TOKEN_INTEGER = 331,
+ TOKEN_INTERVAL = 332,
+ TOKEN_INTO = 333,
+ TOKEN_JOIN = 334,
+ TOKEN_KEY = 335,
+ TOKEN_LAST = 336,
+ TOKEN_LEFT = 337,
+ TOKEN_LIMIT = 338,
+ TOKEN_LONG = 339,
+ TOKEN_MINUTE = 340,
+ TOKEN_MONTH = 341,
+ TOKEN_NULL = 342,
+ TOKEN_NULLS = 343,
+ TOKEN_OFF = 344,
+ TOKEN_ON = 345,
+ TOKEN_ORDER = 346,
+ TOKEN_OUTER = 347,
+ TOKEN_OVER = 348,
+ TOKEN_PARTITION = 349,
+ TOKEN_PARTITIONS = 350,
+ TOKEN_PERCENT = 351,
+ TOKEN_PRECEDING = 352,
+ TOKEN_PRIMARY = 353,
+ TOKEN_PRIORITY = 354,
+ TOKEN_QUIT = 355,
+ TOKEN_RANGE = 356,
+ TOKEN_REAL = 357,
+ TOKEN_REFERENCES = 358,
+ TOKEN_RIGHT = 359,
+ TOKEN_ROW = 360,
+ TOKEN_ROW_DELIMITER = 361,
+ TOKEN_ROWS = 362,
+ TOKEN_SECOND = 363,
+ TOKEN_SELECT = 364,
+ TOKEN_SET = 365,
+ TOKEN_SMA = 366,
+ TOKEN_SMALLINT = 367,
+ TOKEN_SUBSTRING = 368,
+ TOKEN_TABLE = 369,
+ TOKEN_THEN = 370,
+ TOKEN_TIME = 371,
+ TOKEN_TIMESTAMP = 372,
+ TOKEN_TRUE = 373,
+ TOKEN_TUPLESAMPLE = 374,
+ TOKEN_UNBOUNDED = 375,
+ TOKEN_UNIQUE = 376,
+ TOKEN_UPDATE = 377,
+ TOKEN_USING = 378,
+ TOKEN_VALUES = 379,
+ TOKEN_VARCHAR = 380,
+ TOKEN_WHEN = 381,
+ TOKEN_WHERE = 382,
+ TOKEN_WINDOW = 383,
+ TOKEN_WITH = 384,
+ TOKEN_YEAR = 385,
+ TOKEN_YEARMONTH = 386,
+ TOKEN_EOF = 387,
+ TOKEN_LEX_ERROR = 388
+ };
#endif
-
+/* Value type. */
#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
-typedef union YYSTYPE
+typedef union YYSTYPE YYSTYPE;
+union YYSTYPE
{
-/* Line 387 of yacc.c */
-#line 120 "../SqlParser.ypp"
+#line 120 "../SqlParser.ypp" /* yacc.c:355 */
quickstep::ParseString *string_value_;
@@ -432,55 +426,40 @@ typedef union YYSTYPE
quickstep::ParsePriority *opt_priority_clause_;
-
-/* Line 387 of yacc.c */
-#line 438 "SqlParser_gen.cpp"
-} YYSTYPE;
+#line 430 "SqlParser_gen.cpp" /* yacc.c:355 */
+};
# define YYSTYPE_IS_TRIVIAL 1
-# define yystype YYSTYPE /* obsolescent; will be withdrawn */
# define YYSTYPE_IS_DECLARED 1
#endif
+/* Location type. */
#if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED
-typedef struct YYLTYPE
+typedef struct YYLTYPE YYLTYPE;
+struct YYLTYPE
{
int first_line;
int first_column;
int last_line;
int last_column;
-} YYLTYPE;
-# define yyltype YYLTYPE /* obsolescent; will be withdrawn */
+};
# define YYLTYPE_IS_DECLARED 1
# define YYLTYPE_IS_TRIVIAL 1
#endif
-#ifdef YYPARSE_PARAM
-#if defined __STDC__ || defined __cplusplus
-int quickstep_yyparse (void *YYPARSE_PARAM);
-#else
-int quickstep_yyparse ();
-#endif
-#else /* ! YYPARSE_PARAM */
-#if defined __STDC__ || defined __cplusplus
+
int quickstep_yyparse (yyscan_t yyscanner, quickstep::ParseStatement **parsedStatement);
-#else
-int quickstep_yyparse ();
-#endif
-#endif /* ! YYPARSE_PARAM */
#endif /* !YY_QUICKSTEP_YY_SQLPARSER_GEN_HPP_INCLUDED */
/* Copy the second part of user declarations. */
-/* Line 390 of yacc.c */
-#line 220 "../SqlParser.ypp"
+#line 220 "../SqlParser.ypp" /* yacc.c:358 */
/* This header needs YYSTYPE, which is defined by the %union directive above */
#include "SqlLexer_gen.hpp"
void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const std::string &feature);
-/* Line 390 of yacc.c */
-#line 484 "SqlParser_gen.cpp"
+#line 463 "SqlParser_gen.cpp" /* yacc.c:358 */
#ifdef short
# undef short
@@ -494,11 +473,8 @@ typedef unsigned char yytype_uint8;
#ifdef YYTYPE_INT8
typedef YYTYPE_INT8 yytype_int8;
-#elif (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-typedef signed char yytype_int8;
#else
-typedef short int yytype_int8;
+typedef signed char yytype_int8;
#endif
#ifdef YYTYPE_UINT16
@@ -518,8 +494,7 @@ typedef short int yytype_int16;
# define YYSIZE_T __SIZE_TYPE__
# elif defined size_t
# define YYSIZE_T size_t
-# elif ! defined YYSIZE_T && (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
+# elif ! defined YYSIZE_T
# include <stddef.h> /* INFRINGES ON USER NAME SPACE */
# define YYSIZE_T size_t
# else
@@ -541,6 +516,33 @@ typedef short int yytype_int16;
# endif
#endif
+#ifndef YY_ATTRIBUTE
+# if (defined __GNUC__ \
+ && (2 < __GNUC__ || (__GNUC__ == 2 && 96 <= __GNUC_MINOR__))) \
+ || defined __SUNPRO_C && 0x5110 <= __SUNPRO_C
+# define YY_ATTRIBUTE(Spec) __attribute__(Spec)
+# else
+# define YY_ATTRIBUTE(Spec) /* empty */
+# endif
+#endif
+
+#ifndef YY_ATTRIBUTE_PURE
+# define YY_ATTRIBUTE_PURE YY_ATTRIBUTE ((__pure__))
+#endif
+
+#ifndef YY_ATTRIBUTE_UNUSED
+# define YY_ATTRIBUTE_UNUSED YY_ATTRIBUTE ((__unused__))
+#endif
+
+#if !defined _Noreturn \
+ && (!defined __STDC_VERSION__ || __STDC_VERSION__ < 201112)
+# if defined _MSC_VER && 1200 <= _MSC_VER
+# define _Noreturn __declspec (noreturn)
+# else
+# define _Noreturn YY_ATTRIBUTE ((__noreturn__))
+# endif
+#endif
+
/* Suppress unused-variable warnings by "using" E. */
#if ! defined lint || defined __GNUC__
# define YYUSE(E) ((void) (E))
@@ -548,24 +550,26 @@ typedef short int yytype_int16;
# define YYUSE(E) /* empty */
#endif
-/* Identity function, used to suppress warnings about constant conditions. */
-#ifndef lint
-# define YYID(N) (N)
-#else
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-static int
-YYID (int yyi)
+#if defined __GNUC__ && 407 <= __GNUC__ * 100 + __GNUC_MINOR__
+/* Suppress an incorrect diagnostic about yylval being uninitialized. */
+# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \
+ _Pragma ("GCC diagnostic push") \
+ _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"")\
+ _Pragma ("GCC diagnostic ignored \"-Wmaybe-uninitialized\"")
+# define YY_IGNORE_MAYBE_UNINITIALIZED_END \
+ _Pragma ("GCC diagnostic pop")
#else
-static int
-YYID (yyi)
- int yyi;
+# define YY_INITIAL_VALUE(Value) Value
#endif
-{
- return yyi;
-}
+#ifndef YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN
+# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN
+# define YY_IGNORE_MAYBE_UNINITIALIZED_END
+#endif
+#ifndef YY_INITIAL_VALUE
+# define YY_INITIAL_VALUE(Value) /* Nothing. */
#endif
+
#if ! defined yyoverflow || YYERROR_VERBOSE
/* The parser invokes alloca or malloc; define the necessary symbols. */
@@ -583,8 +587,7 @@ YYID (yyi)
# define alloca _alloca
# else
# define YYSTACK_ALLOC alloca
-# if ! defined _ALLOCA_H && ! defined EXIT_SUCCESS && (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
+# if ! defined _ALLOCA_H && ! defined EXIT_SUCCESS
# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */
/* Use EXIT_SUCCESS as a witness for stdlib.h. */
# ifndef EXIT_SUCCESS
@@ -596,8 +599,8 @@ YYID (yyi)
# endif
# ifdef YYSTACK_ALLOC
- /* Pacify GCC's `empty if-body' warning. */
-# define YYSTACK_FREE(Ptr) do { /* empty */; } while (YYID (0))
+ /* Pacify GCC's 'empty if-body' warning. */
+# define YYSTACK_FREE(Ptr) do { /* empty */; } while (0)
# ifndef YYSTACK_ALLOC_MAXIMUM
/* The OS might guarantee only one guard page at the bottom of the stack,
and a page size can be as small as 4096 bytes. So we cannot safely
@@ -613,7 +616,7 @@ YYID (yyi)
# endif
# if (defined __cplusplus && ! defined EXIT_SUCCESS \
&& ! ((defined YYMALLOC || defined malloc) \
- && (defined YYFREE || defined free)))
+ && (defined YYFREE || defined free)))
# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */
# ifndef EXIT_SUCCESS
# define EXIT_SUCCESS 0
@@ -621,15 +624,13 @@ YYID (yyi)
# endif
# ifndef YYMALLOC
# define YYMALLOC malloc
-# if ! defined malloc && ! defined EXIT_SUCCESS && (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
+# if ! defined malloc && ! defined EXIT_SUCCESS
void *malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */
# endif
# endif
# ifndef YYFREE
# define YYFREE free
-# if ! defined free && ! defined EXIT_SUCCESS && (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
+# if ! defined free && ! defined EXIT_SUCCESS
void free (void *); /* INFRINGES ON USER NAME SPACE */
# endif
# endif
@@ -639,8 +640,8 @@ void free (void *); /* INFRINGES ON USER NAME SPACE */
#if (! defined yyoverflow \
&& (! defined __cplusplus \
- || (defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL \
- && defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL)))
+ || (defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL \
+ && defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL)))
/* A type that is properly aligned for any stack member. */
union yyalloc
@@ -666,16 +667,16 @@ union yyalloc
elements in the stack, and YYPTR gives the new location of the
stack. Advance YYPTR to a properly aligned location for the next
stack. */
-# define YYSTACK_RELOCATE(Stack_alloc, Stack) \
- do \
- { \
- YYSIZE_T yynewbytes; \
- YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \
- Stack = &yyptr->Stack_alloc; \
- yynewbytes = yystacksize * sizeof (*Stack) + YYSTACK_GAP_MAXIMUM; \
- yyptr += yynewbytes / sizeof (*yyptr); \
- } \
- while (YYID (0))
+# define YYSTACK_RELOCATE(Stack_alloc, Stack) \
+ do \
+ { \
+ YYSIZE_T yynewbytes; \
+ YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \
+ Stack = &yyptr->Stack_alloc; \
+ yynewbytes = yystacksize * sizeof (*Stack) + YYSTACK_GAP_MAXIMUM; \
+ yyptr += yynewbytes / sizeof (*yyptr); \
+ } \
+ while (0)
#endif
@@ -694,7 +695,7 @@ union yyalloc
for (yyi = 0; yyi < (Count); yyi++) \
(Dst)[yyi] = (Src)[yyi]; \
} \
- while (YYID (0))
+ while (0)
# endif
# endif
#endif /* !YYCOPY_NEEDED */
@@ -710,17 +711,19 @@ union yyalloc
#define YYNNTS 107
/* YYNRULES -- Number of rules. */
#define YYNRULES 289
-/* YYNRULES -- Number of states. */
+/* YYNSTATES -- Number of states. */
#define YYNSTATES 536
-/* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */
+/* YYTRANSLATE[YYX] -- Symbol number corresponding to YYX as returned
+ by yylex, with out-of-bounds checking. */
#define YYUNDEFTOK 2
#define YYMAXUTOK 388
-#define YYTRANSLATE(YYX) \
+#define YYTRANSLATE(YYX) \
((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK)
-/* YYTRANSLATE[YYLEX] -- Bison symbol number corresponding to YYLEX. */
+/* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM
+ as returned by yylex, without out-of-bounds checking. */
static const yytype_uint8 yytranslate[] =
{
0, 2, 2, 2, 2, 2, 2, 2, 2, 2,
@@ -765,139 +768,7 @@ static const yytype_uint8 yytranslate[] =
};
#if YYDEBUG
-/* YYPRHS[YYN] -- Index of the first RHS symbol of rule number YYN in
- YYRHS. */
-static const yytype_uint16 yyprhs[] =
-{
- 0, 0, 3, 6, 9, 12, 15, 17, 19, 21,
- 23, 25, 27, 29, 31, 33, 35, 37, 39, 41,
- 48, 55, 62, 69, 79, 89, 93, 97, 99, 103,
- 105, 107, 109, 111, 113, 115, 117, 119, 121, 123,
- 125, 127, 129, 131, 134, 137, 142, 147, 149, 152,
- 154, 157, 160, 165, 171, 174, 176, 177, 179, 184,
- 190, 201, 206, 210, 212, 213, 215, 216, 220, 221,
- 227, 228, 237, 239, 241, 243, 247, 249, 251, 253,
- 256, 259, 264, 267, 269, 271, 273, 275, 276, 280,
- 291, 299, 304, 310, 316, 317, 322, 325, 328, 333,
- 338, 344, 349, 353, 355, 359, 362, 366, 367, 371,
- 374, 376, 380, 384, 395, 396, 398, 400, 402, 404,
- 406, 410, 414, 417, 419, 422, 426, 427, 431, 435,
- 436, 438, 440, 443, 445, 448, 450, 453, 460, 462,
- 465, 469, 472, 475, 477, 481, 483, 486, 488, 493,
- 495, 499, 500, 504, 505, 508, 509, 513, 514, 517,
- 518, 520, 522, 525, 532, 536, 537, 541, 542, 546,
- 547, 553, 555, 557, 560, 563, 566, 569, 572, 575,
- 577, 581, 585, 586, 588, 590, 591, 594, 597, 598,
- 600, 603, 607, 609, 613, 615, 618, 620, 626, 633,
- 638, 642, 646, 650, 653, 657, 663, 668, 675, 679,
- 681, 685, 687, 690, 692, 694, 696, 698, 702, 708,
- 710, 712, 714, 718, 720, 724, 729, 734, 740, 747,
- 754, 763, 769, 774, 776, 779, 784, 786, 789, 794,
- 795, 798, 800, 804, 806, 808, 811, 814, 816, 819,
- 823, 826, 828, 830, 832, 834, 836, 838, 840, 844,
- 846, 850, 852, 856, 858, 860, 862, 864, 866, 868,
- 870, 873, 875, 878, 880, 882, 884, 886, 888, 890,
- 892, 896, 898, 900, 902, 904, 906, 908, 911, 914
-};
-
-/* YYRHS -- A `-1'-separated list of the rules' RHS. */
-static const yytype_int16 yyrhs[] =
-{
- 146, 0, -1, 147, 139, -1, 147, 137, -1, 250,
- 140, -1, 250, 137, -1, 1, -1, 137, -1, 149,
- -1, 174, -1, 150, -1, 151, -1, 178, -1, 152,
- -1, 173, -1, 148, -1, 181, -1, 177, -1, 105,
- -1, 30, 119, 248, 28, 44, 153, -1, 30, 119,
- 248, 28, 45, 159, -1, 30, 119, 248, 59, 44,
- 248, -1, 30, 119, 248, 59, 45, 248, -1, 47,
- 119, 248, 141, 154, 142, 161, 163, 164, -1, 47,
- 78, 248, 95, 248, 162, 128, 171, 172, -1, 59,
- 119, 248, -1, 248, 155, 158, -1, 153, -1, 154,
- 143, 153, -1, 34, -1, 49, -1, 50, -1, 121,
- -1, 122, -1, 52, -1, 107, -1, 58, -1, 67,
- -1, 117, -1, 81, -1, 33, -1, 89, -1, 82,
- -1, 50, 82, -1, 136, 82, -1, 42, 141, 7,
- 142, -1, 130, 141, 7, 142, -1, 92, -1, 10,
- 92, -1, 126, -1, 103, 85, -1, 53, 238, -1,
- 43, 141, 220, 142, -1, 108, 248, 141, 248, 142,
- -1, 157, 156, -1, 156, -1, -1, 157, -1, 126,
- 141, 247, 142, -1, 103, 85, 141, 247, 142, -1,
- 70, 85, 141, 247, 142, 108, 248, 141, 247, 142,
- -1, 43, 141, 220, 142, -1, 160, 143, 159, -1,
- 159, -1, -1, 160, -1, -1, 141, 242, 142, -1,
- -1, 134, 36, 141, 166, 142, -1, -1, 99, 40,
- 165, 141, 247, 142, 100, 7, -1, 74, -1, 106,
- -1, 167, -1, 166, 143, 167, -1, 168, -1, 169,
- -1, 170, -1, 248, 248, -1, 248, 29, -1, 248,
- 141, 247, 142, -1, 248, 7, -1, 35, -1, 38,
- -1, 39, -1, 116, -1, -1, 141, 166, 142, -1,
- 80, 83, 248, 141, 247, 142, 129, 141, 240, 142,
- -1, 80, 83, 248, 129, 141, 240, 142, -1, 80,
- 83, 248, 186, -1, 183, 80, 83, 248, 186, -1,
- 46, 248, 71, 5, 175, -1, -1, 134, 141, 176,
- 142, -1, 55, 5, -1, 62, 249, -1, 176, 143,
- 55, 5, -1, 176, 143, 62, 249, -1, 127, 248,
- 115, 179, 218, -1, 54, 71, 248, 218, -1, 179,
- 143, 180, -1, 180, -1, 248, 11, 224, -1, 186,
- 182, -1, 183, 186, 182, -1, -1, 134, 104, 7,
- -1, 134, 184, -1, 185, -1, 184, 143, 185, -1,
- 198, 31, 192, -1, 114, 187, 188, 191, 218, 200,
- 201, 202, 203, 204, -1, -1, 29, -1, 57, -1,
- 23, -1, 189, -1, 190, -1, 189, 143, 190, -1,
- 224, 31, 248, -1, 224, 248, -1, 224, -1, 71,
- 199, -1, 141, 186, 142, -1, -1, 37, 7, 101,
- -1, 124, 7, 101, -1, -1, 79, -1, 87, -1,
- 87, 97, -1, 109, -1, 109, 97, -1, 72, -1,
- 72, 97, -1, 195, 194, 84, 196, 95, 220, -1,
- 196, -1, 192, 197, -1, 248, 193, 197, -1, 248,
- 193, -1, 228, 197, -1, 228, -1, 141, 195, 142,
- -1, 198, -1, 31, 198, -1, 248, -1, 248, 141,
- 247, 142, -1, 195, -1, 199, 143, 195, -1, -1,
- 73, 40, 237, -1, -1, 75, 220, -1, -1, 96,
- 40, 214, -1, -1, 88, 7, -1, -1, 205, -1,
- 206, -1, 205, 206, -1, 133, 248, 31, 141, 207,
- 142, -1, 208, 209, 210, -1, -1, 99, 40, 237,
- -1, -1, 96, 40, 214, -1, -1, 211, 19, 212,
- 9, 213, -1, 112, -1, 106, -1, 7, 102, -1,
- 125, 102, -1, 48, 110, -1, 7, 68, -1, 125,
- 68, -1, 48, 110, -1, 215, -1, 214, 143, 215,
- -1, 224, 216, 217, -1, -1, 32, -1, 56, -1,
- -1, 93, 66, -1, 93, 86, -1, -1, 219, -1,
- 132, 220, -1, 220, 8, 221, -1, 221, -1, 221,
- 9, 222, -1, 222, -1, 10, 223, -1, 223, -1,
- 224, 19, 224, 9, 224, -1, 224, 10, 19, 224,
- 9, 224, -1, 241, 20, 10, 92, -1, 241, 20,
- 92, -1, 224, 243, 224, -1, 141, 220, 142, -1,
- 63, 192, -1, 224, 77, 192, -1, 224, 77, 141,
- 237, 142, -1, 224, 10, 77, 192, -1, 224, 10,
- 77, 141, 237, 142, -1, 224, 245, 225, -1, 225,
- -1, 225, 246, 226, -1, 226, -1, 244, 227, -1,
- 227, -1, 241, -1, 238, -1, 228, -1, 228, 98,
- 248, -1, 228, 98, 141, 207, 142, -1, 229, -1,
- 230, -1, 231, -1, 141, 224, 142, -1, 192, -1,
- 248, 141, 142, -1, 248, 141, 23, 142, -1, 248,
- 141, 237, 142, -1, 248, 141, 57, 237, 142, -1,
- 64, 141, 239, 71, 224, 142, -1, 118, 141, 224,
- 71, 7, 142, -1, 118, 141, 224, 71, 7, 69,
- 7, 142, -1, 41, 224, 232, 236, 61, -1, 41,
- 234, 236, 61, -1, 233, -1, 232, 233, -1, 131,
- 224, 120, 224, -1, 235, -1, 234, 235, -1, 131,
- 220, 120, 224, -1, -1, 60, 224, -1, 224, -1,
- 237, 143, 224, -1, 92, -1, 7, -1, 21, 7,
- -1, 22, 7, -1, 5, -1, 82, 5, -1, 82,
- 5, 239, -1, 155, 5, -1, 135, -1, 91, -1,
- 51, -1, 76, -1, 90, -1, 113, -1, 238, -1,
- 240, 143, 238, -1, 248, -1, 248, 27, 248, -1,
- 241, -1, 242, 143, 241, -1, 11, -1, 12, -1,
- 16, -1, 15, -1, 14, -1, 13, -1, 18, -1,
- 10, 18, -1, 17, -1, 10, 17, -1, 22, -1,
- 21, -1, 22, -1, 144, -1, 23, -1, 24, -1,
- 248, -1, 247, 143, 248, -1, 4, -1, 6, -1,
- 123, -1, 95, -1, 65, -1, 94, -1, 3, 251,
- -1, 251, 3, -1, -1
-};
-
-/* YYRLINE[YYN] -- source line where rule number YYN was defined. */
+ /* YYRLINE[YYN] -- Source line where rule number YYN was defined. */
static const yytype_uint16 yyrline[] =
{
0, 629, 629, 633, 637, 641, 645, 648, 655, 658,
@@ -940,9 +811,9 @@ static const char *const yytname[] =
"$end", "error", "$undefined", "TOKEN_COMMAND", "TOKEN_NAME",
"TOKEN_STRING_SINGLE_QUOTED", "TOKEN_STRING_DOUBLE_QUOTED",
"TOKEN_UNSIGNED_NUMVAL", "TOKEN_OR", "TOKEN_AND", "TOKEN_NOT",
- "TOKEN_EQ", "TOKEN_NEQ", "TOKEN_GEQ", "TOKEN_GT", "TOKEN_LEQ",
- "TOKEN_LT", "TOKEN_REGEXP", "TOKEN_LIKE", "TOKEN_BETWEEN", "TOKEN_IS",
- "'+'", "'-'", "'*'", "'/'", "UNARY_MINUS", "UNARY_PLUS", "'.'",
+ "TOKEN_EQ", "TOKEN_LT", "TOKEN_LEQ", "TOKEN_GT", "TOKEN_GEQ",
+ "TOKEN_NEQ", "TOKEN_LIKE", "TOKEN_REGEXP", "TOKEN_BETWEEN", "TOKEN_IS",
+ "'+'", "'-'", "'*'", "'/'", "UNARY_PLUS", "UNARY_MINUS", "'.'",
"TOKEN_ADD", "TOKEN_ALL", "TOKEN_ALTER", "TOKEN_AS", "TOKEN_ASC",
"TOKEN_BIGINT", "TOKEN_BIT", "TOKEN_BITWEAVING", "TOKEN_BLOCKPROPERTIES",
"TOKEN_BLOCKSAMPLE", "TOKEN_BLOOM_FILTER", "TOKEN_CSB_TREE", "TOKEN_BY",
@@ -1004,13 +875,13 @@ static const char *const yytname[] =
"literal_value_commalist", "attribute_ref", "attribute_ref_list",
"comparison_operation", "unary_operation", "add_operation",
"multiply_operation", "name_commalist", "any_name", "boolean_value",
- "command", "command_argument_list", YY_NULL
+ "command", "command_argument_list", YY_NULLPTR
};
#endif
# ifdef YYPRINT
-/* YYTOKNUM[YYLEX-NUM] -- Internal token number corresponding to
- token YYLEX-NUM. */
+/* YYTOKNUM[NUM] -- (External) token number corresponding to the
+ (internal) symbol number NUM (which must be that of a token). */
static const yytype_uint16 yytoknum[] =
{
0, 256, 257, 258, 259, 260, 261, 262, 263, 264,
@@ -1031,154 +902,18 @@ static const yytype_uint16 yytoknum[] =
};
# endif
-/* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */
-static const yytype_uint8 yyr1[] =
-{
- 0, 145, 146, 146, 146, 146, 146, 146, 147, 147,
- 147, 147, 147, 147, 147, 147, 147, 147, 148, 149,
- 149, 149, 149, 150, 151, 152, 153, 154, 154, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 156, 156, 156,
- 156, 156, 156, 156, 157, 157, 158, 158, 159, 159,
- 159, 159, 160, 160, 161, 161, 162, 162, 163, 163,
- 164, 164, 165, 165, 166, 166, 167, 167, 167, 168,
- 168, 169, 170, 171, 171, 171, 171, 172, 172, 173,
- 173, 173, 173, 174, 175, 175, 176, 176, 176, 176,
- 177, 178, 179, 179, 180, 181, 181, 182, 182, 183,
- 184, 184, 185, 186, 187, 187, 187, 188, 188, 189,
- 189, 190, 190, 190, 191, 192, 193, 193, 193, 194,
- 194, 194, 194, 194, 194, 194, 194, 195, 195, 196,
- 196, 196, 196, 196, 196, 197, 197, 198, 198, 199,
- 199, 200, 200, 201, 201, 202, 202, 203, 203, 204,
- 204, 205, 205, 206, 207, 208, 208, 209, 209, 210,
- 210, 211, 211, 212, 212, 212, 213, 213, 213, 214,
- 214, 215, 216, 216, 216, 217, 217, 217, 218, 218,
- 219, 220, 220, 221, 221, 222, 222, 223, 223, 223,
- 223, 223, 223, 223, 223, 223, 223, 223, 224, 224,
- 225, 225, 226, 226, 227, 227, 227, 227, 227, 227,
- 227, 227, 227, 227, 228, 228, 228, 228, 229, 230,
- 230, 231, 231, 232, 232, 233, 234, 234, 235, 236,
- 236, 237, 237, 238, 238, 238, 238, 238, 238, 238,
- 238, 239, 239, 239, 239, 239, 239, 240, 240, 241,
- 241, 242, 242, 243, 243, 243, 243, 243, 243, 243,
- 243, 243, 243, 244, 245, 245, 246, 246, 246, 247,
- 247, 248, 248, 249, 249, 249, 249, 250, 251, 251
-};
+#define YYPACT_NINF -234
-/* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */
-static const yytype_uint8 yyr2[] =
-{
- 0, 2, 2, 2, 2, 2, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 6,
- 6, 6, 6, 9, 9, 3, 3, 1, 3, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 2, 2, 4, 4, 1, 2, 1,
- 2, 2, 4, 5, 2, 1, 0, 1, 4, 5,
- 10, 4, 3, 1, 0, 1, 0, 3, 0, 5,
- 0, 8, 1, 1, 1, 3, 1, 1, 1, 2,
- 2, 4, 2, 1, 1, 1, 1, 0, 3, 10,
- 7, 4, 5, 5, 0, 4, 2, 2, 4, 4,
- 5, 4, 3, 1, 3, 2, 3, 0, 3, 2,
- 1, 3, 3, 10, 0, 1, 1, 1, 1, 1,
- 3, 3, 2, 1, 2, 3, 0, 3, 3, 0,
- 1, 1, 2, 1, 2, 1, 2, 6, 1, 2,
- 3, 2, 2, 1, 3, 1, 2, 1, 4, 1,
- 3, 0, 3, 0, 2, 0, 3, 0, 2, 0,
- 1, 1, 2, 6, 3, 0, 3, 0, 3, 0,
- 5, 1, 1, 2, 2, 2, 2, 2, 2, 1,
- 3, 3, 0, 1, 1, 0, 2, 2, 0, 1,
- 2, 3, 1, 3, 1, 2, 1, 5, 6, 4,
- 3, 3, 3, 2, 3, 5, 4, 6, 3, 1,
- 3, 1, 2, 1, 1, 1, 1, 3, 5, 1,
- 1, 1, 3, 1, 3, 4, 4, 5, 6, 6,
- 8, 5, 4, 1, 2, 4, 1, 2, 4, 0,
- 2, 1, 3, 1, 1, 2, 2, 1, 2, 3,
- 2, 1, 1, 1, 1, 1, 1, 1, 3, 1,
- 3, 1, 3, 1, 1, 1, 1, 1, 1, 1,
- 2, 1, 2, 1, 1, 1, 1, 1, 1, 1,
- 3, 1, 1, 1, 1, 1, 1, 2, 2, 0
-};
+#define yypact_value_is_default(Yystate) \
+ (!!((Yystate) == (-234)))
-/* YYDEFACT[STATE-NAME] -- Default reduction number in state STATE-NUM.
- Performed when YYTABLE doesn't specify something else to do. Zero
- means the default is an error. */
-static const yytype_uint16 yydefact[] =
-{
- 0, 6, 289, 0, 0, 0, 0, 0, 0, 18,
- 114, 0, 0, 7, 0, 0, 15, 8, 10, 11,
- 13, 14, 9, 17, 12, 16, 0, 107, 0, 287,
- 0, 281, 282, 0, 0, 0, 0, 0, 0, 115,
- 116, 0, 0, 109, 110, 0, 147, 1, 3, 2,
- 0, 107, 0, 105, 5, 4, 288, 0, 0, 0,
- 0, 188, 25, 0, 247, 244, 0, 273, 117, 40,
- 29, 0, 0, 30, 31, 34, 36, 0, 37, 39,
- 0, 41, 243, 35, 38, 0, 32, 33, 0, 0,
- 0, 0, 0, 118, 119, 223, 123, 209, 211, 213,
- 216, 219, 220, 221, 215, 214, 0, 259, 0, 0,
- 0, 0, 0, 106, 0, 0, 0, 94, 0, 0,
- 0, 101, 189, 0, 0, 91, 245, 246, 0, 0,
- 239, 236, 0, 43, 0, 248, 0, 0, 44, 0,
- 0, 250, 0, 188, 0, 274, 275, 0, 0, 122,
- 277, 278, 276, 0, 0, 0, 212, 0, 0, 188,
- 103, 0, 111, 0, 112, 0, 279, 0, 108, 0,
- 0, 0, 0, 0, 93, 66, 27, 0, 0, 0,
- 0, 0, 190, 192, 194, 196, 0, 214, 0, 0,
- 0, 0, 239, 233, 0, 237, 0, 0, 253, 254,
- 255, 252, 256, 251, 0, 249, 0, 0, 125, 222,
- 0, 0, 149, 138, 124, 143, 126, 151, 120, 121,
- 208, 210, 165, 217, 260, 0, 0, 224, 241, 0,
- 0, 100, 0, 148, 0, 92, 19, 0, 0, 0,
- 0, 20, 21, 22, 0, 0, 0, 64, 0, 42,
- 56, 195, 203, 0, 0, 0, 0, 0, 263, 264,
- 268, 267, 266, 265, 271, 269, 0, 0, 0, 0,
- 257, 0, 0, 0, 0, 234, 0, 240, 232, 45,
- 0, 0, 46, 129, 0, 139, 145, 135, 130, 131,
- 133, 0, 0, 142, 0, 0, 141, 0, 153, 0,
- 0, 167, 225, 0, 226, 0, 102, 104, 280, 0,
- 0, 0, 0, 0, 0, 0, 261, 0, 259, 0,
- 63, 65, 68, 28, 0, 0, 0, 47, 0, 0,
- 49, 55, 57, 26, 202, 191, 193, 272, 270, 0,
- 0, 0, 0, 204, 201, 0, 200, 90, 0, 0,
- 238, 0, 231, 0, 0, 144, 146, 136, 132, 134,
- 0, 150, 0, 0, 140, 0, 0, 155, 0, 218,
- 0, 169, 227, 242, 0, 0, 0, 0, 96, 285,
- 286, 284, 283, 97, 95, 0, 67, 0, 83, 84,
- 85, 86, 87, 0, 0, 70, 48, 0, 51, 50,
- 0, 54, 0, 0, 206, 0, 0, 199, 258, 0,
- 235, 228, 0, 229, 0, 127, 128, 152, 154, 0,
- 157, 166, 0, 172, 171, 164, 0, 61, 0, 0,
- 58, 0, 0, 262, 0, 24, 62, 0, 0, 23,
- 0, 0, 0, 0, 197, 205, 0, 0, 0, 0,
- 0, 159, 168, 179, 182, 0, 0, 59, 98, 99,
- 0, 74, 76, 77, 78, 0, 0, 0, 52, 0,
- 198, 207, 89, 230, 137, 156, 158, 0, 113, 160,
- 161, 0, 183, 184, 185, 0, 0, 0, 0, 0,
- 88, 0, 82, 80, 0, 79, 0, 72, 73, 0,
- 53, 0, 162, 180, 0, 181, 173, 175, 174, 0,
- 0, 75, 0, 69, 0, 0, 186, 187, 0, 0,
- 0, 170, 0, 81, 0, 165, 176, 178, 177, 0,
- 0, 0, 60, 0, 163, 71
-};
+#define YYTABLE_NINF -130
-/* YYDEFGOTO[NTERM-NUM]. */
-static const yytype_int16 yydefgoto[] =
-{
- -1, 14, 15, 16, 17, 18, 19, 20, 176, 177,
- 91, 331, 332, 333, 241, 321, 322, 246, 395, 439,
- 499, 460, 461, 462, 463, 464, 392, 435, 21, 22,
- 174, 315, 23, 24, 159, 160, 25, 53, 26, 43,
- 44, 139, 41, 92, 93, 94, 143, 95, 296, 291,
- 212, 213, 285, 286, 214, 298, 367, 420, 451, 478,
- 479, 480, 300, 301, 371, 425, 426, 488, 521, 452,
- 453, 484, 505, 121, 122, 182, 183, 184, 185, 186,
- 97, 98, 99, 100, 101, 102, 103, 192, 193, 130,
- 131, 196, 229, 104, 204, 271, 105, 317, 268, 106,
- 148, 153, 165, 107, 383, 28, 29
-};
+#define yytable_value_is_error(Yytable_value) \
+ 0
-/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing
- STATE-NUM. */
-#define YYPACT_NINF -234
+ /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing
+ STATE-NUM. */
static const yytype_int16 yypact[] =
{
168, -234, -234, -58, 181, -19, 40, -37, 59, -234,
@@ -1237,7 +972,68 @@ static const yytype_int16 yypact[] =
411, 377, -234, 505, -234, -234
};
-/* YYPGOTO[NTERM-NUM]. */
+ /* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM.
+ Performed when YYTABLE does not specify something else to do. Zero
+ means the default is an error. */
+static const yytype_uint16 yydefact[] =
+{
+ 0, 6, 289, 0, 0, 0, 0, 0, 0, 18,
+ 114, 0, 0, 7, 0, 0, 15, 8, 10, 11,
+ 13, 14, 9, 17, 12, 16, 0, 107, 0, 287,
+ 0, 281, 282, 0, 0, 0, 0, 0, 0, 115,
+ 116, 0, 0, 109, 110, 0, 147, 1, 3, 2,
+ 0, 107, 0, 105, 5, 4, 288, 0, 0, 0,
+ 0, 188, 25, 0, 247, 244, 0, 273, 117, 40,
+ 29, 0, 0, 30, 31, 34, 36, 0, 37, 39,
+ 0, 41, 243, 35, 38, 0, 32, 33, 0, 0,
+ 0, 0, 0, 118, 119, 223, 123, 209, 211, 213,
+ 216, 219, 220, 221, 215, 214, 0, 259, 0, 0,
+ 0, 0, 0, 106, 0, 0, 0, 94, 0, 0,
+ 0, 101, 189, 0, 0, 91, 245, 246, 0, 0,
+ 239, 236, 0, 43, 0, 248, 0, 0, 44, 0,
+ 0, 250, 0, 188, 0, 274, 275, 0, 0, 122,
+ 277, 278, 276, 0, 0, 0, 212, 0, 0, 188,
+ 103, 0, 111, 0, 112, 0, 279, 0, 108, 0,
+ 0, 0, 0, 0, 93, 66, 27, 0, 0, 0,
+ 0, 0, 190, 192, 194, 196, 0, 214, 0, 0,
+ 0, 0, 239, 233, 0, 237, 0, 0, 253, 254,
+ 255, 252, 256, 251, 0, 249, 0, 0, 125, 222,
+ 0, 0, 149, 138, 124, 143, 126, 151, 120, 121,
+ 208, 210, 165, 217, 260, 0, 0, 224, 241, 0,
+ 0, 100, 0, 148, 0, 92, 19, 0, 0, 0,
+ 0, 20, 21, 22, 0, 0, 0, 64, 0, 42,
+ 56, 195, 203, 0, 0, 0, 0, 0, 263, 265,
+ 266, 267, 268, 264, 269, 271, 0, 0, 0, 0,
+ 257, 0, 0, 0, 0, 234, 0, 240, 232, 45,
+ 0, 0, 46, 129, 0, 139, 145, 135, 130, 131,
+ 133, 0, 0, 142, 0, 0, 141, 0, 153, 0,
+ 0, 167, 225, 0, 226, 0, 102, 104, 280, 0,
+ 0, 0, 0, 0, 0, 0, 261, 0, 259, 0,
+ 63, 65, 68, 28, 0, 0, 0, 47, 0, 0,
+ 49, 55, 57, 26, 202, 191, 193, 270, 272, 0,
+ 0, 0, 0, 204, 201, 0, 200, 90, 0, 0,
+ 238, 0, 231, 0, 0, 144, 146, 136, 132, 134,
+ 0, 150, 0, 0, 140, 0, 0, 155, 0, 218,
+ 0, 169, 227, 242, 0, 0, 0, 0, 96, 285,
+ 286, 284, 283, 97, 95, 0, 67, 0, 83, 84,
+ 85, 86, 87, 0, 0, 70, 48, 0, 51, 50,
+ 0, 54, 0, 0, 206, 0, 0, 199, 258, 0,
+ 235, 228, 0, 229, 0, 127, 128, 152, 154, 0,
+ 157, 166, 0, 172, 171, 164, 0, 61, 0, 0,
+ 58, 0, 0, 262, 0, 24, 62, 0, 0, 23,
+ 0, 0, 0, 0, 197, 205, 0, 0, 0, 0,
+ 0, 159, 168, 179, 182, 0, 0, 59, 98, 99,
+ 0, 74, 76, 77, 78, 0, 0, 0, 52, 0,
+ 198, 207, 89, 230, 137, 156, 158, 0, 113, 160,
+ 161, 0, 183, 184, 185, 0, 0, 0, 0, 0,
+ 88, 0, 82, 80, 0, 79, 0, 72, 73, 0,
+ 53, 0, 162, 180, 0, 181, 173, 175, 174, 0,
+ 0, 75, 0, 69, 0, 0, 186, 187, 0, 0,
+ 0, 170, 0, 81, 0, 165, 176, 178, 177, 0,
+ 0, 0, 60, 0, 163, 71
+};
+
+ /* YYPGOTO[NTERM-NUM]. */
static const yytype_int16 yypgoto[] =
{
-234, -234, -234, -234, -234, -234, -234, -234, -94, -234,
@@ -1253,14 +1049,29 @@ static const yytype_int16 yypgoto[] =
-234, -234, -120, -4, 120, -234, -234
};
-/* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If
- positive, shift that token. If negative, reduce the rule which
- number is the opposite. If YYTABLE_NINF, syntax error. */
-#define YYTABLE_NINF -130
-static const yytype_int16 yytable[] =
+ /* YYDEFGOTO[NTERM-NUM]. */
+static const yytype_int16 yydefgoto[] =
{
- 33, 187, 45, 270, 189, 215, 96, 42, 46, 187,
- 255, 27, 190, 164, 320, 255, 255, 255, 157, 303,
+ -1, 14, 15, 16, 17, 18, 19, 20, 176, 177,
+ 91, 331, 332, 333, 241, 321, 322, 246, 395, 439,
+ 499, 460, 461, 462, 463, 464, 392, 435, 21, 22,
+ 174, 315, 23, 24, 159, 160, 25, 53, 26, 43,
+ 44, 139, 41, 92, 93, 94, 143, 95, 296, 291,
+ 212, 213, 285, 286, 214, 298, 367, 420, 451, 478,
+ 479, 480, 300, 301, 371, 425, 426, 488, 521, 452,
+ 453, 484, 505, 121, 122, 182, 183, 184, 185, 186,
+ 97, 98, 99, 100, 101, 102, 103, 192, 193, 130,
+ 131, 196, 229, 104, 204, 271, 105, 317, 268, 106,
+ 148, 153, 165, 107, 383, 28, 29
+};
+
+ /* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If
+ positive, shift that token. If negative, reduce the rule whose
+ number is the opposite. If YYTABLE_NINF, syntax error. */
+static const yytype_int16 yytable[] =
+{
+ 33, 187, 45, 270, 189, 215, 96, 42, 46, 187,
+ 255, 27, 190, 164, 320, 255, 255, 255, 157, 303,
31, 283, 32, 294, 324, 31, 57, 32, 492, 485,
59, 60, 61, 62, 63, 293, 129, 51, 31, 31,
32, 32, 237, 150, 151, 211, 145, 146, 518, 345,
@@ -1396,12 +1207,6 @@ static const yytype_int16 yytable[] =
88, 0, 0, 0, 0, 0, 89
};
-#define yypact_value_is_default(Yystate) \
- (!!((Yystate) == (-234)))
-
-#define yytable_value_is_error(Yytable_value) \
- YYID (0)
-
static const yytype_int16 yycheck[] =
{
4, 120, 12, 188, 124, 142, 41, 11, 12, 128,
@@ -1541,8 +1346,8 @@ static const yytype_int16 yycheck[] =
130, -1, -1, -1, -1, -1, 136
};
-/* YYSTOS[STATE-NUM] -- The (internal number of the) accessing
- symbol of state STATE-NUM. */
+ /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing
+ symbol of state STATE-NUM. */
static const yytype_uint8 yystos[] =
{
0, 1, 3, 30, 46, 47, 54, 59, 80, 105,
@@ -1601,30 +1406,84 @@ static const yytype_uint8 yystos[] =
142, 207, 142, 100, 142, 7
};
-#define yyerrok (yyerrstatus = 0)
-#define yyclearin (yychar = YYEMPTY)
-#define YYEMPTY (-2)
-#define YYEOF 0
-
-#define YYACCEPT goto yyacceptlab
-#define YYABORT goto yyabortlab
-#define YYERROR goto yyerrorlab
-
-
-/* Like YYERROR except do call yyerror. This remains here temporarily
- to ease the transition to the new meaning of YYERROR, for GCC.
- Once GCC version 2 has supplanted version 1, this can go. However,
- YYFAIL appears to be in use. Nevertheless, it is formally deprecated
- in Bison 2.4.2's NEWS entry, where a plan to phase it out is
- discussed. */
-
-#define YYFAIL goto yyerrlab
-#if defined YYFAIL
- /* This is here to suppress warnings from the GCC cpp's
- -Wunused-macros. Normally we don't worry about that warning, but
- some users do, and we want to make it easy for users to remove
- YYFAIL uses, which will produce warnings from Bison 2.5. */
-#endif
+ /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */
+static const yytype_uint8 yyr1[] =
+{
+ 0, 145, 146, 146, 146, 146, 146, 146, 147, 147,
+ 147, 147, 147, 147, 147, 147, 147, 147, 148, 149,
+ 149, 149, 149, 150, 151, 152, 153, 154, 154, 155,
+ 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
+ 155, 155, 155, 155, 155, 155, 155, 156, 156, 156,
+ 156, 156, 156, 156, 157, 157, 158, 158, 159, 159,
+ 159, 159, 160, 160, 161, 161, 162, 162, 163, 163,
+ 164, 164, 165, 165, 166, 166, 167, 167, 167, 168,
+ 168, 169, 170, 171, 171, 171, 171, 172, 172, 173,
+ 173, 173, 173, 174, 175, 175, 176, 176, 176, 176,
+ 177, 178, 179, 179, 180, 181, 181, 182, 182, 183,
+ 184, 184, 185, 186, 187, 187, 187, 188, 188, 189,
+ 189, 190, 190, 190, 191, 192, 193, 193, 193, 194,
+ 194, 194, 194, 194, 194, 194, 194, 195, 195, 196,
+ 196, 196, 196, 196, 196, 197, 197, 198, 198, 199,
+ 199, 200, 200, 201, 201, 202, 202, 203, 203, 204,
+ 204, 205, 205, 206, 207, 208, 208, 209, 209, 210,
+ 210, 211, 211, 212, 212, 212, 213, 213, 213, 214,
+ 214, 215, 216, 216, 216, 217, 217, 217, 218, 218,
+ 219, 220, 220, 221, 221, 222, 222, 223, 223, 223,
+ 223, 223, 223, 223, 223, 223, 223, 223, 224, 224,
+ 225, 225, 226, 226, 227, 227, 227, 227, 227, 227,
+ 227, 227, 227, 227, 228, 228, 228, 228, 229, 230,
+ 230, 231, 231, 232, 232, 233, 234, 234, 235, 236,
+ 236, 237, 237, 238, 238, 238, 238, 238, 238, 238,
+ 238, 239, 239, 239, 239, 239, 239, 240, 240, 241,
+ 241, 242, 242, 243, 243, 243, 243, 243, 243, 243,
+ 243, 243, 243, 244, 245, 245, 246, 246, 246, 247,
+ 247, 248, 248, 249, 249, 249, 249, 250, 251, 251
+};
+
+ /* YYR2[YYN] -- Number of symbols on the right hand side of rule YYN. */
+static const yytype_uint8 yyr2[] =
+{
+ 0, 2, 2, 2, 2, 2, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 6,
+ 6, 6, 6, 9, 9, 3, 3, 1, 3, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 2, 2, 4, 4, 1, 2, 1,
+ 2, 2, 4, 5, 2, 1, 0, 1, 4, 5,
+ 10, 4, 3, 1, 0, 1, 0, 3, 0, 5,
+ 0, 8, 1, 1, 1, 3, 1, 1, 1, 2,
+ 2, 4, 2, 1, 1, 1, 1, 0, 3, 10,
+ 7, 4, 5, 5, 0, 4, 2, 2, 4, 4,
+ 5, 4, 3, 1, 3, 2, 3, 0, 3, 2,
+ 1, 3, 3, 10, 0, 1, 1, 1, 1, 1,
+ 3, 3, 2, 1, 2, 3, 0, 3, 3, 0,
+ 1, 1, 2, 1, 2, 1, 2, 6, 1, 2,
+ 3, 2, 2, 1, 3, 1, 2, 1, 4, 1,
+ 3, 0, 3, 0, 2, 0, 3, 0, 2, 0,
+ 1, 1, 2, 6, 3, 0, 3, 0, 3, 0,
+ 5, 1, 1, 2, 2, 2, 2, 2, 2, 1,
+ 3, 3, 0, 1, 1, 0, 2, 2, 0, 1,
+ 2, 3, 1, 3, 1, 2, 1, 5, 6, 4,
+ 3, 3, 3, 2, 3, 5, 4, 6, 3, 1,
+ 3, 1, 2, 1, 1, 1, 1, 3, 5, 1,
+ 1, 1, 3, 1, 3, 4, 4, 5, 6, 6,
+ 8, 5, 4, 1, 2, 4, 1, 2, 4, 0,
+ 2, 1, 3, 1, 1, 2, 2, 1, 2, 3,
+ 2, 1, 1, 1, 1, 1, 1, 1, 3, 1,
+ 3, 1, 3, 1, 1, 1, 1, 1, 1, 1,
+ 2, 1, 2, 1, 1, 1, 1, 1, 1, 1,
+ 3, 1, 1, 1, 1, 1, 1, 2, 2, 0
+};
+
+
+#define yyerrok (yyerrstatus = 0)
+#define yyclearin (yychar = YYEMPTY)
+#define YYEMPTY (-2)
+#define YYEOF 0
+
+#define YYACCEPT goto yyacceptlab
+#define YYABORT goto yyabortlab
+#define YYERROR goto yyerrorlab
+
#define YYRECOVERING() (!!yyerrstatus)
@@ -1641,13 +1500,13 @@ do \
else \
{ \
yyerror (&yylloc, yyscanner, parsedStatement, YY_("syntax error: cannot back up")); \
- YYERROR; \
- } \
-while (YYID (0))
+ YYERROR; \
+ } \
+while (0)
/* Error token number */
-#define YYTERROR 1
-#define YYERRCODE 256
+#define YYTERROR 1
+#define YYERRCODE 256
/* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N].
@@ -1657,7 +1516,7 @@ while (YYID (0))
#ifndef YYLLOC_DEFAULT
# define YYLLOC_DEFAULT(Current, Rhs, N) \
do \
- if (YYID (N)) \
+ if (N) \
{ \
(Current).first_line = YYRHSLOC (Rhs, 1).first_line; \
(Current).first_column = YYRHSLOC (Rhs, 1).first_column; \
@@ -1671,12 +1530,27 @@ while (YYID (0))
(Current).first_column = (Current).last_column = \
YYRHSLOC (Rhs, 0).last_column; \
} \
- while (YYID (0))
+ while (0)
#endif
#define YYRHSLOC(Rhs, K) ((Rhs)[K])
+/* Enable debugging if requested. */
+#if YYDEBUG
+
+# ifndef YYFPRINTF
+# include <stdio.h> /* INFRINGES ON USER NAME SPACE */
+# define YYFPRINTF fprintf
+# endif
+
+# define YYDPRINTF(Args) \
+do { \
+ if (yydebug) \
+ YYFPRINTF Args; \
+} while (0)
+
+
/* YY_LOCATION_PRINT -- Print the location on the stream.
This macro was not mandated originally: define only if we know
we won't break user code: when these are the locations we know. */
@@ -1686,36 +1560,28 @@ while (YYID (0))
/* Print *YYLOCP on YYO. Private, do not rely on its existence. */
-__attribute__((__unused__))
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
+YY_ATTRIBUTE_UNUSED
static unsigned
yy_location_print_ (FILE *yyo, YYLTYPE const * const yylocp)
-#else
-static unsigned
-yy_location_print_ (yyo, yylocp)
- FILE *yyo;
- YYLTYPE const * const yylocp;
-#endif
{
unsigned res = 0;
int end_col = 0 != yylocp->last_column ? yylocp->last_column - 1 : 0;
if (0 <= yylocp->first_line)
{
- res += fprintf (yyo, "%d", yylocp->first_line);
+ res += YYFPRINTF (yyo, "%d", yylocp->first_line);
if (0 <= yylocp->first_column)
- res += fprintf (yyo, ".%d", yylocp->first_column);
+ res += YYFPRINTF (yyo, ".%d", yylocp->first_column);
}
if (0 <= yylocp->last_line)
{
if (yylocp->first_line < yylocp->last_line)
{
- res += fprintf (yyo, "-%d", yylocp->last_line);
+ res += YYFPRINTF (yyo, "-%d", yylocp->last_line);
if (0 <= end_col)
- res += fprintf (yyo, ".%d", end_col);
+ res += YYFPRINTF (yyo, ".%d", end_col);
}
else if (0 <= end_col && yylocp->first_column < end_col)
- res += fprintf (yyo, "-%d", end_col);
+ res += YYFPRINTF (yyo, "-%d", end_col);
}
return res;
}
@@ -1729,77 +1595,37 @@ yy_location_print_ (yyo, yylocp)
#endif
-/* YYLEX -- calling `yylex' with the right arguments. */
-#ifdef YYLEX_PARAM
-# define YYLEX yylex (&yylval, &yylloc, YYLEX_PARAM)
-#else
-# define YYLEX yylex (&yylval, &yylloc, yyscanner)
-#endif
+# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \
+do { \
+ if (yydebug) \
+ { \
+ YYFPRINTF (stderr, "%s ", Title); \
+ yy_symbol_print (stderr, \
+ Type, Value, Location, yyscanner, parsedStatement); \
+ YYFPRINTF (stderr, "\n"); \
+ } \
+} while (0)
-/* Enable debugging if requested. */
-#if YYDEBUG
-# ifndef YYFPRINTF
-# include <stdio.h> /* INFRINGES ON USER NAME SPACE */
-# define YYFPRINTF fprintf
-# endif
+/*----------------------------------------.
+| Print this symbol's value on YYOUTPUT. |
+`----------------------------------------*/
-# define YYDPRINTF(Args) \
-do { \
- if (yydebug) \
- YYFPRINTF Args; \
-} while (YYID (0))
-
-# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \
-do { \
- if (yydebug) \
- { \
- YYFPRINTF (stderr, "%s ", Title); \
- yy_symbol_print (stderr, \
- Type, Value, Location, yyscanner, parsedStatement); \
- YYFPRINTF (stderr, "\n"); \
- } \
-} while (YYID (0))
-
-
-/*--------------------------------.
-| Print this symbol on YYOUTPUT. |
-`--------------------------------*/
-
-/*ARGSUSED*/
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
static void
yy_symbol_value_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp, yyscan_t yyscanner, quickstep::ParseStatement **parsedStatement)
-#else
-static void
-yy_symbol_value_print (yyoutput, yytype, yyvaluep, yylocationp, yyscanner, parsedStatement)
- FILE *yyoutput;
- int yytype;
- YYSTYPE const * const yyvaluep;
- YYLTYPE const * const yylocationp;
- yyscan_t yyscanner;
- quickstep::ParseStatement **parsedStatement;
-#endif
{
FILE *yyo = yyoutput;
YYUSE (yyo);
- if (!yyvaluep)
- return;
YYUSE (yylocationp);
YYUSE (yyscanner);
YYUSE (parsedStatement);
+ if (!yyvaluep)
+ return;
# ifdef YYPRINT
if (yytype < YYNTOKENS)
YYPRINT (yyoutput, yytoknum[yytype], *yyvaluep);
-# else
- YYUSE (yyoutput);
# endif
- switch (yytype)
- {
- default:
- break;
- }
+ YYUSE (yytype);
}
@@ -1807,25 +1633,11 @@ yy_symbol_value_print (yyoutput, yytype, yyvaluep, yylocationp, yyscanner, parse
| Print this symbol on YYOUTPUT. |
`--------------------------------*/
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
static void
yy_symbol_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp, yyscan_t yyscanner, quickstep::ParseStatement **parsedStatement)
-#else
-static void
-yy_symbol_print (yyoutput, yytype, yyvaluep, yylocationp, yyscanner, parsedStatement)
- FILE *yyoutput;
- int yytype;
- YYSTYPE const * const yyvaluep;
- YYLTYPE const * const yylocationp;
- yyscan_t yyscanner;
- quickstep::ParseStatement **parsedStatement;
-#endif
{
- if (yytype < YYNTOKENS)
- YYFPRINTF (yyoutput, "token %s (", yytname[yytype]);
- else
- YYFPRINTF (yyoutput, "nterm %s (", yytname[yytype]);
+ YYFPRINTF (yyoutput, "%s %s (",
+ yytype < YYNTOKENS ? "token" : "nterm", yytname[yytype]);
YY_LOCATION_PRINT (yyoutput, *yylocationp);
YYFPRINTF (yyoutput, ": ");
@@ -1838,16 +1650,8 @@ yy_symbol_print (yyoutput, yytype, yyvaluep, yylocationp, yyscanner, parsedState
| TOP (included). |
`------------------------------------------------------------------*/
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
static void
yy_stack_print (yytype_int16 *yybottom, yytype_int16 *yytop)
-#else
-static void
-yy_stack_print (yybottom, yytop)
- yytype_int16 *yybottom;
- yytype_int16 *yytop;
-#endif
{
YYFPRINTF (stderr, "Stack now");
for (; yybottom <= yytop; yybottom++)
@@ -1858,52 +1662,42 @@ yy_stack_print (yybottom, yytop)
YYFPRINTF (stderr, "\n");
}
-# define YY_STACK_PRINT(Bottom, Top) \
-do { \
- if (yydebug) \
- yy_stack_print ((Bottom), (Top)); \
-} while (YYID (0))
+# define YY_STACK_PRINT(Bottom, Top) \
+do { \
+ if (yydebug) \
+ yy_stack_print ((Bottom), (Top)); \
+} while (0)
/*------------------------------------------------.
| Report that the YYRULE is going to be reduced. |
`------------------------------------------------*/
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
static void
-yy_reduce_print (YYSTYPE *yyvsp, YYLTYPE *yylsp, int yyrule, yyscan_t yyscanner, quickstep::ParseStatement **parsedStatement)
-#else
-static void
-yy_reduce_print (yyvsp, yylsp, yyrule, yyscanner, parsedStatement)
- YYSTYPE *yyvsp;
- YYLTYPE *yylsp;
- int yyrule;
- yyscan_t yyscanner;
- quickstep::ParseStatement **parsedStatement;
-#endif
+yy_reduce_print (yytype_int16 *yyssp, YYSTYPE *yyvsp, YYLTYPE *yylsp, int yyrule, yyscan_t yyscanner, quickstep::ParseStatement **parsedStatement)
{
+ unsigned long int yylno = yyrline[yyrule];
int yynrhs = yyr2[yyrule];
int yyi;
- unsigned long int yylno = yyrline[yyrule];
YYFPRINTF (stderr, "Reducing stack by rule %d (line %lu):\n",
- yyrule - 1, yylno);
+ yyrule - 1, yylno);
/* The symbols being reduced. */
for (yyi = 0; yyi < yynrhs; yyi++)
{
YYFPRINTF (stderr, " $%d = ", yyi + 1);
- yy_symbol_print (stderr, yyrhs[yyprhs[yyrule] + yyi],
- &(yyvsp[(yyi + 1) - (yynrhs)])
- , &(yylsp[(yyi + 1) - (yynrhs)]) , yyscanner, parsedStatement);
+ yy_symbol_print (stderr,
+ yystos[yyssp[yyi + 1 - yynrhs]],
+ &(yyvsp[(yyi + 1) - (yynrhs)])
+ , &(yylsp[(yyi + 1) - (yynrhs)]) , yyscanner, parsedStatement);
YYFPRINTF (stderr, "\n");
}
}
-# define YY_REDUCE_PRINT(Rule) \
-do { \
- if (yydebug) \
- yy_reduce_print (yyvsp, yylsp, Rule, yyscanner, parsedStatement); \
-} while (YYID (0))
+# define YY_REDUCE_PRINT(Rule) \
+do { \
+ if (yydebug) \
+ yy_reduce_print (yyssp, yyvsp, yylsp, Rule, yyscanner, parsedStatement); \
+} while (0)
/* Nonzero means print parse trace. It is left uninitialized so that
multiple parsers can coexist. */
@@ -1917,7 +1711,7 @@ int yydebug;
/* YYINITDEPTH -- initial size of the parser's stacks. */
-#ifndef YYINITDEPTH
+#ifndef YYINITDEPTH
# define YYINITDEPTH 200
#endif
@@ -1940,15 +1734,8 @@ int yydebug;
# define yystrlen strlen
# else
/* Return the length of YYSTR. */
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
static YYSIZE_T
yystrlen (const char *yystr)
-#else
-static YYSIZE_T
-yystrlen (yystr)
- const char *yystr;
-#endif
{
YYSIZE_T yylen;
for (yylen = 0; yystr[yylen]; yylen++)
@@ -1964,16 +1751,8 @@ yystrlen (yystr)
# else
/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in
YYDEST. */
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
static char *
yystpcpy (char *yydest, const char *yysrc)
-#else
-static char *
-yystpcpy (yydest, yysrc)
- char *yydest;
- const char *yysrc;
-#endif
{
char *yyd = yydest;
const char *yys = yysrc;
@@ -2003,27 +1782,27 @@ yytnamerr (char *yyres, const char *yystr)
char const *yyp = yystr;
for (;;)
- switch (*++yyp)
- {
- case '\'':
- case ',':
- goto do_not_strip_quotes;
-
- case '\\':
- if (*++yyp != '\\')
- goto do_not_strip_quotes;
- /* Fall through. */
- default:
- if (yyres)
- yyres[yyn] = *yyp;
- yyn++;
- break;
-
- case '"':
- if (yyres)
- yyres[yyn] = '\0';
- return yyn;
- }
+ switch (*++yyp)
+ {
+ case '\'':
+ case ',':
+ goto do_not_strip_quotes;
+
+ case '\\':
+ if (*++yyp != '\\')
+ goto do_not_strip_quotes;
+ /* Fall through. */
+ default:
+ if (yyres)
+ yyres[yyn] = *yyp;
+ yyn++;
+ break;
+
+ case '"':
+ if (yyres)
+ yyres[yyn] = '\0';
+ return yyn;
+ }
do_not_strip_quotes: ;
}
@@ -2046,11 +1825,11 @@ static int
yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg,
yytype_int16 *yyssp, int yytoken)
{
- YYSIZE_T yysize0 = yytnamerr (YY_NULL, yytname[yytoken]);
+ YYSIZE_T yysize0 = yytnamerr (YY_NULLPTR, yytname[yytoken]);
YYSIZE_T yysize = yysize0;
enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 };
/* Internationalized format string. */
- const char *yyformat = YY_NULL;
+ const char *yyformat = YY_NULLPTR;
/* Arguments of yyformat. */
char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM];
/* Number of reported tokens (one for the "unexpected", one per
@@ -2058,10 +1837,6 @@ yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg,
int yycount = 0;
/* There are many possibilities here to consider:
- - Assume YYFAIL is not used. It's too flawed to consider. See
- <http://lists.gnu.org/archive/html/bison-patches/2009-12/msg00024.html>
- for details. YYERROR is fine as it does not invoke this
- function.
- If this state is a consistent state with a default action, then
the only way this function was invoked is if the default action
is an error action. In that case, don't check for expected
@@ -2111,7 +1886,7 @@ yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg,
}
yyarg[yycount++] = yytname[yyx];
{
- YYSIZE_T yysize1 = yysize + yytnamerr (YY_NULL, yytname[yyx]);
+ YYSIZE_T yysize1 = yysize + yytnamerr (YY_NULLPTR, yytname[yyx]);
if (! (yysize <= yysize1
&& yysize1 <= YYSTACK_ALLOC_MAXIMUM))
return 2;
@@ -2178,1175 +1953,1057 @@ yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg,
| Release the memory associated to this symbol. |
`-----------------------------------------------*/
-/*ARGSUSED*/
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
static void
yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep, YYLTYPE *yylocationp, yyscan_t yyscanner, quickstep::ParseStatement **parsedStatement)
-#else
-static void
-yydestruct (yymsg, yytype, yyvaluep, yylocationp, yyscanner, parsedStatement)
- const char *yymsg;
- int yytype;
- YYSTYPE *yyvaluep;
- YYLTYPE *yylocationp;
- yyscan_t yyscanner;
- quickstep::ParseStatement **parsedStatement;
-#endif
{
YYUSE (yyvaluep);
YYUSE (yylocationp);
YYUSE (yyscanner);
YYUSE (parsedStatement);
-
if (!yymsg)
yymsg = "Deleting";
YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp);
+ YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN
switch (yytype)
{
- case 3: /* TOKEN_COMMAND */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+ case 3: /* TOKEN_COMMAND */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2218 "SqlParser_gen.cpp"
+}
+#line 1978 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 4: /* TOKEN_NAME */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 4: /* TOKEN_NAME */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2229 "SqlParser_gen.cpp"
+}
+#line 1988 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 5: /* TOKEN_STRING_SINGLE_QUOTED */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 5: /* TOKEN_STRING_SINGLE_QUOTED */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2240 "SqlParser_gen.cpp"
+}
+#line 1998 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 6: /* TOKEN_STRING_DOUBLE_QUOTED */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 6: /* TOKEN_STRING_DOUBLE_QUOTED */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2251 "SqlParser_gen.cpp"
+}
+#line 2008 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 7: /* TOKEN_UNSIGNED_NUMVAL */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 7: /* TOKEN_UNSIGNED_NUMVAL */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).numeric_literal_value_) != nullptr) {
delete ((*yyvaluep).numeric_literal_value_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2262 "SqlParser_gen.cpp"
+}
+#line 2018 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 147: /* sql_statement */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 147: /* sql_statement */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).statement_) != nullptr) {
delete ((*yyvaluep).statement_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2273 "SqlParser_gen.cpp"
+}
+#line 2028 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 148: /* quit_statement */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 148: /* quit_statement */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).quit_statement_) != nullptr) {
delete ((*yyvaluep).quit_statement_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2284 "SqlParser_gen.cpp"
+}
+#line 2038 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 149: /* alter_table_statement */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 149: /* alter_table_statement */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).statement_) != nullptr) {
delete ((*yyvaluep).statement_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2295 "SqlParser_gen.cpp"
+}
+#line 2048 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 150: /* create_table_statement */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 150: /* create_table_statement */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).create_table_statement_) != nullptr) {
delete ((*yyvaluep).create_table_statement_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2306 "SqlParser_gen.cpp"
+}
+#line 2058 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 151: /* create_index_statement */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 151: /* create_index_statement */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).statement_) != nullptr) {
delete ((*yyvaluep).statement_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2317 "SqlParser_gen.cpp"
+}
+#line 2068 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 152: /* drop_table_statement */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 152: /* drop_table_statement */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).drop_table_statement_) != nullptr) {
delete ((*yyvaluep).drop_table_statement_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2328 "SqlParser_gen.cpp"
+}
+#line 2078 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 153: /* column_def */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 153: /* column_def */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).attribute_definition_) != nullptr) {
delete ((*yyvaluep).attribute_definition_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2339 "SqlParser_gen.cpp"
+}
+#line 2088 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 154: /* column_def_commalist */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 154: /* column_def_commalist */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).attribute_definition_list_) != nullptr) {
delete ((*yyvaluep).attribute_definition_list_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2350 "SqlParser_gen.cpp"
+}
+#line 2098 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 155: /* data_type */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 155: /* data_type */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).data_type_) != nullptr) {
delete ((*yyvaluep).data_type_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2361 "SqlParser_gen.cpp"
+}
+#line 2108 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 156: /* column_constraint_def */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 156: /* column_constraint_def */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).column_constraint_) != nullptr) {
delete ((*yyvaluep).column_constraint_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2372 "SqlParser_gen.cpp"
+}
+#line 2118 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 157: /* column_constraint_def_list */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 157: /* column_constraint_def_list */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).column_constraint_list_) != nullptr) {
delete ((*yyvaluep).column_constraint_list_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2383 "SqlParser_gen.cpp"
+}
+#line 2128 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 158: /* opt_column_constraint_def_list */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 158: /* opt_column_constraint_def_list */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).column_constraint_list_) != nullptr) {
delete ((*yyvaluep).column_constraint_list_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2394 "SqlParser_gen.cpp"
+}
+#line 2138 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 162: /* opt_column_list */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 162: /* opt_column_list */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).attribute_list_) != nullptr) {
delete ((*yyvaluep).attribute_list_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2405 "SqlParser_gen.cpp"
+}
+#line 2148 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 163: /* opt_block_properties */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 163: /* opt_block_properties */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).block_properties_) != nullptr) {
delete ((*yyvaluep).block_properties_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2416 "SqlParser_gen.cpp"
+}
+#line 2158 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 164: /* opt_partition_clause */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 164: /* opt_partition_clause */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).partition_clause_) != nullptr) {
delete ((*yyvaluep).partition_clause_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2427 "SqlParser_gen.cpp"
+}
+#line 2168 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 165: /* partition_type */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 165: /* partition_type */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2438 "SqlParser_gen.cpp"
+}
+#line 2178 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 166: /* key_value_list */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 166: /* key_value_list */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).key_value_list_) != nullptr) {
delete ((*yyvaluep).key_value_list_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2449 "SqlParser_gen.cpp"
+}
+#line 2188 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 167: /* key_value */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 167: /* key_value */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).key_value_) != nullptr) {
delete ((*yyvaluep).key_value_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2460 "SqlParser_gen.cpp"
+}
+#line 2198 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 168: /* key_string_value */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 168: /* key_string_value */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).key_string_value_) != nullptr) {
delete ((*yyvaluep).key_string_value_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2471 "SqlParser_gen.cpp"
+}
+#line 2208 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 169: /* key_string_list */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 169: /* key_string_list */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).key_string_list_) != nullptr) {
delete ((*yyvaluep).key_string_list_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2482 "SqlParser_gen.cpp"
+}
+#line 2218 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 170: /* key_integer_value */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 170: /* key_integer_value */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).key_integer_value_) != nullptr) {
delete ((*yyvaluep).key_integer_value_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2493 "SqlParser_gen.cpp"
+}
+#line 2228 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 171: /* index_type */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 171: /* index_type */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2504 "SqlParser_gen.cpp"
+}
+#line 2238 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 172: /* opt_index_properties */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 172: /* opt_index_properties */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).key_value_list_) != nullptr) {
delete ((*yyvaluep).key_value_list_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2515 "SqlParser_gen.cpp"
+}
+#line 2248 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 173: /* insert_statement */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 173: /* insert_statement */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).insert_statement_) != nullptr) {
delete ((*yyvaluep).insert_statement_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2526 "SqlParser_gen.cpp"
+}
+#line 2258 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 174: /* copy_from_statement */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 174: /* copy_from_statement */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).copy_from_statement_) != nullptr) {
delete ((*yyvaluep).copy_from_statement_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2537 "SqlParser_gen.cpp"
+}
+#line 2268 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 175: /* opt_copy_from_params */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 175: /* opt_copy_from_params */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).copy_from_params_) != nullptr) {
delete ((*yyvaluep).copy_from_params_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2548 "SqlParser_gen.cpp"
+}
+#line 2278 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 176: /* copy_from_params */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 176: /* copy_from_params */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).copy_from_params_) != nullptr) {
delete ((*yyvaluep).copy_from_params_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2559 "SqlParser_gen.cpp"
+}
+#line 2288 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 177: /* update_statement */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 177: /* update_statement */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).update_statement_) != nullptr) {
delete ((*yyvaluep).update_statement_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2570 "SqlParser_gen.cpp"
+}
+#line 2298 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 178: /* delete_statement */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 178: /* delete_statement */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).delete_statement_) != nullptr) {
delete ((*yyvaluep).delete_statement_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2581 "SqlParser_gen.cpp"
+}
+#line 2308 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 179: /* assignment_list */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 179: /* assignment_list */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).assignment_list_) != nullptr) {
delete ((*yyvaluep).assignment_list_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2592 "SqlParser_gen.cpp"
+}
+#line 2318 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 180: /* assignment_item */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 180: /* assignment_item */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).assignment_) != nullptr) {
delete ((*yyvaluep).assignment_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2603 "SqlParser_gen.cpp"
+}
+#line 2328 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 181: /* select_statement */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 181: /* select_statement */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).select_statement_) != nullptr) {
delete ((*yyvaluep).select_statement_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2614 "SqlParser_gen.cpp"
+}
+#line 2338 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 182: /* opt_priority_clause */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 182: /* opt_priority_clause */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).opt_priority_clause_) != nullptr) {
delete ((*yyvaluep).opt_priority_clause_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2625 "SqlParser_gen.cpp"
+}
+#line 2348 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 183: /* with_clause */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 183: /* with_clause */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).with_list_) != nullptr) {
delete ((*yyvaluep).with_list_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2636 "SqlParser_gen.cpp"
+}
+#line 2358 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 184: /* with_list */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 184: /* with_list */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).with_list_) != nullptr) {
delete ((*yyvaluep).with_list_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2647 "SqlParser_gen.cpp"
+}
+#line 2368 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 185: /* with_list_element */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 185: /* with_list_element */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).with_list_element_) != nullptr) {
delete ((*yyvaluep).with_list_element_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2658 "SqlParser_gen.cpp"
+}
+#line 2378 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 186: /* select_query */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 186: /* select_query */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257 */
+ {
if (((*yyvaluep).select_query_) != nullptr) {
delete ((*yyvaluep).select_query_);
}
-};
-/* Line 1398 of yacc.c */
-#line 2669 "SqlParser_gen.cpp"
+}
+#line 2388 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 188: /* selection */
-/* Line 1398 of yacc.c */
-#line 620 "../SqlParser.ypp"
- {
+
+ case 188: /* selection */
+#line 620 "../SqlParser.ypp" /* yacc.c:1257
<TRUNCATED>
[14/17] incubator-quickstep git commit: Enable \analyze command to be
applied to specific tables
Posted by ji...@apache.org.
Enable \analyze command to be applied to specific tables
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4126c4f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4126c4f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4126c4f2
Branch: refs/heads/lip-refactor
Commit: 4126c4f2b9677800995c258df3795526b1e49aa8
Parents: 0820e3d
Author: Jianqiao <ji...@cs.wisc.edu>
Authored: Tue Oct 4 00:12:41 2016 -0500
Committer: Jianqiao <ji...@cs.wisc.edu>
Committed: Tue Oct 4 00:22:23 2016 -0500
----------------------------------------------------------------------
cli/CommandExecutor.cpp | 99 ++++++++++++++++++++++++++++++++------------
1 file changed, 73 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4126c4f2/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 78fbe6f..fea4b74 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -197,9 +197,9 @@ void executeDescribeTable(
}
/**
- * @brief A helper function that executes a SQL query to obtain a scalar result.
+ * @brief A helper function that executes a SQL query to obtain a row of results.
*/
-inline TypedValue executeQueryForSingleResult(
+inline std::vector<TypedValue> executeQueryForSingleRow(
const tmb::client_id main_thread_client_id,
const tmb::client_id foreman_client_id,
const std::string &query_string,
@@ -232,22 +232,29 @@ inline TypedValue executeQueryForSingleResult(
const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
DCHECK(query_result_relation != nullptr);
- TypedValue value;
+ std::vector<TypedValue> values;
{
std::vector<block_id> blocks = query_result_relation->getBlocksSnapshot();
DCHECK_EQ(1u, blocks.size());
+
BlockReference block = storage_manager->getBlock(blocks[0], *query_result_relation);
const TupleStorageSubBlock &tuple_store = block->getTupleStorageSubBlock();
DCHECK_EQ(1, tuple_store.numTuples());
- DCHECK_EQ(1u, tuple_store.getRelation().size());
+ const std::size_t num_columns = tuple_store.getRelation().size();
if (tuple_store.isPacked()) {
- value = tuple_store.getAttributeValueTyped(0, 0);
+ for (std::size_t i = 0; i < num_columns; ++i) {
+ values.emplace_back(tuple_store.getAttributeValueTyped(0, i));
+ values[i].ensureNotReference();
+ }
} else {
std::unique_ptr<TupleIdSequence> existence_map(tuple_store.getExistenceMap());
- value = tuple_store.getAttributeValueTyped(*existence_map->begin(), 0);
+ for (std::size_t i = 0; i < num_columns; ++i) {
+ values.emplace_back(
+ tuple_store.getAttributeValueTyped(*existence_map->begin(), i));
+ values[i].ensureNotReference();
+ }
}
- value.ensureNotReference();
}
// Drop the result relation.
@@ -255,10 +262,34 @@ inline TypedValue executeQueryForSingleResult(
query_processor->getDefaultDatabase(),
query_processor->getStorageManager());
- return value;
+ return values;
+}
+
+/**
+ * @brief A helper function that executes a SQL query to obtain a scalar result.
+ */
+inline TypedValue executeQueryForSingleResult(
+ const tmb::client_id main_thread_client_id,
+ const tmb::client_id foreman_client_id,
+ const std::string &query_string,
+ tmb::MessageBus *bus,
+ StorageManager *storage_manager,
+ QueryProcessor *query_processor,
+ SqlParserWrapper *parser_wrapper) {
+ std::vector<TypedValue> results =
+ executeQueryForSingleRow(main_thread_client_id,
+ foreman_client_id,
+ query_string,
+ bus,
+ storage_manager,
+ query_processor,
+ parser_wrapper);
+ DCHECK_EQ(1u, results.size());
+ return results[0];
}
-void executeAnalyze(const tmb::client_id main_thread_client_id,
+void executeAnalyze(const PtrVector<ParseString> *arguments,
+ const tmb::client_id main_thread_client_id,
const tmb::client_id foreman_client_id,
MessageBus *bus,
QueryProcessor *query_processor,
@@ -267,8 +298,19 @@ void executeAnalyze(const tmb::client_id main_thread_client_id,
StorageManager *storage_manager = query_processor->getStorageManager();
std::unique_ptr<SqlParserWrapper> parser_wrapper(new SqlParserWrapper());
- std::vector<std::reference_wrapper<const CatalogRelation>> relations(
- database.begin(), database.end());
+ std::vector<std::reference_wrapper<const CatalogRelation>> relations;
+ if (arguments->size() == 0) {
+ relations.insert(relations.begin(), database.begin(), database.end());
+ } else {
+ for (const auto &rel_name : *arguments) {
+ const CatalogRelation *rel = database.getRelationByName(rel_name.value());
+ if (rel == nullptr) {
+ THROW_SQL_ERROR_AT(&rel_name) << "Table does not exist";
+ } else {
+ relations.emplace_back(*rel);
+ }
+ }
+ }
// Analyze each relation in the database.
for (const CatalogRelation &relation : relations) {
@@ -286,19 +328,21 @@ void executeAnalyze(const tmb::client_id main_thread_client_id,
query_string.append(relation.getName());
query_string.append(";");
- TypedValue num_distinct_values =
- executeQueryForSingleResult(main_thread_client_id,
- foreman_client_id,
- query_string,
- bus,
- storage_manager,
- query_processor,
- parser_wrapper.get());
-
- DCHECK(num_distinct_values.getTypeID() == TypeID::kLong);
- mutable_relation->getStatisticsMutable()->setNumDistinctValues(
- attribute.getID(),
- num_distinct_values.getLiteral<std::int64_t>());
+ std::vector<TypedValue> results =
+ executeQueryForSingleRow(main_thread_client_id,
+ foreman_client_id,
+ query_string,
+ bus,
+ storage_manager,
+ query_processor,
+ parser_wrapper.get());
+
+ auto *stat = mutable_relation->getStatisticsMutable();
+ const attribute_id attr_id = attribute.getID();
+
+ DCHECK(results[0].getTypeID() == TypeID::kLong);
+ stat->setNumDistinctValues(attr_id,
+ results[0].getLiteral<std::int64_t>());
}
// Get the number of tuples for the relation.
@@ -348,8 +392,11 @@ void executeCommand(const ParseStatement &statement,
executeDescribeTable(arguments, catalog_database, out);
}
} else if (command_str == C::kAnalyzeCommand) {
- executeAnalyze(
- main_thread_client_id, foreman_client_id, bus, query_processor, out);
+ executeAnalyze(arguments,
+ main_thread_client_id,
+ foreman_client_id,
+ bus,
+ query_processor, out);
} else {
THROW_SQL_ERROR_AT(command.command()) << "Invalid Command";
}
[07/17] incubator-quickstep git commit: Initial commit for
QUICKSTEP-28 and QUICKSTEP-29.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp b/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
index 6565a41..78bd249 100644
--- a/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
@@ -29,6 +29,8 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleCount.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/AggregationOperationState.hpp"
+#include "storage/FastHashTableFactory.hpp"
#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DoubleType.hpp"
@@ -50,85 +52,94 @@
namespace quickstep {
-class AggregationHandleCountTest : public::testing::Test {
+class AggregationHandleCountTest : public ::testing::Test {
protected:
const Type &dummy_type = TypeFactory::GetType(kInt);
void iterateHandleNullary(AggregationState *state) {
- static_cast<const AggregationHandleCount<true, false>&>(
- *aggregation_handle_count_).iterateNullaryInl(
- static_cast<AggregationStateCount*>(state));
+ static_cast<const AggregationHandleCount<true, false> &>(
+ *aggregation_handle_count_)
+ .iterateNullaryInl(static_cast<AggregationStateCount *>(state));
}
// Helper method that calls AggregationHandleCount::iterateUnaryInl() to
// aggregate 'value' into '*state'.
void iterateHandle(AggregationState *state, const TypedValue &value) {
- static_cast<const AggregationHandleCount<false, true>&>(
- *aggregation_handle_count_).iterateUnaryInl(
- static_cast<AggregationStateCount*>(state),
- value);
+ static_cast<const AggregationHandleCount<false, true> &>(
+ *aggregation_handle_count_)
+ .iterateUnaryInl(static_cast<AggregationStateCount *>(state), value);
}
void initializeHandle(const Type *argument_type) {
if (argument_type == nullptr) {
aggregation_handle_count_.reset(
- AggregateFunctionFactory::Get(AggregationID::kCount).createHandle(
- std::vector<const Type*>()));
+ AggregateFunctionFactory::Get(AggregationID::kCount)
+ .createHandle(std::vector<const Type *>()));
} else {
aggregation_handle_count_.reset(
- AggregateFunctionFactory::Get(AggregationID::kCount).createHandle(
- std::vector<const Type*>(1, argument_type)));
+ AggregateFunctionFactory::Get(AggregationID::kCount)
+ .createHandle(std::vector<const Type *>(1, argument_type)));
}
aggregation_handle_count_state_.reset(
aggregation_handle_count_->createInitialState());
}
static bool ApplyToTypesTest(TypeID typeID) {
- const Type &type = (typeID == kChar || typeID == kVarChar) ?
- TypeFactory::GetType(typeID, static_cast<std::size_t>(10)) :
- TypeFactory::GetType(typeID);
+ const Type &type =
+ (typeID == kChar || typeID == kVarChar)
+ ? TypeFactory::GetType(typeID, static_cast<std::size_t>(10))
+ : TypeFactory::GetType(typeID);
- return AggregateFunctionFactory::Get(AggregationID::kCount).canApplyToTypes(
- std::vector<const Type*>(1, &type));
+ return AggregateFunctionFactory::Get(AggregationID::kCount)
+ .canApplyToTypes(std::vector<const Type *>(1, &type));
}
static bool ResultTypeForArgumentTypeTest(TypeID input_type_id,
TypeID output_type_id) {
- const Type *result_type
- = AggregateFunctionFactory::Get(AggregationID::kCount).resultTypeForArgumentTypes(
- std::vector<const Type*>(1, &TypeFactory::GetType(input_type_id)));
+ const Type *result_type =
+ AggregateFunctionFactory::Get(AggregationID::kCount)
+ .resultTypeForArgumentTypes(std::vector<const Type *>(
+ 1, &TypeFactory::GetType(input_type_id)));
return (result_type->getTypeID() == output_type_id);
}
- static void CheckCountValue(
- std::int64_t expected,
- const AggregationHandle &handle,
- const AggregationState &state) {
+ static void CheckCountValue(std::int64_t expected,
+ const AggregationHandle &handle,
+ const AggregationState &state) {
EXPECT_EQ(expected, handle.finalize(state).getLiteral<std::int64_t>());
}
+ static void CheckCountValue(std::int64_t expected, const TypedValue &value) {
+ EXPECT_EQ(expected, value.getLiteral<std::int64_t>());
+ }
+
void checkAggregationCountNullary(int test_count) {
initializeHandle(nullptr);
- CheckCountValue(0, *aggregation_handle_count_, *aggregation_handle_count_state_);
+ CheckCountValue(
+ 0, *aggregation_handle_count_, *aggregation_handle_count_state_);
for (int i = 0; i < test_count; ++i) {
iterateHandleNullary(aggregation_handle_count_state_.get());
}
- CheckCountValue(test_count, *aggregation_handle_count_, *aggregation_handle_count_state_);
+ CheckCountValue(test_count,
+ *aggregation_handle_count_,
+ *aggregation_handle_count_state_);
// Test mergeStates.
std::unique_ptr<AggregationState> merge_state(
aggregation_handle_count_->createInitialState());
- aggregation_handle_count_->mergeStates(*merge_state,
- aggregation_handle_count_state_.get());
+ aggregation_handle_count_->mergeStates(
+ *merge_state, aggregation_handle_count_state_.get());
for (int i = 0; i < test_count; ++i) {
iterateHandleNullary(merge_state.get());
}
- aggregation_handle_count_->mergeStates(*merge_state,
- aggregation_handle_count_state_.get());
- CheckCountValue(2 * test_count, *aggregation_handle_count_, *aggregation_handle_count_state_);
+ aggregation_handle_count_->mergeStates(
+ *merge_state, aggregation_handle_count_state_.get());
+ CheckCountValue(2 * test_count,
+ *aggregation_handle_count_,
+ *aggregation_handle_count_state_);
}
void checkAggregationCountNullaryAccumulate(int test_count) {
@@ -139,12 +150,10 @@ class AggregationHandleCountTest : public::testing::Test {
// Test the state generated directly by accumulateNullary(), and also test
// after merging back.
- CheckCountValue(test_count,
- *aggregation_handle_count_,
- *accumulated_state);
+ CheckCountValue(test_count, *aggregation_handle_count_, *accumulated_state);
- aggregation_handle_count_->mergeStates(*accumulated_state,
- aggregation_handle_count_state_.get());
+ aggregation_handle_count_->mergeStates(
+ *accumulated_state, aggregation_handle_count_state_.get());
CheckCountValue(test_count,
*aggregation_handle_count_,
*aggregation_handle_count_state_);
@@ -154,24 +163,27 @@ class AggregationHandleCountTest : public::testing::Test {
void checkAggregationCountNumeric(int test_count) {
const NumericType &type = NumericType::Instance(true);
initializeHandle(&type);
- CheckCountValue(0, *aggregation_handle_count_, *aggregation_handle_count_state_);
+ CheckCountValue(
+ 0, *aggregation_handle_count_, *aggregation_handle_count_state_);
typename NumericType::cpptype val = 0;
int count = 0;
iterateHandle(aggregation_handle_count_state_.get(), type.makeNullValue());
for (int i = 0; i < test_count; ++i) {
- iterateHandle(aggregation_handle_count_state_.get(), type.makeValue(&val));
+ iterateHandle(aggregation_handle_count_state_.get(),
+ type.makeValue(&val));
++count;
}
iterateHandle(aggregation_handle_count_state_.get(), type.makeNullValue());
- CheckCountValue(count, *aggregation_handle_count_, *aggregation_handle_count_state_);
+ CheckCountValue(
+ count, *aggregation_handle_count_, *aggregation_handle_count_state_);
// Test mergeStates.
std::unique_ptr<AggregationState> merge_state(
aggregation_handle_count_->createInitialState());
- aggregation_handle_count_->mergeStates(*merge_state,
- aggregation_handle_count_state_.get());
+ aggregation_handle_count_->mergeStates(
+ *merge_state, aggregation_handle_count_state_.get());
iterateHandle(merge_state.get(), type.makeNullValue());
for (int i = 0; i < test_count; ++i) {
@@ -180,13 +192,14 @@ class AggregationHandleCountTest : public::testing::Test {
}
iterateHandle(merge_state.get(), type.makeNullValue());
- aggregation_handle_count_->mergeStates(*merge_state,
- aggregation_handle_count_state_.get());
- CheckCountValue(count, *aggregation_handle_count_, *aggregation_handle_count_state_);
+ aggregation_handle_count_->mergeStates(
+ *merge_state, aggregation_handle_count_state_.get());
+ CheckCountValue(
+ count, *aggregation_handle_count_, *aggregation_handle_count_state_);
}
template <typename NumericType>
- ColumnVector *createColumnVectorNumeric(const Type &type, int test_count) {
+ ColumnVector* createColumnVectorNumeric(const Type &type, int test_count) {
NativeColumnVector *column = new NativeColumnVector(type, test_count + 3);
typename NumericType::cpptype val = 0;
@@ -194,7 +207,7 @@ class AggregationHandleCountTest : public::testing::Test {
for (int i = 0; i < test_count; ++i) {
column->appendTypedValue(type.makeValue(&val));
// One NULL in the middle.
- if (i == test_count/2) {
+ if (i == test_count / 2) {
column->appendTypedValue(type.makeNullValue());
}
}
@@ -206,21 +219,22 @@ class AggregationHandleCountTest : public::testing::Test {
void checkAggregationCountNumericColumnVector(int test_count) {
const NumericType &type = NumericType::Instance(true);
initializeHandle(&type);
- CheckCountValue(0, *aggregation_handle_count_, *aggregation_handle_count_state_);
+ CheckCountValue(
+ 0, *aggregation_handle_count_, *aggregation_handle_count_state_);
std::vector<std::unique_ptr<ColumnVector>> column_vectors;
- column_vectors.emplace_back(createColumnVectorNumeric<NumericType>(type, test_count));
+ column_vectors.emplace_back(
+ createColumnVectorNumeric<NumericType>(type, test_count));
std::unique_ptr<AggregationState> cv_state(
aggregation_handle_count_->accumulateColumnVectors(column_vectors));
// Test the state generated directly by accumulateColumnVectors(), and also
// test after merging back.
- CheckCountValue(test_count,
- *aggregation_handle_count_,
- *cv_state);
+ CheckCountValue(test_count, *aggregation_handle_count_, *cv_state);
- aggregation_handle_count_->mergeStates(*cv_state, aggregation_handle_count_state_.get());
+ aggregation_handle_count_->mergeStates(
+ *cv_state, aggregation_handle_count_state_.get());
CheckCountValue(test_count,
*aggregation_handle_count_,
*aggregation_handle_count_state_);
@@ -231,22 +245,24 @@ class AggregationHandleCountTest : public::testing::Test {
void checkAggregationCountNumericValueAccessor(int test_count) {
const NumericType &type = NumericType::Instance(true);
initializeHandle(&type);
- CheckCountValue(0, *aggregation_handle_count_, *aggregation_handle_count_state_);
+ CheckCountValue(
+ 0, *aggregation_handle_count_, *aggregation_handle_count_state_);
- std::unique_ptr<ColumnVectorsValueAccessor> accessor(new ColumnVectorsValueAccessor());
- accessor->addColumn(createColumnVectorNumeric<NumericType>(type, test_count));
+ std::unique_ptr<ColumnVectorsValueAccessor> accessor(
+ new ColumnVectorsValueAccessor());
+ accessor->addColumn(
+ createColumnVectorNumeric<NumericType>(type, test_count));
std::unique_ptr<AggregationState> va_state(
- aggregation_handle_count_->accumulateValueAccessor(accessor.get(),
- std::vector<attribute_id>(1, 0)));
+ aggregation_handle_count_->accumulateValueAccessor(
+ accessor.get(), std::vector<attribute_id>(1, 0)));
// Test the state generated directly by accumulateValueAccessor(), and also
// test after merging back.
- CheckCountValue(test_count,
- *aggregation_handle_count_,
- *va_state);
+ CheckCountValue(test_count, *aggregation_handle_count_, *va_state);
- aggregation_handle_count_->mergeStates(*va_state, aggregation_handle_count_state_.get());
+ aggregation_handle_count_->mergeStates(
+ *va_state, aggregation_handle_count_state_.get());
CheckCountValue(test_count,
*aggregation_handle_count_,
*aggregation_handle_count_state_);
@@ -257,7 +273,8 @@ class AggregationHandleCountTest : public::testing::Test {
void checkAggregationCountString(int test_count) {
const StringType &type = StringType::Instance(10, true);
initializeHandle(&type);
- CheckCountValue(0, *aggregation_handle_count_, *aggregation_handle_count_state_);
+ CheckCountValue(
+ 0, *aggregation_handle_count_, *aggregation_handle_count_state_);
std::string string_literal = "test_str";
int count = 0;
@@ -269,7 +286,8 @@ class AggregationHandleCountTest : public::testing::Test {
++count;
}
iterateHandle(aggregation_handle_count_state_.get(), type.makeNullValue());
- CheckCountValue(count, *aggregation_handle_count_, *aggregation_handle_count_state_);
+ CheckCountValue(
+ count, *aggregation_handle_count_, *aggregation_handle_count_state_);
// Test mergeStates().
std::unique_ptr<AggregationState> merge_state(
@@ -277,18 +295,20 @@ class AggregationHandleCountTest : public::testing::Test {
iterateHandle(merge_state.get(), type.makeNullValue());
for (int i = 0; i < test_count; ++i) {
- iterateHandle(merge_state.get(), type.makeValue(string_literal.c_str(), 10));
+ iterateHandle(merge_state.get(),
+ type.makeValue(string_literal.c_str(), 10));
++count;
}
iterateHandle(merge_state.get(), type.makeNullValue());
- aggregation_handle_count_->mergeStates(*merge_state,
- aggregation_handle_count_state_.get());
- CheckCountValue(count, *aggregation_handle_count_, *aggregation_handle_count_state_);
+ aggregation_handle_count_->mergeStates(
+ *merge_state, aggregation_handle_count_state_.get());
+ CheckCountValue(
+ count, *aggregation_handle_count_, *aggregation_handle_count_state_);
}
template <typename ColumnVectorType>
- ColumnVector *createColumnVectorString(const Type &type, int test_count) {
+ ColumnVector* createColumnVectorString(const Type &type, int test_count) {
ColumnVectorType *column = new ColumnVectorType(type, test_count + 3);
std::string string_literal = "test_str";
@@ -296,7 +316,7 @@ class AggregationHandleCountTest : public::testing::Test {
for (int i = 0; i < test_count; ++i) {
column->appendTypedValue(type.makeValue(string_literal.c_str(), 10));
// One NULL in the middle.
- if (i == test_count/2) {
+ if (i == test_count / 2) {
column->appendTypedValue(type.makeNullValue());
}
}
@@ -309,21 +329,22 @@ class AggregationHandleCountTest : public::testing::Test {
void checkAggregationCountStringColumnVector(int test_count) {
const StringType &type = StringType::Instance(10, true);
initializeHandle(&type);
- CheckCountValue(0, *aggregation_handle_count_, *aggregation_handle_count_state_);
+ CheckCountValue(
+ 0, *aggregation_handle_count_, *aggregation_handle_count_state_);
std::vector<std::unique_ptr<ColumnVector>> column_vectors;
- column_vectors.emplace_back(createColumnVectorString<ColumnVectorType>(type, test_count));
+ column_vectors.emplace_back(
+ createColumnVectorString<ColumnVectorType>(type, test_count));
std::unique_ptr<AggregationState> cv_state(
aggregation_handle_count_->accumulateColumnVectors(column_vectors));
// Test the state generated directly by accumulateColumnVectors(), and also
// test after merging back.
- CheckCountValue(test_count,
- *aggregation_handle_count_,
- *cv_state);
+ CheckCountValue(test_count, *aggregation_handle_count_, *cv_state);
- aggregation_handle_count_->mergeStates(*cv_state, aggregation_handle_count_state_.get());
+ aggregation_handle_count_->mergeStates(
+ *cv_state, aggregation_handle_count_state_.get());
CheckCountValue(test_count,
*aggregation_handle_count_,
*aggregation_handle_count_state_);
@@ -334,22 +355,24 @@ class AggregationHandleCountTest : public::testing::Test {
void checkAggregationCountStringValueAccessor(int test_count) {
const StringType &type = StringType::Instance(10, true);
initializeHandle(&type);
- CheckCountValue(0, *aggregation_handle_count_, *aggregation_handle_count_state_);
+ CheckCountValue(
+ 0, *aggregation_handle_count_, *aggregation_handle_count_state_);
- std::unique_ptr<ColumnVectorsValueAccessor> accessor(new ColumnVectorsValueAccessor());
- accessor->addColumn(createColumnVectorString<ColumnVectorType>(type, test_count));
+ std::unique_ptr<ColumnVectorsValueAccessor> accessor(
+ new ColumnVectorsValueAccessor());
+ accessor->addColumn(
+ createColumnVectorString<ColumnVectorType>(type, test_count));
std::unique_ptr<AggregationState> va_state(
- aggregation_handle_count_->accumulateValueAccessor(accessor.get(),
- std::vector<attribute_id>(1, 0)));
+ aggregation_handle_count_->accumulateValueAccessor(
+ accessor.get(), std::vector<attribute_id>(1, 0)));
// Test the state generated directly by accumulateValueAccessor(), and also
// test after merging back.
- CheckCountValue(test_count,
- *aggregation_handle_count_,
- *va_state);
+ CheckCountValue(test_count, *aggregation_handle_count_, *va_state);
- aggregation_handle_count_->mergeStates(*va_state, aggregation_handle_count_state_.get());
+ aggregation_handle_count_->mergeStates(
+ *va_state, aggregation_handle_count_state_.get());
CheckCountValue(test_count,
*aggregation_handle_count_,
*aggregation_handle_count_state_);
@@ -364,13 +387,12 @@ class AggregationHandleCountTest : public::testing::Test {
typedef AggregationHandleCountTest AggregationHandleCountDeathTest;
TEST_F(AggregationHandleCountTest, CountStarTest) {
- checkAggregationCountNullary(0),
- checkAggregationCountNullary(10000);
+ checkAggregationCountNullary(0), checkAggregationCountNullary(10000);
}
TEST_F(AggregationHandleCountTest, CountStarAccumulateTest) {
checkAggregationCountNullaryAccumulate(0),
- checkAggregationCountNullaryAccumulate(10000);
+ checkAggregationCountNullaryAccumulate(10000);
}
TEST_F(AggregationHandleCountTest, IntTypeTest) {
@@ -430,7 +452,8 @@ TEST_F(AggregationHandleCountTest, CharTypeColumnVectorTest) {
TEST_F(AggregationHandleCountTest, VarCharTypeColumnVectorTest) {
checkAggregationCountStringColumnVector<VarCharType, IndirectColumnVector>(0);
- checkAggregationCountStringColumnVector<VarCharType, IndirectColumnVector>(10000);
+ checkAggregationCountStringColumnVector<VarCharType, IndirectColumnVector>(
+ 10000);
}
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -460,8 +483,10 @@ TEST_F(AggregationHandleCountTest, CharTypeValueAccessorTest) {
}
TEST_F(AggregationHandleCountTest, VarCharTypeValueAccessorTest) {
- checkAggregationCountStringValueAccessor<VarCharType, IndirectColumnVector>(0);
- checkAggregationCountStringValueAccessor<VarCharType, IndirectColumnVector>(10000);
+ checkAggregationCountStringValueAccessor<VarCharType, IndirectColumnVector>(
+ 0);
+ checkAggregationCountStringValueAccessor<VarCharType, IndirectColumnVector>(
+ 10000);
}
#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -486,25 +511,28 @@ TEST_F(AggregationHandleCountTest, GroupByTableMergeTestCount) {
initializeHandle(&long_non_null_type);
storage_manager_.reset(new StorageManager("./test_count_data"));
std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
- aggregation_handle_count_->createGroupByHashTable(
- HashTableImplType::kSimpleScalarSeparateChaining,
+ AggregationStateFastHashTableFactory::CreateResizable(
+ HashTableImplType::kSeparateChaining,
std::vector<const Type *>(1, &long_non_null_type),
10,
+ {aggregation_handle_count_.get()->getPayloadSize()},
+ {aggregation_handle_count_.get()},
storage_manager_.get()));
std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
- aggregation_handle_count_->createGroupByHashTable(
- HashTableImplType::kSimpleScalarSeparateChaining,
+ AggregationStateFastHashTableFactory::CreateResizable(
+ HashTableImplType::kSeparateChaining,
std::vector<const Type *>(1, &long_non_null_type),
10,
+ {aggregation_handle_count_.get()->getPayloadSize()},
+ {aggregation_handle_count_.get()},
storage_manager_.get()));
- AggregationStateHashTable<AggregationStateCount> *destination_hash_table_derived =
- static_cast<AggregationStateHashTable<AggregationStateCount> *>(
+ AggregationStateFastHashTable *destination_hash_table_derived =
+ static_cast<AggregationStateFastHashTable *>(
destination_hash_table.get());
- AggregationStateHashTable<AggregationStateCount> *source_hash_table_derived =
- static_cast<AggregationStateHashTable<AggregationStateCount> *>(
- source_hash_table.get());
+ AggregationStateFastHashTable *source_hash_table_derived =
+ static_cast<AggregationStateFastHashTable *>(source_hash_table.get());
// TODO(harshad) - Use TemplateUtil::CreateBoolInstantiatedInstance to
// generate all the combinations of the bool template arguments and test them.
@@ -530,7 +558,8 @@ TEST_F(AggregationHandleCountTest, GroupByTableMergeTestCount) {
TypedValue exclusive_key_source_count_val(exclusive_key_source_count);
const std::int64_t exclusive_key_destination_count = 1;
- TypedValue exclusive_key_destination_count_val(exclusive_key_destination_count);
+ TypedValue exclusive_key_destination_count_val(
+ exclusive_key_destination_count);
std::unique_ptr<AggregationStateCount> common_key_source_state(
static_cast<AggregationStateCount *>(
@@ -546,62 +575,86 @@ TEST_F(AggregationHandleCountTest, GroupByTableMergeTestCount) {
aggregation_handle_count_->createInitialState()));
// Create count value states for keys.
- aggregation_handle_count_derived->iterateUnaryInl(common_key_source_state.get(),
- common_key_source_count_val);
- std::int64_t actual_val = aggregation_handle_count_->finalize(*common_key_source_state)
- .getLiteral<std::int64_t>();
+ aggregation_handle_count_derived->iterateUnaryInl(
+ common_key_source_state.get(), common_key_source_count_val);
+ std::int64_t actual_val =
+ aggregation_handle_count_->finalize(*common_key_source_state)
+ .getLiteral<std::int64_t>();
EXPECT_EQ(common_key_source_count_val.getLiteral<std::int64_t>(), actual_val);
aggregation_handle_count_derived->iterateUnaryInl(
common_key_destination_state.get(), common_key_destination_count_val);
- actual_val = aggregation_handle_count_->finalize(*common_key_destination_state)
- .getLiteral<std::int64_t>();
- EXPECT_EQ(common_key_destination_count_val.getLiteral<std::int64_t>(), actual_val);
+ actual_val =
+ aggregation_handle_count_->finalize(*common_key_destination_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(common_key_destination_count_val.getLiteral<std::int64_t>(),
+ actual_val);
aggregation_handle_count_derived->iterateUnaryInl(
- exclusive_key_destination_state.get(), exclusive_key_destination_count_val);
+ exclusive_key_destination_state.get(),
+ exclusive_key_destination_count_val);
actual_val =
aggregation_handle_count_->finalize(*exclusive_key_destination_state)
.getLiteral<std::int64_t>();
- EXPECT_EQ(exclusive_key_destination_count_val.getLiteral<std::int64_t>(), actual_val);
+ EXPECT_EQ(exclusive_key_destination_count_val.getLiteral<std::int64_t>(),
+ actual_val);
aggregation_handle_count_derived->iterateUnaryInl(
exclusive_key_source_state.get(), exclusive_key_source_count_val);
actual_val = aggregation_handle_count_->finalize(*exclusive_key_source_state)
.getLiteral<std::int64_t>();
- EXPECT_EQ(exclusive_key_source_count_val.getLiteral<std::int64_t>(), actual_val);
+ EXPECT_EQ(exclusive_key_source_count_val.getLiteral<std::int64_t>(),
+ actual_val);
// Add the key-state pairs to the hash tables.
- source_hash_table_derived->putCompositeKey(common_key,
- *common_key_source_state);
- destination_hash_table_derived->putCompositeKey(
- common_key, *common_key_destination_state);
- source_hash_table_derived->putCompositeKey(exclusive_source_key,
- *exclusive_key_source_state);
- destination_hash_table_derived->putCompositeKey(
- exclusive_destination_key, *exclusive_key_destination_state);
+ unsigned char buffer[100];
+ buffer[0] = '\0';
+ memcpy(buffer + 1,
+ common_key_source_state.get()->getPayloadAddress(),
+ aggregation_handle_count_.get()->getPayloadSize());
+ source_hash_table_derived->putCompositeKey(common_key, buffer);
+
+ memcpy(buffer + 1,
+ common_key_destination_state.get()->getPayloadAddress(),
+ aggregation_handle_count_.get()->getPayloadSize());
+ destination_hash_table_derived->putCompositeKey(common_key, buffer);
+
+ memcpy(buffer + 1,
+ exclusive_key_source_state.get()->getPayloadAddress(),
+ aggregation_handle_count_.get()->getPayloadSize());
+ source_hash_table_derived->putCompositeKey(exclusive_source_key, buffer);
+
+ memcpy(buffer + 1,
+ exclusive_key_destination_state.get()->getPayloadAddress(),
+ aggregation_handle_count_.get()->getPayloadSize());
+ destination_hash_table_derived->putCompositeKey(exclusive_destination_key,
+ buffer);
EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
EXPECT_EQ(2u, source_hash_table_derived->numEntries());
- aggregation_handle_count_->mergeGroupByHashTables(*source_hash_table,
- destination_hash_table.get());
+ AggregationOperationState::mergeGroupByHashTables(
+ source_hash_table.get(), destination_hash_table.get());
EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
CheckCountValue(
common_key_destination_count_val.getLiteral<std::int64_t>() +
common_key_source_count_val.getLiteral<std::int64_t>(),
- *aggregation_handle_count_derived,
- *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
- CheckCountValue(exclusive_key_destination_count_val.getLiteral<std::int64_t>(),
- *aggregation_handle_count_derived,
- *(destination_hash_table_derived->getSingleCompositeKey(
- exclusive_destination_key)));
+ aggregation_handle_count_derived->finalizeHashTableEntryFast(
+ destination_hash_table_derived->getSingleCompositeKey(common_key) +
+ 1));
+ CheckCountValue(
+ exclusive_key_destination_count_val.getLiteral<std::int64_t>(),
+ aggregation_handle_count_derived->finalizeHashTableEntryFast(
+ destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key) +
+ 1));
CheckCountValue(exclusive_key_source_count_val.getLiteral<std::int64_t>(),
- *aggregation_handle_count_derived,
- *(source_hash_table_derived->getSingleCompositeKey(
- exclusive_source_key)));
+ aggregation_handle_count_derived->finalizeHashTableEntryFast(
+ source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key) +
+ 1));
}
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ac3512ce/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp b/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
index b7cf02a..026bd1d 100644
--- a/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
@@ -31,6 +31,8 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleMax.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/AggregationOperationState.hpp"
+#include "storage/FastHashTableFactory.hpp"
#include "storage/HashTableBase.hpp"
#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
@@ -70,54 +72,59 @@ class AggregationHandleMaxTest : public ::testing::Test {
// Helper method that calls AggregationHandleMax::iterateUnaryInl() to
// aggregate 'value' into '*state'.
void iterateHandle(AggregationState *state, const TypedValue &value) {
- static_cast<const AggregationHandleMax&>(*aggregation_handle_max_).iterateUnaryInl(
- static_cast<AggregationStateMax*>(state),
- value);
+ static_cast<const AggregationHandleMax &>(*aggregation_handle_max_)
+ .iterateUnaryInl(static_cast<AggregationStateMax *>(state), value);
}
void initializeHandle(const Type &type) {
aggregation_handle_max_.reset(
- AggregateFunctionFactory::Get(AggregationID::kMax).createHandle(
- std::vector<const Type*>(1, &type)));
+ AggregateFunctionFactory::Get(AggregationID::kMax)
+ .createHandle(std::vector<const Type *>(1, &type)));
aggregation_handle_max_state_.reset(
aggregation_handle_max_->createInitialState());
}
static bool ApplyToTypesTest(TypeID typeID) {
- const Type &type = (typeID == kChar || typeID == kVarChar) ?
- TypeFactory::GetType(typeID, static_cast<std::size_t>(10)) :
- TypeFactory::GetType(typeID);
+ const Type &type =
+ (typeID == kChar || typeID == kVarChar)
+ ? TypeFactory::GetType(typeID, static_cast<std::size_t>(10))
+ : TypeFactory::GetType(typeID);
- return AggregateFunctionFactory::Get(AggregationID::kMax).canApplyToTypes(
- std::vector<const Type*>(1, &type));
+ return AggregateFunctionFactory::Get(AggregationID::kMax)
+ .canApplyToTypes(std::vector<const Type *>(1, &type));
}
static bool ResultTypeForArgumentTypeTest(TypeID input_type_id,
TypeID output_type_id) {
- const Type *result_type
- = AggregateFunctionFactory::Get(AggregationID::kMax).resultTypeForArgumentTypes(
- std::vector<const Type*>(1, &TypeFactory::GetType(input_type_id)));
+ const Type *result_type =
+ AggregateFunctionFactory::Get(AggregationID::kMax)
+ .resultTypeForArgumentTypes(std::vector<const Type *>(
+ 1, &TypeFactory::GetType(input_type_id)));
return (result_type->getTypeID() == output_type_id);
}
template <typename CppType>
- static void CheckMaxValue(
- CppType expected,
- const AggregationHandle &handle,
- const AggregationState &state) {
+ static void CheckMaxValue(CppType expected,
+ const AggregationHandle &handle,
+ const AggregationState &state) {
EXPECT_EQ(expected, handle.finalize(state).getLiteral<CppType>());
}
- static void CheckMaxString(
- const std::string &expected,
- const AggregationHandle &handle,
- const AggregationState &state) {
+ template <typename CppType>
+ static void CheckMaxValue(CppType expected, const TypedValue &value) {
+ EXPECT_EQ(expected, value.getLiteral<CppType>());
+ }
+
+ static void CheckMaxString(const std::string &expected,
+ const AggregationHandle &handle,
+ const AggregationState &state) {
TypedValue value = handle.finalize(state);
ASSERT_EQ(expected.length(), value.getAsciiStringLength());
- EXPECT_EQ(0, std::strncmp(expected.c_str(),
- static_cast<const char*>(value.getDataPtr()),
- value.getAsciiStringLength()));
+ EXPECT_EQ(0,
+ std::strncmp(expected.c_str(),
+ static_cast<const char *>(value.getDataPtr()),
+ value.getAsciiStringLength()));
}
// Static templated method to initialize data types.
@@ -130,7 +137,9 @@ class AggregationHandleMaxTest : public ::testing::Test {
void checkAggregationMaxGeneric() {
const GenericType &type = GenericType::Instance(true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_max_->finalize(*aggregation_handle_max_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_max_->finalize(*aggregation_handle_max_state_)
+ .isNull());
typename GenericType::cpptype val;
typename GenericType::cpptype max;
@@ -142,16 +151,18 @@ class AggregationHandleMaxTest : public ::testing::Test {
if (type.getTypeID() == kInt || type.getTypeID() == kLong) {
SetDataType(i * kNumSamples + j - 10, &val);
} else {
- SetDataType(static_cast<float>(i * kNumSamples + j - 10)/10, &val);
+ SetDataType(static_cast<float>(i * kNumSamples + j - 10) / 10, &val);
}
- iterateHandle(aggregation_handle_max_state_.get(), type.makeValue(&val));
+ iterateHandle(aggregation_handle_max_state_.get(),
+ type.makeValue(&val));
if (max < val) {
max = val;
}
}
}
iterateHandle(aggregation_handle_max_state_.get(), type.makeNullValue());
- CheckMaxValue<typename GenericType::cpptype>(max, *aggregation_handle_max_, *aggregation_handle_max_state_);
+ CheckMaxValue<typename GenericType::cpptype>(
+ max, *aggregation_handle_max_, *aggregation_handle_max_state_);
// Test mergeStates().
std::unique_ptr<AggregationState> merge_state(
@@ -165,7 +176,7 @@ class AggregationHandleMaxTest : public ::testing::Test {
if (type.getTypeID() == kInt || type.getTypeID() == kLong) {
SetDataType(i * kNumSamples + j - 20, &val);
} else {
- SetDataType(static_cast<float>(i * kNumSamples + j - 20)/10, &val);
+ SetDataType(static_cast<float>(i * kNumSamples + j - 20) / 10, &val);
}
iterateHandle(merge_state.get(), type.makeValue(&val));
if (max < val) {
@@ -176,14 +187,14 @@ class AggregationHandleMaxTest : public ::testing::Test {
aggregation_handle_max_->mergeStates(*merge_state,
aggregation_handle_max_state_.get());
CheckMaxValue<typename GenericType::cpptype>(
- max,
- *aggregation_handle_max_,
- *aggregation_handle_max_state_);
+ max, *aggregation_handle_max_, *aggregation_handle_max_state_);
}
template <typename GenericType>
- ColumnVector *createColumnVectorGeneric(const Type &type, typename GenericType::cpptype *max) {
- NativeColumnVector *column = new NativeColumnVector(type, kIterations * kNumSamples + 3);
+ ColumnVector* createColumnVectorGeneric(const Type &type,
+ typename GenericType::cpptype *max) {
+ NativeColumnVector *column =
+ new NativeColumnVector(type, kIterations * kNumSamples + 3);
typename GenericType::cpptype val;
SetDataType(0, max);
@@ -194,7 +205,7 @@ class AggregationHandleMaxTest : public ::testing::Test {
if (type.getTypeID() == kInt || type.getTypeID() == kLong) {
SetDataType(i * kNumSamples + j - 10, &val);
} else {
- SetDataType(static_cast<float>(i * kNumSamples + j - 10)/10, &val);
+ SetDataType(static_cast<float>(i * kNumSamples + j - 10) / 10, &val);
}
column->appendTypedValue(type.makeValue(&val));
if (*max < val) {
@@ -202,7 +213,7 @@ class AggregationHandleMaxTest : public ::testing::Test {
}
}
// One NULL in the middle.
- if (i == kIterations/2) {
+ if (i == kIterations / 2) {
column->appendTypedValue(type.makeNullValue());
}
}
@@ -215,11 +226,14 @@ class AggregationHandleMaxTest : public ::testing::Test {
void checkAggregationMaxGenericColumnVector() {
const GenericType &type = GenericType::Instance(true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_max_->finalize(*aggregation_handle_max_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_max_->finalize(*aggregation_handle_max_state_)
+ .isNull());
typename GenericType::cpptype max;
std::vector<std::unique_ptr<ColumnVector>> column_vectors;
- column_vectors.emplace_back(createColumnVectorGeneric<GenericType>(type, &max));
+ column_vectors.emplace_back(
+ createColumnVectorGeneric<GenericType>(type, &max));
std::unique_ptr<AggregationState> cv_state(
aggregation_handle_max_->accumulateColumnVectors(column_vectors));
@@ -227,15 +241,12 @@ class AggregationHandleMaxTest : public ::testing::Test {
// Test the state generated directly by accumulateColumnVectors(), and also
// test after merging back.
CheckMaxValue<typename GenericType::cpptype>(
- max,
- *aggregation_handle_max_,
- *cv_state);
+ max, *aggregation_handle_max_, *cv_state);
- aggregation_handle_max_->mergeStates(*cv_state, aggregation_handle_max_state_.get());
+ aggregation_handle_max_->mergeStates(*cv_state,
+ aggregation_handle_max_state_.get());
CheckMaxValue<typename GenericType::cpptype>(
- max,
- *aggregation_handle_max_,
- *aggregation_handle_max_state_);
+ max, *aggregation_handle_max_, *aggregation_handle_max_state_);
}
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -243,29 +254,29 @@ class AggregationHandleMaxTest : public ::testing::Test {
void checkAggregationMaxGenericValueAccessor() {
const GenericType &type = GenericType::Instance(true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_max_->finalize(*aggregation_handle_max_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_max_->finalize(*aggregation_handle_max_state_)
+ .isNull());
- std::unique_ptr<ColumnVectorsValueAccessor> accessor(new ColumnVectorsValueAccessor());
+ std::unique_ptr<ColumnVectorsValueAccessor> accessor(
+ new ColumnVectorsValueAccessor());
typename GenericType::cpptype max;
accessor->addColumn(createColumnVectorGeneric<GenericType>(type, &max));
std::unique_ptr<AggregationState> va_state(
- aggregation_handle_max_->accumulateValueAccessor(accessor.get(),
- std::vector<attribute_id>(1, 0)));
+ aggregation_handle_max_->accumulateValueAccessor(
+ accessor.get(), std::vector<attribute_id>(1, 0)));
// Test the state generated directly by accumulateValueAccessor(), and also
// test after merging back.
CheckMaxValue<typename GenericType::cpptype>(
- max,
- *aggregation_handle_max_,
- *va_state);
+ max, *aggregation_handle_max_, *va_state);
- aggregation_handle_max_->mergeStates(*va_state, aggregation_handle_max_state_.get());
+ aggregation_handle_max_->mergeStates(*va_state,
+ aggregation_handle_max_state_.get());
CheckMaxValue<typename GenericType::cpptype>(
- max,
- *aggregation_handle_max_,
- *aggregation_handle_max_state_);
+ max, *aggregation_handle_max_, *aggregation_handle_max_state_);
}
#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -273,11 +284,14 @@ class AggregationHandleMaxTest : public ::testing::Test {
void checkAggregationMaxString() {
const StringType &type = StringType::Instance(10, true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_max_->finalize(*aggregation_handle_max_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_max_->finalize(*aggregation_handle_max_state_)
+ .isNull());
std::unique_ptr<UncheckedComparator> fast_comparator_;
- fast_comparator_.reset(ComparisonFactory::GetComparison(ComparisonID::kGreater)
- .makeUncheckedComparatorForTypes(type, type));
+ fast_comparator_.reset(
+ ComparisonFactory::GetComparison(ComparisonID::kGreater)
+ .makeUncheckedComparatorForTypes(type, type));
std::string string_literal;
std::string max = "";
int val;
@@ -291,15 +305,17 @@ class AggregationHandleMaxTest : public ::testing::Test {
iterateHandle(
aggregation_handle_max_state_.get(),
- type.makeValue(string_literal.c_str(),
- string_literal.length() + 1).ensureNotReference());
- if (fast_comparator_->compareDataPtrs(string_literal.c_str(), max.c_str())) {
+ type.makeValue(string_literal.c_str(), string_literal.length() + 1)
+ .ensureNotReference());
+ if (fast_comparator_->compareDataPtrs(string_literal.c_str(),
+ max.c_str())) {
max = string_literal;
}
}
}
iterateHandle(aggregation_handle_max_state_.get(), type.makeNullValue());
- CheckMaxString(max, *aggregation_handle_max_, *aggregation_handle_max_state_);
+ CheckMaxString(
+ max, *aggregation_handle_max_, *aggregation_handle_max_state_);
// Test mergeStates().
std::unique_ptr<AggregationState> merge_state(
@@ -317,24 +333,28 @@ class AggregationHandleMaxTest : public ::testing::Test {
iterateHandle(
merge_state.get(),
- type.makeValue(string_literal.c_str(),
- string_literal.length() + 1).ensureNotReference());
- if (fast_comparator_->compareDataPtrs(string_literal.c_str(), max.c_str())) {
+ type.makeValue(string_literal.c_str(), string_literal.length() + 1)
+ .ensureNotReference());
+ if (fast_comparator_->compareDataPtrs(string_literal.c_str(),
+ max.c_str())) {
max = string_literal;
}
}
}
aggregation_handle_max_->mergeStates(*merge_state,
aggregation_handle_max_state_.get());
- CheckMaxString(max, *aggregation_handle_max_, *aggregation_handle_max_state_);
+ CheckMaxString(
+ max, *aggregation_handle_max_, *aggregation_handle_max_state_);
}
template <typename ColumnVectorType>
- ColumnVector *createColumnVectorString(const Type &type, std::string *max) {
- ColumnVectorType *column = new ColumnVectorType(type, kIterations * kNumSamples + 3);
+ ColumnVector* createColumnVectorString(const Type &type, std::string *max) {
+ ColumnVectorType *column =
+ new ColumnVectorType(type, kIterations * kNumSamples + 3);
std::unique_ptr<UncheckedComparator> fast_comparator_;
- fast_comparator_.reset(ComparisonFactory::GetComparison(ComparisonID::kGreater)
- .makeUncheckedComparatorForTypes(type, type));
+ fast_comparator_.reset(
+ ComparisonFactory::GetComparison(ComparisonID::kGreater)
+ .makeUncheckedComparatorForTypes(type, type));
std::string string_literal;
*max = "";
int val;
@@ -346,14 +366,16 @@ class AggregationHandleMaxTest : public ::testing::Test {
oss << "max" << val;
string_literal = oss.str();
- column->appendTypedValue(type.makeValue(string_literal.c_str(), string_literal.length() + 1)
- .ensureNotReference());
- if (fast_comparator_->compareDataPtrs(string_literal.c_str(), max->c_str())) {
+ column->appendTypedValue(
+ type.makeValue(string_literal.c_str(), string_literal.length() + 1)
+ .ensureNotReference());
+ if (fast_comparator_->compareDataPtrs(string_literal.c_str(),
+ max->c_str())) {
*max = string_literal;
}
}
// One NULL in the middle.
- if (i == kIterations/2) {
+ if (i == kIterations / 2) {
column->appendTypedValue(type.makeNullValue());
}
}
@@ -366,25 +388,26 @@ class AggregationHandleMaxTest : public ::testing::Test {
void checkAggregationMaxStringColumnVector() {
const StringType &type = StringType::Instance(10, true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_max_->finalize(*aggregation_handle_max_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_max_->finalize(*aggregation_handle_max_state_)
+ .isNull());
std::string max;
std::vector<std::unique_ptr<ColumnVector>> column_vectors;
- column_vectors.emplace_back(createColumnVectorString<ColumnVectorType>(type, &max));
+ column_vectors.emplace_back(
+ createColumnVectorString<ColumnVectorType>(type, &max));
std::unique_ptr<AggregationState> cv_state(
aggregation_handle_max_->accumulateColumnVectors(column_vectors));
// Test the state generated directly by accumulateColumnVectors(), and also
// test after merging back.
- CheckMaxString(max,
- *aggregation_handle_max_,
- *cv_state);
-
- aggregation_handle_max_->mergeStates(*cv_state, aggregation_handle_max_state_.get());
- CheckMaxString(max,
- *aggregation_handle_max_,
- *aggregation_handle_max_state_);
+ CheckMaxString(max, *aggregation_handle_max_, *cv_state);
+
+ aggregation_handle_max_->mergeStates(*cv_state,
+ aggregation_handle_max_state_.get());
+ CheckMaxString(
+ max, *aggregation_handle_max_, *aggregation_handle_max_state_);
}
#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -392,26 +415,27 @@ class AggregationHandleMaxTest : public ::testing::Test {
void checkAggregationMaxStringValueAccessor() {
const StringType &type = StringType::Instance(10, true);
initializeHandle(type);
- EXPECT_TRUE(aggregation_handle_max_->finalize(*aggregation_handle_max_state_).isNull());
+ EXPECT_TRUE(
+ aggregation_handle_max_->finalize(*aggregation_handle_max_state_)
+ .isNull());
std::string max;
- std::unique_ptr<ColumnVectorsValueAccessor> accessor(new ColumnVectorsValueAccessor());
+ std::unique_ptr<ColumnVectorsValueAccessor> accessor(
+ new ColumnVectorsValueAccessor());
accessor->addColumn(createColumnVectorString<ColumnVectorType>(type, &max));
std::unique_ptr<AggregationState> va_state(
- aggregation_handle_max_->accumulateValueAccessor(accessor.get(),
- std::vector<attribute_id>(1, 0)));
+ aggregation_handle_max_->accumulateValueAccessor(
+ accessor.get(), std::vector<attribute_id>(1, 0)));
// Test the state generated directly by accumulateValueAccessor(), and also
// test after merging back.
- CheckMaxString(max,
- *aggregation_handle_max_,
- *va_state);
-
- aggregation_handle_max_->mergeStates(*va_state, aggregation_handle_max_state_.get());
- CheckMaxString(max,
- *aggregation_handle_max_,
- *aggregation_handle_max_state_);
+ CheckMaxString(max, *aggregation_handle_max_, *va_state);
+
+ aggregation_handle_max_->mergeStates(*va_state,
+ aggregation_handle_max_state_.get());
+ CheckMaxString(
+ max, *aggregation_handle_max_, *aggregation_handle_max_state_);
}
#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
@@ -422,9 +446,7 @@ class AggregationHandleMaxTest : public ::testing::Test {
template <>
void AggregationHandleMaxTest::CheckMaxValue<float>(
- float val,
- const AggregationHandle &handle,
- const AggregationState &state) {
+ float val, const AggregationHandle &handle, const AggregationState &state) {
EXPECT_FLOAT_EQ(val, handle.finalize(state).getLiteral<float>());
}
@@ -437,17 +459,20 @@ void AggregationHandleMaxTest::CheckMaxValue<double>(
}
template <>
-void AggregationHandleMaxTest::SetDataType<DatetimeLit>(int value, DatetimeLit *data) {
+void AggregationHandleMaxTest::SetDataType<DatetimeLit>(int value,
+ DatetimeLit *data) {
data->ticks = value;
}
template <>
-void AggregationHandleMaxTest::SetDataType<DatetimeIntervalLit>(int value, DatetimeIntervalLit *data) {
+void AggregationHandleMaxTest::SetDataType<DatetimeIntervalLit>(
+ int value, DatetimeIntervalLit *data) {
data->interval_ticks = value;
}
template <>
-void AggregationHandleMaxTest::SetDataType<YearMonthIntervalLit>(int value, YearMonthIntervalLit *data) {
+void AggregationHandleMaxTest::SetDataType<YearMonthIntervalLit>(
+ int value, YearMonthIntervalLit *data) {
data->months = value;
}
@@ -579,50 +604,67 @@ TEST_F(AggregationHandleMaxDeathTest, WrongTypeTest) {
float float_val = 0;
// Passes.
- iterateHandle(aggregation_handle_max_state_.get(), int_non_null_type.makeValue(&int_val));
+ iterateHandle(aggregation_handle_max_state_.get(),
+ int_non_null_type.makeValue(&int_val));
- EXPECT_DEATH(iterateHandle(aggregation_handle_max_state_.get(), long_type.makeValue(&long_val)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_max_state_.get(), double_type.makeValue(&double_val)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_max_state_.get(), float_type.makeValue(&float_val)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_max_state_.get(), char_type.makeValue("asdf", 5)), "");
- EXPECT_DEATH(iterateHandle(aggregation_handle_max_state_.get(), varchar_type.makeValue("asdf", 5)), "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_max_state_.get(),
+ long_type.makeValue(&long_val)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_max_state_.get(),
+ double_type.makeValue(&double_val)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_max_state_.get(),
+ float_type.makeValue(&float_val)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_max_state_.get(),
+ char_type.makeValue("asdf", 5)),
+ "");
+ EXPECT_DEATH(iterateHandle(aggregation_handle_max_state_.get(),
+ varchar_type.makeValue("asdf", 5)),
+ "");
// Test mergeStates() with incorrectly typed handles.
std::unique_ptr<AggregationHandle> aggregation_handle_max_long(
- AggregateFunctionFactory::Get(AggregationID::kMax).createHandle(
- std::vector<const Type*>(1, &long_type)));
+ AggregateFunctionFactory::Get(AggregationID::kMax)
+ .createHandle(std::vector<const Type *>(1, &long_type)));
std::unique_ptr<AggregationState> aggregation_state_max_merge_long(
aggregation_handle_max_long->createInitialState());
- static_cast<const AggregationHandleMax&>(*aggregation_handle_max_long).iterateUnaryInl(
- static_cast<AggregationStateMax*>(aggregation_state_max_merge_long.get()),
- long_type.makeValue(&long_val));
- EXPECT_DEATH(aggregation_handle_max_->mergeStates(*aggregation_state_max_merge_long,
- aggregation_handle_max_state_.get()),
- "");
+ static_cast<const AggregationHandleMax &>(*aggregation_handle_max_long)
+ .iterateUnaryInl(static_cast<AggregationStateMax *>(
+ aggregation_state_max_merge_long.get()),
+ long_type.makeValue(&long_val));
+ EXPECT_DEATH(
+ aggregation_handle_max_->mergeStates(*aggregation_state_max_merge_long,
+ aggregation_handle_max_state_.get()),
+ "");
std::unique_ptr<AggregationHandle> aggregation_handle_max_double(
- AggregateFunctionFactory::Get(AggregationID::kMax).createHandle(
- std::vector<const Type*>(1, &double_type)));
+ AggregateFunctionFactory::Get(AggregationID::kMax)
+ .createHandle(std::vector<const Type *>(1, &double_type)));
std::unique_ptr<AggregationState> aggregation_state_max_merge_double(
aggregation_handle_max_double->createInitialState());
- static_cast<const AggregationHandleMax&>(*aggregation_handle_max_double).iterateUnaryInl(
- static_cast<AggregationStateMax*>(aggregation_state_max_merge_double.get()),
- double_type.makeValue(&double_val));
- EXPECT_DEATH(aggregation_handle_max_->mergeStates(*aggregation_state_max_merge_double,
- aggregation_handle_max_state_.get()),
- "");
+ static_cast<const AggregationHandleMax &>(*aggregation_handle_max_double)
+ .iterateUnaryInl(static_cast<AggregationStateMax *>(
+ aggregation_state_max_merge_double.get()),
+ double_type.makeValue(&double_val));
+ EXPECT_DEATH(
+ aggregation_handle_max_->mergeStates(*aggregation_state_max_merge_double,
+ aggregation_handle_max_state_.get()),
+ "");
std::unique_ptr<AggregationHandle> aggregation_handle_max_float(
- AggregateFunctionFactory::Get(AggregationID::kMax).createHandle(
- std::vector<const Type*>(1, &float_type)));
+ AggregateFunctionFactory::Get(AggregationID::kMax)
+ .createHandle(std::vector<const Type *>(1, &float_type)));
std::unique_ptr<AggregationState> aggregation_state_max_merge_float(
aggregation_handle_max_float->createInitialState());
- static_cast<const AggregationHandleMax&>(*aggregation_handle_max_float).iterateUnaryInl(
- static_cast<AggregationStateMax*>(aggregation_state_max_merge_float.get()),
- float_type.makeValue(&float_val));
- EXPECT_DEATH(aggregation_handle_max_->mergeStates(*aggregation_state_max_merge_float,
- aggregation_handle_max_state_.get()),
- "");
+ static_cast<const AggregationHandleMax &>(*aggregation_handle_max_float)
+ .iterateUnaryInl(static_cast<AggregationStateMax *>(
+ aggregation_state_max_merge_float.get()),
+ float_type.makeValue(&float_val));
+ EXPECT_DEATH(
+ aggregation_handle_max_->mergeStates(*aggregation_state_max_merge_float,
+ aggregation_handle_max_state_.get()),
+ "");
}
#endif
@@ -647,25 +689,28 @@ TEST_F(AggregationHandleMaxTest, GroupByTableMergeTest) {
initializeHandle(int_non_null_type);
storage_manager_.reset(new StorageManager("./test_max_data"));
std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
- aggregation_handle_max_->createGroupByHashTable(
- HashTableImplType::kSimpleScalarSeparateChaining,
+ AggregationStateFastHashTableFactory::CreateResizable(
+ HashTableImplType::kSeparateChaining,
std::vector<const Type *>(1, &int_non_null_type),
10,
+ {aggregation_handle_max_.get()->getPayloadSize()},
+ {aggregation_handle_max_.get()},
storage_manager_.get()));
std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
- aggregation_handle_max_->createGroupByHashTable(
- HashTableImplType::kSimpleScalarSeparateChaining,
+ AggregationStateFastHashTableFactory::CreateResizable(
+ HashTableImplType::kSeparateChaining,
std::vector<const Type *>(1, &int_non_null_type),
10,
+ {aggregation_handle_max_.get()->getPayloadSize()},
+ {aggregation_handle_max_.get()},
storage_manager_.get()));
- AggregationStateHashTable<AggregationStateMax> *destination_hash_table_derived =
- static_cast<AggregationStateHashTable<AggregationStateMax> *>(
+ AggregationStateFastHashTable *destination_hash_table_derived =
+ static_cast<AggregationStateFastHashTable *>(
destination_hash_table.get());
- AggregationStateHashTable<AggregationStateMax> *source_hash_table_derived =
- static_cast<AggregationStateHashTable<AggregationStateMax> *>(
- source_hash_table.get());
+ AggregationStateFastHashTable *source_hash_table_derived =
+ static_cast<AggregationStateFastHashTable *>(source_hash_table.get());
AggregationHandleMax *aggregation_handle_max_derived =
static_cast<AggregationHandleMax *>(aggregation_handle_max_.get());
@@ -730,35 +775,52 @@ TEST_F(AggregationHandleMaxTest, GroupByTableMergeTest) {
EXPECT_EQ(exclusive_key_source_max_val.getLiteral<int>(), actual_val);
// Add the key-state pairs to the hash tables.
- source_hash_table_derived->putCompositeKey(common_key,
- *common_key_source_state);
- destination_hash_table_derived->putCompositeKey(
- common_key, *common_key_destination_state);
- source_hash_table_derived->putCompositeKey(exclusive_source_key,
- *exclusive_key_source_state);
- destination_hash_table_derived->putCompositeKey(
- exclusive_destination_key, *exclusive_key_destination_state);
+ unsigned char buffer[100];
+ buffer[0] = '\0';
+ memcpy(buffer + 1,
+ common_key_source_state.get()->getPayloadAddress(),
+ aggregation_handle_max_.get()->getPayloadSize());
+ source_hash_table_derived->putCompositeKey(common_key, buffer);
+
+ memcpy(buffer + 1,
+ common_key_destination_state.get()->getPayloadAddress(),
+ aggregation_handle_max_.get()->getPayloadSize());
+ destination_hash_table_derived->putCompositeKey(common_key, buffer);
+
+ memcpy(buffer + 1,
+ exclusive_key_source_state.get()->getPayloadAddress(),
+ aggregation_handle_max_.get()->getPayloadSize());
+ source_hash_table_derived->putCompositeKey(exclusive_source_key, buffer);
+
+ memcpy(buffer + 1,
+ exclusive_key_destination_state.get()->getPayloadAddress(),
+ aggregation_handle_max_.get()->getPayloadSize());
+ destination_hash_table_derived->putCompositeKey(exclusive_destination_key,
+ buffer);
EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
EXPECT_EQ(2u, source_hash_table_derived->numEntries());
- aggregation_handle_max_->mergeGroupByHashTables(*source_hash_table,
- destination_hash_table.get());
+ AggregationOperationState::mergeGroupByHashTables(
+ source_hash_table.get(), destination_hash_table.get());
EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
CheckMaxValue<int>(
common_key_destination_max_val.getLiteral<int>(),
- *aggregation_handle_max_derived,
- *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ aggregation_handle_max_derived->finalizeHashTableEntryFast(
+ destination_hash_table_derived->getSingleCompositeKey(common_key) +
+ 1));
CheckMaxValue<int>(exclusive_key_destination_max_val.getLiteral<int>(),
- *aggregation_handle_max_derived,
- *(destination_hash_table_derived->getSingleCompositeKey(
- exclusive_destination_key)));
+ aggregation_handle_max_derived->finalizeHashTableEntryFast(
+ destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key) +
+ 1));
CheckMaxValue<int>(exclusive_key_source_max_val.getLiteral<int>(),
- *aggregation_handle_max_derived,
- *(source_hash_table_derived->getSingleCompositeKey(
- exclusive_source_key)));
+ aggregation_handle_max_derived->finalizeHashTableEntryFast(
+ source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key) +
+ 1));
}
} // namespace quickstep