You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by hb...@apache.org on 2016/06/08 13:21:01 UTC
[01/50] [abbrv] incubator-quickstep git commit: Fixed IWYU include
paths. (#194)
Repository: incubator-quickstep
Updated Branches:
refs/heads/query-manager-used-in-foreman 24c93ca5a -> bef0ae1d8
Fixed IWYU include paths. (#194)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d353a642
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d353a642
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d353a642
Branch: refs/heads/query-manager-used-in-foreman
Commit: d353a642d02d5150f3949fd4575a76a1d06bfc6e
Parents: b9a21b4
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Wed Apr 27 03:23:43 2016 -0700
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Wed Apr 27 05:23:43 2016 -0500
----------------------------------------------------------------------
third_party/iwyu/iwyu_helper.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d353a642/third_party/iwyu/iwyu_helper.py
----------------------------------------------------------------------
diff --git a/third_party/iwyu/iwyu_helper.py b/third_party/iwyu/iwyu_helper.py
index dff1837..a204c50 100755
--- a/third_party/iwyu/iwyu_helper.py
+++ b/third_party/iwyu/iwyu_helper.py
@@ -21,9 +21,10 @@ QUICKSTEP_INCLUDES = [ '.',
'./build/third_party',
'./build/third_party/protobuf/include',
'./build/third_party/gflags/include',
- './third_party/gtest/include',
- './third_party/glog/src',
'./third_party/benchmark/include',
+ './third_party/glog/src',
+ './third_party/googletest/googletest/include',
+ './third_party/re2',
'./third_party/tmb/include']
QUICKSTEP_DEFINES = [ '-DQUICKSTEP_DEBUG',
'-DQUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION', ]
[17/50] [abbrv] incubator-quickstep git commit: Change default
aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining (#210)
Posted by hb...@apache.org.
Change default aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining (#210)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/6c9108a6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/6c9108a6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/6c9108a6
Branch: refs/heads/query-manager-used-in-foreman
Commit: 6c9108a6d468345baa0bab2f9901cf48b84363fa
Parents: bbaff7a
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 5 09:58:45 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:17 2016 -0700
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 69 ++++++++++++--------
.../tests/execution_generator/Select.test | 23 +++----
storage/AggregationOperationState.cpp | 28 +++++++-
storage/AggregationOperationState.hpp | 5 ++
storage/AggregationOperationState.proto | 5 ++
5 files changed, 90 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c9108a6/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index c34f084..077d35d 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -135,7 +135,7 @@ static const volatile bool join_hashtable_type_dummy
= gflags::RegisterFlagValidator(&FLAGS_join_hashtable_type,
&ValidateHashTableImplTypeString);
-DEFINE_string(aggregate_hashtable_type, "LinearOpenAddressing",
+DEFINE_string(aggregate_hashtable_type, "SeparateChaining",
"HashTable implementation to use for aggregates with GROUP BY "
"(valid options are SeparateChaining or LinearOpenAddressing)");
static const volatile bool aggregate_hashtable_type_dummy
@@ -1285,6 +1285,35 @@ void ExecutionGenerator::convertAggregate(
findRelationInfoOutputByPhysical(physical_plan->input());
aggr_state_proto->set_relation_id(input_relation_info->relation->getID());
+ std::vector<const Type*> group_by_types;
+ for (const E::NamedExpressionPtr &grouping_expression : physical_plan->grouping_expressions()) {
+ unique_ptr<const Scalar> execution_group_by_expression;
+ E::AliasPtr alias;
+ if (E::SomeAlias::MatchesWithConditionalCast(grouping_expression, &alias)) {
+ E::ScalarPtr scalar;
+ // NOTE(zuyu): For aggregate expressions, all child expressions of an
+ // Alias should be a Scalar.
+ CHECK(E::SomeScalar::MatchesWithConditionalCast(alias->expression(), &scalar))
+ << alias->toString();
+ execution_group_by_expression.reset(scalar->concretize(attribute_substitution_map_));
+ } else {
+ execution_group_by_expression.reset(
+ grouping_expression->concretize(attribute_substitution_map_));
+ }
+ aggr_state_proto->add_group_by_expressions()->CopyFrom(execution_group_by_expression->getProto());
+ group_by_types.push_back(&execution_group_by_expression->getType());
+ }
+
+ if (!group_by_types.empty()) {
+ // SimplifyHashTableImplTypeProto() switches the hash table implementation
+ // from SeparateChaining to SimpleScalarSeparateChaining when there is a
+ // single scalar key type with a reversible hash function.
+ aggr_state_proto->set_hash_table_impl_type(
+ SimplifyHashTableImplTypeProto(
+ HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
+ group_by_types));
+ }
+
for (const E::AliasPtr &named_aggregate_expression : physical_plan->aggregate_expressions()) {
const E::AggregateFunctionPtr unnamed_aggregate_expression =
std::static_pointer_cast<const E::AggregateFunction>(named_aggregate_expression->expression());
@@ -1304,25 +1333,19 @@ void ExecutionGenerator::convertAggregate(
// Set whether it is a DISTINCT aggregation.
aggr_proto->set_is_distinct(unnamed_aggregate_expression->is_distinct());
- }
- std::vector<const Type*> group_by_types;
- for (const E::NamedExpressionPtr &grouping_expression : physical_plan->grouping_expressions()) {
- unique_ptr<const Scalar> execution_group_by_expression;
- E::AliasPtr alias;
- if (E::SomeAlias::MatchesWithConditionalCast(grouping_expression, &alias)) {
- E::ScalarPtr scalar;
- // NOTE(zuyu): For aggregate expressions, all child expressions of an
- // Alias should be a Scalar.
- CHECK(E::SomeScalar::MatchesWithConditionalCast(alias->expression(), &scalar))
- << alias->toString();
- execution_group_by_expression.reset(scalar->concretize(attribute_substitution_map_));
- } else {
- execution_group_by_expression.reset(
- grouping_expression->concretize(attribute_substitution_map_));
+ // Add distinctify hash table impl type if it is a DISTINCT aggregation.
+ if (unnamed_aggregate_expression->is_distinct()) {
+ if (group_by_types.empty()) {
+ aggr_state_proto->add_distinctify_hash_table_impl_types(
+ SimplifyHashTableImplTypeProto(
+ HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
+ {&unnamed_aggregate_expression->getValueType()}));
+ } else {
+ aggr_state_proto->add_distinctify_hash_table_impl_types(
+ HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type));
+ }
}
- aggr_state_proto->add_group_by_expressions()->CopyFrom(execution_group_by_expression->getProto());
- group_by_types.push_back(&execution_group_by_expression->getType());
}
if (physical_plan->filter_predicate() != nullptr) {
@@ -1332,16 +1355,6 @@ void ExecutionGenerator::convertAggregate(
aggr_state_proto->set_estimated_num_entries(cost_model_->estimateCardinality(physical_plan));
- if (!group_by_types.empty()) {
- // SimplifyHashTableImplTypeProto() switches the hash table implementation
- // from SeparateChaining to SimpleScalarSeparateChaining when there is a
- // single scalar key type with a reversible hash function.
- aggr_state_proto->set_hash_table_impl_type(
- SimplifyHashTableImplTypeProto(
- HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
- group_by_types));
- }
-
const QueryPlan::DAGNodeIndex aggregation_operator_index =
execution_plan_->addRelationalOperator(
new AggregationOperator(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c9108a6/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index a08b012..390b7b6 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -535,21 +535,12 @@ WHERE int_col = 2;
SELECT int_col
FROM test
-GROUP BY int_col;
+GROUP BY int_col
+ORDER BY int_col;
--
+-----------+
|int_col |
+-----------+
-| 2|
-| 4|
-| 6|
-| 8|
-| 12|
-| 14|
-| 16|
-| 18|
-| 22|
-| 24|
| -23|
| -21|
| -19|
@@ -562,6 +553,16 @@ GROUP BY int_col;
| -5|
| -3|
| -1|
+| 2|
+| 4|
+| 6|
+| 8|
+| 12|
+| 14|
+| 16|
+| 18|
+| 22|
+| 24|
+-----------+
==
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c9108a6/storage/AggregationOperationState.cpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.cpp b/storage/AggregationOperationState.cpp
index a3a669c..d209ceb 100644
--- a/storage/AggregationOperationState.cpp
+++ b/storage/AggregationOperationState.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -64,6 +66,7 @@ AggregationOperationState::AggregationOperationState(
const Predicate *predicate,
const std::size_t estimated_num_entries,
const HashTableImplType hash_table_impl_type,
+ const std::vector<HashTableImplType> &distinctify_hash_table_impl_types,
StorageManager *storage_manager)
: input_relation_(input_relation),
predicate_(predicate),
@@ -101,6 +104,8 @@ AggregationOperationState::AggregationOperationState(
std::vector<std::vector<std::unique_ptr<const Scalar>>>::const_iterator args_it
= arguments_.begin();
std::vector<bool>::const_iterator is_distinct_it = is_distinct_.begin();
+ std::vector<HashTableImplType>::const_iterator distinctify_hash_table_impl_types_it
+ = distinctify_hash_table_impl_types.begin();
for (; agg_func_it != aggregate_functions.end(); ++agg_func_it, ++args_it, ++is_distinct_it) {
// Get the Types of this aggregate's arguments so that we can create an
// AggregationHandle.
@@ -161,10 +166,11 @@ AggregationOperationState::AggregationOperationState(
// query optimization, if it worths.
distinctify_hashtables_.emplace_back(
handles_.back()->createDistinctifyHashTable(
- hash_table_impl_type,
+ *distinctify_hash_table_impl_types_it,
key_types,
estimated_num_entries,
storage_manager));
+ ++distinctify_hash_table_impl_types_it;
} else {
distinctify_hashtables_.emplace_back(nullptr);
}
@@ -182,6 +188,8 @@ AggregationOperationState* AggregationOperationState::ReconstructFromProto(
std::vector<const AggregateFunction*> aggregate_functions;
std::vector<std::vector<std::unique_ptr<const Scalar>>> arguments;
std::vector<bool> is_distinct;
+ std::vector<HashTableImplType> distinctify_hash_table_impl_types;
+ std::size_t distinctify_hash_table_impl_type_index = 0;
for (int agg_idx = 0; agg_idx < proto.aggregates_size(); ++agg_idx) {
const serialization::Aggregate &agg_proto = proto.aggregates(agg_idx);
@@ -197,6 +205,13 @@ AggregationOperationState* AggregationOperationState::ReconstructFromProto(
}
is_distinct.emplace_back(agg_proto.is_distinct());
+
+ if (agg_proto.is_distinct()) {
+ distinctify_hash_table_impl_types.emplace_back(
+ HashTableImplTypeFromProto(
+ proto.distinctify_hash_table_impl_types(distinctify_hash_table_impl_type_index)));
+ ++distinctify_hash_table_impl_type_index;
+ }
}
std::vector<std::unique_ptr<const Scalar>> group_by_expressions;
@@ -223,6 +238,7 @@ AggregationOperationState* AggregationOperationState::ReconstructFromProto(
predicate.release(),
proto.estimated_num_entries(),
HashTableImplTypeFromProto(proto.hash_table_impl_type()),
+ distinctify_hash_table_impl_types,
storage_manager);
}
@@ -234,6 +250,8 @@ bool AggregationOperationState::ProtoIsValid(const serialization::AggregationOpe
return false;
}
+ std::size_t num_distinctify_hash_tables = proto.distinctify_hash_table_impl_types_size();
+ std::size_t distinctify_hash_table_impl_type_index = 0;
for (int i = 0; i < proto.aggregates_size(); ++i) {
if (!AggregateFunctionFactory::ProtoIsValid(proto.aggregates(i).function())) {
return false;
@@ -251,6 +269,14 @@ bool AggregationOperationState::ProtoIsValid(const serialization::AggregationOpe
return false;
}
}
+
+ if (proto.aggregates(i).is_distinct()) {
+ if (distinctify_hash_table_impl_type_index >= num_distinctify_hash_tables ||
+ !serialization::HashTableImplType_IsValid(
+ proto.distinctify_hash_table_impl_types(distinctify_hash_table_impl_type_index))) {
+ return false;
+ }
+ }
}
for (int i = 0; i < proto.group_by_expressions_size(); ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c9108a6/storage/AggregationOperationState.hpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.hpp b/storage/AggregationOperationState.hpp
index b883ed1..c3a1278 100644
--- a/storage/AggregationOperationState.hpp
+++ b/storage/AggregationOperationState.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -93,6 +95,8 @@ class AggregationOperationState {
* in the input relation.
* @param hash_table_impl_type The HashTable implementation to use for
* GROUP BY. Ignored if group_by is empty.
+ * @param distinctify_hash_table_impl_type The HashTable implementation to use
+ * for the distinctify phase of each DISTINCT aggregation.
* @param storage_manager The StorageManager to use for allocating hash
* tables. Single aggregation state (when GROUP BY list is not
* specified) is not allocated using memory from storage manager.
@@ -105,6 +109,7 @@ class AggregationOperationState {
const Predicate *predicate,
const std::size_t estimated_num_entries,
const HashTableImplType hash_table_impl_type,
+ const std::vector<HashTableImplType> &distinctify_hash_table_impl_types,
StorageManager *storage_manager);
~AggregationOperationState() {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c9108a6/storage/AggregationOperationState.proto
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.proto b/storage/AggregationOperationState.proto
index 031f782..bf78e3a 100644
--- a/storage/AggregationOperationState.proto
+++ b/storage/AggregationOperationState.proto
@@ -1,5 +1,7 @@
// Copyright 2011-2015 Quickstep Technologies LLC.
// Copyright 2015-2016 Pivotal Software, Inc.
+// Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+// University of Wisconsin\u2014Madison.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -37,4 +39,7 @@ message AggregationOperationState {
// NOTE(chasseur): 'hash_table_impl_type' is marked optional, but it is
// needed if 'group_by_expressions' is non-empty, and ignored otherwise.
optional HashTableImplType hash_table_impl_type = 6;
+
+ // Each DISTINCT aggregation has its distinctify hash table impl type.
+ repeated HashTableImplType distinctify_hash_table_impl_types = 7;
}
[43/50] [abbrv] incubator-quickstep git commit: Revert "Explicitly
specify where tcmalloc comes from" (#237)
Posted by hb...@apache.org.
Revert "Explicitly specify where tcmalloc comes from" (#237)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5bda90ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5bda90ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5bda90ea
Branch: refs/heads/query-manager-used-in-foreman
Commit: 5bda90ea8cb478e37e9a5de31a46c70a85c732cc
Parents: 908d367
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Mon May 23 22:56:22 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:53 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5bda90ea/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2ab0f57..dc51ca6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -490,7 +490,9 @@ if(USE_TCMALLOC)
CXXFLAGS=${THIRD_PARTY_CXX_FLAGS}
BUILD_COMMAND make
BUILD_IN_SOURCE 0
- BUILD_BYPRODUCTS <INSTALL_DIR>/lib/libtcmalloc_minimal.a
+ # Uncomment the next line to change the path of the build by products
+ # as some generators, e.g. Ninja, may need it to build properly
+ # BUILD_BYPRODUCTS <INSTALL_DIR>/lib/libtcmalloc_minimal.a
)
# Static libtcmalloc_minimal.a
add_library(libtcmalloc_minimal STATIC IMPORTED)
[32/50] [abbrv] incubator-quickstep git commit: Quickstep print
catalog (#222)
Posted by hb...@apache.org.
Quickstep print catalog (#222)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3c845376
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3c845376
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3c845376
Branch: refs/heads/query-manager-used-in-foreman
Commit: 3c8453766249d070b034b7a16e48aeff6a840e42
Parents: 2c0722e
Author: Rogers Jeffrey Leo John <ro...@gmail.com>
Authored: Tue May 17 18:43:02 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:45 2016 -0700
----------------------------------------------------------------------
cli/CMakeLists.txt | 3 +-
cli/CommandExecutor.cpp | 50 +++++++++++++++++++++++-----
cli/CommandExecutor.hpp | 8 +++--
cli/PrintToScreen.cpp | 11 ++++++
cli/PrintToScreen.hpp | 8 +++++
cli/QuickstepCli.cpp | 4 ++-
cli/tests/CommandExecutorTestRunner.cpp | 1 +
cli/tests/command_executor/D.test | 35 ++++++++++++-------
cli/tests/command_executor/Dt.test | 36 +++++++++++++++-----
9 files changed, 124 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index a1989d5..761b6d8 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -73,8 +73,9 @@ target_link_libraries(quickstep_cli_CommandExecutor
quickstep_catalog_CatalogDatabase
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogRelationSchema
+ quickstep_cli_PrintToScreen
quickstep_parser_ParseStatement
- quickstep_cli_PrintToScreen
+ quickstep_storage_StorageBlockInfo
quickstep_utility_Macros
quickstep_utility_PtrVector
quickstep_utility_SqlError)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index f38121f..026922a 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -30,6 +30,7 @@
#include "catalog/CatalogRelationSchema.hpp"
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
+#include "storage/StorageBlockInfo.hpp"
#include "utility/PtrVector.hpp"
#include "utility/Macros.hpp"
#include "utility/SqlError.hpp"
@@ -52,15 +53,22 @@ namespace C = ::quickstep::cli;
void executeDescribeDatabase(
const PtrVector<ParseString> *arguments,
- const CatalogDatabase &catalog_database, FILE *out) {
+ const CatalogDatabase &catalog_database,
+ StorageManager *storage_manager,
+ FILE *out) {
// Column width initialized to 6 to take into account the header name
// and the column value table
int max_column_width = C::kInitMaxColumnWidth;
+ vector<std::size_t> num_tuples;
+ vector<std::size_t> num_blocks;
const CatalogRelation *relation = nullptr;
if (arguments->size() == 0) {
for (const CatalogRelation &rel : catalog_database) {
max_column_width =
std::max(static_cast<int>(rel.getName().length()), max_column_width);
+ num_blocks.push_back(rel.size_blocks());
+ num_tuples.push_back(
+ PrintToScreen::GetNumTuplesInRelation(rel, storage_manager));
}
} else {
const ParseString &table_name = arguments->front();
@@ -72,26 +80,51 @@ void executeDescribeDatabase(
}
max_column_width = std::max(static_cast<int>(relation->getName().length()),
max_column_width);
+ num_blocks.push_back(relation->size_blocks());
+ num_tuples.push_back(PrintToScreen::GetNumTuplesInRelation(
+ *relation,
+ storage_manager));
}
// Only if we have relations work on the printing logic.
if (catalog_database.size() > 0) {
+ const std::size_t max_num_blocks = *std::max_element(num_blocks.begin(), num_blocks.end());
+ const std::size_t max_num_rows = *std::max_element(num_tuples.begin(), num_tuples.end());
+ const int max_num_rows_digits = std::max(PrintToScreen::GetNumberOfDigits(max_num_rows),
+ C::kInitMaxColumnWidth);
+ const int max_num_blocks_digits = std::max(PrintToScreen::GetNumberOfDigits(max_num_blocks),
+ C::kInitMaxColumnWidth+2);
+
vector<int> column_widths;
- column_widths.push_back(max_column_width+1);
- column_widths.push_back(C::kInitMaxColumnWidth+1);
+ column_widths.push_back(max_column_width +1);
+ column_widths.push_back(C::kInitMaxColumnWidth + 1);
+ column_widths.push_back(max_num_blocks_digits + 1);
+ column_widths.push_back(max_num_rows_digits + 1);
fputs(" List of relations\n\n", out);
fprintf(out, "%-*s |", max_column_width+1, " Name");
- fprintf(out, "%-*s\n", C::kInitMaxColumnWidth, " Type");
+ fprintf(out, "%-*s |", C::kInitMaxColumnWidth, " Type");
+ fprintf(out, "%-*s |", max_num_blocks_digits, " Blocks");
+ fprintf(out, "%-*s\n", max_num_rows_digits, " Rows");
PrintToScreen::printHBar(column_widths, out);
// If there are no argument print the entire list of tables
// else print the particular table only.
+ vector<std::size_t>::const_iterator num_tuples_it = num_tuples.begin();
+ vector<std::size_t>::const_iterator num_blocks_it = num_blocks.begin();
if (arguments->size() == 0) {
for (const CatalogRelation &rel : catalog_database) {
fprintf(out, " %-*s |", max_column_width, rel.getName().c_str());
- fprintf(out, " %-*s\n", C::kInitMaxColumnWidth, "table");
+ fprintf(out, " %-*s |", C::kInitMaxColumnWidth - 1, "table");
+ fprintf(out, " %-*lu |", max_num_blocks_digits - 1, *num_blocks_it);
+ fprintf(out, " %-*lu\n", max_num_rows_digits - 1, *num_tuples_it);
+ ++num_tuples_it;
+ ++num_blocks_it;
}
} else {
fprintf(out, " %-*s |", max_column_width, relation->getName().c_str());
- fprintf(out, " %-*s\n", C::kInitMaxColumnWidth, "table");
+ fprintf(out, " %-*s |", C::kInitMaxColumnWidth -1, "table");
+ fprintf(out, " %-*lu |", max_num_blocks_digits - 1, *num_blocks_it);
+ fprintf(out, " %-*lu\n", max_num_rows_digits - 1, *num_tuples_it);
+ ++num_tuples_it;
+ ++num_blocks_it;
}
fputc('\n', out);
}
@@ -166,15 +199,16 @@ void executeDescribeTable(
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
+ StorageManager *storage_manager,
FILE *out) {
const ParseCommand &command = static_cast<const ParseCommand &>(statement);
const PtrVector<ParseString> *arguments = command.arguments();
const std::string &command_str = command.command()->value();
if (command_str == C::kDescribeDatabaseCommand) {
- executeDescribeDatabase(arguments, catalog_database, out);
+ executeDescribeDatabase(arguments, catalog_database, storage_manager, out);
} else if (command_str == C::kDescribeTableCommand) {
if (arguments->size() == 0) {
- executeDescribeDatabase(arguments, catalog_database, out);
+ executeDescribeDatabase(arguments, catalog_database, storage_manager, out);
} else {
executeDescribeTable(arguments, catalog_database, out);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/CommandExecutor.hpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.hpp b/cli/CommandExecutor.hpp
index 21eee6a..f367ca1 100644
--- a/cli/CommandExecutor.hpp
+++ b/cli/CommandExecutor.hpp
@@ -19,9 +19,11 @@
#define QUICKSTEP_CLI_COMMAND_COMMAND_EXECUTOR_HPP_
#include <cstdio>
+#include <limits>
#include <string>
#include "parser/ParseStatement.hpp"
+#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
using std::fprintf;
@@ -33,6 +35,7 @@ namespace quickstep {
class CatalogDatabase;
class CatalogAttribute;
class CatalogRelation;
+class StorageManager;
namespace cli {
/** \addtogroup CLI
@@ -49,13 +52,14 @@ constexpr char kDescribeTableCommand[] = "\\d";
/**
* @brief Executes the command by calling the command handler.
- *
+ *
* @param statement The parsed statement from the cli.
* @param catalog_database The catalog information about the current database.
- * @param out The stream where the output of the command has to be redirected to.
+ * @param out The stream where the output of the command has to be redirected to.
*/
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
+ StorageManager *storage_manager,
FILE *out);
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/PrintToScreen.cpp
----------------------------------------------------------------------
diff --git a/cli/PrintToScreen.cpp b/cli/PrintToScreen.cpp
index 227ff39..76e90eb 100644
--- a/cli/PrintToScreen.cpp
+++ b/cli/PrintToScreen.cpp
@@ -19,6 +19,7 @@
#include <cstddef>
#include <cstdio>
+#include <cmath>
#include <memory>
#include <vector>
@@ -47,6 +48,16 @@ DEFINE_bool(printing_enabled, true,
"If true, print query results to screen normally. If false, skip "
"printing output (e.g. for benchmarking).");
+int PrintToScreen::GetNumberOfDigits(int number) {
+ if (number > 0) {
+ return static_cast<int>(log10 (number)) + 1;
+ } else if (number < 0) {
+ return static_cast<int>(log10 ( abs(number) )) + 2;
+ } else {
+ return 1;
+ }
+}
+
void PrintToScreen::PrintRelation(const CatalogRelation &relation,
StorageManager *storage_manager,
FILE *out) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/PrintToScreen.hpp
----------------------------------------------------------------------
diff --git a/cli/PrintToScreen.hpp b/cli/PrintToScreen.hpp
index 0b57b4b..6a29426 100644
--- a/cli/PrintToScreen.hpp
+++ b/cli/PrintToScreen.hpp
@@ -69,6 +69,14 @@ class PrintToScreen {
StorageManager *storage_manager,
FILE *out);
+ /**
+ * @brief Return the number of digits in a number
+ *
+ * @param number The input number.
+ * @param out The number of digits in the input number.
+ **/
+ static int GetNumberOfDigits(int number);
+
private:
// Undefined default constructor. Class is all-static and should not be
// instantiated.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 5881b3e..b7b28ba 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -365,7 +365,9 @@ int main(int argc, char* argv[]) {
try {
quickstep::cli::executeCommand(
*result.parsed_statement,
- *(query_processor->getDefaultDatabase()), stdout);
+ *(query_processor->getDefaultDatabase()),
+ query_processor->getStorageManager(),
+ stdout);
} catch (const quickstep::SqlError &sql_error) {
fprintf(stderr, "%s",
sql_error.formatMessage(*command_string).c_str());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/tests/CommandExecutorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/cli/tests/CommandExecutorTestRunner.cpp b/cli/tests/CommandExecutorTestRunner.cpp
index 49930e1..73d2092 100644
--- a/cli/tests/CommandExecutorTestRunner.cpp
+++ b/cli/tests/CommandExecutorTestRunner.cpp
@@ -87,6 +87,7 @@ void CommandExecutorTestRunner::runTestCase(
quickstep::cli::executeCommand(
*result.parsed_statement,
*(test_database_loader_.catalog_database()),
+ test_database_loader_.storage_manager(),
output_stream.file());
} else {
QueryHandle query_handle(optimizer_context.query_id());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/tests/command_executor/D.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/D.test b/cli/tests/command_executor/D.test
index 1d500df..45f8d8b 100644
--- a/cli/tests/command_executor/D.test
+++ b/cli/tests/command_executor/D.test
@@ -34,9 +34,21 @@ CREATE TABLE foo4 (col1 INT,
col3 DOUBLE,
col4 FLOAT,
col5 CHAR(5));
-CREATE INDEX foo4_index_1 ON foo4 (col1, col2) USING CSBTREE;
-CREATE INDEX foo4_index_2 ON foo4 (col3, col4) USING CSBTREE;
+CREATE INDEX foo4_index_1 ON foo4 (col1, col2) USING CSBTREE;
+CREATE INDEX foo4_index_2 ON foo4 (col3, col4) USING CSBTREE;
CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
+DROP TABLE TEST;
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (1);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (2);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (3);
+INSERT INTO foo values(1, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(2, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(3, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(4, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo2 values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo2 values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
--
==
\d foo
@@ -50,7 +62,7 @@ CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
col4 | Float
col5 | Char(5)
==
-\d foo2
+\d foo2
--
Table "foo2"
Column | Type
@@ -73,7 +85,7 @@ CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
col5 | Char(5)
Indexes
"foo3_index_1" CSB_TREE (col1)
-==
+==
\d foo4
--
Table "foo4"
@@ -92,14 +104,13 @@ CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
--
List of relations
- Name | Type
-+--------------------------------------+-------+
- Test | table
- foo | table
- foo2 | table
- foo3 | table
- foo4 | table
- averylongtablenamethatseemstoneverend | table
+ Name | Type | Blocks | Rows
++--------------------------------------+-------+---------+-------+
+ foo | table | 1 | 5
+ foo2 | table | 1 | 2
+ foo3 | table | 1 | 1
+ foo4 | table | 0 | 0
+ averylongtablenamethatseemstoneverend | table | 1 | 3
==
\d invalidtable
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/tests/command_executor/Dt.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/Dt.test b/cli/tests/command_executor/Dt.test
index 6458e15..1de6360 100644
--- a/cli/tests/command_executor/Dt.test
+++ b/cli/tests/command_executor/Dt.test
@@ -33,21 +33,41 @@ CREATE TABLE foo4 (col1 INT,
col3 DOUBLE,
col4 FLOAT,
col5 CHAR(5));
+DROP TABLE TEST;
CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (1);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (2);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (3);
+INSERT INTO foo values(1, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(2, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(3, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(4, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo2 values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo2 values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
--
==
\dt
--
List of relations
- Name | Type
-+--------------------------------------+-------+
- Test | table
- foo | table
- foo2 | table
- foo3 | table
- foo4 | table
- averylongtablenamethatseemstoneverend | table
+ Name | Type | Blocks | Rows
++--------------------------------------+-------+---------+-------+
+ foo | table | 1 | 5
+ foo2 | table | 1 | 2
+ foo3 | table | 1 | 1
+ foo4 | table | 0 | 0
+ averylongtablenamethatseemstoneverend | table | 1 | 3
+
+==
+\dt foo
+--
+ List of relations
+
+ Name | Type | Blocks | Rows
++-------+-------+---------+-------+
+ foo | table | 1 | 5
==
\dt invalidtable
[38/50] [abbrv] incubator-quickstep git commit: Visualize optimized
physical plan in DOT format (#232)
Posted by hb...@apache.org.
Visualize optimized physical plan in DOT format (#232)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1605fd84
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1605fd84
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1605fd84
Branch: refs/heads/query-manager-used-in-foreman
Commit: 1605fd8445a59a7cc38bf23088d9af434a2b75eb
Parents: d136e1d
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Fri May 20 16:47:31 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:52 2016 -0700
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 3 +-
query_optimizer/PhysicalGenerator.cpp | 13 ++-
utility/CMakeLists.txt | 13 +++
utility/PlanVisualizer.cpp | 161 +++++++++++++++++++++++++++++
utility/PlanVisualizer.hpp | 94 +++++++++++++++++
utility/StringUtil.hpp | 30 +++++-
6 files changed, 310 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1605fd84/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index aa2873e..5c9438d 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -194,7 +194,8 @@ target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
quickstep_queryoptimizer_strategy_Selection
quickstep_queryoptimizer_strategy_Strategy
quickstep_queryoptimizer_Validator
- quickstep_utility_Macros)
+ quickstep_utility_Macros
+ quickstep_utility_PlanVisualizer)
target_link_libraries(quickstep_queryoptimizer_QueryHandle
quickstep_catalog_Catalog_proto
quickstep_queryexecution_QueryContext_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1605fd84/query_optimizer/PhysicalGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/PhysicalGenerator.cpp b/query_optimizer/PhysicalGenerator.cpp
index 662236f..75a7bc9 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -33,6 +33,7 @@
#include "query_optimizer/strategy/OneToOne.hpp"
#include "query_optimizer/strategy/Selection.hpp"
#include "query_optimizer/strategy/Strategy.hpp"
+#include "utility/PlanVisualizer.hpp"
#include "gflags/gflags.h"
@@ -45,7 +46,12 @@ DEFINE_bool(reorder_hash_joins, true,
"If true, apply hash join order optimization to each group of hash "
"joins. The optimization applies a greedy algorithm to favor smaller "
"cardinality and selective tables to be joined first, which is suitable "
- "for queries on star-schema tables");
+ "for queries on star-schema tables.");
+
+DEFINE_bool(visualize_plan, false,
+ "If true, visualize the final physical plan into a graph in DOT format "
+ "(DOT is a plain text graph description language). Then print the "
+ "generated graph through stderr.");
namespace L = ::quickstep::optimizer::logical;
namespace P = ::quickstep::optimizer::physical;
@@ -101,6 +107,11 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
DVLOG(4) << "Optimized physical plan:\n" << physical_plan_->toString();
+ if (FLAGS_visualize_plan) {
+ quickstep::PlanVisualizer plan_visualizer;
+ std::cerr << "\n" << plan_visualizer.visualize(physical_plan_) << "\n";
+ }
+
#ifdef QUICKSTEP_DEBUG
Validate(physical_plan_);
#endif
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1605fd84/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index 6d1eeab..2d3db8f 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -171,6 +171,7 @@ add_library(quickstep_utility_Glob Glob.cpp Glob.hpp)
add_library(quickstep_utility_HashPair ../empty_src.cpp HashPair.hpp)
add_library(quickstep_utility_Macros ../empty_src.cpp Macros.hpp)
add_library(quickstep_utility_MemStream ../empty_src.cpp MemStream.hpp)
+add_library(quickstep_utility_PlanVisualizer PlanVisualizer.cpp PlanVisualizer.hpp)
add_library(quickstep_utility_PrimeNumber PrimeNumber.cpp PrimeNumber.hpp)
add_library(quickstep_utility_PtrList ../empty_src.cpp PtrList.hpp)
add_library(quickstep_utility_PtrMap ../empty_src.cpp PtrMap.hpp)
@@ -231,6 +232,17 @@ target_link_libraries(quickstep_utility_MemStream
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_PrimeNumber
glog)
+target_link_libraries(quickstep_utility_PlanVisualizer
+ quickstep_catalog_CatalogRelation
+ quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_queryoptimizer_physical_TableReference
+ quickstep_queryoptimizer_physical_TopLevelPlan
+ quickstep_utility_Macros
+ quickstep_utility_StringUtil)
target_link_libraries(quickstep_utility_PtrList
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_PtrMap
@@ -295,6 +307,7 @@ target_link_libraries(quickstep_utility
quickstep_utility_HashPair
quickstep_utility_Macros
quickstep_utility_MemStream
+ quickstep_utility_PlanVisualizer
quickstep_utility_PrimeNumber
quickstep_utility_PtrList
quickstep_utility_PtrMap
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1605fd84/utility/PlanVisualizer.cpp
----------------------------------------------------------------------
diff --git a/utility/PlanVisualizer.cpp b/utility/PlanVisualizer.cpp
new file mode 100644
index 0000000..962d577
--- /dev/null
+++ b/utility/PlanVisualizer.cpp
@@ -0,0 +1,161 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "utility/PlanVisualizer.hpp"
+
+#include <cstddef>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "catalog/CatalogRelation.hpp"
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/PhysicalType.hpp"
+#include "query_optimizer/physical/TableReference.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+#include "utility/StringUtil.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+namespace E = ::quickstep::optimizer::expressions;
+namespace P = ::quickstep::optimizer::physical;
+namespace C = ::quickstep::optimizer::cost;
+
+std::string PlanVisualizer::visualize(const P::PhysicalPtr &input) {
+ DCHECK(input->getPhysicalType() == P::PhysicalType::kTopLevelPlan);
+ cost_model_.reset(
+ new C::StarSchemaSimpleCostModel(
+ std::static_pointer_cast<const P::TopLevelPlan>(input)->shared_subplans()));
+
+ color_map_["TableReference"] = "skyblue";
+ color_map_["Selection"] = "#90EE90";
+ color_map_["HashJoin"] = "red";
+ color_map_["HashLeftOuterJoin"] = "orange";
+ color_map_["HashLeftSemiJoin"] = "orange";
+ color_map_["HashLeftAntiJoin"] = "orange";
+
+ visit(input);
+
+ // Format output graph
+ std::ostringstream graph_oss;
+ graph_oss << "digraph g {\n";
+ graph_oss << " rankdir=BT\n";
+ graph_oss << " node [penwidth=2]\n";
+ graph_oss << " edge [fontsize=16 fontcolor=gray penwidth=2]\n\n";
+
+ // Format nodes
+ for (const NodeInfo &node_info : nodes_) {
+ graph_oss << " " << node_info.id << " [";
+ if (!node_info.labels.empty()) {
+ graph_oss << "label=\""
+ << EscapeSpecialChars(JoinToString(node_info.labels, " "))
+ << "\"";
+ }
+ if (!node_info.color.empty()) {
+ graph_oss << " style=filled fillcolor=\"" << node_info.color << "\"";
+ }
+ graph_oss << "]\n";
+ }
+ graph_oss << "\n";
+
+ // Format edges
+ for (const EdgeInfo &edge_info : edges_) {
+ graph_oss << " " << edge_info.src_node_id << " -> "
+ << edge_info.dst_node_id << " [";
+ if (!edge_info.labels.empty()) {
+ graph_oss << "label=\""
+ << EscapeSpecialChars(JoinToString(edge_info.labels, " "))
+ << "\"";
+ }
+ graph_oss << "]\n";
+ }
+
+ graph_oss << "}\n";
+
+ return graph_oss.str();
+}
+
+void PlanVisualizer::visit(const P::PhysicalPtr &input) {
+ int node_id = ++id_counter_;
+ node_id_map_.emplace(input, node_id);
+
+ for (const auto &child : input->children()) {
+ visit(child);
+
+ int child_id = node_id_map_[child];
+
+ edges_.emplace_back(EdgeInfo());
+ EdgeInfo &edge_info = edges_.back();
+ edge_info.src_node_id = child_id;
+ edge_info.dst_node_id = node_id;
+
+ // Print output attributes except for TableReference -- there are just too many
+ // attributes out of TableReference.
+ if (child->getPhysicalType() != P::PhysicalType::kTableReference) {
+ for (const auto &attr : child->getOutputAttributes()) {
+ edge_info.labels.emplace_back(attr->attribute_alias());
+ }
+ }
+ }
+
+ nodes_.emplace_back(NodeInfo());
+ NodeInfo &node_info = nodes_.back();
+ node_info.id = node_id;
+ if (color_map_.find(input->getName()) != color_map_.end()) {
+ node_info.color = color_map_[input->getName()];
+ }
+
+ switch (input->getPhysicalType()) {
+ case P::PhysicalType::kTableReference: {
+ const P::TableReferencePtr table_reference =
+ std::static_pointer_cast<const P::TableReference>(input);
+ node_info.labels.emplace_back(table_reference->relation()->getName());
+ break;
+ }
+ case P::PhysicalType::kHashJoin: {
+ const P::HashJoinPtr hash_join =
+ std::static_pointer_cast<const P::HashJoin>(input);
+ node_info.labels.emplace_back(input->getName());
+
+ const auto &left_attributes = hash_join->left_join_attributes();
+ const auto &right_attributes = hash_join->right_join_attributes();
+ for (std::size_t i = 0; i < left_attributes.size(); ++i) {
+ node_info.labels.emplace_back(
+ left_attributes[i]->attribute_alias() + " = " + right_attributes[i]->attribute_alias());
+ }
+ break;
+ }
+ default: {
+ node_info.labels.emplace_back(input->getName());
+ break;
+ }
+ }
+ node_info.labels.emplace_back(
+ "est. # = " + std::to_string(cost_model_->estimateCardinality(input)));
+ node_info.labels.emplace_back(
+ "est. Selectivity = " + std::to_string(cost_model_->estimateSelectivity(input)));
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1605fd84/utility/PlanVisualizer.hpp
----------------------------------------------------------------------
diff --git a/utility/PlanVisualizer.hpp b/utility/PlanVisualizer.hpp
new file mode 100644
index 0000000..080b7de
--- /dev/null
+++ b/utility/PlanVisualizer.hpp
@@ -0,0 +1,94 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_UTILITY_PLAN_VISUALIZER_HPP_
+#define QUICKSTEP_UTILITY_PLAN_VISUALIZER_HPP_
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+/** \addtogroup Utility
+ * @{
+ */
+
+/**
+ * @brief A query plan visualizer that converts a physical plan into a graph in
+ * DOT format. Note that DOT is a plain text graph description language.
+ *
+ * @note This utility tool can be further extended to be more generic.
+ */
+class PlanVisualizer {
+ public:
+ PlanVisualizer()
+ : id_counter_(0) {}
+
+ ~PlanVisualizer() {}
+
+ /**
+ * @brief Visualize the query plan into a graph in DOT format (DOT is a plain
+ * text graph description language).
+ *
+ * @return The visualized query plan graph in DOT format.
+ */
+ std::string visualize(const optimizer::physical::PhysicalPtr &input);
+
+ private:
+ /**
+ * @brief Information of a graph node.
+ */
+ struct NodeInfo {
+ int id;
+ std::vector<std::string> labels;
+ std::string color;
+ };
+
+ /**
+ * @brief Information of a graph edge.
+ */
+ struct EdgeInfo {
+ int src_node_id;
+ int dst_node_id;
+ std::vector<std::string> labels;
+ };
+
+ void visit(const optimizer::physical::PhysicalPtr &input);
+
+ int id_counter_;
+ std::unordered_map<optimizer::physical::PhysicalPtr, int> node_id_map_;
+ std::unordered_map<std::string, std::string> color_map_;
+
+ std::vector<NodeInfo> nodes_;
+ std::vector<EdgeInfo> edges_;
+
+ std::unique_ptr<optimizer::cost::StarSchemaSimpleCostModel> cost_model_;
+
+ DISALLOW_COPY_AND_ASSIGN(PlanVisualizer);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif /* QUICKSTEP_UTILITY_PLAN_VISUALIZER_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1605fd84/utility/StringUtil.hpp
----------------------------------------------------------------------
diff --git a/utility/StringUtil.hpp b/utility/StringUtil.hpp
index bd138bb..6477ded 100644
--- a/utility/StringUtil.hpp
+++ b/utility/StringUtil.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,6 +21,7 @@
#define QUICKSTEP_UTILITY_STRING_UTIL_HPP_
#include <cstdint>
+#include <sstream>
#include <string>
#include <vector>
@@ -34,7 +37,7 @@ namespace quickstep {
* @param str The string to be converted.
* @return The converted string with all lower case characters bing converted to upper case characters.
*/
-extern std::string ToLower(const std::string& str);
+extern std::string ToLower(const std::string &str);
/**
* @brief Converts special characters to escape characters.
@@ -42,7 +45,30 @@ extern std::string ToLower(const std::string& str);
* @param text The string to be unescaped.
* @return Unescaped string.
*/
-extern std::string EscapeSpecialChars(const std::string& text);
+extern std::string EscapeSpecialChars(const std::string &text);
+
+/**
+ * @brief Join all objects in a iterable container into a single string. The object
+ * must have implemented the operator<< overloading with std::stringstream.
+ *
+ * @param container The iterable container of objects.
+ * @param separator A string to separate each object.
+ */
+template <typename ContainerType>
+std::string JoinToString(const ContainerType &container,
+ const std::string &separator) {
+ std::ostringstream oss;
+ bool is_first = true;
+ for (const auto &item : container) {
+ if (is_first) {
+ is_first = false;
+ } else {
+ oss << separator;
+ }
+ oss << item;
+ }
+ return oss.str();
+}
/**
* @brief Parse a string of base-10 integers separated by delimiter characters
[27/50] [abbrv] incubator-quickstep git commit: Print number of rows
in the output (#219)
Posted by hb...@apache.org.
Print number of rows in the output (#219)
- Provide an API to calculate the number of rows in a table
- Print output time in milliseconds, upto 3 decimal places.
- Simplify output messages.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/6b8cc265
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/6b8cc265
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/6b8cc265
Branch: refs/heads/query-manager-used-in-foreman
Commit: 6b8cc2650e4d99c580dcd2618ababcc6b403b166
Parents: 21b8508
Author: Harshad Deshmukh <d....@gmail.com>
Authored: Tue May 10 16:27:06 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:47 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 3 ++-
cli/CMakeLists.txt | 1 +
cli/PrintToScreen.cpp | 22 ++++++++++++++++++++++
cli/PrintToScreen.hpp | 23 +++++++++++++++++++++++
cli/QuickstepCli.cpp | 12 +++++++++---
storage/StorageBlock.cpp | 5 +++++
storage/StorageBlock.hpp | 11 ++++++++---
7 files changed, 70 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c9578b0..87a8f7c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -712,7 +712,8 @@ target_link_libraries(quickstep_cli_shell
quickstep_threading_ThreadIDBasedMap
quickstep_utility_Macros
quickstep_utility_PtrVector
- quickstep_utility_SqlError)
+ quickstep_utility_SqlError
+ quickstep_utility_StringUtil)
if (ENABLE_HDFS)
target_link_libraries(quickstep_cli_shell
quickstep_storage_FileManagerHdfs)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 9c4073b..a1989d5 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -102,6 +102,7 @@ target_link_libraries(quickstep_cli_PrintToScreen
quickstep_storage_StorageManager
quickstep_storage_TupleIdSequence
quickstep_storage_TupleStorageSubBlock
+ quickstep_types_IntType
quickstep_types_Type
quickstep_types_TypedValue
quickstep_utility_Macros)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/cli/PrintToScreen.cpp
----------------------------------------------------------------------
diff --git a/cli/PrintToScreen.cpp b/cli/PrintToScreen.cpp
index c543b70..227ff39 100644
--- a/cli/PrintToScreen.cpp
+++ b/cli/PrintToScreen.cpp
@@ -29,6 +29,7 @@
#include "storage/StorageManager.hpp"
#include "storage/TupleIdSequence.hpp"
#include "storage/TupleStorageSubBlock.hpp"
+#include "types/IntType.hpp"
#include "types/Type.hpp"
#include "types/TypedValue.hpp"
#include "utility/Macros.hpp"
@@ -147,4 +148,25 @@ void PrintToScreen::printTuple(const TupleStorageSubBlock &tuple_store,
fputc('\n', out);
}
+std::size_t PrintToScreen::GetNumTuplesInRelation(
+ const CatalogRelation &relation, StorageManager *storage_manager) {
+ const std::vector<block_id> &blocks = relation.getBlocksSnapshot();
+ std::size_t total_num_tuples = 0;
+ for (block_id block : blocks) {
+ total_num_tuples +=
+ storage_manager->getBlock(block, relation)->getNumTuples();
+ }
+ return total_num_tuples;
+}
+
+void PrintToScreen::PrintOutputSize(const CatalogRelation &relation,
+ StorageManager *storage_manager,
+ FILE *out) {
+ const std::size_t num_rows = GetNumTuplesInRelation(relation, storage_manager);
+ fprintf(out,
+ "(%lu %s)\n",
+ num_rows,
+ (num_rows == 1) ? "row" : "rows");
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/cli/PrintToScreen.hpp
----------------------------------------------------------------------
diff --git a/cli/PrintToScreen.hpp b/cli/PrintToScreen.hpp
index 3c9afae..0b57b4b 100644
--- a/cli/PrintToScreen.hpp
+++ b/cli/PrintToScreen.hpp
@@ -46,6 +46,29 @@ class PrintToScreen {
static void printHBar(const std::vector<int> &column_widths,
FILE *out);
+
+ /**
+ * @brief Get the total number of tuples in the given relation.
+ *
+ * @param relation The given relation.
+ * @param storage_manager The storage manager.
+ *
+ * @return The total number of tuples in the relation.
+ **/
+ static std::size_t GetNumTuplesInRelation(const CatalogRelation &relation,
+ StorageManager *storage_manager);
+
+ /**
+ * @brief Print the size of the output (i.e. number of rows in the relation).
+ *
+ * @param relation The given relation.
+ * @param storage_manager The storage manager.
+ * @param out The output stream.
+ **/
+ static void PrintOutputSize(const CatalogRelation &relation,
+ StorageManager *storage_manager,
+ FILE *out);
+
private:
// Undefined default constructor. Class is all-static and should not be
// instantiated.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 4c0a166..5881b3e 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -72,6 +72,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "utility/Macros.hpp"
#include "utility/PtrVector.hpp"
#include "utility/SqlError.hpp"
+#include "utility/StringUtil.hpp"
#include "gflags/gflags.h"
@@ -399,6 +400,10 @@ int main(int argc, char* argv[]) {
PrintToScreen::PrintRelation(*query_result_relation,
query_processor->getStorageManager(),
stdout);
+ PrintToScreen::PrintOutputSize(
+ *query_result_relation,
+ query_processor->getStorageManager(),
+ stdout);
DropRelation::Drop(*query_result_relation,
query_processor->getDefaultDatabase(),
@@ -406,13 +411,14 @@ int main(int argc, char* argv[]) {
}
query_processor->saveCatalog();
- printf("Execution time: %g seconds\n",
- std::chrono::duration<double>(end - start).count());
+ std::chrono::duration<double, std::milli> time_ms = end - start;
+ printf("Time: %s ms\n",
+ quickstep::DoubleToStringWithSignificantDigits(
+ time_ms.count(), 3).c_str());
} catch (const std::exception &e) {
fprintf(stderr, "QUERY EXECUTION ERROR: %s\n", e.what());
break;
}
- printf("Query Complete\n");
} else {
if (result.condition == ParseResult::kError) {
fprintf(stderr, "%s", result.error_message.c_str());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/storage/StorageBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.cpp b/storage/StorageBlock.cpp
index 476a130..fdd438d 100644
--- a/storage/StorageBlock.cpp
+++ b/storage/StorageBlock.cpp
@@ -1361,4 +1361,9 @@ void StorageBlock::invalidateAllIndexes() {
}
}
+const std::size_t StorageBlock::getNumTuples() const {
+ DCHECK(tuple_store_ != nullptr);
+ return tuple_store_->numTuples();
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/storage/StorageBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.hpp b/storage/StorageBlock.hpp
index 97813e2..3ae3812 100644
--- a/storage/StorageBlock.hpp
+++ b/storage/StorageBlock.hpp
@@ -311,17 +311,17 @@ class StorageBlock : public StorageBlockBase {
ValueAccessor *accessor);
/**
- * @brief Perform a random sampling of data on the StorageBlock. The number
+ * @brief Perform a random sampling of data on the StorageBlock. The number
* of records sampled is determined by the sample percentage in case of
* tuple sample. For block sample all the records in a block are taken.
*
* @param is_block_sample Flag to indicate if the Sampling method is a tuple
* sample or block sample.
* @param percentage The percentage of tuples to be sampled.Used only when the
- * sampling method is tuple sample.
+ * sampling method is tuple sample.
* @param destination Where to insert the tuples resulting from the SAMPLE.
*
- * @return Randomly sampled tuples whose count is determined by the
+ * @return Randomly sampled tuples whose count is determined by the
* sampling percentage. In the case of block sample the entire
* block is returned.
**/
@@ -583,6 +583,11 @@ class StorageBlock : public StorageBlockBase {
return rebuildIndexes(false);
}
+ /**
+ * @brief Get the number of tuples in this storage block.
+ **/
+ const std::size_t getNumTuples() const;
+
private:
static TupleStorageSubBlock* CreateTupleStorageSubBlock(
const CatalogRelationSchema &relation,
[48/50] [abbrv] incubator-quickstep git commit: Added Async
DataExchange Service.
Posted by hb...@apache.org.
Added Async DataExchange Service.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/50b4e55a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/50b4e55a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/50b4e55a
Branch: refs/heads/query-manager-used-in-foreman
Commit: 50b4e55a48c97fd5ce9800d8db92d64ee59ad2e3
Parents: 2221b7e
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Sat May 28 22:55:05 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:54 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 11 +-
storage/CMakeLists.txt | 69 +++++++-
storage/DataExchange.proto | 31 ++++
storage/DataExchangerAsync.cpp | 165 ++++++++++++++++++
storage/DataExchangerAsync.hpp | 97 +++++++++++
storage/StorageManager.cpp | 113 ++++++++++++-
storage/StorageManager.hpp | 61 ++++++-
storage/tests/DataExchange_unittest.cpp | 240 +++++++++++++++++++++++++++
third_party/iwyu/iwyu_helper.py | 3 +-
validate_cmakelists.py | 5 +-
10 files changed, 787 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index dc51ca6..ef7fd50 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -662,7 +662,12 @@ set(ENABLE_PUREMEMORY ON CACHE BOOL "Enable PureMemory TMB")
set(ENABLE_LEVELDB OFF CACHE BOOL "Enable LevelDB TMB")
set(ENABLE_MEMORYMIRROR OFF CACHE BOOL "Enable MemoryMirror TMB")
set(ENABLE_NATIVELOG OFF CACHE BOOL "Enable NativeLog TMB")
-set(ENABLE_NATIVENET OFF CACHE BOOL "Enable NativeNet TMB")
+
+# The distributed version requires to use the NativeNet implementation.
+if (NOT ENABLE_DISTRIBUTED)
+ set(ENABLE_NATIVENET OFF CACHE BOOL "Enable NativeNet TMB")
+endif()
+
set(ENABLE ENABLE_SQLITE OFF CACHE BOOL "Enable SQLite TMB")
set(ENABLE_VOLTDB OFF CACHE BOOL "Enable VoltDB TMB")
set(ENABLE_ZOOKEEPER OFF CACHE BOOL "Enable Zookeeper TMB")
@@ -670,6 +675,10 @@ set(ENABLE_ZOOKEEPER OFF CACHE BOOL "Enable Zookeeper TMB")
add_subdirectory("${THIRD_PARTY_SOURCE_DIR}/tmb" "${CMAKE_CURRENT_BINARY_DIR}/third_party/tmb")
include_directories(${TMB_INCLUDE_DIRS})
+if (ENABLE_DISTRIBUTED)
+ include_directories(${CMAKE_CURRENT_BINARY_DIR}/third_party/tmb/include)
+endif()
+
# Add all of the module subdirectories. CMakeLists.txt in each of the subdirectories
# defines how to build that module's libraries.
add_subdirectory(catalog)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 4da16ea..a77976a 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -126,6 +126,13 @@ QS_PROTOBUF_GENERATE_CPP(storage_StorageBlockLayout_proto_srcs
storage_StorageBlockLayout_proto_hdrs
StorageBlockLayout.proto)
+if (ENABLE_DISTRIBUTED)
+ GRPC_GENERATE_CPP(storage_DataExchange_proto_srcs
+ storage_DataExchange_proto_hdrs
+ .
+ DataExchange.proto)
+endif()
+
# Declare micro-libs:
add_library(quickstep_storage_AggregationOperationState
AggregationOperationState.cpp
@@ -171,6 +178,14 @@ add_library(quickstep_storage_CompressedTupleStorageSubBlock
CompressedTupleStorageSubBlock.hpp)
add_library(quickstep_storage_CountedReference ../empty_src.cpp CountedReference.hpp)
add_library(quickstep_storage_CSBTreeIndexSubBlock CSBTreeIndexSubBlock.cpp CSBTreeIndexSubBlock.hpp)
+
+if (ENABLE_DISTRIBUTED)
+ add_library(quickstep_storage_DataExchange_proto
+ ${storage_DataExchange_proto_srcs}
+ ${storage_DataExchange_proto_hdrs})
+ add_library(quickstep_storage_DataExchangerAsync DataExchangerAsync.cpp DataExchangerAsync.hpp)
+endif()
+
add_library(quickstep_storage_EvictionPolicy EvictionPolicy.cpp EvictionPolicy.hpp)
add_library(quickstep_storage_FileManager ../empty_src.cpp FileManager.hpp)
if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
@@ -575,6 +590,19 @@ target_link_libraries(quickstep_storage_CSBTreeIndexSubBlock
quickstep_utility_Macros
quickstep_utility_PtrVector
quickstep_utility_ScopedBuffer)
+
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_storage_DataExchange_proto
+ ${PROTOBUF3_LIBRARY})
+ target_link_libraries(quickstep_storage_DataExchangerAsync
+ glog
+ quickstep_storage_DataExchange_proto
+ quickstep_storage_StorageManager
+ quickstep_threading_Thread
+ quickstep_utility_Macros
+ ${GRPCPLUSPLUS_LIBRARIES})
+endif()
+
target_link_libraries(quickstep_storage_EvictionPolicy
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageConstants
@@ -925,6 +953,7 @@ target_link_libraries(quickstep_storage_StorageManager
gflags_nothreads-static
glog
gtest
+ quickstep_catalog_CatalogTypedefs
quickstep_storage_CountedReference
quickstep_storage_EvictionPolicy
quickstep_storage_FileManager
@@ -955,7 +984,9 @@ if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_storage_StorageManager
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
- quickstep_queryexecution_QueryExecutionUtil)
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_DataExchange_proto
+ ${GRPCPLUSPLUS_LIBRARIES})
endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_storage_SubBlockTypeRegistry
glog
@@ -1071,6 +1102,11 @@ elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS)
target_link_libraries(quickstep_storage
quickstep_storage_FileManagerWindows)
endif()
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_storage
+ quickstep_storage_DataExchange_proto
+ quickstep_storage_DataExchangerAsync)
+endif()
# CMAKE_VALIDATE_IGNORE_BEGIN
if(QUICKSTEP_HAVE_BITWEAVING)
target_link_libraries(quickstep_storage
@@ -1340,6 +1376,37 @@ target_link_libraries(CompressedPackedRowStoreTupleStorageSubBlock_unittest
${LIBS})
add_test(CompressedPackedRowStoreTupleStorageSubBlock_unittest CompressedPackedRowStoreTupleStorageSubBlock_unittest)
+if (ENABLE_DISTRIBUTED)
+ add_executable(DataExchange_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/DataExchange_unittest.cpp")
+ target_link_libraries(DataExchange_unittest
+ gflags_nothreads-static
+ glog
+ gtest
+ quickstep_catalog_CatalogAttribute
+ quickstep_catalog_CatalogRelation
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_CountedReference
+ quickstep_storage_DataExchangerAsync
+ quickstep_storage_StorageBlob
+ quickstep_storage_StorageBlock
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_storage_StorageManager
+ quickstep_storage_TupleStorageSubBlock
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ quickstep_types_TypedValue
+ quickstep_types_containers_Tuple
+ tmb
+ ${LIBS})
+ add_test(DataExchange_unittest DataExchange_unittest)
+endif(ENABLE_DISTRIBUTED)
+
add_executable(CSBTreeIndexSubBlock_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/CSBTreeIndexSubBlock_unittest.cpp")
target_link_libraries(CSBTreeIndexSubBlock_unittest
gtest
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/DataExchange.proto
----------------------------------------------------------------------
diff --git a/storage/DataExchange.proto b/storage/DataExchange.proto
new file mode 100644
index 0000000..a2636e5
--- /dev/null
+++ b/storage/DataExchange.proto
@@ -0,0 +1,31 @@
+// Copyright 2016 Pivotal Software, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package quickstep;
+
+service DataExchange {
+ rpc Pull (PullRequest) returns (PullResponse) {}
+}
+
+message PullRequest {
+ fixed64 block_id = 1;
+}
+
+message PullResponse {
+ bool is_valid = 1;
+ uint64 num_slots = 2;
+ bytes block = 3;
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/DataExchangerAsync.cpp
----------------------------------------------------------------------
diff --git a/storage/DataExchangerAsync.cpp b/storage/DataExchangerAsync.cpp
new file mode 100644
index 0000000..78c6565
--- /dev/null
+++ b/storage/DataExchangerAsync.cpp
@@ -0,0 +1,165 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "storage/DataExchangerAsync.hpp"
+
+#include <grpc++/grpc++.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+
+#include "storage/DataExchange.grpc.pb.h"
+#include "storage/DataExchange.pb.h"
+#include "storage/StorageManager.hpp"
+
+#include "glog/logging.h"
+
+using grpc::ServerCompletionQueue;
+
+namespace quickstep {
+namespace {
+
+/**
+ * @brief RPC Request Context Instance.
+ **/
+class CallContext {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param service The async service.
+ * @param queue The RPC request queue.
+ * @param storage_manager The StorageManager to use.
+ **/
+ CallContext(DataExchange::AsyncService *service,
+ ServerCompletionQueue *queue,
+ StorageManager *storage_manager)
+ : service_(service),
+ queue_(queue),
+ storage_manager_(DCHECK_NOTNULL(storage_manager)),
+ responder_(&context_),
+ status_(CallStatus::CREATE) {
+ Proceed();
+ }
+
+ /**
+ * @brief Process the RPC request.
+ **/
+ void Proceed();
+
+ private:
+ DataExchange::AsyncService *service_;
+ ServerCompletionQueue *queue_;
+
+ StorageManager *storage_manager_;
+
+ grpc::ServerContext context_;
+
+ PullRequest request_;
+ PullResponse response_;
+
+ grpc::ServerAsyncResponseWriter<PullResponse> responder_;
+
+ enum class CallStatus {
+ CREATE = 0,
+ PROCESS,
+ FINISH,
+ };
+ CallStatus status_;
+};
+
+void CallContext::Proceed() {
+ switch (status_) {
+ case CallStatus::CREATE: {
+ // Change this instance progress to the PROCESS state.
+ status_ = CallStatus::PROCESS;
+
+ // As part of the initial CREATE state, we *request* that the system
+ // start processing Pull requests. In this request, "this" acts are
+ // the tag uniquely identifying the request (so that different CallContext
+ // instances can serve different requests concurrently), in this case
+ // the memory address of this CallContext instance.
+ service_->RequestPull(&context_, &request_, &responder_, queue_, queue_, this);
+ break;
+ }
+ case CallStatus::PROCESS: {
+ // Spawn a new CallContext instance to serve new clients while we process
+ // the one for this CallContext. The instance will deallocate itself as
+ // part of its FINISH state.
+ new CallContext(service_, queue_, storage_manager_);
+
+ // The actual processing.
+ storage_manager_->pullBlockOrBlob(request_.block_id(), &response_);
+
+ // And we are done! Let the gRPC runtime know we've finished, using the
+ // memory address of this instance as the uniquely identifying tag for
+ // the event.
+ status_ = CallStatus::FINISH;
+ responder_.Finish(response_, grpc::Status::OK, this);
+ break;
+ }
+ case CallStatus::FINISH: {
+ // Once in the FINISH state, deallocate ourselves (CallContext).
+ delete this;
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown call status.";
+ }
+}
+
+} // namespace
+
+const char *DataExchangerAsync::kLocalNetworkAddress = "0.0.0.0:";
+
+DataExchangerAsync::DataExchangerAsync() {
+ grpc::ServerBuilder builder;
+ builder.AddListeningPort(kLocalNetworkAddress, grpc::InsecureServerCredentials(), &port_);
+ builder.RegisterService(&service_);
+
+ queue_ = builder.AddCompletionQueue();
+ server_ = builder.BuildAndStart();
+
+ DCHECK_GT(port_, 0);
+ server_address_ = kLocalNetworkAddress + std::to_string(port_);
+ LOG(INFO) << "DataExchangerAsync Service listening on " << server_address_;
+}
+
+void DataExchangerAsync::run() {
+ // Self-destruct upon success.
+ new CallContext(&service_, queue_.get(), storage_manager_);
+
+ void *tag = nullptr; // Uniquely identify a request.
+ bool ok = false;
+
+ while (true) {
+ if (queue_->Next(&tag, &ok)) {
+ CallContext *call_context = static_cast<CallContext*>(tag);
+ if (ok) {
+ call_context->Proceed();
+ } else {
+ LOG(WARNING) << "Not ok\n";
+ delete call_context;
+ }
+ } else {
+ LOG(INFO) << "Shutdown\n";
+ return;
+ }
+ }
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/DataExchangerAsync.hpp
----------------------------------------------------------------------
diff --git a/storage/DataExchangerAsync.hpp b/storage/DataExchangerAsync.hpp
new file mode 100644
index 0000000..75a4e4d
--- /dev/null
+++ b/storage/DataExchangerAsync.hpp
@@ -0,0 +1,97 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_DATA_EXCHANGER_ASYNC_HPP_
+#define QUICKSTEP_STORAGE_DATA_EXCHANGER_ASYNC_HPP_
+
+#include <grpc++/grpc++.h>
+
+#include <memory>
+#include <string>
+
+#include "storage/DataExchange.grpc.pb.h"
+#include "threading/Thread.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class StorageManager;
+
+/**
+ * @brief A class which exchanges data from a StorageManager to its peer.
+ **/
+class DataExchangerAsync final : public Thread {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ DataExchangerAsync();
+
+ ~DataExchangerAsync() override {}
+
+ /**
+ * @brief Set the local StorageManager.
+ *
+ * @param storage_manager The StorageManager to use.
+ **/
+ void set_storage_manager(StorageManager *storage_manager) {
+ storage_manager_ = storage_manager;
+ }
+
+ /**
+ * @brief Return its network address for peers to connect.
+ *
+ * @return Its network address.
+ **/
+ std::string network_address() const {
+ DCHECK(!server_address_.empty());
+ return server_address_;
+ }
+
+ /**
+ * @brief Shutdown itself.
+ **/
+ void shutdown() {
+ server_->Shutdown();
+ // Always shutdown the completion queue after the server.
+ queue_->Shutdown();
+ }
+
+ protected:
+ void run() override;
+
+ private:
+ static const char *kLocalNetworkAddress;
+
+ DataExchange::AsyncService service_;
+
+ int port_ = -1;
+ // Format IP:port, i.e., "0.0.0.0:0".
+ std::string server_address_;
+
+ std::unique_ptr<grpc::ServerCompletionQueue> queue_;
+ std::unique_ptr<grpc::Server> server_;
+
+ StorageManager *storage_manager_ = nullptr;
+
+ DISALLOW_COPY_AND_ASSIGN(DataExchangerAsync);
+};
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_STORAGE_DATA_EXCHANGER_ASYNC_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 15e2503..8fc1224 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -41,6 +41,10 @@
#include <numaif.h>
#endif
+#ifdef QUICKSTEP_DISTRIBUTED
+#include <grpc++/grpc++.h>
+#endif
+
#include <atomic>
#include <cerrno>
#include <cstddef>
@@ -53,6 +57,8 @@
#include <unordered_map>
#include <vector>
+#include "catalog/CatalogTypedefs.hpp"
+
#ifdef QUICKSTEP_DISTRIBUTED
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionTypedefs.hpp"
@@ -60,6 +66,12 @@
#endif
#include "storage/CountedReference.hpp"
+
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "storage/DataExchange.grpc.pb.h"
+#include "storage/DataExchange.pb.h"
+#endif
+
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManagerLocal.hpp"
#include "storage/StorageBlob.hpp"
@@ -456,6 +468,80 @@ block_id StorageManager::allocateNewBlockOrBlob(const std::size_t num_slots,
}
#ifdef QUICKSTEP_DISTRIBUTED
+void StorageManager::pullBlockOrBlob(const block_id block,
+ PullResponse *response) const {
+ SpinSharedMutexSharedLock<false> read_lock(blocks_shared_mutex_);
+ std::unordered_map<block_id, BlockHandle>::const_iterator cit = blocks_.find(block);
+ if (cit != blocks_.end()) {
+ response->set_is_valid(true);
+
+ const BlockHandle &block_handle = cit->second;
+ const std::size_t num_slots = block_handle.block_memory_size;
+
+ response->set_num_slots(num_slots);
+ response->set_block(block_handle.block_memory,
+ num_slots * kSlotSizeBytes);
+ } else {
+ response->set_is_valid(false);
+ }
+}
+
+StorageManager::DataExchangerClientAsync::DataExchangerClientAsync(const std::shared_ptr<grpc::Channel> &channel,
+ StorageManager *storage_manager)
+ : stub_(DataExchange::NewStub(channel)),
+ storage_manager_(storage_manager) {
+}
+
+bool StorageManager::DataExchangerClientAsync::Pull(const block_id block,
+ const numa_node_id numa_node,
+ BlockHandle *block_handle) {
+ grpc::ClientContext context;
+
+ PullRequest request;
+ request.set_block_id(block);
+
+ grpc::CompletionQueue queue;
+
+ unique_ptr<grpc::ClientAsyncResponseReader<PullResponse>> rpc(
+ stub_->AsyncPull(&context, request, &queue));
+
+ PullResponse response;
+ grpc::Status status;
+
+ rpc->Finish(&response, &status, reinterpret_cast<void*>(1));
+
+ void *got_tag;
+ bool ok = false;
+
+ queue.Next(&got_tag, &ok);
+ CHECK(got_tag == reinterpret_cast<void*>(1));
+ CHECK(ok);
+
+ if (!status.ok()) {
+ LOG(ERROR) << "DataExchangerClientAsync Pull error: RPC failed";
+ return false;
+ }
+
+ if (!response.is_valid()) {
+ LOG(INFO) << "The pulling block not found in all the peers";
+ return false;
+ }
+
+ const size_t num_slots = response.num_slots();
+ DCHECK_NE(num_slots, 0u);
+
+ const string &block_content = response.block();
+ DCHECK_EQ(kSlotSizeBytes * num_slots, block_content.size());
+
+ void *block_buffer = storage_manager_->allocateSlots(num_slots, numa_node);
+
+ block_handle->block_memory =
+ std::memcpy(block_buffer, block_content.c_str(), block_content.size());
+ block_handle->block_memory_size = num_slots;
+
+ return true;
+}
+
vector<string> StorageManager::getPeerDomainNetworkAddresses(const block_id block) {
serialization::BlockMessage proto;
proto.set_block_id(block);
@@ -541,14 +627,37 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
// The caller of this function holds an exclusive lock on this block/blob's
// mutex in the lock manager. The caller has ensured that the block is not
// already loaded before this function gets called.
- size_t num_slots = file_manager_->numSlots(block);
+ BlockHandle loaded_handle;
+
+#ifdef QUICKSTEP_DISTRIBUTED
+ // TODO(quickstep-team): Use a cost model to determine whether to load from
+ // a remote peer or the disk.
+ if (BlockIdUtil::Domain(block) != block_domain_) {
+ LOG(INFO) << "Pulling Block " << BlockIdUtil::ToString(block) << " from a remote peer";
+ const vector<string> peer_domain_network_addresses = getPeerDomainNetworkAddresses(block);
+ for (const string &peer_domain_network_address : peer_domain_network_addresses) {
+ DataExchangerClientAsync client(
+ grpc::CreateChannel(peer_domain_network_address, grpc::InsecureChannelCredentials()),
+ this);
+
+ if (client.Pull(block, numa_node, &loaded_handle)) {
+ sendBlockLocationMessage(block, kAddBlockLocationMessage);
+ return loaded_handle;
+ }
+ }
+
+ LOG(INFO) << "Failed to pull Block " << BlockIdUtil::ToString(block)
+ << " from remote peers, so try to load from disk.";
+ }
+#endif
+
+ const size_t num_slots = file_manager_->numSlots(block);
DEBUG_ASSERT(num_slots != 0);
void *block_buffer = allocateSlots(num_slots, numa_node);
const bool status = file_manager_->readBlockOrBlob(block, block_buffer, kSlotSizeBytes * num_slots);
CHECK(status) << "Failed to read block from persistent storage: " << block;
- BlockHandle loaded_handle;
loaded_handle.block_memory = block_buffer;
loaded_handle.block_memory_size = num_slots;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index 55a011e..50ddb0f 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -26,9 +26,14 @@
#include <unordered_map>
#include <vector>
+#include "catalog/CatalogTypedefs.hpp"
#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED
-
#include "storage/CountedReference.hpp"
+
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "storage/DataExchange.grpc.pb.h"
+#endif
+
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManager.hpp"
#include "storage/StorageBlob.hpp"
@@ -45,6 +50,10 @@
#include "tmb/id_typedefs.h"
+#ifdef QUICKSTEP_DISTRIBUTED
+namespace grpc { class Channel; }
+#endif
+
namespace tmb { class MessageBus; }
namespace quickstep {
@@ -58,6 +67,10 @@ DECLARE_bool(use_hdfs);
class CatalogRelationSchema;
+#ifdef QUICKSTEP_DISTRIBUTED
+class PullResponse;
+#endif
+
class StorageBlockLayout;
/** \addtogroup Storage
@@ -365,6 +378,16 @@ class StorageManager {
**/
bool blockOrBlobIsLoadedAndDirty(const block_id block);
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Pull a block or a blob. Used by DataExchangerAsync.
+ *
+ * @param block The id of the block or blob.
+ * @param response Where to store the pulled block content.
+ **/
+ void pullBlockOrBlob(const block_id block, PullResponse *response) const;
+#endif
+
private:
struct BlockHandle {
void *block_memory;
@@ -374,6 +397,42 @@ class StorageManager {
#ifdef QUICKSTEP_DISTRIBUTED
/**
+ * @brief A class which connects to DataExchangerAsync to exchange data from
+ * remote peers.
+ **/
+ class DataExchangerClientAsync {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param channel The RPC channel to connect DataExchangerAsync.
+ * @param storage_manager The StorageManager to use.
+ */
+ DataExchangerClientAsync(const std::shared_ptr<grpc::Channel> &channel,
+ StorageManager *storage_manager);
+
+ /**
+ * @brief Pull a block or blob from a remote StorageManager.
+ *
+ * @param block The block or blob to pull.
+ * @param numa_node The NUMA node for placing this block.
+ * @param block_handle Where the pulled block or blob stores.
+ *
+ * @return Whether the pull operation is successful or not.
+ */
+ bool Pull(const block_id block,
+ const numa_node_id numa_node,
+ BlockHandle *block_handle);
+
+ private:
+ std::unique_ptr<DataExchange::Stub> stub_;
+
+ StorageManager *storage_manager_;
+
+ DISALLOW_COPY_AND_ASSIGN(DataExchangerClientAsync);
+ };
+
+ /**
* @brief Get the network info of all the remote StorageManagers which may
* load the given block in the buffer pool.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/tests/DataExchange_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/DataExchange_unittest.cpp b/storage/tests/DataExchange_unittest.cpp
new file mode 100644
index 0000000..38d12f6
--- /dev/null
+++ b/storage/tests/DataExchange_unittest.cpp
@@ -0,0 +1,240 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogAttribute.hpp"
+#include "catalog/CatalogRelation.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/BlockLocator.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "storage/CountedReference.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageBlob.hpp"
+#include "storage/StorageBlock.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageConstants.hpp"
+#include "storage/StorageManager.hpp"
+#include "storage/TupleStorageSubBlock.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "types/containers/Tuple.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+
+using std::free;
+using std::malloc;
+using std::move;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+using tmb::MessageBus;
+using tmb::TaggedMessage;
+
+namespace quickstep {
+
+class DataExchangeTest : public ::testing::Test {
+ protected:
+ static const char kStoragePath[];
+ static const char kCheckedDomainNetworkAddress[];
+
+ ~DataExchangeTest() {
+ data_exchanger_expected_.join();
+ locator_->join();
+ }
+
+ virtual void SetUp() {
+ bus_.Initialize();
+
+ locator_.reset(new BlockLocator(&bus_));
+ locator_client_id_ = locator_->getBusClientID();
+ locator_->start();
+
+ worker_client_id_ = bus_.Connect();
+ bus_.RegisterClientAsSender(worker_client_id_, kBlockDomainRegistrationMessage);
+ bus_.RegisterClientAsReceiver(worker_client_id_, kBlockDomainRegistrationResponseMessage);
+
+ bus_.RegisterClientAsSender(worker_client_id_, kPoisonMessage);
+
+ storage_manager_expected_.reset(new StorageManager(
+ kStoragePath,
+ getBlockDomain(data_exchanger_expected_.network_address()),
+ locator_client_id_,
+ &bus_));
+
+ data_exchanger_expected_.set_storage_manager(storage_manager_expected_.get());
+ data_exchanger_expected_.start();
+
+ storage_manager_checked_.reset(new StorageManager(
+ kStoragePath,
+ getBlockDomain(kCheckedDomainNetworkAddress),
+ locator_client_id_,
+ &bus_));
+ }
+
+ virtual void TearDown() {
+ storage_manager_checked_.reset();
+
+ data_exchanger_expected_.shutdown();
+ storage_manager_expected_.reset();
+
+ serialization::EmptyMessage proto;
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kPoisonMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent PoisonMessage (typed '" << kPoisonMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+ }
+
+ unique_ptr<StorageManager> storage_manager_expected_, storage_manager_checked_;
+
+ private:
+ block_id_domain getBlockDomain(const string &network_address) {
+ serialization::BlockDomainRegistrationMessage proto;
+ proto.set_domain_network_address(network_address);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainRegistrationMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+
+ const tmb::AnnotatedMessage annotated_message(bus_.Receive(worker_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ EXPECT_EQ(locator_client_id_, annotated_message.sender);
+ EXPECT_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') received BlockDomainRegistrationResponseMessage (typed '"
+ << kBlockDomainRegistrationResponseMessage
+ << "') from BlockLocator";
+
+ serialization::BlockDomainMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ return static_cast<block_id_domain>(response_proto.block_domain());
+ }
+
+ MessageBusImpl bus_;
+
+ unique_ptr<BlockLocator> locator_;
+ tmb::client_id locator_client_id_;
+
+ tmb::client_id worker_client_id_;
+
+ DataExchangerAsync data_exchanger_expected_;
+};
+
+const char DataExchangeTest::kStoragePath[] = "./data_exchange_test_data/";
+const char DataExchangeTest::kCheckedDomainNetworkAddress[] = "ip:port";
+
+TEST_F(DataExchangeTest, BlockPull) {
+ CatalogRelation relation(nullptr, "rel");
+ const attribute_id attr_id =
+ relation.addAttribute(new CatalogAttribute(nullptr, "attr_int", TypeFactory::GetType(kInt)));
+
+ const block_id block =
+ storage_manager_expected_->createBlock(relation, relation.getDefaultStorageBlockLayout());
+
+ {
+ MutableBlockReference block_expected = storage_manager_expected_->getBlockMutable(block, relation);
+
+ // Insert a tuple.
+ const int value_expected = -1;
+ {
+ vector<TypedValue> attrs(1, TypedValue(value_expected));
+ const Tuple tuple(move(attrs));
+
+ EXPECT_TRUE(block_expected->insertTuple(tuple));
+ }
+
+ const BlockReference block_checked = storage_manager_checked_->getBlock(block, relation);
+ EXPECT_FALSE(block_checked->isBlob());
+
+ const TupleStorageSubBlock &tuple_store_checked = block_checked->getTupleStorageSubBlock();
+
+ EXPECT_EQ(1, tuple_store_checked.numTuples());
+ EXPECT_EQ(value_expected, tuple_store_checked.getAttributeValueTyped(0 /* tuple_id */, attr_id).getLiteral<int>());
+ }
+
+ storage_manager_checked_->deleteBlockOrBlobFile(block);
+ storage_manager_expected_->deleteBlockOrBlobFile(block);
+}
+
+TEST_F(DataExchangeTest, BlobPull) {
+ const block_id blob = storage_manager_expected_->createBlob(kDefaultBlockSizeInSlots);
+ {
+ const BlobReference blob_expected = storage_manager_expected_->getBlob(blob);
+ const BlobReference blob_checked = storage_manager_checked_->getBlob(blob);
+ EXPECT_TRUE(blob_checked->isBlob());
+ EXPECT_EQ(blob, blob_checked->getID());
+ }
+
+ storage_manager_checked_->deleteBlockOrBlobFile(blob);
+ storage_manager_expected_->deleteBlockOrBlobFile(blob);
+}
+
+} // namespace quickstep
+
+int main(int argc, char **argv) {
+ google::InitGoogleLogging(argv[0]);
+ // Honor FLAGS_buffer_pool_slots in StorageManager.
+ gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+ ::testing::InitGoogleTest(&argc, argv);
+
+ return RUN_ALL_TESTS();
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/third_party/iwyu/iwyu_helper.py
----------------------------------------------------------------------
diff --git a/third_party/iwyu/iwyu_helper.py b/third_party/iwyu/iwyu_helper.py
index a204c50..13697be 100755
--- a/third_party/iwyu/iwyu_helper.py
+++ b/third_party/iwyu/iwyu_helper.py
@@ -19,8 +19,9 @@ import sys
QUICKSTEP_INCLUDES = [ '.',
'./build',
'./build/third_party',
- './build/third_party/protobuf/include',
'./build/third_party/gflags/include',
+ './build/third_party/protobuf/include',
+ './build/third_party/tmb/include',
'./third_party/benchmark/include',
'./third_party/glog/src',
'./third_party/googletest/googletest/include',
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/validate_cmakelists.py
----------------------------------------------------------------------
diff --git a/validate_cmakelists.py b/validate_cmakelists.py
index c7b5883..7dd6fc5 100755
--- a/validate_cmakelists.py
+++ b/validate_cmakelists.py
@@ -17,7 +17,7 @@ TODO List / Known Issues & Limitations:
"""
# Copyright 2011-2015 Quickstep Technologies LLC.
-# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2015-2016 Pivotal Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -40,7 +40,8 @@ EXCLUDED_TOP_LEVEL_DIRS = ["build", "third_party"]
# Explicitly ignored dependencies (special headers with no other quickstep
# dependencies).
IGNORED_DEPENDENCIES = frozenset(
- ["quickstep_threading_WinThreadsAPI",
+ ["quickstep_storage_DataExchange.grpc_proto",
+ "quickstep_threading_WinThreadsAPI",
"quickstep_utility_textbasedtest_TextBasedTest",
"quickstep_utility_textbasedtest_TextBasedTestDriver",
"quickstep_storage_bitweaving_BitWeavingHIndexSubBlock",
[09/50] [abbrv] incubator-quickstep git commit: Partition aware
selection using NUMA-awareness (#175)
Posted by hb...@apache.org.
Partition aware selection using NUMA-awareness (#175)
Link: https://github.com/pivotalsoftware/quickstep/pull/175
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/89acf230
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/89acf230
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/89acf230
Branch: refs/heads/query-manager-used-in-foreman
Commit: 89acf230e85f5b3d1bdf01bb5decdf76f6dec128
Parents: 5ea898b
Author: Adalbert Gerald Soosai Raj <ad...@gmail.com>
Authored: Sun May 1 15:38:38 2016 -0500
Committer: Harshad Deshmukh <d....@gmail.com>
Committed: Sun May 1 15:38:38 2016 -0500
----------------------------------------------------------------------
catalog/CMakeLists.txt | 5 +-
catalog/CatalogRelation.hpp | 9 ++
relational_operators/CMakeLists.txt | 5 ++
relational_operators/SelectOperator.cpp | 129 +++++++++++++++++++++------
relational_operators/SelectOperator.hpp | 115 +++++++++++++++++++++---
5 files changed, 226 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/89acf230/catalog/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/catalog/CMakeLists.txt b/catalog/CMakeLists.txt
index 8c89d7e..94da838 100644
--- a/catalog/CMakeLists.txt
+++ b/catalog/CMakeLists.txt
@@ -175,9 +175,12 @@ target_link_libraries(quickstep_catalog
quickstep_catalog_CatalogRelationSchema
quickstep_catalog_CatalogTypedefs
quickstep_catalog_IndexScheme
- quickstep_catalog_NUMAPlacementScheme
quickstep_catalog_PartitionScheme
quickstep_catalog_PartitionSchemeHeader)
+if(QUICKSTEP_HAVE_LIBNUMA)
+target_link_libraries(quickstep_catalog
+ quickstep_catalog_NUMAPlacementScheme)
+endif()
# Tests:
add_executable(Catalog_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/Catalog_unittest.cpp")
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/89acf230/catalog/CatalogRelation.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.hpp b/catalog/CatalogRelation.hpp
index 4cc8d79..3701090 100644
--- a/catalog/CatalogRelation.hpp
+++ b/catalog/CatalogRelation.hpp
@@ -165,6 +165,15 @@ class CatalogRelation : public CatalogRelationSchema {
}
/**
+ * @brief Get the NUMA placement scheme of the relation.
+ *
+ * @return A pointer to a const NUMA placement scheme.
+ **/
+ const NUMAPlacementScheme* getNUMAPlacementSchemePtr() const {
+ return placement_scheme_.get();
+ }
+
+ /**
* @brief Set the NUMA placement scheme for the catalog relation.
*
* @param placement_scheme The NUMA placement scheme object for the relation,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/89acf230/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index a4600e6..eec5300 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -266,6 +266,7 @@ target_link_libraries(quickstep_relationaloperators_SelectOperator
glog
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
+ quickstep_catalog_PartitionSchemeHeader
quickstep_queryexecution_QueryContext
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
@@ -276,6 +277,10 @@ target_link_libraries(quickstep_relationaloperators_SelectOperator
quickstep_storage_StorageManager
quickstep_utility_Macros
tmb)
+if(QUICKSTEP_HAVE_LIBNUMA)
+target_link_libraries(quickstep_relationaloperators_SelectOperator
+ quickstep_catalog_NUMAPlacementScheme)
+endif()
target_link_libraries(quickstep_relationaloperators_SortMergeRunOperator
glog
quickstep_catalog_CatalogRelation
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/89acf230/relational_operators/SelectOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.cpp b/relational_operators/SelectOperator.cpp
index 3cac199..69bb434 100644
--- a/relational_operators/SelectOperator.cpp
+++ b/relational_operators/SelectOperator.cpp
@@ -35,6 +35,93 @@ namespace quickstep {
class Predicate;
+void SelectOperator::addWorkOrders(WorkOrdersContainer *container,
+ StorageManager *storage_manager,
+ const Predicate *predicate,
+ const std::vector<std::unique_ptr<const Scalar>> *selection,
+ InsertDestination *output_destination) {
+ if (input_relation_is_stored_) {
+ for (const block_id input_block_id : input_relation_block_ids_) {
+ container->addNormalWorkOrder(
+ new SelectWorkOrder(input_relation_,
+ input_block_id,
+ predicate,
+ simple_projection_,
+ simple_selection_,
+ selection,
+ output_destination,
+ storage_manager),
+ op_index_);
+ }
+ } else {
+ while (num_workorders_generated_ < input_relation_block_ids_.size()) {
+ container->addNormalWorkOrder(
+ new SelectWorkOrder(
+ input_relation_,
+ input_relation_block_ids_[num_workorders_generated_],
+ predicate,
+ simple_projection_,
+ simple_selection_,
+ selection,
+ output_destination,
+ storage_manager),
+ op_index_);
+ ++num_workorders_generated_;
+ }
+ }
+}
+
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+void SelectOperator::addPartitionAwareWorkOrders(WorkOrdersContainer *container,
+ StorageManager *storage_manager,
+ const Predicate *predicate,
+ const std::vector<std::unique_ptr<const Scalar>> *selection,
+ InsertDestination *output_destination) {
+ DCHECK(placement_scheme_ != nullptr);
+ const std::size_t num_partitions = input_relation_.getPartitionScheme().getPartitionSchemeHeader().getNumPartitions();
+ if (input_relation_is_stored_) {
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ for (const block_id input_block_id :
+ input_relation_block_ids_in_partition_[part_id]) {
+ container->addNormalWorkOrder(
+ new SelectWorkOrder(
+ input_relation_,
+ input_block_id,
+ predicate,
+ simple_projection_,
+ simple_selection_,
+ selection,
+ output_destination,
+ storage_manager,
+ placement_scheme_->getNUMANodeForBlock(input_block_id)),
+ op_index_);
+ }
+ }
+ } else {
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ while (num_workorders_generated_in_partition_[part_id] <
+ input_relation_block_ids_in_partition_[part_id].size()) {
+ block_id block_in_partition
+ = input_relation_block_ids_in_partition_[part_id][num_workorders_generated_in_partition_[part_id]];
+ container->addNormalWorkOrder(
+ new SelectWorkOrder(
+ input_relation_,
+ block_in_partition,
+ predicate,
+ simple_projection_,
+ simple_selection_,
+ selection,
+ output_destination,
+ storage_manager,
+ placement_scheme_->getNUMANodeForBlock(block_in_partition)),
+ op_index_);
+ ++num_workorders_generated_in_partition_[part_id];
+ }
+ }
+ }
+}
+#endif
+
bool SelectOperator::getAllWorkOrders(
WorkOrdersContainer *container,
QueryContext *query_context,
@@ -54,35 +141,27 @@ bool SelectOperator::getAllWorkOrders(
if (input_relation_is_stored_) {
if (!started_) {
- for (const block_id input_block_id : input_relation_block_ids_) {
- container->addNormalWorkOrder(
- new SelectWorkOrder(input_relation_,
- input_block_id,
- predicate,
- simple_projection_,
- simple_selection_,
- selection,
- output_destination,
- storage_manager),
- op_index_);
+ if (input_relation_.hasPartitionScheme()) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ if (input_relation_.hasNUMAPlacementScheme()) {
+ addPartitionAwareWorkOrders(container, storage_manager, predicate, selection, output_destination);
+ }
+#endif
+ } else {
+ addWorkOrders(container, storage_manager, predicate, selection, output_destination);
}
started_ = true;
}
return started_;
} else {
- while (num_workorders_generated_ < input_relation_block_ids_.size()) {
- container->addNormalWorkOrder(
- new SelectWorkOrder(
- input_relation_,
- input_relation_block_ids_[num_workorders_generated_],
- predicate,
- simple_projection_,
- simple_selection_,
- selection,
- output_destination,
- storage_manager),
- op_index_);
- ++num_workorders_generated_;
+ if (input_relation_.hasPartitionScheme()) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ if (input_relation_.hasNUMAPlacementScheme()) {
+ addPartitionAwareWorkOrders(container, storage_manager, predicate, selection, output_destination);
+ }
+#endif
+ } else {
+ addWorkOrders(container, storage_manager, predicate, selection, output_destination);
}
return done_feeding_input_relation_;
}
@@ -90,7 +169,7 @@ bool SelectOperator::getAllWorkOrders(
void SelectWorkOrder::execute() {
BlockReference block(
- storage_manager_->getBlock(input_block_id_, input_relation_));
+ storage_manager_->getBlock(input_block_id_, input_relation_, getPreferredNUMANodes()[0]));
if (simple_projection_) {
block->selectSimple(simple_selection_,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/89acf230/relational_operators/SelectOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.hpp b/relational_operators/SelectOperator.hpp
index 3da496c..76f4cb6 100644
--- a/relational_operators/SelectOperator.hpp
+++ b/relational_operators/SelectOperator.hpp
@@ -24,6 +24,12 @@
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
+
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include "catalog/NUMAPlacementScheme.hpp"
+#endif
+
+#include "catalog/PartitionSchemeHeader.hpp"
#include "query_execution/QueryContext.hpp"
#include "relational_operators/RelationalOperator.hpp"
#include "relational_operators/WorkOrder.hpp"
@@ -87,7 +93,28 @@ class SelectOperator : public RelationalOperator {
num_workorders_generated_(0),
simple_projection_(false),
input_relation_is_stored_(input_relation_is_stored),
- started_(false) {}
+ started_(false) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ placement_scheme_ = input_relation.getNUMAPlacementSchemePtr();
+#endif
+ if (input_relation.hasPartitionScheme()) {
+ const PartitionScheme &part_scheme = input_relation.getPartitionScheme();
+ const PartitionSchemeHeader &part_scheme_header = part_scheme.getPartitionSchemeHeader();
+ const std::size_t num_partitions = part_scheme_header.getNumPartitions();
+ input_relation_block_ids_in_partition_.resize(num_partitions);
+ num_workorders_generated_in_partition_.resize(num_partitions);
+ num_workorders_generated_in_partition_.assign(num_partitions, 0);
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ if (input_relation_is_stored) {
+ input_relation_block_ids_in_partition_[part_id] =
+ part_scheme.getBlocksInPartition(part_id);
+ } else {
+ input_relation_block_ids_in_partition_[part_id] =
+ std::vector<block_id>();
+ }
+ }
+ }
+ }
/**
* @brief Constructor for selection with simple projection of attributes.
@@ -124,7 +151,28 @@ class SelectOperator : public RelationalOperator {
num_workorders_generated_(0),
simple_projection_(true),
input_relation_is_stored_(input_relation_is_stored),
- started_(false) {}
+ started_(false) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ placement_scheme_ = input_relation.getNUMAPlacementSchemePtr();
+#endif
+ if (input_relation.hasPartitionScheme()) {
+ const PartitionScheme &part_scheme = input_relation.getPartitionScheme();
+ const PartitionSchemeHeader &part_scheme_header = part_scheme.getPartitionSchemeHeader();
+ const std::size_t num_partitions = part_scheme_header.getNumPartitions();
+ input_relation_block_ids_in_partition_.resize(num_partitions);
+ num_workorders_generated_in_partition_.resize(num_partitions);
+ num_workorders_generated_in_partition_.assign(num_partitions, 0);
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ if (input_relation_is_stored) {
+ input_relation_block_ids_in_partition_[part_id] =
+ part_scheme.getBlocksInPartition(part_id);
+ } else {
+ input_relation_block_ids_in_partition_[part_id] =
+ std::vector<block_id>();
+ }
+ }
+ }
+ }
~SelectOperator() override {}
@@ -135,13 +183,33 @@ class SelectOperator : public RelationalOperator {
tmb::MessageBus *bus) override;
void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
- input_relation_block_ids_.push_back(input_block_id);
+ if (input_relation_.hasPartitionScheme()) {
+ const partition_id part_id =
+ input_relation_.getPartitionScheme().getPartitionForBlock(input_block_id);
+ input_relation_block_ids_in_partition_[part_id].push_back(input_block_id);
+ } else {
+ input_relation_block_ids_.push_back(input_block_id);
+ }
}
+ // TODO(gerald): Each call to getPartitionForBlock() involves grabbing shared
+ // locks on each partition's mutex, checking if the block belongs to the
+ // partition. Instead, we can provide a method getPartitionsForBlocks() which
+ // accepts a list of blocks and returns corresponding list of their partition IDs.
+ // Therefore, once we grab a lock for a partition, we search for all the blocks
+ // and then release the lock.
void feedInputBlocks(const relation_id rel_id, std::vector<block_id> *partially_filled_blocks) override {
- input_relation_block_ids_.insert(input_relation_block_ids_.end(),
- partially_filled_blocks->begin(),
- partially_filled_blocks->end());
+ if (input_relation_.hasPartitionScheme()) {
+ for (auto it = partially_filled_blocks->begin(); it != partially_filled_blocks->end(); ++it) {
+ const partition_id part_id = input_relation_.getPartitionScheme().getPartitionForBlock((*it));
+ input_relation_block_ids_in_partition_[part_id].insert(input_relation_block_ids_in_partition_[part_id].end(),
+ *it);
+ }
+ } else {
+ input_relation_block_ids_.insert(input_relation_block_ids_.end(),
+ partially_filled_blocks->begin(),
+ partially_filled_blocks->end());
+ }
}
QueryContext::insert_destination_id getInsertDestinationID() const override {
@@ -152,9 +220,20 @@ class SelectOperator : public RelationalOperator {
return output_relation_.getID();
}
+ void addWorkOrders(WorkOrdersContainer *container,
+ StorageManager *storage_manager,
+ const Predicate *predicate,
+ const std::vector<std::unique_ptr<const Scalar>> *selection,
+ InsertDestination *output_destination);
+
+ void addPartitionAwareWorkOrders(WorkOrdersContainer *container,
+ StorageManager *storage_manager,
+ const Predicate *predicate,
+ const std::vector<std::unique_ptr<const Scalar>> *selection,
+ InsertDestination *output_destination);
+
private:
const CatalogRelation &input_relation_;
-
const CatalogRelation &output_relation_;
const QueryContext::insert_destination_id output_destination_index_;
const QueryContext::predicate_id predicate_index_;
@@ -163,12 +242,20 @@ class SelectOperator : public RelationalOperator {
const std::vector<attribute_id> simple_selection_;
std::vector<block_id> input_relation_block_ids_;
+ // A vector of vectors V where V[i] indicates the list of block IDs of the
+ // input relation that belong to the partition i.
+ std::vector<std::vector<block_id>> input_relation_block_ids_in_partition_;
// A single workorder is generated for each block of input relation.
std::vector<block_id>::size_type num_workorders_generated_;
+ // A single workorder is generated for each block in each partition of input relation.
+ std::vector<std::size_t> num_workorders_generated_in_partition_;
const bool simple_projection_;
const bool input_relation_is_stored_;
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ const NUMAPlacementScheme *placement_scheme_;
+#endif
bool started_;
DISALLOW_COPY_AND_ASSIGN(SelectOperator);
@@ -205,7 +292,8 @@ class SelectWorkOrder : public WorkOrder {
const std::vector<attribute_id> &simple_selection,
const std::vector<std::unique_ptr<const Scalar>> *selection,
InsertDestination *output_destination,
- StorageManager *storage_manager)
+ StorageManager *storage_manager,
+ const numa_node_id numa_node = 0)
: input_relation_(input_relation),
input_block_id_(input_block_id),
predicate_(predicate),
@@ -213,7 +301,9 @@ class SelectWorkOrder : public WorkOrder {
simple_selection_(simple_selection),
selection_(selection),
output_destination_(DCHECK_NOTNULL(output_destination)),
- storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
+ storage_manager_(DCHECK_NOTNULL(storage_manager)) {
+ preferred_numa_nodes_.push_back(numa_node);
+ }
/**
* @brief Constructor for the distributed version.
@@ -241,7 +331,8 @@ class SelectWorkOrder : public WorkOrder {
std::vector<attribute_id> &&simple_selection,
const std::vector<std::unique_ptr<const Scalar>> *selection,
InsertDestination *output_destination,
- StorageManager *storage_manager)
+ StorageManager *storage_manager,
+ const numa_node_id numa_node = 0)
: input_relation_(input_relation),
input_block_id_(input_block_id),
predicate_(predicate),
@@ -249,7 +340,9 @@ class SelectWorkOrder : public WorkOrder {
simple_selection_(std::move(simple_selection)),
selection_(selection),
output_destination_(DCHECK_NOTNULL(output_destination)),
- storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
+ storage_manager_(DCHECK_NOTNULL(storage_manager)) {
+ preferred_numa_nodes_.push_back(numa_node);
+ }
~SelectWorkOrder() override {}
[47/50] [abbrv] incubator-quickstep git commit: Added BlockLocator.
Posted by hb...@apache.org.
Added BlockLocator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/2221b7e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/2221b7e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/2221b7e4
Branch: refs/heads/query-manager-used-in-foreman
Commit: 2221b7e45bf5b01897e7ccee3016107d78fcea3e
Parents: 8f8a03a
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Sat May 28 14:24:13 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:54 2016 -0700
----------------------------------------------------------------------
query_execution/BlockLocator.cpp | 223 +++++++++++++++
query_execution/BlockLocator.hpp | 125 +++++++++
query_execution/CMakeLists.txt | 48 ++++
query_execution/QueryExecutionMessages.proto | 34 +++
query_execution/QueryExecutionTypedefs.hpp | 16 ++
query_execution/tests/BlockLocator_unittest.cpp | 270 +++++++++++++++++++
storage/CMakeLists.txt | 9 +-
storage/StorageManager.cpp | 190 ++++++++++++-
storage/StorageManager.hpp | 76 +++++-
9 files changed, 984 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/query_execution/BlockLocator.cpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocator.cpp b/query_execution/BlockLocator.cpp
new file mode 100644
index 0000000..6cf5249
--- /dev/null
+++ b/query_execution/BlockLocator.cpp
@@ -0,0 +1,223 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_execution/BlockLocator.hpp"
+
+#include <cstdlib>
+#include <string>
+#include <utility>
+
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "threading/ThreadUtil.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+
+using std::free;
+using std::malloc;
+using std::move;
+
+using tmb::TaggedMessage;
+using tmb::client_id;
+
+namespace quickstep {
+
+void BlockLocator::run() {
+ if (cpu_id_ >= 0) {
+ ThreadUtil::BindToCPU(cpu_id_);
+ }
+
+ for (;;) {
+ // Receive() is a blocking call, causing this thread to sleep until next
+ // message is received.
+ const tmb::AnnotatedMessage annotated_message = bus_->Receive(locator_client_id_, 0, true);
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ const client_id sender = annotated_message.sender;
+ LOG(INFO) << "BlockLocator received the typed '" << tagged_message.message_type()
+ << "' message from TMB Client " << sender;
+ switch (tagged_message.message_type()) {
+ case kBlockDomainRegistrationMessage: {
+ serialization::BlockDomainRegistrationMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processBlockDomainRegistrationMessage(sender, proto.domain_network_address());
+ break;
+ }
+ case kAddBlockLocationMessage: {
+ serialization::BlockLocationMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ const block_id block = proto.block_id();
+ const block_id_domain domain = proto.block_domain();
+
+ const auto result_block_locations = block_locations_[block].insert(domain);
+ const auto result_domain_blocks = domain_blocks_[domain].insert(block);
+ DCHECK_EQ(result_block_locations.second, result_domain_blocks.second);
+
+ if (result_domain_blocks.second) {
+ LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " loaded in Domain " << domain;
+ } else {
+ LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " existed in Domain " << domain;
+ }
+ break;
+ }
+ case kDeleteBlockLocationMessage: {
+ serialization::BlockLocationMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ const block_id block = proto.block_id();
+ const block_id_domain domain = proto.block_domain();
+
+ const auto cit = block_locations_[block].find(domain);
+ if (cit != block_locations_[block].end()) {
+ block_locations_[block].erase(domain);
+ domain_blocks_[domain].erase(block);
+
+ LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " evicted in Domain " << domain;
+ } else {
+ LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " not found in Domain " << domain;
+ }
+ break;
+ }
+ case kLocateBlockMessage: {
+ serialization::BlockMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processLocateBlockMessage(sender, proto.block_id());
+ break;
+ }
+ case kGetPeerDomainNetworkAddressesMessage: {
+ serialization::BlockMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processGetPeerDomainNetworkAddressesMessage(sender, proto.block_id());
+ break;
+ }
+ case kBlockDomainUnregistrationMessage: {
+ serialization::BlockDomainMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ const block_id_domain domain = proto.block_domain();
+
+ domain_network_addresses_.erase(domain);
+
+ for (const block_id block : domain_blocks_[domain]) {
+ block_locations_[block].erase(domain);
+ }
+ domain_blocks_.erase(domain);
+
+ LOG(INFO) << "Unregistered Domain " << domain;
+ break;
+ }
+ case kPoisonMessage: {
+ return;
+ }
+ }
+ }
+}
+
+void BlockLocator::processBlockDomainRegistrationMessage(const client_id receiver,
+ const std::string &network_address) {
+ DCHECK_LT(block_domain_, kMaxDomain);
+
+ domain_network_addresses_.emplace(++block_domain_, network_address);
+ domain_blocks_[block_domain_];
+
+ serialization::BlockDomainMessage proto;
+ proto.set_block_domain(block_domain_);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainRegistrationResponseMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent BlockDomainRegistrationResponseMessage (typed '"
+ << kBlockDomainRegistrationResponseMessage
+ << "') to Worker (id '" << receiver << "')";
+ CHECK(tmb::MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ locator_client_id_,
+ receiver,
+ move(message)));
+}
+
+void BlockLocator::processLocateBlockMessage(const client_id receiver,
+ const block_id block) {
+ serialization::LocateBlockResponseMessage proto;
+
+ for (const block_id_domain domain : block_locations_[block]) {
+ proto.add_block_domains(domain);
+ }
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kLocateBlockResponseMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent LocateBlockResponseMessage (typed '" << kLocateBlockResponseMessage
+ << "') to StorageManager (id '" << receiver << "')";
+ CHECK(tmb::MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ locator_client_id_,
+ receiver,
+ move(message)));
+}
+
+void BlockLocator::processGetPeerDomainNetworkAddressesMessage(const client_id receiver,
+ const block_id block) {
+ serialization::GetPeerDomainNetworkAddressesResponseMessage proto;
+
+ for (const block_id_domain domain : block_locations_[block]) {
+ proto.add_domain_network_addresses(domain_network_addresses_[domain]);
+ }
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kGetPeerDomainNetworkAddressesResponseMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent GetPeerDomainNetworkAddressesResponseMessage (typed '"
+ << kGetPeerDomainNetworkAddressesResponseMessage
+ << "') to StorageManager (id '" << receiver << "')";
+ CHECK(tmb::MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ locator_client_id_,
+ receiver,
+ move(message)));
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/query_execution/BlockLocator.hpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocator.hpp b/query_execution/BlockLocator.hpp
new file mode 100644
index 0000000..bbd9b8f
--- /dev/null
+++ b/query_execution/BlockLocator.hpp
@@ -0,0 +1,125 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_HPP_
+
+#include <atomic>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageConstants.hpp"
+#include "threading/Thread.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+
+namespace quickstep {
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief A class for keeping trace of blocks loaded in a Worker's buffer pool
+ * in the distributed version.
+ **/
+class BlockLocator : public Thread {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param bus A pointer to the TMB.
+ * @param cpu_id The ID of the CPU to which the BlockLocator thread can be pinned.
+ *
+ * @note If cpu_id is not specified, BlockLocator thread can be possibly moved
+ * around on different CPUs by the OS.
+ **/
+ BlockLocator(tmb::MessageBus *bus,
+ const int cpu_id = -1)
+ : bus_(DCHECK_NOTNULL(bus)),
+ cpu_id_(cpu_id),
+ block_domain_(0) {
+ locator_client_id_ = bus_->Connect();
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kBlockDomainRegistrationMessage);
+ bus_->RegisterClientAsSender(locator_client_id_, kBlockDomainRegistrationResponseMessage);
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kAddBlockLocationMessage);
+ bus_->RegisterClientAsReceiver(locator_client_id_, kDeleteBlockLocationMessage);
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kLocateBlockMessage);
+ bus_->RegisterClientAsSender(locator_client_id_, kLocateBlockResponseMessage);
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kGetPeerDomainNetworkAddressesMessage);
+ bus_->RegisterClientAsSender(locator_client_id_, kGetPeerDomainNetworkAddressesResponseMessage);
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kBlockDomainUnregistrationMessage);
+ bus_->RegisterClientAsReceiver(locator_client_id_, kPoisonMessage);
+ }
+
+ ~BlockLocator() override {}
+
+ /**
+ * @brief Get the TMB client ID of BlockLocator thread.
+ *
+ * @return TMB client ID of BlockLocator thread.
+ **/
+ tmb::client_id getBusClientID() const {
+ return locator_client_id_;
+ }
+
+ protected:
+ void run() override;
+
+ private:
+ void processBlockDomainRegistrationMessage(const tmb::client_id receiver, const std::string &network_address);
+ void processLocateBlockMessage(const tmb::client_id receiver, const block_id block);
+ void processGetPeerDomainNetworkAddressesMessage(const tmb::client_id receiver, const block_id block);
+
+ tmb::MessageBus *bus_;
+
+ // The ID of the CPU that the BlockLocator thread can optionally be pinned to.
+ const int cpu_id_;
+
+ alignas(kCacheLineBytes) std::atomic<block_id_domain> block_domain_;
+
+ // From a block domain to its network info in the ip:port format, i.e.,
+ // "0.0.0.0:0".
+ std::unordered_map<block_id_domain, const std::string> domain_network_addresses_;
+
+ // From a block to its domains.
+ std::unordered_map<block_id, std::unordered_set<block_id_domain>> block_locations_;
+
+ // From a block domain to all blocks loaded in its buffer pool.
+ std::unordered_map<block_id_domain, std::unordered_set<block_id>> domain_blocks_;
+
+ tmb::client_id locator_client_id_;
+
+ DISALLOW_COPY_AND_ASSIGN(BlockLocator);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 04a0348..7d9d601 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -20,6 +20,9 @@ QS_PROTOBUF_GENERATE_CPP(queryexecution_QueryExecutionMessages_proto_srcs
QueryExecutionMessages.proto)
# Declare micro-libs:
+if (ENABLE_DISTRIBUTED)
+ add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp BlockLocator.hpp)
+endif()
add_library(quickstep_queryexecution_Foreman Foreman.cpp Foreman.hpp)
add_library(quickstep_queryexecution_ForemanLite ../empty_src.cpp ForemanLite.hpp)
add_library(quickstep_queryexecution_QueryContext QueryContext.cpp QueryContext.hpp)
@@ -40,6 +43,19 @@ add_library(quickstep_queryexecution_WorkerMessage ../empty_src.cpp WorkerMessag
add_library(quickstep_queryexecution_WorkerSelectionPolicy ../empty_src.cpp WorkerSelectionPolicy.hpp)
# Link dependencies:
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryexecution_BlockLocator
+ glog
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_threading_Thread
+ quickstep_threading_ThreadUtil
+ quickstep_utility_Macros
+ tmb)
+endif()
target_link_libraries(quickstep_queryexecution_Foreman
glog
gtest
@@ -176,7 +192,37 @@ target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_WorkerDirectory
quickstep_queryexecution_WorkerMessage
quickstep_queryexecution_WorkerSelectionPolicy)
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryexecution
+ quickstep_queryexecution_BlockLocator)
+endif()
+
# Tests:
+if (ENABLE_DISTRIBUTED)
+ add_executable(BlockLocator_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/BlockLocator_unittest.cpp")
+ target_link_libraries(BlockLocator_unittest
+ gflags_nothreads-static
+ glog
+ gtest
+ quickstep_catalog_CatalogAttribute
+ quickstep_catalog_CatalogRelation
+ quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_StorageBlob
+ quickstep_storage_StorageBlock
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_storage_StorageManager
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ tmb
+ ${LIBS})
+ add_test(BlockLocator_unittest BlockLocator_unittest)
+endif()
+
add_executable(Foreman_unittest
"${CMAKE_CURRENT_SOURCE_DIR}/tests/Foreman_unittest.cpp")
target_link_libraries(Foreman_unittest
@@ -269,3 +315,5 @@ target_link_libraries(WorkerSelectionPolicy_unittest
quickstep_queryexecution_WorkerDirectory
quickstep_queryexecution_WorkerSelectionPolicy)
add_test(WorkerSelectionPolicy_unittest WorkerSelectionPolicy_unittest)
+
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/block_locator_test_data/)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 8d2efd0..15803cf 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -16,6 +16,10 @@ syntax = "proto2";
package quickstep.serialization;
+// Used for any messages that do not carry payloads.
+message EmptyMessage {
+}
+
// Used for both Normal WorkOrders and RebuildWorkOrders.
// NOTE(zuyu): we might need to seperate the completion messages to contain
// run-time information for Foreman to make better decisions on scheduling
@@ -42,3 +46,33 @@ message DataPipelineMessage {
message WorkOrdersAvailableMessage {
required uint64 operator_index = 1;
}
+
+// BlockLocator related messages.
+message BlockDomainRegistrationMessage {
+ // Format IP:Port, i.e., "0.0.0.0:0".
+ required string domain_network_address = 1;
+}
+
+// Used for RegistrationResponse, Unregistration, and FailureReport.
+message BlockDomainMessage {
+ required uint32 block_domain = 1;
+}
+
+// Used when StorageManager loads or evicts a block or a blob from its buffer
+// pool.
+message BlockLocationMessage {
+ required fixed64 block_id = 1;
+ required uint32 block_domain = 2;
+}
+
+message BlockMessage {
+ required fixed64 block_id = 1;
+}
+
+message LocateBlockResponseMessage {
+ repeated uint32 block_domains = 1;
+}
+
+message GetPeerDomainNetworkAddressesResponseMessage {
+ repeated string domain_network_addresses = 1;
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionTypedefs.hpp b/query_execution/QueryExecutionTypedefs.hpp
index 36cfd82..fc253bc 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -18,6 +18,7 @@
#ifndef QUICKSTEP_QUERY_EXECUTION_QUERY_EXECUTION_TYPEDEFS_HPP_
#define QUICKSTEP_QUERY_EXECUTION_QUERY_EXECUTION_TYPEDEFS_HPP_
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED
#include "threading/ThreadIDBasedMap.hpp"
#include "tmb/address.h"
@@ -55,6 +56,7 @@ using ClientIDMap = ThreadIDBasedMap<client_id,
'a',
'p'>;
+// We sort the following message types in the order of a life cycle of a query.
enum QueryExecutionMessageType : message_type_id {
kWorkOrderMessage, // From Foreman to Worker.
kWorkOrderCompleteMessage, // From Worker to Foreman.
@@ -66,6 +68,20 @@ enum QueryExecutionMessageType : message_type_id {
kRebuildWorkOrderMessage, // From Foreman to Worker.
kRebuildWorkOrderCompleteMessage, // From Worker to Foreman.
kPoisonMessage, // From the CLI shell to Foreman, then from Foreman to Workers.
+
+#ifdef QUICKSTEP_DISTRIBUTED
+ // BlockLocator related messages, sorted in a life cycle of StorageManager
+ // with a unique block domain.
+ kBlockDomainRegistrationMessage, // From Worker to BlockLocator.
+ kBlockDomainRegistrationResponseMessage, // From BlockLocator to Worker.
+ kAddBlockLocationMessage, // From StorageManager to BlockLocator.
+ kDeleteBlockLocationMessage, // From StorageManager to BlockLocator.
+ kLocateBlockMessage, // From StorageManager to BlockLocator.
+ kLocateBlockResponseMessage, // From BlockLocator to StorageManager.
+ kGetPeerDomainNetworkAddressesMessage, // From StorageManager to BlockLocator.
+ kGetPeerDomainNetworkAddressesResponseMessage, // From BlockLocator to StorageManager.
+ kBlockDomainUnregistrationMessage, // From StorageManager to BlockLocator.
+#endif
};
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/query_execution/tests/BlockLocator_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/BlockLocator_unittest.cpp b/query_execution/tests/BlockLocator_unittest.cpp
new file mode 100644
index 0000000..fe7b86b
--- /dev/null
+++ b/query_execution/tests/BlockLocator_unittest.cpp
@@ -0,0 +1,270 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogAttribute.hpp"
+#include "catalog/CatalogRelation.hpp"
+#include "query_execution/BlockLocator.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "storage/StorageBlob.hpp"
+#include "storage/StorageBlock.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageConstants.hpp"
+#include "storage/StorageManager.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+
+using std::free;
+using std::malloc;
+using std::move;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+using tmb::AnnotatedMessage;
+using tmb::MessageBus;
+using tmb::TaggedMessage;
+
+namespace quickstep {
+
+class BlockLocatorTest : public ::testing::Test {
+ protected:
+ static const char kStoragePath[];
+ static const char kDomainNetworkAddress[];
+
+ ~BlockLocatorTest() {
+ locator_->join();
+ }
+
+ virtual void SetUp() {
+ bus_.Initialize();
+
+ locator_.reset(new BlockLocator(&bus_));
+ locator_client_id_ = locator_->getBusClientID();
+ locator_->start();
+
+ worker_client_id_ = bus_.Connect();
+
+ bus_.RegisterClientAsSender(worker_client_id_, kBlockDomainRegistrationMessage);
+ bus_.RegisterClientAsReceiver(worker_client_id_, kBlockDomainRegistrationResponseMessage);
+
+ bus_.RegisterClientAsSender(worker_client_id_, kLocateBlockMessage);
+ bus_.RegisterClientAsReceiver(worker_client_id_, kLocateBlockResponseMessage);
+
+ bus_.RegisterClientAsSender(worker_client_id_, kPoisonMessage);
+
+ block_domain_ = getBlockDomain(kDomainNetworkAddress);
+
+ storage_manager_.reset(
+ new StorageManager(kStoragePath, block_domain_, locator_client_id_, &bus_));
+ }
+
+ virtual void TearDown() {
+ storage_manager_.reset();
+
+ serialization::EmptyMessage proto;
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kPoisonMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent PoisonMessage (typed '" << kPoisonMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+ }
+
+ vector<block_id_domain> getPeerDomains(const block_id block) {
+ serialization::BlockMessage proto;
+ proto.set_block_id(block);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kLocateBlockMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent LocateBlockMessage (typed '" << kLocateBlockMessage
+ << "') to BlockLocator";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+
+ const AnnotatedMessage annotated_message(bus_.Receive(worker_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ CHECK_EQ(kLocateBlockResponseMessage, tagged_message.message_type());
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') received LocateBlockResponseMessage from BlockLocator";
+
+ serialization::LocateBlockResponseMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ vector<block_id_domain> domains;
+ for (int i = 0; i < response_proto.block_domains_size(); ++i) {
+ domains.push_back(response_proto.block_domains(i));
+ }
+
+ return domains;
+ }
+
+ void checkLoaded(const block_id block) {
+ const vector<string> peer_domain_network_addresses = storage_manager_->getPeerDomainNetworkAddresses(block);
+ EXPECT_EQ(1u, peer_domain_network_addresses.size());
+ EXPECT_STREQ(kDomainNetworkAddress, peer_domain_network_addresses[0].data());
+
+ const vector<block_id_domain> domains = getPeerDomains(block);
+ EXPECT_EQ(1u, domains.size());
+ EXPECT_EQ(block_domain_, domains[0]);
+ }
+
+ void checkEvicted(const block_id block) {
+ const vector<string> peer_domain_network_addresses = storage_manager_->getPeerDomainNetworkAddresses(block);
+ EXPECT_TRUE(peer_domain_network_addresses.empty());
+
+ const vector<block_id_domain> domains = getPeerDomains(block);
+ EXPECT_TRUE(domains.empty());
+ }
+
+ tmb::client_id worker_client_id_;
+
+ block_id_domain block_domain_;
+ unique_ptr<StorageManager> storage_manager_;
+
+ private:
+ block_id_domain getBlockDomain(const string &network_address) {
+ serialization::BlockDomainRegistrationMessage proto;
+ proto.set_domain_network_address(network_address);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainRegistrationMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+
+ const AnnotatedMessage annotated_message(bus_.Receive(worker_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ EXPECT_EQ(locator_client_id_, annotated_message.sender);
+ EXPECT_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') received BlockDomainRegistrationResponseMessage from BlockLocator";
+
+ serialization::BlockDomainMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ return static_cast<block_id_domain>(response_proto.block_domain());
+ }
+
+ MessageBusImpl bus_;
+
+ unique_ptr<BlockLocator> locator_;
+ tmb::client_id locator_client_id_;
+};
+
+const char BlockLocatorTest::kStoragePath[] = "./block_locator_test_data/";
+const char BlockLocatorTest::kDomainNetworkAddress[] = "ip:port";
+
+TEST_F(BlockLocatorTest, BlockTest) {
+ CatalogRelation relation(nullptr, "rel");
+ relation.addAttribute(new CatalogAttribute(nullptr, "attr_int", TypeFactory::GetType(kInt)));
+
+ const block_id block =
+ storage_manager_->createBlock(relation, relation.getDefaultStorageBlockLayout());
+ checkLoaded(block);
+
+ ASSERT_TRUE(storage_manager_->saveBlockOrBlob(block));
+ storage_manager_->evictBlockOrBlob(block);
+ checkEvicted(block);
+
+ {
+ const BlockReference block_ref = storage_manager_->getBlock(block, relation);
+ }
+ checkLoaded(block);
+
+ storage_manager_->deleteBlockOrBlobFile(block);
+ checkEvicted(block);
+}
+
+TEST_F(BlockLocatorTest, BlobTest) {
+ const block_id blob = storage_manager_->createBlob(kDefaultBlockSizeInSlots);
+ checkLoaded(blob);
+
+ ASSERT_TRUE(storage_manager_->saveBlockOrBlob(blob));
+ storage_manager_->evictBlockOrBlob(blob);
+ checkEvicted(blob);
+
+ {
+ const BlobReference blob_ref = storage_manager_->getBlob(blob);
+ }
+ checkLoaded(blob);
+
+ storage_manager_->deleteBlockOrBlobFile(blob);
+ checkEvicted(blob);
+}
+
+} // namespace quickstep
+
+int main(int argc, char **argv) {
+ google::InitGoogleLogging(argv[0]);
+ // Honor FLAGS_buffer_pool_slots in StorageManager.
+ gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+ ::testing::InitGoogleTest(&argc, argv);
+
+ return RUN_ALL_TESTS();
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 87a5e54..4da16ea 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -941,7 +941,8 @@ target_link_libraries(quickstep_storage_StorageManager
quickstep_utility_Alignment
quickstep_utility_CalculateInstalledMemory
quickstep_utility_Macros
- quickstep_utility_ShardedLockManager)
+ quickstep_utility_ShardedLockManager
+ tmb)
if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
target_link_libraries(quickstep_storage_StorageManager
quickstep_storage_FileManagerHdfs)
@@ -950,6 +951,12 @@ if (QUICKSTEP_HAVE_LIBNUMA)
target_link_libraries(quickstep_storage_StorageManager
${LIBNUMA_LIBRARY})
endif()
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_storage_StorageManager
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil)
+endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_storage_SubBlockTypeRegistry
glog
quickstep_storage_StorageBlockLayout_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 5d91052..15e2503 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -18,6 +18,7 @@
// This is included before other files so that we can conditionally determine
// what else to include.
#include "catalog/CatalogConfig.h"
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED
#include "storage/StorageConfig.h"
// Define feature test macros to enable large page support for mmap.
@@ -52,6 +53,12 @@
#include <unordered_map>
#include <vector>
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#endif
+
#include "storage/CountedReference.hpp"
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManagerLocal.hpp"
@@ -74,6 +81,13 @@
#include "gflags/gflags.h"
#include "glog/logging.h"
+#include "tmb/id_typedefs.h"
+
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+#endif
+
using std::free;
using std::int32_t;
using std::memset;
@@ -81,6 +95,15 @@ using std::size_t;
using std::string;
using std::vector;
+#ifdef QUICKSTEP_DISTRIBUTED
+using std::malloc;
+using std::move;
+using std::unique_ptr;
+
+using tmb::MessageBus;
+using tmb::TaggedMessage;
+#endif
+
namespace quickstep {
static bool ValidateBlockDomain(const char *flagname,
@@ -157,14 +180,21 @@ DEFINE_bool(use_hdfs, false, "Use HDFS as the persistent storage, instead of the
#endif
StorageManager::StorageManager(
- const std::string &storage_path,
- const block_id_domain block_domain,
- const size_t max_memory_usage,
- EvictionPolicy *eviction_policy)
+ const std::string &storage_path,
+ const block_id_domain block_domain,
+ const size_t max_memory_usage,
+ EvictionPolicy *eviction_policy,
+ const tmb::client_id block_locator_client_id,
+ tmb::MessageBus *bus)
: storage_path_(storage_path),
total_memory_usage_(0),
max_memory_usage_(max_memory_usage),
- eviction_policy_(eviction_policy) {
+ eviction_policy_(eviction_policy),
+#ifdef QUICKSTEP_DISTRIBUTED
+ block_domain_(block_domain),
+#endif
+ block_locator_client_id_(block_locator_client_id),
+ bus_(bus) {
#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
if (FLAGS_use_hdfs) {
file_manager_.reset(new FileManagerHdfs(storage_path));
@@ -175,10 +205,55 @@ StorageManager::StorageManager(
file_manager_.reset(new FileManagerLocal(storage_path));
#endif
+#ifdef QUICKSTEP_DISTRIBUTED
+ // NOTE(zuyu): The following if-condition is a workaround to bypass code for
+ // the distributed version in some unittests that does not use TMB. The
+ // end-to-end functional tests for the distributed version, however, would not
+ // be affected.
+ if (bus_) {
+ storage_manager_client_id_ = bus_->Connect();
+
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kGetPeerDomainNetworkAddressesMessage);
+ bus_->RegisterClientAsReceiver(storage_manager_client_id_, kGetPeerDomainNetworkAddressesResponseMessage);
+
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kAddBlockLocationMessage);
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kDeleteBlockLocationMessage);
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kBlockDomainUnregistrationMessage);
+
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') starts with Domain " << block_domain;
+ }
+#endif
+
block_index_ = BlockIdUtil::GetBlockId(block_domain, file_manager_->getMaxUsedBlockCounter(block_domain));
}
StorageManager::~StorageManager() {
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ serialization::BlockDomainMessage proto;
+ proto.set_block_domain(block_domain_);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainUnregistrationMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent BlockDomainUnregistrationMessage (typed '" << kBlockDomainUnregistrationMessage
+ << "') to BlockLocator";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ storage_manager_client_id_,
+ block_locator_client_id_,
+ move(message)));
+ }
+#endif
+
for (std::unordered_map<block_id, BlockHandle>::iterator it = blocks_.begin();
it != blocks_.end();
++it) {
@@ -222,6 +297,12 @@ block_id StorageManager::createBlock(const CatalogRelationSchema &relation,
// Make '*eviction_policy_' aware of the new block's existence.
eviction_policy_->blockCreated(new_block_id);
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ sendBlockLocationMessage(new_block_id, kAddBlockLocationMessage);
+ }
+#endif
+
return new_block_id;
}
@@ -249,6 +330,12 @@ block_id StorageManager::createBlob(const std::size_t num_slots,
// Make '*eviction_policy_' aware of the new blob's existence.
eviction_policy_->blockCreated(new_block_id);
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ sendBlockLocationMessage(new_block_id, kAddBlockLocationMessage);
+ }
+#endif
+
return new_block_id;
}
@@ -315,6 +402,12 @@ bool StorageManager::saveBlockOrBlob(const block_id block, const bool force) {
}
void StorageManager::evictBlockOrBlob(const block_id block) {
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ sendBlockLocationMessage(block, kDeleteBlockLocationMessage);
+ }
+#endif
+
BlockHandle handle;
{
SpinSharedMutexExclusiveLock<false> write_lock(blocks_shared_mutex_);
@@ -362,6 +455,87 @@ block_id StorageManager::allocateNewBlockOrBlob(const std::size_t num_slots,
return ++block_index_;
}
+#ifdef QUICKSTEP_DISTRIBUTED
+vector<string> StorageManager::getPeerDomainNetworkAddresses(const block_id block) {
+ serialization::BlockMessage proto;
+ proto.set_block_id(block);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kGetPeerDomainNetworkAddressesMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent GetPeerDomainNetworkAddressesMessage (typed '" << kGetPeerDomainNetworkAddressesMessage
+ << "') to BlockLocator";
+
+ DCHECK_NE(block_locator_client_id_, tmb::kClientIdNone);
+ DCHECK(bus_ != nullptr);
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ storage_manager_client_id_,
+ block_locator_client_id_,
+ move(message)));
+
+ const tmb::AnnotatedMessage annotated_message(bus_->Receive(storage_manager_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ CHECK_EQ(block_locator_client_id_, annotated_message.sender);
+ CHECK_EQ(kGetPeerDomainNetworkAddressesResponseMessage, tagged_message.message_type());
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') received GetPeerDomainNetworkAddressesResponseMessage from BlockLocator";
+
+ serialization::GetPeerDomainNetworkAddressesResponseMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ vector<string> domain_network_addresses;
+ for (int i = 0; i < response_proto.domain_network_addresses_size(); ++i) {
+ domain_network_addresses.push_back(response_proto.domain_network_addresses(i));
+ }
+
+ return domain_network_addresses;
+}
+
+void StorageManager::sendBlockLocationMessage(const block_id block,
+ const tmb::message_type_id message_type) {
+ switch (message_type) {
+ case kAddBlockLocationMessage:
+ LOG(INFO) << "Loaded Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
+ break;
+ case kDeleteBlockLocationMessage:
+ LOG(INFO) << "Evicted Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
+ break;
+ default:
+ LOG(FATAL) << "Unknown message type " << message_type;
+ }
+
+ serialization::BlockLocationMessage proto;
+ proto.set_block_id(block);
+ proto.set_block_domain(block_domain_);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ message_type);
+ free(proto_bytes);
+
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent BlockLocationMessage (typed '" << message_type
+ << "') to BlockLocator";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ storage_manager_client_id_,
+ block_locator_client_id_,
+ move(message)));
+}
+#endif
+
StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
const block_id block, const int numa_node) {
// The caller of this function holds an exclusive lock on this block/blob's
@@ -378,6 +552,12 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
loaded_handle.block_memory = block_buffer;
loaded_handle.block_memory_size = num_slots;
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ sendBlockLocationMessage(block, kAddBlockLocationMessage);
+ }
+#endif
+
return loaded_handle;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index 52326c2..55a011e 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -20,11 +20,14 @@
#include <atomic>
#include <chrono>
+#include <cstddef>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED
+
#include "storage/CountedReference.hpp"
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManager.hpp"
@@ -40,6 +43,10 @@
#include "gflags/gflags.h"
#include "gtest/gtest_prod.h"
+#include "tmb/id_typedefs.h"
+
+namespace tmb { class MessageBus; }
+
namespace quickstep {
DECLARE_int32(block_domain);
@@ -50,6 +57,7 @@ DECLARE_bool(use_hdfs);
#endif
class CatalogRelationSchema;
+
class StorageBlockLayout;
/** \addtogroup Storage
@@ -104,6 +112,33 @@ class StorageManager {
std::chrono::milliseconds(200))) {
}
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Constructor.
+ * @param storage_path The filesystem directory where blocks have persistent
+ * storage.
+ * @param block_domain The unique block domain.
+ * @param block_locator_client_id The TMB client ID of the block locator.
+ * @param bus A pointer to the TMB.
+ *
+ * @exception CorruptPersistentStorage The storage directory layout is not
+ * in the expected format.
+ **/
+ StorageManager(const std::string &storage_path,
+ const block_id_domain block_domain,
+ const tmb::client_id block_locator_client_id,
+ tmb::MessageBus *bus)
+ : StorageManager(storage_path,
+ block_domain,
+ FLAGS_buffer_pool_slots,
+ LRUKEvictionPolicyFactory::ConstructLRUKEvictionPolicy(
+ 2,
+ std::chrono::milliseconds(200)),
+ block_locator_client_id,
+ bus) {
+ }
+#endif
+
/**
* @brief Constructor.
* @param storage_path The filesystem directory where blocks have persistent
@@ -121,13 +156,18 @@ class StorageManager {
* @param eviction_policy The eviction policy that the storage manager should
* use to manage the cache. The storage manager takes
* ownership of *eviction_policy.
+ * @param block_locator_client_id The TMB client ID of the block locator.
+ * @param bus A pointer to the TMB.
+ *
* @exception CorruptPersistentStorage The storage directory layout is not
* in the expected format.
**/
StorageManager(const std::string &storage_path,
const block_id_domain block_domain,
const size_t max_memory_usage,
- EvictionPolicy *eviction_policy);
+ EvictionPolicy *eviction_policy,
+ const tmb::client_id block_locator_client_id = tmb::kClientIdNone,
+ tmb::MessageBus *bus = nullptr);
/**
* @brief Destructor which also destroys all managed blocks.
@@ -332,6 +372,27 @@ class StorageManager {
StorageBlockBase *block;
};
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Get the network info of all the remote StorageManagers which may
+ * load the given block in the buffer pool.
+ *
+ * @param block The block or blob to pull.
+ *
+ * @return The network info of all the possible peers to pull.
+ **/
+ std::vector<std::string> getPeerDomainNetworkAddresses(const block_id block);
+
+ /**
+ * @brief Update the block location info in BlockLocator.
+ *
+ * @param block The given block or blob.
+ * @param message_type Indicate whether to add or delete a block location.
+ **/
+ void sendBlockLocationMessage(const block_id block,
+ const tmb::message_type_id message_type);
+#endif
+
// Helper for createBlock() and createBlob(). Allocates a block ID and memory
// slots for a new StorageBlock or StorageBlob. Returns the allocated ID and
// writes the allocated slot range into 'handle->slot_index_low' and
@@ -459,6 +520,15 @@ class StorageManager {
std::unique_ptr<EvictionPolicy> eviction_policy_;
+#ifdef QUICKSTEP_DISTRIBUTED
+ const block_id_domain block_domain_;
+
+ tmb::client_id storage_manager_client_id_;
+#endif
+
+ const tmb::client_id block_locator_client_id_;
+ tmb::MessageBus *bus_;
+
std::unique_ptr<FileManager> file_manager_;
// Used to generate unique IDs in allocateNewBlockOrBlob().
@@ -486,6 +556,10 @@ class StorageManager {
static constexpr std::size_t kLockManagerNumShards = 0x2000-1;
ShardedLockManager<block_id, kLockManagerNumShards, SpinSharedMutex<false>> lock_manager_;
+ friend class BlockLocatorTest;
+ FRIEND_TEST(BlockLocatorTest, BlockTest);
+ FRIEND_TEST(BlockLocatorTest, BlobTest);
+
FRIEND_TEST(StorageManagerTest, DifferentNUMANodeBlobTestWithEviction);
FRIEND_TEST(StorageManagerTest, EvictFromSameShardTest);
[50/50] [abbrv] incubator-quickstep git commit: Merge branch 'master'
into query-manager-used-in-foreman
Posted by hb...@apache.org.
Merge branch 'master' into query-manager-used-in-foreman
Conflicts:
cli/QuickstepCli.cpp
query_execution/CMakeLists.txt
query_execution/QueryContext.proto
query_execution/QueryExecutionTypedefs.hpp
query_optimizer/ExecutionGenerator.hpp
storage/StorageManager.hpp
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/bef0ae1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/bef0ae1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/bef0ae1d
Branch: refs/heads/query-manager-used-in-foreman
Commit: bef0ae1d83649b1d1a288f94e4be6bca28ea6037
Parents: 24c93ca 69c6551
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Wed Jun 8 08:15:49 2016 -0500
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Wed Jun 8 08:15:49 2016 -0500
----------------------------------------------------------------------
CMakeLists.txt | 17 +-
DISCLAIMER | 11 +
README.md | 25 +-
catalog/CMakeLists.txt | 16 +-
catalog/Catalog.proto | 13 +-
catalog/CatalogRelation.cpp | 11 +
catalog/CatalogRelation.hpp | 52 +-
catalog/CatalogRelationStatistics.cpp | 49 +
catalog/CatalogRelationStatistics.hpp | 122 +
cli/CMakeLists.txt | 23 +-
cli/CommandExecutor.cpp | 189 +-
cli/CommandExecutor.hpp | 28 +-
cli/DefaultsConfigurator.hpp | 16 +
cli/InputParserUtil.cpp | 1 +
cli/PrintToScreen.cpp | 33 +
cli/PrintToScreen.hpp | 31 +
cli/QuickstepCli.cpp | 57 +-
cli/tests/CommandExecutorTestRunner.cpp | 5 +
cli/tests/command_executor/D.test | 35 +-
cli/tests/command_executor/Dt.test | 36 +-
.../aggregation/AggregationConcreteHandle.hpp | 105 +
expressions/aggregation/AggregationHandle.hpp | 15 +-
.../aggregation/AggregationHandleAvg.cpp | 9 +
.../aggregation/AggregationHandleAvg.hpp | 4 +
.../aggregation/AggregationHandleCount.cpp | 11 +
.../aggregation/AggregationHandleCount.hpp | 4 +
.../aggregation/AggregationHandleDistinct.hpp | 7 +
.../aggregation/AggregationHandleMax.cpp | 9 +
.../aggregation/AggregationHandleMax.hpp | 4 +
.../aggregation/AggregationHandleMin.cpp | 9 +
.../aggregation/AggregationHandleMin.hpp | 4 +
.../aggregation/AggregationHandleSum.cpp | 11 +-
.../aggregation/AggregationHandleSum.hpp | 4 +
expressions/aggregation/CMakeLists.txt | 2 +
.../tests/AggregationHandleAvg_unittest.cpp | 109 +
.../tests/AggregationHandleCount_unittest.cpp | 126 +-
.../tests/AggregationHandleMax_unittest.cpp | 122 +
.../tests/AggregationHandleMin_unittest.cpp | 121 +
.../tests/AggregationHandleSum_unittest.cpp | 124 +
parser/ParseBasicExpressions.cpp | 31 +
parser/ParseBasicExpressions.hpp | 77 +
parser/ParseExpression.hpp | 1 +
parser/ParseIndexProperties.hpp | 2 +-
parser/SqlLexer.lpp | 2 +
parser/SqlParser.ypp | 16 +
parser/preprocessed/SqlLexer_gen.cpp | 1173 +++----
parser/preprocessed/SqlLexer_gen.hpp | 2 +-
parser/preprocessed/SqlParser_gen.cpp | 2920 +++++++++---------
parser/preprocessed/SqlParser_gen.hpp | 126 +-
parser/tests/Index.test | 11 +
query_execution/AdmitRequestMessage.hpp | 2 +
query_execution/BlockLocator.cpp | 223 ++
query_execution/BlockLocator.hpp | 125 +
query_execution/CMakeLists.txt | 51 +
query_execution/Foreman.cpp | 3 +
query_execution/PolicyEnforcer.cpp | 9 +-
query_execution/QueryContext.cpp | 16 +-
query_execution/QueryContext.hpp | 57 +-
query_execution/QueryContext.proto | 20 +-
query_execution/QueryExecutionMessages.proto | 34 +
query_execution/QueryExecutionTypedefs.hpp | 20 +-
query_execution/QueryExecutionUtil.hpp | 45 +-
query_execution/tests/BlockLocator_unittest.cpp | 270 ++
query_optimizer/CMakeLists.txt | 16 +-
query_optimizer/ExecutionGenerator.cpp | 139 +-
query_optimizer/ExecutionGenerator.hpp | 15 +-
query_optimizer/ExecutionHeuristics.cpp | 127 +
query_optimizer/ExecutionHeuristics.hpp | 155 +
query_optimizer/PhysicalGenerator.cpp | 25 +
query_optimizer/QueryProcessor.hpp | 10 +
query_optimizer/cost_model/CMakeLists.txt | 33 +-
query_optimizer/cost_model/SimpleCostModel.cpp | 9 +
query_optimizer/cost_model/SimpleCostModel.hpp | 8 +-
.../cost_model/StarSchemaSimpleCostModel.cpp | 258 ++
.../cost_model/StarSchemaSimpleCostModel.hpp | 115 +
.../logical/SharedSubplanReference.cpp | 7 +-
.../logical/SharedSubplanReference.hpp | 34 +-
query_optimizer/physical/Aggregate.cpp | 14 +
.../physical/SharedSubplanReference.cpp | 5 +
.../physical/SharedSubplanReference.hpp | 34 +-
query_optimizer/resolver/CMakeLists.txt | 1 +
query_optimizer/resolver/Resolver.cpp | 84 +-
query_optimizer/rules/CMakeLists.txt | 20 +-
query_optimizer/rules/RuleHelper.cpp | 18 +-
.../StarSchemaHashJoinOrderOptimization.cpp | 309 ++
.../StarSchemaHashJoinOrderOptimization.hpp | 136 +
query_optimizer/strategy/OneToOne.cpp | 1 +
query_optimizer/tests/CMakeLists.txt | 17 +
.../tests/ExecutionGeneratorTestRunner.cpp | 17 +-
.../tests/ExecutionHeuristics_unittest.cpp | 301 ++
query_optimizer/tests/OptimizerTextTest.cpp | 14 +
.../tests/execution_generator/CMakeLists.txt | 2 +-
.../tests/execution_generator/Select.test | 101 +-
.../tests/logical_generator/Create.test | 4 +-
.../tests/logical_generator/Select.test | 106 +-
.../tests/physical_generator/Create.test | 6 +-
.../tests/physical_generator/Select.test | 181 +-
query_optimizer/tests/resolver/Create.test | 28 +-
query_optimizer/tests/resolver/Select.test | 259 +-
relational_operators/CMakeLists.txt | 8 +
relational_operators/HashJoinOperator.cpp | 7 +-
relational_operators/SelectOperator.cpp | 129 +-
relational_operators/SelectOperator.hpp | 115 +-
relational_operators/WorkOrder.proto | 98 +-
relational_operators/WorkOrderFactory.cpp | 362 +--
storage/AggregationOperationState.cpp | 112 +-
storage/AggregationOperationState.hpp | 9 +
storage/AggregationOperationState.proto | 5 +
storage/CMakeLists.txt | 116 +-
storage/DataExchange.proto | 31 +
storage/DataExchangerAsync.cpp | 165 +
storage/DataExchangerAsync.hpp | 97 +
storage/EvictionPolicy.hpp | 6 +
storage/FileManager.hpp | 9 +-
storage/FileManagerHdfs.cpp | 59 +-
storage/FileManagerPosix.cpp | 55 +-
storage/FileManagerPosix.hpp | 6 +-
storage/FileManagerWindows.cpp | 55 +-
storage/FileManagerWindows.hpp | 4 +-
storage/HashTable.hpp | 104 +
storage/HashTable.proto | 8 +
storage/HashTableFactory.hpp | 44 +-
storage/HashTablePool.hpp | 166 +
storage/PreloaderThread.cpp | 98 +-
storage/PreloaderThread.hpp | 30 +
storage/SMAIndexSubBlock.cpp | 48 +-
storage/SMAIndexSubBlock.hpp | 3 +-
.../SimpleScalarSeparateChainingHashTable.hpp | 10 +-
storage/StorageBlock.cpp | 5 +
storage/StorageBlock.hpp | 11 +-
storage/StorageConstants.hpp | 15 +-
storage/StorageManager.cpp | 473 ++-
storage/StorageManager.hpp | 182 +-
storage/tests/DataExchange_unittest.cpp | 240 ++
storage/tests/SMAIndexSubBlock_unittest.cpp | 52 +
storage/tests/StorageManager_unittest.cpp | 42 +
third_party/iwyu/iwyu_helper.py | 8 +-
transaction/AccessMode.hpp | 76 +-
transaction/CMakeLists.txt | 59 +-
transaction/CycleDetector.cpp | 120 +
transaction/CycleDetector.hpp | 83 +
transaction/DeadLockDetector.cpp | 177 ++
transaction/DeadLockDetector.hpp | 156 +
transaction/DirectedGraph.hpp | 56 +-
transaction/LockManager.cpp | 237 ++
transaction/LockManager.hpp | 128 +
transaction/LockTable.cpp | 22 +-
transaction/LockTable.hpp | 36 +-
transaction/ResourceId.hpp | 9 +-
transaction/StronglyConnectedComponents.cpp | 1 -
transaction/Transaction.cpp | 48 -
transaction/TransactionTable.cpp | 8 +-
transaction/TransactionTable.hpp | 11 +-
transaction/tests/AccessMode_unittest.cpp | 12 +-
transaction/tests/CycleDetector_unittest.cpp | 157 +
transaction/tests/DeadLockDetector_unittest.cpp | 96 +
transaction/tests/DirectedGraph_unittest.cpp | 113 +-
transaction/tests/LockRequest_unittest.cpp | 4 +-
transaction/tests/LockTable_unittest.cpp | 63 +-
transaction/tests/Lock_unittest.cpp | 13 +-
.../StronglyConnectedComponents_unittest.cpp | 15 +-
transaction/tests/TransactionTable_unittest.cpp | 102 +-
types/operations/Operation.proto | 9 +
.../operations/unary_operations/CMakeLists.txt | 22 +
.../unary_operations/SubstringOperation.cpp | 214 ++
.../unary_operations/SubstringOperation.hpp | 234 ++
.../unary_operations/UnaryOperation.cpp | 4 +
.../unary_operations/UnaryOperationFactory.cpp | 12 +
.../unary_operations/UnaryOperationID.cpp | 6 +-
.../unary_operations/UnaryOperationID.hpp | 3 +
utility/BloomFilter.hpp | 198 +-
utility/BloomFilter.proto | 30 +
utility/CMakeLists.txt | 31 +
utility/PlanVisualizer.cpp | 161 +
utility/PlanVisualizer.hpp | 94 +
utility/ScopedBuffer.hpp | 19 +-
utility/ShardedLockManager.hpp | 71 +-
utility/StringUtil.cpp | 14 +
utility/StringUtil.hpp | 48 +-
validate_cmakelists.py | 5 +-
180 files changed, 12093 insertions(+), 3286 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/CMakeLists.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --cc cli/CommandExecutor.cpp
index f38121f,ddcd38f..cfb8c5b
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@@ -28,15 -29,28 +29,30 @@@
#include "catalog/CatalogDatabase.hpp"
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogRelationSchema.hpp"
+ #include "cli/DropRelation.hpp"
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
+ #include "parser/ParseString.hpp"
+ #include "parser/SqlParserWrapper.hpp"
+ #include "query_execution/Foreman.hpp"
+ #include "query_optimizer/QueryHandle.hpp"
+ #include "query_optimizer/QueryPlan.hpp"
+ #include "query_optimizer/QueryProcessor.hpp"
+ #include "storage/StorageBlock.hpp"
+ #include "storage/StorageBlockInfo.hpp"
+ #include "storage/StorageManager.hpp"
+ #include "storage/TupleIdSequence.hpp"
+ #include "storage/TupleStorageSubBlock.hpp"
+ #include "types/Type.hpp"
+ #include "types/TypeID.hpp"
+ #include "types/TypedValue.hpp"
#include "utility/PtrVector.hpp"
- #include "utility/Macros.hpp"
#include "utility/SqlError.hpp"
- #include "gflags/gflags.h"
#include "glog/logging.h"
++#include "tmb/id_typedefs.h"
++
using std::fprintf;
using std::fputc;
using std::fputs;
@@@ -44,6 -58,6 +60,8 @@@ using std::size_t
using std::string;
using std::vector;
++namespace tmb { class MessageBus; }
++
namespace quickstep {
namespace cli {
namespace {
@@@ -162,10 -191,130 +195,143 @@@ void executeDescribeTable
}
}
+ /**
+ * @brief A helper function that executes a SQL query to obtain a scalar result.
+ */
-inline TypedValue executeQueryForSingleResult(const std::string &query_string,
- StorageManager *storage_manager,
- QueryProcessor *query_processor,
- SqlParserWrapper *parser_wrapper,
- Foreman *foreman) {
++inline TypedValue executeQueryForSingleResult(
++ const tmb::client_id main_thread_client_id,
++ const tmb::client_id foreman_client_id,
++ const std::string &query_string,
++ tmb::MessageBus *bus,
++ StorageManager *storage_manager,
++ QueryProcessor *query_processor,
++ SqlParserWrapper *parser_wrapper) {
+ parser_wrapper->feedNextBuffer(new std::string(query_string));
+
+ ParseResult result = parser_wrapper->getNextStatement();
+ DCHECK(result.condition == ParseResult::kSuccess);
+
+ // Generate the query plan.
+ std::unique_ptr<QueryHandle> query_handle(
+ query_processor->generateQueryHandle(*result.parsed_statement));
+ DCHECK(query_handle->getQueryPlanMutable() != nullptr);
+
+ // Use foreman to execute the query plan.
- foreman->setQueryPlan(query_handle->getQueryPlanMutable()->getQueryPlanDAGMutable());
++ /*foreman->setQueryPlan(query_handle->getQueryPlanMutable()->getQueryPlanDAGMutable());
+ foreman->reconstructQueryContextFromProto(query_handle->getQueryContextProto());
+
+ foreman->start();
- foreman->join();
++ foreman->join();*/
++ QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
++ main_thread_client_id, foreman_client_id, query_handle.get(), bus);
+
+ // Retrieve the scalar result from the result relation.
+ const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
+ DCHECK(query_result_relation != nullptr);
+
+ TypedValue value;
+ {
+ std::vector<block_id> blocks = query_result_relation->getBlocksSnapshot();
+ DCHECK_EQ(1u, blocks.size());
+ BlockReference block = storage_manager->getBlock(blocks[0], *query_result_relation);
+ const TupleStorageSubBlock &tuple_store = block->getTupleStorageSubBlock();
+ DCHECK_EQ(1, tuple_store.numTuples());
+ DCHECK_EQ(1u, tuple_store.getRelation().size());
+
+ if (tuple_store.isPacked()) {
+ value = tuple_store.getAttributeValueTyped(0, 0);
+ } else {
+ std::unique_ptr<TupleIdSequence> existence_map(tuple_store.getExistenceMap());
+ value = tuple_store.getAttributeValueTyped(*existence_map->begin(), 0);
+ }
+ value.ensureNotReference();
+ }
+
+ // Drop the result relation.
+ DropRelation::Drop(*query_result_relation,
+ query_processor->getDefaultDatabase(),
+ query_processor->getStorageManager());
+
+ return value;
+ }
+
-void executeAnalyze(QueryProcessor *query_processor,
- Foreman *foreman,
++void executeAnalyze(const tmb::client_id main_thread_client_id,
++ const tmb::client_id foreman_client_id,
++ MessageBus *bus,
++ QueryProcessor *query_processor,
+ FILE *out) {
+ const CatalogDatabase &database = *query_processor->getDefaultDatabase();
+ StorageManager *storage_manager = query_processor->getStorageManager();
+
+ std::unique_ptr<SqlParserWrapper> parser_wrapper(new SqlParserWrapper());
+ std::vector<std::reference_wrapper<const CatalogRelation>> relations(
+ database.begin(), database.end());
+
+ // Analyze each relation in the database.
+ for (const CatalogRelation &relation : relations) {
+ fprintf(out, "Analyzing %s ... ", relation.getName().c_str());
+ fflush(out);
+
+ CatalogRelation *mutable_relation =
+ query_processor->getDefaultDatabase()->getRelationByIdMutable(relation.getID());
+
+ // Get the number of distinct values for each column.
+ for (const CatalogAttribute &attribute : relation) {
+ std::string query_string = "SELECT COUNT(DISTINCT ";
+ query_string.append(attribute.getName());
+ query_string.append(") FROM ");
+ query_string.append(relation.getName());
+ query_string.append(";");
+
+ TypedValue num_distinct_values =
- executeQueryForSingleResult(query_string,
++ executeQueryForSingleResult(main_thread_client_id,
++ foreman_client_id,
++ query_string,
++ bus,
+ storage_manager,
+ query_processor,
- parser_wrapper.get(),
- foreman);
++ parser_wrapper.get());
+
+ DCHECK(num_distinct_values.getTypeID() == TypeID::kLong);
+ mutable_relation->getStatisticsMutable()->setNumDistinctValues(
+ attribute.getID(),
+ num_distinct_values.getLiteral<std::int64_t>());
+ }
+
+ // Get the number of tuples for the relation.
+ std::string query_string = "SELECT COUNT(*) FROM ";
+ query_string.append(relation.getName());
+ query_string.append(";");
+
+ TypedValue num_tuples =
- executeQueryForSingleResult(query_string,
++ executeQueryForSingleResult(main_thread_client_id,
++ foreman_client_id,
++ query_string,
++ bus,
+ storage_manager,
+ query_processor,
- parser_wrapper.get(),
- foreman);
++ parser_wrapper.get());
+
+ DCHECK(num_tuples.getTypeID() == TypeID::kLong);
+ mutable_relation->getStatisticsMutable()->setNumTuples(
+ num_tuples.getLiteral<std::int64_t>());
+
+ fprintf(out, "done\n");
+ fflush(out);
+ }
+ query_processor->markCatalogAltered();
+ query_processor->saveCatalog();
+ }
+
} // namespace
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
++ const tmb::client_id main_thread_client_id,
++ const tmb::client_id foreman_client_id,
++ MessageBus *bus,
+ StorageManager *storage_manager,
+ QueryProcessor *query_processor,
- Foreman *foreman,
FILE *out) {
const ParseCommand &command = static_cast<const ParseCommand &>(statement);
const PtrVector<ParseString> *arguments = command.arguments();
@@@ -178,6 -327,8 +344,9 @@@
} else {
executeDescribeTable(arguments, catalog_database, out);
}
+ } else if (command_str == C::kAnalyzeCommand) {
- executeAnalyze(query_processor, foreman, out);
++ executeAnalyze(
++ main_thread_client_id, foreman_client_id, bus, query_processor, out);
} else {
THROW_SQL_ERROR_AT(command.command()) << "Invalid Command";
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/cli/CommandExecutor.hpp
----------------------------------------------------------------------
diff --cc cli/CommandExecutor.hpp
index 21eee6a,c819981..3435aeb
--- a/cli/CommandExecutor.hpp
+++ b/cli/CommandExecutor.hpp
@@@ -21,13 -21,10 +21,14 @@@
#include <cstdio>
#include <string>
- #include "parser/ParseStatement.hpp"
- #include "utility/Macros.hpp"
++#include "tmb/id_typedefs.h"
+
using std::fprintf;
using std::fputc;
using std::string;
++namespace tmb { class MessageBus; }
++
namespace quickstep {
class CatalogDatabase;
@@@ -49,13 -50,19 +54,24 @@@ constexpr char kAnalyzeCommand[] = "\\a
/**
* @brief Executes the command by calling the command handler.
- *
+ *
* @param statement The parsed statement from the cli.
* @param catalog_database The catalog information about the current database.
- * @param out The stream where the output of the command has to be redirected to.
++ * @param main_thread_client_id The TMB client ID of the main thread.
++ * @param foreman_client_id The TMB client ID of the Foreman thread.
++ * @param bus A pointer to the TMB.
+ * @param storage_manager The current StorageManager.
+ * @param query_processor The query processor to generate plans for SQL queries.
+ * @param foreman The foreman to execute query plans.
+ * @param out The stream where the output of the command has to be redirected to.
*/
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
++ const tmb::client_id main_thread_client_id,
++ const tmb::client_id foreman_client_id,
++ tmb::MessageBus *bus,
+ StorageManager *storage_manager,
+ QueryProcessor *query_processor,
- Foreman *foreman,
FILE *out);
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --cc cli/QuickstepCli.cpp
index 66e58fe,558d6eb..6f954fe
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@@ -320,12 -323,7 +326,13 @@@ int main(int argc, char* argv[])
worker_client_ids,
worker_numa_nodes);
- foreman.setWorkerDirectory(&worker_directory);
+ Foreman foreman(main_thread_client_id,
+ &worker_directory,
+ &bus,
+ query_processor->getDefaultDatabase(),
+ query_processor->getStorageManager(),
- num_numa_nodes_covered);
++ -1,
++ num_numa_nodes_system);
// Start the worker threads.
for (Worker &worker : workers) {
@@@ -365,7 -365,11 +376,13 @@@
try {
quickstep::cli::executeCommand(
*result.parsed_statement,
- *(query_processor->getDefaultDatabase()), stdout);
+ *(query_processor->getDefaultDatabase()),
++ main_thread_client_id,
++ foreman.getBusClientID(),
++ &bus,
+ query_processor->getStorageManager(),
+ query_processor.get(),
- &foreman,
+ stdout);
} catch (const quickstep::SqlError &sql_error) {
fprintf(stderr, "%s",
sql_error.formatMessage(*command_string).c_str());
@@@ -385,20 -389,14 +402,25 @@@
}
DCHECK(query_handle->getQueryPlanMutable() != nullptr);
- AdmitRequestMessage request_message(query_handle.get());
- TaggedMessage admit_tagged_message(
- &request_message, sizeof(request_message), kAdmitRequestMessage);
- foreman.setQueryPlan(query_handle->getQueryPlanMutable()->getQueryPlanDAGMutable());
-
- foreman.reconstructQueryContextFromProto(query_handle->getQueryContextProto());
--
++ // TODO(harshad) - In the future when queries are not admitted
++ // immediately, calculate their waiting time separately.
++ LOG(INFO) << "Address of query handle in QuickstepCli: " << query_handle.get();
+ start = std::chrono::steady_clock::now();
- QueryExecutionUtil::SendTMBMessage(&bus,
- main_thread_client_id,
- foreman.getBusClientID(),
- std::move(admit_tagged_message));
++ const tmb::MessageBus::SendStatus send_status =
++ QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
++ main_thread_client_id,
++ foreman.getBusClientID(),
++ query_handle.get(),
++ &bus);
++ if (send_status != tmb::MessageBus::SendStatus::kOK) {
++ fprintf(stderr, "\nQuery %s could not be admitted to the system\n", command_string->c_str());
++ continue;
++ }
try {
- start = std::chrono::steady_clock::now();
- foreman.start();
- foreman.join();
+ const AnnotatedMessage annotated_msg =
+ bus.Receive(main_thread_client_id, 0, true);
+ const TaggedMessage &tagged_message = annotated_msg.tagged_message;
+ DCHECK_EQ(kWorkloadCompletionMessage, tagged_message.message_type());
end = std::chrono::steady_clock::now();
const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/cli/tests/CommandExecutorTestRunner.cpp
----------------------------------------------------------------------
diff --cc cli/tests/CommandExecutorTestRunner.cpp
index f5c37c3,9cd493e..794f7e1
--- a/cli/tests/CommandExecutorTestRunner.cpp
+++ b/cli/tests/CommandExecutorTestRunner.cpp
@@@ -92,6 -87,9 +92,11 @@@ void CommandExecutorTestRunner::runTest
quickstep::cli::executeCommand(
*result.parsed_statement,
*(test_database_loader_.catalog_database()),
++ main_thread_client_id_,
++ foreman_->getBusClientID(),
++ &bus_,
+ test_database_loader_.storage_manager(),
+ nullptr,
- nullptr,
output_stream.file());
} else {
QueryHandle query_handle(optimizer_context.query_id());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_execution/AdmitRequestMessage.hpp
----------------------------------------------------------------------
diff --cc query_execution/AdmitRequestMessage.hpp
index e33b354,0000000..e2a1077
mode 100644,000000..100644
--- a/query_execution/AdmitRequestMessage.hpp
+++ b/query_execution/AdmitRequestMessage.hpp
@@@ -1,73 -1,0 +1,75 @@@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_ADMIT_REQUEST_MESSAGE_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_ADMIT_REQUEST_MESSAGE_HPP_
+
+#include <vector>
+
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+class QueryHandle;
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief A message requesting a query or queries to be admitted to the system.
+ **/
+class AdmitRequestMessage {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param query_handles The handles of the queries requesting to be admitted
+ * to the system.
+ **/
+ explicit AdmitRequestMessage(const std::vector<QueryHandle*> &query_handles)
+ : query_handles_(query_handles) {}
+
+ /**
+ * @brief Constructor for requesting single query admission.
+ *
+ * @param query_handle The handle of the query requesting to be admitted.
+ **/
+ explicit AdmitRequestMessage(QueryHandle *query_handle) {
+ query_handles_.push_back(query_handle);
+ }
+
+ /**
+ * @brief Get the query handles from this message.
+ **/
+ const std::vector<QueryHandle*>& getQueryHandles() const {
++ LOG(INFO) << "Query handle in getQueryHandles(): " << query_handles_.front()
++ << " [0] " << query_handles_[0];
+ return query_handles_;
+ }
+
+ private:
+ std::vector<QueryHandle*> query_handles_;
+
+ DISALLOW_COPY_AND_ASSIGN(AdmitRequestMessage);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_ADMIT_REQUEST_MESSAGE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --cc query_execution/CMakeLists.txt
index 9658c08,7d9d601..c8c8161
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@@ -20,10 -20,11 +20,13 @@@ QS_PROTOBUF_GENERATE_CPP(queryexecution
QueryExecutionMessages.proto)
# Declare micro-libs:
+add_library(quickstep_queryexecution_AdmitRequestMessage ../empty_src.cpp AdmitRequestMessage.hpp)
+ if (ENABLE_DISTRIBUTED)
+ add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp BlockLocator.hpp)
+ endif()
add_library(quickstep_queryexecution_Foreman Foreman.cpp Foreman.hpp)
add_library(quickstep_queryexecution_ForemanLite ../empty_src.cpp ForemanLite.hpp)
+add_library(quickstep_queryexecution_PolicyEnforcer PolicyEnforcer.cpp PolicyEnforcer.hpp)
add_library(quickstep_queryexecution_QueryContext QueryContext.cpp QueryContext.hpp)
add_library(quickstep_queryexecution_QueryContext_proto
${queryexecution_QueryContext_proto_srcs}
@@@ -42,20 -43,43 +45,33 @@@ add_library(quickstep_queryexecution_Wo
add_library(quickstep_queryexecution_WorkerSelectionPolicy ../empty_src.cpp WorkerSelectionPolicy.hpp)
# Link dependencies:
+target_link_libraries(quickstep_queryexecution_AdmitRequestMessage
+ quickstep_utility_Macros)
+ if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryexecution_BlockLocator
+ glog
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_threading_Thread
+ quickstep_threading_ThreadUtil
+ quickstep_utility_Macros
+ tmb)
+ endif()
target_link_libraries(quickstep_queryexecution_Foreman
+ gflags_nothreads-static
glog
- gtest
- quickstep_catalog_CatalogDatabase
- quickstep_catalog_CatalogRelation
- quickstep_catalog_CatalogTypedefs
- quickstep_catalog_PartitionScheme
+ quickstep_queryexecution_AdmitRequestMessage
quickstep_queryexecution_ForemanLite
- quickstep_queryexecution_QueryContext
- quickstep_queryexecution_QueryExecutionMessages_proto
- quickstep_queryexecution_QueryExecutionState
+ quickstep_queryexecution_PolicyEnforcer
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
- quickstep_queryexecution_WorkOrdersContainer
quickstep_queryexecution_WorkerDirectory
quickstep_queryexecution_WorkerMessage
- quickstep_relationaloperators_RebuildWorkOrder
- quickstep_relationaloperators_RelationalOperator
- quickstep_relationaloperators_WorkOrder
- quickstep_storage_InsertDestination
- quickstep_storage_StorageBlock
- quickstep_storage_StorageBlockInfo
quickstep_threading_ThreadUtil
- quickstep_utility_DAG
+ quickstep_utility_EqualsAnyConstant
quickstep_utility_Macros
tmb)
target_link_libraries(quickstep_queryexecution_ForemanLite
@@@ -112,8 -128,7 +130,9 @@@ target_link_libraries(quickstep_queryex
quickstep_threading_ThreadIDBasedMap
tmb)
target_link_libraries(quickstep_queryexecution_QueryExecutionUtil
++ quickstep_queryexecution_AdmitRequestMessage
quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_WorkerMessage
quickstep_utility_Macros
tmb)
target_link_libraries(quickstep_queryexecution_QueryManager
@@@ -179,7 -192,67 +198,37 @@@ target_link_libraries(quickstep_queryex
quickstep_queryexecution_WorkerDirectory
quickstep_queryexecution_WorkerMessage
quickstep_queryexecution_WorkerSelectionPolicy)
+ if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryexecution
+ quickstep_queryexecution_BlockLocator)
+ endif()
+
# Tests:
+ if (ENABLE_DISTRIBUTED)
+ add_executable(BlockLocator_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/BlockLocator_unittest.cpp")
+ target_link_libraries(BlockLocator_unittest
+ gflags_nothreads-static
+ glog
+ gtest
+ quickstep_catalog_CatalogAttribute
+ quickstep_catalog_CatalogRelation
+ quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_StorageBlob
+ quickstep_storage_StorageBlock
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_storage_StorageManager
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ tmb
+ ${LIBS})
+ add_test(BlockLocator_unittest BlockLocator_unittest)
+ endif()
+
-add_executable(Foreman_unittest
- "${CMAKE_CURRENT_SOURCE_DIR}/tests/Foreman_unittest.cpp")
-target_link_libraries(Foreman_unittest
- glog
- gtest
- gtest_main
- quickstep_catalog_CatalogDatabase
- quickstep_catalog_CatalogRelation
- quickstep_catalog_CatalogTypedefs
- quickstep_queryexecution_Foreman
- quickstep_queryexecution_QueryContext
- quickstep_queryexecution_QueryContext_proto
- quickstep_queryexecution_QueryExecutionState
- quickstep_queryexecution_QueryExecutionTypedefs
- quickstep_queryexecution_WorkOrdersContainer
- quickstep_queryexecution_WorkerDirectory
- quickstep_queryexecution_WorkerMessage
- quickstep_queryoptimizer_QueryPlan
- quickstep_relationaloperators_RelationalOperator
- quickstep_relationaloperators_WorkOrder
- quickstep_storage_InsertDestination
- quickstep_storage_InsertDestination_proto
- quickstep_storage_StorageBlock
- quickstep_storage_StorageBlockInfo
- quickstep_storage_StorageManager
- quickstep_utility_DAG
- quickstep_utility_Macros
- tmb)
-add_test(Foreman_unittest Foreman_unittest)
-
add_executable(QueryManager_unittest
"${CMAKE_CURRENT_SOURCE_DIR}/tests/QueryManager_unittest.cpp")
target_link_libraries(QueryManager_unittest
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_execution/Foreman.cpp
----------------------------------------------------------------------
diff --cc query_execution/Foreman.cpp
index 3609120,304c429..6cec70a
--- a/query_execution/Foreman.cpp
+++ b/query_execution/Foreman.cpp
@@@ -97,40 -58,161 +97,43 @@@ void Foreman::run()
// We can pin the foreman thread to a CPU if specified.
ThreadUtil::BindToCPU(cpu_id_);
}
- initializeState();
-
- DEBUG_ASSERT(query_dag_ != nullptr);
- const dag_node_index dag_size = query_dag_->size();
-
- // Collect all the workorders from all the relational operators in the DAG.
- for (dag_node_index index = 0; index < dag_size; ++index) {
- if (checkAllBlockingDependenciesMet(index)) {
- query_dag_->getNodePayloadMutable(index)->informAllBlockingDependenciesMet();
- processOperator(index, false);
- }
- }
-
- // Dispatch the WorkOrders generated so far.
- dispatchWorkerMessages(0, 0);
-}
-
-void Foreman::processWorkOrderCompleteMessage(const dag_node_index op_index,
- const size_t worker_thread_index) {
- query_exec_state_->decrementNumQueuedWorkOrders(op_index);
-
- // As the given worker finished executing a WorkOrder, decrement its number
- // of queued WorkOrders.
- workers_->decrementNumQueuedWorkOrders(worker_thread_index);
-
- // Check if new work orders are available and fetch them if so.
- fetchNormalWorkOrders(op_index);
-
- if (checkRebuildRequired(op_index)) {
- if (checkNormalExecutionOver(op_index)) {
- if (!checkRebuildInitiated(op_index)) {
- if (initiateRebuild(op_index)) {
- // Rebuild initiated and completed right away.
- markOperatorFinished(op_index);
- } else {
- // Rebuild under progress.
- }
- } else if (checkRebuildOver(op_index)) {
- // Rebuild was under progress and now it is over.
- markOperatorFinished(op_index);
- }
- } else {
- // Normal execution under progress for this operator.
- }
- } else if (checkOperatorExecutionOver(op_index)) {
- // Rebuild not required for this operator and its normal execution is
- // complete.
- markOperatorFinished(op_index);
- }
-
- for (const pair<dag_node_index, bool> &dependent_link :
- query_dag_->getDependents(op_index)) {
- const dag_node_index dependent_op_index = dependent_link.first;
- if (checkAllBlockingDependenciesMet(dependent_op_index)) {
- // Process the dependent operator (of the operator whose WorkOrder
- // was just executed) for which all the dependencies have been met.
- processOperator(dependent_op_index, true);
- }
- }
-
- // Dispatch the WorkerMessages to the workers. We prefer to start the search
- // for the schedulable WorkOrders beginning from 'op_index'. The first
- // candidate worker to receive the next WorkOrder is the one that sent the
- // response message to Foreman.
- dispatchWorkerMessages(worker_thread_index, op_index);
-}
-
-void Foreman::processRebuildWorkOrderCompleteMessage(const dag_node_index op_index,
- const size_t worker_thread_index) {
- query_exec_state_->decrementNumRebuildWorkOrders(op_index);
- workers_->decrementNumQueuedWorkOrders(worker_thread_index);
-
- if (checkRebuildOver(op_index)) {
- markOperatorFinished(op_index);
-
- for (const pair<dag_node_index, bool> &dependent_link :
- query_dag_->getDependents(op_index)) {
- const dag_node_index dependent_op_index = dependent_link.first;
- if (checkAllBlockingDependenciesMet(dependent_op_index)) {
- processOperator(dependent_op_index, true);
- }
- }
- }
-
- // Dispatch the WorkerMessages to the workers. We prefer to start the search
- // for the schedulable WorkOrders beginning from 'op_index'. The first
- // candidate worker to receive the next WorkOrder is the one that sent the
- // response message to Foreman.
- dispatchWorkerMessages(worker_thread_index, op_index);
-}
-
-void Foreman::processDataPipelineMessage(const dag_node_index op_index,
- const block_id block,
- const relation_id rel_id) {
- for (const dag_node_index consumer_index :
- output_consumers_[op_index]) {
- // Feed the streamed block to the consumer. Note that 'output_consumers_'
- // only contain those dependents of operator with index = op_index which are
- // eligible to receive streamed input.
- query_dag_->getNodePayloadMutable(consumer_index)->feedInputBlock(block, rel_id);
- // Because of the streamed input just fed, check if there are any new
- // WorkOrders available and if so, fetch them.
- fetchNormalWorkOrders(consumer_index);
- }
-
- // Dispatch the WorkerMessages to the workers. We prefer to start the search
- // for the schedulable WorkOrders beginning from 'op_index'. The first
- // candidate worker to receive the next WorkOrder is the one that sent the
- // response message to Foreman.
- // TODO(zuyu): Improve the data locality for the next WorkOrder.
- dispatchWorkerMessages(0, op_index);
-}
-
-void Foreman::processFeedbackMessage(const WorkOrder::FeedbackMessage &msg) {
- RelationalOperator *op =
- query_dag_->getNodePayloadMutable(msg.header().rel_op_index);
- op->receiveFeedbackMessage(msg);
-}
-
-void Foreman::run() {
- // Initialize before for Foreman eventloop.
- initialize();
// Event loop
- while (!query_exec_state_->hasQueryExecutionFinished()) {
+ for (;;) {
// Receive() causes this thread to sleep until next message is received.
- AnnotatedMessage annotated_msg = bus_->Receive(foreman_client_id_, 0, true);
+ const AnnotatedMessage annotated_msg =
+ bus_->Receive(foreman_client_id_, 0, true);
const TaggedMessage &tagged_message = annotated_msg.tagged_message;
- switch (tagged_message.message_type()) {
- case kWorkOrderCompleteMessage: {
- serialization::WorkOrderCompletionMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
-
- processWorkOrderCompleteMessage(proto.operator_index(), proto.worker_thread_index());
- break;
- }
- case kRebuildWorkOrderCompleteMessage: {
- serialization::WorkOrderCompletionMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
-
- processRebuildWorkOrderCompleteMessage(proto.operator_index(), proto.worker_thread_index());
+ const tmb::message_type_id message_type = tagged_message.message_type();
+ switch (message_type) {
+ case kCatalogRelationNewBlockMessage: // Fall through
+ case kDataPipelineMessage:
+ case kRebuildWorkOrderCompleteMessage:
+ case kWorkOrderCompleteMessage:
+ case kWorkOrderFeedbackMessage:
+ case kWorkOrdersAvailableMessage: {
+ policy_enforcer_->processMessage(tagged_message);
break;
}
- case kCatalogRelationNewBlockMessage: {
- serialization::CatalogRelationNewBlockMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
-
- const block_id block = proto.block_id();
-
- CatalogRelation *relation =
- static_cast<CatalogDatabase*>(catalog_database_)->getRelationByIdMutable(proto.relation_id());
- relation->addBlock(block);
-
- if (proto.has_partition_id()) {
- relation->getPartitionSchemeMutable()->addBlockToPartition(proto.partition_id(), block);
+ case kAdmitRequestMessage: {
+ const AdmitRequestMessage *msg =
+ static_cast<const AdmitRequestMessage *>(tagged_message.message());
+ const vector<QueryHandle *> &query_handles = msg->getQueryHandles();
+
++ LOG(INFO) << "Address of query handle in foreman front: " << query_handles.front() <<
++ " [0]: " << query_handles[0];
+ DCHECK(!query_handles.empty());
+ bool all_queries_admitted = true;
+ if (query_handles.size() == 1u) {
++ LOG(INFO) << "Address of query handle in foreman: " << query_handles.front();
+ all_queries_admitted =
+ policy_enforcer_->admitQuery(query_handles.front());
+ } else {
+ all_queries_admitted = policy_enforcer_->admitQueries(query_handles);
+ }
+ if (!all_queries_admitted) {
+ LOG(WARNING) << "The scheduler could not admit all the queries";
+ // TODO(harshad) - Inform the main thread about the failure.
}
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_execution/PolicyEnforcer.cpp
----------------------------------------------------------------------
diff --cc query_execution/PolicyEnforcer.cpp
index 1ee1df9,0000000..15b90a1
mode 100644,000000..100644
--- a/query_execution/PolicyEnforcer.cpp
+++ b/query_execution/PolicyEnforcer.cpp
@@@ -1,177 -1,0 +1,180 @@@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_execution/PolicyEnforcer.hpp"
+
+#include <cstddef>
+#include <memory>
+#include <queue>
+#include <utility>
+#include <unordered_map>
+#include <vector>
+
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryManager.hpp"
+#include "query_optimizer/QueryHandle.hpp"
+#include "relational_operators/WorkOrder.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+bool PolicyEnforcer::admitQuery(QueryHandle *query_handle) {
+ if (admitted_queries_.size() < kMaxConcurrentQueries) {
+ // Ok to admit the query.
+ const std::size_t query_id = query_handle->query_id();
+ if (admitted_queries_.find(query_id) == admitted_queries_.end()) {
- admitted_queries_[query_id].reset(
- new QueryManager(foreman_client_id_, num_numa_nodes_, query_handle,
- catalog_database_, storage_manager_, bus_));
++ admitted_queries_[query_id].reset(new QueryManager(foreman_client_id_,
++ num_numa_nodes_,
++ query_handle,
++ catalog_database_,
++ storage_manager_,
++ bus_));
+ return true;
+ } else {
+ LOG(ERROR) << "Query with the same ID " << query_id << " exists";
+ return false;
+ }
+ } else {
+ // This query will have to wait.
+ waiting_queries_.push(query_handle);
+ return false;
+ }
+}
+
+void PolicyEnforcer::processMessage(const TaggedMessage &tagged_message) {
+ // TODO(harshad) : Provide processXMessage() public functions in
+ // QueryManager, so that we need to extract message from the
+ // TaggedMessage only once.
+ std::size_t query_id;
+ switch (tagged_message.message_type()) {
+ case kWorkOrderCompleteMessage: // Fall through.
+ case kRebuildWorkOrderCompleteMessage: {
+ serialization::WorkOrderCompletionMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+ query_id = proto.query_id();
+ break;
+ }
+ case kCatalogRelationNewBlockMessage: {
+ serialization::CatalogRelationNewBlockMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+ query_id = proto.query_id();
+ break;
+ }
+ case kDataPipelineMessage: {
+ serialization::DataPipelineMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+ query_id = proto.query_id();
+ break;
+ }
+ case kWorkOrdersAvailableMessage: {
+ serialization::WorkOrdersAvailableMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+ query_id = proto.query_id();
+ break;
+ }
+ case kWorkOrderFeedbackMessage: {
+ // TODO(harshad) Add query ID to FeedbackMessage.
+ WorkOrder::FeedbackMessage msg(const_cast<void *>(tagged_message.message()), tagged_message.message_bytes());
+ query_id = msg.header().query_id;
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown message type found in PolicyEnforcer";
+ }
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+ const QueryManager::QueryStatusCode return_code =
+ admitted_queries_[query_id]->processMessage(tagged_message);
+ if (return_code == QueryManager::QueryStatusCode::kQueryExecuted) {
+ removeQuery(query_id);
+ if (!waiting_queries_.empty()) {
+ // Admit the earliest waiting query.
+ QueryHandle *new_query = waiting_queries_.front();
+ waiting_queries_.pop();
+ admitQuery(new_query);
+ }
+ }
+}
+
+void PolicyEnforcer::getWorkerMessages(
+ std::vector<std::unique_ptr<WorkerMessage>> *worker_messages) {
+ // Iterate over admitted queries until either there are no more
+ // messages available, or the maximum number of messages have
+ // been collected.
+ DCHECK(worker_messages->empty());
+ // TODO(harshad) - Make this function generic enough so that it
+ // works well when multiple queries are getting executed.
+ std::size_t per_query_share = 0;
+ if (!admitted_queries_.empty()) {
+ per_query_share = kMaxNumWorkerMessages / admitted_queries_.size();
+ } else {
+ LOG(WARNING) << "Requesting WorkerMessages when no query is running";
+ return;
+ }
+ DCHECK_GT(per_query_share, 0u);
+ std::vector<std::size_t> finished_queries_ids;
+
+ for (const auto &admitted_query_info : admitted_queries_) {
+ QueryManager *curr_query_manager = admitted_query_info.second.get();
+ DCHECK(curr_query_manager != nullptr);
+ std::size_t messages_collected_curr_query = 0;
+ while (messages_collected_curr_query < per_query_share) {
+ WorkerMessage *next_worker_message =
+ curr_query_manager->getNextWorkerMessage(0, -1);
+ if (next_worker_message != nullptr) {
+ ++messages_collected_curr_query;
+ worker_messages->push_back(std::unique_ptr<WorkerMessage>(next_worker_message));
+ } else {
+ // No more work ordes from the current query at this time.
+ // Check if the query's execution is over.
+ if (curr_query_manager->getQueryExecutionState().hasQueryExecutionFinished()) {
+ // If the query has been executed, remove it.
+ finished_queries_ids.push_back(admitted_query_info.first);
+ }
+ break;
+ }
+ }
+ }
+ for (std::size_t finished_qid : finished_queries_ids) {
+ removeQuery(finished_qid);
+ }
+}
+
+void PolicyEnforcer::removeQuery(const std::size_t query_id) {
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+ if (!admitted_queries_[query_id]->getQueryExecutionState().hasQueryExecutionFinished()) {
+ LOG(WARNING) << "Removing query with ID " << query_id
+ << " that hasn't finished its execution";
+ }
+ admitted_queries_.erase(query_id);
+}
+
+bool PolicyEnforcer::admitQueries(
+ const std::vector<QueryHandle *> &query_handles) {
+ for (QueryHandle *curr_query : query_handles) {
+ if (!admitQuery(curr_query)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_execution/QueryContext.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_execution/QueryContext.hpp
----------------------------------------------------------------------
diff --cc query_execution/QueryContext.hpp
index e170494,7d5628d..83627bc
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@@ -406,15 -460,9 +460,16 @@@ class QueryContext
return update_groups_[id];
}
+ /**
+ * @param Get the ID of this query.
+ **/
+ inline const std::size_t getQueryID() const {
+ return query_id_;
+ }
+
private:
std::vector<std::unique_ptr<AggregationOperationState>> aggregation_states_;
+ std::vector<std::unique_ptr<BloomFilter>> bloom_filters_;
std::vector<std::unique_ptr<const GeneratorFunctionHandle>> generator_functions_;
std::vector<std::unique_ptr<InsertDestination>> insert_destinations_;
std::vector<std::unique_ptr<JoinHashTable>> join_hash_tables_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_execution/QueryContext.proto
----------------------------------------------------------------------
diff --cc query_execution/QueryContext.proto
index 5632482,b37286c..98cd0b6
--- a/query_execution/QueryContext.proto
+++ b/query_execution/QueryContext.proto
@@@ -42,16 -43,15 +43,17 @@@ message QueryContext
}
repeated AggregationOperationState aggregation_states = 1;
- repeated HashTable join_hash_tables = 2;
- repeated InsertDestination insert_destinations = 3;
- repeated Predicate predicates = 4;
- repeated ScalarGroup scalar_groups = 5;
- repeated SortConfiguration sort_configs = 6;
- repeated Tuple tuples = 7;
- repeated GeneratorFunctionHandle generator_functions = 8;
+ repeated BloomFilter bloom_filters = 2;
+ repeated GeneratorFunctionHandle generator_functions = 3;
+ repeated HashTable join_hash_tables = 4;
+ repeated InsertDestination insert_destinations = 5;
+ repeated Predicate predicates = 6;
+ repeated ScalarGroup scalar_groups = 7;
+ repeated SortConfiguration sort_configs = 8;
+ repeated Tuple tuples = 9;
// NOTE(zuyu): For UpdateWorkOrder only.
- repeated UpdateGroup update_groups = 9;
+ repeated UpdateGroup update_groups = 10;
+
- required uint64 query_id = 10;
++ required uint64 query_id = 11;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --cc query_execution/QueryExecutionMessages.proto
index f8628fa,15803cf..9d9a9e5
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@@ -44,5 -45,34 +48,35 @@@ message DataPipelineMessage
message WorkOrdersAvailableMessage {
required uint64 operator_index = 1;
+ required uint64 query_id = 2;
}
+
+ // BlockLocator related messages.
+ message BlockDomainRegistrationMessage {
+ // Format IP:Port, i.e., "0.0.0.0:0".
+ required string domain_network_address = 1;
+ }
+
+ // Used for RegistrationResponse, Unregistration, and FailureReport.
+ message BlockDomainMessage {
+ required uint32 block_domain = 1;
+ }
+
+ // Used when StorageManager loads or evicts a block or a blob from its buffer
+ // pool.
+ message BlockLocationMessage {
+ required fixed64 block_id = 1;
+ required uint32 block_domain = 2;
+ }
+
+ message BlockMessage {
+ required fixed64 block_id = 1;
+ }
+
+ message LocateBlockResponseMessage {
+ repeated uint32 block_domains = 1;
+ }
+
+ message GetPeerDomainNetworkAddressesResponseMessage {
+ repeated string domain_network_addresses = 1;
+ }
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --cc query_execution/QueryExecutionTypedefs.hpp
index 6ba792e,fc253bc..fa97158
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@@ -55,7 -56,8 +56,9 @@@ using ClientIDMap = ThreadIDBasedMap<cl
'a',
'p'>;
+ // We sort the following message types in the order of a life cycle of a query.
enum QueryExecutionMessageType : message_type_id {
++ kAdmitRequestMessage, // Requesting a query (or queries) to be admitted.
kWorkOrderMessage, // From Foreman to Worker.
kWorkOrderCompleteMessage, // From Worker to Foreman.
kCatalogRelationNewBlockMessage, // From InsertDestination to Foreman.
@@@ -65,9 -67,21 +68,22 @@@
// their corresponding RelationalOperators.
kRebuildWorkOrderMessage, // From Foreman to Worker.
kRebuildWorkOrderCompleteMessage, // From Worker to Foreman.
- kPoisonMessage, // From the CLI shell to Foreman, then from Foreman to Workers.
++ kWorkloadCompletionMessage, // From Foreman to main thread.
+ kPoisonMessage, // From the main thread to Foreman and workers.
- kAdmitRequestMessage, // Requesting a query (or queries) to be admitted.
- kWorkloadCompletionMessage // From Foreman to main thread.
+
+ #ifdef QUICKSTEP_DISTRIBUTED
+ // BlockLocator related messages, sorted in a life cycle of StorageManager
+ // with a unique block domain.
+ kBlockDomainRegistrationMessage, // From Worker to BlockLocator.
+ kBlockDomainRegistrationResponseMessage, // From BlockLocator to Worker.
+ kAddBlockLocationMessage, // From StorageManager to BlockLocator.
+ kDeleteBlockLocationMessage, // From StorageManager to BlockLocator.
+ kLocateBlockMessage, // From StorageManager to BlockLocator.
+ kLocateBlockResponseMessage, // From BlockLocator to StorageManager.
+ kGetPeerDomainNetworkAddressesMessage, // From StorageManager to BlockLocator.
+ kGetPeerDomainNetworkAddressesResponseMessage, // From BlockLocator to StorageManager.
+ kBlockDomainUnregistrationMessage, // From StorageManager to BlockLocator.
+ #endif
};
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_execution/QueryExecutionUtil.hpp
----------------------------------------------------------------------
diff --cc query_execution/QueryExecutionUtil.hpp
index 0cd0a4e,a8b6a38..267bbe6
--- a/query_execution/QueryExecutionUtil.hpp
+++ b/query_execution/QueryExecutionUtil.hpp
@@@ -17,11 -17,9 +17,12 @@@
#ifndef QUICKSTEP_QUERY_EXECUTION_QUERY_EXECUTION_UTIL_HPP_
#define QUICKSTEP_QUERY_EXECUTION_QUERY_EXECUTION_UTIL_HPP_
+#include <memory>
#include <utility>
++#include "query_execution/AdmitRequestMessage.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/WorkerMessage.hpp"
#include "utility/Macros.hpp"
#include "tmb/address.h"
@@@ -31,6 -29,6 +32,8 @@@
namespace quickstep {
++class QueryHandle;
++
/**
* @brief A static class for reusable methods in query_execution module.
**/
@@@ -62,27 -60,6 +65,67 @@@ class QueryExecutionUtil
std::move(tagged_message));
}
- static void BroadcastPoisonMessage(const tmb::client_id sender_id, tmb::MessageBus *bus) {
++ /**
++ * @brief Construct and send an AdmitRequestMessage from a given sender to a
++ * given recipient.
++ *
++ * @param sender_id The TMB client ID of the sender.
++ * @param receiver_id The TMB client ID of the receiver.
++ * @param query_handle The QueryHandle used in the AdmitRequestMessage.
++ * @param bus A pointer to the TMB.
++ * @param tagged_message A moved from reference to the tagged message.
++ *
++ * @return A status code indicating the result of the message delivery.
++ * The caller should ensure that the status is SendStatus::kOK.
++ **/
++ static tmb::MessageBus::SendStatus ConstructAndSendAdmitRequestMessage(
++ const tmb::client_id sender_id,
++ const tmb::client_id receiver_id,
++ QueryHandle *query_handle,
++ MessageBus *bus) {
++ LOG(INFO) << "Address of query handle in QExecUtil: " << query_handle;
++ std::unique_ptr<AdmitRequestMessage> request_message(new AdmitRequestMessage(query_handle));
++ const std::vector<QueryHandle *> &query_handles = request_message->getQueryHandles();
++ LOG(INFO) << "Address of query handle in foreman front: " << query_handles.front() << " [0]: " << query_handles[0];
++ const std::size_t size_of_request_msg = sizeof(*request_message);
++ TaggedMessage admit_tagged_message(
++ request_message.release(), size_of_request_msg, kAdmitRequestMessage);
++
++ return QueryExecutionUtil::SendTMBMessage(
++ bus, sender_id, receiver_id, std::move(admit_tagged_message));
++ }
++
++ /**
++ * @brief Broadcast a poison message from a given sender.
++ *
++ * @note This message will be received by all the clients that have registered
++ * as recipients of the poison message type.
++ *
++ * @param sender_id The TMB client ID of the sender.
++ * @param bus A pointer to the TMB.
++ **/
++ static void BroadcastPoisonMessage(const tmb::client_id sender_id,
++ tmb::MessageBus *bus) {
+ // Terminate all threads.
+ // The sender thread broadcasts poison message to the workers and foreman.
+ // Each worker dies after receiving poison message. The order of workers'
+ // death is irrelavant.
+ MessageStyle style;
+ style.Broadcast(true);
+ Address address;
+ address.All(true);
+ std::unique_ptr<WorkerMessage> poison_message(WorkerMessage::PoisonMessage());
+ TaggedMessage poison_tagged_message(poison_message.get(),
+ sizeof(*poison_message),
+ kPoisonMessage);
+
+ const tmb::MessageBus::SendStatus send_status = bus->Send(
+ sender_id, address, style, std::move(poison_tagged_message));
+ CHECK(send_status == tmb::MessageBus::SendStatus::kOK) <<
+ "Broadcast poison message from sender with TMB client ID " << sender_id
+ << " failed";
+ }
+
private:
/**
* @brief Constructor. Made private to avoid instantiation.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_optimizer/ExecutionGenerator.hpp
----------------------------------------------------------------------
diff --cc query_optimizer/ExecutionGenerator.hpp
index 0056cbf,0630bca..9639cf1
--- a/query_optimizer/ExecutionGenerator.hpp
+++ b/query_optimizer/ExecutionGenerator.hpp
@@@ -100,8 -103,8 +103,9 @@@ class ExecutionGenerator
: optimizer_context_(DCHECK_NOTNULL(optimizer_context)),
query_handle_(DCHECK_NOTNULL(query_handle)),
execution_plan_(DCHECK_NOTNULL(query_handle->getQueryPlanMutable())),
- query_context_proto_(DCHECK_NOTNULL(query_handle->getQueryContextProtoMutable())) {
+ query_context_proto_(DCHECK_NOTNULL(query_handle->getQueryContextProtoMutable())),
+ execution_heuristics_(new ExecutionHeuristics()) {
+ query_context_proto_->set_query_id(query_handle->query_id());
#ifdef QUICKSTEP_DISTRIBUTED
catalog_database_cache_proto_ = DCHECK_NOTNULL(query_handle->getCatalogDatabaseCacheProtoMutable());
#endif
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --cc query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
index 930087a,56b53ba..59ba5c1
--- a/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
@@@ -94,20 -90,13 +94,23 @@@ void ExecutionGeneratorTestRunner::runT
physical_generator.generatePlan(
logical_generator.generatePlan(*result.parsed_statement));
execution_generator.generatePlan(physical_plan);
- foreman_->setQueryPlan(
- query_handle.getQueryPlanMutable()->getQueryPlanDAGMutable());
- AdmitRequestMessage request_message(&query_handle);
- TaggedMessage admit_tagged_message(
- &request_message, sizeof(request_message), kAdmitRequestMessage);
- QueryExecutionUtil::SendTMBMessage(&bus_,
- main_thread_client_id_,
- foreman_->getBusClientID(),
- std::move(admit_tagged_message));
- foreman_->reconstructQueryContextFromProto(query_handle.getQueryContextProto());
++ const tmb::MessageBus::SendStatus send_status =
++ QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
++ main_thread_client_id_,
++ foreman_->getBusClientID(),
++ &query_handle,
++ &bus_);
++ if (send_status != tmb::MessageBus::SendStatus::kOK) {
++ std::fprintf(stderr, "\nQuery %s could not be admitted to the system\n", input.c_str());
++ continue;
++ }
- foreman_->start();
- foreman_->join();
+ // Receive workload completion message from Foreman.
+ const AnnotatedMessage annotated_msg =
+ bus_.Receive(main_thread_client_id_, 0, true);
+ const TaggedMessage &tagged_message = annotated_msg.tagged_message;
+ DCHECK(tagged_message.message_type() == kWorkloadCompletionMessage);
const CatalogRelation *query_result_relation = query_handle.getQueryResultRelation();
if (query_result_relation) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bef0ae1d/relational_operators/WorkOrderFactory.cpp
----------------------------------------------------------------------
[14/50] [abbrv] incubator-quickstep git commit: Revert "Change
default aggregate_hashtable_type from LinearOpenAddressing to
SeparateChaining" (#208)
Posted by hb...@apache.org.
Revert "Change default aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining" (#208)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a3889a39
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a3889a39
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a3889a39
Branch: refs/heads/query-manager-used-in-foreman
Commit: a3889a392abde3b04633465404d6e889e6c6343d
Parents: 8444e2d
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Wed May 4 14:21:58 2016 -0500
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Wed May 4 14:21:58 2016 -0500
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a3889a39/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 3698701..c34f084 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -135,7 +135,7 @@ static const volatile bool join_hashtable_type_dummy
= gflags::RegisterFlagValidator(&FLAGS_join_hashtable_type,
&ValidateHashTableImplTypeString);
-DEFINE_string(aggregate_hashtable_type, "SeparateChaining",
+DEFINE_string(aggregate_hashtable_type, "LinearOpenAddressing",
"HashTable implementation to use for aggregates with GROUP BY "
"(valid options are SeparateChaining or LinearOpenAddressing)");
static const volatile bool aggregate_hashtable_type_dummy
[08/50] [abbrv] incubator-quickstep git commit: Refactored
HashJoinWorkOrder protos. (#203)
Posted by hb...@apache.org.
Refactored HashJoinWorkOrder protos. (#203)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5ea898b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5ea898b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5ea898b2
Branch: refs/heads/query-manager-used-in-foreman
Commit: 5ea898b228df3a2166ed82d242f0a50a78f1c290
Parents: abfc5f2
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Fri Apr 29 21:11:23 2016 -0700
Committer: Jianqiao Zhu <ji...@cs.wisc.edu>
Committed: Fri Apr 29 23:11:23 2016 -0500
----------------------------------------------------------------------
relational_operators/CMakeLists.txt | 3 +
relational_operators/HashJoinOperator.cpp | 5 +-
relational_operators/WorkOrder.proto | 98 +++----
relational_operators/WorkOrderFactory.cpp | 362 +++++++++----------------
4 files changed, 162 insertions(+), 306 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ea898b2/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index 759a233..a4600e6 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -181,6 +181,9 @@ target_link_libraries(quickstep_relationaloperators_HashJoinOperator
quickstep_storage_TupleReference
quickstep_storage_TupleStorageSubBlock
quickstep_storage_ValueAccessor
+ quickstep_types_Type
+ quickstep_types_TypedValue
+ quickstep_types_containers_ColumnVector
quickstep_types_containers_ColumnVectorsValueAccessor
quickstep_utility_Macros
tmb)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ea898b2/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index 82f6b2a..104a02d 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -42,6 +42,9 @@
#include "storage/TupleReference.hpp"
#include "storage/TupleStorageSubBlock.hpp"
#include "storage/ValueAccessor.hpp"
+#include "types/Type.hpp"
+#include "types/TypedValue.hpp"
+#include "types/containers/ColumnVector.hpp"
#include "types/containers/ColumnVectorsValueAccessor.hpp"
#include "gflags/gflags.h"
@@ -820,7 +823,7 @@ void HashOuterJoinWorkOrder::execute() {
// where x is an attribute of the build relation.
// In that case, this HashOuterJoinWorkOrder needs to be updated to
// correctly handle the selections.
- const Type& column_type = selection_[i]->getType().getNullableVersion();
+ const Type &column_type = selection_[i]->getType().getNullableVersion();
if (NativeColumnVector::UsableForType(column_type)) {
NativeColumnVector *result = new NativeColumnVector(
column_type, num_tuples_without_matches);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ea898b2/relational_operators/WorkOrder.proto
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrder.proto b/relational_operators/WorkOrder.proto
index 8ed2080..5d0619a 100644
--- a/relational_operators/WorkOrder.proto
+++ b/relational_operators/WorkOrder.proto
@@ -29,21 +29,18 @@ enum WorkOrderType {
DESTROY_HASH = 6;
DROP_TABLE = 7;
FINALIZE_AGGREGATION = 8;
- HASH_ANTI_JOIN = 9;
- HASH_INNER_JOIN = 10;
- HASH_OUTER_JOIN = 11;
- HASH_SEMI_JOIN = 12;
- INSERT = 13;
- NESTED_LOOP_JOIN = 14;
- SAMPLE = 15;
- SAVE_BLOCKS = 16;
- SELECT = 17;
- SORT_MERGE_RUN = 18;
- SORT_RUN_GENERATION = 19;
- TABLE_GENERATOR = 20;
- TEXT_SCAN = 21;
- TEXT_SPLIT = 22;
- UPDATE = 23;
+ HASH_JOIN = 9;
+ INSERT = 10;
+ NESTED_LOOP_JOIN = 11;
+ SAMPLE = 12;
+ SAVE_BLOCKS = 13;
+ SELECT = 14;
+ SORT_MERGE_RUN = 15;
+ SORT_RUN_GENERATION = 16;
+ TABLE_GENERATOR = 17;
+ TEXT_SCAN = 18;
+ TEXT_SPLIT = 19;
+ UPDATE = 20;
}
message WorkOrder {
@@ -107,63 +104,30 @@ message FinalizeAggregationWorkOrder {
}
}
-message HashInnerJoinWorkOrder {
- extend WorkOrder {
- // All required.
- optional int32 build_relation_id = 160;
- optional int32 probe_relation_id = 161;
- repeated int32 join_key_attributes = 162;
- optional bool any_join_key_attributes_nullable = 163;
- optional int32 insert_destination_index = 164;
- optional uint32 join_hash_table_index = 165;
- optional int32 residual_predicate_index = 166;
- optional int32 selection_index = 167;
- optional fixed64 block_id = 168;
- }
-}
-
-message HashAntiJoinWorkOrder {
- extend WorkOrder {
- // All required.
- optional int32 build_relation_id = 350;
- optional int32 probe_relation_id = 351;
- repeated int32 join_key_attributes = 352;
- optional bool any_join_key_attributes_nullable = 353;
- optional int32 insert_destination_index = 354;
- optional uint32 join_hash_table_index = 355;
- optional int32 residual_predicate_index = 356;
- optional int32 selection_index = 357;
- optional fixed64 block_id = 358;
+message HashJoinWorkOrder {
+ enum HashJoinWorkOrderType {
+ HASH_ANTI_JOIN = 0;
+ HASH_INNER_JOIN = 1;
+ HASH_OUTER_JOIN = 2;
+ HASH_SEMI_JOIN = 3;
}
-}
-message HashSemiJoinWorkOrder {
extend WorkOrder {
// All required.
- optional int32 build_relation_id = 360;
- optional int32 probe_relation_id = 361;
- repeated int32 join_key_attributes = 362;
- optional bool any_join_key_attributes_nullable = 363;
- optional int32 insert_destination_index = 364;
- optional uint32 join_hash_table_index = 365;
- optional int32 residual_predicate_index = 366;
- optional int32 selection_index = 367;
- optional fixed64 block_id = 368;
- }
-}
+ optional HashJoinWorkOrderType hash_join_work_order_type = 160;
+ optional int32 build_relation_id = 161;
+ optional int32 probe_relation_id = 162;
+ repeated int32 join_key_attributes = 163;
+ optional bool any_join_key_attributes_nullable = 164;
+ optional int32 insert_destination_index = 165;
+ optional uint32 join_hash_table_index = 166;
+ optional int32 selection_index = 167;
+ optional fixed64 block_id = 168;
-message HashOuterJoinWorkOrder {
- extend WorkOrder {
- // All required.
- optional int32 build_relation_id = 370;
- optional int32 probe_relation_id = 371;
- repeated int32 join_key_attributes = 372;
- optional bool any_join_key_attributes_nullable = 373;
- optional int32 insert_destination_index = 374;
- optional uint32 join_hash_table_index = 375;
- optional int32 selection_index = 376;
- repeated bool is_selection_on_build = 377;
- optional fixed64 block_id = 378;
+ // Used by all but HashOuterJoinWorkOrder.
+ optional int32 residual_predicate_index = 169;
+ // Used by HashOuterJoinWorkOrder only.
+ repeated bool is_selection_on_build = 170;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ea898b2/relational_operators/WorkOrderFactory.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrderFactory.cpp b/relational_operators/WorkOrderFactory.cpp
index 964c11c..4157d0f 100644
--- a/relational_operators/WorkOrderFactory.cpp
+++ b/relational_operators/WorkOrderFactory.cpp
@@ -16,6 +16,7 @@
#include "relational_operators/WorkOrderFactory.hpp"
+#include <memory>
#include <utility>
#include <vector>
@@ -55,6 +56,10 @@ using std::vector;
namespace quickstep {
+class InsertDestination;
+class Predicate;
+class Scalar;
+
WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder &proto,
CatalogDatabaseLite *catalog_database,
QueryContext *query_context,
@@ -135,123 +140,115 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
query_context->getInsertDestination(
proto.GetExtension(serialization::FinalizeAggregationWorkOrder::insert_destination_index)));
}
- case serialization::HASH_ANTI_JOIN: {
- LOG(INFO) << "Creating HashAntiJoinWorkOrder";
- vector<attribute_id> join_key_attributes;
- const int join_key_attributes_size =
- proto.ExtensionSize(serialization::HashAntiJoinWorkOrder::join_key_attributes);
- for (int i = 0; i < join_key_attributes_size; ++i) {
- join_key_attributes.push_back(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::join_key_attributes, i));
- }
+ case serialization::HASH_JOIN: {
+ const auto hash_join_work_order_type =
+ proto.GetExtension(serialization::HashJoinWorkOrder::hash_join_work_order_type);
- return new HashAntiJoinWorkOrder(
+ const CatalogRelationSchema &build_relation =
catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::build_relation_id)),
+ proto.GetExtension(serialization::HashJoinWorkOrder::build_relation_id));
+ const CatalogRelationSchema &probe_relation =
catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::probe_relation_id)),
- move(join_key_attributes),
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::any_join_key_attributes_nullable),
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::block_id),
- query_context->getPredicate(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::residual_predicate_index)),
- query_context->getScalarGroup(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::selection_index)),
- *query_context->getJoinHashTable(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::join_hash_table_index)),
- query_context->getInsertDestination(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::insert_destination_index)),
- storage_manager);
- }
- case serialization::HASH_INNER_JOIN: {
- LOG(INFO) << "Creating HashInnerJoinWorkOrder";
- vector<attribute_id> join_key_attributes;
- const int join_key_attributes_size =
- proto.ExtensionSize(serialization::HashInnerJoinWorkOrder::join_key_attributes);
- for (int i = 0; i < join_key_attributes_size; ++i) {
- join_key_attributes.push_back(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::join_key_attributes, i));
- }
+ proto.GetExtension(serialization::HashJoinWorkOrder::probe_relation_id));
- return new HashInnerJoinWorkOrder(
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::build_relation_id)),
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::probe_relation_id)),
- move(join_key_attributes),
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::any_join_key_attributes_nullable),
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::block_id),
- query_context->getPredicate(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::residual_predicate_index)),
- query_context->getScalarGroup(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::selection_index)),
- *query_context->getJoinHashTable(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::join_hash_table_index)),
- query_context->getInsertDestination(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::insert_destination_index)),
- storage_manager);
- }
- case serialization::HASH_OUTER_JOIN: {
- LOG(INFO) << "Creating HashOuterJoinWorkOrder";
vector<attribute_id> join_key_attributes;
const int join_key_attributes_size =
- proto.ExtensionSize(serialization::HashOuterJoinWorkOrder::join_key_attributes);
+ proto.ExtensionSize(serialization::HashJoinWorkOrder::join_key_attributes);
for (int i = 0; i < join_key_attributes_size; ++i) {
join_key_attributes.push_back(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::join_key_attributes, i));
- }
- vector<bool> is_selection_on_build;
- const int is_selection_on_build_size =
- proto.ExtensionSize(serialization::HashOuterJoinWorkOrder::is_selection_on_build);
- for (int i = 0; i < is_selection_on_build_size; ++i) {
- is_selection_on_build.push_back(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::is_selection_on_build, i));
+ proto.GetExtension(serialization::HashJoinWorkOrder::join_key_attributes, i));
}
- return new HashOuterJoinWorkOrder(
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::build_relation_id)),
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::probe_relation_id)),
- move(join_key_attributes),
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::any_join_key_attributes_nullable),
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::block_id),
- query_context->getScalarGroup(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::selection_index)),
- move(is_selection_on_build),
- *query_context->getJoinHashTable(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::join_hash_table_index)),
- query_context->getInsertDestination(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::insert_destination_index)),
- storage_manager);
- }
- case serialization::HASH_SEMI_JOIN: {
- LOG(INFO) << "Creating HashSemiJoinWorkOrder";
- vector<attribute_id> join_key_attributes;
- const int join_key_attributes_size =
- proto.ExtensionSize(serialization::HashSemiJoinWorkOrder::join_key_attributes);
- for (int i = 0; i < join_key_attributes_size; ++i) {
- join_key_attributes.push_back(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::join_key_attributes, i));
+ const bool any_join_key_attributes_nullable =
+ proto.GetExtension(serialization::HashJoinWorkOrder::any_join_key_attributes_nullable);
+ const block_id lookup_block_id =
+ proto.GetExtension(serialization::HashJoinWorkOrder::block_id);
+
+ const Predicate *residual_predicate = nullptr;
+ if (hash_join_work_order_type != serialization::HashJoinWorkOrder::HASH_OUTER_JOIN) {
+ residual_predicate =
+ query_context->getPredicate(
+ proto.GetExtension(serialization::HashJoinWorkOrder::residual_predicate_index));
}
- return new HashSemiJoinWorkOrder(
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::build_relation_id)),
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::probe_relation_id)),
- move(join_key_attributes),
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::any_join_key_attributes_nullable),
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::block_id),
- query_context->getPredicate(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::residual_predicate_index)),
+ const std::vector<std::unique_ptr<const Scalar>> &selection =
query_context->getScalarGroup(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::selection_index)),
+ proto.GetExtension(serialization::HashJoinWorkOrder::selection_index));
+ const JoinHashTable &hash_table =
*query_context->getJoinHashTable(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::join_hash_table_index)),
+ proto.GetExtension(serialization::HashJoinWorkOrder::join_hash_table_index));
+ InsertDestination *output_destination =
query_context->getInsertDestination(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::insert_destination_index)),
- storage_manager);
+ proto.GetExtension(serialization::HashJoinWorkOrder::insert_destination_index));
+
+ switch (hash_join_work_order_type) {
+ case serialization::HashJoinWorkOrder::HASH_ANTI_JOIN: {
+ LOG(INFO) << "Creating HashAntiJoinWorkOrder";
+ return new HashAntiJoinWorkOrder(
+ build_relation,
+ probe_relation,
+ move(join_key_attributes),
+ any_join_key_attributes_nullable,
+ lookup_block_id,
+ residual_predicate,
+ selection,
+ hash_table,
+ output_destination,
+ storage_manager);
+ }
+ case serialization::HashJoinWorkOrder::HASH_INNER_JOIN: {
+ LOG(INFO) << "Creating HashInnerJoinWorkOrder";
+ return new HashInnerJoinWorkOrder(
+ build_relation,
+ probe_relation,
+ move(join_key_attributes),
+ any_join_key_attributes_nullable,
+ lookup_block_id,
+ residual_predicate,
+ selection,
+ hash_table,
+ output_destination,
+ storage_manager);
+ }
+ case serialization::HashJoinWorkOrder::HASH_OUTER_JOIN: {
+ vector<bool> is_selection_on_build;
+ const int is_selection_on_build_size =
+ proto.ExtensionSize(serialization::HashJoinWorkOrder::is_selection_on_build);
+ for (int i = 0; i < is_selection_on_build_size; ++i) {
+ is_selection_on_build.push_back(
+ proto.GetExtension(serialization::HashJoinWorkOrder::is_selection_on_build, i));
+ }
+
+ LOG(INFO) << "Creating HashOuterJoinWorkOrder";
+ return new HashOuterJoinWorkOrder(
+ build_relation,
+ probe_relation,
+ move(join_key_attributes),
+ any_join_key_attributes_nullable,
+ lookup_block_id,
+ selection,
+ move(is_selection_on_build),
+ hash_table,
+ output_destination,
+ storage_manager);
+ }
+ case serialization::HashJoinWorkOrder::HASH_SEMI_JOIN: {
+ LOG(INFO) << "Creating HashSemiJoinWorkOrder";
+ return new HashSemiJoinWorkOrder(
+ build_relation,
+ probe_relation,
+ move(join_key_attributes),
+ any_join_key_attributes_nullable,
+ lookup_block_id,
+ residual_predicate,
+ selection,
+ hash_table,
+ output_destination,
+ storage_manager);
+ }
+ default:
+ LOG(FATAL) << "Unknown HashJoinWorkOrder Type in WorkOrderFactory::ReconstructFromProto";
+ }
}
case serialization::INSERT: {
LOG(INFO) << "Creating InsertWorkOrder";
@@ -486,179 +483,68 @@ bool WorkOrderFactory::ProtoIsValid(const serialization::WorkOrder &proto,
query_context.isValidInsertDestinationId(
proto.GetExtension(serialization::FinalizeAggregationWorkOrder::insert_destination_index));
}
- case serialization::HASH_ANTI_JOIN: {
- if (!proto.HasExtension(serialization::HashAntiJoinWorkOrder::build_relation_id) ||
- !proto.HasExtension(serialization::HashAntiJoinWorkOrder::probe_relation_id)) {
+ case serialization::HASH_JOIN: {
+ if (!proto.HasExtension(serialization::HashJoinWorkOrder::hash_join_work_order_type)) {
return false;
}
- const relation_id build_relation_id =
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::build_relation_id);
- if (!catalog_database.hasRelationWithId(build_relation_id)) {
+ const auto hash_join_work_order_type =
+ proto.GetExtension(serialization::HashJoinWorkOrder::hash_join_work_order_type);
+ if (!serialization::HashJoinWorkOrder_HashJoinWorkOrderType_IsValid(hash_join_work_order_type)) {
return false;
}
- const relation_id probe_relation_id =
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::probe_relation_id);
- if (!catalog_database.hasRelationWithId(probe_relation_id)) {
- return false;
- }
-
- const CatalogRelationSchema &build_relation = catalog_database.getRelationSchemaById(build_relation_id);
- const CatalogRelationSchema &probe_relation = catalog_database.getRelationSchemaById(probe_relation_id);
- for (int i = 0; i < proto.ExtensionSize(serialization::HashAntiJoinWorkOrder::join_key_attributes); ++i) {
- const attribute_id attr_id =
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::join_key_attributes, i);
- if (!build_relation.hasAttributeWithId(attr_id) ||
- !probe_relation.hasAttributeWithId(attr_id)) {
- return false;
- }
- }
-
- return proto.HasExtension(serialization::HashAntiJoinWorkOrder::any_join_key_attributes_nullable) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::insert_destination_index) &&
- query_context.isValidInsertDestinationId(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::insert_destination_index)) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::join_hash_table_index) &&
- query_context.isValidJoinHashTableId(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::join_hash_table_index)) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::residual_predicate_index) &&
- query_context.isValidPredicate(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::residual_predicate_index)) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::selection_index) &&
- query_context.isValidScalarGroupId(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::selection_index)) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::block_id);
- }
- case serialization::HASH_INNER_JOIN: {
- if (!proto.HasExtension(serialization::HashInnerJoinWorkOrder::build_relation_id) ||
- !proto.HasExtension(serialization::HashInnerJoinWorkOrder::probe_relation_id)) {
+ if (!proto.HasExtension(serialization::HashJoinWorkOrder::build_relation_id) ||
+ !proto.HasExtension(serialization::HashJoinWorkOrder::probe_relation_id)) {
return false;
}
const relation_id build_relation_id =
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::build_relation_id);
+ proto.GetExtension(serialization::HashJoinWorkOrder::build_relation_id);
if (!catalog_database.hasRelationWithId(build_relation_id)) {
return false;
}
const relation_id probe_relation_id =
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::probe_relation_id);
+ proto.GetExtension(serialization::HashJoinWorkOrder::probe_relation_id);
if (!catalog_database.hasRelationWithId(probe_relation_id)) {
return false;
}
const CatalogRelationSchema &build_relation = catalog_database.getRelationSchemaById(build_relation_id);
const CatalogRelationSchema &probe_relation = catalog_database.getRelationSchemaById(probe_relation_id);
- for (int i = 0; i < proto.ExtensionSize(serialization::HashInnerJoinWorkOrder::join_key_attributes); ++i) {
+ for (int i = 0; i < proto.ExtensionSize(serialization::HashJoinWorkOrder::join_key_attributes); ++i) {
const attribute_id attr_id =
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::join_key_attributes, i);
+ proto.GetExtension(serialization::HashJoinWorkOrder::join_key_attributes, i);
if (!build_relation.hasAttributeWithId(attr_id) ||
!probe_relation.hasAttributeWithId(attr_id)) {
return false;
}
}
- return proto.HasExtension(serialization::HashInnerJoinWorkOrder::any_join_key_attributes_nullable) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::insert_destination_index) &&
- query_context.isValidInsertDestinationId(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::insert_destination_index)) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::join_hash_table_index) &&
- query_context.isValidJoinHashTableId(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::join_hash_table_index)) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::residual_predicate_index) &&
- query_context.isValidPredicate(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::residual_predicate_index)) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::selection_index) &&
- query_context.isValidScalarGroupId(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::selection_index)) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::block_id);
- }
- case serialization::HASH_OUTER_JOIN: {
- if (!proto.HasExtension(serialization::HashOuterJoinWorkOrder::build_relation_id) ||
- !proto.HasExtension(serialization::HashOuterJoinWorkOrder::probe_relation_id)) {
- return false;
- }
-
- const relation_id build_relation_id =
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::build_relation_id);
- if (!catalog_database.hasRelationWithId(build_relation_id)) {
- return false;
- }
-
- const relation_id probe_relation_id =
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::probe_relation_id);
- if (!catalog_database.hasRelationWithId(probe_relation_id)) {
- return false;
- }
-
- const CatalogRelationSchema &build_relation = catalog_database.getRelationSchemaById(build_relation_id);
- const CatalogRelationSchema &probe_relation = catalog_database.getRelationSchemaById(probe_relation_id);
- for (int i = 0; i < proto.ExtensionSize(serialization::HashOuterJoinWorkOrder::join_key_attributes); ++i) {
- const attribute_id attr_id =
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::join_key_attributes, i);
- if (!build_relation.hasAttributeWithId(attr_id) ||
- !probe_relation.hasAttributeWithId(attr_id)) {
+ if (hash_join_work_order_type == serialization::HashJoinWorkOrder::HASH_OUTER_JOIN) {
+ if (!proto.HasExtension(serialization::HashJoinWorkOrder::is_selection_on_build)) {
return false;
}
- }
-
- return proto.HasExtension(serialization::HashOuterJoinWorkOrder::any_join_key_attributes_nullable) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::insert_destination_index) &&
- query_context.isValidInsertDestinationId(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::insert_destination_index)) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::join_hash_table_index) &&
- query_context.isValidJoinHashTableId(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::join_hash_table_index)) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::selection_index) &&
- query_context.isValidScalarGroupId(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::selection_index)) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::is_selection_on_build) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::block_id);
- }
- case serialization::HASH_SEMI_JOIN: {
- if (!proto.HasExtension(serialization::HashSemiJoinWorkOrder::build_relation_id) ||
- !proto.HasExtension(serialization::HashSemiJoinWorkOrder::probe_relation_id)) {
- return false;
- }
-
- const relation_id build_relation_id =
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::build_relation_id);
- if (!catalog_database.hasRelationWithId(build_relation_id)) {
- return false;
- }
-
- const relation_id probe_relation_id =
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::probe_relation_id);
- if (!catalog_database.hasRelationWithId(probe_relation_id)) {
- return false;
- }
-
- const CatalogRelationSchema &build_relation = catalog_database.getRelationSchemaById(build_relation_id);
- const CatalogRelationSchema &probe_relation = catalog_database.getRelationSchemaById(probe_relation_id);
- for (int i = 0; i < proto.ExtensionSize(serialization::HashSemiJoinWorkOrder::join_key_attributes); ++i) {
- const attribute_id attr_id =
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::join_key_attributes, i);
- if (!build_relation.hasAttributeWithId(attr_id) ||
- !probe_relation.hasAttributeWithId(attr_id)) {
+ } else {
+ if (!proto.HasExtension(serialization::HashJoinWorkOrder::residual_predicate_index) ||
+ !query_context.isValidPredicate(
+ proto.GetExtension(serialization::HashJoinWorkOrder::residual_predicate_index))) {
return false;
}
}
- return proto.HasExtension(serialization::HashSemiJoinWorkOrder::any_join_key_attributes_nullable) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::insert_destination_index) &&
+ return proto.HasExtension(serialization::HashJoinWorkOrder::any_join_key_attributes_nullable) &&
+ proto.HasExtension(serialization::HashJoinWorkOrder::insert_destination_index) &&
query_context.isValidInsertDestinationId(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::insert_destination_index)) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::join_hash_table_index) &&
+ proto.GetExtension(serialization::HashJoinWorkOrder::insert_destination_index)) &&
+ proto.HasExtension(serialization::HashJoinWorkOrder::join_hash_table_index) &&
query_context.isValidJoinHashTableId(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::join_hash_table_index)) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::residual_predicate_index) &&
- query_context.isValidPredicate(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::residual_predicate_index)) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::selection_index) &&
+ proto.GetExtension(serialization::HashJoinWorkOrder::join_hash_table_index)) &&
+ proto.HasExtension(serialization::HashJoinWorkOrder::selection_index) &&
query_context.isValidScalarGroupId(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::selection_index)) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::block_id);
+ proto.GetExtension(serialization::HashJoinWorkOrder::selection_index)) &&
+ proto.HasExtension(serialization::HashJoinWorkOrder::block_id);
}
case serialization::INSERT: {
return proto.HasExtension(serialization::InsertWorkOrder::insert_destination_index) &&
[29/50] [abbrv] incubator-quickstep git commit: Added a quick fix to
the count(*) problem (#220)
Posted by hb...@apache.org.
Added a quick fix to the count(*) problem (#220)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ee54e1d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ee54e1d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ee54e1d8
Branch: refs/heads/query-manager-used-in-foreman
Commit: ee54e1d8bacc6a951ee34f07a9901a8c49d4aa34
Parents: 6b8cc26
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Tue May 10 17:12:31 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:27 2016 -0700
----------------------------------------------------------------------
query_optimizer/physical/Aggregate.cpp | 14 ++++++++++++++
query_optimizer/tests/execution_generator/Select.test | 9 +++------
2 files changed, 17 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee54e1d8/query_optimizer/physical/Aggregate.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/Aggregate.cpp b/query_optimizer/physical/Aggregate.cpp
index 9bf606c..c582bba 100644
--- a/query_optimizer/physical/Aggregate.cpp
+++ b/query_optimizer/physical/Aggregate.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -70,6 +72,18 @@ std::vector<E::AttributeReferencePtr> Aggregate::getReferencedAttributes()
referenced_attributes_in_predicate.begin(),
referenced_attributes_in_predicate.end());
}
+
+ // TODO(jianqiao): This is a quick fix to the COUNT(*) problem that we retain
+ // at least one column from the child plan to survive the PruneColumns physical
+ // optimization. The comprehensive approach should also try to optimize the
+ // child plan with regard to the knowledge that the aggregation actually don't
+ // care about the values of the child plan's output but needs only the number
+ // of rows of the output.
+ if (referenced_attributes.size() == 0) {
+ DCHECK_GT(input_->getOutputAttributes().size(), static_cast<std::size_t>(0));
+ referenced_attributes.emplace_back(input_->getOutputAttributes()[0]);
+ }
+
return referenced_attributes;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee54e1d8/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index 023ad2e..05f7108 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -464,8 +464,6 @@ WHERE a.int_col = b.int_col
+-----------+-----------+---------------+-----------------------+---------------+------------------------+-----------+
==
-# FIXME(qzeng): Inner nested-loops join has no output column, causing query
-# result to be INCORRECT.
SELECT COUNT(*)
FROM
(SELECT a.float_col
@@ -476,7 +474,7 @@ FROM
+--------------------+
|COUNT(*) |
+--------------------+
-| 0|
+| 575|
+--------------------+
==
@@ -941,15 +939,14 @@ ORDER BY x;
+-----------+-----------+
==
-# TODO(team): Fix Issue #9 to enable COUNT(*).
-SELECT COUNT(long_col)
+SELECT COUNT(*)
FROM test,
(SELECT AVG(long_col) a FROM test) subquery
WHERE double_col < 0
AND long_col > subquery.a;
--
+--------------------+
-|COUNT(long_col) |
+|COUNT(*) |
+--------------------+
| 5|
+--------------------+
[19/50] [abbrv] incubator-quickstep git commit: Added support for the
substring function. (#211)
Posted by hb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/preprocessed/SqlParser_gen.cpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlParser_gen.cpp b/parser/preprocessed/SqlParser_gen.cpp
index f2cb8ca..038db14 100644
--- a/parser/preprocessed/SqlParser_gen.cpp
+++ b/parser/preprocessed/SqlParser_gen.cpp
@@ -244,67 +244,69 @@ extern int quickstep_yydebug;
TOKEN_FALSE = 314,
TOKEN_FIRST = 315,
TOKEN_FLOAT = 316,
- TOKEN_FOREIGN = 317,
- TOKEN_FROM = 318,
- TOKEN_FULL = 319,
- TOKEN_GROUP = 320,
- TOKEN_HASH = 321,
- TOKEN_HAVING = 322,
- TOKEN_HOUR = 323,
- TOKEN_IN = 324,
- TOKEN_INDEX = 325,
- TOKEN_INNER = 326,
- TOKEN_INSERT = 327,
- TOKEN_INTEGER = 328,
- TOKEN_INTERVAL = 329,
- TOKEN_INTO = 330,
- TOKEN_JOIN = 331,
- TOKEN_KEY = 332,
- TOKEN_LAST = 333,
- TOKEN_LEFT = 334,
- TOKEN_LIMIT = 335,
- TOKEN_LONG = 336,
- TOKEN_MINUTE = 337,
- TOKEN_MONTH = 338,
- TOKEN_NULL = 339,
- TOKEN_NULLS = 340,
- TOKEN_OFF = 341,
- TOKEN_ON = 342,
- TOKEN_ORDER = 343,
- TOKEN_OUTER = 344,
- TOKEN_PARTITION = 345,
- TOKEN_PARTITIONS = 346,
- TOKEN_PERCENT = 347,
- TOKEN_PRIMARY = 348,
- TOKEN_QUIT = 349,
- TOKEN_RANGE = 350,
- TOKEN_REAL = 351,
- TOKEN_REFERENCES = 352,
- TOKEN_RIGHT = 353,
- TOKEN_ROW_DELIMITER = 354,
- TOKEN_SECOND = 355,
- TOKEN_SELECT = 356,
- TOKEN_SET = 357,
- TOKEN_SMA = 358,
- TOKEN_SMALLINT = 359,
- TOKEN_TABLE = 360,
- TOKEN_THEN = 361,
- TOKEN_TIME = 362,
- TOKEN_TIMESTAMP = 363,
- TOKEN_TRUE = 364,
- TOKEN_TUPLESAMPLE = 365,
- TOKEN_UNIQUE = 366,
- TOKEN_UPDATE = 367,
- TOKEN_USING = 368,
- TOKEN_VALUES = 369,
- TOKEN_VARCHAR = 370,
- TOKEN_WHEN = 371,
- TOKEN_WHERE = 372,
- TOKEN_WITH = 373,
- TOKEN_YEAR = 374,
- TOKEN_YEARMONTH = 375,
- TOKEN_EOF = 376,
- TOKEN_LEX_ERROR = 377
+ TOKEN_FOR = 317,
+ TOKEN_FOREIGN = 318,
+ TOKEN_FROM = 319,
+ TOKEN_FULL = 320,
+ TOKEN_GROUP = 321,
+ TOKEN_HASH = 322,
+ TOKEN_HAVING = 323,
+ TOKEN_HOUR = 324,
+ TOKEN_IN = 325,
+ TOKEN_INDEX = 326,
+ TOKEN_INNER = 327,
+ TOKEN_INSERT = 328,
+ TOKEN_INTEGER = 329,
+ TOKEN_INTERVAL = 330,
+ TOKEN_INTO = 331,
+ TOKEN_JOIN = 332,
+ TOKEN_KEY = 333,
+ TOKEN_LAST = 334,
+ TOKEN_LEFT = 335,
+ TOKEN_LIMIT = 336,
+ TOKEN_LONG = 337,
+ TOKEN_MINUTE = 338,
+ TOKEN_MONTH = 339,
+ TOKEN_NULL = 340,
+ TOKEN_NULLS = 341,
+ TOKEN_OFF = 342,
+ TOKEN_ON = 343,
+ TOKEN_ORDER = 344,
+ TOKEN_OUTER = 345,
+ TOKEN_PARTITION = 346,
+ TOKEN_PARTITIONS = 347,
+ TOKEN_PERCENT = 348,
+ TOKEN_PRIMARY = 349,
+ TOKEN_QUIT = 350,
+ TOKEN_RANGE = 351,
+ TOKEN_REAL = 352,
+ TOKEN_REFERENCES = 353,
+ TOKEN_RIGHT = 354,
+ TOKEN_ROW_DELIMITER = 355,
+ TOKEN_SECOND = 356,
+ TOKEN_SELECT = 357,
+ TOKEN_SET = 358,
+ TOKEN_SMA = 359,
+ TOKEN_SMALLINT = 360,
+ TOKEN_SUBSTRING = 361,
+ TOKEN_TABLE = 362,
+ TOKEN_THEN = 363,
+ TOKEN_TIME = 364,
+ TOKEN_TIMESTAMP = 365,
+ TOKEN_TRUE = 366,
+ TOKEN_TUPLESAMPLE = 367,
+ TOKEN_UNIQUE = 368,
+ TOKEN_UPDATE = 369,
+ TOKEN_USING = 370,
+ TOKEN_VALUES = 371,
+ TOKEN_VARCHAR = 372,
+ TOKEN_WHEN = 373,
+ TOKEN_WHERE = 374,
+ TOKEN_WITH = 375,
+ TOKEN_YEAR = 376,
+ TOKEN_YEARMONTH = 377,
+ TOKEN_EOF = 378,
+ TOKEN_LEX_ERROR = 379
};
#endif
@@ -405,7 +407,7 @@ union YYSTYPE
quickstep::PtrVector<quickstep::ParseSubqueryTableReference> *with_list_;
quickstep::ParseSubqueryTableReference *with_list_element_;
-#line 409 "SqlParser_gen.cpp" /* yacc.c:355 */
+#line 411 "SqlParser_gen.cpp" /* yacc.c:355 */
};
typedef union YYSTYPE YYSTYPE;
@@ -440,7 +442,7 @@ int quickstep_yyparse (yyscan_t yyscanner, quickstep::ParseStatement **parsedSta
#include "SqlLexer_gen.hpp"
void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const std::string &feature);
-#line 444 "SqlParser_gen.cpp" /* yacc.c:358 */
+#line 446 "SqlParser_gen.cpp" /* yacc.c:358 */
#ifdef short
# undef short
@@ -684,21 +686,21 @@ union yyalloc
/* YYFINAL -- State number of the termination state. */
#define YYFINAL 47
/* YYLAST -- Last index in YYTABLE. */
-#define YYLAST 1191
+#define YYLAST 1215
/* YYNTOKENS -- Number of terminals. */
-#define YYNTOKENS 134
+#define YYNTOKENS 136
/* YYNNTS -- Number of nonterminals. */
-#define YYNNTS 95
+#define YYNNTS 96
/* YYNRULES -- Number of rules. */
-#define YYNRULES 262
+#define YYNRULES 265
/* YYNSTATES -- Number of states. */
-#define YYNSTATES 478
+#define YYNSTATES 488
/* YYTRANSLATE[YYX] -- Symbol number corresponding to YYX as returned
by yylex, with out-of-bounds checking. */
#define YYUNDEFTOK 2
-#define YYMAXUTOK 377
+#define YYMAXUTOK 379
#define YYTRANSLATE(YYX) \
((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK)
@@ -708,11 +710,11 @@ union yyalloc
static const yytype_uint8 yytranslate[] =
{
0, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 129, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 131, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 133, 2, 2,
- 130, 131, 23, 21, 132, 22, 27, 24, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 128,
+ 2, 2, 2, 2, 2, 2, 2, 135, 2, 2,
+ 132, 133, 23, 21, 134, 22, 27, 24, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 130,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
@@ -744,40 +746,40 @@ static const yytype_uint8 yytranslate[] =
90, 91, 92, 93, 94, 95, 96, 97, 98, 99,
100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
- 120, 121, 122, 123, 124, 125, 126, 127
+ 120, 121, 122, 123, 124, 125, 126, 127, 128, 129
};
#if YYDEBUG
/* YYRLINE[YYN] -- Source line where rule number YYN was defined. */
static const yytype_uint16 yyrline[] =
{
- 0, 582, 582, 586, 590, 594, 598, 601, 608, 611,
- 614, 617, 620, 623, 626, 629, 632, 635, 641, 647,
- 654, 660, 667, 676, 681, 690, 695, 700, 704, 710,
- 715, 718, 721, 726, 729, 732, 735, 738, 741, 744,
- 747, 750, 753, 765, 768, 771, 789, 809, 812, 815,
- 820, 825, 831, 837, 846, 850, 856, 859, 864, 869,
- 874, 881, 888, 892, 898, 901, 906, 909, 914, 917,
- 922, 925, 944, 948, 954, 958, 964, 967, 970, 975,
- 978, 985, 990, 1001, 1006, 1010, 1014, 1020, 1023, 1029,
- 1037, 1040, 1043, 1049, 1054, 1057, 1062, 1066, 1070, 1074,
- 1080, 1085, 1090, 1094, 1100, 1106, 1109, 1114, 1119, 1123,
- 1129, 1135, 1141, 1144, 1148, 1154, 1157, 1162, 1166, 1172,
- 1175, 1178, 1183, 1188, 1193, 1196, 1199, 1204, 1207, 1210,
- 1213, 1216, 1219, 1222, 1225, 1230, 1233, 1238, 1242, 1246,
- 1249, 1253, 1256, 1261, 1264, 1269, 1272, 1277, 1281, 1287,
- 1290, 1295, 1298, 1303, 1306, 1311, 1314, 1333, 1337, 1343,
- 1350, 1353, 1356, 1361, 1364, 1367, 1373, 1376, 1381, 1386,
- 1395, 1400, 1409, 1414, 1417, 1422, 1425, 1430, 1436, 1442,
- 1445, 1448, 1451, 1454, 1457, 1463, 1472, 1475, 1480, 1483,
- 1488, 1491, 1496, 1499, 1502, 1505, 1508, 1511, 1514, 1519,
- 1523, 1527, 1530, 1535, 1540, 1543, 1548, 1552, 1558, 1563,
- 1567, 1573, 1578, 1581, 1586, 1590, 1596, 1599, 1602, 1605,
- 1617, 1621, 1640, 1653, 1668, 1671, 1674, 1677, 1680, 1683,
- 1688, 1692, 1698, 1701, 1706, 1710, 1717, 1720, 1723, 1726,
- 1729, 1732, 1735, 1738, 1741, 1744, 1749, 1760, 1763, 1768,
- 1771, 1774, 1780, 1784, 1790, 1793, 1801, 1804, 1807, 1810,
- 1816, 1821, 1826
+ 0, 585, 585, 589, 593, 597, 601, 604, 611, 614,
+ 617, 620, 623, 626, 629, 632, 635, 638, 644, 650,
+ 657, 663, 670, 679, 684, 693, 698, 703, 707, 713,
+ 718, 721, 724, 729, 732, 735, 738, 741, 744, 747,
+ 750, 753, 756, 768, 771, 774, 792, 812, 815, 818,
+ 823, 828, 834, 840, 849, 853, 859, 862, 867, 872,
+ 877, 884, 891, 895, 901, 904, 909, 912, 917, 920,
+ 925, 928, 947, 951, 957, 961, 967, 970, 973, 978,
+ 981, 988, 993, 1004, 1009, 1013, 1017, 1023, 1026, 1032,
+ 1040, 1043, 1046, 1052, 1057, 1060, 1065, 1069, 1073, 1077,
+ 1083, 1088, 1093, 1097, 1103, 1109, 1112, 1117, 1122, 1126,
+ 1132, 1138, 1144, 1147, 1151, 1157, 1160, 1165, 1169, 1175,
+ 1178, 1181, 1186, 1191, 1196, 1199, 1202, 1207, 1210, 1213,
+ 1216, 1219, 1222, 1225, 1228, 1233, 1236, 1241, 1245, 1249,
+ 1252, 1256, 1259, 1264, 1267, 1272, 1275, 1280, 1284, 1290,
+ 1293, 1298, 1301, 1306, 1309, 1314, 1317, 1336, 1340, 1346,
+ 1353, 1356, 1359, 1364, 1367, 1370, 1376, 1379, 1384, 1389,
+ 1398, 1403, 1412, 1417, 1420, 1425, 1428, 1433, 1439, 1445,
+ 1448, 1451, 1454, 1457, 1460, 1466, 1475, 1478, 1483, 1486,
+ 1491, 1494, 1499, 1502, 1505, 1508, 1511, 1514, 1517, 1520,
+ 1525, 1529, 1533, 1536, 1541, 1546, 1550, 1556, 1559, 1564,
+ 1568, 1574, 1579, 1583, 1589, 1594, 1597, 1602, 1606, 1612,
+ 1615, 1618, 1621, 1633, 1637, 1656, 1669, 1684, 1687, 1690,
+ 1693, 1696, 1699, 1704, 1708, 1714, 1717, 1722, 1726, 1733,
+ 1736, 1739, 1742, 1745, 1748, 1751, 1754, 1757, 1760, 1765,
+ 1776, 1779, 1784, 1787, 1790, 1796, 1800, 1806, 1809, 1817,
+ 1820, 1823, 1826, 1832, 1837, 1842
};
#endif
@@ -801,7 +803,7 @@ static const char *const yytname[] =
"TOKEN_DELETE", "TOKEN_DELIMITER", "TOKEN_DESC", "TOKEN_DISTINCT",
"TOKEN_DOUBLE", "TOKEN_DROP", "TOKEN_ELSE", "TOKEN_END",
"TOKEN_ESCAPE_STRINGS", "TOKEN_EXISTS", "TOKEN_EXTRACT", "TOKEN_FALSE",
- "TOKEN_FIRST", "TOKEN_FLOAT", "TOKEN_FOREIGN", "TOKEN_FROM",
+ "TOKEN_FIRST", "TOKEN_FLOAT", "TOKEN_FOR", "TOKEN_FOREIGN", "TOKEN_FROM",
"TOKEN_FULL", "TOKEN_GROUP", "TOKEN_HASH", "TOKEN_HAVING", "TOKEN_HOUR",
"TOKEN_IN", "TOKEN_INDEX", "TOKEN_INNER", "TOKEN_INSERT",
"TOKEN_INTEGER", "TOKEN_INTERVAL", "TOKEN_INTO", "TOKEN_JOIN",
@@ -811,26 +813,27 @@ static const char *const yytname[] =
"TOKEN_PARTITIONS", "TOKEN_PERCENT", "TOKEN_PRIMARY", "TOKEN_QUIT",
"TOKEN_RANGE", "TOKEN_REAL", "TOKEN_REFERENCES", "TOKEN_RIGHT",
"TOKEN_ROW_DELIMITER", "TOKEN_SECOND", "TOKEN_SELECT", "TOKEN_SET",
- "TOKEN_SMA", "TOKEN_SMALLINT", "TOKEN_TABLE", "TOKEN_THEN", "TOKEN_TIME",
- "TOKEN_TIMESTAMP", "TOKEN_TRUE", "TOKEN_TUPLESAMPLE", "TOKEN_UNIQUE",
- "TOKEN_UPDATE", "TOKEN_USING", "TOKEN_VALUES", "TOKEN_VARCHAR",
- "TOKEN_WHEN", "TOKEN_WHERE", "TOKEN_WITH", "TOKEN_YEAR",
- "TOKEN_YEARMONTH", "TOKEN_EOF", "TOKEN_LEX_ERROR", "';'", "'\\n'", "'('",
- "')'", "','", "'%'", "$accept", "start", "sql_statement",
- "quit_statement", "alter_table_statement", "create_table_statement",
- "create_index_statement", "drop_table_statement", "column_def",
- "column_def_commalist", "data_type", "column_constraint_def",
- "column_constraint_def_list", "opt_column_constraint_def_list",
- "table_constraint_def", "table_constraint_def_commalist",
- "opt_table_constraint_def_commalist", "opt_column_list",
- "opt_block_properties", "opt_partition_clause", "partition_type",
- "key_value_list", "key_value", "key_string_value", "key_string_list",
- "key_integer_value", "index_type", "opt_index_properties",
- "insert_statement", "copy_from_statement", "opt_copy_from_params",
- "copy_from_params", "update_statement", "delete_statement",
- "assignment_list", "assignment_item", "select_statement", "with_clause",
- "with_list", "with_list_element", "select_query", "opt_all_distinct",
- "selection", "selection_item_commalist", "selection_item", "from_clause",
+ "TOKEN_SMA", "TOKEN_SMALLINT", "TOKEN_SUBSTRING", "TOKEN_TABLE",
+ "TOKEN_THEN", "TOKEN_TIME", "TOKEN_TIMESTAMP", "TOKEN_TRUE",
+ "TOKEN_TUPLESAMPLE", "TOKEN_UNIQUE", "TOKEN_UPDATE", "TOKEN_USING",
+ "TOKEN_VALUES", "TOKEN_VARCHAR", "TOKEN_WHEN", "TOKEN_WHERE",
+ "TOKEN_WITH", "TOKEN_YEAR", "TOKEN_YEARMONTH", "TOKEN_EOF",
+ "TOKEN_LEX_ERROR", "';'", "'\\n'", "'('", "')'", "','", "'%'", "$accept",
+ "start", "sql_statement", "quit_statement", "alter_table_statement",
+ "create_table_statement", "create_index_statement",
+ "drop_table_statement", "column_def", "column_def_commalist",
+ "data_type", "column_constraint_def", "column_constraint_def_list",
+ "opt_column_constraint_def_list", "table_constraint_def",
+ "table_constraint_def_commalist", "opt_table_constraint_def_commalist",
+ "opt_column_list", "opt_block_properties", "opt_partition_clause",
+ "partition_type", "key_value_list", "key_value", "key_string_value",
+ "key_string_list", "key_integer_value", "index_type",
+ "opt_index_properties", "insert_statement", "copy_from_statement",
+ "opt_copy_from_params", "copy_from_params", "update_statement",
+ "delete_statement", "assignment_list", "assignment_item",
+ "select_statement", "with_clause", "with_list", "with_list_element",
+ "select_query", "opt_all_distinct", "selection",
+ "selection_item_commalist", "selection_item", "from_clause",
"subquery_expression", "opt_sample_clause", "join_type",
"joined_table_reference", "table_reference", "table_reference_signature",
"table_reference_signature_primary", "joined_table_reference_commalist",
@@ -840,14 +843,14 @@ static const char *const yytname[] =
"where_clause", "or_expression", "and_expression", "not_expression",
"predicate_expression_base", "add_expression", "multiply_expression",
"unary_expression", "expression_base", "function_call",
- "extract_function", "case_expression", "simple_when_clause_list",
- "simple_when_clause", "searched_when_clause_list",
- "searched_when_clause", "opt_else_clause", "expression_list",
- "literal_value", "datetime_unit", "literal_value_commalist",
- "attribute_ref", "attribute_ref_list", "comparison_operation",
- "unary_operation", "add_operation", "multiply_operation",
- "name_commalist", "any_name", "boolean_value", "command",
- "command_argument_list", YY_NULLPTR
+ "extract_function", "substr_function", "case_expression",
+ "simple_when_clause_list", "simple_when_clause",
+ "searched_when_clause_list", "searched_when_clause", "opt_else_clause",
+ "expression_list", "literal_value", "datetime_unit",
+ "literal_value_commalist", "attribute_ref", "attribute_ref_list",
+ "comparison_operation", "unary_operation", "add_operation",
+ "multiply_operation", "name_commalist", "any_name", "boolean_value",
+ "command", "command_argument_list", YY_NULLPTR
};
#endif
@@ -868,15 +871,15 @@ static const yytype_uint16 yytoknum[] =
340, 341, 342, 343, 344, 345, 346, 347, 348, 349,
350, 351, 352, 353, 354, 355, 356, 357, 358, 359,
360, 361, 362, 363, 364, 365, 366, 367, 368, 369,
- 370, 371, 372, 373, 374, 375, 376, 377, 59, 10,
- 40, 41, 44, 37
+ 370, 371, 372, 373, 374, 375, 376, 377, 378, 379,
+ 59, 10, 40, 41, 44, 37
};
# endif
-#define YYPACT_NINF -223
+#define YYPACT_NINF -230
#define yypact_value_is_default(Yystate) \
- (!!((Yystate) == (-223)))
+ (!!((Yystate) == (-230)))
#define YYTABLE_NINF -128
@@ -887,54 +890,55 @@ static const yytype_uint16 yytoknum[] =
STATE-NUM. */
static const yytype_int16 yypact[] =
{
- 921, -223, -223, -82, 231, -14, 56, 22, 74, -223,
- 70, 231, 231, -223, 135, 120, -223, -223, -223, -223,
- -223, -223, -223, -223, -223, -223, -35, -223, -73, 177,
- 231, -223, -223, 121, 231, 231, 231, 231, 231, -223,
- -223, 576, 85, 63, -223, 173, 77, -223, -223, -223,
- 140, -223, -223, -223, -223, 18, 218, 144, 97, 119,
- -223, 4, -223, -223, 240, 245, -223, -223, -223, 642,
- 134, -223, 187, -223, -223, 146, -223, -223, 265, -223,
- -223, -223, -223, -223, -223, 164, 203, 708, 290, 230,
- 176, -223, -223, 256, 20, -223, -223, -223, -223, -223,
- -223, -223, 840, -11, 231, 231, 182, 231, 231, 167,
- 206, 204, 231, 231, 483, -223, -223, 205, 231, -223,
- -223, -223, 483, 47, -10, -223, 329, -223, 129, 129,
- 330, -223, 207, 26, -223, 32, 119, 906, -223, -223,
- 231, 906, -223, -223, -223, -223, 906, 245, -223, 231,
- 324, -70, -223, 331, -223, 234, -223, 125, -223, 234,
- 231, 49, 231, 231, 211, -223, 213, -223, 137, 991,
- 774, 182, 390, 336, 339, -223, -223, 1117, 335, 1001,
- 142, 10, 906, -9, -223, 906, -223, 296, 228, -223,
- -223, -223, -223, -223, -223, 292, -223, 232, -223, -223,
- 21, 163, 122, -223, 229, 163, -13, 294, -223, -223,
- 20, -223, -223, 236, 906, -223, 259, 152, 231, -223,
- 906, -223, 231, -223, -223, 238, 287, 288, 241, -223,
- -223, -223, 61, 231, 258, 49, 231, -223, 156, -223,
- -223, -6, 69, 483, 483, 55, -223, -223, -223, -223,
- -223, -223, -223, -223, 906, 244, 906, 5, -223, 154,
- 260, 906, 37, -223, 317, 259, -223, -223, 906, -223,
- 102, 231, -223, -223, 284, -223, 291, 297, 301, 32,
- -223, 377, 379, 163, 348, 320, -223, 157, -223, 906,
- -223, 259, -223, 483, 263, 268, 231, 394, 151, 159,
- -223, 168, 378, 25, -223, 272, 283, -223, 318, 279,
- 1001, -223, 332, 231, -223, -223, 156, -223, -223, 339,
- -223, -223, -223, 906, 285, 161, 708, -223, 259, 327,
- -223, -223, 1001, 289, 259, 906, -223, 33, -223, -223,
- -223, -223, -223, 32, 122, 321, 323, -223, 906, 483,
- 328, -223, 259, 13, 231, 231, 170, -223, -223, -223,
- -223, -223, -223, -223, 76, -223, 231, -223, -223, -223,
- -223, 299, 49, 381, 340, -223, 483, -223, -223, 300,
- -223, 200, 708, -223, 906, 172, -223, -223, 1001, 259,
- -223, 342, -223, -223, 311, 336, 382, 361, -223, 175,
- 179, -223, 443, 151, -223, 231, -223, -223, 333, 410,
- -223, 29, 231, 906, 183, 259, -223, 185, 483, 906,
- 444, -223, 355, -223, -223, -223, 188, -223, -223, -223,
- -223, 16, 231, -5, -223, 334, 259, -223, -223, 336,
- 326, -223, 155, -223, 231, -223, 231, -223, -223, 231,
- -223, 190, -223, -223, 337, -223, 906, -223, -223, 369,
- 341, -223, 192, -223, 231, -223, 110, -223, 231, -223,
- 194, -223, -223, 201, 365, -223, 455, -223
+ 162, -230, -230, -75, 181, 7, 17, 23, 19, -230,
+ 41, 181, 181, -230, 151, 129, -230, -230, -230, -230,
+ -230, -230, -230, -230, -230, -230, -38, -230, -36, 166,
+ 181, -230, -230, 138, 181, 181, 181, 181, 181, -230,
+ -230, 644, 111, 93, -230, 220, 128, -230, -230, -230,
+ 202, -230, -230, -230, -230, 65, 273, 192, 163, 172,
+ -230, 11, -230, -230, 295, 300, -230, -230, -230, 699,
+ 207, -230, 233, -230, -230, 218, -230, -230, 310, -230,
+ -230, -230, -230, 219, -230, -230, 229, 245, 786, 322,
+ 261, 224, -230, -230, 182, 35, -230, -230, -230, -230,
+ -230, -230, -230, -230, 928, -12, 181, 181, 235, 181,
+ 181, 194, 226, 238, 181, 181, 542, -230, -230, 237,
+ 181, -230, -230, -230, 542, 53, -34, -230, 359, -230,
+ 105, 105, 983, 363, -230, 241, 40, -230, 18, 172,
+ 983, -230, -230, 181, 983, -230, -230, -230, -230, 983,
+ 300, -230, 181, 331, 82, -230, 360, -230, 268, -230,
+ -7, -230, 268, 181, 146, 181, 181, 244, -230, 246,
+ -230, 143, 1088, 841, 235, 455, 369, 372, -230, -230,
+ 624, 364, 1069, 155, 15, 983, 8, -230, 983, -230,
+ 323, 252, -230, -230, -230, -230, -230, -230, 317, -230,
+ 60, 256, -230, -230, 10, 90, 197, -230, 257, 90,
+ -10, 319, -230, -230, 35, -230, -230, 259, 983, -230,
+ 270, 160, 181, -230, 983, -230, 181, -230, -230, 263,
+ 313, 315, 267, -230, -230, -230, 174, 181, 281, 146,
+ 181, -230, 81, -230, -230, 20, 34, 542, 542, 61,
+ -230, -230, -230, -230, -230, -230, -230, -230, 983, 271,
+ 983, 47, -230, 165, 283, 983, 55, -230, 342, 270,
+ -230, -230, 983, 398, -230, 125, 181, -230, -230, 311,
+ -230, 312, 318, 326, 18, -230, 405, 407, 90, 375,
+ 344, -230, 171, -230, 983, -230, 270, -230, 542, 287,
+ 288, 181, 417, 157, 175, -230, 177, 396, 49, -230,
+ 290, 301, -230, 335, 296, 1069, -230, 346, 181, -230,
+ -230, 81, -230, -230, 372, -230, -230, -230, 983, 298,
+ 203, 786, -230, 270, 337, -230, -230, 1069, 299, 270,
+ 983, -230, 42, -25, -230, -230, -230, -230, -230, 18,
+ 197, 334, 336, -230, 983, 542, 341, -230, 270, 21,
+ 181, 181, 183, -230, -230, -230, -230, -230, -230, -230,
+ 193, -230, 181, -230, -230, -230, -230, 307, 146, 404,
+ 347, -230, 542, -230, -230, 316, -230, 234, 786, -230,
+ 983, 185, -230, -230, 1069, 270, -230, 437, -230, 354,
+ -230, -230, 320, 369, 409, 365, -230, 188, 190, -230,
+ 451, 157, -230, 181, -230, -230, 325, 426, -230, 31,
+ 181, 983, 195, 270, -230, 198, 338, 542, 983, 460,
+ -230, 366, -230, -230, -230, 200, -230, -230, -230, -230,
+ 14, 181, 13, -230, 339, 270, -230, -230, -230, 369,
+ 345, -230, 231, -230, 181, -230, 181, -230, -230, 181,
+ -230, 208, -230, -230, 343, -230, 983, -230, -230, 377,
+ 348, -230, 210, -230, 181, -230, 113, -230, 181, -230,
+ 213, -230, -230, 215, 376, -230, 471, -230
};
/* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM.
@@ -942,50 +946,51 @@ static const yytype_int16 yypact[] =
means the default is an error. */
static const yytype_uint16 yydefact[] =
{
- 0, 6, 262, 0, 0, 0, 0, 0, 0, 18,
+ 0, 6, 265, 0, 0, 0, 0, 0, 0, 18,
112, 0, 0, 7, 0, 0, 15, 8, 10, 11,
- 13, 14, 9, 17, 12, 16, 0, 105, 0, 260,
- 0, 254, 255, 0, 0, 0, 0, 0, 0, 113,
+ 13, 14, 9, 17, 12, 16, 0, 105, 0, 263,
+ 0, 257, 258, 0, 0, 0, 0, 0, 0, 113,
114, 0, 0, 107, 108, 0, 145, 1, 3, 2,
- 0, 106, 5, 4, 261, 0, 0, 0, 0, 166,
- 25, 0, 220, 217, 0, 246, 115, 40, 29, 0,
+ 0, 106, 5, 4, 264, 0, 0, 0, 0, 166,
+ 25, 0, 223, 220, 0, 249, 115, 40, 29, 0,
0, 30, 31, 34, 36, 0, 37, 39, 0, 41,
- 216, 35, 38, 32, 33, 0, 0, 0, 0, 0,
- 116, 117, 198, 121, 187, 189, 191, 194, 195, 196,
- 193, 192, 0, 232, 0, 0, 0, 0, 0, 0,
- 0, 94, 0, 0, 0, 101, 167, 0, 0, 91,
- 218, 219, 0, 0, 212, 209, 0, 43, 0, 221,
- 0, 44, 0, 0, 223, 0, 166, 0, 247, 248,
- 0, 0, 120, 250, 251, 249, 0, 0, 190, 0,
- 0, 166, 103, 0, 109, 0, 110, 0, 252, 0,
- 0, 0, 0, 0, 0, 93, 66, 27, 0, 0,
- 0, 0, 0, 168, 170, 172, 174, 0, 192, 0,
- 0, 0, 0, 212, 206, 0, 210, 0, 0, 226,
- 227, 228, 225, 229, 224, 0, 222, 0, 123, 197,
- 0, 0, 147, 136, 122, 141, 124, 149, 118, 119,
- 186, 188, 233, 0, 0, 199, 214, 0, 0, 100,
- 0, 146, 0, 92, 19, 0, 0, 0, 0, 20,
- 21, 22, 0, 0, 0, 64, 0, 42, 56, 173,
- 181, 0, 0, 0, 0, 0, 236, 238, 239, 240,
- 241, 237, 242, 244, 0, 0, 0, 0, 230, 0,
- 0, 0, 0, 207, 0, 213, 205, 45, 0, 46,
- 127, 0, 137, 143, 133, 128, 129, 131, 0, 0,
- 140, 0, 0, 139, 0, 151, 200, 0, 201, 0,
- 102, 104, 253, 0, 0, 0, 0, 0, 0, 0,
- 234, 0, 232, 0, 63, 65, 68, 28, 0, 0,
- 0, 47, 0, 0, 49, 55, 57, 26, 180, 169,
- 171, 243, 245, 0, 0, 0, 0, 182, 179, 0,
- 178, 90, 0, 0, 211, 0, 204, 0, 142, 144,
- 134, 130, 132, 0, 148, 0, 0, 138, 0, 0,
- 153, 202, 215, 0, 0, 0, 0, 96, 258, 259,
- 257, 256, 97, 95, 0, 67, 0, 83, 84, 85,
- 86, 87, 0, 0, 70, 48, 0, 51, 50, 0,
- 54, 0, 0, 184, 0, 0, 177, 231, 0, 208,
- 203, 0, 125, 126, 150, 152, 0, 155, 61, 0,
- 0, 58, 0, 0, 235, 0, 24, 62, 0, 0,
- 23, 0, 0, 0, 0, 175, 183, 0, 0, 0,
- 0, 111, 0, 59, 98, 99, 0, 74, 76, 77,
- 78, 0, 0, 0, 52, 0, 176, 185, 89, 135,
+ 219, 35, 38, 0, 32, 33, 0, 0, 0, 0,
+ 0, 116, 117, 199, 121, 187, 189, 191, 194, 195,
+ 196, 197, 193, 192, 0, 235, 0, 0, 0, 0,
+ 0, 0, 0, 94, 0, 0, 0, 101, 167, 0,
+ 0, 91, 221, 222, 0, 0, 215, 212, 0, 43,
+ 0, 224, 0, 0, 44, 0, 0, 226, 0, 166,
+ 0, 250, 251, 0, 0, 120, 253, 254, 252, 0,
+ 0, 190, 0, 0, 166, 103, 0, 109, 0, 110,
+ 0, 255, 0, 0, 0, 0, 0, 0, 93, 66,
+ 27, 0, 0, 0, 0, 0, 168, 170, 172, 174,
+ 0, 192, 0, 0, 0, 0, 215, 209, 0, 213,
+ 0, 0, 229, 230, 231, 228, 232, 227, 0, 225,
+ 0, 0, 123, 198, 0, 0, 147, 136, 122, 141,
+ 124, 149, 118, 119, 186, 188, 236, 0, 0, 200,
+ 217, 0, 0, 100, 0, 146, 0, 92, 19, 0,
+ 0, 0, 0, 20, 21, 22, 0, 0, 0, 64,
+ 0, 42, 56, 173, 181, 0, 0, 0, 0, 0,
+ 239, 241, 242, 243, 244, 240, 245, 247, 0, 0,
+ 0, 0, 233, 0, 0, 0, 0, 210, 0, 216,
+ 208, 45, 0, 0, 46, 127, 0, 137, 143, 133,
+ 128, 129, 131, 0, 0, 140, 0, 0, 139, 0,
+ 151, 201, 0, 202, 0, 102, 104, 256, 0, 0,
+ 0, 0, 0, 0, 0, 237, 0, 235, 0, 63,
+ 65, 68, 28, 0, 0, 0, 47, 0, 0, 49,
+ 55, 57, 26, 180, 169, 171, 246, 248, 0, 0,
+ 0, 0, 182, 179, 0, 178, 90, 0, 0, 214,
+ 0, 207, 0, 0, 142, 144, 134, 130, 132, 0,
+ 148, 0, 0, 138, 0, 0, 153, 203, 218, 0,
+ 0, 0, 0, 96, 261, 262, 260, 259, 97, 95,
+ 0, 67, 0, 83, 84, 85, 86, 87, 0, 0,
+ 70, 48, 0, 51, 50, 0, 54, 0, 0, 184,
+ 0, 0, 177, 234, 0, 211, 204, 0, 205, 0,
+ 125, 126, 150, 152, 0, 155, 61, 0, 0, 58,
+ 0, 0, 238, 0, 24, 62, 0, 0, 23, 0,
+ 0, 0, 0, 175, 183, 0, 0, 0, 0, 0,
+ 111, 0, 59, 98, 99, 0, 74, 76, 77, 78,
+ 0, 0, 0, 52, 0, 176, 185, 89, 206, 135,
154, 157, 160, 156, 0, 88, 0, 82, 80, 0,
79, 0, 72, 73, 0, 53, 0, 161, 162, 163,
0, 75, 0, 69, 0, 158, 0, 159, 0, 81,
@@ -995,31 +1000,31 @@ static const yytype_uint16 yydefact[] =
/* YYPGOTO[NTERM-NUM]. */
static const yytype_int16 yypgoto[] =
{
- -223, -223, -223, -223, -223, -223, -223, -223, -131, -223,
- 303, 150, -223, -223, -222, -223, -223, -223, -223, -223,
- -223, 38, 27, -223, -223, -223, -223, -223, -223, -223,
- -223, -223, -223, -223, -223, 257, -223, -223, -223, 372,
- 14, -223, -223, -223, 343, -223, -94, -223, -223, -181,
- 131, -170, -8, -223, -223, -223, -223, -223, -223, 28,
- -223, -223, -58, -223, -121, 235, 237, 312, -30, 344,
- 346, 384, -130, -223, -223, -223, 314, -223, 359, 315,
- -208, -169, 366, 106, -105, -223, -223, -223, -223, -223,
- -115, -4, 98, -223, -223
+ -230, -230, -230, -230, -230, -230, -230, -230, -127, -230,
+ 309, 161, -230, -230, -229, -230, -230, -230, -230, -230,
+ -230, 43, 27, -230, -230, -230, -230, -230, -230, -230,
+ -230, -230, -230, -230, -230, 264, -230, -230, -230, 378,
+ 12, -230, -230, -230, 350, -230, -103, -230, -230, -139,
+ 142, -190, -8, -230, -230, -230, -230, -230, -230, 26,
+ -230, -230, 44, -230, -121, 240, 247, 321, -28, 349,
+ 351, 394, -132, -230, -230, -230, -230, 324, -230, 373,
+ 327, -216, -171, 370, 108, -107, -230, -230, -230, -230,
+ -230, -119, -4, 94, -230, -230
};
/* YYDEFGOTO[NTERM-NUM]. */
static const yytype_int16 yydefgoto[] =
{
- -1, 14, 15, 16, 17, 18, 19, 20, 167, 168,
- 88, 315, 316, 317, 229, 305, 306, 234, 374, 410,
- 454, 426, 427, 428, 429, 430, 371, 406, 21, 22,
- 165, 299, 23, 24, 151, 152, 25, 26, 43, 44,
- 132, 41, 89, 90, 91, 136, 92, 283, 278, 202,
- 203, 272, 273, 204, 285, 350, 397, 421, 440, 441,
- 459, 467, 115, 116, 173, 174, 175, 176, 177, 94,
- 95, 96, 97, 98, 99, 183, 184, 124, 125, 187,
- 217, 100, 195, 259, 101, 301, 256, 102, 141, 146,
- 157, 103, 362, 28, 29
+ -1, 14, 15, 16, 17, 18, 19, 20, 170, 171,
+ 89, 320, 321, 322, 233, 310, 311, 238, 380, 418,
+ 464, 435, 436, 437, 438, 439, 377, 414, 21, 22,
+ 168, 304, 23, 24, 154, 155, 25, 26, 43, 44,
+ 135, 41, 90, 91, 92, 139, 93, 288, 283, 206,
+ 207, 277, 278, 208, 290, 356, 405, 430, 450, 451,
+ 469, 477, 117, 118, 176, 177, 178, 179, 180, 95,
+ 96, 97, 98, 99, 100, 101, 186, 187, 126, 127,
+ 190, 221, 102, 198, 263, 103, 306, 260, 104, 144,
+ 149, 160, 105, 368, 28, 29
};
/* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If
@@ -1027,336 +1032,341 @@ static const yytype_int16 yydefgoto[] =
number is the opposite. If YYTABLE_NINF, syntax error. */
static const yytype_int16 yytable[] =
{
- 33, 181, 243, 180, 45, 205, 287, 42, 46, 178,
- 258, 93, 156, 304, 27, 329, 149, 178, 243, 270,
- 31, 243, 32, 447, 281, 31, 55, 32, 30, 224,
- 57, 58, 59, 60, 61, 280, 31, 243, 32, 123,
- 51, 201, 50, 143, 144, 448, 109, 138, 139, 185,
- 185, 241, 114, 52, 138, 139, 53, 133, 138, 139,
- 367, 34, 218, 368, 369, 178, 452, 178, 138, 139,
- 205, 10, 321, 322, 323, 119, 110, 240, 207, 245,
- 246, 247, 248, 249, 250, 251, 252, 253, 254, 142,
- 138, 139, 225, 219, 330, 453, 35, 45, 344, 39,
- 153, 46, 282, 158, 159, 307, 201, 93, 166, 169,
- 10, 122, 182, 347, 158, 297, 226, 150, 385, 150,
- 216, 261, 298, 117, 36, 318, 40, 10, 300, 324,
- 402, 206, 37, 370, 118, 47, 209, 403, 178, 178,
- 394, 377, 242, 255, 398, 212, 449, 227, 335, 205,
- 407, 200, 262, 145, 38, 265, 169, 199, 230, 231,
- 434, 327, 200, 387, 390, 228, 308, 31, 182, 32,
- 384, 274, 353, 223, 414, 471, 138, 139, 275, 189,
- 54, 356, 138, 139, 216, 201, 276, 457, 178, 56,
- 291, 274, 104, 472, 271, 105, 206, 46, 275, 309,
- 199, 46, 190, -127, 106, 277, 276, 107, 310, 413,
- 458, 160, 161, 205, 153, 358, 191, 192, 292, 258,
- 108, 138, 139, 111, 325, 277, 328, 113, 395, 302,
- 383, 334, 169, 338, 193, 31, 112, 32, 337, 399,
- 400, 114, 359, 360, 178, 311, 48, 120, 49, 201,
- 162, 163, 121, 194, 312, 411, 221, 222, 313, 352,
- 31, 404, 32, 339, 126, 361, 127, 46, 235, 236,
- 129, 178, 314, 260, 222, 206, 128, 138, 139, 46,
- 138, 139, 131, 288, 289, 331, 332, 140, 351, 289,
- 363, 364, 158, 381, 130, 134, 216, 439, 135, 365,
- 366, 401, 222, 416, 289, 389, 422, 222, 137, 379,
- 423, 222, 155, 178, 437, 289, 438, 332, 216, 445,
- 446, 463, 446, 469, 222, 474, 222, 164, 31, 62,
- 32, 63, 475, 222, 462, 179, 188, 197, 198, 206,
- 10, 232, 220, 233, 243, 64, 65, 213, 244, 470,
- 158, 158, 216, 473, 415, 257, 266, 67, 68, 267,
- 268, 279, 302, 269, 284, 69, 70, 286, 293, 294,
- 295, 296, 71, 72, 326, 73, 303, 336, 340, 333,
- 214, 74, 343, 436, 345, 341, 346, 75, 348, 442,
- 76, 342, 349, 354, 31, 62, 32, 63, 355, 357,
- 170, 431, 77, 78, 372, 149, 373, 375, 435, 376,
- 79, 64, 65, 80, 378, 382, 386, 408, 392, 388,
- 393, 396, 419, 67, 68, 81, 442, 450, 431, 405,
- 412, 69, 70, 82, 418, 409, 83, 84, 71, 72,
- 460, 73, 431, 289, 85, 158, 420, 74, 424, 86,
- 433, 443, 171, 75, 87, 215, 76, 444, 456, 466,
- 158, 476, 477, 432, 158, 455, 380, 464, 77, 78,
- 451, 468, 238, 461, 391, 290, 79, 154, 319, 80,
- 208, 320, 239, 186, 465, 210, 148, 31, 62, 32,
- 63, 81, 211, 170, 417, 196, 10, 263, 264, 82,
- 0, 425, 83, 84, 64, 65, 0, 0, 0, 0,
- 85, 0, 0, 0, 0, 86, 67, 68, 0, 0,
- 172, 0, 0, 0, 69, 70, 0, 0, 0, 0,
- 0, 71, 72, 0, 73, 0, 0, 0, 0, 0,
- 74, 0, 0, 0, 0, 171, 75, 0, 0, 76,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 77, 78, 0, 0, 0, 0, 0, 0, 79,
- 0, 0, 80, 0, 0, 0, 0, 0, 0, 0,
- 31, 62, 32, 63, 81, 0, 0, 0, 0, 0,
- 0, 0, 82, 0, 0, 83, 84, 64, 65, 66,
- 0, 0, 0, 85, 0, 0, 0, 0, 86, 67,
- 68, 0, 0, 172, 0, 0, 0, 69, 70, 0,
- 0, 0, 0, 0, 71, 72, 0, 73, 0, 0,
- 0, 0, 0, 74, 0, 0, 0, 0, 0, 75,
- 0, 0, 76, 0, 0, 0, 31, 62, 32, 63,
+ 33, 183, 292, 184, 45, 159, 209, 42, 46, 181,
+ 309, 262, 27, 94, 31, 152, 32, 181, 31, 285,
+ 32, 457, 31, 247, 32, 188, 55, 286, 247, 247,
+ 57, 58, 59, 60, 61, 205, 228, 30, 51, 247,
+ 50, 125, 397, 458, 249, 250, 251, 252, 253, 254,
+ 255, 256, 257, 258, 245, 141, 142, 334, 146, 147,
+ 136, 141, 142, 141, 142, 275, 181, 188, 181, 10,
+ 39, 244, 209, 121, 141, 142, 141, 142, 326, 327,
+ 328, 141, 142, 34, 373, 462, 36, 374, 375, 124,
+ 145, 313, 52, 111, 31, 53, 32, 40, 353, 45,
+ 38, 205, 156, 46, 200, 161, 162, 287, 398, 259,
+ 169, 172, 94, 312, 463, 391, 161, 10, 10, 35,
+ 153, 276, 153, 112, 314, 220, 225, 226, 265, 273,
+ 305, 185, 119, 315, 210, 37, 329, 335, 402, 213,
+ 181, 181, 204, 120, 383, 350, 459, 246, 216, 415,
+ 204, 47, 209, 323, 406, 192, 332, 266, 376, 172,
+ 269, 234, 235, 1, 443, 2, 393, 203, 340, 54,
+ 148, 316, 422, 203, 227, 396, 185, 359, 481, 193,
+ 317, 205, 362, 211, 318, 31, 31, 32, 32, 229,
+ 220, 181, 3, 194, 195, 279, 296, 482, 223, 319,
+ 210, 46, 280, 141, 142, 46, 116, 56, 4, 5,
+ 281, 196, 390, 143, 230, 6, 222, 209, 156, 106,
+ 7, 364, 297, 262, 141, 142, 389, 107, 302, 282,
+ 330, 197, 333, 307, 403, 303, 172, 339, 163, 164,
+ 8, 407, 408, 421, 342, 231, 205, 410, 181, 365,
+ 366, 108, 141, 142, 411, 141, 142, 48, 344, 49,
+ 109, 419, 9, 467, 232, 412, 358, 279, 345, 10,
+ 165, 166, 46, 367, 280, 181, 239, 240, 113, -127,
+ 210, 11, 281, 110, 46, 114, 468, 12, 264, 226,
+ 13, 141, 142, 293, 294, 115, 116, 161, 336, 337,
+ 387, 282, 122, 220, 357, 294, 449, 123, 369, 370,
+ 371, 372, 395, 129, 385, 131, 409, 226, 424, 294,
+ 181, 431, 226, 432, 226, 134, 220, 137, 446, 294,
+ 138, 447, 337, 455, 456, 31, 62, 32, 63, 128,
+ 472, 473, 456, 479, 226, 210, 484, 226, 485, 226,
+ 130, 132, 64, 65, 217, 480, 161, 161, 140, 483,
+ 220, 133, 423, 167, 67, 68, 191, 158, 307, 182,
+ 201, 224, 69, 70, 202, 10, 236, 247, 237, 71,
+ 72, 248, 73, 270, 261, 271, 272, 218, 74, 274,
+ 289, 284, 291, 445, 75, 298, 299, 76, 300, 301,
+ 452, 308, 341, 331, 338, 343, 346, 347, 349, 440,
+ 77, 78, 351, 348, 352, 354, 444, 355, 79, 360,
+ 361, 80, 363, 152, 378, 381, 379, 392, 382, 384,
+ 388, 394, 400, 81, 401, 404, 460, 440, 452, 413,
+ 416, 82, 83, 417, 426, 84, 85, 427, 420, 428,
+ 470, 429, 440, 86, 294, 161, 433, 441, 87, 31,
+ 62, 32, 63, 88, 219, 173, 442, 453, 476, 454,
+ 161, 448, 465, 486, 161, 474, 64, 65, 487, 466,
+ 478, 242, 386, 471, 461, 157, 295, 324, 67, 68,
+ 212, 399, 475, 214, 243, 325, 69, 70, 151, 189,
+ 215, 199, 425, 71, 72, 434, 73, 0, 0, 0,
+ 267, 0, 74, 268, 0, 0, 0, 174, 75, 0,
+ 0, 76, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 77, 78, 0, 0, 0, 0,
- 0, 0, 79, 64, 65, 80, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 67, 68, 81, 0, 0,
- 0, 0, 0, 69, 70, 82, 0, 0, 83, 84,
- 71, 72, 0, 73, 0, 0, 85, 0, 0, 74,
- 0, 86, 0, 0, 0, 75, 87, 0, 76, 0,
- 0, 0, 31, 62, 32, 63, 0, 0, 0, 0,
- 77, 78, 0, 0, 0, 0, 0, 0, 79, 64,
- 65, 80, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 67, 68, 81, 0, 0, 0, 0, 0, 69,
- 70, 82, 0, 0, 83, 84, 71, 72, 0, 73,
- 0, 0, 85, 122, 0, 74, 0, 86, 0, 0,
- 0, 75, 87, 0, 76, 0, 0, 0, 31, 62,
- 32, 63, 0, 0, 0, 0, 77, 78, 0, 0,
- 0, 0, 0, 0, 79, 64, 65, 80, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 67, 68, 81,
- 0, 0, 0, 0, 10, 69, 70, 82, 0, 0,
- 83, 84, 71, 72, 0, 73, 0, 0, 85, 0,
- 0, 74, 0, 86, 0, 0, 171, 75, 87, 0,
- 76, 0, 0, 0, 31, 62, 32, 63, 0, 0,
- 0, 0, 77, 78, 0, 0, 0, 0, 0, 0,
- 79, 64, 147, 80, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 67, 68, 81, 0, 0, 0, 0,
- 0, 69, 70, 82, 0, 0, 83, 84, 71, 72,
- 0, 73, 0, 0, 85, 0, 0, 74, 0, 86,
- 0, 0, 0, 75, 172, 0, 76, 0, 0, 0,
- 31, 62, 32, 63, 0, 0, 0, 0, 77, 78,
- 0, 0, 1, 0, 2, 0, 79, 64, 65, 80,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 67,
- 68, 81, 0, 0, 0, 0, 0, 69, 70, 82,
- 0, 3, 83, 84, 71, 72, 0, 73, 0, 0,
- 85, 0, 0, 74, 0, 86, 0, 4, 5, 75,
- 87, 0, 76, 0, 6, 0, 0, 0, 0, 7,
- 0, 0, 0, 0, 77, 78, 0, 0, 0, 0,
- 0, 0, 79, 0, 0, 80, 0, 0, 8, 0,
- 0, 0, 0, 0, 0, 0, 62, 81, 63, 0,
- 0, 0, 0, 0, 0, 82, 0, 0, 83, 84,
- 9, 0, 64, 147, 67, 68, 85, 10, 0, 0,
- 0, 86, 0, 70, 67, 68, 87, 0, 11, 71,
- 72, 0, 73, 70, 12, 0, 0, 13, 74, 71,
- 72, 0, 73, 0, 0, 0, 0, 76, 74, 0,
- 0, 0, 0, 0, 0, 0, 0, 76, 0, 77,
- 237, 0, 0, 0, 0, 0, 0, 79, 0, 77,
- 78, 0, 0, 0, 0, 0, 0, 79, 0, 0,
- 80, 0, 81, 0, 0, 0, 0, 0, 0, 0,
- 82, 0, 81, 83, 84, 0, 0, 0, 0, 0,
- 82, 85, 0, 83, 84, 0, 86, 0, 0, 0,
- 0, 85, 0, 0, 0, 0, 86, 245, 246, 247,
- 248, 249, 250, 251, 252, 253, 254, 0, 138, 139,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 79, 0, 0, 80, 31, 62, 32, 63,
+ 0, 0, 173, 0, 0, 0, 0, 81, 0, 0,
+ 0, 0, 10, 64, 65, 82, 83, 0, 0, 84,
+ 85, 0, 0, 0, 0, 67, 68, 86, 0, 0,
+ 0, 0, 87, 69, 70, 0, 0, 175, 0, 0,
+ 71, 72, 0, 73, 0, 0, 0, 0, 0, 74,
+ 0, 0, 0, 0, 174, 75, 0, 0, 76, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 255
+ 0, 77, 78, 0, 0, 0, 0, 0, 0, 79,
+ 0, 0, 80, 0, 249, 250, 251, 252, 253, 254,
+ 255, 256, 257, 258, 81, 141, 142, 0, 31, 62,
+ 32, 63, 82, 83, 0, 0, 84, 85, 0, 0,
+ 0, 0, 0, 0, 86, 64, 65, 66, 0, 87,
+ 0, 0, 0, 0, 175, 0, 0, 67, 68, 0,
+ 0, 0, 0, 0, 0, 69, 70, 0, 0, 0,
+ 0, 0, 71, 72, 0, 73, 0, 0, 0, 259,
+ 0, 74, 0, 31, 62, 32, 63, 75, 0, 0,
+ 76, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 64, 65, 0, 77, 78, 0, 0, 0, 0, 0,
+ 0, 79, 67, 68, 80, 0, 0, 0, 0, 0,
+ 69, 70, 0, 0, 0, 0, 81, 71, 72, 0,
+ 73, 0, 0, 0, 82, 83, 74, 0, 84, 85,
+ 0, 0, 75, 0, 0, 76, 86, 0, 0, 0,
+ 0, 87, 0, 0, 0, 0, 88, 0, 77, 78,
+ 0, 0, 0, 0, 0, 0, 79, 0, 0, 80,
+ 31, 62, 32, 63, 0, 0, 0, 0, 0, 0,
+ 0, 81, 0, 0, 0, 0, 0, 64, 65, 82,
+ 83, 0, 0, 84, 85, 0, 0, 0, 0, 67,
+ 68, 86, 124, 0, 0, 0, 87, 69, 70, 0,
+ 0, 88, 0, 0, 71, 72, 0, 73, 0, 0,
+ 0, 0, 0, 74, 0, 31, 62, 32, 63, 75,
+ 0, 0, 76, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 64, 65, 0, 77, 78, 0, 0, 0,
+ 0, 0, 0, 79, 67, 68, 80, 0, 0, 0,
+ 0, 0, 69, 70, 0, 0, 0, 0, 81, 71,
+ 72, 0, 73, 10, 0, 0, 82, 83, 74, 0,
+ 84, 85, 0, 174, 75, 0, 0, 76, 86, 0,
+ 0, 0, 0, 87, 0, 0, 0, 0, 88, 0,
+ 77, 78, 0, 0, 0, 0, 0, 0, 79, 0,
+ 0, 80, 31, 62, 32, 63, 0, 0, 0, 0,
+ 0, 0, 0, 81, 0, 0, 0, 0, 0, 64,
+ 150, 82, 83, 0, 0, 84, 85, 0, 0, 0,
+ 0, 67, 68, 86, 0, 0, 0, 0, 87, 69,
+ 70, 0, 0, 175, 0, 0, 71, 72, 0, 73,
+ 0, 0, 0, 0, 0, 74, 0, 31, 62, 32,
+ 63, 75, 0, 0, 76, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 64, 65, 0, 77, 78, 0,
+ 0, 0, 0, 0, 0, 79, 67, 68, 80, 0,
+ 0, 0, 0, 0, 69, 70, 0, 0, 0, 0,
+ 81, 71, 72, 0, 73, 0, 0, 0, 82, 83,
+ 74, 0, 84, 85, 0, 0, 75, 0, 0, 76,
+ 86, 0, 0, 0, 0, 87, 0, 0, 0, 0,
+ 88, 0, 77, 78, 0, 0, 0, 0, 0, 0,
+ 79, 0, 0, 80, 62, 0, 63, 0, 0, 0,
+ 0, 0, 0, 0, 0, 81, 0, 0, 0, 0,
+ 64, 150, 0, 82, 83, 0, 0, 84, 85, 0,
+ 0, 0, 67, 68, 0, 86, 0, 0, 0, 0,
+ 87, 70, 0, 0, 0, 88, 0, 71, 72, 0,
+ 73, 67, 68, 0, 0, 0, 74, 0, 0, 0,
+ 70, 0, 0, 0, 0, 76, 71, 72, 0, 73,
+ 0, 0, 0, 0, 0, 74, 0, 0, 77, 78,
+ 0, 0, 0, 0, 76, 0, 79, 0, 0, 80,
+ 0, 0, 0, 0, 0, 0, 0, 77, 241, 0,
+ 0, 81, 0, 0, 0, 79, 0, 0, 0, 82,
+ 0, 0, 0, 84, 85, 0, 0, 0, 0, 0,
+ 81, 86, 0, 0, 0, 0, 87, 0, 82, 0,
+ 0, 0, 84, 85, 0, 0, 0, 0, 0, 0,
+ 86, 0, 0, 0, 0, 87
};
static const yytype_int16 yycheck[] =
{
- 4, 122, 8, 118, 12, 135, 214, 11, 12, 114,
- 179, 41, 106, 235, 0, 10, 27, 122, 8, 200,
- 4, 8, 6, 7, 37, 4, 30, 6, 110, 160,
- 34, 35, 36, 37, 38, 205, 4, 8, 6, 69,
- 26, 135, 77, 23, 24, 29, 28, 21, 22, 59,
- 59, 172, 122, 126, 21, 22, 129, 87, 21, 22,
- 35, 75, 132, 38, 39, 170, 71, 172, 21, 22,
- 200, 106, 17, 18, 19, 61, 58, 171, 136, 10,
- 11, 12, 13, 14, 15, 16, 17, 18, 19, 93,
- 21, 22, 43, 151, 89, 100, 110, 105, 279, 29,
- 104, 105, 115, 107, 108, 236, 200, 137, 112, 113,
- 106, 121, 121, 283, 118, 54, 67, 130, 326, 130,
- 150, 111, 61, 119, 68, 131, 56, 106, 233, 74,
- 54, 135, 110, 108, 130, 0, 140, 61, 243, 244,
- 348, 310, 172, 74, 131, 149, 130, 98, 111, 279,
- 372, 130, 182, 133, 80, 185, 160, 131, 162, 163,
- 131, 255, 130, 332, 131, 116, 10, 4, 121, 6,
- 9, 69, 293, 159, 382, 65, 21, 22, 76, 50,
- 3, 296, 21, 22, 214, 279, 84, 32, 293, 68,
- 220, 69, 107, 83, 31, 132, 200, 201, 76, 43,
- 131, 205, 73, 81, 31, 103, 84, 130, 52, 9,
- 55, 44, 45, 343, 218, 64, 87, 88, 222, 388,
- 80, 21, 22, 5, 254, 103, 256, 130, 349, 233,
- 324, 261, 236, 131, 105, 4, 92, 6, 268, 354,
- 355, 122, 91, 92, 349, 89, 126, 7, 128, 343,
- 44, 45, 7, 124, 98, 376, 131, 132, 102, 289,
- 4, 366, 6, 271, 130, 114, 79, 271, 131, 132,
- 5, 376, 116, 131, 132, 279, 130, 21, 22, 283,
- 21, 22, 79, 131, 132, 131, 132, 31, 131, 132,
- 131, 132, 296, 323, 130, 5, 326, 418, 68, 131,
- 132, 131, 132, 131, 132, 335, 131, 132, 132, 313,
- 131, 132, 130, 418, 131, 132, 131, 132, 348, 131,
- 132, 131, 132, 131, 132, 131, 132, 123, 4, 5,
- 6, 7, 131, 132, 449, 130, 7, 7, 131, 343,
- 106, 130, 11, 130, 8, 21, 22, 23, 9, 464,
- 354, 355, 382, 468, 384, 20, 60, 33, 34, 131,
- 68, 132, 366, 131, 70, 41, 42, 131, 130, 82,
- 82, 130, 48, 49, 130, 51, 118, 60, 94, 119,
- 56, 57, 81, 413, 7, 94, 7, 63, 40, 419,
- 66, 94, 72, 130, 4, 5, 6, 7, 130, 5,
- 10, 405, 78, 79, 132, 27, 123, 89, 412, 130,
- 86, 21, 22, 89, 82, 130, 89, 36, 97, 130,
- 97, 93, 40, 33, 34, 101, 456, 431, 432, 130,
- 130, 41, 42, 109, 92, 95, 112, 113, 48, 49,
- 444, 51, 446, 132, 120, 449, 85, 57, 5, 125,
- 40, 7, 62, 63, 130, 131, 66, 102, 132, 90,
- 464, 96, 7, 130, 468, 131, 316, 130, 78, 79,
- 432, 130, 169, 446, 343, 218, 86, 105, 243, 89,
- 137, 244, 170, 124, 456, 141, 102, 4, 5, 6,
- 7, 101, 146, 10, 388, 129, 106, 183, 183, 109,
- -1, 403, 112, 113, 21, 22, -1, -1, -1, -1,
- 120, -1, -1, -1, -1, 125, 33, 34, -1, -1,
- 130, -1, -1, -1, 41, 42, -1, -1, -1, -1,
- -1, 48, 49, -1, 51, -1, -1, -1, -1, -1,
- 57, -1, -1, -1, -1, 62, 63, -1, -1, 66,
+ 4, 120, 218, 124, 12, 108, 138, 11, 12, 116,
+ 239, 182, 0, 41, 4, 27, 6, 124, 4, 209,
+ 6, 7, 4, 8, 6, 59, 30, 37, 8, 8,
+ 34, 35, 36, 37, 38, 138, 163, 112, 26, 8,
+ 78, 69, 67, 29, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 175, 21, 22, 10, 23, 24,
+ 88, 21, 22, 21, 22, 204, 173, 59, 175, 107,
+ 29, 174, 204, 61, 21, 22, 21, 22, 17, 18,
+ 19, 21, 22, 76, 35, 72, 69, 38, 39, 123,
+ 94, 10, 128, 28, 4, 131, 6, 56, 288, 107,
+ 81, 204, 106, 107, 132, 109, 110, 117, 133, 75,
+ 114, 115, 140, 240, 101, 331, 120, 107, 107, 112,
+ 132, 31, 132, 58, 43, 153, 133, 134, 113, 69,
+ 237, 123, 121, 52, 138, 112, 75, 90, 354, 143,
+ 247, 248, 132, 132, 315, 284, 132, 175, 152, 378,
+ 132, 0, 284, 133, 133, 50, 259, 185, 109, 163,
+ 188, 165, 166, 1, 133, 3, 337, 133, 113, 3,
+ 135, 90, 388, 133, 162, 133, 123, 298, 65, 74,
+ 99, 284, 301, 139, 103, 4, 4, 6, 6, 43,
+ 218, 298, 30, 88, 89, 70, 224, 84, 154, 118,
+ 204, 205, 77, 21, 22, 209, 124, 69, 46, 47,
+ 85, 106, 9, 31, 68, 53, 134, 349, 222, 108,
+ 58, 64, 226, 394, 21, 22, 329, 134, 54, 104,
+ 258, 126, 260, 237, 355, 61, 240, 265, 44, 45,
+ 78, 360, 361, 9, 272, 99, 349, 54, 355, 92,
+ 93, 31, 21, 22, 61, 21, 22, 128, 133, 130,
+ 132, 382, 100, 32, 118, 372, 294, 70, 276, 107,
+ 44, 45, 276, 116, 77, 382, 133, 134, 5, 82,
+ 284, 119, 85, 81, 288, 93, 55, 125, 133, 134,
+ 128, 21, 22, 133, 134, 132, 124, 301, 133, 134,
+ 328, 104, 7, 331, 133, 134, 427, 7, 133, 134,
+ 133, 134, 340, 80, 318, 5, 133, 134, 133, 134,
+ 427, 133, 134, 133, 134, 80, 354, 5, 133, 134,
+ 69, 133, 134, 133, 134, 4, 5, 6, 7, 132,
+ 459, 133, 134, 133, 134, 349, 133, 134, 133, 134,
+ 132, 132, 21, 22, 23, 474, 360, 361, 134, 478,
+ 388, 132, 390, 125, 33, 34, 7, 132, 372, 132,
+ 7, 11, 41, 42, 133, 107, 132, 8, 132, 48,
+ 49, 9, 51, 60, 20, 133, 69, 56, 57, 133,
+ 71, 134, 133, 421, 63, 132, 83, 66, 83, 132,
+ 428, 120, 60, 132, 121, 7, 95, 95, 82, 413,
+ 79, 80, 7, 95, 7, 40, 420, 73, 87, 132,
+ 132, 90, 5, 27, 134, 90, 125, 90, 132, 83,
+ 132, 132, 98, 102, 98, 94, 440, 441, 466, 132,
+ 36, 110, 111, 96, 7, 114, 115, 93, 132, 40,
+ 454, 86, 456, 122, 134, 459, 5, 132, 127, 4,
+ 5, 6, 7, 132, 133, 10, 40, 7, 91, 103,
+ 474, 133, 133, 97, 478, 132, 21, 22, 7, 134,
+ 132, 172, 321, 456, 441, 107, 222, 247, 33, 34,
+ 140, 349, 466, 144, 173, 248, 41, 42, 104, 126,
+ 149, 131, 394, 48, 49, 411, 51, -1, -1, -1,
+ 186, -1, 57, 186, -1, -1, -1, 62, 63, -1,
+ -1, 66, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, 79, 80, -1, -1, -1, -1,
+ -1, -1, 87, -1, -1, 90, 4, 5, 6, 7,
+ -1, -1, 10, -1, -1, -1, -1, 102, -1, -1,
+ -1, -1, 107, 21, 22, 110, 111, -1, -1, 114,
+ 115, -1, -1, -1, -1, 33, 34, 122, -1, -1,
+ -1, -1, 127, 41, 42, -1, -1, 132, -1, -1,
+ 48, 49, -1, 51, -1, -1, -1, -1, -1, 57,
+ -1, -1, -1, -1, 62, 63, -1, -1, 66, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, 78, 79, -1, -1, -1, -1, -1, -1, 86,
- -1, -1, 89, -1, -1, -1, -1, -1, -1, -1,
- 4, 5, 6, 7, 101, -1, -1, -1, -1, -1,
- -1, -1, 109, -1, -1, 112, 113, 21, 22, 23,
- -1, -1, -1, 120, -1, -1, -1, -1, 125, 33,
- 34, -1, -1, 130, -1, -1, -1, 41, 42, -1,
- -1, -1, -1, -1, 48, 49, -1, 51, -1, -1,
- -1, -1, -1, 57, -1, -1, -1, -1, -1, 63,
- -1, -1, 66, -1, -1, -1, 4, 5, 6, 7,
- -1, -1, -1, -1, 78, 79, -1, -1, -1, -1,
- -1, -1, 86, 21, 22, 89, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, 33, 34, 101, -1, -1,
- -1, -1, -1, 41, 42, 109, -1, -1, 112, 113,
- 48, 49, -1, 51, -1, -1, 120, -1, -1, 57,
- -1, 125, -1, -1, -1, 63, 130, -1, 66, -1,
- -1, -1, 4, 5, 6, 7, -1, -1, -1, -1,
- 78, 79, -1, -1, -1, -1, -1, -1, 86, 21,
- 22, 89, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, 33, 34, 101, -1, -1, -1, -1, -1, 41,
- 42, 109, -1, -1, 112, 113, 48, 49, -1, 51,
- -1, -1, 120, 121, -1, 57, -1, 125, -1, -1,
- -1, 63, 130, -1, 66, -1, -1, -1, 4, 5,
- 6, 7, -1, -1, -1, -1, 78, 79, -1, -1,
- -1, -1, -1, -1, 86, 21, 22, 89, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, 33, 34, 101,
- -1, -1, -1, -1, 106, 41, 42, 109, -1, -1,
- 112, 113, 48, 49, -1, 51, -1, -1, 120, -1,
- -1, 57, -1, 125, -1, -1, 62, 63, 130, -1,
- 66, -1, -1, -1, 4, 5, 6, 7, -1, -1,
- -1, -1, 78, 79, -1, -1, -1, -1, -1, -1,
- 86, 21, 22, 89, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, 33, 34, 101, -1, -1, -1, -1,
- -1, 41, 42, 109, -1, -1, 112, 113, 48, 49,
- -1, 51, -1, -1, 120, -1, -1, 57, -1, 125,
- -1, -1, -1, 63, 130, -1, 66, -1, -1, -1,
- 4, 5, 6, 7, -1, -1, -1, -1, 78, 79,
- -1, -1, 1, -1, 3, -1, 86, 21, 22, 89,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, 33,
- 34, 101, -1, -1, -1, -1, -1, 41, 42, 109,
- -1, 30, 112, 113, 48, 49, -1, 51, -1, -1,
- 120, -1, -1, 57, -1, 125, -1, 46, 47, 63,
- 130, -1, 66, -1, 53, -1, -1, -1, -1, 58,
- -1, -1, -1, -1, 78, 79, -1, -1, -1, -1,
- -1, -1, 86, -1, -1, 89, -1, -1, 77, -1,
- -1, -1, -1, -1, -1, -1, 5, 101, 7, -1,
- -1, -1, -1, -1, -1, 109, -1, -1, 112, 113,
- 99, -1, 21, 22, 33, 34, 120, 106, -1, -1,
- -1, 125, -1, 42, 33, 34, 130, -1, 117, 48,
- 49, -1, 51, 42, 123, -1, -1, 126, 57, 48,
- 49, -1, 51, -1, -1, -1, -1, 66, 57, -1,
- -1, -1, -1, -1, -1, -1, -1, 66, -1, 78,
- 79, -1, -1, -1, -1, -1, -1, 86, -1, 78,
- 79, -1, -1, -1, -1, -1, -1, 86, -1, -1,
- 89, -1, 101, -1, -1, -1, -1, -1, -1, -1,
- 109, -1, 101, 112, 113, -1, -1, -1, -1, -1,
- 109, 120, -1, 112, 113, -1, 125, -1, -1, -1,
- -1, 120, -1, -1, -1, -1, 125, 10, 11, 12,
- 13, 14, 15, 16, 17, 18, 19, -1, 21, 22,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, 74
+ -1, 79, 80, -1, -1, -1, -1, -1, -1, 87,
+ -1, -1, 90, -1, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 102, 21, 22, -1, 4, 5,
+ 6, 7, 110, 111, -1, -1, 114, 115, -1, -1,
+ -1, -1, -1, -1, 122, 21, 22, 23, -1, 127,
+ -1, -1, -1, -1, 132, -1, -1, 33, 34, -1,
+ -1, -1, -1, -1, -1, 41, 42, -1, -1, -1,
+ -1, -1, 48, 49, -1, 51, -1, -1, -1, 75,
+ -1, 57, -1, 4, 5, 6, 7, 63, -1, -1,
+ 66, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ 21, 22, -1, 79, 80, -1, -1, -1, -1, -1,
+ -1, 87, 33, 34, 90, -1, -1, -1, -1, -1,
+ 41, 42, -1, -1, -1, -1, 102, 48, 49, -1,
+ 51, -1, -1, -1, 110, 111, 57, -1, 114, 115,
+ -1, -1, 63, -1, -1, 66, 122, -1, -1, -1,
+ -1, 127, -1, -1, -1, -1, 132, -1, 79, 80,
+ -1, -1, -1, -1, -1, -1, 87, -1, -1, 90,
+ 4, 5, 6, 7, -1, -1, -1, -1, -1, -1,
+ -1, 102, -1, -1, -1, -1, -1, 21, 22, 110,
+ 111, -1, -1, 114, 115, -1, -1, -1, -1, 33,
+ 34, 122, 123, -1, -1, -1, 127, 41, 42, -1,
+ -1, 132, -1, -1, 48, 49, -1, 51, -1, -1,
+ -1, -1, -1, 57, -1, 4, 5, 6, 7, 63,
+ -1, -1, 66, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, 21, 22, -1, 79, 80, -1, -1, -1,
+ -1, -1, -1, 87, 33, 34, 90, -1, -1, -1,
+ -1, -1, 41, 42, -1, -1, -1, -1, 102, 48,
+ 49, -1, 51, 107, -1, -1, 110, 111, 57, -1,
+ 114, 115, -1, 62, 63, -1, -1, 66, 122, -1,
+ -1, -1, -1, 127, -1, -1, -1, -1, 132, -1,
+ 79, 80, -1, -1, -1, -1, -1, -1, 87, -1,
+ -1, 90, 4, 5, 6, 7, -1, -1, -1, -1,
+ -1, -1, -1, 102, -1, -1, -1, -1, -1, 21,
+ 22, 110, 111, -1, -1, 114, 115, -1, -1, -1,
+ -1, 33, 34, 122, -1, -1, -1, -1, 127, 41,
+ 42, -1, -1, 132, -1, -1, 48, 49, -1, 51,
+ -1, -1, -1, -1, -1, 57, -1, 4, 5, 6,
+ 7, 63, -1, -1, 66, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, 21, 22, -1, 79, 80, -1,
+ -1, -1, -1, -1, -1, 87, 33, 34, 90, -1,
+ -1, -1, -1, -1, 41, 42, -1, -1, -1, -1,
+ 102, 48, 49, -1, 51, -1, -1, -1, 110, 111,
+ 57, -1, 114, 115, -1, -1, 63, -1, -1, 66,
+ 122, -1, -1, -1, -1, 127, -1, -1, -1, -1,
+ 132, -1, 79, 80, -1, -1, -1, -1, -1, -1,
+ 87, -1, -1, 90, 5, -1, 7, -1, -1, -1,
+ -1, -1, -1, -1, -1, 102, -1, -1, -1, -1,
+ 21, 22, -1, 110, 111, -1, -1, 114, 115, -1,
+ -1, -1, 33, 34, -1, 122, -1, -1, -1, -1,
+ 127, 42, -1, -1, -1, 132, -1, 48, 49, -1,
+ 51, 33, 34, -1, -1, -1, 57, -1, -1, -1,
+ 42, -1, -1, -1, -1, 66, 48, 49, -1, 51,
+ -1, -1, -1, -1, -1, 57, -1, -1, 79, 80,
+ -1, -1, -1, -1, 66, -1, 87, -1, -1, 90,
+ -1, -1, -1, -1, -1, -1, -1, 79, 80, -1,
+ -1, 102, -1, -1, -1, 87, -1, -1, -1, 110,
+ -1, -1, -1, 114, 115, -1, -1, -1, -1, -1,
+ 102, 122, -1, -1, -1, -1, 127, -1, 110, -1,
+ -1, -1, 114, 115, -1, -1, -1, -1, -1, -1,
+ 122, -1, -1, -1, -1, 127
};
/* YYSTOS[STATE-NUM] -- The (internal number of the) accessing
symbol of state STATE-NUM. */
static const yytype_uint8 yystos[] =
{
- 0, 1, 3, 30, 46, 47, 53, 58, 77, 99,
- 106, 117, 123, 126, 135, 136, 137, 138, 139, 140,
- 141, 162, 163, 166, 167, 170, 171, 174, 227, 228,
- 110, 4, 6, 225, 75, 110, 68, 110, 80, 29,
- 56, 175, 225, 172, 173, 186, 225, 0, 126, 128,
- 77, 174, 126, 129, 3, 225, 68, 225, 225, 225,
- 225, 225, 5, 7, 21, 22, 23, 33, 34, 41,
- 42, 48, 49, 51, 57, 63, 66, 78, 79, 86,
- 89, 101, 109, 112, 113, 120, 125, 130, 144, 176,
- 177, 178, 180, 202, 203, 204, 205, 206, 207, 208,
- 215, 218, 221, 225, 107, 132, 31, 130, 80, 28,
- 58, 5, 92, 130, 122, 196, 197, 119, 130, 174,
- 7, 7, 121, 202, 211, 212, 130, 79, 130, 5,
- 130, 79, 174, 202, 5, 68, 179, 132, 21, 22,
- 31, 222, 225, 23, 24, 133, 223, 22, 205, 27,
- 130, 168, 169, 225, 173, 130, 180, 224, 225, 225,
- 44, 45, 44, 45, 123, 164, 225, 142, 143, 225,
- 10, 62, 130, 198, 199, 200, 201, 202, 218, 130,
- 224, 198, 121, 209, 210, 59, 212, 213, 7, 50,
- 73, 87, 88, 105, 124, 216, 216, 7, 131, 131,
- 130, 180, 183, 184, 187, 206, 225, 196, 178, 225,
- 203, 204, 225, 23, 56, 131, 202, 214, 132, 196,
- 11, 131, 132, 174, 142, 43, 67, 98, 116, 148,
- 225, 225, 130, 130, 151, 131, 132, 79, 144, 201,
- 180, 198, 202, 8, 9, 10, 11, 12, 13, 14,
- 15, 16, 17, 18, 19, 74, 220, 20, 215, 217,
- 131, 111, 202, 210, 213, 202, 60, 131, 68, 131,
- 183, 31, 185, 186, 69, 76, 84, 103, 182, 132,
- 185, 37, 115, 181, 70, 188, 131, 214, 131, 132,
- 169, 202, 225, 130, 82, 82, 130, 54, 61, 165,
- 218, 219, 225, 118, 148, 149, 150, 142, 10, 43,
- 52, 89, 98, 102, 116, 145, 146, 147, 131, 199,
- 200, 17, 18, 19, 74, 202, 130, 180, 202, 10,
- 89, 131, 132, 119, 202, 111, 60, 202, 131, 186,
- 94, 94, 94, 81, 183, 7, 7, 185, 40, 72,
- 189, 131, 202, 198, 130, 130, 224, 5, 64, 91,
- 92, 114, 226, 131, 132, 131, 132, 35, 38, 39,
- 108, 160, 132, 123, 152, 89, 130, 215, 82, 225,
- 145, 202, 130, 180, 9, 214, 89, 215, 130, 202,
- 131, 184, 97, 97, 214, 198, 93, 190, 131, 224,
- 224, 131, 54, 61, 218, 130, 161, 148, 36, 95,
- 153, 198, 130, 9, 214, 202, 131, 217, 92, 40,
- 85, 191, 131, 131, 5, 226, 155, 156, 157, 158,
- 159, 225, 130, 40, 131, 225, 202, 131, 131, 198,
- 192, 193, 202, 7, 102, 131, 132, 7, 29, 130,
- 225, 155, 71, 100, 154, 131, 132, 32, 55, 194,
- 225, 156, 224, 131, 130, 193, 90, 195, 130, 131,
- 224, 65, 83, 224, 131, 131, 96, 7
+ 0, 1, 3, 30, 46, 47, 53, 58, 78, 100,
+ 107, 119, 125, 128, 137, 138, 139, 140, 141, 142,
+ 143, 164, 165, 168, 169, 172, 173, 176, 230, 231,
+ 112, 4, 6, 228, 76, 112, 69, 112, 81, 29,
+ 56, 177, 228, 174, 175, 188, 228, 0, 128, 130,
+ 78, 176, 128, 131, 3, 228, 69, 228, 228, 228,
+ 228, 228, 5, 7, 21, 22, 23, 33, 34, 41,
+ 42, 48, 49, 51, 57, 63, 66, 79, 80, 87,
+ 90, 102, 110, 111, 114, 115, 122, 127, 132, 146,
+ 178, 179, 180, 182, 204, 205, 206, 207, 208, 209,
+ 210, 211, 218, 221, 224, 228, 108, 134, 31, 132,
+ 81, 28, 58, 5, 93, 132, 124, 198, 199, 121,
+ 132, 176, 7, 7, 123, 204, 214, 215, 132, 80,
+ 132, 5, 132, 132, 80, 176, 204, 5, 69, 181,
+ 134, 21, 22, 31, 225, 228, 23, 24, 135, 226,
+ 22, 207, 27, 132, 170, 171, 228, 175, 132, 182,
+ 227, 228, 228, 44, 45, 44, 45, 125, 166, 228,
+ 144, 145, 228, 10, 62, 132, 200, 201, 202, 203,
+ 204, 221, 132, 227, 200, 123, 212, 213, 59, 215,
+ 216, 7, 50, 74, 88, 89, 106, 126, 219, 219,
+ 204, 7, 133, 133, 132, 182, 185, 186, 189, 208,
+ 228, 198, 180, 228, 205, 206, 228, 23, 56, 133,
+ 204, 217, 134, 198, 11, 133, 134, 176, 144, 43,
+ 68, 99, 118, 150, 228, 228, 132, 132, 153, 133,
+ 134, 80, 146, 203, 182, 200, 204, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18, 19, 75,
+ 223, 20, 218, 220, 133, 113, 204, 213, 216, 204,
+ 60, 133, 69, 69, 133, 185, 31, 187, 188, 70,
+ 77, 85, 104, 184, 134, 187, 37, 117, 183, 71,
+ 190, 133, 217, 133, 134, 171, 204, 228, 132, 83,
+ 83, 132, 54, 61, 167, 221, 222, 228, 120, 150,
+ 151, 152, 144, 10, 43, 52, 90, 99, 103, 118,
+ 147, 148, 149, 133, 201, 202, 17, 18, 19, 75,
+ 204, 132, 182, 204, 10, 90, 133, 134, 121, 204,
+ 113, 60, 204, 7, 133, 188, 95, 95, 95, 82,
+ 185, 7, 7, 187, 40, 73, 191, 133, 204, 200,
+ 132, 132, 227, 5, 64, 92, 93, 116, 229, 133,
+ 134, 133, 134, 35, 38, 39, 109, 162, 134, 125,
+ 154, 90, 132, 218, 83, 228, 147, 204, 132, 182,
+ 9, 217, 90, 218, 132, 204, 133, 67, 133, 186,
+ 98, 98, 217, 200, 94, 192, 133, 227, 227, 133,
+ 54, 61, 221, 132, 163, 150, 36, 96, 155, 200,
+ 132, 9, 217, 204, 133, 220, 7, 93, 40, 86,
+ 193, 133, 133, 5, 229, 157, 158, 159, 160, 161,
+ 228, 132, 40, 133, 228, 204, 133, 133, 133, 200,
+ 194, 195, 204, 7, 103, 133, 134, 7, 29, 132,
+ 228, 157, 72, 101, 156, 133, 134, 32, 55, 196,
+ 228, 158, 227, 133, 132, 195, 91, 197, 132, 133,
+ 227, 65, 84, 227, 133, 133, 97, 7
};
/* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */
static const yytype_uint8 yyr1[] =
{
- 0, 134, 135, 135, 135, 135, 135, 135, 136, 136,
- 136, 136, 136, 136, 136, 136, 136, 136, 137, 138,
- 138, 138, 138, 139, 140, 141, 142, 143, 143, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 145, 145, 145,
- 145, 145, 145, 145, 146, 146, 147, 147, 148, 148,
- 148, 148, 149, 149, 150, 150, 151, 151, 152, 152,
- 153, 153, 154, 154, 155, 155, 156, 156, 156, 157,
- 157, 158, 159, 160, 160, 160, 160, 161, 161, 162,
- 162, 162, 162, 163, 164, 164, 165, 165, 165, 165,
- 166, 167, 168, 168, 169, 170, 170, 171, 172, 172,
- 173, 174, 175, 175, 175, 176, 176, 177, 177, 178,
- 178, 178, 179, 180, 181, 181, 181, 182, 182, 182,
- 182, 182, 182, 182, 182, 183, 183, 184, 184, 184,
- 184, 184, 184, 185, 185, 186, 186, 187, 187, 188,
- 188, 189, 189, 190, 190, 191, 191, 192, 192, 193,
- 194, 194, 194, 195, 195, 195, 196, 196, 197, 198,
- 198, 199, 199, 200, 200, 201, 201, 201, 201, 201,
- 201, 201, 201, 201, 201, 201, 202, 202, 203, 203,
- 204, 204, 205, 205, 205, 205, 205, 205, 205, 206,
- 206, 206, 206, 207, 208, 208, 209, 209, 210, 211,
- 211, 212, 213, 213, 214, 214, 215, 215, 215, 215,
- 215, 215, 215, 215, 216, 216, 216, 216, 216, 216,
- 217, 217, 218, 218, 219, 219, 220, 220, 220, 220,
- 220, 220, 220, 220, 220, 220, 221, 222, 222, 223,
- 223, 223, 224, 224, 225, 225, 226, 226, 226, 226,
- 227, 228, 228
+ 0, 136, 137, 137, 137, 137, 137, 137, 138, 138,
+ 138, 138, 138, 138, 138, 138, 138, 138, 139, 140,
+ 140, 140, 140, 141, 142, 143, 144, 145, 145, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 147, 147, 147,
+ 147, 147, 147, 147, 148, 148, 149, 149, 150, 150,
+ 150, 150, 151, 151, 152, 152, 153, 153, 154, 154,
+ 155, 155, 156, 156, 157, 157, 158, 158, 158, 159,
+ 159, 160, 161, 162, 162, 162, 162, 163, 163, 164,
+ 164, 164, 164, 165, 166, 166, 167, 167, 167, 167,
+ 168, 169, 170, 170, 171, 172, 172, 173, 174, 174,
+ 175, 176, 177, 177, 177, 178, 178, 179, 179, 180,
+ 180, 180, 181, 182, 183, 183, 183, 184, 184, 184,
+ 184, 184, 184, 184, 184, 185, 185, 186, 186, 186,
+ 186, 186, 186, 187, 187, 188, 188, 189, 189, 190,
+ 190, 191, 191, 192, 192, 193, 193, 194, 194, 195,
+ 196, 196, 196, 197, 197, 197, 198, 198, 199, 200,
+ 200, 201, 201, 202, 202, 203, 203, 203, 203, 203,
+ 203, 203, 203, 203, 203, 203, 204, 204, 205, 205,
+ 206, 206, 207, 207, 207, 207, 207, 207, 207, 207,
+ 208, 208, 208, 208, 209, 210, 210, 211, 211, 212,
+ 212, 213, 214, 214, 215, 216, 216, 217, 217, 218,
+ 218, 218, 218, 218, 218, 218, 218, 219, 219, 219,
+ 219, 219, 219, 220, 220, 221, 221, 222, 222, 223,
+ 223, 223, 223, 223, 223, 223, 223, 223, 223, 224,
+ 225, 225, 226, 226, 226, 227, 227, 228, 228, 229,
+ 229, 229, 229, 230, 231, 231
};
/* YYR2[YYN] -- Number of symbols on the right hand side of rule YYN. */
@@ -1381,14 +1391,14 @@ static const yytype_uint8 yyr2[] =
0, 1, 1, 0, 2, 2, 0, 1, 2, 3,
1, 3, 1, 2, 1, 5, 6, 4, 3, 3,
3, 2, 3, 5, 4, 6, 3, 1, 3, 1,
- 2, 1, 1, 1, 1, 1, 1, 3, 1, 3,
- 4, 4, 5, 6, 5, 4, 1, 2, 4, 1,
- 2, 4, 0, 2, 1, 3, 1, 1, 2, 2,
- 1, 2, 3, 2, 1, 1, 1, 1, 1, 1,
- 1, 3, 1, 3, 1, 3, 1, 1, 1, 1,
- 1, 1, 1, 2, 1, 2, 1, 1, 1, 1,
- 1, 1, 1, 3, 1, 1, 1, 1, 1, 1,
- 2, 2, 0
+ 2, 1, 1, 1, 1, 1, 1, 1, 3, 1,
+ 3, 4, 4, 5, 6, 6, 8, 5, 4, 1,
+ 2, 4, 1, 2, 4, 0, 2, 1, 3, 1,
+ 1, 2, 2, 1, 2, 3, 2, 1, 1, 1,
+ 1, 1, 1, 1, 3, 1, 3, 1, 3, 1,
+ 1, 1, 1, 1, 1, 1, 2, 1, 2, 1,
+ 1, 1, 1, 1, 1, 1, 3, 1, 1, 1,
+ 1, 1, 1, 2, 2, 0
};
@@ -1885,603 +1895,593 @@ yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep, YYLTYPE *yylocatio
switch (yytype)
{
case 3: /* TOKEN_COMMAND */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
}
-#line 1895 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1905 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
case 4: /* TOKEN_NAME */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
}
-#line 1905 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1915 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
case 5: /* TOKEN_STRING_SINGLE_QUOTED */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
}
-#line 1915 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1925 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
case 6: /* TOKEN_STRING_DOUBLE_QUOTED */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
}
-#line 1925 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1935 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
case 7: /* TOKEN_UNSIGNED_NUMVAL */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).numeric_literal_value_) != nullptr) {
delete ((*yyvaluep).numeric_literal_value_);
}
}
-#line 1935 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1945 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 136: /* sql_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 138: /* sql_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).statement_) != nullptr) {
delete ((*yyvaluep).statement_);
}
}
-#line 1945 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1955 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 137: /* quit_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 139: /* quit_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).quit_statement_) != nullptr) {
delete ((*yyvaluep).quit_statement_);
}
}
-#line 1955 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1965 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 138: /* alter_table_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 140: /* alter_table_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).statement_) != nullptr) {
delete ((*yyvaluep).statement_);
}
}
-#line 1965 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1975 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 139: /* create_table_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 141: /* create_table_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).create_table_statement_) != nullptr) {
delete ((*yyvaluep).create_table_statement_);
}
}
-#line 1975 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1985 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 140: /* create_index_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 142: /* create_index_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).statement_) != nullptr) {
delete ((*yyvaluep).statement_);
}
}
-#line 1985 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1995 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 141: /* drop_table_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 143: /* drop_table_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).drop_table_statement_) != nullptr) {
delete ((*yyvaluep).drop_table_statement_);
}
}
-#line 1995 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2005 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 142: /* column_def */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 144: /* column_def */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).attribute_definition_) != nullptr) {
delete ((*yyvaluep).attribute_definition_);
}
}
-#line 2005 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2015 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 143: /* column_def_commalist */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 145: /* column_def_commalist */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).attribute_definition_list_) != nullptr) {
delete ((*yyvaluep).attribute_definition_list_);
}
}
-#line 2015 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2025 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 144: /* data_type */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 146: /* data_type */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).data_type_) != nullptr) {
delete ((*yyvaluep).data_type_);
}
}
-#line 2025 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2035 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 145: /* column_constraint_def */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 147: /* column_constraint_def */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).column_constraint_) != nullptr) {
delete ((*yyvaluep).column_constraint_);
}
}
-#line 2035 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2045 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 146: /* column_constraint_def_list */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 148: /* column_constraint_def_list */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).column_constraint_list_) != nullptr) {
delete ((*yyvaluep).column_constraint_list_);
}
}
-#line 2045 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2055 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 147: /* opt_column_constraint_def_list */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 149: /* opt_column_constraint_def_list */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).column_constraint_list_) != nullptr) {
delete ((*yyvaluep).column_constraint_list_);
}
}
-#line 2055 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2065 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 151: /* opt_column_list */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 153: /* opt_column_list */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).attribute_list_) != nullptr) {
delete ((*yyvaluep).attribute_list_);
}
}
-#line 2065 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2075 "SqlParser_gen.cpp" /*
<TRUNCATED>
[07/50] [abbrv] incubator-quickstep git commit: Set block slots
correctly. (#202)
Posted by hb...@apache.org.
Set block slots correctly. (#202)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/abfc5f27
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/abfc5f27
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/abfc5f27
Branch: refs/heads/query-manager-used-in-foreman
Commit: abfc5f275c500d40962dac70a60a4cd5881b77bc
Parents: 5b75e8e
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Fri Apr 29 12:01:19 2016 -0700
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Fri Apr 29 12:01:19 2016 -0700
----------------------------------------------------------------------
query_optimizer/resolver/Resolver.cpp | 13 +++++++++----
query_optimizer/tests/physical_generator/Create.test | 6 +++---
query_optimizer/tests/resolver/Create.test | 15 ++++++++++++---
storage/StorageConstants.hpp | 3 +--
4 files changed, 25 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/abfc5f27/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index 45ecf33..2667ee9 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -602,16 +602,21 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
// Resolve the Block size (size -> # of slots).
std::int64_t slots = kDefaultBlockSizeInSlots;
if (block_properties->hasBlockSizeMb()) {
- std::int64_t blocksizemb = block_properties->getBlockSizeMbValue();
- if (blocksizemb == -1) {
+ const std::int64_t block_size_in_mega_bytes = block_properties->getBlockSizeMbValue();
+ if (block_size_in_mega_bytes == -1) {
// Indicates an error condition if the property is present but getter returns -1.
THROW_SQL_ERROR_AT(block_properties->getBlockSizeMb())
<< "The BLOCKSIZEMB property must be an integer.";
+ } else if ((block_size_in_mega_bytes * kAMegaByte) % kSlotSizeBytes != 0) {
+ THROW_SQL_ERROR_AT(block_properties->getBlockSizeMb())
+ << "The BLOCKSIZEMB property must be multiple times of "
+ << std::to_string(kSlotSizeBytes / kAMegaByte) << "MB.";
}
- slots = (blocksizemb * kAMegaByte) / kSlotSizeBytes;
+
+ slots = (block_size_in_mega_bytes * kAMegaByte) / kSlotSizeBytes;
DLOG(INFO) << "Resolver using BLOCKSIZEMB of " << slots << " slots"
<< " which is " << (slots * kSlotSizeBytes) << " bytes versus"
- << " user requested " << (blocksizemb * kAMegaByte) << " bytes.";
+ << " user requested " << (block_size_in_mega_bytes * kAMegaByte) << " bytes.";
const std::uint64_t max_size_slots = kBlockSizeUpperBoundBytes / kSlotSizeBytes;
const std::uint64_t min_size_slots = kBlockSizeLowerBoundBytes / kSlotSizeBytes;
if (static_cast<std::uint64_t>(slots) < min_size_slots ||
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/abfc5f27/query_optimizer/tests/physical_generator/Create.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/physical_generator/Create.test b/query_optimizer/tests/physical_generator/Create.test
index 58e15fa..8e6c64b 100644
--- a/query_optimizer/tests/physical_generator/Create.test
+++ b/query_optimizer/tests/physical_generator/Create.test
@@ -1,5 +1,5 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
-# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2015-2016 Pivotal Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -86,8 +86,8 @@ TopLevelPlan
+-AttributeReference[id=10,name=col11,relation=foo,type=Char(5) NULL]
+-AttributeReference[id=11,name=col12,relation=foo,type=VarChar(5) NULL]
==
-CREATE TABLE foo (col1 INT) WITH BLOCKPROPERTIES
- (TYPE compressed_columnstore, SORT col1, COMPRESS ALL, BLOCKSIZEMB 5);
+CREATE TABLE foo (col1 INT) WITH BLOCKPROPERTIES
+ (TYPE compressed_columnstore, SORT col1, COMPRESS ALL, BLOCKSIZEMB 4);
--
[Optimized Logical Plan]
TopLevelPlan
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/abfc5f27/query_optimizer/tests/resolver/Create.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/resolver/Create.test b/query_optimizer/tests/resolver/Create.test
index 18beacd..63f7ac9 100644
--- a/query_optimizer/tests/resolver/Create.test
+++ b/query_optimizer/tests/resolver/Create.test
@@ -1,5 +1,5 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
-# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2015-2016 Pivotal Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -123,7 +123,7 @@ BLOCKPROPERTIES (BLOCKSIZEMB 1...
==
# Rowstores cannot have a sorted attribute.
-CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
+CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
(TYPE rowstore, SORT attr);
--
ERROR: The SORT property does not apply to this block type. (2 : 22)
@@ -167,7 +167,7 @@ ERROR: The COMPRESS property does not apply to this block type. (2 : 7)
==
# Compress property is required for compressed blocks.
-CREATE TABLE foo (attr INT) WITH
+CREATE TABLE foo (attr INT) WITH
BLOCKPROPERTIES (TYPE compressed_rowstore);
--
ERROR: The COMPRESS property must be specified as ALL or a list of attributes. (2 : 1)
@@ -202,6 +202,15 @@ ERROR: The BLOCKSIZEMB property must be an integer. (2 : 17)
^
==
+# BLOCKSIZEMB property must be multiple times of the slot size.
+CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
+(TYPE rowstore, BLOCKSIZEMB 25);
+--
+ERROR: The BLOCKSIZEMB property must be multiple times of 2MB. (2 : 17)
+(TYPE rowstore, BLOCKSIZEMB 25);
+ ^
+==
+
# BLOCKSIZEMB must be greater than the minimum (defined in StorageConstants.hpp).
CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
(TYPE rowstore, BLOCKSIZEMB 0);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/abfc5f27/storage/StorageConstants.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageConstants.hpp b/storage/StorageConstants.hpp
index de54345..154d2f7 100644
--- a/storage/StorageConstants.hpp
+++ b/storage/StorageConstants.hpp
@@ -44,8 +44,7 @@ const std::uint64_t kAMegaByte = (1 << 20);
// the SQL clause BLOCKPROPERTIES.
const std::uint64_t kBlockSizeUpperBoundBytes = kAGigaByte;
-// 2 Megabytes.
-const std::uint64_t kBlockSizeLowerBoundBytes = kAMegaByte << 1;
+const std::uint64_t kBlockSizeLowerBoundBytes = kSlotSizeBytes;
// The default size of a new relation in terms of the number of slots.
const std::uint64_t kDefaultBlockSizeInSlots = 1;
[13/50] [abbrv] incubator-quickstep git commit: Change default
aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining (#207)
Posted by hb...@apache.org.
Change default aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining (#207)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8444e2dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8444e2dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8444e2dc
Branch: refs/heads/query-manager-used-in-foreman
Commit: 8444e2dc85739170f2c00d378513ab71573fb4c1
Parents: 0f261ea
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Wed May 4 13:54:55 2016 -0500
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Wed May 4 13:54:55 2016 -0500
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8444e2dc/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index c34f084..3698701 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -135,7 +135,7 @@ static const volatile bool join_hashtable_type_dummy
= gflags::RegisterFlagValidator(&FLAGS_join_hashtable_type,
&ValidateHashTableImplTypeString);
-DEFINE_string(aggregate_hashtable_type, "LinearOpenAddressing",
+DEFINE_string(aggregate_hashtable_type, "SeparateChaining",
"HashTable implementation to use for aggregates with GROUP BY "
"(valid options are SeparateChaining or LinearOpenAddressing)");
static const volatile bool aggregate_hashtable_type_dummy
[26/50] [abbrv] incubator-quickstep git commit: Added a method for
double parsing with significant number argument. (#216)
Posted by hb...@apache.org.
Added a method for double parsing with significant number argument. (#216)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4937e684
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4937e684
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4937e684
Branch: refs/heads/query-manager-used-in-foreman
Commit: 4937e6840d8a6fbb1034bd9d4ad9ae50b8446e9d
Parents: ab79cf6
Author: Hakan Memisoglu <ha...@gmail.com>
Authored: Sun May 8 08:22:17 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:32 2016 -0700
----------------------------------------------------------------------
utility/StringUtil.cpp | 14 ++++++++++++++
utility/StringUtil.hpp | 18 ++++++++++++++++++
2 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4937e684/utility/StringUtil.cpp
----------------------------------------------------------------------
diff --git a/utility/StringUtil.cpp b/utility/StringUtil.cpp
index 77b0a3a..b38e8f4 100644
--- a/utility/StringUtil.cpp
+++ b/utility/StringUtil.cpp
@@ -22,10 +22,13 @@
#include <algorithm>
#include <cctype>
#include <cinttypes>
+#include <cmath>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
+#include <iomanip>
+#include <sstream>
#include <string>
#include <vector>
@@ -128,4 +131,15 @@ string ToZeroPaddedString(std::uint64_t val, int pad_width) {
return result;
}
+std::string
+DoubleToStringWithSignificantDigits(double val,
+ std::uint64_t significant_digits) {
+ std::uint64_t precision_needed =
+ (val >= 1.0) ? significant_digits
+ : significant_digits + -(std::trunc(std::log10(val)));
+ std::stringstream string_buffer;
+ string_buffer << std::fixed << std::setprecision(precision_needed) << val;
+ return string_buffer.str();
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4937e684/utility/StringUtil.hpp
----------------------------------------------------------------------
diff --git a/utility/StringUtil.hpp b/utility/StringUtil.hpp
index 26c04a4..bd138bb 100644
--- a/utility/StringUtil.hpp
+++ b/utility/StringUtil.hpp
@@ -70,6 +70,24 @@ extern bool ParseIntString(const std::string &int_string,
*/
extern std::string ToZeroPaddedString(std::uint64_t val, int pad_width);
+
+/**
+ * @brief Parse double with significant digits.
+ *
+ * @param val The value that will be parsed.
+ * @param significant_digit The number of non-zero digits that will be parsed
+ * in the fraction part of double.
+ * @return Corresponding string representation of parsed double.
+ * Ex:
+ * DoubleToStringWithSignificantDigits(3.124355123, 3)
+ * => 3.124
+ * DoubleToStringWithSignificantDigits(0.00000000323411, 3)
+ * => 0.00000000323
+ **/
+extern std::string
+DoubleToStringWithSignificantDigits(double val,
+ std::uint64_t significant_digits);
+
/** @} */
} // namespace quickstep
[15/50] [abbrv] incubator-quickstep git commit: Transaction Part 4:
LockManager, CycleDetector and DeadLockDetector. (#187)
Posted by hb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/StronglyConnectedComponents_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/StronglyConnectedComponents_unittest.cpp b/transaction/tests/StronglyConnectedComponents_unittest.cpp
index 79d6881..35ef842 100644
--- a/transaction/tests/StronglyConnectedComponents_unittest.cpp
+++ b/transaction/tests/StronglyConnectedComponents_unittest.cpp
@@ -35,21 +35,20 @@ namespace transaction {
class GraphConfiguration {
public:
GraphConfiguration(DirectedGraph *graph,
- std::size_t no_transaction,
+ const std::size_t num_transactions,
const std::vector<std::pair<transaction_id,
transaction_id>> &mapping)
: graph_(graph) {
- for (std::size_t index = 0; index < no_transaction; ++index) {
- std::unique_ptr<transaction_id> tid =
- std::make_unique<transaction_id>(transaction_id(index));
- transaction_list_.push_back(*tid);
- DirectedGraph::node_id nid = graph->addNodeUnchecked(tid.release());
+ for (std::size_t index = 0; index < num_transactions; ++index) {
+ const transaction_id transaction = static_cast<transaction_id>(index);
+ transaction_list_.push_back(transaction);
+ const DirectedGraph::node_id nid = graph->addNodeUnchecked(transaction);
node_id_list_.push_back(nid);
}
for (const std::pair<transaction_id, transaction_id> &edge : mapping) {
- transaction_id pending = edge.first;
- transaction_id owner = edge.second;
+ const transaction_id pending = edge.first;
+ const transaction_id owner = edge.second;
graph_->addEdgeUnchecked(pending, owner);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/TransactionTable_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/TransactionTable_unittest.cpp b/transaction/tests/TransactionTable_unittest.cpp
index f5b5bc9..cd47a2b 100644
--- a/transaction/tests/TransactionTable_unittest.cpp
+++ b/transaction/tests/TransactionTable_unittest.cpp
@@ -41,91 +41,101 @@ class TransactionTableTest : public ::testing::Test {
};
TEST_F(TransactionTableTest, NormalOperations) {
- EXPECT_EQ(transaction_table_.putOwnEntry(tid_1_,
+ const AccessMode is_lock_mode = AccessMode::IsLockMode();
+ const AccessMode x_lock_mode = AccessMode::XLockMode();
+
+ EXPECT_EQ(TransactionTableResult::kPlacedInOwned,
+ transaction_table_.putOwnEntry(tid_1_,
ResourceId(3),
- AccessMode(AccessModeType::kIsLock)),
- TransactionTableResult::kPlacedInOwned);
+ is_lock_mode));
+
- EXPECT_EQ(transaction_table_.putPendingEntry(tid_1_,
+ EXPECT_EQ(TransactionTableResult::kPlacedInPending,
+ transaction_table_.putPendingEntry(tid_1_,
ResourceId(5),
- AccessMode(AccessModeType::kXLock)),
- TransactionTableResult::kPlacedInPending);
+ x_lock_mode));
}
TEST_F(TransactionTableTest, DeleteEntryOperations) {
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_2_,
+ const AccessMode s_lock_mode = AccessMode::SLockMode();
+ const AccessMode x_lock_mode = AccessMode::XLockMode();
+
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deleteOwnEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.putOwnEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kPlacedInOwned,
+ transaction_table_.putOwnEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kPlacedInOwned);
+ s_lock_mode));
// Tring to delete a lock with different acces mode on same resource id
// will result in an error.
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deleteOwnEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kXLock)),
- TransactionTableResult::kDelError);
+ x_lock_mode));
// Transaction 3 does not have a lock on this resource id.
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_3_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deleteOwnEntry(tid_3_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
// This will result in success since transaction 2 have acquired the lock on
// this resource with the corresponding mode.
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kDelFromOwned,
+ transaction_table_.deleteOwnEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelFromOwned);
+ s_lock_mode));
// Repeat the previous sequence, with pending list.
- EXPECT_EQ(transaction_table_.deletePendingEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deletePendingEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.putPendingEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kPlacedInPending,
+ transaction_table_.putPendingEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kPlacedInPending);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.deletePendingEntry(tid_2_,
+
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deletePendingEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kXLock)),
- TransactionTableResult::kDelError);
+ x_lock_mode));
- EXPECT_EQ(transaction_table_.deletePendingEntry(tid_3_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deletePendingEntry(tid_3_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.deletePendingEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kDelFromPending,
+ transaction_table_.deletePendingEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelFromPending);
+ s_lock_mode));
}
TEST_F(TransactionTableTest, TransactionEntries) {
- EXPECT_EQ(transaction_table_.deleteTransaction(tid_1_),
- TransactionTableResult::kTransactionDeleteError);
+ const AccessMode s_lock_mode = AccessMode::SLockMode();
+
+ EXPECT_EQ(TransactionTableResult::kTransactionDeleteError,
+ transaction_table_.deleteTransaction(tid_1_));
- EXPECT_EQ(transaction_table_.putOwnEntry(tid_1_,
+ EXPECT_EQ(TransactionTableResult::kPlacedInOwned,
+ transaction_table_.putOwnEntry(tid_1_,
ResourceId(4),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kPlacedInOwned);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.deleteTransaction(tid_1_),
- TransactionTableResult::kTransactionDeleteOk);
+ EXPECT_EQ(TransactionTableResult::kTransactionDeleteOk,
+ transaction_table_.deleteTransaction(tid_1_));
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_1_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deleteOwnEntry(tid_1_,
ResourceId(4),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
}
} // namespace transaction
[33/50] [abbrv] incubator-quickstep git commit: Initial support for
collecting table statistics: number of distinct values (#227)
Posted by hb...@apache.org.
Initial support for collecting table statistics: number of distinct values (#227)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/df4a05d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/df4a05d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/df4a05d7
Branch: refs/heads/query-manager-used-in-foreman
Commit: df4a05d7ea95cc65c93015e85eaf6edb824816d4
Parents: a25da39
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 19 10:58:43 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:51 2016 -0700
----------------------------------------------------------------------
catalog/CMakeLists.txt | 11 +++
catalog/Catalog.proto | 13 ++-
catalog/CatalogRelation.cpp | 11 +++
catalog/CatalogRelation.hpp | 24 ++++-
catalog/CatalogRelationStatistics.cpp | 49 ++++++++++
catalog/CatalogRelationStatistics.hpp | 122 +++++++++++++++++++++++
cli/CMakeLists.txt | 21 +++-
cli/CommandExecutor.cpp | 138 ++++++++++++++++++++++++++-
cli/CommandExecutor.hpp | 17 ++--
cli/QuickstepCli.cpp | 2 +
cli/tests/CommandExecutorTestRunner.cpp | 2 +
query_optimizer/ExecutionGenerator.cpp | 6 +-
query_optimizer/QueryProcessor.hpp | 10 ++
13 files changed, 410 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/catalog/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/catalog/CMakeLists.txt b/catalog/CMakeLists.txt
index 94da838..64b4f16 100644
--- a/catalog/CMakeLists.txt
+++ b/catalog/CMakeLists.txt
@@ -1,5 +1,7 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015-2016 Pivotal Software, Inc.
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -36,6 +38,9 @@ add_library(quickstep_catalog_CatalogRelation CatalogRelation.cpp CatalogRelatio
add_library(quickstep_catalog_CatalogRelationSchema
CatalogRelationSchema.cpp
CatalogRelationSchema.hpp)
+add_library(quickstep_catalog_CatalogRelationStatistics
+ CatalogRelationStatistics.cpp
+ CatalogRelationStatistics.hpp)
add_library(quickstep_catalog_CatalogTypedefs ../empty_src.cpp CatalogTypedefs.hpp)
add_library(quickstep_catalog_IndexScheme IndexScheme.cpp IndexScheme.hpp)
if(QUICKSTEP_HAVE_LIBNUMA)
@@ -98,6 +103,7 @@ target_link_libraries(quickstep_catalog_CatalogRelation
glog
quickstep_catalog_CatalogAttribute
quickstep_catalog_CatalogRelationSchema
+ quickstep_catalog_CatalogRelationStatistics
quickstep_catalog_CatalogTypedefs
quickstep_catalog_Catalog_proto
quickstep_catalog_IndexScheme
@@ -111,6 +117,10 @@ target_link_libraries(quickstep_catalog_CatalogRelation
quickstep_threading_SpinSharedMutex
quickstep_utility_Macros
quickstep_utility_PtrVector)
+target_link_libraries(quickstep_catalog_CatalogRelationStatistics
+ quickstep_catalog_CatalogTypedefs
+ quickstep_catalog_Catalog_proto
+ quickstep_utility_Macros)
target_link_libraries(quickstep_catalog_IndexScheme
glog
quickstep_catalog_Catalog_proto
@@ -173,6 +183,7 @@ target_link_libraries(quickstep_catalog
quickstep_catalog_CatalogErrors
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogRelationSchema
+ quickstep_catalog_CatalogRelationStatistics
quickstep_catalog_CatalogTypedefs
quickstep_catalog_IndexScheme
quickstep_catalog_PartitionScheme
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/catalog/Catalog.proto
----------------------------------------------------------------------
diff --git a/catalog/Catalog.proto b/catalog/Catalog.proto
index 81e28cf..ce4bc2e 100644
--- a/catalog/Catalog.proto
+++ b/catalog/Catalog.proto
@@ -1,7 +1,7 @@
// Copyright 2011-2015 Quickstep Technologies LLC.
// Copyright 2015-2016 Pivotal Software, Inc.
// Copyright 2016, Quickstep Research Group, Computer Sciences Department,
-// University of Wisconsin\u2014Madison.
+// University of Wisconsin\u2014Madison.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -80,6 +80,16 @@ message IndexScheme {
repeated IndexEntry index_entries = 1;
}
+message CatalogRelationStatistics {
+ optional fixed64 num_tuples = 1;
+
+ message NumDistinctValuesEntry {
+ required int32 attr_id = 1;
+ required fixed64 num_distinct_values = 2;
+ }
+ repeated NumDistinctValuesEntry num_distinct_values_map = 2;
+}
+
message CatalogRelationSchema {
required int32 relation_id = 1;
required string name = 2;
@@ -99,6 +109,7 @@ message CatalogRelation {
optional IndexScheme index_scheme = 18;
optional PartitionScheme partition_scheme = 19;
optional NUMAPlacementScheme placement_scheme = 20;
+ optional CatalogRelationStatistics statistics = 21;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/catalog/CatalogRelation.cpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.cpp b/catalog/CatalogRelation.cpp
index 36f82d9..01aebb5 100644
--- a/catalog/CatalogRelation.cpp
+++ b/catalog/CatalogRelation.cpp
@@ -132,6 +132,14 @@ CatalogRelation::CatalogRelation(const serialization::CatalogRelationSchema &pro
}
default_layout_.reset(new StorageBlockLayout(*this, proto_default_layout));
+
+ if (proto.HasExtension(serialization::CatalogRelation::statistics)) {
+ statistics_.reset(
+ new CatalogRelationStatistics(
+ proto.GetExtension(serialization::CatalogRelation::statistics)));
+ } else {
+ statistics_.reset(new CatalogRelationStatistics());
+ }
}
serialization::CatalogRelationSchema CatalogRelation::getProto() const {
@@ -177,6 +185,9 @@ serialization::CatalogRelationSchema CatalogRelation::getProto() const {
#endif
}
+ proto.MutableExtension(serialization::CatalogRelation::statistics)
+ ->MergeFrom(statistics_->getProto());
+
return proto;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/catalog/CatalogRelation.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.hpp b/catalog/CatalogRelation.hpp
index 312f3b4..e0d5350 100644
--- a/catalog/CatalogRelation.hpp
+++ b/catalog/CatalogRelation.hpp
@@ -29,6 +29,7 @@
#include "catalog/Catalog.pb.h"
#include "catalog/CatalogConfig.h"
#include "catalog/CatalogRelationSchema.hpp"
+#include "catalog/CatalogRelationStatistics.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "catalog/IndexScheme.hpp"
@@ -79,7 +80,8 @@ class CatalogRelation : public CatalogRelationSchema {
const relation_id id = -1,
bool temporary = false)
: CatalogRelationSchema(parent, name, id, temporary),
- default_layout_(nullptr) {
+ default_layout_(nullptr),
+ statistics_(new CatalogRelationStatistics()) {
}
/**
@@ -377,6 +379,24 @@ class CatalogRelation : public CatalogRelationSchema {
* getDefaultStorageBlockLayout().estimateTuplesPerBlock();
}
+ /**
+ * @brief Get an immutable reference to the statistics of this catalog relation.
+ *
+ * @return A reference to the statistics of this catalog relation.
+ */
+ const CatalogRelationStatistics& getStatistics() const {
+ return *statistics_;
+ }
+
+ /**
+ * @brief Get a mutable pointer to the statistics of this catalog relation.
+ *
+ * @return A pointer to the statistics of this catalog relation.
+ */
+ CatalogRelationStatistics* getStatisticsMutable() {
+ return statistics_.get();
+ }
+
private:
// A list of blocks belonged to the relation.
std::vector<block_id> blocks_;
@@ -397,6 +417,8 @@ class CatalogRelation : public CatalogRelationSchema {
// Mutex for locking the index scheme.
alignas(kCacheLineBytes) mutable SpinSharedMutex<false> index_scheme_mutex_;
+ std::unique_ptr<CatalogRelationStatistics> statistics_;
+
#ifdef QUICKSTEP_HAVE_LIBNUMA
// NUMA placement scheme object which has the mapping between the partitions
// of the relation and the NUMA nodes/sockets. It also maintains a mapping
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/catalog/CatalogRelationStatistics.cpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelationStatistics.cpp b/catalog/CatalogRelationStatistics.cpp
new file mode 100644
index 0000000..2bd92b4
--- /dev/null
+++ b/catalog/CatalogRelationStatistics.cpp
@@ -0,0 +1,49 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "catalog/CatalogRelationStatistics.hpp"
+
+#include "catalog/Catalog.pb.h"
+
+namespace quickstep {
+
+CatalogRelationStatistics::CatalogRelationStatistics(
+ const serialization::CatalogRelationStatistics &proto) {
+ if (proto.has_num_tuples()) {
+ num_tuples_ = proto.num_tuples();
+ }
+ for (int i = 0; i < proto.num_distinct_values_map_size(); ++i) {
+ const auto &entry = proto.num_distinct_values_map(i);
+ num_distinct_values_map_.emplace(entry.attr_id(),
+ entry.num_distinct_values());
+ }
+}
+
+serialization::CatalogRelationStatistics CatalogRelationStatistics::getProto() const {
+ serialization::CatalogRelationStatistics proto;
+ if (num_tuples_ != 0) {
+ proto.set_num_tuples(num_tuples_);
+ }
+ for (const auto &pair : num_distinct_values_map_) {
+ auto entry = proto.add_num_distinct_values_map();
+ entry->set_attr_id(pair.first);
+ entry->set_num_distinct_values(pair.second);
+ }
+ return proto;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/catalog/CatalogRelationStatistics.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelationStatistics.hpp b/catalog/CatalogRelationStatistics.hpp
new file mode 100644
index 0000000..572d141
--- /dev/null
+++ b/catalog/CatalogRelationStatistics.hpp
@@ -0,0 +1,122 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_CATALOG_CATALOG_RELATION_STATISTICS_HPP_
+#define QUICKSTEP_CATALOG_CATALOG_RELATION_STATISTICS_HPP_
+
+#include <cstddef>
+#include <unordered_map>
+#include <utility>
+
+#include "catalog/Catalog.pb.h"
+#include "catalog/CatalogTypedefs.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+/** \addtogroup Catalog
+ * @{
+ */
+
+/**
+ * @brief Statistics of a catalog relation. E.g. total number of tuples,
+ * number of distinct values for each column.
+ **/
+class CatalogRelationStatistics {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ CatalogRelationStatistics()
+ : num_tuples_(0) {}
+
+ /**
+ * @brief Reconstruct a CatalogRelationStatistics object from its serialized
+ * Protocol Buffer form.
+ *
+ * @param proto The Protocol Buffer serialization of a CatalogRelationStatistics
+ * object, previously produced by getProto().
+ **/
+ explicit CatalogRelationStatistics(const serialization::CatalogRelationStatistics &proto);
+
+ /**
+ * @brief Serialize the CatalogRelationStatistics object as Protocol Buffer.
+ *
+ * @return The Protocol Buffer representation of the CatalogRelationStatistics
+ * object.
+ **/
+ serialization::CatalogRelationStatistics getProto() const;
+
+ /**
+ * @brief Set the number of tuples statistic.
+ *
+ * @param num_tuples The number of tuples statistic.
+ */
+ void setNumTuples(std::size_t num_tuples) {
+ num_tuples_ = num_tuples;
+ }
+
+ /**
+ * @brief Get the number of tuples statistic.
+ *
+ * @return The number of tuples. Returns 0 if the statistic is not set.
+ */
+ std::size_t getNumTuples() const {
+ return num_tuples_;
+ }
+
+ /**
+ * @brief Set the number of distinct values statistic for a column (catalog attribute).
+ *
+ * @param attr_id The id of the column.
+ * @param num_distinct_values The number of distinct values statistic.
+ */
+ void setNumDistinctValues(attribute_id attr_id, std::size_t num_distinct_values) {
+ num_distinct_values_map_[attr_id] = num_distinct_values;
+ }
+
+ /**
+ * @brief Get the number of distinct values statistic for a column (catalog attribute).
+ *
+ * @param The id of the column.
+ * @return The number of distinct values statistic for the column. Returns 0
+ * if the statistic is not set.
+ */
+ std::size_t getNumDistinctValues(attribute_id attr_id) const {
+ const auto it = num_distinct_values_map_.find(attr_id);
+ if (it == num_distinct_values_map_.end()) {
+ return static_cast<std::size_t>(0);
+ } else {
+ return it->second;
+ }
+ }
+
+ private:
+ // Total number of tuples in the relation.
+ std::size_t num_tuples_;
+
+ // Number of distinct values for each column.
+ std::unordered_map<attribute_id, std::size_t> num_distinct_values_map_;
+
+ DISALLOW_COPY_AND_ASSIGN(CatalogRelationStatistics);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_CATALOG_CATALOG_RELATION_STATISTICS_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 761b6d8..8fee7a4 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -1,5 +1,7 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -73,11 +75,24 @@ target_link_libraries(quickstep_cli_CommandExecutor
quickstep_catalog_CatalogDatabase
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogRelationSchema
- quickstep_cli_PrintToScreen
+ quickstep_cli_DropRelation
+ quickstep_cli_PrintToScreen
quickstep_parser_ParseStatement
+ quickstep_parser_SqlParserWrapper
+ quickstep_queryexecution_Foreman
+ quickstep_queryoptimizer_QueryHandle
+ quickstep_queryoptimizer_QueryPlan
+ quickstep_queryoptimizer_QueryProcessor
+ quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
- quickstep_utility_Macros
- quickstep_utility_PtrVector
+ quickstep_storage_StorageManager
+ quickstep_storage_TupleIdSequence
+ quickstep_storage_TupleStorageSubBlock
+ quickstep_parser_ParseString
+ quickstep_types_Type
+ quickstep_types_TypeID
+ quickstep_types_TypedValue
+ quickstep_utility_PtrVector
quickstep_utility_SqlError)
target_link_libraries(quickstep_cli_DefaultsConfigurator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 026922a..3cb3f86 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -19,6 +19,7 @@
#include <algorithm>
#include <cstddef>
+#include <cstdint>
#include <cstdio>
#include <memory>
#include <string>
@@ -28,14 +29,26 @@
#include "catalog/CatalogDatabase.hpp"
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogRelationSchema.hpp"
+#include "cli/DropRelation.hpp"
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
+#include "parser/ParseString.hpp"
+#include "parser/SqlParserWrapper.hpp"
+#include "query_execution/Foreman.hpp"
+#include "query_optimizer/QueryHandle.hpp"
+#include "query_optimizer/QueryPlan.hpp"
+#include "query_optimizer/QueryProcessor.hpp"
+#include "storage/StorageBlock.hpp"
#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageManager.hpp"
+#include "storage/TupleIdSequence.hpp"
+#include "storage/TupleStorageSubBlock.hpp"
+#include "types/Type.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
#include "utility/PtrVector.hpp"
-#include "utility/Macros.hpp"
#include "utility/SqlError.hpp"
-#include "gflags/gflags.h"
#include "glog/logging.h"
using std::fprintf;
@@ -195,11 +208,130 @@ void executeDescribeTable(
}
}
+/**
+ * @brief A helper function that executes a SQL query to obtain a scalar result.
+ */
+inline TypedValue executeQueryForSingleResult(const std::string &query_string,
+ StorageManager *storage_manager,
+ QueryProcessor *query_processor,
+ SqlParserWrapper *parser_wrapper,
+ Foreman *foreman) {
+ parser_wrapper->feedNextBuffer(new std::string(query_string));
+
+ ParseResult result = parser_wrapper->getNextStatement();
+ DCHECK(result.condition == ParseResult::kSuccess);
+
+ // Generate the query plan.
+ std::unique_ptr<QueryHandle> query_handle(
+ query_processor->generateQueryHandle(*result.parsed_statement));
+ DCHECK(query_handle->getQueryPlanMutable() != nullptr);
+
+ // Use foreman to execute the query plan.
+ foreman->setQueryPlan(query_handle->getQueryPlanMutable()->getQueryPlanDAGMutable());
+ foreman->reconstructQueryContextFromProto(query_handle->getQueryContextProto());
+
+ foreman->start();
+ foreman->join();
+
+ // Retrieve the scalar result from the result relation.
+ const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
+ DCHECK(query_result_relation != nullptr);
+
+ TypedValue value;
+ {
+ std::vector<block_id> blocks = query_result_relation->getBlocksSnapshot();
+ DCHECK_EQ(1u, blocks.size());
+ BlockReference block = storage_manager->getBlock(blocks[0], *query_result_relation);
+ const TupleStorageSubBlock &tuple_store = block->getTupleStorageSubBlock();
+ DCHECK_EQ(1, tuple_store.numTuples());
+ DCHECK_EQ(1u, tuple_store.getRelation().size());
+
+ if (tuple_store.isPacked()) {
+ value = tuple_store.getAttributeValueTyped(0, 0);
+ } else {
+ std::unique_ptr<TupleIdSequence> existence_map(tuple_store.getExistenceMap());
+ value = tuple_store.getAttributeValueTyped(*existence_map->begin(), 0);
+ }
+ value.ensureNotReference();
+ }
+
+ // Drop the result relation.
+ DropRelation::Drop(*query_result_relation,
+ query_processor->getDefaultDatabase(),
+ query_processor->getStorageManager());
+
+ return value;
+}
+
+void executeAnalyze(QueryProcessor *query_processor,
+ Foreman *foreman,
+ FILE *out) {
+ const CatalogDatabase &database = *query_processor->getDefaultDatabase();
+ StorageManager *storage_manager = query_processor->getStorageManager();
+
+ std::unique_ptr<SqlParserWrapper> parser_wrapper(new SqlParserWrapper());
+ std::vector<std::reference_wrapper<const CatalogRelation>> relations(
+ database.begin(), database.end());
+
+ // Analyze each relation in the database.
+ for (const CatalogRelation &relation : relations) {
+ fprintf(out, "Analyzing %s ... ", relation.getName().c_str());
+ fflush(out);
+
+ CatalogRelation *mutable_relation =
+ query_processor->getDefaultDatabase()->getRelationByIdMutable(relation.getID());
+
+ // Get the number of distinct values for each column.
+ for (const CatalogAttribute &attribute : relation) {
+ std::string query_string = "SELECT COUNT(DISTINCT ";
+ query_string.append(attribute.getName());
+ query_string.append(") FROM ");
+ query_string.append(relation.getName());
+ query_string.append(";");
+
+ TypedValue num_distinct_values =
+ executeQueryForSingleResult(query_string,
+ storage_manager,
+ query_processor,
+ parser_wrapper.get(),
+ foreman);
+
+ DCHECK(num_distinct_values.getTypeID() == TypeID::kLong);
+ mutable_relation->getStatisticsMutable()->setNumDistinctValues(
+ attribute.getID(),
+ num_distinct_values.getLiteral<std::int64_t>());
+ }
+
+ // Get the number of tuples for the relation.
+ std::string query_string = "SELECT COUNT(*) FROM ";
+ query_string.append(relation.getName());
+ query_string.append(";");
+
+ TypedValue num_tuples =
+ executeQueryForSingleResult(query_string,
+ storage_manager,
+ query_processor,
+ parser_wrapper.get(),
+ foreman);
+
+ DCHECK(num_tuples.getTypeID() == TypeID::kLong);
+ mutable_relation->getStatisticsMutable()->setNumTuples(
+ num_tuples.getLiteral<std::int64_t>());
+
+ fprintf(out, "done\n");
+ fflush(out);
+ }
+ query_processor->markCatalogAltered();
+ query_processor->saveCatalog();
+}
+
} // namespace
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
StorageManager *storage_manager,
+ QueryProcessor *query_processor,
+ Foreman *foreman,
FILE *out) {
const ParseCommand &command = static_cast<const ParseCommand &>(statement);
const PtrVector<ParseString> *arguments = command.arguments();
@@ -212,6 +344,8 @@ void executeCommand(const ParseStatement &statement,
} else {
executeDescribeTable(arguments, catalog_database, out);
}
+ } else if (command_str == C::kAnalyzeCommand) {
+ executeAnalyze(query_processor, foreman, out);
} else {
THROW_SQL_ERROR_AT(command.command()) << "Invalid Command";
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/cli/CommandExecutor.hpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.hpp b/cli/CommandExecutor.hpp
index f367ca1..c819981 100644
--- a/cli/CommandExecutor.hpp
+++ b/cli/CommandExecutor.hpp
@@ -19,13 +19,8 @@
#define QUICKSTEP_CLI_COMMAND_COMMAND_EXECUTOR_HPP_
#include <cstdio>
-#include <limits>
#include <string>
-#include "parser/ParseStatement.hpp"
-#include "storage/StorageBlockInfo.hpp"
-#include "utility/Macros.hpp"
-
using std::fprintf;
using std::fputc;
using std::string;
@@ -33,11 +28,13 @@ using std::string;
namespace quickstep {
class CatalogDatabase;
-class CatalogAttribute;
-class CatalogRelation;
+class Foreman;
+class ParseStatement;
+class QueryProcessor;
class StorageManager;
namespace cli {
+
/** \addtogroup CLI
* @{
*/
@@ -49,17 +46,23 @@ constexpr int kInitMaxColumnWidth = 6;
constexpr char kDescribeDatabaseCommand[] = "\\dt";
constexpr char kDescribeTableCommand[] = "\\d";
+constexpr char kAnalyzeCommand[] = "\\analyze";
/**
* @brief Executes the command by calling the command handler.
*
* @param statement The parsed statement from the cli.
* @param catalog_database The catalog information about the current database.
+ * @param storage_manager The current StorageManager.
+ * @param query_processor The query processor to generate plans for SQL queries.
+ * @param foreman The foreman to execute query plans.
* @param out The stream where the output of the command has to be redirected to.
*/
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
StorageManager *storage_manager,
+ QueryProcessor *query_processor,
+ Foreman *foreman,
FILE *out);
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index b7b28ba..558d6eb 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -367,6 +367,8 @@ int main(int argc, char* argv[]) {
*result.parsed_statement,
*(query_processor->getDefaultDatabase()),
query_processor->getStorageManager(),
+ query_processor.get(),
+ &foreman,
stdout);
} catch (const quickstep::SqlError &sql_error) {
fprintf(stderr, "%s",
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/cli/tests/CommandExecutorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/cli/tests/CommandExecutorTestRunner.cpp b/cli/tests/CommandExecutorTestRunner.cpp
index 73d2092..9cd493e 100644
--- a/cli/tests/CommandExecutorTestRunner.cpp
+++ b/cli/tests/CommandExecutorTestRunner.cpp
@@ -88,6 +88,8 @@ void CommandExecutorTestRunner::runTestCase(
*result.parsed_statement,
*(test_database_loader_.catalog_database()),
test_database_loader_.storage_manager(),
+ nullptr,
+ nullptr,
output_stream.file());
} else {
QueryHandle query_handle(optimizer_context.query_id());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 7209cfa..c590b6e 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1389,11 +1389,13 @@ void ExecutionGenerator::convertAggregate(
// Add distinctify hash table impl type if it is a DISTINCT aggregation.
if (unnamed_aggregate_expression->is_distinct()) {
- if (group_by_types.empty()) {
+ const std::vector<E::ScalarPtr> &arguments = unnamed_aggregate_expression->getArguments();
+ DCHECK_GE(arguments.size(), 1u);
+ if (group_by_types.empty() && arguments.size() == 1) {
aggr_state_proto->add_distinctify_hash_table_impl_types(
SimplifyHashTableImplTypeProto(
HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
- {&unnamed_aggregate_expression->getValueType()}));
+ {&arguments[0]->getValueType()}));
} else {
aggr_state_proto->add_distinctify_hash_table_impl_types(
HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/query_optimizer/QueryProcessor.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryProcessor.hpp b/query_optimizer/QueryProcessor.hpp
index 4514f45..32739dc 100644
--- a/query_optimizer/QueryProcessor.hpp
+++ b/query_optimizer/QueryProcessor.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -159,6 +161,14 @@ class QueryProcessor {
void saveCatalog();
/**
+ * @brief Set \p catalog_altered_ to true to indicate that the catalog
+ * has been altered.
+ */
+ void markCatalogAltered() {
+ catalog_altered_ = true;
+ }
+
+ /**
* @brief Get the default database in the Catalog held by this
* QueryProcessor.
**/
[06/50] [abbrv] incubator-quickstep git commit: Increased the size of
the sharded lock manager to a large primary number (#198)
Posted by hb...@apache.org.
Increased the size of the sharded lock manager to a large primary number (#198)
Having small number of entries in the sharded lock manager introduces artificial conflict causing the buffer pool to grow. The size of an entry in the lock_manager_ is small, so don't be so stingy with the size of the lock manager. By default, we want to run well on large memory boxes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5b75e8ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5b75e8ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5b75e8ec
Branch: refs/heads/query-manager-used-in-foreman
Commit: 5b75e8ec24366e2379624804db9cb118cbe0bcd3
Parents: c5460f4
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Thu Apr 28 16:11:22 2016 -0500
Committer: Zuyu ZHANG <zu...@users.noreply.github.com>
Committed: Thu Apr 28 14:11:22 2016 -0700
----------------------------------------------------------------------
storage/StorageManager.hpp | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5b75e8ec/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index 0b68b76..52326c2 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -479,7 +479,11 @@ class StorageManager {
// (2) If it is not safe to evict a block, then either that block's
// reference count is greater than 0 or a shared lock is held on the
// block's lock shard.
- static constexpr std::size_t kLockManagerNumShards = 256;
+ // TODO(jmp): Would be good to set this more intelligently in the future
+ // based on the hardware concurrency, the amount of main memory
+ // and slot size. For now pick the largest prime that is less
+ // than 8K.
+ static constexpr std::size_t kLockManagerNumShards = 0x2000-1;
ShardedLockManager<block_id, kLockManagerNumShards, SpinSharedMutex<false>> lock_manager_;
FRIEND_TEST(StorageManagerTest, DifferentNUMANodeBlobTestWithEviction);
[12/50] [abbrv] incubator-quickstep git commit: Support for NUMA
aware preloading (#206)
Posted by hb...@apache.org.
Support for NUMA aware preloading (#206)
* Support for preloading of NUMA-partitioned relations.
- Stored relations which have been NUMA-partitioned, can now be
preloaded. The preloading can maintain the same NUMA placement of
storage blocks.
* Added missing glog library in CMakeLists
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0f261ea1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0f261ea1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0f261ea1
Branch: refs/heads/query-manager-used-in-foreman
Commit: 0f261ea16a9b8cdb10ca7d5c2c64a906b71ae9a2
Parents: 456b434
Author: Harshad Deshmukh <d....@gmail.com>
Authored: Wed May 4 10:32:47 2016 -0500
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Wed May 4 10:32:47 2016 -0500
----------------------------------------------------------------------
storage/CMakeLists.txt | 5 +++
storage/PreloaderThread.cpp | 86 +++++++++++++++++++++++++++++++++++-----
storage/PreloaderThread.hpp | 23 +++++++++++
3 files changed, 105 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0f261ea1/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index ed23802..dacacfa 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -741,6 +741,7 @@ target_link_libraries(quickstep_storage_PackedRowStoreValueAccessor
quickstep_utility_BitVector
quickstep_utility_Macros)
target_link_libraries(quickstep_storage_PreloaderThread
+ glog
quickstep_catalog_CatalogDatabase
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
@@ -750,6 +751,10 @@ target_link_libraries(quickstep_storage_PreloaderThread
quickstep_threading_Thread
quickstep_threading_ThreadUtil
quickstep_utility_Macros)
+if (QUICKSTEP_HAVE_LIBNUMA)
+ target_link_libraries(quickstep_storage_PreloaderThread
+ quickstep_catalog_NUMAPlacementScheme)
+endif()
target_link_libraries(quickstep_storage_SMAIndexSubBlock
glog
quickstep_catalog_CatalogAttribute
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0f261ea1/storage/PreloaderThread.cpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.cpp b/storage/PreloaderThread.cpp
index d5dc55b..8f600b8 100644
--- a/storage/PreloaderThread.cpp
+++ b/storage/PreloaderThread.cpp
@@ -17,8 +17,13 @@
#include "storage/PreloaderThread.hpp"
+#include <cstddef>
#include <vector>
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include <unordered_map>
+#endif
+
#include "catalog/CatalogDatabase.hpp"
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
@@ -27,6 +32,12 @@
#include "storage/StorageManager.hpp"
#include "threading/ThreadUtil.hpp"
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include "catalog/NUMAPlacementScheme.hpp"
+#endif
+
+#include "glog/logging.h"
+
namespace quickstep {
void PreloaderThread::run() {
@@ -38,24 +49,81 @@ void PreloaderThread::run() {
std::size_t blocks_loaded = 0;
for (const CatalogRelation &relation : database_) {
- std::vector<block_id> blocks = relation.getBlocksSnapshot();
- for (block_id current_block_id : blocks) {
+ if (relation.hasPartitionScheme()) {
+ blocks_loaded += preloadNUMAAware(relation, blocks_loaded, num_slots);
+ } else {
+ std::vector<block_id> blocks = relation.getBlocksSnapshot();
+ for (block_id current_block_id : blocks) {
+ try {
+ BlockReference current_block = storage_manager_->getBlock(current_block_id, relation);
+ } catch (...) {
+ LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks\n";
+ throw;
+ }
+ ++blocks_loaded;
+ if (blocks_loaded == num_slots) {
+ // The buffer pool has filled up. But, some database blocks are not loaded.
+ printf(" The database is larger than the buffer pool. Only %lu blocks were loaded ",
+ blocks_loaded);
+ return;
+ }
+ }
+ }
+ }
+ printf(" Loaded %lu blocks ", blocks_loaded);
+}
+
+std::size_t PreloaderThread::preloadNUMAAware(
+ const CatalogRelation &relation,
+ const std::size_t num_previously_loaded_blocks,
+ const std::size_t num_slots) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ std::size_t blocks_loaded = 0;
+ const NUMAPlacementScheme *placement_scheme = relation.getNUMAPlacementSchemePtr();
+ DCHECK(placement_scheme != nullptr);
+ DCHECK(relation.hasPartitionScheme());
+ const PartitionScheme &part_scheme = relation.getPartitionScheme();
+ const PartitionSchemeHeader &part_scheme_header =
+ part_scheme.getPartitionSchemeHeader();
+ const std::size_t num_partitions = part_scheme_header.getNumPartitions();
+ // Key = NUMA node ID, value = number of blocks loaded from that NUMA node.
+ std::unordered_map<numa_node_id, std::size_t> num_blocks_loaded;
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ const numa_node_id partition_numa_node_id =
+ placement_scheme->getNUMANodeForPartition(part_id);
+ for (block_id curr_block_id : part_scheme.getBlocksInPartition(part_id)) {
try {
- BlockReference current_block = storage_manager_->getBlock(current_block_id, relation);
+ BlockReference current_block = storage_manager_->getBlock(
+ curr_block_id, relation, partition_numa_node_id);
} catch (...) {
- LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks\n";
+ LOG(ERROR) << "Error after loading "
+ << blocks_loaded + num_previously_loaded_blocks
+ << " blocks\n";
throw;
}
++blocks_loaded;
- if (blocks_loaded == num_slots) {
+ num_blocks_loaded[partition_numa_node_id]++;
+ if ((blocks_loaded + num_previously_loaded_blocks) == num_slots) {
// The buffer pool has filled up. But, some database blocks are not loaded.
- printf(" The database is larger than the buffer pool. Only %lu blocks were loaded ",
- blocks_loaded);
- return;
+ printf(
+ " The database is larger than the buffer pool. Only %lu blocks "
+ "were loaded ",
+ blocks_loaded + num_previously_loaded_blocks);
+ return blocks_loaded;
}
}
}
- printf(" Loaded %lu blocks ", blocks_loaded);
+ LOG(INFO) << "For relation: " << relation.getName();
+ for (auto numa_block_loaded_info : num_blocks_loaded) {
+ LOG(INFO) << "NUMA node: " << numa_block_loaded_info.first
+ << " Number of loaded blocks: " << numa_block_loaded_info.second;
+ }
+ return blocks_loaded;
+#else
+ LOG(INFO) << "Relation: " << relation.getName()
+ << " has partition scheme but the system doesn't support NUMA";
+ return 0;
+#endif
}
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0f261ea1/storage/PreloaderThread.hpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.hpp b/storage/PreloaderThread.hpp
index ed866b4..f16fd50 100644
--- a/storage/PreloaderThread.hpp
+++ b/storage/PreloaderThread.hpp
@@ -18,12 +18,15 @@
#ifndef QUICKSTEP_STORAGE_PRELOADER_THREAD_HPP_
#define QUICKSTEP_STORAGE_PRELOADER_THREAD_HPP_
+#include <cstddef>
+
#include "threading/Thread.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
class CatalogDatabase;
+class CatalogRelation;
class StorageManager;
/** \addtogroup Storage
@@ -65,6 +68,26 @@ class PreloaderThread : public Thread {
void run() override;
private:
+ /**
+ * @brief Preload a relation which has a partition and a NUMA placement scheme.
+ *
+ * @param relation The relation to be preloaded.
+ * @param num_previously_loaded_blocks The number of blocks already preloaded.
+ * @param num_slots The maximum number of slots in the StorageManager.
+ *
+ * @warning This function may not detect skew on NUMA sockets, i.e. if a given
+ * NUMA socket has large number of blocks, preloading may cause the
+ * memory on that NUMA socket to be full. It is recommended to use
+ * this preloading when we are sure that each NUMA socket has been
+ * allocated sufficient amount of memory so as not to exceed that
+ * socket's memory limit.
+ *
+ * @return The number of blocks loaded during this function call.
+ **/
+ std::size_t preloadNUMAAware(const CatalogRelation &relation,
+ const std::size_t num_previously_loaded_blocks,
+ const std::size_t num_slots);
+
const CatalogDatabase &database_;
StorageManager *storage_manager_;
[02/50] [abbrv] incubator-quickstep git commit: Fixed the deadlock
when loading while evicting. (#196)
Posted by hb...@apache.org.
Fixed the deadlock when loading while evicting. (#196)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/7ac1d22e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/7ac1d22e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/7ac1d22e
Branch: refs/heads/query-manager-used-in-foreman
Commit: 7ac1d22e0ae156730d14b86855df76bc10cefc47
Parents: d353a64
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Wed Apr 27 22:59:25 2016 -0700
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Thu Apr 28 00:59:25 2016 -0500
----------------------------------------------------------------------
storage/StorageManager.cpp | 20 ++++++++++++------
utility/CMakeLists.txt | 2 ++
utility/ShardedLockManager.hpp | 41 +++++++++++++++++++++++++------------
3 files changed, 44 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7ac1d22e/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index a3f265d..b98a28c 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -494,12 +494,16 @@ MutableBlockReference StorageManager::getBlockInternal(
// To be safe, release the block's shard after 'eviction_lock' destructs.
lock_manager_.release(block);
+ if (ret.valid()) {
+ return ret;
+ }
+
// Note that there is no way for the block to be evicted between the call to
// loadBlock and the call to EvictionPolicy::blockReferenced from
// MutableBlockReference's constructor; this is because EvictionPolicy
// doesn't know about the block until blockReferenced is called, so
// chooseBlockToEvict shouldn't return the block.
- if (!ret.valid()) {
+ do {
SpinSharedMutexExclusiveLock<false> io_lock(*lock_manager_.get(block));
{
// Check one more time if the block got loaded in memory by someone else.
@@ -508,12 +512,12 @@ MutableBlockReference StorageManager::getBlockInternal(
if (it != blocks_.end()) {
DEBUG_ASSERT(!it->second.block->isBlob());
ret = MutableBlockReference(static_cast<StorageBlock*>(it->second.block), eviction_policy_.get());
- return ret;
+ break;
}
}
// No other thread loaded the block before us.
ret = MutableBlockReference(loadBlock(block, relation, numa_node), eviction_policy_.get());
- }
+ } while (false);
// To be safe, release the block's shard after 'io_lock' destructs.
lock_manager_.release(block);
@@ -535,7 +539,11 @@ MutableBlobReference StorageManager::getBlobInternal(const block_id blob,
// To be safe, release the blob's shard after 'eviction_lock' destructs.
lock_manager_.release(blob);
- if (!ret.valid()) {
+ if (ret.valid()) {
+ return ret;
+ }
+
+ do {
SpinSharedMutexExclusiveLock<false> io_lock(*lock_manager_.get(blob));
// Note that there is no way for the block to be evicted between the call to
// loadBlob and the call to EvictionPolicy::blockReferenced from
@@ -548,12 +556,12 @@ MutableBlobReference StorageManager::getBlobInternal(const block_id blob,
if (it != blocks_.end()) {
DEBUG_ASSERT(it->second.block->isBlob());
ret = MutableBlobReference(static_cast<StorageBlob*>(it->second.block), eviction_policy_.get());
- return ret;
+ break;
}
}
// No other thread loaded the blob before us.
ret = MutableBlobReference(loadBlob(blob, numa_node), eviction_policy_.get());
- }
+ } while (false);
// To be safe, release the blob's shard after 'io_lock' destructs.
lock_manager_.release(blob);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7ac1d22e/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index 4ff9254..bb59f65 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -244,7 +244,9 @@ target_link_libraries(quickstep_utility_SortConfiguration_proto
${PROTOBUF_LIBRARY})
target_link_libraries(quickstep_utility_ShardedLockManager
quickstep_storage_StorageConstants
+ quickstep_threading_Mutex
quickstep_threading_SharedMutex
+ quickstep_threading_SpinSharedMutex
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_StringUtil
glog)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7ac1d22e/utility/ShardedLockManager.hpp
----------------------------------------------------------------------
diff --git a/utility/ShardedLockManager.hpp b/utility/ShardedLockManager.hpp
index 1d59acb..e3eba85 100644
--- a/utility/ShardedLockManager.hpp
+++ b/utility/ShardedLockManager.hpp
@@ -21,9 +21,12 @@
#include <array>
#include <cstddef>
#include <functional>
+#include <unordered_map>
#include "storage/StorageConstants.hpp"
+#include "threading/Mutex.hpp"
#include "threading/SharedMutex.hpp"
+#include "threading/SpinSharedMutex.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
@@ -64,24 +67,29 @@ class ShardedLockManager {
if (has_collision != nullptr) {
// In StorageManager::makeRoomForBlock, check whether the evicting block
// or blob has a shard collision with existing referenced shards.
- SpinSharedMutexSharedLock<false> read_lock(shards_mutex_);
- if (shards_.find(shard) != shards_.end()) {
+ SpinSharedMutexSharedLock<false> read_lock(shard_count_mutex_);
+ if (shard_count_.find(shard) != shard_count_.end()) {
*has_collision = true;
return &collision_mutex_;
}
}
{
- SpinSharedMutexExclusiveLock<false> write_lock(shards_mutex_);
+ SpinSharedMutexExclusiveLock<false> write_lock(shard_count_mutex_);
// Check one more time for the evicting block or blob if there is a shard
// collision.
- if (has_collision != nullptr && shards_.find(shard) != shards_.end()) {
- *has_collision = true;
- return &collision_mutex_;
+ auto it = shard_count_.find(shard);
+ if (it != shard_count_.end()) {
+ if (has_collision != nullptr) {
+ *has_collision = true;
+ return &collision_mutex_;
+ }
+
+ ++it->second;
+ } else {
+ shard_count_.emplace(shard, 1);
}
-
- shards_.insert(shard);
}
return &sharded_mutexes_[shard];
}
@@ -91,8 +99,13 @@ class ShardedLockManager {
* @param key The key to compute the shard.
*/
void release(const T key) {
- SpinSharedMutexExclusiveLock<false> write_lock(shards_mutex_);
- shards_.erase(hash_(key) % N);
+ SpinSharedMutexExclusiveLock<false> write_lock(shard_count_mutex_);
+ auto it = shard_count_.find(hash_(key) % N);
+ DCHECK(it != shard_count_.end());
+
+ if (--it->second == 0) {
+ shard_count_.erase(it);
+ }
}
private:
@@ -102,9 +115,11 @@ class ShardedLockManager {
// The placeholder mutex used whenever there is a hash collision.
SharedMutexT collision_mutex_;
- // Bookkeep all shards referenced by StorageManager in multiple threads.
- std::unordered_set<std::size_t> shards_;
- alignas(kCacheLineBytes) mutable SpinSharedMutex<false> shards_mutex_;
+ // Count all shards referenced by StorageManager in multiple threads.
+ // The key is the shard, while the value is the count. If the count equals to
+ // zero, we delete the shard entry.
+ std::unordered_map<std::size_t, std::size_t> shard_count_;
+ alignas(kCacheLineBytes) mutable SpinSharedMutex<false> shard_count_mutex_;
DISALLOW_COPY_AND_ASSIGN(ShardedLockManager);
};
[10/50] [abbrv] incubator-quickstep git commit: Fixes bug which
disallows the creation of BITWEAVING_H indices. (#204)
Posted by hb...@apache.org.
Fixes bug which disallows the creation of BITWEAVING_H indices. (#204)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/29a71ac2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/29a71ac2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/29a71ac2
Branch: refs/heads/query-manager-used-in-foreman
Commit: 29a71ac21f746e74dc7a544c8d8e09f71854158c
Parents: 89acf23
Author: Marc S <cr...@users.noreply.github.com>
Authored: Sun May 1 17:14:20 2016 -0500
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Sun May 1 17:14:20 2016 -0500
----------------------------------------------------------------------
parser/ParseIndexProperties.hpp | 2 +-
parser/tests/Index.test | 11 +++++++++++
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29a71ac2/parser/ParseIndexProperties.hpp
----------------------------------------------------------------------
diff --git a/parser/ParseIndexProperties.hpp b/parser/ParseIndexProperties.hpp
index 9ca1419..2cb5df9 100644
--- a/parser/ParseIndexProperties.hpp
+++ b/parser/ParseIndexProperties.hpp
@@ -220,7 +220,7 @@ class BitWeavingIndexProperties : public IndexProperties {
}
const std::string key = ToLower(key_value.key()->value());
const std::string value = ToLower(
- static_cast<const ParseKeyStringValue&>(key_value).key()->value());
+ static_cast<const ParseKeyStringValue&>(key_value).value()->value());
if (key.compare(kBitWeavingType) == 0) {
if (value.compare("h") == 0) {
index_sub_block_description_->set_sub_block_type(IndexSubBlockDescription::BITWEAVING_H);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29a71ac2/parser/tests/Index.test
----------------------------------------------------------------------
diff --git a/parser/tests/Index.test b/parser/tests/Index.test
index 605768b..39cd523 100644
--- a/parser/tests/Index.test
+++ b/parser/tests/Index.test
@@ -66,3 +66,14 @@ CREATE INDEX bwIndex ON test(int_col) USING bitweaving;
CreateIndexStatement[index_name=bwIndex,relation_name=test,index_type=bitweaving]
+-attribute_list=
+-AttributeReference[attribute_name=int_col]
+==
+CREATE INDEX bwhIndex ON test(int_col) USING bitweaving (TYPE H);
+--
+CreateIndexStatement[index_name=bwhIndex,relation_name=test,
+ index_type=bitweaving]
++-attribute_list=
+| +-AttributeReference[attribute_name=int_col]
++-index_property_list=
+ +-IndexProperties
+ +-index_property=KeyStringValue[key=TYPE]
+ +-value=String[value=H]
[44/50] [abbrv] incubator-quickstep git commit: Fixed Travis CI link
in Readme.
Posted by hb...@apache.org.
Fixed Travis CI link in Readme.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8f8a03a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8f8a03a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8f8a03a2
Branch: refs/heads/query-manager-used-in-foreman
Commit: 8f8a03a2f91592ebc07389268a47219d458832d6
Parents: fe80c01
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Mon May 16 13:39:12 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:53 2016 -0700
----------------------------------------------------------------------
README.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f8a03a2/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 7cea80c..04d5d66 100644
--- a/README.md
+++ b/README.md
@@ -2,8 +2,8 @@
[![Travis Widget]][Travis]
-[Travis]: https://travis-ci.org/pivotalsoftware/quickstep
-[Travis Widget]: https://travis-ci.org/pivotalsoftware/quickstep.svg?branch=master
+[Travis]: https://travis-ci.org/apache/incubator-quickstep
+[Travis Widget]: https://travis-ci.org/apache/incubator-quickstep.svg?branch=master
Apache Quickstep is an experimental high-performance database engine designed with the
aim of Data at Bare-Metal Speed. It began life in 2011 as a
[34/50] [abbrv] incubator-quickstep git commit: Change the default
value of Joined Tuple Collector (#226)
Posted by hb...@apache.org.
Change the default value of Joined Tuple Collector (#226)
On NUMA boxes, the vector based joined tuple collector slows down queries.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a25da390
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a25da390
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a25da390
Branch: refs/heads/query-manager-used-in-foreman
Commit: a25da39001017360907ee523d94b48cb4417fe24
Parents: 3c84537
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Wed May 18 11:56:53 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:51 2016 -0700
----------------------------------------------------------------------
relational_operators/HashJoinOperator.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a25da390/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index 104a02d..aa03794 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -59,7 +59,7 @@ namespace quickstep {
namespace {
-DEFINE_bool(vector_based_joined_tuple_collector, true,
+DEFINE_bool(vector_based_joined_tuple_collector, false,
"If true, use simple vector-based joined tuple collector in "
"hash join, with a final sort pass to group joined tuple pairs "
"by inner block. If false, use unordered_map based collector that "
[30/50] [abbrv] incubator-quickstep git commit: Fixed a g++ compiler
warning (#221)
Posted by hb...@apache.org.
Fixed a g++ compiler warning (#221)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ba25b132
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ba25b132
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ba25b132
Branch: refs/heads/query-manager-used-in-foreman
Commit: ba25b1326e8063f41a4d89b62b6860b6ef8d0e52
Parents: ee54e1d
Author: Marc S <cr...@users.noreply.github.com>
Authored: Wed May 11 14:34:37 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:27 2016 -0700
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ba25b132/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 7f26e85..7209cfa 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -588,8 +588,8 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::vector<attribute_id> probe_original_attribute_ids;
std::vector<attribute_id> build_original_attribute_ids;
- const CatalogRelation *referenced_stored_probe_relation;
- const CatalogRelation *referenced_stored_build_relation;
+ const CatalogRelation *referenced_stored_probe_relation = nullptr;
+ const CatalogRelation *referenced_stored_build_relation = nullptr;
bool any_probe_attributes_nullable = false;
bool any_build_attributes_nullable = false;
[16/50] [abbrv] incubator-quickstep git commit: Transaction Part 4:
LockManager, CycleDetector and DeadLockDetector. (#187)
Posted by hb...@apache.org.
Transaction Part 4: LockManager, CycleDetector and DeadLockDetector. (#187)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/bbaff7a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/bbaff7a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/bbaff7a2
Branch: refs/heads/query-manager-used-in-foreman
Commit: bbaff7a2aa84b0792a33ca652da701c1b3498008
Parents: a3889a3
Author: Hakan Memisoglu <ha...@gmail.com>
Authored: Wed May 4 23:30:51 2016 -0500
Committer: Zuyu ZHANG <zu...@users.noreply.github.com>
Committed: Wed May 4 21:30:51 2016 -0700
----------------------------------------------------------------------
transaction/AccessMode.hpp | 76 +++++-
transaction/CMakeLists.txt | 59 ++++-
transaction/CycleDetector.cpp | 120 ++++++++++
transaction/CycleDetector.hpp | 83 +++++++
transaction/DeadLockDetector.cpp | 177 ++++++++++++++
transaction/DeadLockDetector.hpp | 156 ++++++++++++
transaction/DirectedGraph.hpp | 56 ++---
transaction/LockManager.cpp | 237 +++++++++++++++++++
transaction/LockManager.hpp | 128 ++++++++++
transaction/LockTable.cpp | 22 +-
transaction/LockTable.hpp | 36 +--
transaction/ResourceId.hpp | 9 +-
transaction/StronglyConnectedComponents.cpp | 1 -
transaction/Transaction.cpp | 48 ----
transaction/TransactionTable.cpp | 8 +-
transaction/TransactionTable.hpp | 11 +-
transaction/tests/AccessMode_unittest.cpp | 12 +-
transaction/tests/CycleDetector_unittest.cpp | 157 ++++++++++++
transaction/tests/DeadLockDetector_unittest.cpp | 96 ++++++++
transaction/tests/DirectedGraph_unittest.cpp | 113 +++++----
transaction/tests/LockRequest_unittest.cpp | 4 +-
transaction/tests/LockTable_unittest.cpp | 63 ++---
transaction/tests/Lock_unittest.cpp | 13 +-
.../StronglyConnectedComponents_unittest.cpp | 15 +-
transaction/tests/TransactionTable_unittest.cpp | 102 ++++----
25 files changed, 1512 insertions(+), 290 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/AccessMode.hpp
----------------------------------------------------------------------
diff --git a/transaction/AccessMode.hpp b/transaction/AccessMode.hpp
index 34ace36..bb06689 100644
--- a/transaction/AccessMode.hpp
+++ b/transaction/AccessMode.hpp
@@ -32,12 +32,12 @@ namespace transaction {
* @brief Represents mode type. Possible options are NL, IS, IX, S, SIX, X.
**/
enum class AccessModeType : std::uint8_t {
- kNoLock = 0,
- kIsLock,
- kIxLock,
- kSLock,
- kSixLock,
- kXLock,
+ kNoLockMode = 0,
+ kIsLockMode,
+ kIxLockMode,
+ kSLockMode,
+ kSixLockMode,
+ kXLockMode,
kNumAccessModeTypes,
};
@@ -55,6 +55,60 @@ class AccessMode {
: access_mode_(access_mode) {}
/**
+ * @brief Factory method for NoLockMode.
+ *
+ * @return NoLockMode instance.
+ **/
+ static AccessMode NoLockMode() {
+ return AccessMode(AccessModeType::kNoLockMode);
+ }
+
+ /**
+ * @brief Factory method for IsLockMode.
+ *
+ * @return IsLockMode instance.
+ **/
+ static AccessMode IsLockMode() {
+ return AccessMode(AccessModeType::kIsLockMode);
+ }
+
+ /**
+ * @brief Factory method for IxLockMode.
+ *
+ * @return IxLockMode instance.
+ **/
+ static AccessMode IxLockMode() {
+ return AccessMode(AccessModeType::kIxLockMode);
+ }
+
+ /**
+ * @brief Factory method for SixLockMode.
+ *
+ * @return SixLockMode instance.
+ **/
+ static AccessMode SixLockMode() {
+ return AccessMode(AccessModeType::kSixLockMode);
+ }
+
+ /**
+ * @brief Factory method for SLockMode.
+ *
+ * @return SLockMode instance.
+ **/
+ static AccessMode SLockMode() {
+ return AccessMode(AccessModeType::kSLockMode);
+ }
+
+ /**
+ * @brief Factory method for XLockMode.
+ *
+ * @return XLockMode instance.
+ **/
+ static AccessMode XLockMode() {
+ return AccessMode(AccessModeType::kXLockMode);
+ }
+
+ /**
* @brief Checks whether this access mode is compatible with the other.
*
* @param other Other access mode that will be checked against to this one.
@@ -74,7 +128,7 @@ class AccessMode {
* @return True if it is IS mode, false otherwise.
**/
inline bool isIntentionShareLock() const {
- return access_mode_ == AccessModeType::kIsLock;
+ return access_mode_ == AccessModeType::kIsLockMode;
}
/**
@@ -83,7 +137,7 @@ class AccessMode {
* @return True if it is IX mode, false otherwise.
**/
inline bool isIntentionExclusiveLock() const {
- return access_mode_ == AccessModeType::kIxLock;
+ return access_mode_ == AccessModeType::kIxLockMode;
}
/**
@@ -92,7 +146,7 @@ class AccessMode {
* @return True if it is SIX mode, false otherwise.
**/
inline bool isShareAndIntentionExclusiveLock() const {
- return access_mode_ == AccessModeType::kSixLock;
+ return access_mode_ == AccessModeType::kSixLockMode;
}
/**
@@ -101,7 +155,7 @@ class AccessMode {
* @return True if it is S mode, false otherwise.
**/
inline bool isShareLock() const {
- return access_mode_ == AccessModeType::kSLock;
+ return access_mode_ == AccessModeType::kSLockMode;
}
/**
@@ -110,7 +164,7 @@ class AccessMode {
* @return True if it is X mode, false otherwise.
**/
inline bool isExclusiveLock() const {
- return access_mode_ == AccessModeType::kXLock;
+ return access_mode_ == AccessModeType::kXLockMode;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/transaction/CMakeLists.txt b/transaction/CMakeLists.txt
index 05fc96a..c6c87b6 100644
--- a/transaction/CMakeLists.txt
+++ b/transaction/CMakeLists.txt
@@ -16,12 +16,21 @@
add_library(quickstep_transaction_AccessMode
AccessMode.cpp
AccessMode.hpp)
+add_library(quickstep_transaction_CycleDetector
+ CycleDetector.cpp
+ CycleDetector.hpp)
+add_library(quickstep_transaction_DeadLockDetector
+ DeadLockDetector.cpp
+ DeadLockDetector.cpp)
add_library(quickstep_transaction_DirectedGraph
../empty_src.cpp
DirectedGraph.hpp)
add_library(quickstep_transaction_Lock
../empty_src.cpp
Lock.hpp)
+add_library(quickstep_transaction_LockManager
+ LockManager.hpp
+ LockManager.cpp)
add_library(quickstep_transaction_LockRequest
../empty_src.cpp
LockRequest.hpp)
@@ -40,7 +49,17 @@ add_library(quickstep_transaction_Transaction
add_library(quickstep_transaction_TransactionTable
TransactionTable.cpp
TransactionTable.hpp)
-
+
+target_link_libraries(quickstep_transaction_CycleDetector
+ quickstep_transaction_DirectedGraph
+ quickstep_transaction_StronglyConnectedComponents
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_transaction_DeadLockDetector
+ glog
+ quickstep_transaction_CycleDetector
+ quickstep_transaction_DirectedGraph
+ quickstep_transaction_LockTable
+ quickstep_transaction_Transaction)
target_link_libraries(quickstep_transaction_DirectedGraph
glog
quickstep_transaction_Transaction
@@ -48,6 +67,18 @@ target_link_libraries(quickstep_transaction_DirectedGraph
target_link_libraries(quickstep_transaction_Lock
quickstep_transaction_AccessMode
quickstep_transaction_ResourceId)
+target_link_libraries(quickstep_transaction_LockManager
+ gflags_nothreads-static
+ glog
+ quickstep_utility_ThreadSafeQueue
+ quickstep_threading_Thread
+ quickstep_transaction_AccessMode
+ quickstep_transaction_DeadLockDetector
+ quickstep_transaction_LockRequest
+ quickstep_transaction_LockTable
+ quickstep_transaction_ResourceId
+ quickstep_transaction_Transaction
+ quickstep_transaction_TransactionTable)
target_link_libraries(quickstep_transaction_LockRequest
quickstep_transaction_AccessMode
quickstep_transaction_ResourceId
@@ -80,8 +111,11 @@ add_library(quickstep_transaction
TransactionModule.hpp)
target_link_libraries(quickstep_transaction
quickstep_transaction_AccessMode
+ quickstep_transaction_CycleDetector
+ quickstep_transaction_DeadLockDetector
quickstep_transaction_DirectedGraph
quickstep_transaction_Lock
+ quickstep_transaction_LockManager
quickstep_transaction_LockRequest
quickstep_transaction_LockTable
quickstep_transaction_ResourceId
@@ -97,6 +131,29 @@ target_link_libraries(AccessMode_unittest
quickstep_transaction_AccessMode)
add_test(AccessMode_unittest AccessMode_unittest)
+add_executable(CycleDetector_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/CycleDetector_unittest.cpp")
+target_link_libraries(CycleDetector_unittest
+ gtest
+ gtest_main
+ quickstep_transaction_CycleDetector
+ quickstep_transaction_DirectedGraph
+ quickstep_transaction_StronglyConnectedComponents)
+add_test(CycleDetector_unittest CycleDetector_unittest)
+
+add_executable(DeadLockDetector_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/DeadLockDetector_unittest.cpp")
+target_link_libraries(DeadLockDetector_unittest
+ gtest
+ gtest_main
+ quickstep_threading_Thread
+ quickstep_transaction_AccessMode
+ quickstep_transaction_DeadLockDetector
+ quickstep_transaction_LockTable
+ quickstep_transaction_ResourceId
+ quickstep_transaction_Transaction)
+add_test(DeadLockDetector_unittest DeadLockDetector_unittest)
+
add_executable(DirectedGraph_unittest
"${CMAKE_CURRENT_SOURCE_DIR}/tests/DirectedGraph_unittest.cpp")
target_link_libraries(DirectedGraph_unittest
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/CycleDetector.cpp
----------------------------------------------------------------------
diff --git a/transaction/CycleDetector.cpp b/transaction/CycleDetector.cpp
new file mode 100644
index 0000000..b12897f
--- /dev/null
+++ b/transaction/CycleDetector.cpp
@@ -0,0 +1,120 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/CycleDetector.hpp"
+
+#include <cstdint>
+#include <memory>
+#include <stack>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/StronglyConnectedComponents.hpp"
+
+namespace quickstep {
+namespace transaction {
+
+CycleDetector::CycleDetector(DirectedGraph *wait_for_graph)
+ : wait_for_graph_(wait_for_graph),
+ strongly_connected_components_(
+ std::make_unique<StronglyConnectedComponents>(*wait_for_graph)) {
+}
+
+std::vector<DirectedGraph::node_id>
+CycleDetector::chooseVictimsToBreakCycle() const {
+ std::vector<DirectedGraph::node_id> nodes_to_kill;
+ const std::unordered_map<std::uint64_t, std::vector<DirectedGraph::node_id>>
+ component_mapping = strongly_connected_components_->getComponentMapping();
+ for (const auto &entry : component_mapping) {
+ // One node means no cycle.
+ if (entry.second.size() == 1) {
+ continue;
+ }
+ const std::vector<DirectedGraph::node_id> nodes =
+ chooseVictimsInComponent(entry.second);
+ nodes_to_kill.insert(nodes_to_kill.end(), nodes.begin(), nodes.end());
+ }
+ return nodes_to_kill;
+}
+
+std::vector<DirectedGraph::node_id> CycleDetector::chooseVictimsInComponent(
+ const std::vector<DirectedGraph::node_id> &nodes) const {
+ std::vector<DirectedGraph::node_id> targets;
+ // Convert it to set to ensure defensively that the elements are unique.
+ std::unordered_set<DirectedGraph::node_id> nodes_set(nodes.begin(),
+ nodes.end());
+
+ while (true) {
+ if (!hasCycle(nodes_set)) {
+ break;
+ }
+ // Connected component still has a cycle, therefore choose a
+ // victim and keep trying to remove nodes until there is no cycle.
+ const DirectedGraph::node_id victim = chooseVictim(nodes_set);
+ // Remove the victim node from the connected component.
+ nodes_set.erase(victim);
+ // Removed node is a victim now.
+ targets.push_back(victim);
+ }
+ return targets;
+}
+
+bool CycleDetector::hasCycle(
+ const std::unordered_set<DirectedGraph::node_id> &nodes) const {
+ // Keeps track of the nodes the algorithms visited.
+ std::unordered_set<DirectedGraph::node_id> visited;
+ for (const DirectedGraph::node_id node_id : nodes) {
+ // If it is visited, then pass to the next one.
+ if (visited.count(node_id) == 1) {
+ continue;
+ }
+ // Save the backtracking information.
+ std::stack<DirectedGraph::node_id> to_visit;
+ // Mark this id as "to be visited".
+ to_visit.push(node_id);
+ // Start to visit nodes until it is done.
+ while (!to_visit.empty()) {
+ const DirectedGraph::node_id current_node = to_visit.top();
+ to_visit.pop();
+ // Mark the node coming from stack as "visited".
+ visited.insert(current_node);
+ // For all adjacent nodes of this "visited" node,
+ const std::vector<DirectedGraph::node_id> adjacents
+ = wait_for_graph_->getAdjacentNodes(current_node);
+ for (const DirectedGraph::node_id adjacent : adjacents) {
+ if (visited.count(adjacent) == 1) {
+ // If this adjacent node is a node we already visited, then
+ // there is a cycle.
+ return true;
+ } else if (nodes.count(adjacent) == 1 && visited.count(adjacent) == 0) {
+ // Otherwise, if it is a node that we did not visit before
+ // mark this nodes as "to be visited".
+ to_visit.push(adjacent);
+ }
+ }
+ }
+ }
+ // If we have already visited all nodes and could not find a cycle,
+ // then we should return "no cycle" result.
+ return false;
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/CycleDetector.hpp
----------------------------------------------------------------------
diff --git a/transaction/CycleDetector.hpp b/transaction/CycleDetector.hpp
new file mode 100644
index 0000000..6865e2d
--- /dev/null
+++ b/transaction/CycleDetector.hpp
@@ -0,0 +1,83 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_TRANSACTION_CYCLE_DETECTOR_HPP_
+#define QUICKSTEP_TRANSACTION_CYCLE_DETECTOR_HPP_
+
+#include <memory>
+#include <unordered_set>
+#include <vector>
+
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/StronglyConnectedComponents.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+namespace transaction {
+
+/** \addtogroup Transaction
+ * @{
+ */
+
+/**
+ * @brief Class for running cycle detection algorithm on directed graph.
+ */
+class CycleDetector {
+ public:
+ /**
+ * @brief Constructor for DirectedGraph.
+ *
+ * @param wait_for_graph Pointer to a directed wait-for graph.
+ */
+ explicit CycleDetector(DirectedGraph *wait_for_graph);
+
+ /**
+ * @brief Calculate which nodes should be killed to eliminate all cycles
+ * in the graph.
+ *
+ * @return Vector of node ids that should be killed to break all cycles.
+ */
+ std::vector<DirectedGraph::node_id> chooseVictimsToBreakCycle() const;
+
+ private:
+ std::vector<DirectedGraph::node_id> chooseVictimsInComponent(
+ const std::vector<DirectedGraph::node_id> &nodes) const;
+
+ inline DirectedGraph::node_id chooseVictim(
+ const std::unordered_set<DirectedGraph::node_id> &nodes_set) const {
+ // TODO(Hakan): This is very inefficient scheme, however in the
+ // future, we can use the transaction's priority
+ // as the victim selection parameter.
+ return *(nodes_set.begin());
+ }
+
+ // Checks whether the nodes in the set make a cycle.
+ bool hasCycle(const std::unordered_set<DirectedGraph::node_id> &within) const;
+
+ DirectedGraph *wait_for_graph_;
+
+ std::unique_ptr<StronglyConnectedComponents> strongly_connected_components_;
+
+ DISALLOW_COPY_AND_ASSIGN(CycleDetector);
+};
+
+/** @} */
+
+} // namespace transaction
+} // namespace quickstep
+
+#endif // QUICKSTEP_TRANSACTION_CYCLE_DETECTOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/DeadLockDetector.cpp
----------------------------------------------------------------------
diff --git a/transaction/DeadLockDetector.cpp b/transaction/DeadLockDetector.cpp
new file mode 100644
index 0000000..26ab115
--- /dev/null
+++ b/transaction/DeadLockDetector.cpp
@@ -0,0 +1,177 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/DeadLockDetector.hpp"
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <memory>
+#include <thread> // NOLINT(build/c++11)
+#include <utility>
+#include <vector>
+
+#include "transaction/CycleDetector.hpp"
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/LockTable.hpp"
+#include "transaction/Transaction.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace transaction {
+
+constexpr std::int64_t DeadLockDetector::kSleepDurationInSeconds;
+
+DeadLockDetector::DeadLockDetector(LockTable *lock_table,
+ std::atomic<DeadLockDetectorStatus> *status,
+ std::vector<DirectedGraph::node_id> *victims)
+ : tid_node_mapping_(std::make_unique<transaction_id_node_map>()),
+ lock_table_(lock_table),
+ status_(status),
+ victims_(victims) {
+}
+
+void DeadLockDetector::run() {
+ while (true) {
+ if (status_->load() == DeadLockDetectorStatus::kQuit) {
+ // DeadLockDetector should stop.
+ return;
+ }
+ while (status_->load() == DeadLockDetectorStatus::kDone) {
+ // LockTable has not process the previous batch yet.
+ }
+
+ // TODO(Hakan): Implement logging mechanism for deadlock detection
+ // start and end times.
+ std::vector<DirectedGraph::node_id> victim_new_batch = getAllVictims();
+
+ // Swap new batch with old batch to make LockTable to see new victims.
+ std::swap(victim_new_batch, *victims_);
+
+ // Signal LockTable that new batch is ready.
+ status_->store(DeadLockDetectorStatus::kDone);
+
+ // DeadLockDetector should run once in a predefined interval.
+ std::this_thread::sleep_for(
+ std::chrono::seconds(kSleepDurationInSeconds));
+ }
+}
+
+void DeadLockDetector::addPendingInfo(const transaction_id pending,
+ const transaction_id owner) {
+ const DirectedGraph::node_id pending_node_id = getNodeId(pending);
+ const DirectedGraph::node_id owner_node_id = getNodeId(owner);
+
+ // TODO(Hakan): Check first whether link is already created. Use checked
+ // version for adding an edge.
+ wait_for_graph_->addEdgeUnchecked(pending_node_id, owner_node_id);
+}
+
+void DeadLockDetector::deletePendingInfo(const transaction_id pending,
+ const transaction_id owner) {
+ LOG(FATAL) << "Not implemented";
+}
+
+bool DeadLockDetector::isDependent(const transaction_id pending,
+ const transaction_id owner) const {
+ LOG(FATAL) << "Not implemented";
+}
+
+std::vector<transaction_id>
+DeadLockDetector::getAllDependents(const transaction_id owner) const {
+ LOG(FATAL) << "Not implemented";
+}
+
+std::vector<transaction_id>
+DeadLockDetector::getAllDependees(transaction_id pending) {
+ const DirectedGraph::node_id pending_node_id = getNodeId(pending);
+ const std::vector<DirectedGraph::node_id> nodes
+ = wait_for_graph_->getAdjacentNodes(pending_node_id);
+ std::vector<transaction_id> transactions;
+ transactions.reserve(nodes.size());
+ for (const DirectedGraph::node_id node_id : nodes) {
+ const transaction_id tid = wait_for_graph_->getDataFromNode(node_id);
+ transactions.push_back(tid);
+ }
+ return transactions;
+}
+
+DirectedGraph::node_id DeadLockDetector::getNodeId(const transaction_id tid) {
+ DirectedGraph::node_id node_id;
+ if (tid_node_mapping_->count(tid) == 0) {
+ // If it is not created, create it.
+ node_id = addNode(tid);
+ } else {
+ // Otherwise find it in the map.
+ node_id = (*tid_node_mapping_)[tid];
+ }
+ return node_id;
+}
+
+
+DirectedGraph::node_id DeadLockDetector::addNode(const transaction_id tid) {
+ const DirectedGraph::node_id node_id =
+ wait_for_graph_->addNodeUnchecked(tid);
+ tid_node_mapping_->emplace(tid, node_id);
+ return node_id;
+}
+
+std::vector<transaction_id> DeadLockDetector::getAllVictims() {
+ std::vector<transaction_id> result_victims;
+ wait_for_graph_.reset(new DirectedGraph());
+
+ // Critical region on LockTable starts here.
+ lock_table_->latchShared();
+ for (const auto &lock_control_block : *lock_table_) {
+ const LockTable::lock_own_list &own_list = lock_control_block.second.first;
+ const LockTable::lock_pending_list &pending_list =
+ lock_control_block.second.second;
+
+ for (const auto &owned_lock_info : own_list) {
+ const transaction_id owned_transaction = owned_lock_info.first;
+ const DirectedGraph::node_id owned_node = getNodeId(owned_transaction);
+
+ for (const auto &pending_lock_info : pending_list) {
+ const transaction_id pending_transaction = pending_lock_info.first;
+ const DirectedGraph::node_id pending_node = getNodeId(pending_transaction);
+
+ wait_for_graph_->addEdgeUnchecked(pending_node, owned_node);
+ }
+ }
+ }
+
+ lock_table_->unlatchShared();
+ // Critical region on LockTable ends here.
+
+ const CycleDetector cycle_detector(wait_for_graph_.get());
+ const std::vector<DirectedGraph::node_id> victim_nodes =
+ cycle_detector.chooseVictimsToBreakCycle();
+ for (const DirectedGraph::node_id node_id : victim_nodes) {
+ const transaction_id victim_tid = wait_for_graph_->getDataFromNode(node_id);
+ result_victims.push_back(victim_tid);
+ }
+
+ // Destroy the wait graph. It will be reconstructed kSleepDurationSeconds
+ // seconds later.
+ wait_for_graph_.reset();
+
+ return result_victims;
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/DeadLockDetector.hpp
----------------------------------------------------------------------
diff --git a/transaction/DeadLockDetector.hpp b/transaction/DeadLockDetector.hpp
new file mode 100644
index 0000000..6897afb
--- /dev/null
+++ b/transaction/DeadLockDetector.hpp
@@ -0,0 +1,156 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_TRANSACTION_DEAD_LOCK_DETECTOR_HPP_
+#define QUICKSTEP_TRANSACTION_DEAD_LOCK_DETECTOR_HPP_
+
+#include <atomic>
+#include <cstdint>
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "threading/Thread.hpp"
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/Transaction.hpp"
+
+namespace quickstep {
+namespace transaction {
+
+class LockTable;
+
+/** \addtogroup Transaction
+ * @{
+ */
+
+/**
+ * @brief Notification mechanism between LockManager and DeadLockDetector.
+ **/
+enum class DeadLockDetectorStatus {
+ kNotReady = 0,
+ kDone,
+ kQuit,
+};
+
+/**
+ * @brief Class for deadlock detection on wait-for graph.
+ **/
+class DeadLockDetector : public Thread {
+ public:
+ typedef std::unordered_map<transaction_id, DirectedGraph::node_id>
+ transaction_id_node_map;
+
+ /**
+ * @brief Constructor for DeadLockDetector.
+ *
+ * @param lock_table Pointer to lock table, which this class gets the
+ * necessary information.
+ * @param status Pointer to status object which will act as a message
+ * passing algorithm between LockManager.
+ * @param victims Message passing buffer betwen DeadLockDetector and
+ * LockManager.
+ **/
+ DeadLockDetector(LockTable *lock_table,
+ std::atomic<DeadLockDetectorStatus> *status,
+ std::vector<DirectedGraph::node_id> *victims);
+
+ void run() override;
+
+ /**
+ * @brief Adds pending information based on a resource conflict.
+ *
+ * @param pending Id of the transaction that waits for the resource lock.
+ * @param owner Id of the transaction that owns the resource lock.
+ */
+ void addPendingInfo(const transaction_id pending,
+ const transaction_id owner);
+
+ /**
+ * @brief Deletes pending information on a resource.
+ *
+ * @param pending Id of the transaction that waits for the resource lock.
+ * @param owner Id of the transaction that owns the resource lock.
+ *
+ * @warning This method is not implemented yet.
+ */
+ void deletePendingInfo(const transaction_id pending,
+ const transaction_id owner);
+
+ /**
+ * @brief Check whether first transaction waits for the latter.
+ *
+ * @param pending Id of the transaction which will be checked whether
+ * it waits for the other.
+ * @param owner Id of the transaction which will be checked whether
+ * it is waited by the first.
+ *
+ * @warning This method is not implemented yet.
+ */
+ bool isDependent(const transaction_id pending,
+ const transaction_id owner) const;
+
+ /**
+ * @brief Gives the ids of transactions that wait for the owner transaction.
+ *
+ * @param owner Id of the transaction whose the penders will be returned.
+ * @return Vector of transaction ids that wait for owner.
+ */
+ std::vector<transaction_id> getAllDependents(const transaction_id owner) const;
+
+ /**
+ * @brief Gives the ids of transaction that the pending transaction waits for.
+ * @warning This method is not implemented yet.
+
+ * @param pending Id of the transaction that is pending.
+ * @return Vector of transaction ids which the pending transaction waits for.
+ */
+ std::vector<transaction_id> getAllDependees(transaction_id pending);
+
+ /**
+ * @brief Gives the list of victims whose the terminations will end the cycle.
+ *
+ * @return Vector of victim transaction ids.
+ */
+ std::vector<transaction_id> getAllVictims();
+
+ private:
+ static constexpr std::int64_t kSleepDurationInSeconds = 5;
+
+ DirectedGraph::node_id getNodeId(const transaction_id tid);
+
+ DirectedGraph::node_id addNode(const transaction_id tid);
+
+ // Owned pointer to wait-for graph.
+ std::unique_ptr<DirectedGraph> wait_for_graph_;
+
+ // Mapping from TransactioId to NodeId in graph.
+ std::unique_ptr<transaction_id_node_map> tid_node_mapping_;
+
+ // Pointer to lock table. Dependence edges will be created
+ // by the information got from lock table.
+ LockTable *lock_table_;
+
+ std::atomic<DeadLockDetectorStatus> *status_;
+ std::vector<DirectedGraph::node_id> *victims_;
+};
+
+/** @} */
+
+} // namespace transaction
+} // namespace quickstep
+
+#endif // QUICKSTEP_TRANSACTION_DEAD_LOCK_DETECTOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/DirectedGraph.hpp
----------------------------------------------------------------------
diff --git a/transaction/DirectedGraph.hpp b/transaction/DirectedGraph.hpp
index 89ce9c6..16b551a 100644
--- a/transaction/DirectedGraph.hpp
+++ b/transaction/DirectedGraph.hpp
@@ -21,8 +21,8 @@
#include <algorithm>
#include <cstddef>
#include <cstdint>
+#include <iterator>
#include <memory>
-#include <stack>
#include <unordered_set>
#include <vector>
@@ -39,7 +39,7 @@ namespace transaction {
*/
/**
- * @brief Class for representing a directed graph. Vertices are transaction
+ * @brief Class for representing a directed graph. Vertices are transaction
* ids, edges are wait-for relations.
**/
class DirectedGraph {
@@ -54,34 +54,31 @@ class DirectedGraph {
/**
* @brief Adds a new node to the graph with the given transaction id.
* It does not check whether the transaction id is valid or not.
- * @warning Pointer ownership will pass to the graph, therefore it
- * should not be deleted.
*
- * @param data Pointer to the transaction id that will be contained
+ * @param transaction_id_payload Transaction id that will be contained
* in the node.
* @return Id of the newly created node.
**/
- inline node_id addNodeUnchecked(transaction_id *data) {
- nodes_.emplace_back(data);
+ inline
+ node_id addNodeUnchecked(const transaction_id transaction_id_payload) {
+ nodes_.emplace_back(transaction_id_payload);
return nodes_.size() - 1;
}
/**
* @brief Adds a new node to the graph with the given transaction id.
* It checks whether the transaction id is valid or not.
- * @warning Pointer ownership will pass to the graph, therefore it
- * should not be deleted.
*
- * @param data Pointer to the transaction id that will be contained
+ * @param transaction_id_payload Transaction id that will be contained
* in the node.
* @return Id of the newly created node.
**/
- inline node_id addNodeCheckExists(transaction_id *data) {
- for (std::vector<DirectedGraphNode>::const_iterator
- it = nodes_.cbegin(); it != nodes_.cend(); ++it) {
- CHECK(*data != it->getData());
+ node_id addNodeCheckExists(const transaction_id transaction_id_payload) {
+ for (const auto &node : nodes_) {
+ CHECK(transaction_id_payload != node.getData());
}
- nodes_.emplace_back(data);
+
+ nodes_.emplace_back(transaction_id_payload);
return nodes_.size() - 1;
}
@@ -91,10 +88,10 @@ class DirectedGraph {
* @warning Does not check arguments are legit. It may cause
* out of range errors.
*
- * @param fromNode The node that edge is orginated.
+ * @param fromNode The node that edge is originated.
* @param toNode The node that edge is ended.
**/
- inline void addEdgeUnchecked(node_id from_node, node_id to_node) {
+ void addEdgeUnchecked(node_id from_node, node_id to_node) {
nodes_[from_node].addOutgoingEdge(to_node);
}
@@ -105,7 +102,7 @@ class DirectedGraph {
* @param fromNode The node that edge is orginated.
* @param toNode The node that edge is ended.
**/
- inline void addEdgeCheckExists(node_id from_node, node_id to_node) {
+ void addEdgeCheckExists(node_id from_node, node_id to_node) {
CHECK(from_node < getNumNodes() && to_node < getNumNodes());
nodes_[from_node].addOutgoingEdge(to_node);
}
@@ -119,7 +116,7 @@ class DirectedGraph {
* @param toNode Id of the node that edge is ended.
* @return True if there is an edge, false otherwise.
**/
- inline bool hasEdge(node_id from_node, node_id to_node) const {
+ bool hasEdge(node_id from_node, node_id to_node) const {
DCHECK(from_node < getNumNodes() && to_node < getNumNodes());
return nodes_[from_node].hasOutgoingEdge(to_node);
}
@@ -130,7 +127,7 @@ class DirectedGraph {
* @param node Id of the node that the data is got from.
* @return Id of the transaction that this node contains.
**/
- inline transaction_id getDataFromNode(node_id node) const {
+ transaction_id getDataFromNode(node_id node) const {
DCHECK(node < getNumNodes());
return nodes_[node].getData();
}
@@ -140,7 +137,7 @@ class DirectedGraph {
*
* @return The number of nodes the graph has.
**/
- inline std::size_t getNumNodes() const {
+ std::size_t getNumNodes() const {
return nodes_.size();
}
@@ -158,18 +155,18 @@ class DirectedGraph {
// Class for representing a graph node.
class DirectedGraphNode {
public:
- explicit DirectedGraphNode(transaction_id *data)
- : data_(data) {}
+ explicit DirectedGraphNode(const transaction_id payload)
+ : transaction_id_payload_(payload) {}
- inline void addOutgoingEdge(node_id to_node) {
+ void addOutgoingEdge(node_id to_node) {
outgoing_edges_.insert(to_node);
}
- inline bool hasOutgoingEdge(node_id to_node) const {
+ bool hasOutgoingEdge(node_id to_node) const {
return outgoing_edges_.count(to_node) == 1;
}
- inline std::vector<node_id> getOutgoingEdges() const {
+ std::vector<node_id> getOutgoingEdges() const {
// TODO(hakan): Benchmark this version and the alternative which the
// function returns const reference and the uniqueness
// is imposed in the outgoing_edges_ as a vector.
@@ -179,13 +176,12 @@ class DirectedGraph {
return result;
}
- inline transaction_id getData() const {
- return *(data_.get());
+ transaction_id getData() const {
+ return transaction_id_payload_;
}
private:
- // Owner pointer to transaction id.
- std::unique_ptr<transaction_id> data_;
+ const transaction_id transaction_id_payload_;
// Endpoint nodes of outgoing edges originated from this node.
std::unordered_set<node_id> outgoing_edges_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/LockManager.cpp
----------------------------------------------------------------------
diff --git a/transaction/LockManager.cpp b/transaction/LockManager.cpp
new file mode 100644
index 0000000..da6181a
--- /dev/null
+++ b/transaction/LockManager.cpp
@@ -0,0 +1,237 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/LockManager.hpp"
+
+#include <cstdint>
+#include <stack>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "transaction/AccessMode.hpp"
+#include "transaction/DeadLockDetector.hpp"
+#include "transaction/LockRequest.hpp"
+#include "transaction/LockTable.hpp"
+#include "transaction/ResourceId.hpp"
+#include "transaction/Transaction.hpp"
+#include "transaction/TransactionTable.hpp"
+#include "utility/ThreadSafeQueue.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace transaction {
+
+DEFINE_uint64(max_try_incoming, 10000,
+ "The maximum number of tries that lock manager checks incoming "
+ "request buffer until the buffer is empty.");
+
+DEFINE_uint64(max_try_inner, 6000,
+ "The maximum number of tries that lock manager checks inner "
+ "request buffer until the buffer is empty.");
+
+LockManager::LockManager(ThreadSafeQueue<LockRequest> *incoming_requests,
+ ThreadSafeQueue<LockRequest> *permitted_requests)
+ : lock_table_(std::make_unique<LockTable>()),
+ transaction_table_(std::make_unique<TransactionTable>()),
+ detector_status_(DeadLockDetectorStatus::kNotReady),
+ deadlock_detector_(std::make_unique<DeadLockDetector>(lock_table_.get(),
+ &detector_status_,
+ &victim_result_)),
+ incoming_requests_(*incoming_requests),
+ permitted_requests_(*permitted_requests),
+ inner_pending_requests_() {
+}
+
+LockManager::~LockManager() {
+ deadlock_detector_->join();
+}
+
+void LockManager::run() {
+ deadlock_detector_->start();
+
+ const std::uint64_t kMaxTryIncoming =
+ static_cast<std::uint64_t>(FLAGS_max_try_incoming);
+ const std::uint64_t kMaxTryInner =
+ static_cast<std::uint64_t>(FLAGS_max_try_incoming);
+
+ while (true) {
+ for (std::uint64_t tries = 0; tries < kMaxTryIncoming; ++tries) {
+ if (!incoming_requests_.empty()) {
+ const LockRequest request = incoming_requests_.popOne();
+ if (request.getRequestType() == RequestType::kReleaseLocks) {
+ CHECK(releaseAllLocks(request.getTransactionId()))
+ << "Unexpected condition occured.";
+
+ } else if (acquireLock(request.getTransactionId(),
+ request.getResourceId(),
+ request.getAccessMode())) {
+ LOG(INFO) << "Transaction "
+ << std::to_string(request.getTransactionId())
+ << " is waiting " + request.getResourceId().toString();
+
+ inner_pending_requests_.push(request);
+ } else {
+ LOG(INFO) << "Transaction "
+ << std::to_string(request.getTransactionId())
+ << " acquired " + request.getResourceId().toString();
+
+ permitted_requests_.push(request);
+ }
+ }
+ }
+
+ for (std::uint64_t tries = 0; tries < kMaxTryInner; ++tries) {
+ if (!inner_pending_requests_.empty()) {
+ const LockRequest request = inner_pending_requests_.front();
+
+ if (acquireLock(request.getTransactionId(), request.getResourceId(),
+ request.getAccessMode())) {
+ inner_pending_requests_.pop();
+ permitted_requests_.push(request);
+ }
+ }
+ }
+
+ // Resolve deadlocks.
+ killVictims();
+ }
+}
+
+bool LockManager::acquireLock(const transaction_id tid,
+ const ResourceId &rid,
+ const AccessMode &access_mode) {
+ std::stack<std::pair<ResourceId, AccessMode>> stack;
+ ResourceId current_rid = rid;
+ AccessMode current_access_mode = access_mode;
+ stack.push(std::make_pair(current_rid, current_access_mode));
+
+ while (current_rid.hasParent()) {
+ current_rid = current_rid.getParentResourceId();
+ current_access_mode = (current_access_mode.isShareLock() ||
+ current_access_mode.isIntentionShareLock())
+ ? AccessMode(AccessMode::IsLockMode())
+ : AccessMode(AccessMode::IxLockMode());
+
+ stack.push(std::make_pair(current_rid, current_access_mode));
+ }
+
+ lock_table_->latchExclusive();
+
+ while (!stack.empty()) {
+ const std::pair<ResourceId, AccessMode> pair_to_pick = stack.top();
+ const ResourceId rid_to_pick = pair_to_pick.first;
+ const AccessMode access_mode_to_pick = pair_to_pick.second;
+
+ if (!acquireLockInternal(tid, rid_to_pick, access_mode_to_pick)) {
+ lock_table_->unlatchExclusive();
+ return false;
+ }
+ stack.pop();
+ }
+ lock_table_->unlatchExclusive();
+ return true;
+}
+
+bool LockManager::releaseAllLocks(const transaction_id tid,
+ const bool latch_table) {
+ const std::vector<ResourceId> resource_ids
+ = transaction_table_->getResourceIdList(tid);
+ const TransactionTableResult transaction_deleted
+ = transaction_table_->deleteTransaction(tid);
+
+ CHECK(transaction_deleted != TransactionTableResult::kTransactionDeleteError)
+ << "In LockManager.releaseAllLocks: Transaction could not be deleted!";
+
+ if (latch_table) {
+ lock_table_->latchExclusive();
+ }
+ for (const auto &resource_id : resource_ids) {
+ const LockTableResult lock_deleted = lock_table_->deleteLock(tid, resource_id);
+
+ LOG(INFO) << "Transaction "
+ << std::to_string(tid)
+ << " released lock:"
+ << resource_id.toString();
+ CHECK(lock_deleted != LockTableResult::kDeleteError)
+ << "In LockManager.releaseAllLock lock could not be deleted from "
+ "LockTable";
+ }
+ if (latch_table) {
+ lock_table_->unlatchExclusive();
+ }
+ return true;
+}
+
+bool LockManager::acquireLockInternal(const transaction_id tid,
+ const ResourceId &rid,
+ const AccessMode &access_mode) {
+ const LockTableResult lock_result = lock_table_->putLock(tid, rid, access_mode);
+ CHECK(lock_result != LockTableResult::kPutError)
+ << "Unexpected result in LockManager.acquireLockInternal";
+
+ switch (lock_result) {
+ case LockTableResult::kAlreadyInOwned: {
+ return true;
+ }
+ case LockTableResult::kPlacedInOwned: {
+ const TransactionTableResult transaction_result
+ = transaction_table_->putOwnEntry(tid, rid, access_mode);
+ CHECK(transaction_result == TransactionTableResult::kPlacedInOwned)
+ << "Unexpected result in LockManager.acquireLockInternal: "
+ "Mismatch of table results: LockTable entry is owned, "
+ "whereas TransactionTable entry is not owned.";
+ return true;
+ }
+ case LockTableResult::kAlreadyInPending: {
+ return false;
+ }
+ case LockTableResult::kPlacedInPending: {
+ const TransactionTableResult transaction_result =
+ transaction_table_->putPendingEntry(tid, rid, access_mode);
+ CHECK(transaction_result == TransactionTableResult::kPlacedInPending)
+ << "Unexpected result in LockManager.acquireLockInternal: "
+ "Mismatch of table results: LockTable entry is pending, "
+ "whereas TransactionTable entry is not pending";
+ return false;
+ }
+ default: {
+ return false;
+ }
+ }
+}
+
+void LockManager::killVictims() {
+ if (detector_status_.load() == DeadLockDetectorStatus::kDone) {
+ lock_table_->latchExclusive();
+ for (const auto victim_transaction_id : victim_result_) {
+ releaseAllLocks(victim_transaction_id, false);
+ // TODO(Hakan): Find a way to kill transaction, so that requests with this
+ // tid should be ignored.
+ LOG(INFO) << "Killed transaction "
+ << std::to_string(victim_transaction_id);
+ }
+ lock_table_->unlatchExclusive();
+ }
+ victim_result_.clear();
+ detector_status_.store(DeadLockDetectorStatus::kNotReady);
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/LockManager.hpp
----------------------------------------------------------------------
diff --git a/transaction/LockManager.hpp b/transaction/LockManager.hpp
new file mode 100644
index 0000000..40ee6c8
--- /dev/null
+++ b/transaction/LockManager.hpp
@@ -0,0 +1,128 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_TRANSACTION_LOCK_MANAGER_HPP_
+#define QUICKSTEP_TRANSACTION_LOCK_MANAGER_HPP_
+
+#include <atomic>
+#include <memory>
+#include <queue>
+#include <vector>
+
+#include "threading/Thread.hpp"
+#include "transaction/DeadLockDetector.hpp"
+#include "transaction/LockRequest.hpp"
+#include "transaction/Transaction.hpp"
+
+namespace quickstep {
+
+template <typename T> class ThreadSafeQueue;
+
+namespace transaction {
+
+class AccessMode;
+class LockTable;
+class ResourceId;
+class TransactionTable;
+
+/** \addtogroup Transaction
+ * @{
+ **/
+
+/**
+ * @brief Class for centralized location of acquisition and releasing
+ * of resource locks.
+ **/
+class LockManager : public Thread {
+ public:
+ /**
+ * @brief Constructor
+ *
+ * @param incoming_requests Queue for the lock requests that are waiting
+ * for the permission.
+ * @param permitted_requests Queue for the lock requests that are granted
+ * permission.
+ **/
+ LockManager(ThreadSafeQueue<LockRequest> *incoming_requests,
+ ThreadSafeQueue<LockRequest> *permitted_requests);
+
+ /**
+ * @brief Destructor for LockManager. It handles the thread
+ * joins that it owns.
+ **/
+ ~LockManager();
+
+ /**
+ * @brief Method for defining the LockManager's thread main logic.
+ *
+ * @warning Users must not use this function directly. Instead use
+ * start() method inherited from the Thread class.
+ **/
+ void run() override;
+
+ /**
+ * @brief Acquires the lock on resource with specified access mode.
+ *
+ * @param tid Id of the transaction which the resource lock is acquired for.
+ * @param rid Id of the resource on which the resource lock is acquired.
+ * @param access_mode Permissible access mode on resource.
+ *
+ * @return True if it can acquire the lock from root to leaf lock hierarchy,
+ * false otherwise.
+ **/
+ bool acquireLock(const transaction_id tid,
+ const ResourceId &rid,
+ const AccessMode &access_mode);
+
+ /**
+ * @brief Releases all locks hold by the transaction.
+ *
+ * @param tid Id of the transaction whose locks will be released.
+ * @param latch_table If it is true, the method latch the whole
+ * lock table, which is default.
+ * @return True if tid releases all of its locks.
+ **/
+ bool releaseAllLocks(const transaction_id tid,
+ const bool latch_table = true);
+
+ /**
+ * @brief Release the locks acquired by the transactions contained
+ * in victim buffer to break the deadlock.
+ **/
+ void killVictims();
+
+ private:
+ bool acquireLockInternal(const transaction_id tid,
+ const ResourceId &rid,
+ const AccessMode &access_mode);
+
+ std::unique_ptr<LockTable> lock_table_;
+ std::unique_ptr<TransactionTable> transaction_table_;
+ std::atomic<DeadLockDetectorStatus> detector_status_;
+ std::vector<transaction_id> victim_result_;
+ std::unique_ptr<DeadLockDetector> deadlock_detector_;
+ ThreadSafeQueue<LockRequest> &incoming_requests_;
+ ThreadSafeQueue<LockRequest> &permitted_requests_;
+ std::queue<LockRequest> inner_pending_requests_;
+};
+
+/** @} */
+
+} // namespace transaction
+} // namespace quickstep
+
+#endif // QUICKSTEP_TRANSACTION_LOCK_MANAGER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/LockTable.cpp
----------------------------------------------------------------------
diff --git a/transaction/LockTable.cpp b/transaction/LockTable.cpp
index 77986f6..7b568fc 100644
--- a/transaction/LockTable.cpp
+++ b/transaction/LockTable.cpp
@@ -18,15 +18,11 @@
#include "transaction/LockTable.hpp"
#include <list>
-#include <unordered_map>
#include <utility>
-#include "threading/SharedMutex.hpp"
#include "transaction/AccessMode.hpp"
#include "transaction/Lock.hpp"
-#include "transaction/ResourceId.hpp"
#include "transaction/Transaction.hpp"
-#include "utility/Macros.hpp"
namespace quickstep {
namespace transaction {
@@ -34,7 +30,7 @@ namespace transaction {
LockTableResult
LockTable::putLock(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
// TODO(hakan): Lock upgrade is not supported.
lock_list_pair &lock_list_pair = internal_map_[rid];
@@ -47,7 +43,7 @@ LockTable::putLock(const transaction_id tid,
for (lock_own_list::const_iterator it = lock_own_list.cbegin();
it != lock_own_list.cend(); ++it) {
if (it->first == tid && it->second.getAccessMode() == access_mode) {
- return LockTableResult::kALREADY_IN_OWNED;
+ return LockTableResult::kAlreadyInOwned;
}
}
@@ -56,7 +52,7 @@ LockTable::putLock(const transaction_id tid,
for (lock_pending_list::const_iterator it = lock_pending_list.cbegin();
it != lock_pending_list.cend(); ++it) {
if (it->first == tid && it->second.getAccessMode() == access_mode) {
- return LockTableResult::kALREADY_IN_PENDING;
+ return LockTableResult::kAlreadyInPending;
}
}
@@ -68,18 +64,18 @@ LockTable::putLock(const transaction_id tid,
if (!access_mode.isCompatible(it->second.getAccessMode())) {
lock_pending_list.push_back(std::make_pair(tid,
Lock(rid, access_mode)));
- return LockTableResult::kPLACED_IN_PENDING;
+ return LockTableResult::kPlacedInPending;
}
}
lock_own_list.push_back(std::make_pair(tid, Lock(rid, access_mode)));
- return LockTableResult::kPLACED_IN_OWNED;
+ return LockTableResult::kPlacedInOwned;
} else {
// If the pending list is not empty, even if the lock request is compatible
// with other owned lock entries, we put the new request into the pending
// list to eliminate starvation.
lock_pending_list.push_back(std::make_pair(tid, Lock(rid, access_mode)));
- return LockTableResult::kPLACED_IN_PENDING;
+ return LockTableResult::kPlacedInPending;
}
}
@@ -105,7 +101,7 @@ LockTable::deleteLock(const transaction_id tid,
// compatible with the remaining owned entries.
movePendingToOwned(rid);
- return LockTableResult::kDEL_FROM_OWNED;
+ return LockTableResult::kDeleteFromOwned;
}
}
@@ -116,13 +112,13 @@ LockTable::deleteLock(const transaction_id tid,
if (it->first == tid) {
// If it exists, erase it from pending list.
lock_pending_list.erase(it);
- return LockTableResult::kDEL_FROM_PENDING;
+ return LockTableResult::kDeleteFromPending;
}
}
// Execution reaches here, if we cannot find the corresponding lock entry
// in the both list.
- return LockTableResult::kDEL_ERROR;
+ return LockTableResult::kDeleteError;
}
void LockTable::movePendingToOwned(const ResourceId &rid) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/LockTable.hpp
----------------------------------------------------------------------
diff --git a/transaction/LockTable.hpp b/transaction/LockTable.hpp
index 5a0612e..529db12 100644
--- a/transaction/LockTable.hpp
+++ b/transaction/LockTable.hpp
@@ -23,7 +23,6 @@
#include <utility>
#include "threading/SharedMutex.hpp"
-#include "transaction/AccessMode.hpp"
#include "transaction/Lock.hpp"
#include "transaction/ResourceId.hpp"
#include "transaction/Transaction.hpp"
@@ -32,6 +31,8 @@
namespace quickstep {
namespace transaction {
+class AccessMode;
+
/** \addtogroup Transaction
* @{
*/
@@ -40,14 +41,14 @@ namespace transaction {
* @brief Represents different results for LockTable's methods.
**/
enum class LockTableResult {
- kPLACED_IN_OWNED = 0,
- kPLACED_IN_PENDING,
- kALREADY_IN_OWNED,
- kALREADY_IN_PENDING,
- kDEL_FROM_OWNED,
- kDEL_FROM_PENDING,
- kDEL_ERROR,
- kPUT_ERROR,
+ kPlacedInOwned = 0,
+ kPlacedInPending,
+ kAlreadyInOwned,
+ kAlreadyInPending,
+ kDeleteFromOwned,
+ kDeleteFromPending,
+ kDeleteError,
+ kPutError,
};
/**
@@ -79,32 +80,31 @@ class LockTable {
* @param rid Id of the resource to be locked.
* @param access_mode Access mode of the lock.
*
- * @return LockTableResult::kPLACED_IN_OWNED if lock is granted,
- * LockTableResult::kPLACED_IN_PENDING if lock is not granted,
- * LockTableResult::kALREADY_IN_OWNED if lock has been
+ * @return LockTableResult::kPlacedInOwned if lock is granted,
+ * LockTableResult::kPlacedInPending if lock is not granted,
+ * LockTableResult::kAlreadyInOwned if lock has been
* already granted,
- * LockTableResult::kALREADY_IN_PENDING if lock has been
+ * LockTableResult::kAlreadyInPending if lock has been
* already pending.
**/
LockTableResult putLock(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Deletes the lock entry.
*
* @param tid Id of the transaction that owns or awaits.
* @param rid Id of resource that the lock covers.
*
- * @return LockTableResult::kDEL_FROM_OWNED if the lock is deleted from
+ * @return LockTableResult::kDelFromOwned if the lock is deleted from
* owned list,
- * LockTableResult::kDEL_FROM_PENDING if the lock is deleted from
+ * LockTableResult::kDelFromPending if the lock is deleted from
* pending list,
- * LockTableResult::kDEL_ERROR if the lock cannot be found
+ * LockTableResult::kDelError if the lock cannot be found
**/
LockTableResult deleteLock(const transaction_id tid,
const ResourceId &rid);
-
/**
* @brief Iterator for begin position.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/ResourceId.hpp
----------------------------------------------------------------------
diff --git a/transaction/ResourceId.hpp b/transaction/ResourceId.hpp
index b9d1cdf..3a770dd 100644
--- a/transaction/ResourceId.hpp
+++ b/transaction/ResourceId.hpp
@@ -19,7 +19,6 @@
#define QUICKSTEP_TRANSACTION_RESOURCE_ID_HPP_
#include <cstddef>
-#include <limits>
#include <string>
#include "catalog/CatalogTypedefs.hpp"
@@ -202,10 +201,10 @@ class ResourceId {
return tuple_id_ == kTupleIdPlaceholder;
}
- const database_id db_id_;
- const relation_id rel_id_;
- const block_id block_id_;
- const tuple_id tuple_id_;
+ database_id db_id_;
+ relation_id rel_id_;
+ block_id block_id_;
+ tuple_id tuple_id_;
};
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/StronglyConnectedComponents.cpp
----------------------------------------------------------------------
diff --git a/transaction/StronglyConnectedComponents.cpp b/transaction/StronglyConnectedComponents.cpp
index 89daf46..f50ed85 100644
--- a/transaction/StronglyConnectedComponents.cpp
+++ b/transaction/StronglyConnectedComponents.cpp
@@ -17,7 +17,6 @@
#include "transaction/StronglyConnectedComponents.hpp"
-#include <cstddef>
#include <cstdint>
#include <stack>
#include <unordered_map>
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/Transaction.cpp
----------------------------------------------------------------------
diff --git a/transaction/Transaction.cpp b/transaction/Transaction.cpp
deleted file mode 100644
index 3478d01..0000000
--- a/transaction/Transaction.cpp
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#include "transaction/Transaction.hpp"
-
-#include <functional>
-
-namespace quickstep {
-
-namespace transaction {
-
-TransactionId Transaction::getTransactionId() const {
- return tid_;
-}
-
-void Transaction::setStatus(TransactionStatus status) {
- status_ = status;
-}
-
-TransactionStatus Transaction::getStatus() const {
- return status_;
-}
-
-bool Transaction::operator==(const Transaction &other) const {
- return tid_ == other.tid_;
-}
-
-std::size_t Transaction::TransactionHasher::operator()(const Transaction &transaction) const {
- return std::hash<TransactionId>()(transaction.tid_);
-}
-
-} // namespace transaction
-
-} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/TransactionTable.cpp
----------------------------------------------------------------------
diff --git a/transaction/TransactionTable.cpp b/transaction/TransactionTable.cpp
index 993703a..3e37439 100644
--- a/transaction/TransactionTable.cpp
+++ b/transaction/TransactionTable.cpp
@@ -33,7 +33,7 @@ namespace transaction {
TransactionTableResult
TransactionTable::putOwnEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
transaction_list_pair &transaction_list_pair = internal_map_[tid];
transaction_own_list &transaction_own_list = transaction_list_pair.first;
@@ -45,7 +45,7 @@ TransactionTable::putOwnEntry(const transaction_id tid,
TransactionTableResult
TransactionTable::putPendingEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
transaction_list_pair &transaction_list_pair = internal_map_[tid];
transaction_pending_list &transaction_pending_list
= transaction_list_pair.second;
@@ -59,7 +59,7 @@ TransactionTable::putPendingEntry(const transaction_id tid,
TransactionTableResult
TransactionTable::deleteOwnEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
transaction_list_pair &transaction_list_pair = internal_map_[tid];
transaction_own_list &transaction_own_list = transaction_list_pair.first;
@@ -79,7 +79,7 @@ TransactionTable::deleteOwnEntry(const transaction_id tid,
TransactionTableResult
TransactionTable::deletePendingEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
transaction_list_pair &transaction_list_pair = internal_map_[tid];
transaction_pending_list &transaction_pending_list
= transaction_list_pair.second;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/TransactionTable.hpp
----------------------------------------------------------------------
diff --git a/transaction/TransactionTable.hpp b/transaction/TransactionTable.hpp
index a5e1da4..29df536 100644
--- a/transaction/TransactionTable.hpp
+++ b/transaction/TransactionTable.hpp
@@ -23,7 +23,6 @@
#include <utility>
#include <vector>
-#include "transaction/AccessMode.hpp"
#include "transaction/Lock.hpp"
#include "transaction/ResourceId.hpp"
#include "transaction/Transaction.hpp"
@@ -32,6 +31,8 @@
namespace quickstep {
namespace transaction {
+class AccessMode;
+
/** \addtogroup Transaction
* @{
*/
@@ -81,7 +82,7 @@ class TransactionTable {
**/
TransactionTableResult putOwnEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Puts a pending entry of the given resource id in the given
@@ -95,7 +96,7 @@ class TransactionTable {
**/
TransactionTableResult putPendingEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Deletes the owned entry corresponding to the resource id
@@ -110,7 +111,7 @@ class TransactionTable {
**/
TransactionTableResult deleteOwnEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Deletes the pending entry corresponding to the resource id
@@ -124,7 +125,7 @@ class TransactionTable {
**/
TransactionTableResult deletePendingEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Returns a vector of resource ids which the corresponding transaction
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/AccessMode_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/AccessMode_unittest.cpp b/transaction/tests/AccessMode_unittest.cpp
index fa51525..3287fb0 100644
--- a/transaction/tests/AccessMode_unittest.cpp
+++ b/transaction/tests/AccessMode_unittest.cpp
@@ -25,12 +25,12 @@ namespace transaction {
class AccessModeTest : public ::testing::Test {
protected:
AccessModeTest()
- : nl_mode_(AccessModeType::kNoLock),
- is_mode_(AccessModeType::kIsLock),
- ix_mode_(AccessModeType::kIxLock),
- s_mode_(AccessModeType::kSLock),
- six_mode_(AccessModeType::kSixLock),
- x_mode_(AccessModeType::kXLock) {
+ : nl_mode_(AccessMode::NoLockMode()),
+ is_mode_(AccessMode::IsLockMode()),
+ ix_mode_(AccessMode::IxLockMode()),
+ s_mode_(AccessMode::SLockMode()),
+ six_mode_(AccessMode::SixLockMode()),
+ x_mode_(AccessMode::XLockMode()) {
}
const AccessMode nl_mode_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/CycleDetector_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/CycleDetector_unittest.cpp b/transaction/tests/CycleDetector_unittest.cpp
new file mode 100644
index 0000000..6edaa63
--- /dev/null
+++ b/transaction/tests/CycleDetector_unittest.cpp
@@ -0,0 +1,157 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/CycleDetector.hpp"
+
+#include <cstdint>
+#include <memory>
+#include <stack>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/Transaction.hpp"
+
+#include "gtest/gtest.h"
+
+namespace quickstep {
+namespace transaction {
+
+class CycleDetectorTest : public testing::Test {
+ protected:
+ const std::uint64_t kNumberOfTransactions = 12;
+
+ CycleDetectorTest()
+ : wait_for_graph_(std::make_unique<DirectedGraph>()) {
+ }
+
+ virtual void SetUp() {
+ std::vector<transaction_id> transactions(kNumberOfTransactions);
+ for (std::uint64_t i = 0; i < kNumberOfTransactions; ++i) {
+ transactions.push_back(transaction_id(i));
+ }
+
+ std::vector<DirectedGraph::node_id> node_ids;
+ for (std::uint64_t i = 0; i < kNumberOfTransactions; ++i) {
+ node_ids.push_back(wait_for_graph_->addNodeUnchecked(transactions[i]));
+ }
+ }
+
+ void initializeCycleDetector() {
+ for (const auto &edge : edges_) {
+ wait_for_graph_->addEdgeUnchecked(edge.first, edge.second);
+ }
+
+ cycle_detector_.reset(new CycleDetector(wait_for_graph_.get()));
+ }
+
+ void checkVictims(
+ const std::unordered_set<DirectedGraph::node_id> &expected_victims) {
+ const std::vector<DirectedGraph::node_id> victims =
+ cycle_detector_->chooseVictimsToBreakCycle();
+
+ std::unordered_set<DirectedGraph::node_id> remaining_nodes;
+
+ for (DirectedGraph::node_id node = 0; node < wait_for_graph_->getNumNodes();
+ ++node) {
+ if (std::find(victims.begin(), victims.end(), node) == victims.end()) {
+ // Node is not in victims, then insert it to remaining set.
+ remaining_nodes.insert(node);
+ }
+ }
+
+ for (const auto node : remaining_nodes) {
+ ASSERT_FALSE(isSelfReachableNode(node, remaining_nodes));
+ }
+ }
+
+ bool isSelfReachableNode(
+ const DirectedGraph::node_id start_node,
+ const std::unordered_set<DirectedGraph::node_id> &node_set) {
+ std::unordered_set<DirectedGraph::node_id> marked_nodes;
+ std::stack<DirectedGraph::node_id> to_be_visied_nodes;
+
+ const std::vector<DirectedGraph::node_id> neighbors_of_start_node =
+ wait_for_graph_->getAdjacentNodes(start_node);
+ for (const auto node : neighbors_of_start_node) {
+ marked_nodes.insert(node);
+ to_be_visied_nodes.push(node);
+ }
+
+ while (!to_be_visied_nodes.empty()) {
+ const DirectedGraph::node_id current_node = to_be_visied_nodes.top();
+ to_be_visied_nodes.pop();
+ if (current_node == start_node) {
+ return true;
+ }
+ if (node_set.count(current_node) == 1 &&
+ marked_nodes.count(current_node) == 0) {
+ // Means, we did not visited this node yet, and it is in the node set,
+ // so we should process it (mark it and push all of its neighbors
+ // into stack).
+ marked_nodes.insert(current_node);
+ const auto neighbors = wait_for_graph_->getAdjacentNodes(current_node);
+ for (const auto neighbor : neighbors) {
+ to_be_visied_nodes.push(neighbor);
+ }
+ }
+ }
+ return false;
+ }
+
+ std::vector<std::pair<DirectedGraph::node_id, DirectedGraph::node_id>> edges_;
+ std::unique_ptr<DirectedGraph> wait_for_graph_;
+ std::unique_ptr<CycleDetector> cycle_detector_;
+};
+
+TEST_F(CycleDetectorTest, Interleaving) {
+ edges_ = {{0, 1},
+ {1, 0}};
+
+ initializeCycleDetector();
+
+ std::unordered_set<DirectedGraph::node_id> expected_victims = {1};
+
+ checkVictims(expected_victims);
+}
+
+TEST_F(CycleDetectorTest, MultipleCycle) {
+ // This edge contains lots of cycles of degree 1, 2 and 3.
+ edges_ = {{0, 1},
+ {1, 2}, {1, 3}, {1, 4},
+ {2, 5},
+ {3, 4}, {3, 6},
+ {4, 1}, {4, 5}, {4, 6},
+ {5, 2}, {5, 7},
+ {6, 7}, {6, 9},
+ {7, 6},
+ {8, 6},
+ {9, 8}, {9, 10},
+ {10, 11},
+ {11, 9}};
+
+ initializeCycleDetector();
+
+ std::unordered_set<DirectedGraph::node_id> expected_victims
+ = {4, 5, 7, 8, 9, 10, 11};
+
+ checkVictims(expected_victims);
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/DeadLockDetector_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/DeadLockDetector_unittest.cpp b/transaction/tests/DeadLockDetector_unittest.cpp
new file mode 100644
index 0000000..bc65ef5
--- /dev/null
+++ b/transaction/tests/DeadLockDetector_unittest.cpp
@@ -0,0 +1,96 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/DeadLockDetector.hpp"
+
+#include <atomic>
+#include <memory>
+#include <vector>
+
+#include "transaction/AccessMode.hpp"
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/LockTable.hpp"
+#include "transaction/ResourceId.hpp"
+#include "transaction/Transaction.hpp"
+
+#include "gtest/gtest.h"
+
+namespace quickstep {
+namespace transaction {
+
+class DeadLockDetectorTest : public ::testing::Test {
+ protected:
+ DeadLockDetectorTest()
+ : lock_table_(std::make_unique<LockTable>()),
+ status_(DeadLockDetectorStatus::kDone) {
+ }
+
+ std::unique_ptr<LockTable> lock_table_;
+ std::atomic<DeadLockDetectorStatus> status_;
+ std::vector<DirectedGraph::node_id> victims_;
+};
+
+TEST_F(DeadLockDetectorTest, SimpleCycle) {
+ const transaction_id transaction_one(1), transaction_two(2);
+ const ResourceId resource_one(1, 2), resource_two(4, 5);
+
+ const AccessMode x_lock_mode(AccessMode::XLockMode());
+
+ // Produce a conflicting schedule.
+ // Transaction 1 will acquire X lock on resource 1.
+ lock_table_->putLock(transaction_one,
+ resource_one,
+ x_lock_mode);
+
+ // Transaction 2 will acquire X lock on resource 2.
+ lock_table_->putLock(transaction_two,
+ resource_two,
+ x_lock_mode);
+
+ // Transaction 1 will try to acquire X lock on resource 2,
+ // but it will fail since Transaction 2 has already acquired
+ // X lock on resource 2.
+ lock_table_->putLock(transaction_one,
+ resource_two,
+ x_lock_mode);
+
+ // Transaction 2 will try to acquire X lock on resource 1,
+ // but it will fail since Transaction 1 has already acquired
+ // X lock on resource 2.
+ lock_table_->putLock(transaction_two,
+ resource_one,
+ x_lock_mode);
+
+ // Run deadlock detector.
+ DeadLockDetector deadlock_detector(lock_table_.get(), &status_, &victims_);
+ status_.store(DeadLockDetectorStatus::kNotReady);
+
+ deadlock_detector.start();
+
+ // Signal deadlock detector.
+ while (status_.load() == DeadLockDetectorStatus::kNotReady) {
+ }
+
+ status_.store(DeadLockDetectorStatus::kQuit);
+ deadlock_detector.join();
+
+ // Victim size must be 1.
+ ASSERT_EQ(1u, victims_.size());
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/DirectedGraph_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/DirectedGraph_unittest.cpp b/transaction/tests/DirectedGraph_unittest.cpp
index 43ad972..00fe276 100644
--- a/transaction/tests/DirectedGraph_unittest.cpp
+++ b/transaction/tests/DirectedGraph_unittest.cpp
@@ -26,101 +26,96 @@
namespace quickstep {
namespace transaction {
-TEST(DirectedGraphTest, AddNode) {
- // Prepare the data, but do not include in the graph.
- DirectedGraph wait_for_graph;
- transaction_id *tid3 = new transaction_id(3);
- transaction_id *tid4 = new transaction_id(4);
- transaction_id *tid5 = new transaction_id(5);
- transaction_id *tid6 = new transaction_id(6);
-
+class DirectedGraphTest : public ::testing::Test {
+ protected:
+ DirectedGraphTest()
+ : tid3_(3),
+ tid4_(4),
+ tid5_(5),
+ tid6_(6) {
+ }
+
+ DirectedGraph wait_for_graph_;
+ transaction_id tid3_;
+ transaction_id tid4_;
+ transaction_id tid5_;
+ transaction_id tid6_;
+};
+
+TEST_F(DirectedGraphTest, AddNode) {
// The nodes are not added yet, total no of nodesshould be zero.
- EXPECT_EQ(0u, wait_for_graph.getNumNodes());
+ EXPECT_EQ(0u, wait_for_graph_.getNumNodes());
- wait_for_graph.addNodeUnchecked(tid3);
+ wait_for_graph_.addNodeUnchecked(tid3_);
// One node is added.
- EXPECT_EQ(1u, wait_for_graph.getNumNodes());
+ EXPECT_EQ(1u, wait_for_graph_.getNumNodes());
- wait_for_graph.addNodeUnchecked(tid4);
+ wait_for_graph_.addNodeUnchecked(tid4_);
// Another node is added.
- EXPECT_EQ(2u, wait_for_graph.getNumNodes());
+ EXPECT_EQ(2u, wait_for_graph_.getNumNodes());
- wait_for_graph.addNodeUnchecked(tid5);
- wait_for_graph.addNodeUnchecked(tid6);
+ wait_for_graph_.addNodeUnchecked(tid5_);
+ wait_for_graph_.addNodeUnchecked(tid6_);
// Total no of nodes should be 4 right now.
- EXPECT_EQ(4u, wait_for_graph.getNumNodes());
+ EXPECT_EQ(4u, wait_for_graph_.getNumNodes());
}
-TEST(DirectedGraphTest, AddEdge) {
- // Prepare the graph.
- DirectedGraph wait_for_graph;
- transaction_id *tid3 = new transaction_id(3);
- transaction_id *tid4 = new transaction_id(4);
- transaction_id *tid5 = new transaction_id(5);
- transaction_id *tid6 = new transaction_id(6);
-
- DirectedGraph::node_id nid3 = wait_for_graph.addNodeUnchecked(tid3);
- DirectedGraph::node_id nid6 = wait_for_graph.addNodeUnchecked(tid6);
- DirectedGraph::node_id nid4 = wait_for_graph.addNodeUnchecked(tid4);
- DirectedGraph::node_id nid5 = wait_for_graph.addNodeUnchecked(tid5);
+TEST_F(DirectedGraphTest, AddEdge) {
+ DirectedGraph::node_id nid3 = wait_for_graph_.addNodeUnchecked(tid3_);
+ DirectedGraph::node_id nid6 = wait_for_graph_.addNodeUnchecked(tid6_);
+ DirectedGraph::node_id nid4 = wait_for_graph_.addNodeUnchecked(tid4_);
+ DirectedGraph::node_id nid5 = wait_for_graph_.addNodeUnchecked(tid5_);
// Add edges.
- wait_for_graph.addEdgeUnchecked(nid3, nid5);
- wait_for_graph.addEdgeUnchecked(nid6, nid4);
- wait_for_graph.addEdgeUnchecked(nid3, nid6);
- wait_for_graph.addEdgeUnchecked(nid4, nid6);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid5);
+ wait_for_graph_.addEdgeUnchecked(nid6, nid4);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid6);
+ wait_for_graph_.addEdgeUnchecked(nid4, nid6);
// Check whether the edges are already there.
- EXPECT_TRUE(wait_for_graph.hasEdge(nid3, nid5));
- EXPECT_TRUE(wait_for_graph.hasEdge(nid6, nid4));
- EXPECT_TRUE(wait_for_graph.hasEdge(nid4, nid6));
- EXPECT_TRUE(wait_for_graph.hasEdge(nid3, nid6));
+ EXPECT_TRUE(wait_for_graph_.hasEdge(nid3, nid5));
+ EXPECT_TRUE(wait_for_graph_.hasEdge(nid6, nid4));
+ EXPECT_TRUE(wait_for_graph_.hasEdge(nid4, nid6));
+ EXPECT_TRUE(wait_for_graph_.hasEdge(nid3, nid6));
// Check non-existent edges.
- EXPECT_FALSE(wait_for_graph.hasEdge(nid5, nid3));
- EXPECT_FALSE(wait_for_graph.hasEdge(nid6, nid3));
- EXPECT_FALSE(wait_for_graph.hasEdge(nid4, nid5));
+ EXPECT_FALSE(wait_for_graph_.hasEdge(nid5, nid3));
+ EXPECT_FALSE(wait_for_graph_.hasEdge(nid6, nid3));
+ EXPECT_FALSE(wait_for_graph_.hasEdge(nid4, nid5));
}
-TEST(DirectedGraphTest, GetAdjacentNodes) {
- // Prepare the graph.
- DirectedGraph wait_for_graph;
- transaction_id *tid3 = new transaction_id(3);
- transaction_id *tid4 = new transaction_id(4);
- transaction_id *tid5 = new transaction_id(5);
- transaction_id *tid6 = new transaction_id(6);
-
+TEST_F(DirectedGraphTest, GetAdjacentNodes) {
// Add 4 disconnected nodes to the graph.
- DirectedGraph::node_id nid3 = wait_for_graph.addNodeUnchecked(tid3);
- DirectedGraph::node_id nid6 = wait_for_graph.addNodeUnchecked(tid6);
- DirectedGraph::node_id nid4 = wait_for_graph.addNodeUnchecked(tid4);
- DirectedGraph::node_id nid5 = wait_for_graph.addNodeUnchecked(tid5);
+ DirectedGraph::node_id nid3 = wait_for_graph_.addNodeUnchecked(tid3_);
+ DirectedGraph::node_id nid6 = wait_for_graph_.addNodeUnchecked(tid6_);
+ DirectedGraph::node_id nid4 = wait_for_graph_.addNodeUnchecked(tid4_);
+ DirectedGraph::node_id nid5 = wait_for_graph_.addNodeUnchecked(tid5_);
- std::vector<DirectedGraph::node_id> result1 = wait_for_graph.getAdjacentNodes(nid3);
+ std::vector<DirectedGraph::node_id> result1 = wait_for_graph_.getAdjacentNodes(nid3);
// nid3 has no edge to other nodes.
EXPECT_EQ(0u, result1.size());
// Now nid3 has outgoing edge to nid4 and nid5.
- wait_for_graph.addEdgeUnchecked(nid3, nid4);
- wait_for_graph.addEdgeUnchecked(nid3, nid5);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid4);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid5);
- std::vector<DirectedGraph::node_id> result2 = wait_for_graph.getAdjacentNodes(nid3);
+ std::vector<DirectedGraph::node_id> result2 = wait_for_graph_.getAdjacentNodes(nid3);
// Therefore, number of outgoing edges from nid3 is 2.
EXPECT_EQ(2u, result2.size());
// Add an edge from nid3 to nid6.
- wait_for_graph.addEdgeUnchecked(nid3, nid6);
- std::vector<DirectedGraph::node_id> result3 = wait_for_graph.getAdjacentNodes(nid3);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid6);
+ std::vector<DirectedGraph::node_id> result3 = wait_for_graph_.getAdjacentNodes(nid3);
// Now there are 3 outgoing edges.
EXPECT_EQ(3u, result3.size());
// Again add edge from nid3 to nid6.
- wait_for_graph.addEdgeUnchecked(nid3, nid6);
- std::vector<DirectedGraph::node_id> result4 = wait_for_graph.getAdjacentNodes(nid3);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid6);
+ std::vector<DirectedGraph::node_id> result4 = wait_for_graph_.getAdjacentNodes(nid3);
// Since we have already add same edge before, number of edges are still 3.
EXPECT_EQ(3u, result4.size());
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/LockRequest_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/LockRequest_unittest.cpp b/transaction/tests/LockRequest_unittest.cpp
index 0e4138a..77047d9 100644
--- a/transaction/tests/LockRequest_unittest.cpp
+++ b/transaction/tests/LockRequest_unittest.cpp
@@ -31,7 +31,7 @@ class LockRequestTest : public ::testing::Test {
LockRequestTest()
: lock_request_(transaction_id(3),
ResourceId(5),
- AccessMode(AccessModeType::kSLock),
+ AccessMode::SLockMode(),
RequestType::kAcquireLock) {
}
@@ -41,7 +41,7 @@ class LockRequestTest : public ::testing::Test {
TEST_F(LockRequestTest, CheckGetters) {
EXPECT_EQ(transaction_id(3), lock_request_.getTransactionId());
EXPECT_EQ(ResourceId(5), lock_request_.getResourceId());
- EXPECT_EQ(AccessMode(AccessModeType::kSLock), lock_request_.getAccessMode());
+ EXPECT_EQ(AccessMode::SLockMode(), lock_request_.getAccessMode());
EXPECT_EQ(RequestType::kAcquireLock, lock_request_.getRequestType());
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/LockTable_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/LockTable_unittest.cpp b/transaction/tests/LockTable_unittest.cpp
index 577cb79..1aed0b8 100644
--- a/transaction/tests/LockTable_unittest.cpp
+++ b/transaction/tests/LockTable_unittest.cpp
@@ -41,64 +41,73 @@ class LockTableTest : public ::testing::Test {
};
TEST_F(LockTableTest, CompatibleRequestsFromDifferentTransactions) {
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ const AccessMode is_lock_mode = AccessMode::IsLockMode();
+ const AccessMode s_lock_mode = AccessMode::SLockMode();
+
+ EXPECT_EQ(LockTableResult::kPlacedInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kPLACED_IN_OWNED);
+ is_lock_mode));
// Acquire the same lock mode on same resource.
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ EXPECT_EQ(LockTableResult::kAlreadyInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kALREADY_IN_OWNED);
+ is_lock_mode));
// Another transaction acquires compatible lock on the same resource.
- EXPECT_EQ(lock_table_.putLock(tid_2_,
+ EXPECT_EQ(LockTableResult::kPlacedInOwned,
+ lock_table_.putLock(tid_2_,
ResourceId(2),
- AccessMode(AccessModeType::kSLock)),
- LockTableResult::kPLACED_IN_OWNED);
+ s_lock_mode));
}
TEST_F(LockTableTest, IncompatibleRequestsFromDifferentTransactions) {
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ const AccessMode is_lock_mode = AccessMode::IsLockMode();
+ const AccessMode x_lock_mode = AccessMode::XLockMode();
+
+ EXPECT_EQ(LockTableResult::kPlacedInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kPLACED_IN_OWNED);
+ is_lock_mode));
// Acquire the same lock mode on same resource.
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ EXPECT_EQ(LockTableResult::kAlreadyInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kALREADY_IN_OWNED);
+ is_lock_mode));
// Another transaction acquires incompatible lock on the same resource.
- EXPECT_EQ(lock_table_.putLock(tid_2_,
+ EXPECT_EQ(LockTableResult::kPlacedInPending,
+ lock_table_.putLock(tid_2_,
ResourceId(2),
- AccessMode(AccessModeType::kXLock)),
- LockTableResult::kPLACED_IN_PENDING);
+ x_lock_mode));
}
TEST_F(LockTableTest, StarvationProtection) {
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ const AccessMode is_lock_mode = AccessMode::IsLockMode();
+ const AccessMode x_lock_mode = AccessMode::XLockMode();
+
+ EXPECT_EQ(LockTableResult::kPlacedInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kPLACED_IN_OWNED);
+ is_lock_mode));
// Another transaction requests incompatible lock on the same resource.
// It should wait for the previous transaction.
- EXPECT_EQ(lock_table_.putLock(tid_2_,
+ EXPECT_EQ(LockTableResult::kPlacedInPending,
+ lock_table_.putLock(tid_2_,
ResourceId(2),
- AccessMode(AccessModeType::kXLock)),
- LockTableResult::kPLACED_IN_PENDING);
+ x_lock_mode));
// Another third transaction requests a compatible lock on the same resource.
// Normally, it should acquire the lock, however, there is a pending
// transaction waiting on the same resource. To prevent starvation, we should
// put in the pending list.
- EXPECT_EQ(lock_table_.putLock(tid_3_,
+ EXPECT_EQ(LockTableResult::kPlacedInPending,
+ lock_table_.putLock(tid_3_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kPLACED_IN_PENDING);
+ is_lock_mode));
}
} // namespace transaction
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/Lock_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/Lock_unittest.cpp b/transaction/tests/Lock_unittest.cpp
index 2ab8b3e..59a5e7a 100644
--- a/transaction/tests/Lock_unittest.cpp
+++ b/transaction/tests/Lock_unittest.cpp
@@ -18,6 +18,7 @@
#include "transaction/Lock.hpp"
#include <cstddef>
+#include <memory>
#include <vector>
#include "transaction/AccessMode.hpp"
@@ -31,12 +32,12 @@ namespace transaction {
class LockTest : public ::testing::Test {
protected:
LockTest()
- : modes_({AccessMode(AccessModeType::kNoLock),
- AccessMode(AccessModeType::kIsLock),
- AccessMode(AccessModeType::kIxLock),
- AccessMode(AccessModeType::kSLock),
- AccessMode(AccessModeType::kSixLock),
- AccessMode(AccessModeType::kXLock)}),
+ : modes_({AccessMode::NoLockMode(),
+ AccessMode::IsLockMode(),
+ AccessMode::IxLockMode(),
+ AccessMode::SLockMode(),
+ AccessMode::SixLockMode(),
+ AccessMode::XLockMode()}),
resource_a_(3, 10, 2, 5),
resource_b_(4, 5, 3, 2),
locks_on_resource_a_({Lock(resource_a_, modes_[0]),
[20/50] [abbrv] incubator-quickstep git commit: Added support for the
substring function. (#211)
Posted by hb...@apache.org.
Added support for the substring function. (#211)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/767b2ef1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/767b2ef1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/767b2ef1
Branch: refs/heads/query-manager-used-in-foreman
Commit: 767b2ef1a9ea570c8d51808559c800c01e0a385e
Parents: 6c9108a
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 5 11:17:37 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:31 2016 -0700
----------------------------------------------------------------------
parser/ParseBasicExpressions.cpp | 31 +
parser/ParseBasicExpressions.hpp | 77 +
parser/ParseExpression.hpp | 1 +
parser/SqlLexer.lpp | 2 +
parser/SqlParser.ypp | 16 +
parser/preprocessed/SqlLexer_gen.cpp | 1173 +++----
parser/preprocessed/SqlLexer_gen.hpp | 2 +-
parser/preprocessed/SqlParser_gen.cpp | 2920 +++++++++---------
parser/preprocessed/SqlParser_gen.hpp | 126 +-
query_optimizer/resolver/CMakeLists.txt | 1 +
query_optimizer/resolver/Resolver.cpp | 32 +
.../tests/execution_generator/Select.test | 24 +
query_optimizer/tests/resolver/Select.test | 78 +
types/operations/Operation.proto | 9 +
.../operations/unary_operations/CMakeLists.txt | 22 +
.../unary_operations/SubstringOperation.cpp | 214 ++
.../unary_operations/SubstringOperation.hpp | 234 ++
.../unary_operations/UnaryOperation.cpp | 4 +
.../unary_operations/UnaryOperationFactory.cpp | 12 +
.../unary_operations/UnaryOperationID.cpp | 6 +-
.../unary_operations/UnaryOperationID.hpp | 3 +
21 files changed, 2905 insertions(+), 2082 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/ParseBasicExpressions.cpp
----------------------------------------------------------------------
diff --git a/parser/ParseBasicExpressions.cpp b/parser/ParseBasicExpressions.cpp
index 2b1d7e0..a9d84ea 100644
--- a/parser/ParseBasicExpressions.cpp
+++ b/parser/ParseBasicExpressions.cpp
@@ -189,4 +189,35 @@ void ParseExtractFunction::getFieldStringItems(
non_container_child_fields->push_back(date_expression_.get());
}
+std::string ParseSubstringFunction::generateName() const {
+ std::string name;
+ name.append("SUBSTRING(");
+ name.append(operand_->generateName());
+ name.append(" FROM ");
+ name.append(std::to_string(start_position_));
+ if (length_ != kDefaultLength) {
+ name.append(" FOR ");
+ name.append(std::to_string(length_));
+ }
+ name.push_back(')');
+ return name;
+}
+
+void ParseSubstringFunction::getFieldStringItems(
+ std::vector<std::string> *inline_field_names,
+ std::vector<std::string> *inline_field_values,
+ std::vector<std::string> *non_container_child_field_names,
+ std::vector<const ParseTreeNode*> *non_container_child_fields,
+ std::vector<std::string> *container_child_field_names,
+ std::vector<std::vector<const ParseTreeNode*>> *container_child_fields) const {
+ inline_field_names->push_back("start_position");
+ inline_field_values->push_back(std::to_string(start_position_));
+
+ inline_field_names->push_back("length");
+ inline_field_values->push_back(std::to_string(length_));
+
+ non_container_child_field_names->push_back("operand");
+ non_container_child_fields->push_back(operand_.get());
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/ParseBasicExpressions.hpp
----------------------------------------------------------------------
diff --git a/parser/ParseBasicExpressions.hpp b/parser/ParseBasicExpressions.hpp
index 1886c8b..dea25d7 100644
--- a/parser/ParseBasicExpressions.hpp
+++ b/parser/ParseBasicExpressions.hpp
@@ -20,6 +20,8 @@
#ifndef QUICKSTEP_PARSER_PARSE_BASIC_EXPRESSIONS_HPP_
#define QUICKSTEP_PARSER_PARSE_BASIC_EXPRESSIONS_HPP_
+#include <cstddef>
+#include <limits>
#include <memory>
#include <string>
#include <vector>
@@ -511,6 +513,81 @@ class ParseExtractFunction : public ParseExpression {
DISALLOW_COPY_AND_ASSIGN(ParseExtractFunction);
};
+
+/**
+ * @brief Parsed representation of the substring function.
+ */
+class ParseSubstringFunction : public ParseExpression {
+ public:
+ static constexpr std::size_t kDefaultLength = std::numeric_limits<std::size_t>::max();
+
+ /**
+ * @brief Constructor.
+ *
+ * @param line_number The line number of the first token of the function call.
+ * @param column_number The column number of the first token of the function call.
+ * @param operand The operand of the substring.
+ * @param start_position The 1-based starting position of the substring.
+ * @param length Optional substring length.
+ */
+ ParseSubstringFunction(const int line_number,
+ const int column_number,
+ ParseExpression *operand,
+ const std::size_t start_position,
+ const std::size_t length = kDefaultLength)
+ : ParseExpression(line_number, column_number),
+ operand_(operand),
+ start_position_(start_position),
+ length_(length) {}
+
+ ExpressionType getExpressionType() const override {
+ return kSubstring;
+ }
+
+ std::string getName() const override {
+ return "Substring";
+ }
+
+ /**
+ * @return The operand of the substring.
+ */
+ const ParseExpression* operand() const {
+ return operand_.get();
+ }
+
+ /**
+ * @return The 1-based starting position of the substring.
+ */
+ std::size_t start_position() const {
+ return start_position_;
+ }
+
+ /**
+ * @return Then substring length.
+ */
+ std::size_t length() const {
+ return length_;
+ }
+
+ std::string generateName() const override;
+
+ protected:
+ void getFieldStringItems(
+ std::vector<std::string> *inline_field_names,
+ std::vector<std::string> *inline_field_values,
+ std::vector<std::string> *non_container_child_field_names,
+ std::vector<const ParseTreeNode*> *non_container_child_fields,
+ std::vector<std::string> *container_child_field_names,
+ std::vector<std::vector<const ParseTreeNode*>> *container_child_fields) const override;
+
+ private:
+ std::unique_ptr<ParseExpression> operand_;
+ const std::size_t start_position_;
+ const std::size_t length_;
+
+ DISALLOW_COPY_AND_ASSIGN(ParseSubstringFunction);
+};
+
/** @} */
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/ParseExpression.hpp
----------------------------------------------------------------------
diff --git a/parser/ParseExpression.hpp b/parser/ParseExpression.hpp
index e959e72..3541f83 100644
--- a/parser/ParseExpression.hpp
+++ b/parser/ParseExpression.hpp
@@ -45,6 +45,7 @@ class ParseExpression : public ParseTreeNode {
kSearchedCaseExpression,
kSimpleCaseExpression,
kSubqueryExpression,
+ kSubstring,
kUnaryExpression,
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/SqlLexer.lpp
----------------------------------------------------------------------
diff --git a/parser/SqlLexer.lpp b/parser/SqlLexer.lpp
index a399723..ac1c708 100644
--- a/parser/SqlLexer.lpp
+++ b/parser/SqlLexer.lpp
@@ -209,6 +209,7 @@ unsigned_numeric_literal {exact_numeric_literal}|{approximate_numeric_literal}
"false" return TOKEN_FALSE;
"first" return TOKEN_FIRST;
"float" return TOKEN_FLOAT;
+ "for" return TOKEN_FOR;
"foreign" return TOKEN_FOREIGN;
"from" return TOKEN_FROM;
"full" return TOKEN_FULL;
@@ -258,6 +259,7 @@ unsigned_numeric_literal {exact_numeric_literal}|{approximate_numeric_literal}
"set" return TOKEN_SET;
"sma" return TOKEN_SMA;
"smallint" return TOKEN_SMALLINT;
+ "substring" return TOKEN_SUBSTRING;
"table" return TOKEN_TABLE;
"then" return TOKEN_THEN;
"time" return TOKEN_TIME;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/SqlParser.ypp
----------------------------------------------------------------------
diff --git a/parser/SqlParser.ypp b/parser/SqlParser.ypp
index 1202d66..b07c48e 100644
--- a/parser/SqlParser.ypp
+++ b/parser/SqlParser.ypp
@@ -273,6 +273,7 @@ void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const std::string
%token TOKEN_FALSE;
%token TOKEN_FIRST;
%token TOKEN_FLOAT;
+%token TOKEN_FOR;
%token TOKEN_FOREIGN;
%token TOKEN_FROM;
%token TOKEN_FULL;
@@ -319,6 +320,7 @@ void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const std::string
%token TOKEN_SET;
%token TOKEN_SMA;
%token TOKEN_SMALLINT;
+%token TOKEN_SUBSTRING;
%token TOKEN_TABLE;
%token TOKEN_THEN;
%token TOKEN_TIME;
@@ -361,6 +363,7 @@ void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const std::string
case_expression
opt_else_clause
extract_function
+ substr_function
%type <attribute_>
attribute_ref
@@ -1505,6 +1508,9 @@ expression_base:
| extract_function {
$$ = $1;
}
+ | substr_function {
+ $$ = $1;
+ }
| case_expression {
$$ = $1;
}
@@ -1536,6 +1542,16 @@ extract_function:
$$ = new quickstep::ParseExtractFunction(@1.first_line, @1.first_column, $3, $5);
};
+substr_function:
+ TOKEN_SUBSTRING '(' add_expression TOKEN_FROM TOKEN_UNSIGNED_NUMVAL ')' {
+ $$ = new quickstep::ParseSubstringFunction(
+ @1.first_line, @1.first_column, $3, $5->long_value());
+ }
+ | TOKEN_SUBSTRING '(' add_expression TOKEN_FROM TOKEN_UNSIGNED_NUMVAL TOKEN_FOR TOKEN_UNSIGNED_NUMVAL ')' {
+ $$ = new quickstep::ParseSubstringFunction(
+ @1.first_line, @1.first_column, $3, $5->long_value(), $7->long_value());
+ };
+
case_expression:
TOKEN_CASE add_expression simple_when_clause_list opt_else_clause TOKEN_END {
$$ = new quickstep::ParseSimpleCaseExpression(@1.first_line, @1.first_column, $2, $3, $4);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/preprocessed/SqlLexer_gen.cpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlLexer_gen.cpp b/parser/preprocessed/SqlLexer_gen.cpp
index d836988..db20491 100644
--- a/parser/preprocessed/SqlLexer_gen.cpp
+++ b/parser/preprocessed/SqlLexer_gen.cpp
@@ -381,8 +381,8 @@ static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner );
*yy_cp = '\0'; \
yyg->yy_c_buf_p = yy_cp;
-#define YY_NUM_RULES 150
-#define YY_END_OF_BUFFER 151
+#define YY_NUM_RULES 152
+#define YY_END_OF_BUFFER 153
/* This struct is not used in this scanner,
but its presence is necessary. */
struct yy_trans_info
@@ -390,68 +390,69 @@ struct yy_trans_info
flex_int32_t yy_verify;
flex_int32_t yy_nxt;
};
-static yyconst flex_int16_t yy_accept[545] =
+static yyconst flex_int16_t yy_accept[553] =
{ 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 151, 2, 2, 149, 149, 148, 147, 149,
- 126, 122, 125, 122, 122, 145, 118, 115, 119, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 123, 4, 5, 5, 3, 141,
- 141, 138, 142, 142, 136, 143, 143, 140, 1, 148,
- 116, 146, 145, 145, 145, 0, 120, 117, 121, 144,
- 144, 144, 144, 10, 144, 144, 144, 22, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 124, 144,
-
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 57, 65, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 79, 80, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 4, 5, 3, 141, 137, 142, 135, 135, 127, 129,
- 130, 131, 132, 133, 134, 135, 143, 139, 146, 145,
- 0, 145, 6, 7, 144, 9, 11, 144, 144, 15,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 32, 144, 144, 144, 144, 144, 144, 144, 144, 43,
-
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 61, 144, 67, 144,
- 144, 144, 144, 144, 144, 144, 75, 144, 78, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 96, 97, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 127, 129,
- 128, 144, 144, 144, 144, 144, 144, 144, 20, 23,
- 144, 144, 144, 28, 144, 144, 30, 144, 144, 144,
- 144, 37, 144, 144, 41, 42, 144, 144, 144, 144,
- 144, 144, 144, 51, 52, 144, 54, 144, 56, 144,
-
- 144, 144, 144, 64, 66, 68, 69, 70, 144, 72,
- 144, 144, 76, 144, 144, 144, 144, 144, 87, 144,
- 89, 144, 144, 144, 144, 144, 144, 144, 144, 100,
- 101, 103, 144, 144, 144, 144, 144, 144, 110, 144,
- 112, 113, 127, 128, 8, 144, 144, 144, 144, 144,
- 144, 144, 25, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 47, 48,
- 49, 144, 53, 144, 58, 59, 144, 144, 144, 71,
- 144, 74, 77, 81, 82, 144, 144, 144, 88, 144,
- 144, 92, 144, 144, 144, 144, 99, 144, 144, 144,
-
- 144, 107, 144, 144, 111, 144, 144, 144, 14, 144,
- 144, 144, 144, 144, 26, 144, 29, 144, 144, 144,
- 144, 35, 144, 144, 144, 40, 144, 45, 144, 144,
- 55, 60, 144, 144, 73, 144, 144, 144, 144, 91,
- 144, 94, 95, 144, 144, 144, 105, 106, 108, 144,
- 144, 144, 13, 144, 144, 144, 144, 144, 144, 21,
- 144, 33, 34, 144, 144, 144, 144, 46, 50, 62,
- 144, 144, 85, 86, 144, 144, 144, 144, 144, 109,
- 144, 144, 144, 144, 144, 144, 144, 144, 31, 144,
- 144, 39, 144, 63, 144, 144, 144, 98, 144, 144,
-
- 144, 12, 144, 144, 144, 144, 24, 144, 36, 144,
- 144, 83, 144, 144, 102, 144, 114, 16, 144, 144,
- 144, 27, 38, 144, 84, 90, 144, 144, 144, 18,
- 19, 144, 144, 104, 144, 144, 144, 144, 144, 93,
- 144, 44, 17, 0
+ 0, 0, 153, 2, 2, 151, 151, 150, 149, 151,
+ 128, 124, 127, 124, 124, 147, 120, 117, 121, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 125, 4, 5, 5, 3, 143,
+ 143, 140, 144, 144, 138, 145, 145, 142, 1, 150,
+ 118, 148, 147, 147, 147, 0, 122, 119, 123, 146,
+ 146, 146, 146, 10, 146, 146, 146, 22, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 126, 146,
+
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 58, 66, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 80, 81, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 4, 5, 3, 143, 139, 144, 137, 137, 129,
+ 131, 132, 133, 134, 135, 136, 137, 145, 141, 148,
+ 147, 0, 147, 6, 7, 146, 9, 11, 146, 146,
+ 15, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 32, 146, 146, 146, 146, 146, 146, 146, 146,
+
+ 43, 146, 146, 146, 146, 146, 146, 50, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 62, 146, 68,
+ 146, 146, 146, 146, 146, 146, 146, 76, 146, 79,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 97, 98, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 129, 131, 130, 146, 146, 146, 146, 146, 146, 146,
+ 20, 23, 146, 146, 146, 28, 146, 146, 30, 146,
+ 146, 146, 146, 37, 146, 146, 41, 42, 146, 146,
+ 146, 146, 146, 146, 146, 52, 53, 146, 55, 146,
+
+ 57, 146, 146, 146, 146, 65, 67, 69, 70, 71,
+ 146, 73, 146, 146, 77, 146, 146, 146, 146, 146,
+ 88, 146, 90, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 102, 103, 105, 146, 146, 146, 146, 146,
+ 146, 112, 146, 114, 115, 129, 130, 8, 146, 146,
+ 146, 146, 146, 146, 146, 25, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 47, 48, 49, 146, 54, 146, 59, 60, 146,
+ 146, 146, 72, 146, 75, 78, 82, 83, 146, 146,
+ 146, 89, 146, 146, 93, 146, 146, 146, 146, 146,
+
+ 101, 146, 146, 146, 146, 109, 146, 146, 113, 146,
+ 146, 146, 14, 146, 146, 146, 146, 146, 26, 146,
+ 29, 146, 146, 146, 146, 35, 146, 146, 146, 40,
+ 146, 45, 146, 146, 56, 61, 146, 146, 74, 146,
+ 146, 146, 146, 92, 146, 95, 96, 146, 146, 146,
+ 146, 107, 108, 110, 146, 146, 146, 13, 146, 146,
+ 146, 146, 146, 146, 21, 146, 33, 34, 146, 146,
+ 146, 146, 46, 51, 63, 146, 146, 86, 87, 146,
+ 146, 146, 146, 146, 146, 111, 146, 146, 146, 146,
+ 146, 146, 146, 146, 31, 146, 146, 39, 146, 64,
+
+ 146, 146, 146, 99, 146, 146, 146, 146, 12, 146,
+ 146, 146, 146, 24, 146, 36, 146, 146, 84, 146,
+ 146, 100, 104, 146, 116, 16, 146, 146, 146, 27,
+ 38, 146, 85, 91, 146, 146, 146, 18, 19, 146,
+ 146, 106, 146, 146, 146, 146, 146, 94, 146, 44,
+ 17, 0
} ;
static yyconst YY_CHAR yy_ec[256] =
@@ -498,143 +499,145 @@ static yyconst YY_CHAR yy_meta[72] =
8
} ;
-static yyconst flex_uint16_t yy_base[560] =
+static yyconst flex_uint16_t yy_base[568] =
{ 0,
0, 1, 46, 0, 117, 163, 2, 3, 128, 132,
- 6, 10, 257, 1212, 1212, 0, 1212, 13, 1212, 233,
- 1212, 1212, 1212, 208, 6, 130, 4, 1212, 195, 124,
+ 6, 10, 257, 1219, 1219, 0, 1219, 13, 1219, 233,
+ 1219, 1219, 1219, 208, 6, 130, 4, 1219, 195, 124,
161, 170, 178, 207, 260, 92, 167, 161, 96, 107,
- 219, 214, 212, 224, 236, 92, 279, 171, 278, 281,
- 128, 227, 0, 125, 1212, 184, 4, 19, 0, 0,
+ 219, 214, 212, 224, 236, 92, 279, 272, 278, 281,
+ 128, 168, 0, 125, 1219, 184, 4, 19, 0, 0,
0, 146, 0, 0, 343, 0, 0, 145, 0, 22,
- 1212, 0, 297, 316, 338, 18, 1212, 1212, 1212, 0,
- 170, 227, 173, 178, 224, 299, 270, 0, 270, 335,
- 330, 286, 320, 327, 376, 308, 316, 326, 1212, 335,
-
- 351, 355, 371, 348, 346, 353, 359, 370, 382, 383,
- 380, 379, 399, 0, 392, 379, 386, 401, 399, 401,
- 402, 407, 402, 413, 420, 0, 431, 417, 420, 422,
- 434, 437, 435, 451, 446, 433, 456, 459, 459, 457,
- 450, 444, 454, 462, 469, 465, 465, 474, 460, 483,
- 148, 29, 0, 0, 1212, 0, 1212, 1212, 22, 24,
- 1212, 1212, 1212, 1212, 1212, 0, 0, 1212, 0, 515,
- 26, 28, 0, 0, 488, 0, 490, 473, 489, 478,
- 501, 502, 496, 512, 496, 499, 494, 520, 503, 521,
- 0, 519, 528, 526, 529, 514, 535, 522, 534, 0,
-
- 539, 522, 524, 532, 534, 553, 551, 546, 550, 544,
- 564, 564, 556, 570, 571, 572, 574, 564, 0, 561,
- 564, 581, 578, 583, 571, 573, 0, 583, 0, 591,
- 592, 578, 596, 587, 589, 604, 600, 609, 612, 612,
- 98, 608, 625, 0, 619, 620, 619, 629, 630, 624,
- 620, 638, 628, 623, 642, 633, 640, 632, 30, 125,
- 0, 635, 640, 650, 642, 652, 647, 654, 0, 668,
- 659, 659, 655, 0, 658, 663, 668, 676, 669, 671,
- 679, 688, 685, 683, 0, 0, 681, 680, 701, 698,
- 685, 686, 699, 0, 0, 693, 0, 697, 0, 688,
-
- 695, 696, 709, 0, 0, 0, 0, 0, 695, 0,
- 705, 720, 711, 715, 718, 730, 741, 746, 0, 743,
- 0, 731, 726, 731, 748, 739, 752, 746, 755, 0,
- 742, 0, 758, 743, 746, 760, 764, 762, 0, 766,
- 0, 759, 136, 1212, 0, 769, 769, 763, 784, 772,
- 780, 791, 0, 783, 786, 800, 801, 798, 807, 797,
- 805, 802, 799, 802, 813, 814, 802, 819, 0, 0,
- 0, 817, 0, 818, 0, 0, 807, 823, 807, 0,
- 825, 0, 0, 0, 0, 811, 818, 823, 0, 838,
- 828, 0, 841, 845, 832, 846, 0, 842, 844, 859,
-
- 860, 0, 847, 866, 0, 853, 860, 857, 0, 852,
- 858, 876, 870, 860, 0, 881, 0, 878, 872, 874,
- 867, 0, 868, 885, 887, 0, 93, 0, 879, 887,
- 0, 0, 884, 903, 0, 898, 890, 888, 906, 0,
- 909, 0, 0, 908, 922, 923, 0, 0, 0, 907,
- 912, 913, 0, 920, 917, 921, 923, 932, 929, 0,
- 935, 0, 0, 936, 934, 924, 926, 0, 0, 0,
- 934, 932, 0, 0, 945, 948, 939, 947, 949, 0,
- 945, 961, 957, 962, 963, 960, 963, 968, 0, 965,
- 970, 0, 965, 0, 973, 985, 979, 0, 977, 979,
-
- 988, 0, 991, 994, 989, 997, 0, 983, 0, 997,
- 987, 987, 996, 1008, 0, 1006, 0, 0, 1002, 1018,
- 1008, 0, 0, 1020, 0, 0, 1016, 1032, 1018, 0,
- 0, 1025, 1035, 0, 1032, 1035, 1025, 1040, 1029, 0,
- 1031, 0, 0, 1212, 1096, 1106, 1116, 1126, 1136, 1140,
- 1143, 1149, 1157, 1167, 1177, 1187, 1197, 1202, 1204
+ 1219, 0, 307, 337, 341, 18, 1219, 1219, 1219, 0,
+ 170, 224, 177, 181, 215, 269, 224, 0, 263, 326,
+ 336, 286, 290, 333, 382, 321, 322, 329, 1219, 328,
+
+ 347, 351, 346, 350, 346, 353, 352, 374, 386, 385,
+ 383, 382, 402, 0, 395, 382, 389, 403, 399, 397,
+ 399, 401, 399, 408, 418, 0, 421, 406, 410, 425,
+ 437, 438, 438, 455, 451, 436, 457, 462, 462, 463,
+ 461, 454, 447, 454, 462, 469, 465, 466, 474, 460,
+ 481, 148, 29, 0, 0, 1219, 0, 1219, 1219, 22,
+ 24, 1219, 1219, 1219, 1219, 1219, 0, 0, 1219, 0,
+ 509, 26, 28, 0, 0, 478, 0, 481, 478, 501,
+ 493, 514, 513, 501, 517, 500, 503, 498, 523, 505,
+ 521, 0, 518, 527, 526, 529, 513, 532, 520, 532,
+
+ 0, 537, 521, 523, 523, 525, 558, 563, 559, 562,
+ 554, 568, 568, 560, 574, 575, 576, 577, 569, 0,
+ 564, 565, 581, 578, 582, 569, 571, 0, 581, 0,
+ 589, 590, 576, 595, 586, 580, 595, 605, 621, 625,
+ 624, 98, 618, 629, 0, 623, 617, 625, 624, 634,
+ 635, 629, 625, 642, 630, 624, 644, 634, 641, 632,
+ 30, 125, 0, 633, 639, 649, 642, 652, 647, 647,
+ 0, 674, 673, 675, 670, 0, 671, 674, 673, 681,
+ 674, 676, 684, 693, 690, 688, 0, 0, 685, 682,
+ 702, 700, 686, 687, 699, 0, 0, 693, 0, 696,
+
+ 0, 687, 694, 696, 726, 0, 0, 0, 0, 0,
+ 696, 0, 698, 734, 727, 730, 731, 741, 746, 751,
+ 0, 748, 0, 736, 731, 736, 753, 744, 756, 748,
+ 741, 758, 0, 745, 0, 760, 745, 747, 761, 764,
+ 762, 0, 767, 0, 761, 136, 1219, 0, 772, 786,
+ 780, 801, 788, 800, 805, 0, 795, 792, 806, 807,
+ 804, 813, 803, 811, 808, 804, 805, 816, 817, 804,
+ 821, 0, 0, 0, 818, 0, 819, 0, 0, 807,
+ 823, 808, 0, 827, 0, 0, 0, 0, 814, 835,
+ 840, 0, 855, 846, 0, 859, 861, 846, 858, 850,
+
+ 0, 849, 851, 866, 867, 0, 854, 873, 0, 860,
+ 867, 863, 0, 857, 862, 880, 873, 863, 0, 883,
+ 0, 880, 873, 876, 870, 0, 872, 903, 913, 0,
+ 93, 0, 898, 906, 0, 0, 904, 922, 0, 915,
+ 905, 901, 913, 0, 916, 0, 0, 915, 921, 930,
+ 931, 0, 0, 0, 915, 920, 921, 0, 928, 923,
+ 926, 928, 936, 933, 0, 938, 0, 0, 939, 937,
+ 928, 931, 0, 0, 0, 953, 959, 0, 0, 973,
+ 969, 960, 967, 969, 970, 0, 964, 978, 972, 971,
+ 972, 969, 972, 977, 0, 974, 979, 0, 975, 0,
+
+ 982, 992, 985, 0, 992, 984, 985, 994, 0, 997,
+ 1001, 996, 1018, 0, 1012, 0, 1027, 1018, 1019, 1020,
+ 1031, 0, 0, 1029, 0, 0, 1024, 1038, 1026, 0,
+ 0, 1036, 0, 0, 1026, 1042, 1028, 0, 0, 1035,
+ 1045, 0, 1042, 1046, 1036, 1050, 1037, 0, 1038, 0,
+ 0, 1219, 1103, 1113, 1123, 1133, 1143, 1147, 1150, 1156,
+ 1164, 1174, 1184, 1194, 1204, 1209, 1211
} ;
-static yyconst flex_int16_t yy_def[560] =
+static yyconst flex_int16_t yy_def[568] =
{ 0,
- 545, 545, 544, 3, 546, 546, 547, 547, 548, 548,
- 549, 549, 544, 544, 544, 550, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 544, 544, 544, 544, 552, 553,
- 553, 544, 554, 554, 555, 556, 556, 544, 550, 544,
- 544, 557, 544, 544, 544, 544, 544, 544, 544, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 544, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 544, 544, 552, 553, 544, 554, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 558, 556, 544, 557, 544,
- 544, 544, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 544, 544,
- 559, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 544, 544, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 0, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544
+ 553, 553, 552, 3, 554, 554, 555, 555, 556, 556,
+ 557, 557, 552, 552, 552, 558, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 552, 552, 552, 552, 560, 561,
+ 561, 552, 562, 562, 563, 564, 564, 552, 558, 552,
+ 552, 565, 552, 552, 552, 552, 552, 552, 552, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 552, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 552, 552, 560, 561, 552, 562, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 566, 564, 552, 565,
+ 552, 552, 552, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 552, 552, 567, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 552, 552, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 0, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552
} ;
-static yyconst flex_uint16_t yy_nxt[1284] =
+static yyconst flex_uint16_t yy_nxt[1291] =
{ 0,
- 544, 544, 15, 15, 61, 61, 152, 152, 67, 62,
- 62, 68, 67, 544, 70, 68, 70, 73, 73, 77,
- 78, 152, 152, 70, 544, 70, 171, 171, 544, 172,
- 172, 152, 152, 259, 260, 260, 260, 172, 172, 172,
- 172, 343, 260, 544, 16, 16, 17, 18, 19, 18,
+ 552, 552, 15, 15, 61, 61, 153, 153, 67, 62,
+ 62, 68, 67, 552, 70, 68, 70, 73, 73, 77,
+ 78, 153, 153, 70, 552, 70, 172, 172, 552, 173,
+ 173, 153, 153, 261, 262, 262, 262, 173, 173, 173,
+ 173, 346, 262, 552, 16, 16, 17, 18, 19, 18,
20, 21, 22, 23, 22, 24, 25, 26, 26, 17,
27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
37, 38, 39, 40, 41, 42, 43, 44, 45, 46,
@@ -644,142 +647,142 @@ static yyconst flex_uint16_t yy_nxt[1284] =
38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
48, 49, 50, 51, 52, 53, 54, 17, 56, 57,
58, 17, 17, 17, 17, 17, 110, 115, 116, 132,
- 64, 17, 17, 17, 64, 62, 260, 260, 467, 62,
- 74, 75, 75, 325, 81, 147, 150, 260, 260, 151,
- 168, 76, 82, 155, 83, 110, 115, 116, 132, 84,
+ 64, 17, 17, 17, 64, 62, 262, 262, 472, 62,
+ 74, 75, 75, 327, 81, 148, 151, 262, 262, 152,
+ 169, 76, 82, 156, 83, 110, 115, 116, 132, 84,
17, 17, 17, 17, 56, 57, 58, 17, 17, 17,
- 17, 17, 65, 81, 147, 150, 65, 17, 17, 17,
- 76, 82, 85, 83, 111, 151, 86, 89, 84, 87,
- 173, 113, 137, 176, 90, 94, 114, 177, 112, 95,
+ 17, 17, 65, 81, 148, 151, 65, 17, 17, 17,
+ 76, 82, 85, 83, 111, 152, 86, 89, 84, 87,
+ 174, 113, 149, 150, 90, 94, 114, 177, 112, 95,
- 138, 91, 88, 96, 92, 93, 17, 17, 17, 97,
- 79, 85, 98, 111, 99, 86, 89, 72, 87, 173,
- 113, 137, 176, 90, 94, 114, 177, 112, 95, 138,
+ 178, 91, 88, 96, 92, 93, 17, 17, 17, 97,
+ 79, 85, 98, 111, 99, 86, 89, 72, 87, 174,
+ 113, 149, 150, 90, 94, 114, 177, 112, 95, 178,
91, 88, 96, 92, 93, 100, 117, 101, 97, 121,
118, 98, 102, 123, 119, 122, 125, 103, 71, 124,
- 120, 148, 149, 129, 126, 174, 544, 130, 127, 544,
- 178, 128, 544, 175, 100, 117, 101, 544, 121, 118,
+ 120, 179, 175, 129, 126, 182, 552, 130, 127, 552,
+ 176, 128, 552, 552, 100, 117, 101, 552, 121, 118,
131, 102, 123, 119, 122, 125, 103, 104, 124, 120,
- 148, 149, 129, 126, 174, 105, 130, 127, 106, 178,
- 128, 107, 175, 544, 108, 139, 133, 109, 544, 131,
-
- 134, 181, 140, 141, 135, 182, 104, 188, 73, 73,
- 136, 144, 142, 145, 105, 143, 146, 106, 76, 544,
- 107, 544, 179, 108, 139, 133, 109, 170, 170, 134,
- 181, 140, 141, 135, 182, 180, 188, 76, 189, 136,
- 144, 142, 145, 196, 143, 146, 158, 76, 74, 75,
- 75, 179, 183, 197, 159, 160, 184, 198, 185, 76,
- 186, 161, 187, 190, 180, 162, 76, 189, 191, 544,
- 199, 200, 196, 163, 201, 544, 204, 164, 544, 165,
- 205, 183, 197, 166, 206, 184, 198, 185, 76, 186,
- 161, 187, 190, 207, 162, 192, 202, 191, 193, 199,
-
- 200, 208, 163, 201, 194, 204, 164, 203, 165, 205,
- 209, 195, 166, 206, 210, 211, 213, 218, 212, 214,
- 219, 220, 207, 221, 192, 202, 222, 193, 223, 215,
- 208, 224, 225, 194, 216, 217, 203, 226, 227, 209,
- 195, 228, 229, 210, 211, 213, 218, 212, 214, 219,
- 220, 230, 221, 231, 232, 222, 233, 223, 215, 234,
- 224, 225, 235, 216, 217, 236, 226, 227, 237, 240,
- 228, 229, 241, 238, 239, 242, 245, 246, 247, 248,
- 230, 249, 231, 232, 243, 233, 250, 251, 234, 252,
- 253, 235, 244, 254, 236, 256, 257, 237, 240, 255,
-
- 258, 241, 238, 239, 242, 245, 246, 247, 248, 262,
- 249, 263, 264, 243, 265, 250, 251, 266, 252, 253,
- 267, 244, 254, 269, 256, 257, 170, 170, 255, 258,
- 270, 271, 268, 272, 273, 274, 76, 275, 262, 276,
- 263, 264, 277, 265, 278, 279, 266, 280, 282, 267,
- 283, 281, 269, 284, 285, 286, 287, 288, 289, 270,
- 271, 268, 272, 273, 274, 76, 275, 290, 276, 291,
- 292, 277, 293, 278, 279, 294, 280, 282, 295, 283,
- 281, 296, 284, 285, 286, 287, 288, 289, 297, 298,
- 299, 300, 301, 302, 305, 303, 290, 306, 291, 292,
-
- 307, 293, 308, 309, 294, 304, 310, 295, 311, 312,
- 296, 313, 314, 315, 316, 317, 318, 297, 298, 299,
- 300, 301, 302, 305, 303, 319, 306, 320, 321, 307,
- 322, 308, 309, 323, 304, 310, 324, 311, 312, 326,
- 313, 314, 315, 316, 317, 318, 327, 328, 329, 330,
- 331, 332, 333, 334, 319, 335, 320, 321, 336, 322,
- 337, 338, 323, 339, 341, 324, 342, 340, 326, 345,
- 346, 347, 348, 349, 350, 327, 328, 329, 330, 331,
- 332, 333, 334, 351, 335, 352, 353, 336, 354, 337,
- 338, 355, 339, 341, 356, 342, 340, 357, 345, 346,
-
- 347, 348, 349, 350, 358, 359, 360, 361, 362, 363,
- 364, 365, 351, 366, 352, 353, 367, 354, 368, 369,
- 355, 370, 371, 356, 372, 373, 357, 374, 375, 376,
- 377, 380, 378, 358, 359, 360, 361, 362, 363, 364,
- 365, 381, 366, 379, 382, 367, 383, 368, 369, 384,
- 370, 371, 385, 372, 373, 386, 374, 375, 376, 377,
- 380, 378, 387, 388, 389, 390, 391, 392, 393, 394,
- 381, 395, 379, 382, 396, 383, 397, 398, 384, 399,
- 400, 385, 401, 402, 386, 403, 404, 405, 406, 407,
- 408, 387, 388, 389, 390, 391, 392, 393, 394, 409,
-
- 395, 410, 413, 396, 411, 397, 398, 412, 399, 400,
- 414, 401, 402, 415, 403, 404, 405, 406, 407, 408,
- 416, 417, 418, 419, 420, 421, 422, 423, 409, 424,
- 410, 413, 425, 411, 426, 427, 412, 428, 429, 414,
- 430, 431, 415, 432, 433, 434, 435, 436, 437, 416,
- 417, 418, 419, 420, 421, 422, 423, 438, 424, 439,
- 440, 425, 441, 426, 427, 442, 428, 429, 443, 430,
- 431, 444, 432, 433, 434, 435, 436, 437, 445, 446,
- 447, 448, 449, 450, 451, 452, 438, 453, 439, 440,
- 454, 441, 455, 456, 442, 457, 458, 443, 459, 460,
-
- 444, 461, 462, 463, 464, 465, 466, 445, 446, 447,
- 448, 449, 450, 451, 452, 468, 453, 469, 470, 454,
- 471, 455, 456, 472, 457, 458, 473, 459, 460, 474,
- 461, 462, 463, 464, 465, 466, 475, 476, 477, 478,
- 479, 480, 481, 482, 468, 483, 469, 470, 484, 471,
- 485, 486, 472, 487, 488, 473, 489, 490, 474, 491,
- 492, 493, 494, 495, 496, 475, 476, 477, 478, 479,
- 480, 481, 482, 497, 483, 498, 499, 484, 500, 485,
- 486, 501, 487, 488, 502, 489, 490, 503, 491, 492,
- 493, 494, 495, 496, 504, 505, 506, 507, 508, 509,
-
- 510, 511, 497, 512, 498, 499, 513, 500, 514, 515,
- 501, 516, 517, 502, 518, 519, 503, 520, 521, 522,
- 523, 524, 525, 504, 505, 506, 507, 508, 509, 510,
- 511, 526, 512, 527, 528, 513, 529, 514, 515, 530,
- 516, 517, 531, 518, 519, 532, 520, 521, 522, 523,
- 524, 525, 533, 534, 535, 536, 537, 538, 539, 540,
- 526, 541, 527, 528, 542, 529, 543, 544, 530, 544,
- 544, 531, 544, 544, 532, 544, 544, 544, 544, 544,
- 544, 533, 534, 535, 536, 537, 538, 539, 540, 544,
- 541, 544, 544, 542, 544, 543, 14, 14, 14, 14,
-
- 14, 14, 14, 14, 14, 14, 59, 59, 59, 59,
- 59, 59, 59, 59, 59, 59, 60, 60, 60, 60,
- 60, 60, 60, 60, 60, 60, 63, 63, 63, 63,
- 63, 63, 63, 63, 63, 63, 66, 66, 66, 66,
- 66, 66, 66, 66, 66, 66, 69, 69, 80, 80,
- 80, 544, 80, 153, 153, 153, 153, 154, 154, 154,
- 544, 154, 154, 154, 154, 154, 154, 156, 156, 156,
- 544, 156, 156, 156, 156, 544, 156, 157, 157, 157,
- 157, 157, 157, 157, 157, 157, 157, 167, 167, 544,
- 167, 167, 167, 167, 167, 167, 167, 169, 544, 169,
-
- 169, 169, 169, 169, 169, 169, 169, 261, 261, 344,
- 344, 13, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544
+ 179, 175, 129, 126, 182, 105, 130, 127, 106, 176,
+ 128, 107, 180, 137, 108, 140, 133, 109, 183, 131,
+
+ 134, 138, 141, 142, 135, 181, 104, 189, 190, 139,
+ 136, 145, 143, 146, 105, 144, 147, 106, 73, 73,
+ 107, 180, 137, 108, 140, 133, 109, 183, 76, 134,
+ 138, 141, 142, 135, 181, 552, 189, 190, 139, 136,
+ 145, 143, 146, 184, 144, 147, 159, 185, 171, 171,
+ 552, 74, 75, 75, 160, 161, 197, 76, 76, 198,
+ 199, 162, 76, 200, 186, 163, 187, 201, 188, 191,
+ 202, 203, 184, 164, 192, 552, 185, 165, 205, 166,
+ 206, 552, 204, 167, 207, 197, 208, 76, 198, 199,
+ 162, 76, 200, 186, 163, 187, 201, 188, 191, 202,
+
+ 203, 193, 164, 192, 194, 209, 165, 205, 166, 206,
+ 195, 204, 167, 207, 210, 208, 211, 196, 212, 214,
+ 219, 213, 215, 220, 221, 222, 223, 225, 224, 226,
+ 193, 227, 216, 194, 209, 228, 229, 217, 218, 195,
+ 230, 231, 232, 210, 233, 211, 196, 212, 214, 219,
+ 213, 215, 220, 221, 222, 223, 225, 224, 226, 234,
+ 227, 216, 235, 236, 228, 229, 217, 218, 237, 230,
+ 231, 232, 238, 233, 241, 242, 243, 239, 240, 246,
+ 247, 248, 249, 250, 251, 244, 252, 253, 234, 254,
+ 255, 235, 236, 245, 256, 258, 259, 237, 260, 264,
+
+ 257, 238, 265, 241, 242, 243, 239, 240, 246, 247,
+ 248, 249, 250, 251, 244, 252, 253, 266, 254, 255,
+ 171, 171, 245, 256, 258, 259, 267, 260, 264, 257,
+ 76, 265, 268, 269, 271, 272, 273, 274, 275, 276,
+ 277, 278, 279, 280, 281, 270, 266, 282, 284, 285,
+ 286, 283, 287, 288, 289, 267, 290, 291, 292, 76,
+ 293, 268, 269, 271, 272, 273, 274, 275, 276, 277,
+ 278, 279, 280, 281, 270, 294, 282, 284, 285, 286,
+ 283, 287, 288, 289, 295, 290, 291, 292, 296, 293,
+ 297, 298, 299, 300, 301, 302, 303, 304, 305, 307,
+
+ 308, 309, 310, 311, 294, 312, 313, 314, 306, 315,
+ 316, 317, 318, 295, 319, 320, 321, 296, 322, 297,
+ 298, 299, 300, 301, 302, 303, 304, 305, 307, 308,
+ 309, 310, 311, 323, 312, 313, 314, 306, 315, 316,
+ 317, 318, 324, 319, 320, 321, 325, 322, 326, 328,
+ 329, 330, 331, 332, 333, 334, 335, 336, 337, 338,
+ 339, 340, 323, 341, 342, 344, 345, 348, 343, 349,
+ 350, 324, 351, 352, 353, 325, 354, 326, 328, 329,
+ 330, 331, 332, 333, 334, 335, 336, 337, 338, 339,
+ 340, 355, 341, 342, 344, 345, 348, 343, 349, 350,
+
+ 356, 351, 352, 353, 357, 354, 358, 359, 360, 361,
+ 362, 363, 364, 365, 366, 367, 368, 369, 370, 371,
+ 355, 372, 373, 374, 375, 376, 377, 378, 379, 356,
+ 380, 552, 383, 357, 384, 358, 359, 360, 361, 362,
+ 363, 364, 365, 366, 367, 368, 369, 370, 371, 381,
+ 372, 373, 374, 375, 376, 377, 378, 379, 385, 380,
+ 382, 383, 386, 384, 387, 388, 389, 390, 391, 392,
+ 393, 394, 395, 396, 397, 398, 399, 400, 381, 401,
+ 402, 403, 404, 405, 406, 407, 408, 385, 409, 382,
+ 410, 386, 411, 387, 388, 389, 390, 391, 392, 393,
+
+ 394, 395, 396, 397, 398, 399, 400, 412, 401, 402,
+ 403, 404, 405, 406, 407, 408, 413, 409, 414, 410,
+ 415, 411, 417, 416, 418, 419, 420, 421, 422, 423,
+ 424, 425, 426, 427, 428, 429, 412, 430, 431, 432,
+ 433, 434, 435, 436, 437, 413, 438, 414, 439, 415,
+ 440, 417, 416, 418, 419, 420, 421, 422, 423, 424,
+ 425, 426, 427, 428, 429, 441, 430, 431, 432, 433,
+ 434, 435, 436, 437, 442, 438, 443, 439, 444, 440,
+ 445, 446, 447, 448, 449, 450, 451, 452, 453, 454,
+ 455, 456, 457, 458, 441, 459, 460, 461, 462, 463,
+
+ 464, 465, 466, 442, 467, 443, 468, 444, 469, 445,
+ 446, 447, 448, 449, 450, 451, 452, 453, 454, 455,
+ 456, 457, 458, 470, 459, 460, 461, 462, 463, 464,
+ 465, 466, 471, 467, 473, 468, 474, 469, 475, 476,
+ 477, 478, 479, 480, 481, 482, 483, 484, 485, 486,
+ 487, 488, 470, 489, 490, 491, 492, 493, 494, 495,
+ 496, 471, 497, 473, 498, 474, 499, 475, 476, 477,
+ 478, 479, 480, 481, 482, 483, 484, 485, 486, 487,
+ 488, 500, 489, 490, 491, 492, 493, 494, 495, 496,
+ 501, 497, 502, 498, 503, 499, 504, 505, 506, 507,
+
+ 508, 509, 510, 511, 512, 513, 514, 515, 516, 517,
+ 500, 518, 519, 520, 521, 522, 523, 524, 525, 501,
+ 526, 502, 527, 503, 528, 504, 505, 506, 507, 508,
+ 509, 510, 511, 512, 513, 514, 515, 516, 517, 529,
+ 518, 519, 520, 521, 522, 523, 524, 525, 530, 526,
+ 531, 527, 532, 528, 533, 534, 535, 536, 537, 538,
+ 539, 540, 541, 542, 543, 544, 545, 546, 529, 547,
+ 548, 549, 550, 551, 552, 552, 552, 530, 552, 531,
+ 552, 532, 552, 533, 534, 535, 536, 537, 538, 539,
+ 540, 541, 542, 543, 544, 545, 546, 552, 547, 548,
+
+ 549, 550, 551, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 59, 59, 59, 59, 59, 59, 59,
+ 59, 59, 59, 60, 60, 60, 60, 60, 60, 60,
+ 60, 60, 60, 63, 63, 63, 63, 63, 63, 63,
+ 63, 63, 63, 66, 66, 66, 66, 66, 66, 66,
+ 66, 66, 66, 69, 69, 80, 80, 80, 552, 80,
+ 154, 154, 154, 154, 155, 155, 155, 552, 155, 155,
+ 155, 155, 155, 155, 157, 157, 157, 552, 157, 157,
+ 157, 157, 552, 157, 158, 158, 158, 158, 158, 158,
+ 158, 158, 158, 158, 168, 168, 552, 168, 168, 168,
+
+ 168, 168, 168, 168, 170, 552, 170, 170, 170, 170,
+ 170, 170, 170, 170, 263, 263, 347, 347, 13, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552
} ;
-static yyconst flex_int16_t yy_chk[1284] =
+static yyconst flex_int16_t yy_chk[1291] =
{ 0,
0, 0, 1, 2, 7, 8, 57, 57, 11, 7,
8, 11, 12, 0, 18, 12, 18, 25, 25, 27,
27, 58, 58, 70, 0, 70, 76, 76, 0, 76,
- 76, 152, 152, 159, 159, 160, 160, 171, 171, 172,
- 172, 259, 259, 0, 1, 2, 3, 3, 3, 3,
+ 76, 153, 153, 160, 160, 161, 161, 172, 172, 173,
+ 173, 261, 261, 0, 1, 2, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
@@ -789,137 +792,137 @@ static yyconst flex_int16_t yy_chk[1284] =
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 5, 5, 5,
5, 5, 5, 5, 5, 5, 36, 39, 40, 46,
- 9, 5, 5, 5, 10, 9, 260, 260, 427, 10,
- 26, 26, 26, 241, 30, 51, 54, 343, 343, 151,
+ 9, 5, 5, 5, 10, 9, 262, 262, 431, 10,
+ 26, 26, 26, 242, 30, 51, 54, 346, 346, 152,
68, 26, 30, 62, 30, 36, 39, 40, 46, 30,
5, 5, 5, 6, 6, 6, 6, 6, 6, 6,
6, 6, 9, 30, 51, 54, 10, 6, 6, 6,
26, 30, 31, 30, 37, 56, 31, 32, 30, 31,
- 81, 38, 48, 83, 32, 33, 38, 84, 37, 33,
+ 81, 38, 52, 52, 32, 33, 38, 83, 37, 33,
- 48, 32, 31, 33, 32, 32, 6, 6, 6, 33,
+ 84, 32, 31, 33, 32, 32, 6, 6, 6, 33,
29, 31, 33, 37, 34, 31, 32, 24, 31, 81,
- 38, 48, 83, 32, 33, 38, 84, 37, 33, 48,
+ 38, 52, 52, 32, 33, 38, 83, 37, 33, 84,
32, 31, 33, 32, 32, 34, 41, 34, 33, 42,
41, 33, 34, 43, 41, 42, 44, 34, 20, 43,
- 41, 52, 52, 45, 44, 82, 13, 45, 44, 0,
- 85, 44, 0, 82, 34, 41, 34, 0, 42, 41,
+ 41, 85, 82, 45, 44, 87, 13, 45, 44, 0,
+ 82, 44, 0, 0, 34, 41, 34, 0, 42, 41,
45, 34, 43, 41, 42, 44, 34, 35, 43, 41,
- 52, 52, 45, 44, 82, 35, 45, 44, 35, 85,
- 44, 35, 82, 0, 35, 49, 47, 35, 0, 45,
-
- 47, 87, 49, 49, 47, 89, 35, 92, 73, 73,
- 47, 50, 49, 50, 35, 49, 50, 35, 73, 0,
- 35, 0, 86, 35, 49, 47, 35, 74, 74, 47,
- 87, 49, 49, 47, 89, 86, 92, 74, 93, 47,
- 50, 49, 50, 96, 49, 50, 65, 73, 75, 75,
- 75, 86, 90, 97, 65, 65, 90, 98, 91, 75,
- 91, 65, 91, 94, 86, 65, 74, 93, 94, 0,
- 100, 101, 96, 65, 102, 0, 104, 65, 0, 65,
- 105, 90, 97, 65, 106, 90, 98, 91, 75, 91,
- 65, 91, 94, 107, 65, 95, 103, 94, 95, 100,
-
- 101, 108, 65, 102, 95, 104, 65, 103, 65, 105,
- 109, 95, 65, 106, 110, 111, 112, 115, 111, 113,
- 116, 117, 107, 118, 95, 103, 119, 95, 119, 113,
- 108, 120, 121, 95, 113, 113, 103, 122, 123, 109,
- 95, 124, 125, 110, 111, 112, 115, 111, 113, 116,
- 117, 127, 118, 128, 129, 119, 130, 119, 113, 131,
- 120, 121, 132, 113, 113, 133, 122, 123, 134, 135,
- 124, 125, 136, 134, 134, 137, 138, 139, 140, 141,
- 127, 142, 128, 129, 137, 130, 143, 144, 131, 145,
- 146, 132, 137, 147, 133, 148, 149, 134, 135, 147,
-
- 150, 136, 134, 134, 137, 138, 139, 140, 141, 175,
- 142, 177, 178, 137, 179, 143, 144, 180, 145, 146,
- 181, 137, 147, 182, 148, 149, 170, 170, 147, 150,
- 183, 184, 181, 185, 186, 187, 170, 188, 175, 189,
- 177, 178, 190, 179, 192, 193, 180, 194, 195, 181,
- 196, 194, 182, 197, 198, 199, 201, 202, 203, 183,
- 184, 181, 185, 186, 187, 170, 188, 204, 189, 205,
- 206, 190, 207, 192, 193, 208, 194, 195, 209, 196,
- 194, 210, 197, 198, 199, 201, 202, 203, 211, 212,
- 213, 214, 215, 216, 218, 217, 204, 220, 205, 206,
-
- 221, 207, 222, 223, 208, 217, 224, 209, 225, 226,
- 210, 228, 230, 231, 232, 233, 234, 211, 212, 213,
- 214, 215, 216, 218, 217, 235, 220, 236, 237, 221,
- 238, 222, 223, 239, 217, 224, 240, 225, 226, 242,
- 228, 230, 231, 232, 233, 234, 243, 245, 246, 247,
- 248, 249, 250, 251, 235, 252, 236, 237, 253, 238,
- 254, 255, 239, 256, 257, 240, 258, 256, 242, 262,
- 263, 264, 265, 266, 267, 243, 245, 246, 247, 248,
- 249, 250, 251, 268, 252, 270, 271, 253, 272, 254,
- 255, 273, 256, 257, 275, 258, 256, 276, 262, 263,
-
- 264, 265, 266, 267, 277, 278, 279, 280, 281, 282,
- 283, 284, 268, 287, 270, 271, 288, 272, 289, 290,
- 273, 291, 292, 275, 293, 296, 276, 298, 300, 301,
- 302, 309, 303, 277, 278, 279, 280, 281, 282, 283,
- 284, 311, 287, 303, 312, 288, 313, 289, 290, 314,
- 291, 292, 315, 293, 296, 316, 298, 300, 301, 302,
- 309, 303, 317, 318, 320, 322, 323, 324, 325, 326,
- 311, 327, 303, 312, 328, 313, 329, 331, 314, 333,
- 334, 315, 335, 336, 316, 337, 338, 340, 342, 346,
- 347, 317, 318, 320, 322, 323, 324, 325, 326, 348,
-
- 327, 349, 351, 328, 350, 329, 331, 350, 333, 334,
- 352, 335, 336, 354, 337, 338, 340, 342, 346, 347,
- 355, 356, 357, 358, 359, 360, 361, 362, 348, 363,
- 349, 351, 364, 350, 365, 366, 350, 367, 368, 352,
- 372, 374, 354, 377, 378, 379, 381, 386, 387, 355,
- 356, 357, 358, 359, 360, 361, 362, 388, 363, 390,
- 391, 364, 393, 365, 366, 394, 367, 368, 395, 372,
- 374, 396, 377, 378, 379, 381, 386, 387, 398, 399,
- 400, 401, 403, 404, 406, 407, 388, 408, 390, 391,
- 410, 393, 411, 412, 394, 413, 414, 395, 416, 418,
-
- 396, 419, 420, 421, 423, 424, 425, 398, 399, 400,
- 401, 403, 404, 406, 407, 429, 408, 430, 433, 410,
- 434, 411, 412, 436, 413, 414, 437, 416, 418, 438,
- 419, 420, 421, 423, 424, 425, 439, 441, 444, 445,
- 446, 450, 451, 452, 429, 454, 430, 433, 455, 434,
- 456, 457, 436, 458, 459, 437, 461, 464, 438, 465,
- 466, 467, 471, 472, 475, 439, 441, 444, 445, 446,
- 450, 451, 452, 476, 454, 477, 478, 455, 479, 456,
- 457, 481, 458, 459, 482, 461, 464, 483, 465, 466,
- 467, 471, 472, 475, 484, 485, 486, 487, 488, 490,
-
- 491, 493, 476, 495, 477, 478, 496, 479, 497, 499,
- 481, 500, 501, 482, 503, 504, 483, 505, 506, 508,
- 510, 511, 512, 484, 485, 486, 487, 488, 490, 491,
- 493, 513, 495, 514, 516, 496, 519, 497, 499, 520,
- 500, 501, 521, 503, 504, 524, 505, 506, 508, 510,
- 511, 512, 527, 528, 529, 532, 533, 535, 536, 537,
- 513, 538, 514, 516, 539, 519, 541, 0, 520, 0,
- 0, 521, 0, 0, 524, 0, 0, 0, 0, 0,
- 0, 527, 528, 529, 532, 533, 535, 536, 537, 0,
- 538, 0, 0, 539, 0, 541, 545, 545, 545, 545,
-
- 545, 545, 545, 545, 545, 545, 546, 546, 546, 546,
- 546, 546, 546, 546, 546, 546, 547, 547, 547, 547,
- 547, 547, 547, 547, 547, 547, 548, 548, 548, 548,
- 548, 548, 548, 548, 548, 548, 549, 549, 549, 549,
- 549, 549, 549, 549, 549, 549, 550, 550, 551, 551,
- 551, 0, 551, 552, 552, 552, 552, 553, 553, 553,
- 0, 553, 553, 553, 553, 553, 553, 554, 554, 554,
- 0, 554, 554, 554, 554, 0, 554, 555, 555, 555,
- 555, 555, 555, 555, 555, 555, 555, 556, 556, 0,
- 556, 556, 556, 556, 556, 556, 556, 557, 0, 557,
-
- 557, 557, 557, 557, 557, 557, 557, 558, 558, 559,
- 559, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544
+ 85, 82, 45, 44, 87, 35, 45, 44, 35, 82,
+ 44, 35, 86, 48, 35, 49, 47, 35, 89, 45,
+
+ 47, 48, 49, 49, 47, 86, 35, 92, 93, 48,
+ 47, 50, 49, 50, 35, 49, 50, 35, 73, 73,
+ 35, 86, 48, 35, 49, 47, 35, 89, 73, 47,
+ 48, 49, 49, 47, 86, 0, 92, 93, 48, 47,
+ 50, 49, 50, 90, 49, 50, 65, 90, 74, 74,
+ 0, 75, 75, 75, 65, 65, 96, 73, 74, 97,
+ 98, 65, 75, 100, 91, 65, 91, 101, 91, 94,
+ 102, 103, 90, 65, 94, 0, 90, 65, 104, 65,
+ 105, 0, 103, 65, 106, 96, 107, 74, 97, 98,
+ 65, 75, 100, 91, 65, 91, 101, 91, 94, 102,
+
+ 103, 95, 65, 94, 95, 108, 65, 104, 65, 105,
+ 95, 103, 65, 106, 109, 107, 110, 95, 111, 112,
+ 115, 111, 113, 116, 117, 118, 119, 120, 119, 121,
+ 95, 122, 113, 95, 108, 123, 124, 113, 113, 95,
+ 125, 127, 128, 109, 129, 110, 95, 111, 112, 115,
+ 111, 113, 116, 117, 118, 119, 120, 119, 121, 130,
+ 122, 113, 131, 132, 123, 124, 113, 113, 133, 125,
+ 127, 128, 134, 129, 135, 136, 137, 134, 134, 138,
+ 139, 140, 141, 142, 143, 137, 144, 145, 130, 146,
+ 147, 131, 132, 137, 148, 149, 150, 133, 151, 176,
+
+ 148, 134, 178, 135, 136, 137, 134, 134, 138, 139,
+ 140, 141, 142, 143, 137, 144, 145, 179, 146, 147,
+ 171, 171, 137, 148, 149, 150, 180, 151, 176, 148,
+ 171, 178, 181, 182, 183, 184, 185, 186, 187, 188,
+ 189, 190, 191, 193, 194, 182, 179, 195, 196, 197,
+ 198, 195, 199, 200, 202, 180, 203, 204, 205, 171,
+ 206, 181, 182, 183, 184, 185, 186, 187, 188, 189,
+ 190, 191, 193, 194, 182, 207, 195, 196, 197, 198,
+ 195, 199, 200, 202, 208, 203, 204, 205, 209, 206,
+ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219,
+
+ 221, 222, 223, 224, 207, 225, 226, 227, 218, 229,
+ 231, 232, 233, 208, 234, 235, 236, 209, 237, 210,
+ 211, 212, 213, 214, 215, 216, 217, 218, 219, 221,
+ 222, 223, 224, 238, 225, 226, 227, 218, 229, 231,
+ 232, 233, 239, 234, 235, 236, 240, 237, 241, 243,
+ 244, 246, 247, 248, 249, 250, 251, 252, 253, 254,
+ 255, 256, 238, 257, 258, 259, 260, 264, 258, 265,
+ 266, 239, 267, 268, 269, 240, 270, 241, 243, 244,
+ 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
+ 256, 272, 257, 258, 259, 260, 264, 258, 265, 266,
+
+ 273, 267, 268, 269, 274, 270, 275, 277, 278, 279,
+ 280, 281, 282, 283, 284, 285, 286, 289, 290, 291,
+ 272, 292, 293, 294, 295, 298, 300, 302, 303, 273,
+ 304, 0, 311, 274, 313, 275, 277, 278, 279, 280,
+ 281, 282, 283, 284, 285, 286, 289, 290, 291, 305,
+ 292, 293, 294, 295, 298, 300, 302, 303, 314, 304,
+ 305, 311, 315, 313, 316, 317, 318, 319, 320, 322,
+ 324, 325, 326, 327, 328, 329, 330, 331, 305, 332,
+ 334, 336, 337, 338, 339, 340, 341, 314, 343, 305,
+ 345, 315, 349, 316, 317, 318, 319, 320, 322, 324,
+
+ 325, 326, 327, 328, 329, 330, 331, 350, 332, 334,
+ 336, 337, 338, 339, 340, 341, 351, 343, 352, 345,
+ 353, 349, 354, 353, 355, 357, 358, 359, 360, 361,
+ 362, 363, 364, 365, 366, 367, 350, 368, 369, 370,
+ 371, 375, 377, 380, 381, 351, 382, 352, 384, 353,
+ 389, 354, 353, 355, 357, 358, 359, 360, 361, 362,
+ 363, 364, 365, 366, 367, 390, 368, 369, 370, 371,
+ 375, 377, 380, 381, 391, 382, 393, 384, 394, 389,
+ 396, 397, 398, 399, 400, 402, 403, 404, 405, 407,
+ 408, 410, 411, 412, 390, 414, 415, 416, 417, 418,
+
+ 420, 422, 423, 391, 424, 393, 425, 394, 427, 396,
+ 397, 398, 399, 400, 402, 403, 404, 405, 407, 408,
+ 410, 411, 412, 428, 414, 415, 416, 417, 418, 420,
+ 422, 423, 429, 424, 433, 425, 434, 427, 437, 438,
+ 440, 441, 442, 443, 445, 448, 449, 450, 451, 455,
+ 456, 457, 428, 459, 460, 461, 462, 463, 464, 466,
+ 469, 429, 470, 433, 471, 434, 472, 437, 438, 440,
+ 441, 442, 443, 445, 448, 449, 450, 451, 455, 456,
+ 457, 476, 459, 460, 461, 462, 463, 464, 466, 469,
+ 477, 470, 480, 471, 481, 472, 482, 483, 484, 485,
+
+ 487, 488, 489, 490, 491, 492, 493, 494, 496, 497,
+ 476, 499, 501, 502, 503, 505, 506, 507, 508, 477,
+ 510, 480, 511, 481, 512, 482, 483, 484, 485, 487,
+ 488, 489, 490, 491, 492, 493, 494, 496, 497, 513,
+ 499, 501, 502, 503, 505, 506, 507, 508, 515, 510,
+ 517, 511, 518, 512, 519, 520, 521, 524, 527, 528,
+ 529, 532, 535, 536, 537, 540, 541, 543, 513, 544,
+ 545, 546, 547, 549, 0, 0, 0, 515, 0, 517,
+ 0, 518, 0, 519, 520, 521, 524, 527, 528, 529,
+ 532, 535, 536, 537, 540, 541, 543, 0, 544, 545,
+
+ 546, 547, 549, 553, 553, 553, 553, 553, 553, 553,
+ 553, 553, 553, 554, 554, 554, 554, 554, 554, 554,
+ 554, 554, 554, 555, 555, 555, 555, 555, 555, 555,
+ 555, 555, 555, 556, 556, 556, 556, 556, 556, 556,
+ 556, 556, 556, 557, 557, 557, 557, 557, 557, 557,
+ 557, 557, 557, 558, 558, 559, 559, 559, 0, 559,
+ 560, 560, 560, 560, 561, 561, 561, 0, 561, 561,
+ 561, 561, 561, 561, 562, 562, 562, 0, 562, 562,
+ 562, 562, 0, 562, 563, 563, 563, 563, 563, 563,
+ 563, 563, 563, 563, 564, 564, 0, 564, 564, 564,
+
+ 564, 564, 564, 564, 565, 0, 565, 565, 565, 565,
+ 565, 565, 565, 565, 566, 566, 567, 567, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552
} ;
/* Table of booleans, true if rule could match eol. */
-static yyconst flex_int32_t yy_rule_can_match_eol[151] =
+static yyconst flex_int32_t yy_rule_can_match_eol[153] =
{ 0,
0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -927,8 +930,8 @@ static yyconst flex_int32_t yy_rule_can_match_eol[151] =
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,
- 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, };
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,
+ 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, };
/* The intent behind this definition is that it'll catch
* any uses of REJECT which flex missed.
@@ -1045,7 +1048,7 @@ class UnaryOperation;
-#line 1049 "SqlLexer_gen.cpp"
+#line 1052 "SqlLexer_gen.cpp"
#define INITIAL 0
#define CONDITION_SQL 1
@@ -1336,7 +1339,7 @@ YY_DECL
#line 128 "../SqlLexer.lpp"
-#line 1340 "SqlLexer_gen.cpp"
+#line 1343 "SqlLexer_gen.cpp"
while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */
{
@@ -1363,13 +1366,13 @@ yy_match:
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 545 )
+ if ( yy_current_state >= 553 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
++yy_cp;
}
- while ( yy_current_state != 544 );
+ while ( yy_current_state != 552 );
yy_cp = yyg->yy_last_accepting_cpos;
yy_current_state = yyg->yy_last_accepting_state;
@@ -1678,62 +1681,62 @@ return TOKEN_FLOAT;
case 50:
YY_RULE_SETUP
#line 212 "../SqlLexer.lpp"
-return TOKEN_FOREIGN;
+return TOKEN_FOR;
YY_BREAK
case 51:
YY_RULE_SETUP
#line 213 "../SqlLexer.lpp"
-return TOKEN_FROM;
+return TOKEN_FOREIGN;
YY_BREAK
case 52:
YY_RULE_SETUP
#line 214 "../SqlLexer.lpp"
-return TOKEN_FULL;
+return TOKEN_FROM;
YY_BREAK
case 53:
YY_RULE_SETUP
#line 215 "../SqlLexer.lpp"
-return TOKEN_GROUP;
+return TOKEN_FULL;
YY_BREAK
case 54:
YY_RULE_SETUP
#line 216 "../SqlLexer.lpp"
-return TOKEN_HASH;
+return TOKEN_GROUP;
YY_BREAK
case 55:
YY_RULE_SETUP
#line 217 "../SqlLexer.lpp"
-return TOKEN_HAVING;
+return TOKEN_HASH;
YY_BREAK
case 56:
YY_RULE_SETUP
#line 218 "../SqlLexer.lpp"
-return TOKEN_HOUR;
+return TOKEN_HAVING;
YY_BREAK
case 57:
YY_RULE_SETUP
#line 219 "../SqlLexer.lpp"
-return TOKEN_IN;
+return TOKEN_HOUR;
YY_BREAK
case 58:
YY_RULE_SETUP
#line 220 "../SqlLexer.lpp"
-return TOKEN_INDEX;
+return TOKEN_IN;
YY_BREAK
case 59:
YY_RULE_SETUP
#line 221 "../SqlLexer.lpp"
-return TOKEN_INNER;
+return TOKEN_INDEX;
YY_BREAK
case 60:
YY_RULE_SETUP
#line 222 "../SqlLexer.lpp"
-return TOKEN_INSERT;
+return TOKEN_INNER;
YY_BREAK
case 61:
YY_RULE_SETUP
#line 223 "../SqlLexer.lpp"
-return TOKEN_INTEGER;
+return TOKEN_INSERT;
YY_BREAK
case 62:
YY_RULE_SETUP
@@ -1743,331 +1746,341 @@ return TOKEN_INTEGER;
case 63:
YY_RULE_SETUP
#line 225 "../SqlLexer.lpp"
-return TOKEN_INTERVAL;
+return TOKEN_INTEGER;
YY_BREAK
case 64:
YY_RULE_SETUP
#line 226 "../SqlLexer.lpp"
-return TOKEN_INTO;
+return TOKEN_INTERVAL;
YY_BREAK
case 65:
YY_RULE_SETUP
#line 227 "../SqlLexer.lpp"
-return TOKEN_IS;
+return TOKEN_INTO;
YY_BREAK
case 66:
YY_RULE_SETUP
#line 228 "../SqlLexer.lpp"
-return TOKEN_JOIN;
+return TOKEN_IS;
YY_BREAK
case 67:
YY_RULE_SETUP
#line 229 "../SqlLexer.lpp"
-return TOKEN_KEY;
+return TOKEN_JOIN;
YY_BREAK
case 68:
YY_RULE_SETUP
#line 230 "../SqlLexer.lpp"
-return TOKEN_LAST;
+return TOKEN_KEY;
YY_BREAK
case 69:
YY_RULE_SETUP
#line 231 "../SqlLexer.lpp"
-return TOKEN_LEFT;
+return TOKEN_LAST;
YY_BREAK
case 70:
YY_RULE_SETUP
#line 232 "../SqlLexer.lpp"
-return TOKEN_LIKE;
+return TOKEN_LEFT;
YY_BREAK
case 71:
YY_RULE_SETUP
#line 233 "../SqlLexer.lpp"
-return TOKEN_LIMIT;
+return TOKEN_LIKE;
YY_BREAK
case 72:
YY_RULE_SETUP
#line 234 "../SqlLexer.lpp"
-return TOKEN_LONG;
+return TOKEN_LIMIT;
YY_BREAK
case 73:
YY_RULE_SETUP
#line 235 "../SqlLexer.lpp"
-return TOKEN_MINUTE;
+return TOKEN_LONG;
YY_BREAK
case 74:
YY_RULE_SETUP
#line 236 "../SqlLexer.lpp"
-return TOKEN_MONTH;
+return TOKEN_MINUTE;
YY_BREAK
case 75:
YY_RULE_SETUP
#line 237 "../SqlLexer.lpp"
-return TOKEN_NOT;
+return TOKEN_MONTH;
YY_BREAK
case 76:
YY_RULE_SETUP
#line 238 "../SqlLexer.lpp"
-return TOKEN_NULL;
+return TOKEN_NOT;
YY_BREAK
case 77:
YY_RULE_SETUP
#line 239 "../SqlLexer.lpp"
-return TOKEN_NULLS;
+return TOKEN_NULL;
YY_BREAK
case 78:
YY_RULE_SETUP
#line 240 "../SqlLexer.lpp"
-return TOKEN_OFF;
+return TOKEN_NULLS;
YY_BREAK
case 79:
YY_RULE_SETUP
#line 241 "../SqlLexer.lpp"
-return TOKEN_ON;
+return TOKEN_OFF;
YY_BREAK
case 80:
YY_RULE_SETUP
#line 242 "../SqlLexer.lpp"
-return TOKEN_OR;
+return TOKEN_ON;
YY_BREAK
case 81:
YY_RULE_SETUP
#line 243 "../SqlLexer.lpp"
-return TOKEN_ORDER;
+return TOKEN_OR;
YY_BREAK
case 82:
YY_RULE_SETUP
#line 244 "../SqlLexer.lpp"
-return TOKEN_OUTER;
+return TOKEN_ORDER;
YY_BREAK
case 83:
YY_RULE_SETUP
#line 245 "../SqlLexer.lpp"
-return TOKEN_PARTITION;
+return TOKEN_OUTER;
YY_BREAK
case 84:
YY_RULE_SETUP
#line 246 "../SqlLexer.lpp"
-return TOKEN_PARTITIONS;
+return TOKEN_PARTITION;
YY_BREAK
case 85:
YY_RULE_SETUP
#line 247 "../SqlLexer.lpp"
-return TOKEN_PERCENT;
+return TOKEN_PARTITIONS;
YY_BREAK
case 86:
YY_RULE_SETUP
#line 248 "../SqlLexer.lpp"
-return TOKEN_PRIMARY;
+return TOKEN_PERCENT;
YY_BREAK
case 87:
YY_RULE_SETUP
#line 249 "../SqlLexer.lpp"
-return TOKEN_QUIT;
+return TOKEN_PRIMARY;
YY_BREAK
case 88:
YY_RULE_SETUP
#line 250 "../SqlLexer.lpp"
-return TOKEN_RANGE;
+return TOKEN_QUIT;
YY_BREAK
case 89:
YY_RULE_SETUP
#line 251 "../SqlLexer.lpp"
-return TOKEN_REAL;
+return TOKEN_RANGE;
YY_BREAK
case 90:
YY_RULE_SETUP
#line 252 "../SqlLexer.lpp"
-return TOKEN_REFERENCES;
+return TOKEN_REAL;
YY_BREAK
case 91:
YY_RULE_SETUP
#line 253 "../SqlLexer.lpp"
-return TOKEN_REGEXP;
+return TOKEN_REFERENCES;
YY_BREAK
case 92:
YY_RULE_SETUP
#line 254 "../SqlLexer.lpp"
-return TOKEN_RIGHT;
+return TOKEN_REGEXP;
YY_BREAK
case 93:
YY_RULE_SETUP
#line 255 "../SqlLexer.lpp"
-return TOKEN_ROW_DELIMITER;
+return TOKEN_RIGHT;
YY_BREAK
case 94:
YY_RULE_SETUP
#line 256 "../SqlLexer.lpp"
-return TOKEN_SECOND;
+return TOKEN_ROW_DELIMITER;
YY_BREAK
case 95:
YY_RULE_SETUP
#line 257 "../SqlLexer.lpp"
-return TOKEN_SELECT;
+return TOKEN_SECOND;
YY_BREAK
case 96:
YY_RULE_SETUP
#line 258 "../SqlLexer.lpp"
-return TOKEN_SET;
+return TOKEN_SELECT;
YY_BREAK
case 97:
YY_RULE_SETUP
#line 259 "../SqlLexer.lpp"
-return TOKEN_SMA;
+return TOKEN_SET;
YY_BREAK
case 98:
YY_RULE_SETUP
#line 260 "../SqlLexer.lpp"
-return TOKEN_SMALLINT;
+return TOKEN_SMA;
YY_BREAK
case 99:
YY_RULE_SETUP
#line 261 "../SqlLexer.lpp"
-return TOKEN_TABLE;
+return TOKEN_SMALLINT;
YY_BREAK
case 100:
YY_RULE_SETUP
#line 262 "../SqlLexer.lpp"
-return TOKEN_THEN;
+return TOKEN_SUBSTRING;
YY_BREAK
case 101:
YY_RULE_SETUP
#line 263 "../SqlLexer.lpp"
-return TOKEN_TIME;
+return TOKEN_TABLE;
YY_BREAK
case 102:
YY_RULE_SETUP
#line 264 "../SqlLexer.lpp"
-return TOKEN_TIMESTAMP;
+return TOKEN_THEN;
YY_BREAK
case 103:
YY_RULE_SETUP
#line 265 "../SqlLexer.lpp"
-return TOKEN_TRUE;
+return TOKEN_TIME;
YY_BREAK
case 104:
YY_RULE_SETUP
#line 266 "../SqlLexer.lpp"
-return TOKEN_TUPLESAMPLE;
+return TOKEN_TIMESTAMP;
YY_BREAK
case 105:
YY_RULE_SETUP
#line 267 "../SqlLexer.lpp"
-return TOKEN_UNIQUE;
+return TOKEN_TRUE;
YY_BREAK
case 106:
YY_RULE_SETUP
#line 268 "../SqlLexer.lpp"
-return TOKEN_UPDATE;
+return TOKEN_TUPLESAMPLE;
YY_BREAK
case 107:
YY_RULE_SETUP
#line 269 "../SqlLexer.lpp"
-return TOKEN_USING;
+return TOKEN_UNIQUE;
YY_BREAK
case 108:
YY_RULE_SETUP
#line 270 "../SqlLexer.lpp"
-return TOKEN_VALUES;
+return TOKEN_UPDATE;
YY_BREAK
case 109:
YY_RULE_SETUP
#line 271 "../SqlLexer.lpp"
-return TOKEN_VARCHAR;
+return TOKEN_USING;
YY_BREAK
case 110:
YY_RULE_SETUP
#line 272 "../SqlLexer.lpp"
-return TOKEN_WHEN;
+return TOKEN_VALUES;
YY_BREAK
case 111:
YY_RULE_SETUP
#line 273 "../SqlLexer.lpp"
-return TOKEN_WHERE;
+return TOKEN_VARCHAR;
YY_BREAK
case 112:
YY_RULE_SETUP
#line 274 "../SqlLexer.lpp"
-return TOKEN_WITH;
+return TOKEN_WHEN;
YY_BREAK
case 113:
YY_RULE_SETUP
#line 275 "../SqlLexer.lpp"
-return TOKEN_YEAR;
+return TOKEN_WHERE;
YY_BREAK
case 114:
YY_RULE_SETUP
#line 276 "../SqlLexer.lpp"
-return TOKEN_YEARMONTH;
+return TOKEN_WITH;
YY_BREAK
case 115:
YY_RULE_SETUP
-#line 278 "../SqlLexer.lpp"
-return TOKEN_EQ;
+#line 277 "../SqlLexer.lpp"
+return TOKEN_YEAR;
YY_BREAK
case 116:
YY_RULE_SETUP
-#line 279 "../SqlLexer.lpp"
-return TOKEN_NEQ;
+#line 278 "../SqlLexer.lpp"
+return TOKEN_YEARMONTH;
YY_BREAK
case 117:
YY_RULE_SETUP
#line 280 "../SqlLexer.lpp"
-return TOKEN_NEQ;
+return TOKEN_EQ;
YY_BREAK
case 118:
YY_RULE_SETUP
#line 281 "../SqlLexer.lpp"
-return TOKEN_LT;
+return TOKEN_NEQ;
YY_BREAK
case 119:
YY_RULE_SETUP
#line 282 "../SqlLexer.lpp"
-return TOKEN_GT;
+return TOKEN_NEQ;
YY_BREAK
case 120:
YY_RULE_SETUP
#line 283 "../SqlLexer.lpp"
-return TOKEN_LEQ;
+return TOKEN_LT;
YY_BREAK
case 121:
YY_RULE_SETUP
#line 284 "../SqlLexer.lpp"
-return TOKEN_GEQ;
+return TOKEN_GT;
YY_BREAK
case 122:
YY_RULE_SETUP
+#line 285 "../SqlLexer.lpp"
+return TOKEN_LEQ;
+ YY_BREAK
+case 123:
+YY_RULE_SETUP
#line 286 "../SqlLexer.lpp"
+return TOKEN_GEQ;
+ YY_BREAK
+case 124:
+YY_RULE_SETUP
+#line 288 "../SqlLexer.lpp"
return yytext[0];
YY_BREAK
-case 123:
+case 125:
YY_RULE_SETUP
-#line 287 "../SqlLexer.lpp"
+#line 289 "../SqlLexer.lpp"
return yytext[0];
YY_BREAK
/**
* Quoted strings. Prefacing a string with an 'e' or 'E' causes escape
* sequences to be processed (as in PostgreSQL).
**/
-case 124:
+case 126:
YY_RULE_SETUP
-#line 293 "../SqlLexer.lpp"
+#line 295 "../SqlLexer.lpp"
{
yylval->string_value_ = new quickstep::ParseString(yylloc->first_line, yylloc->first_column);
BEGIN(CONDITION_STRING_SINGLE_QUOTED_ESCAPED);
}
YY_BREAK
-case 125:
+case 127:
YY_RULE_SETUP
-#line 298 "../SqlLexer.lpp"
+#line 300 "../SqlLexer.lpp"
{
yylval->string_value_ = new quickstep::ParseString(yylloc->first_line, yylloc->first_column);
BEGIN(CONDITION_STRING_SINGLE_QUOTED);
}
YY_BREAK
-case 126:
+case 128:
YY_RULE_SETUP
-#line 303 "../SqlLexer.lpp"
+#line 305 "../SqlLexer.lpp"
{
yylval->string_value_ = new quickstep::ParseString(yylloc->first_line, yylloc->first_column);
BEGIN(CONDITION_STRING_DOUBLE_QUOTED);
@@ -2079,7 +2092,7 @@ YY_RULE_SETUP
case YY_STATE_EOF(CONDITION_STRING_SINGLE_QUOTED):
case YY_STATE_EOF(CONDITION_STRING_SINGLE_QUOTED_ESCAPED):
case YY_STATE_EOF(CONDITION_STRING_DOUBLE_QUOTED):
-#line 312 "../SqlLexer.lpp"
+#line 314 "../SqlLexer.lpp"
{
delete yylval->string_value_;
BEGIN(INITIAL);
@@ -2090,9 +2103,9 @@ case YY_STATE_EOF(CONDITION_STRING_DOUBLE_QUOTED):
/* Process escape sequences. */
-case 127:
+case 129:
YY_RULE_SETUP
-#line 322 "../SqlLexer.lpp"
+#line 324 "../SqlLexer.lpp"
{
/* Octal code */
unsigned int code;
@@ -2106,9 +2119,9 @@ YY_RULE_SETUP
yylval->string_value_->push_back(code);
}
YY_BREAK
-case 128:
+case 130:
YY_RULE_SETUP
-#line 334 "../SqlLexer.lpp"
+#line 336 "../SqlLexer.lpp"
{
/* Hexadecimal code */
unsigned int code;
@@ -2116,9 +2129,9 @@ YY_RULE_SETUP
yylval->string_value_->push_back(code);
}
YY_BREAK
-case 129:
+case 131:
YY_RULE_SETUP
-#line 340 "../SqlLexer.lpp"
+#line 342 "../SqlLexer.lpp"
{
/* A numeric escape sequence that isn't correctly specified. */
delete yylval->string_value_;
@@ -2127,58 +2140,58 @@ YY_RULE_SETUP
return TOKEN_LEX_ERROR;
}
YY_BREAK
-case 130:
+case 132:
YY_RULE_SETUP
-#line 347 "../SqlLexer.lpp"
+#line 349 "../SqlLexer.lpp"
{
/* Backspace */
yylval->string_value_->push_back('\b');
}
YY_BREAK
-case 131:
+case 133:
YY_RULE_SETUP
-#line 351 "../SqlLexer.lpp"
+#line 353 "../SqlLexer.lpp"
{
/* Form-feed */
yylval->string_value_->push_back('\f');
}
YY_BREAK
-case 132:
+case 134:
YY_RULE_SETUP
-#line 355 "../SqlLexer.lpp"
+#line 357 "../SqlLexer.lpp"
{
/* Newline */
yylval->string_value_->push_back('\n');
}
YY_BREAK
-case 133:
+case 135:
YY_RULE_SETUP
-#line 359 "../SqlLexer.lpp"
+#line 361 "../SqlLexer.lpp"
{
/* Carriage-return */
yylval->string_value_->push_back('\r');
}
YY_BREAK
-case 134:
+case 136:
YY_RULE_SETUP
-#line 363 "../SqlLexer.lpp"
+#line 365 "../SqlLexer.lpp"
{
/* Horizontal Tab */
yylval->string_value_->push_back('\t');
}
YY_BREAK
-case 135:
-/* rule 135 can match eol */
+case 137:
+/* rule 137 can match eol */
YY_RULE_SETUP
-#line 367 "../SqlLexer.lpp"
+#line 369 "../SqlLexer.lpp"
{
/* Any other character (including actual newline or carriage return) */
yylval->string_value_->push_back(yytext[1]);
}
YY_BREAK
-case 136:
+case 138:
YY_RULE_SETUP
-#line 371 "../SqlLexer.lpp"
+#line 373 "../SqlLexer.lpp"
{
/* This should only be encountered right before an EOF. */
delete yylval->string_value_;
@@ -2189,17 +2202,17 @@ YY_RULE_SETUP
YY_BREAK
-case 137:
+case 139:
YY_RULE_SETUP
-#line 381 "../SqlLexer.lpp"
+#line 383 "../SqlLexer.lpp"
{
/* Two quotes in a row become a single quote (this is specified by the SQL standard). */
yylval->string_value_->push_back('\'');
}
YY_BREAK
-case 138:
+case 140:
YY_RULE_SETUP
-#line 385 "../SqlLexer.lpp"
+#line 387 "../SqlLexer.lpp"
{
/* End string */
BEGIN(CONDITION_SQL);
@@ -2208,17 +2221,17 @@ YY_RULE_SETUP
YY_BREAK
-case 139:
+case 141:
YY_RULE_SETUP
-#line 393 "../SqlLexer.lpp"
+#line 395 "../SqlLexer.lpp"
{
/* Two quotes in a row become a single quote (this is specified by the SQL standard). */
yylval->string_value_->push_back('"');
}
YY_BREAK
-case 140:
+case 142:
YY_RULE_SETUP
-#line 397 "../SqlLexer.lpp"
+#line 399 "../SqlLexer.lpp"
{
/* End string */
BEGIN(CONDITION_SQL);
@@ -2226,94 +2239,94 @@ YY_RULE_SETUP
}
YY_BREAK
-case 141:
-/* rule 141 can match eol */
+case 143:
+/* rule 143 can match eol */
YY_RULE_SETUP
-#line 404 "../SqlLexer.lpp"
+#line 406 "../SqlLexer.lpp"
{
/* Scan up to a quote. */
yylval->string_value_->append(yytext, yyleng);
}
YY_BREAK
-case 142:
-/* rule 142 can match eol */
+case 144:
+/* rule 144 can match eol */
YY_RULE_SETUP
-#line 409 "../SqlLexer.lpp"
+#line 411 "../SqlLexer.lpp"
{
/* Scan up to a quote or escape sequence. */
yylval->string_value_->append(yytext, yyleng);
}
YY_BREAK
-case 143:
-/* rule 143 can match eol */
+case 145:
+/* rule 145 can match eol */
YY_RULE_SETUP
-#line 414 "../SqlLexer.lpp"
+#line 416 "../SqlLexer.lpp"
{
/* Scan up to a quote. */
yylval->string_value_->append(yytext, yyleng);
}
YY_BREAK
-case 144:
+case 146:
YY_RULE_SETUP
-#line 420 "../SqlLexer.lpp"
+#line 422 "../SqlLexer.lpp"
{
yylval->string_value_ = new quickstep::ParseString(
yylloc->first_line, yylloc->first_column, std::string(yytext, yyleng));
return TOKEN_NAME;
}
YY_BREAK
-case 145:
+case 147:
YY_RULE_SETUP
-#line 426 "../SqlLexer.lpp"
+#line 428 "../SqlLexer.lpp"
{
yylval->numeric_literal_value_ = new quickstep::NumericParseLiteralValue(
yylloc->first_line, yylloc->first_column, yytext);
return TOKEN_UNSIGNED_NUMVAL;
}
YY_BREAK
-case 146:
+case 148:
YY_RULE_SETUP
-#line 432 "../SqlLexer.lpp"
+#line 434 "../SqlLexer.lpp"
/* comment */
YY_BREAK
-case 147:
-/* rule 147 can match eol */
+case 149:
+/* rule 149 can match eol */
YY_RULE_SETUP
-#line 434 "../SqlLexer.lpp"
+#line 436 "../SqlLexer.lpp"
{ yycolumn = 0; }
YY_BREAK
-case 148:
+case 150:
YY_RULE_SETUP
-#line 436 "../SqlLexer.lpp"
+#line 438 "../SqlLexer.lpp"
; /* ignore white space */
YY_BREAK
/* CONDITION_SQL */
case YY_STATE_EOF(INITIAL):
case YY_STATE_EOF(CONDITION_COMMAND):
case YY_STATE_EOF(CONDITION_SQL):
-#line 440 "../SqlLexer.lpp"
+#line 442 "../SqlLexer.lpp"
{
/* All conditions except for mutli-state string extracting conditions. */
BEGIN(INITIAL);
return TOKEN_EOF;
}
YY_BREAK
-case 149:
+case 151:
YY_RULE_SETUP
-#line 446 "../SqlLexer.lpp"
+#line 448 "../SqlLexer.lpp"
{
BEGIN(INITIAL);
quickstep_yyerror(NULL, yyscanner, NULL, "illegal character");
return TOKEN_LEX_ERROR;
}
YY_BREAK
-case 150:
+case 152:
YY_RULE_SETUP
-#line 452 "../SqlLexer.lpp"
+#line 454 "../SqlLexer.lpp"
YY_FATAL_ERROR( "flex scanner jammed" );
YY_BREAK
-#line 2317 "SqlLexer_gen.cpp"
+#line 2330 "SqlLexer_gen.cpp"
case YY_END_OF_BUFFER:
{
@@ -2607,7 +2620,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 545 )
+ if ( yy_current_state >= 553 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
@@ -2636,11 +2649,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 545 )
+ if ( yy_current_state >= 553 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
- yy_is_jam = (yy_current_state == 544);
+ yy_is_jam = (yy_current_state == 552);
(void)yyg;
return yy_is_jam ? 0 : yy_current_state;
@@ -3474,7 +3487,7 @@ void quickstep_yyfree (void * ptr , yyscan_t yyscanner)
#define YYTABLES_NAME "yytables"
-#line 452 "../SqlLexer.lpp"
+#line 454 "../SqlLexer.lpp"
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/preprocessed/SqlLexer_gen.hpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlLexer_gen.hpp b/parser/preprocessed/SqlLexer_gen.hpp
index d629f04..c14559b 100644
--- a/parser/preprocessed/SqlLexer_gen.hpp
+++ b/parser/preprocessed/SqlLexer_gen.hpp
@@ -360,7 +360,7 @@ extern int quickstep_yylex \
#undef YY_DECL
#endif
-#line 452 "../SqlLexer.lpp"
+#line 454 "../SqlLexer.lpp"
#line 367 "SqlLexer_gen.hpp"
[37/50] [abbrv] incubator-quickstep git commit: Revert "Quickstep gen
stats" (#231)
Posted by hb...@apache.org.
Revert "Quickstep gen stats" (#231)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d136e1dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d136e1dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d136e1dc
Branch: refs/heads/query-manager-used-in-foreman
Commit: d136e1dc955969b9d6ab65e69913ca78634c49b3
Parents: 30f0981
Author: Rogers Jeffrey Leo John <ro...@gmail.com>
Authored: Fri May 20 14:51:35 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:52 2016 -0700
----------------------------------------------------------------------
catalog/Catalog.proto | 2 +-
query_optimizer/CMakeLists.txt | 1 -
query_optimizer/ExecutionGenerator.cpp | 10 ---
relational_operators/CMakeLists.txt | 11 ---
.../GenerateNumRowsStatsOperator.cpp | 42 -----------
.../GenerateNumRowsStatsOperator.hpp | 79 --------------------
6 files changed, 1 insertion(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d136e1dc/catalog/Catalog.proto
----------------------------------------------------------------------
diff --git a/catalog/Catalog.proto b/catalog/Catalog.proto
index 8e44181..ce4bc2e 100644
--- a/catalog/Catalog.proto
+++ b/catalog/Catalog.proto
@@ -82,7 +82,7 @@ message IndexScheme {
message CatalogRelationStatistics {
optional fixed64 num_tuples = 1;
-
+
message NumDistinctValuesEntry {
required int32 attr_id = 1;
required fixed64 num_distinct_values = 2;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d136e1dc/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 1cc38d1..aa2873e 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -111,7 +111,6 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_relationaloperators_DestroyHashOperator
quickstep_relationaloperators_DropTableOperator
quickstep_relationaloperators_FinalizeAggregationOperator
- quickstep_relationaloperators_GenerateNumRowsStatsOperator
quickstep_relationaloperators_HashJoinOperator
quickstep_relationaloperators_InsertOperator
quickstep_relationaloperators_NestedLoopsJoinOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d136e1dc/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 612efd9..c590b6e 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -91,7 +91,6 @@
#include "relational_operators/DestroyHashOperator.hpp"
#include "relational_operators/DropTableOperator.hpp"
#include "relational_operators/FinalizeAggregationOperator.hpp"
-#include "relational_operators/GenerateNumRowsStatsOperator.hpp"
#include "relational_operators/HashJoinOperator.hpp"
#include "relational_operators/InsertOperator.hpp"
#include "relational_operators/NestedLoopsJoinOperator.hpp"
@@ -948,15 +947,6 @@ void ExecutionGenerator::convertCopyFrom(
execution_plan_->addDirectDependency(save_blocks_operator_index,
scan_operator_index,
false /* is_pipeline_breaker */);
-
- const QueryPlan::DAGNodeIndex num_rows_operator_index =
- execution_plan_->addRelationalOperator(new GenerateNumRowsStatsOperator(
- optimizer_context_->catalog_database()->getRelationByIdMutable(
- output_relation->getID())));
- insert_destination_proto->set_relational_op_index(num_rows_operator_index);
- execution_plan_->addDirectDependency(num_rows_operator_index,
- scan_operator_index,
- true /* is_pipeline_breaker */);
}
void ExecutionGenerator::convertCreateIndex(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d136e1dc/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index e211630..eec5300 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -34,9 +34,6 @@ add_library(quickstep_relationaloperators_DropTableOperator DropTableOperator.cp
add_library(quickstep_relationaloperators_FinalizeAggregationOperator
FinalizeAggregationOperator.cpp
FinalizeAggregationOperator.hpp)
-add_library(quickstep_relationaloperators_GenerateNumRowsStatsOperator
- GenerateNumRowsStatsOperator.cpp
- GenerateNumRowsStatsOperator.hpp)
add_library(quickstep_relationaloperators_HashJoinOperator HashJoinOperator.cpp HashJoinOperator.hpp)
add_library(quickstep_relationaloperators_InsertOperator InsertOperator.cpp InsertOperator.hpp)
add_library(quickstep_relationaloperators_NestedLoopsJoinOperator
@@ -162,13 +159,6 @@ target_link_libraries(quickstep_relationaloperators_FinalizeAggregationOperator
quickstep_storage_AggregationOperationState
quickstep_utility_Macros
tmb)
-target_link_libraries(quickstep_relationaloperators_GenerateNumRowsStatsOperator
- glog
- quickstep_catalog_CatalogRelation
- quickstep_cli_PrintToScreen
- quickstep_relationaloperators_RelationalOperator
- quickstep_utility_Macros
- tmb)
target_link_libraries(quickstep_relationaloperators_HashJoinOperator
gflags_nothreads-static
glog
@@ -456,7 +446,6 @@ target_link_libraries(quickstep_relationaloperators
quickstep_relationaloperators_DestroyHashOperator
quickstep_relationaloperators_DropTableOperator
quickstep_relationaloperators_FinalizeAggregationOperator
- quickstep_relationaloperators_GenerateNumRowsStatsOperator
quickstep_relationaloperators_HashJoinOperator
quickstep_relationaloperators_InsertOperator
quickstep_relationaloperators_NestedLoopsJoinOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d136e1dc/relational_operators/GenerateNumRowsStatsOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/GenerateNumRowsStatsOperator.cpp b/relational_operators/GenerateNumRowsStatsOperator.cpp
deleted file mode 100644
index 074e1ca..0000000
--- a/relational_operators/GenerateNumRowsStatsOperator.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#include "relational_operators/GenerateNumRowsStatsOperator.hpp"
-
-#include <memory>
-
-#include "catalog/CatalogRelation.hpp"
-#include "cli/PrintToScreen.hpp"
-
-#include "tmb/id_typedefs.h"
-
-namespace quickstep {
-
-bool GenerateNumRowsStatsOperator::getAllWorkOrders(
- WorkOrdersContainer *container,
- QueryContext *query_context,
- StorageManager *storage_manager,
- const tmb::client_id scheduler_client_id,
- tmb::MessageBus *bus) {
- std::size_t num_tuples =
- PrintToScreen::GetNumTuplesInRelation(*relation_, storage_manager);
- relation_->getStatisticsMutable()->setNumTuples(num_tuples);
- return true;
-}
-
-} // namespace quickstep
-
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d136e1dc/relational_operators/GenerateNumRowsStatsOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/GenerateNumRowsStatsOperator.hpp b/relational_operators/GenerateNumRowsStatsOperator.hpp
deleted file mode 100644
index 8622a63..0000000
--- a/relational_operators/GenerateNumRowsStatsOperator.hpp
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#ifndef QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
-#define QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
-
-#include <memory>
-
-#include "catalog/CatalogRelation.hpp"
-#include "relational_operators/RelationalOperator.hpp"
-#include "utility/Macros.hpp"
-
-#include "glog/logging.h"
-
-#include "tmb/id_typedefs.h"
-
-namespace tmb { class MessageBus; }
-
-namespace quickstep {
-
-class CatalogRelation;
-class QueryContext;
-class StorageManager;
-class WorkOrdersContainer;
-
-/** \addtogroup RelationalOperators
- * @{
- */
-
-/**
- * @brief An operator that gets the number of rows after loading a relation.
- **/
-class GenerateNumRowsStatsOperator : public RelationalOperator {
- public:
- /**
- * @brief Constructor.
- *
- * @param relation The relation to get the number of rows from.
- * This GenNumRowStatsOperator owns relation until
- * the WorkOrder it produces is successfully executed.
- **/
- explicit GenerateNumRowsStatsOperator(CatalogRelation *relation)
- : relation_(relation) {}
- ~GenerateNumRowsStatsOperator() override {}
-
- /**
- * @note no WorkOrder is generated for this operator.
- **/
- bool getAllWorkOrders(WorkOrdersContainer *container,
- QueryContext *query_context,
- StorageManager *storage_manager,
- const tmb::client_id scheduler_client_id,
- tmb::MessageBus *bus) override;
-
- private:
- CatalogRelation *relation_;
-
- DISALLOW_COPY_AND_ASSIGN(GenerateNumRowsStatsOperator);
-};
-
-/** @} */
-
-} // namespace quickstep
-
-#endif // QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
[35/50] [abbrv] incubator-quickstep git commit: In ScopedBuffer,
initialize the allocated memory bytes to 0 (#230)
Posted by hb...@apache.org.
In ScopedBuffer, initialize the allocated memory bytes to 0 (#230)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/be35bb5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/be35bb5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/be35bb5b
Branch: refs/heads/query-manager-used-in-foreman
Commit: be35bb5b008d974faf6638cf40f446128f38030a
Parents: 0fffe50
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 19 18:20:53 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:52 2016 -0700
----------------------------------------------------------------------
utility/ScopedBuffer.hpp | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/be35bb5b/utility/ScopedBuffer.hpp
----------------------------------------------------------------------
diff --git a/utility/ScopedBuffer.hpp b/utility/ScopedBuffer.hpp
index b6fddc5..03290ff 100644
--- a/utility/ScopedBuffer.hpp
+++ b/utility/ScopedBuffer.hpp
@@ -43,9 +43,13 @@ class ScopedBuffer {
* size.
*
* @param alloc_size The number of bytes of memory to allocate.
+ * @param initialize If true, initialize all the bytes of the allocated memory to 0.
**/
- explicit ScopedBuffer(const std::size_t alloc_size) {
+ explicit ScopedBuffer(const std::size_t alloc_size, bool initialize = true) {
internal_ptr_ = std::malloc(alloc_size);
+ if (initialize) {
+ memset(internal_ptr_, 0x0, alloc_size);
+ }
}
/**
@@ -54,10 +58,15 @@ class ScopedBuffer {
*
* @param alloc_size The number of bytes of memory to allocate.
* @param alloc_alignment The alignment of the memory to allocate.
+ * @param initialize If true, initialize all the bytes of the allocated memory to 0.
**/
ScopedBuffer(const std::size_t alloc_size,
- const std::size_t alloc_alignment) {
+ const std::size_t alloc_alignment,
+ const bool initialize = true) {
internal_ptr_ = malloc_with_alignment(alloc_size, alloc_alignment);
+ if (initialize) {
+ memset(internal_ptr_, 0x0, alloc_size);
+ }
}
/**
@@ -110,12 +119,16 @@ class ScopedBuffer {
* size.
*
* @param alloc_size The number of bytes of memory to allocate.
+ * @param initialize If true, initialize all the bytes of the allocated memory to 0.
**/
- void reset(const std::size_t alloc_size) {
+ void reset(const std::size_t alloc_size, const bool initialize = true) {
if (internal_ptr_ != nullptr) {
std::free(internal_ptr_);
}
internal_ptr_ = std::malloc(alloc_size);
+ if (initialize) {
+ memset(internal_ptr_, 0x0, alloc_size);
+ }
}
/**
[46/50] [abbrv] incubator-quickstep git commit: Resolves SMA bugs
revealed by TPCH. (#234)
Posted by hb...@apache.org.
Resolves SMA bugs revealed by TPCH. (#234)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5a81b803
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5a81b803
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5a81b803
Branch: refs/heads/query-manager-used-in-foreman
Commit: 5a81b803ac205785987ebd7eb5d73ee76d1d5c1e
Parents: 1d8c682
Author: Marc S <cr...@users.noreply.github.com>
Authored: Mon May 23 11:11:44 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:53 2016 -0700
----------------------------------------------------------------------
storage/CMakeLists.txt | 1 +
storage/SMAIndexSubBlock.cpp | 55 +++++++++++++++++++-----
storage/SMAIndexSubBlock.hpp | 3 +-
storage/tests/SMAIndexSubBlock_unittest.cpp | 52 ++++++++++++++++++++++
4 files changed, 99 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5a81b803/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 26a3e32..a3093df 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -1451,6 +1451,7 @@ target_link_libraries(SMAIndexSubBlock_unittest
quickstep_types_CharType
quickstep_types_DoubleType
quickstep_types_FloatType
+ quickstep_types_IntType
quickstep_types_LongType
quickstep_types_TypeFactory
quickstep_types_TypeID
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5a81b803/storage/SMAIndexSubBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/SMAIndexSubBlock.cpp b/storage/SMAIndexSubBlock.cpp
index aa9bc54..3a1ccc2 100644
--- a/storage/SMAIndexSubBlock.cpp
+++ b/storage/SMAIndexSubBlock.cpp
@@ -181,19 +181,22 @@ Selectivity getSelectivity(const TypedValue &literal,
}
SMAPredicate* SMAPredicate::ExtractSMAPredicate(const ComparisonPredicate &predicate) {
- if (predicate.getLeftOperand().hasStaticValue()) {
- DLOG_IF(FATAL, predicate.getRightOperand().getDataSource() != Scalar::kAttribute);
+ // Check that the predicate is contains an attribute and a literal value.
+ if (predicate.getLeftOperand().hasStaticValue() &&
+ predicate.getRightOperand().getDataSource() == Scalar::kAttribute) {
return new SMAPredicate(
static_cast<const ScalarAttribute&>(predicate.getRightOperand()).getAttribute().getID(),
flipComparisonID(predicate.getComparison().getComparisonID()),
predicate.getLeftOperand().getStaticValue().makeReferenceToThis());
- } else {
- DLOG_IF(FATAL, predicate.getLeftOperand().getDataSource() != Scalar::kAttribute);
+ } else if (predicate.getRightOperand().hasStaticValue() &&
+ predicate.getLeftOperand().getDataSource() == Scalar::kAttribute) {
return new SMAPredicate(
static_cast<const ScalarAttribute&>(predicate.getLeftOperand()).getAttribute().getID(),
predicate.getComparison().getComparisonID(),
predicate.getRightOperand().getStaticValue().makeReferenceToThis());
}
+ // Predicate is improper form, so return nullptr.
+ return nullptr;
}
/**
@@ -603,12 +606,47 @@ Selectivity SMAIndexSubBlock::getSelectivityForPredicate(const ComparisonPredica
}
std::unique_ptr<SMAPredicate> sma_predicate(SMAPredicate::ExtractSMAPredicate(predicate));
+ // The predicate did not contain a static value to compare against.
+ if (!sma_predicate) {
+ return Selectivity::kUnknown;
+ }
+
const SMAEntry *entry = getEntryChecked(sma_predicate->attribute);
// The attribute wasn't indexed.
if (entry == nullptr || !entry->min_entry_ref.valid || !entry->max_entry_ref.valid) {
return Selectivity::kUnknown;
}
+
+ // Check that the type of the comparison is the same or can be coerced the
+ // type of the underlying attribute.
+ if (sma_predicate->literal.getTypeID() != entry->type_id) {
+ // Try to coerce the literal type to the entry type.
+ // It may have to account for variable length attributes.
+ int literal_length = -1;
+ if (TypeFactory::TypeRequiresLengthParameter(sma_predicate->literal.getTypeID())) {
+ literal_length = sma_predicate->literal.getAsciiStringLength();
+ }
+
+ const Type &literal_type = literal_length == -1 ?
+ TypeFactory::GetType(sma_predicate->literal.getTypeID(), false) :
+ TypeFactory::GetType(sma_predicate->literal.getTypeID(), static_cast<std::size_t>(literal_length), false);
+ const Type &attribute_type = literal_length == -1 ?
+ TypeFactory::GetType(entry->type_id, false) :
+ TypeFactory::GetType(entry->type_id, static_cast<std::size_t>(literal_length), false);
+ if (attribute_type.isSafelyCoercibleFrom(literal_type)) {
+ // Convert the literal's type inside the predicate.
+ SMAPredicate *replacement = new SMAPredicate(
+ sma_predicate->attribute,
+ sma_predicate->comparison,
+ attribute_type.coerceValue(sma_predicate->literal, literal_type));
+ sma_predicate.reset(replacement);
+ } else {
+ // The literal type cannot be converted, so do not evaluate with the SMA.
+ return Selectivity::kUnknown;
+ }
+ }
+
return sma_internal::getSelectivity(
sma_predicate->literal,
sma_predicate->comparison,
@@ -623,12 +661,9 @@ predicate_cost_t SMAIndexSubBlock::estimatePredicateEvaluationCost(
DCHECK(initialized_);
// Check that at least one of the operands has a static value.
- if (predicate.getLeftOperand().hasStaticValue() ||
- predicate.getRightOperand().hasStaticValue()) {
- Selectivity selectivity = getSelectivityForPredicate(predicate);
- if (selectivity == Selectivity::kAll || selectivity == Selectivity::kNone) {
- return predicate_cost::kConstantTime;
- }
+ Selectivity selectivity = getSelectivityForPredicate(predicate);
+ if (selectivity == Selectivity::kAll || selectivity == Selectivity::kNone) {
+ return predicate_cost::kConstantTime;
}
return predicate_cost::kInfinite;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5a81b803/storage/SMAIndexSubBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/SMAIndexSubBlock.hpp b/storage/SMAIndexSubBlock.hpp
index 71ca75b..895add9 100644
--- a/storage/SMAIndexSubBlock.hpp
+++ b/storage/SMAIndexSubBlock.hpp
@@ -120,7 +120,6 @@ struct SMAPredicate {
// that the predicate had not been used previously.
Selectivity selectivity;
- private:
SMAPredicate(const attribute_id attr,
const ComparisonID comp,
const TypedValue lit)
@@ -376,7 +375,7 @@ class SMAIndexSubBlock : public IndexSubBlock {
private:
bool requiresRebuild() const;
- inline sma_internal::Selectivity getSelectivityForPredicate(const ComparisonPredicate &predicate) const;
+ sma_internal::Selectivity getSelectivityForPredicate(const ComparisonPredicate &predicate) const;
// Retrieves an entry, first checking if the given attribute is indexed.
inline const sma_internal::SMAEntry* getEntryChecked(attribute_id attribute) const {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5a81b803/storage/tests/SMAIndexSubBlock_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/SMAIndexSubBlock_unittest.cpp b/storage/tests/SMAIndexSubBlock_unittest.cpp
index b709712..ad842c2 100644
--- a/storage/tests/SMAIndexSubBlock_unittest.cpp
+++ b/storage/tests/SMAIndexSubBlock_unittest.cpp
@@ -41,6 +41,7 @@
#include "types/CharType.hpp"
#include "types/DoubleType.hpp"
#include "types/FloatType.hpp"
+#include "types/IntType.hpp"
#include "types/LongType.hpp"
#include "types/TypeFactory.hpp"
#include "types/TypeID.hpp"
@@ -301,6 +302,10 @@ class SMAIndexSubBlockTest : public ::testing::Test {
return !index_->header_->index_consistent;
}
+ Selectivity getSelectivityForPredicate(const ComparisonPredicate &predicate) const {
+ return index_->getSelectivityForPredicate(predicate);
+ }
+
std::unique_ptr<CatalogRelation> relation_;
std::unique_ptr<MockTupleStorageSubBlock> tuple_store_;
ScopedBuffer index_memory_;
@@ -359,6 +364,53 @@ TEST_F(SMAIndexSubBlockTest, TestConstructor) {
EXPECT_FALSE(requiresRebuild());
}
+TEST_F(SMAIndexSubBlockTest, TestGetSelectivityForPredicate) {
+ // Index long, float, bigchar, and varchar type.
+ createIndex({0, 2, 5, 6}, kIndexSubBlockSize);
+ int min = 0, max = 9010, step = 10;
+ for (std::size_t i = min; i <= static_cast<std::size_t>(max); i += step) {
+ generateAndInsertTuple(i, false, "suffix");
+ }
+ index_->rebuild();
+
+ // Tuple storage block now contains long values [0, 9010], and float values
+ // [0, 2252.5].
+ // Test several value types.
+ // Test with inline values, returning both possible value types.
+ std::unique_ptr<ComparisonPredicate> predicate(
+ generateNumericComparisonPredicate<LongType>(ComparisonID::kLess, 0, min));
+ Selectivity cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kNone, cost);
+
+ // Test on the float attribute with a type that will need coercion. In this
+ // case, a long type should not coerce to an integer type.
+ predicate.reset(generateNumericComparisonPredicate<LongType>(ComparisonID::kLess, 2, min));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kUnknown, cost);
+
+ predicate.reset(generateNumericComparisonPredicate<IntType>(ComparisonID::kLess, 2, min));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kNone, cost);
+
+ // This should be unknown because of the loss of precision on coercion.
+ predicate.reset(generateNumericComparisonPredicate<FloatType>(ComparisonID::kLess, 0, min + 0.5));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kUnknown, cost);
+
+ // Test coercion with String types as well.
+ predicate.reset(generateStringComparisonPredicate(ComparisonID::kLess, 6, ""));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kNone, cost);
+
+ predicate.reset(generateStringComparisonPredicate(ComparisonID::kLess, 5, ""));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kNone, cost);
+
+ predicate.reset(generateStringComparisonPredicate(ComparisonID::kLess, 5, "999999999999999"));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kAll, cost);
+}
+
TEST_F(SMAIndexSubBlockTest, TestRebuild) {
// This test is checking to see if the min/max values are set correctly on
// a rebuild of the index.
[36/50] [abbrv] incubator-quickstep git commit: Quickstep gen stats
(#225)
Posted by hb...@apache.org.
Quickstep gen stats (#225)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/30f0981d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/30f0981d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/30f0981d
Branch: refs/heads/query-manager-used-in-foreman
Commit: 30f0981db21cb53c9edbb7799a90f0de8979dde7
Parents: be35bb5
Author: Rogers Jeffrey Leo John <ro...@gmail.com>
Authored: Thu May 19 20:31:37 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:52 2016 -0700
----------------------------------------------------------------------
catalog/Catalog.proto | 2 +-
query_optimizer/CMakeLists.txt | 1 +
query_optimizer/ExecutionGenerator.cpp | 10 +++
relational_operators/CMakeLists.txt | 11 +++
.../GenerateNumRowsStatsOperator.cpp | 42 +++++++++++
.../GenerateNumRowsStatsOperator.hpp | 79 ++++++++++++++++++++
6 files changed, 144 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30f0981d/catalog/Catalog.proto
----------------------------------------------------------------------
diff --git a/catalog/Catalog.proto b/catalog/Catalog.proto
index ce4bc2e..8e44181 100644
--- a/catalog/Catalog.proto
+++ b/catalog/Catalog.proto
@@ -82,7 +82,7 @@ message IndexScheme {
message CatalogRelationStatistics {
optional fixed64 num_tuples = 1;
-
+
message NumDistinctValuesEntry {
required int32 attr_id = 1;
required fixed64 num_distinct_values = 2;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30f0981d/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index aa2873e..1cc38d1 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -111,6 +111,7 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_relationaloperators_DestroyHashOperator
quickstep_relationaloperators_DropTableOperator
quickstep_relationaloperators_FinalizeAggregationOperator
+ quickstep_relationaloperators_GenerateNumRowsStatsOperator
quickstep_relationaloperators_HashJoinOperator
quickstep_relationaloperators_InsertOperator
quickstep_relationaloperators_NestedLoopsJoinOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30f0981d/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index c590b6e..612efd9 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -91,6 +91,7 @@
#include "relational_operators/DestroyHashOperator.hpp"
#include "relational_operators/DropTableOperator.hpp"
#include "relational_operators/FinalizeAggregationOperator.hpp"
+#include "relational_operators/GenerateNumRowsStatsOperator.hpp"
#include "relational_operators/HashJoinOperator.hpp"
#include "relational_operators/InsertOperator.hpp"
#include "relational_operators/NestedLoopsJoinOperator.hpp"
@@ -947,6 +948,15 @@ void ExecutionGenerator::convertCopyFrom(
execution_plan_->addDirectDependency(save_blocks_operator_index,
scan_operator_index,
false /* is_pipeline_breaker */);
+
+ const QueryPlan::DAGNodeIndex num_rows_operator_index =
+ execution_plan_->addRelationalOperator(new GenerateNumRowsStatsOperator(
+ optimizer_context_->catalog_database()->getRelationByIdMutable(
+ output_relation->getID())));
+ insert_destination_proto->set_relational_op_index(num_rows_operator_index);
+ execution_plan_->addDirectDependency(num_rows_operator_index,
+ scan_operator_index,
+ true /* is_pipeline_breaker */);
}
void ExecutionGenerator::convertCreateIndex(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30f0981d/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index eec5300..e211630 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -34,6 +34,9 @@ add_library(quickstep_relationaloperators_DropTableOperator DropTableOperator.cp
add_library(quickstep_relationaloperators_FinalizeAggregationOperator
FinalizeAggregationOperator.cpp
FinalizeAggregationOperator.hpp)
+add_library(quickstep_relationaloperators_GenerateNumRowsStatsOperator
+ GenerateNumRowsStatsOperator.cpp
+ GenerateNumRowsStatsOperator.hpp)
add_library(quickstep_relationaloperators_HashJoinOperator HashJoinOperator.cpp HashJoinOperator.hpp)
add_library(quickstep_relationaloperators_InsertOperator InsertOperator.cpp InsertOperator.hpp)
add_library(quickstep_relationaloperators_NestedLoopsJoinOperator
@@ -159,6 +162,13 @@ target_link_libraries(quickstep_relationaloperators_FinalizeAggregationOperator
quickstep_storage_AggregationOperationState
quickstep_utility_Macros
tmb)
+target_link_libraries(quickstep_relationaloperators_GenerateNumRowsStatsOperator
+ glog
+ quickstep_catalog_CatalogRelation
+ quickstep_cli_PrintToScreen
+ quickstep_relationaloperators_RelationalOperator
+ quickstep_utility_Macros
+ tmb)
target_link_libraries(quickstep_relationaloperators_HashJoinOperator
gflags_nothreads-static
glog
@@ -446,6 +456,7 @@ target_link_libraries(quickstep_relationaloperators
quickstep_relationaloperators_DestroyHashOperator
quickstep_relationaloperators_DropTableOperator
quickstep_relationaloperators_FinalizeAggregationOperator
+ quickstep_relationaloperators_GenerateNumRowsStatsOperator
quickstep_relationaloperators_HashJoinOperator
quickstep_relationaloperators_InsertOperator
quickstep_relationaloperators_NestedLoopsJoinOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30f0981d/relational_operators/GenerateNumRowsStatsOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/GenerateNumRowsStatsOperator.cpp b/relational_operators/GenerateNumRowsStatsOperator.cpp
new file mode 100644
index 0000000..074e1ca
--- /dev/null
+++ b/relational_operators/GenerateNumRowsStatsOperator.cpp
@@ -0,0 +1,42 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "relational_operators/GenerateNumRowsStatsOperator.hpp"
+
+#include <memory>
+
+#include "catalog/CatalogRelation.hpp"
+#include "cli/PrintToScreen.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace quickstep {
+
+bool GenerateNumRowsStatsOperator::getAllWorkOrders(
+ WorkOrdersContainer *container,
+ QueryContext *query_context,
+ StorageManager *storage_manager,
+ const tmb::client_id scheduler_client_id,
+ tmb::MessageBus *bus) {
+ std::size_t num_tuples =
+ PrintToScreen::GetNumTuplesInRelation(*relation_, storage_manager);
+ relation_->getStatisticsMutable()->setNumTuples(num_tuples);
+ return true;
+}
+
+} // namespace quickstep
+
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30f0981d/relational_operators/GenerateNumRowsStatsOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/GenerateNumRowsStatsOperator.hpp b/relational_operators/GenerateNumRowsStatsOperator.hpp
new file mode 100644
index 0000000..8622a63
--- /dev/null
+++ b/relational_operators/GenerateNumRowsStatsOperator.hpp
@@ -0,0 +1,79 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
+#define QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
+
+#include <memory>
+
+#include "catalog/CatalogRelation.hpp"
+#include "relational_operators/RelationalOperator.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+
+namespace tmb { class MessageBus; }
+
+namespace quickstep {
+
+class CatalogRelation;
+class QueryContext;
+class StorageManager;
+class WorkOrdersContainer;
+
+/** \addtogroup RelationalOperators
+ * @{
+ */
+
+/**
+ * @brief An operator that gets the number of rows after loading a relation.
+ **/
+class GenerateNumRowsStatsOperator : public RelationalOperator {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param relation The relation to get the number of rows from.
+ * This GenNumRowStatsOperator owns relation until
+ * the WorkOrder it produces is successfully executed.
+ **/
+ explicit GenerateNumRowsStatsOperator(CatalogRelation *relation)
+ : relation_(relation) {}
+ ~GenerateNumRowsStatsOperator() override {}
+
+ /**
+ * @note no WorkOrder is generated for this operator.
+ **/
+ bool getAllWorkOrders(WorkOrdersContainer *container,
+ QueryContext *query_context,
+ StorageManager *storage_manager,
+ const tmb::client_id scheduler_client_id,
+ tmb::MessageBus *bus) override;
+
+ private:
+ CatalogRelation *relation_;
+
+ DISALLOW_COPY_AND_ASSIGN(GenerateNumRowsStatsOperator);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
[25/50] [abbrv] incubator-quickstep git commit: Refactor
hasNUMAPlacementScheme. (#215)
Posted by hb...@apache.org.
Refactor hasNUMAPlacementScheme. (#215)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ab79cf64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ab79cf64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ab79cf64
Branch: refs/heads/query-manager-used-in-foreman
Commit: ab79cf646eedc931ab011bd1f78cb73ed87a0436
Parents: 8939c25
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Sun May 8 06:21:51 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:32 2016 -0700
----------------------------------------------------------------------
catalog/CatalogRelation.hpp | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ab79cf64/catalog/CatalogRelation.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.hpp b/catalog/CatalogRelation.hpp
index 3701090..312f3b4 100644
--- a/catalog/CatalogRelation.hpp
+++ b/catalog/CatalogRelation.hpp
@@ -31,6 +31,11 @@
#include "catalog/CatalogRelationSchema.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "catalog/IndexScheme.hpp"
+
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include "catalog/NUMAPlacementScheme.hpp"
+#endif // QUICKSTEP_HAVE_LIBNUMA
+
#include "catalog/PartitionScheme.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageBlockLayout.hpp"
@@ -40,10 +45,6 @@
#include "threading/SpinSharedMutex.hpp"
#include "utility/Macros.hpp"
-#ifdef QUICKSTEP_HAVE_LIBNUMA
-#include "catalog/NUMAPlacementScheme.hpp"
-#endif
-
namespace quickstep {
class CatalogDatabase;
@@ -135,16 +136,20 @@ class CatalogRelation : public CatalogRelationSchema {
return partition_scheme_.get();
}
-#ifdef QUICKSTEP_HAVE_LIBNUMA
/**
* @brief Check if a NUMA placement scheme is available for the relation.
*
* @return True if the relation has a NUMA placement scheme, false otherwise.
**/
bool hasNUMAPlacementScheme() const {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
return placement_scheme_ != nullptr;
+#else
+ return false;
+#endif // QUICKSTEP_HAVE_LIBNUMA
}
+#ifdef QUICKSTEP_HAVE_LIBNUMA
/**
* @brief Get the NUMA placement scheme of the catalog relation.
* @warning This is only safe if hasNUMAPlacementScheme() is true.
@@ -182,7 +187,7 @@ class CatalogRelation : public CatalogRelationSchema {
void setNUMAPlacementScheme(NUMAPlacementScheme *placement_scheme) {
placement_scheme_.reset(placement_scheme);
}
-#endif
+#endif // QUICKSTEP_HAVE_LIBNUMA
/**
* @brief Check if an index scheme is available for the relation.
@@ -397,7 +402,7 @@ class CatalogRelation : public CatalogRelationSchema {
// of the relation and the NUMA nodes/sockets. It also maintains a mapping
// between the blocks of the relation and the NUMA nodes..
std::unique_ptr<NUMAPlacementScheme> placement_scheme_;
-#endif
+#endif // QUICKSTEP_HAVE_LIBNUMA
friend class CatalogTest;
[45/50] [abbrv] incubator-quickstep git commit: Explicitly specify
where tcmalloc comes from (#235)
Posted by hb...@apache.org.
Explicitly specify where tcmalloc comes from (#235)
This change adds a declaration that `libtcmalloc_minimal.a` is generated
by the external project. This is needed by the Ninja generator to
properly build.
To reproduce:
run `cmake -GNinja path/to/quickstep`
then `cmake --build path/to/build/dir` and see it fails.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/908d3675
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/908d3675
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/908d3675
Branch: refs/heads/query-manager-used-in-foreman
Commit: 908d367510052a68ed245ef0ce3dd3c834250850
Parents: 5a81b80
Author: Jesse Zhang <jz...@pivotallabs.com>
Authored: Mon May 23 13:38:59 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:53 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/908d3675/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 87a8f7c..2ab0f57 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -490,6 +490,7 @@ if(USE_TCMALLOC)
CXXFLAGS=${THIRD_PARTY_CXX_FLAGS}
BUILD_COMMAND make
BUILD_IN_SOURCE 0
+ BUILD_BYPRODUCTS <INSTALL_DIR>/lib/libtcmalloc_minimal.a
)
# Static libtcmalloc_minimal.a
add_library(libtcmalloc_minimal STATIC IMPORTED)
[42/50] [abbrv] incubator-quickstep git commit: QUICKSTEP-1: Add
incubation disclaimer to README.md.
Posted by hb...@apache.org.
QUICKSTEP-1: Add incubation disclaimer to README.md.
This closes #2
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/fe80c013
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/fe80c013
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/fe80c013
Branch: refs/heads/query-manager-used-in-foreman
Commit: fe80c013a7a466e72e63bd2641b75a165028f4e4
Parents: 2ddb67b
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Tue Apr 19 12:23:51 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:53 2016 -0700
----------------------------------------------------------------------
DISCLAIMER | 11 +++++++++++
README.md | 21 ++++++++++++++++++---
2 files changed, 29 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fe80c013/DISCLAIMER
----------------------------------------------------------------------
diff --git a/DISCLAIMER b/DISCLAIMER
new file mode 100644
index 0000000..09d629d
--- /dev/null
+++ b/DISCLAIMER
@@ -0,0 +1,11 @@
+Apache Quickstep is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further
+review indicates that the infrastructure, communications, and decision
+making process have stabilized in a manner consistent with other
+successful ASF projects.
+
+While incubation status is not necessarily a reflection of the
+completeness or stability of the code, it does indicate that the
+project has yet to be fully endorsed by the ASF.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fe80c013/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index c44e0b9..7cea80c 100644
--- a/README.md
+++ b/README.md
@@ -1,14 +1,16 @@
-# Quickstep
+# Apache Quickstep (Incubating)
[![Travis Widget]][Travis]
[Travis]: https://travis-ci.org/pivotalsoftware/quickstep
[Travis Widget]: https://travis-ci.org/pivotalsoftware/quickstep.svg?branch=master
-Quickstep is an experimental high-performance database engine designed with the
-aim of Data at Bare-Metal Speed. It began life in 2011 as a
+Apache Quickstep is an experimental high-performance database engine designed with the
+aim of Data at Bare-Metal Speed. It began life in 2011 as a
[research project at the University of Wisconsin](https://quickstep.cs.wisc.edu)
and was acquired by [Pivotal](https://pivotal.io) in 2015.
+Quickstep entered incubation at the
+[Apache Software Foundation](https://www.apache.org) in April, 2016.
## Getting Started (Building)
@@ -91,3 +93,16 @@ concerns of a database system. The main modules are:
## Licensing
Quickstep is licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/pivotalsoftware/quickstep/blob/master/LICENSE) for the full license text.
+
+## Disclaimer
+Apache Quickstep is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further
+review indicates that the infrastructure, communications, and decision
+making process have stabilized in a manner consistent with other
+successful ASF projects.
+
+While incubation status is not necessarily a reflection of the
+completeness or stability of the code, it does indicate that the
+project has yet to be fully endorsed by the ASF.
[40/50] [abbrv] incubator-quickstep git commit: Reverting the PR that
gets the number of rows (#233)
Posted by hb...@apache.org.
Reverting the PR that gets the number of rows (#233)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1d8c6822
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1d8c6822
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1d8c6822
Branch: refs/heads/query-manager-used-in-foreman
Commit: 1d8c6822896ea7032b1f9f623f481177e534d7e4
Parents: 1605fd8
Author: Rogers Jeffrey Leo John <ro...@gmail.com>
Authored: Sat May 21 07:36:06 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:52 2016 -0700
----------------------------------------------------------------------
cli/CommandExecutor.cpp | 23 +++--------------------
cli/tests/command_executor/D.test | 14 +++++++-------
cli/tests/command_executor/Dt.test | 20 ++++++++++----------
3 files changed, 20 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1d8c6822/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 3cb3f86..ddcd38f 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -72,7 +72,6 @@ void executeDescribeDatabase(
// Column width initialized to 6 to take into account the header name
// and the column value table
int max_column_width = C::kInitMaxColumnWidth;
- vector<std::size_t> num_tuples;
vector<std::size_t> num_blocks;
const CatalogRelation *relation = nullptr;
if (arguments->size() == 0) {
@@ -80,8 +79,6 @@ void executeDescribeDatabase(
max_column_width =
std::max(static_cast<int>(rel.getName().length()), max_column_width);
num_blocks.push_back(rel.size_blocks());
- num_tuples.push_back(
- PrintToScreen::GetNumTuplesInRelation(rel, storage_manager));
}
} else {
const ParseString &table_name = arguments->front();
@@ -94,49 +91,35 @@ void executeDescribeDatabase(
max_column_width = std::max(static_cast<int>(relation->getName().length()),
max_column_width);
num_blocks.push_back(relation->size_blocks());
- num_tuples.push_back(PrintToScreen::GetNumTuplesInRelation(
- *relation,
- storage_manager));
}
// Only if we have relations work on the printing logic.
if (catalog_database.size() > 0) {
const std::size_t max_num_blocks = *std::max_element(num_blocks.begin(), num_blocks.end());
- const std::size_t max_num_rows = *std::max_element(num_tuples.begin(), num_tuples.end());
- const int max_num_rows_digits = std::max(PrintToScreen::GetNumberOfDigits(max_num_rows),
- C::kInitMaxColumnWidth);
const int max_num_blocks_digits = std::max(PrintToScreen::GetNumberOfDigits(max_num_blocks),
C::kInitMaxColumnWidth+2);
-
vector<int> column_widths;
column_widths.push_back(max_column_width +1);
column_widths.push_back(C::kInitMaxColumnWidth + 1);
column_widths.push_back(max_num_blocks_digits + 1);
- column_widths.push_back(max_num_rows_digits + 1);
fputs(" List of relations\n\n", out);
fprintf(out, "%-*s |", max_column_width+1, " Name");
fprintf(out, "%-*s |", C::kInitMaxColumnWidth, " Type");
- fprintf(out, "%-*s |", max_num_blocks_digits, " Blocks");
- fprintf(out, "%-*s\n", max_num_rows_digits, " Rows");
+ fprintf(out, "%-*s\n", max_num_blocks_digits, " Blocks");
PrintToScreen::printHBar(column_widths, out);
// If there are no argument print the entire list of tables
// else print the particular table only.
- vector<std::size_t>::const_iterator num_tuples_it = num_tuples.begin();
vector<std::size_t>::const_iterator num_blocks_it = num_blocks.begin();
if (arguments->size() == 0) {
for (const CatalogRelation &rel : catalog_database) {
fprintf(out, " %-*s |", max_column_width, rel.getName().c_str());
fprintf(out, " %-*s |", C::kInitMaxColumnWidth - 1, "table");
- fprintf(out, " %-*lu |", max_num_blocks_digits - 1, *num_blocks_it);
- fprintf(out, " %-*lu\n", max_num_rows_digits - 1, *num_tuples_it);
- ++num_tuples_it;
+ fprintf(out, " %-*lu\n", max_num_blocks_digits - 1, *num_blocks_it);
++num_blocks_it;
}
} else {
fprintf(out, " %-*s |", max_column_width, relation->getName().c_str());
fprintf(out, " %-*s |", C::kInitMaxColumnWidth -1, "table");
- fprintf(out, " %-*lu |", max_num_blocks_digits - 1, *num_blocks_it);
- fprintf(out, " %-*lu\n", max_num_rows_digits - 1, *num_tuples_it);
- ++num_tuples_it;
+ fprintf(out, " %-*lu\n", max_num_blocks_digits - 1, *num_blocks_it);
++num_blocks_it;
}
fputc('\n', out);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1d8c6822/cli/tests/command_executor/D.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/D.test b/cli/tests/command_executor/D.test
index 45f8d8b..d18ab47 100644
--- a/cli/tests/command_executor/D.test
+++ b/cli/tests/command_executor/D.test
@@ -104,13 +104,13 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
--
List of relations
- Name | Type | Blocks | Rows
-+--------------------------------------+-------+---------+-------+
- foo | table | 1 | 5
- foo2 | table | 1 | 2
- foo3 | table | 1 | 1
- foo4 | table | 0 | 0
- averylongtablenamethatseemstoneverend | table | 1 | 3
+ Name | Type | Blocks
++--------------------------------------+-------+---------+
+ foo | table | 1
+ foo2 | table | 1
+ foo3 | table | 1
+ foo4 | table | 0
+ averylongtablenamethatseemstoneverend | table | 1
==
\d invalidtable
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1d8c6822/cli/tests/command_executor/Dt.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/Dt.test b/cli/tests/command_executor/Dt.test
index 1de6360..65c6be3 100644
--- a/cli/tests/command_executor/Dt.test
+++ b/cli/tests/command_executor/Dt.test
@@ -52,22 +52,22 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
--
List of relations
- Name | Type | Blocks | Rows
-+--------------------------------------+-------+---------+-------+
- foo | table | 1 | 5
- foo2 | table | 1 | 2
- foo3 | table | 1 | 1
- foo4 | table | 0 | 0
- averylongtablenamethatseemstoneverend | table | 1 | 3
+ Name | Type | Blocks
++--------------------------------------+-------+---------+
+ foo | table | 1
+ foo2 | table | 1
+ foo3 | table | 1
+ foo4 | table | 0
+ averylongtablenamethatseemstoneverend | table | 1
==
\dt foo
--
List of relations
- Name | Type | Blocks | Rows
-+-------+-------+---------+-------+
- foo | table | 1 | 5
+ Name | Type | Blocks
++-------+-------+---------+
+ foo | table | 1
==
\dt invalidtable
[22/50] [abbrv] incubator-quickstep git commit: Added print_query
gflag (#213)
Posted by hb...@apache.org.
Added print_query gflag (#213)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8939c253
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8939c253
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8939c253
Branch: refs/heads/query-manager-used-in-foreman
Commit: 8939c25397c55149ed14834e7b4a41d88c6f9d00
Parents: 32e7c1b
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Sat May 7 09:44:56 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:31 2016 -0700
----------------------------------------------------------------------
cli/QuickstepCli.cpp | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8939c253/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index ec195f7..4c0a166 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -144,6 +146,10 @@ DEFINE_string(worker_affinities, "",
"means that they will all be runable on any CPU according to "
"the kernel's own scheduling policy).");
DEFINE_bool(initialize_db, false, "If true, initialize a database.");
+DEFINE_bool(print_query, false,
+ "Print each input query statement. This is useful when running a "
+ "large number of queries in a batch.");
+
} // namespace quickstep
int main(int argc, char* argv[]) {
@@ -336,6 +342,10 @@ int main(int argc, char* argv[]) {
break;
}
+ if (quickstep::FLAGS_print_query) {
+ printf("\n%s\n", command_string->c_str());
+ }
+
parser_wrapper->feedNextBuffer(command_string);
bool quitting = false;
[21/50] [abbrv] incubator-quickstep git commit: Cleanup in Preloader
class (#209)
Posted by hb...@apache.org.
Cleanup in Preloader class (#209)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/32e7c1bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/32e7c1bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/32e7c1bd
Branch: refs/heads/query-manager-used-in-foreman
Commit: 32e7c1bd8159b7c3883e0cb8213d330276b4cf41
Parents: 767b2ef
Author: Harshad Deshmukh <d....@gmail.com>
Authored: Fri May 6 22:29:41 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:31 2016 -0700
----------------------------------------------------------------------
storage/PreloaderThread.cpp | 44 ++++++++++++++++++++++++----------------
storage/PreloaderThread.hpp | 7 +++++++
2 files changed, 34 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32e7c1bd/storage/PreloaderThread.cpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.cpp b/storage/PreloaderThread.cpp
index 8f600b8..4bfa9c5 100644
--- a/storage/PreloaderThread.cpp
+++ b/storage/PreloaderThread.cpp
@@ -49,37 +49,46 @@ void PreloaderThread::run() {
std::size_t blocks_loaded = 0;
for (const CatalogRelation &relation : database_) {
- if (relation.hasPartitionScheme()) {
+ if (relation.hasPartitionScheme() && relation.hasNUMAPlacementScheme()) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
blocks_loaded += preloadNUMAAware(relation, blocks_loaded, num_slots);
+#endif
} else {
+ // NUMA agnostic preloading of relation.
std::vector<block_id> blocks = relation.getBlocksSnapshot();
for (block_id current_block_id : blocks) {
try {
- BlockReference current_block = storage_manager_->getBlock(current_block_id, relation);
+ BlockReference current_block =
+ storage_manager_->getBlock(current_block_id, relation);
} catch (...) {
- LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks\n";
+ LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks";
throw;
}
++blocks_loaded;
if (blocks_loaded == num_slots) {
- // The buffer pool has filled up. But, some database blocks are not loaded.
- printf(" The database is larger than the buffer pool. Only %lu blocks were loaded ",
- blocks_loaded);
+ // The buffer pool has filled up. But, some database blocks are not
+ // loaded.
+ printf(
+ " The database is larger than the buffer pool. Only %lu blocks "
+ "were loaded ", blocks_loaded);
return;
}
}
+ LOG(INFO) << "Relation " << relation.getName()
+ << " completely preloaded in buffer pool";
}
}
printf(" Loaded %lu blocks ", blocks_loaded);
}
+#ifdef QUICKSTEP_HAVE_LIBNUMA
std::size_t PreloaderThread::preloadNUMAAware(
const CatalogRelation &relation,
const std::size_t num_previously_loaded_blocks,
const std::size_t num_slots) {
-#ifdef QUICKSTEP_HAVE_LIBNUMA
std::size_t blocks_loaded = 0;
- const NUMAPlacementScheme *placement_scheme = relation.getNUMAPlacementSchemePtr();
+ const NUMAPlacementScheme *placement_scheme =
+ relation.getNUMAPlacementSchemePtr();
DCHECK(placement_scheme != nullptr);
DCHECK(relation.hasPartitionScheme());
const PartitionScheme &part_scheme = relation.getPartitionScheme();
@@ -96,15 +105,17 @@ std::size_t PreloaderThread::preloadNUMAAware(
BlockReference current_block = storage_manager_->getBlock(
curr_block_id, relation, partition_numa_node_id);
} catch (...) {
- LOG(ERROR) << "Error after loading "
+ LOG(ERROR) << "Error while preloading: After loading total "
<< blocks_loaded + num_previously_loaded_blocks
- << " blocks\n";
+ << " blocks and " << blocks_loaded
+ << " blocks of relation " << relation.getName();
throw;
}
++blocks_loaded;
num_blocks_loaded[partition_numa_node_id]++;
if ((blocks_loaded + num_previously_loaded_blocks) == num_slots) {
- // The buffer pool has filled up. But, some database blocks are not loaded.
+ // The buffer pool has filled up. But, some database blocks are not
+ // loaded.
printf(
" The database is larger than the buffer pool. Only %lu blocks "
"were loaded ",
@@ -116,14 +127,13 @@ std::size_t PreloaderThread::preloadNUMAAware(
LOG(INFO) << "For relation: " << relation.getName();
for (auto numa_block_loaded_info : num_blocks_loaded) {
LOG(INFO) << "NUMA node: " << numa_block_loaded_info.first
- << " Number of loaded blocks: " << numa_block_loaded_info.second;
+ << " Number of loaded blocks: "
+ << numa_block_loaded_info.second;
}
+ LOG(INFO) << "Relation " << relation.getName()
+ << " completely preloaded in buffer pool in a NUMA aware fashion";
return blocks_loaded;
-#else
- LOG(INFO) << "Relation: " << relation.getName()
- << " has partition scheme but the system doesn't support NUMA";
- return 0;
-#endif
}
+#endif
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32e7c1bd/storage/PreloaderThread.hpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.hpp b/storage/PreloaderThread.hpp
index f16fd50..16df34a 100644
--- a/storage/PreloaderThread.hpp
+++ b/storage/PreloaderThread.hpp
@@ -20,6 +20,7 @@
#include <cstddef>
+#include "storage/StorageConfig.h"
#include "threading/Thread.hpp"
#include "utility/Macros.hpp"
@@ -68,6 +69,7 @@ class PreloaderThread : public Thread {
void run() override;
private:
+#ifdef QUICKSTEP_HAVE_LIBNUMA
/**
* @brief Preload a relation which has a partition and a NUMA placement scheme.
*
@@ -82,11 +84,16 @@ class PreloaderThread : public Thread {
* allocated sufficient amount of memory so as not to exceed that
* socket's memory limit.
*
+ * TODO(harshad) - Allow multiple preloader threads, each pinned to a NUMA
+ * socket in the system. Each thread should preload only its share of
+ * storage blocks.
+ *
* @return The number of blocks loaded during this function call.
**/
std::size_t preloadNUMAAware(const CatalogRelation &relation,
const std::size_t num_previously_loaded_blocks,
const std::size_t num_slots);
+#endif
const CatalogDatabase &database_;
StorageManager *storage_manager_;
[39/50] [abbrv] incubator-quickstep git commit: Added hash join order
optimization for star schema queries. (#229)
Posted by hb...@apache.org.
Added hash join order optimization for star schema queries. (#229)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0fffe505
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0fffe505
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0fffe505
Branch: refs/heads/query-manager-used-in-foreman
Commit: 0fffe5057da6b6abc1d9cdf0f7dab0accc8a0fe1
Parents: df4a05d
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 19 18:19:28 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:52 2016 -0700
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 2 +
query_optimizer/PhysicalGenerator.cpp | 14 +
query_optimizer/cost_model/CMakeLists.txt | 32 +-
.../cost_model/StarSchemaSimpleCostModel.cpp | 258 ++++++++++++++++
.../cost_model/StarSchemaSimpleCostModel.hpp | 115 +++++++
query_optimizer/rules/CMakeLists.txt | 20 +-
.../StarSchemaHashJoinOrderOptimization.cpp | 309 +++++++++++++++++++
.../StarSchemaHashJoinOrderOptimization.hpp | 136 ++++++++
query_optimizer/tests/CMakeLists.txt | 1 +
query_optimizer/tests/OptimizerTextTest.cpp | 14 +
10 files changed, 899 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index feaecb3..aa2873e 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -182,10 +182,12 @@ target_link_libraries(quickstep_queryoptimizer_OptimizerTree
quickstep_utility_Macros
quickstep_utility_TreeStringSerializable)
target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
+ gflags_nothreads-static
quickstep_queryoptimizer_LogicalToPhysicalMapper
quickstep_queryoptimizer_logical_Logical
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_rules_PruneColumns
+ quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
quickstep_queryoptimizer_strategy_Aggregate
quickstep_queryoptimizer_strategy_Join
quickstep_queryoptimizer_strategy_OneToOne
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/PhysicalGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/PhysicalGenerator.cpp b/query_optimizer/PhysicalGenerator.cpp
index ee52966..662236f 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -25,17 +27,26 @@
#include "query_optimizer/logical/Logical.hpp"
#include "query_optimizer/physical/Physical.hpp"
#include "query_optimizer/rules/PruneColumns.hpp"
+#include "query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp"
#include "query_optimizer/strategy/Aggregate.hpp"
#include "query_optimizer/strategy/Join.hpp"
#include "query_optimizer/strategy/OneToOne.hpp"
#include "query_optimizer/strategy/Selection.hpp"
#include "query_optimizer/strategy/Strategy.hpp"
+#include "gflags/gflags.h"
+
#include "glog/logging.h"
namespace quickstep {
namespace optimizer {
+DEFINE_bool(reorder_hash_joins, true,
+ "If true, apply hash join order optimization to each group of hash "
+ "joins. The optimization applies a greedy algorithm to favor smaller "
+ "cardinality and selective tables to be joined first, which is suitable "
+ "for queries on star-schema tables");
+
namespace L = ::quickstep::optimizer::logical;
namespace P = ::quickstep::optimizer::physical;
namespace S = ::quickstep::optimizer::strategy;
@@ -77,6 +88,9 @@ P::PhysicalPtr PhysicalGenerator::generateInitialPlan(
P::PhysicalPtr PhysicalGenerator::optimizePlan() {
std::vector<std::unique_ptr<Rule<P::Physical>>> rules;
+ if (FLAGS_reorder_hash_joins) {
+ rules.emplace_back(new StarSchemaHashJoinOrderOptimization());
+ }
rules.emplace_back(new PruneColumns());
for (std::unique_ptr<Rule<P::Physical>> &rule : rules) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/cost_model/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/CMakeLists.txt b/query_optimizer/cost_model/CMakeLists.txt
index 6697d52..6bf5240 100644
--- a/query_optimizer/cost_model/CMakeLists.txt
+++ b/query_optimizer/cost_model/CMakeLists.txt
@@ -1,5 +1,7 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -16,6 +18,9 @@
# Declare micro-libs:
add_library(quickstep_queryoptimizer_costmodel_CostModel ../../empty_src.cpp CostModel.hpp)
add_library(quickstep_queryoptimizer_costmodel_SimpleCostModel SimpleCostModel.cpp SimpleCostModel.hpp)
+add_library(quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ StarSchemaSimpleCostModel.cpp
+ StarSchemaSimpleCostModel.hpp)
# Link dependencies:
target_link_libraries(quickstep_queryoptimizer_costmodel_CostModel
@@ -36,9 +41,34 @@ target_link_libraries(quickstep_queryoptimizer_costmodel_SimpleCostModel
quickstep_queryoptimizer_physical_TableReference
quickstep_queryoptimizer_physical_TopLevelPlan
quickstep_utility_Macros)
+target_link_libraries(quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ glog
+ quickstep_catalog_CatalogRelation
+ quickstep_queryoptimizer_costmodel_CostModel
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_expressions_ComparisonExpression
+ quickstep_queryoptimizer_expressions_ExprId
+ quickstep_queryoptimizer_expressions_ExpressionType
+ quickstep_queryoptimizer_expressions_LogicalAnd
+ quickstep_queryoptimizer_expressions_LogicalOr
+ quickstep_queryoptimizer_expressions_PatternMatcher
+ quickstep_queryoptimizer_expressions_Predicate
+ quickstep_queryoptimizer_physical_Aggregate
+ quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_NestedLoopsJoin
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_queryoptimizer_physical_Selection
+ quickstep_queryoptimizer_physical_SharedSubplanReference
+ quickstep_queryoptimizer_physical_Sort
+ quickstep_queryoptimizer_physical_TableGenerator
+ quickstep_queryoptimizer_physical_TableReference
+ quickstep_queryoptimizer_physical_TopLevelPlan
+ quickstep_utility_Macros)
# Module all-in-one library:
add_library(quickstep_queryoptimizer_costmodel ../../empty_src.cpp CostModelModule.hpp)
target_link_libraries(quickstep_queryoptimizer_costmodel
quickstep_queryoptimizer_costmodel_CostModel
- quickstep_queryoptimizer_costmodel_SimpleCostModel)
+ quickstep_queryoptimizer_costmodel_SimpleCostModel
+ quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
new file mode 100644
index 0000000..eb9fcc1
--- /dev/null
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -0,0 +1,258 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+
+#include <algorithm>
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "catalog/CatalogRelation.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/expressions/ComparisonExpression.hpp"
+#include "query_optimizer/expressions/ExprId.hpp"
+#include "query_optimizer/expressions/ExpressionType.hpp"
+#include "query_optimizer/expressions/LogicalAnd.hpp"
+#include "query_optimizer/expressions/LogicalOr.hpp"
+#include "query_optimizer/expressions/Predicate.hpp"
+#include "query_optimizer/expressions/PatternMatcher.hpp"
+#include "query_optimizer/physical/Aggregate.hpp"
+#include "query_optimizer/physical/NestedLoopsJoin.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/PhysicalType.hpp"
+#include "query_optimizer/physical/Selection.hpp"
+#include "query_optimizer/physical/SharedSubplanReference.hpp"
+#include "query_optimizer/physical/Sort.hpp"
+#include "query_optimizer/physical/TableGenerator.hpp"
+#include "query_optimizer/physical/TableReference.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+namespace cost {
+
+namespace E = ::quickstep::optimizer::expressions;
+namespace P = ::quickstep::optimizer::physical;
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinality(
+ const P::PhysicalPtr &physical_plan) {
+ switch (physical_plan->getPhysicalType()) {
+ case P::PhysicalType::kTopLevelPlan:
+ return estimateCardinalityForTopLevelPlan(
+ std::static_pointer_cast<const P::TopLevelPlan>(physical_plan));
+ case P::PhysicalType::kTableReference:
+ return estimateCardinalityForTableReference(
+ std::static_pointer_cast<const P::TableReference>(physical_plan));
+ case P::PhysicalType::kSelection:
+ return estimateCardinalityForSelection(
+ std::static_pointer_cast<const P::Selection>(physical_plan));
+ case P::PhysicalType::kTableGenerator:
+ return estimateCardinalityForTableGenerator(
+ std::static_pointer_cast<const P::TableGenerator>(physical_plan));
+ case P::PhysicalType::kHashJoin:
+ return estimateCardinalityForHashJoin(
+ std::static_pointer_cast<const P::HashJoin>(physical_plan));
+ case P::PhysicalType::kNestedLoopsJoin:
+ return estimateCardinalityForNestedLoopsJoin(
+ std::static_pointer_cast<const P::NestedLoopsJoin>(physical_plan));
+ case P::PhysicalType::kAggregate:
+ return estimateCardinalityForAggregate(
+ std::static_pointer_cast<const P::Aggregate>(physical_plan));
+ case P::PhysicalType::kSharedSubplanReference: {
+ const P::SharedSubplanReferencePtr shared_subplan_reference =
+ std::static_pointer_cast<const P::SharedSubplanReference>(physical_plan);
+ return estimateCardinality(
+ shared_subplans_[shared_subplan_reference->subplan_id()]);
+ }
+ case P::PhysicalType::kSort:
+ return estimateCardinality(
+ std::static_pointer_cast<const P::Sort>(physical_plan)->input());
+ default:
+ LOG(FATAL) << "Unsupported physical plan:" << physical_plan->toString();
+ }
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForTopLevelPlan(
+ const P::TopLevelPlanPtr &physical_plan) {
+ return estimateCardinality(physical_plan->plan());
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForTableReference(
+ const P::TableReferencePtr &physical_plan) {
+ std::size_t num_tuples = physical_plan->relation()->getStatistics().getNumTuples();
+ if (num_tuples == 0) {
+ num_tuples = physical_plan->relation()->estimateTupleCardinality();
+ }
+ return num_tuples;
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForSelection(
+ const P::SelectionPtr &physical_plan) {
+ double selectivity = estimateSelectivityForSelection(physical_plan);
+ return std::max(static_cast<std::size_t>(estimateCardinality(physical_plan->input()) * selectivity),
+ static_cast<std::size_t>(1));
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForTableGenerator(
+ const P::TableGeneratorPtr &physical_plan) {
+ return physical_plan->generator_function_handle()->getEstimatedCardinality();
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForHashJoin(
+ const P::HashJoinPtr &physical_plan) {
+ std::size_t left_cardinality = estimateCardinality(physical_plan->left());
+ std::size_t right_cardinality = estimateCardinality(physical_plan->right());
+ double left_selectivity = estimateSelectivity(physical_plan->left());
+ double right_selectivity = estimateSelectivity(physical_plan->right());
+ return std::max(static_cast<std::size_t>(left_cardinality * right_selectivity) + 1,
+ static_cast<std::size_t>(right_cardinality * left_selectivity) + 1);
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForNestedLoopsJoin(
+ const P::NestedLoopsJoinPtr &physical_plan) {
+ return std::max(estimateCardinality(physical_plan->left()),
+ estimateCardinality(physical_plan->right()));
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForAggregate(
+ const P::AggregatePtr &physical_plan) {
+ if (physical_plan->grouping_expressions().empty()) {
+ return 1;
+ }
+ return std::max(static_cast<std::size_t>(1),
+ estimateCardinality(physical_plan->input()) / 10);
+}
+
+double StarSchemaSimpleCostModel::estimateSelectivity(
+ const physical::PhysicalPtr &physical_plan) {
+ switch (physical_plan->getPhysicalType()) {
+ case P::PhysicalType::kSelection: {
+ return estimateSelectivityForSelection(
+ std::static_pointer_cast<const P::Selection>(physical_plan));
+ }
+ case P::PhysicalType::kHashJoin: {
+ const P::HashJoinPtr &hash_join =
+ std::static_pointer_cast<const P::HashJoin>(physical_plan);
+ return std::min(estimateSelectivity(hash_join->left()),
+ estimateSelectivity(hash_join->right()));
+ }
+ case P::PhysicalType::kNestedLoopsJoin: {
+ const P::NestedLoopsJoinPtr &nested_loop_join =
+ std::static_pointer_cast<const P::NestedLoopsJoin>(physical_plan);
+ return std::min(estimateSelectivity(nested_loop_join->left()),
+ estimateSelectivity(nested_loop_join->right()));
+ }
+ case P::PhysicalType::kSharedSubplanReference: {
+ const P::SharedSubplanReferencePtr shared_subplan_reference =
+ std::static_pointer_cast<const P::SharedSubplanReference>(physical_plan);
+ return estimateSelectivity(
+ shared_subplans_[shared_subplan_reference->subplan_id()]);
+ }
+ default:
+ return 1.0;
+ }
+}
+
+double StarSchemaSimpleCostModel::estimateSelectivityForSelection(
+ const physical::SelectionPtr &physical_plan) {
+ const E::PredicatePtr &filter_predicate = physical_plan->filter_predicate();
+
+ // If the subplan is a table reference, gather the number of distinct values
+ // statistics for each column (attribute).
+ std::unordered_map<E::ExprId, std::size_t> num_distinct_values_map;
+ if (physical_plan->input()->getPhysicalType() == P::PhysicalType::kTableReference) {
+ const P::TableReferencePtr &table_reference =
+ std::static_pointer_cast<const P::TableReference>(physical_plan->input());
+ const CatalogRelation &relation = *table_reference->relation();
+ const std::vector<E::AttributeReferencePtr> &attributes = table_reference->attribute_list();
+ for (std::size_t i = 0; i < attributes.size(); ++i) {
+ std::size_t num_distinct_values = relation.getStatistics().getNumDistinctValues(i);
+ if (num_distinct_values > 0) {
+ num_distinct_values_map[attributes[i]->id()] = num_distinct_values;
+ }
+ }
+ }
+
+ return estimateSelectivityForPredicate(num_distinct_values_map, filter_predicate);
+}
+
+double StarSchemaSimpleCostModel::estimateSelectivityForPredicate(
+ const std::unordered_map<expressions::ExprId, std::size_t> &num_distinct_values_map,
+ const expressions::PredicatePtr &filter_predicate) {
+ if (filter_predicate == nullptr) {
+ return 1.0;
+ }
+
+ switch (filter_predicate->getExpressionType()) {
+ case E::ExpressionType::kComparisonExpression: {
+ // Case 1 - Number of distinct values statistics available
+ // Case 1.1 - Equality comparison: 1.0 / num_distinct_values
+ // Case 1.2 - Otherwise: 5.0 / num_distinct_values
+ // Case 2 - Number of distinct values statistics not available
+ // Case 2.1 - Equality comparison: 0.1
+ // Case 2.2 - Otherwise: 0.5
+ const E::ComparisonExpressionPtr &comparison_expression =
+ std::static_pointer_cast<const E::ComparisonExpression>(filter_predicate);
+ E::AttributeReferencePtr attr;
+ if ((E::SomeAttributeReference::MatchesWithConditionalCast(comparison_expression->left(), &attr) &&
+ E::SomeScalarLiteral::Matches(comparison_expression->right())) ||
+ (E::SomeAttributeReference::MatchesWithConditionalCast(comparison_expression->right(), &attr) &&
+ E::SomeScalarLiteral::Matches(comparison_expression->left()))) {
+ const auto it = num_distinct_values_map.find(attr->id());
+ if (it != num_distinct_values_map.end() && it->second > 0) {
+ double unit_selectivity = 1.0 / it->second;
+ return comparison_expression->isEqualityComparisonPredicate()
+ ? unit_selectivity
+ : std::min(0.5, unit_selectivity * 5.0);
+ }
+ }
+
+ return comparison_expression->isEqualityComparisonPredicate() ? 0.1 : 0.5;
+ }
+ case E::ExpressionType::kLogicalAnd: {
+ const E::LogicalAndPtr &logical_and =
+ std::static_pointer_cast<const E::LogicalAnd>(filter_predicate);
+ double selectivity = 1.0;
+ for (const auto &predicate : logical_and->operands()) {
+ selectivity =
+ std::min(selectivity,
+ estimateSelectivityForPredicate(num_distinct_values_map, predicate));
+ }
+ return selectivity;
+ }
+ case E::ExpressionType::kLogicalOr: {
+ const E::LogicalOrPtr &logical_or =
+ std::static_pointer_cast<const E::LogicalOr>(filter_predicate);
+ double selectivity = 0;
+ for (const auto &predicate : logical_or->operands()) {
+ selectivity += estimateSelectivityForPredicate(num_distinct_values_map, predicate);
+ }
+ return std::min(selectivity, 1.0);
+ }
+ default:
+ break;
+ }
+ return 1.0;
+}
+
+} // namespace cost
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp b/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp
new file mode 100644
index 0000000..c63e55a
--- /dev/null
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp
@@ -0,0 +1,115 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUERY_OPTIMIZER_COST_MODEL_STAR_SCHEMA_SIMPLE_COST_MODEL_HPP_
+#define QUERY_OPTIMIZER_COST_MODEL_STAR_SCHEMA_SIMPLE_COST_MODEL_HPP_
+
+#include <cstddef>
+#include <unordered_map>
+#include <vector>
+
+#include "query_optimizer/cost_model/CostModel.hpp"
+#include "query_optimizer/expressions/ExprId.hpp"
+#include "query_optimizer/expressions/Predicate.hpp"
+#include "query_optimizer/physical/Aggregate.hpp"
+#include "query_optimizer/physical/NestedLoopsJoin.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/Selection.hpp"
+#include "query_optimizer/physical/TableGenerator.hpp"
+#include "query_optimizer/physical/TableReference.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+namespace optimizer {
+namespace cost {
+
+/** \addtogroup CostModel
+ * @{
+ */
+
+/**
+ * @brief A simple cost model for hash join planning.
+ */
+class StarSchemaSimpleCostModel : public CostModel {
+ public:
+ /**
+ * @brief Constructor.
+ */
+ explicit StarSchemaSimpleCostModel(const std::vector<physical::PhysicalPtr> &shared_subplans)
+ : shared_subplans_(shared_subplans) {}
+
+ /**
+ * @brief Estimate the cardinality of a physical plan.
+ *
+ * @param physical_plan The physical plan.
+ * @return The estimated cardinality.
+ */
+ std::size_t estimateCardinality(
+ const physical::PhysicalPtr &physical_plan) override;
+
+ /**
+ * @brief Estimate the "selectivity" of a physical plan under the assumption
+ * that it acts as a filtered dimension table in a hash join.
+ *
+ * @param phyiscal_plan The physical plan.
+ * @return The estimated selectivity.
+ */
+ double estimateSelectivity(const physical::PhysicalPtr &physical_plan);
+
+ private:
+ std::size_t estimateCardinalityForTopLevelPlan(
+ const physical::TopLevelPlanPtr &physical_plan);
+
+ std::size_t estimateCardinalityForTableReference(
+ const physical::TableReferencePtr &physical_plan);
+
+ std::size_t estimateCardinalityForSelection(
+ const physical::SelectionPtr &physical_plan);
+
+ std::size_t estimateCardinalityForTableGenerator(
+ const physical::TableGeneratorPtr &physical_plan);
+
+ std::size_t estimateCardinalityForHashJoin(
+ const physical::HashJoinPtr &physical_plan);
+
+ std::size_t estimateCardinalityForNestedLoopsJoin(
+ const physical::NestedLoopsJoinPtr &physical_plan);
+
+ std::size_t estimateCardinalityForAggregate(
+ const physical::AggregatePtr &physical_plan);
+
+ double estimateSelectivityForSelection(
+ const physical::SelectionPtr &physical_plan);
+
+ double estimateSelectivityForPredicate(
+ const std::unordered_map<expressions::ExprId, std::size_t> &num_distinct_values_map,
+ const expressions::PredicatePtr &filter_predicate);
+
+ const std::vector<physical::PhysicalPtr> &shared_subplans_;
+
+ DISALLOW_COPY_AND_ASSIGN(StarSchemaSimpleCostModel);
+};
+
+/** @} */
+
+} // namespace cost
+} // namespace optimizer
+} // namespace quickstep
+
+#endif /* QUERY_OPTIMIZER_COST_MODEL_STAR_SCHEMA_SIMPLE_COST_MODEL_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/rules/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/CMakeLists.txt b/query_optimizer/rules/CMakeLists.txt
index 7032af5..1990174 100644
--- a/query_optimizer/rules/CMakeLists.txt
+++ b/query_optimizer/rules/CMakeLists.txt
@@ -26,11 +26,14 @@ add_library(quickstep_queryoptimizer_rules_PushDownFilter PushDownFilter.cpp Pus
add_library(quickstep_queryoptimizer_rules_PushDownSemiAntiJoin PushDownSemiAntiJoin.cpp PushDownSemiAntiJoin.hpp)
add_library(quickstep_queryoptimizer_rules_Rule ../../empty_src.cpp Rule.hpp)
add_library(quickstep_queryoptimizer_rules_RuleHelper RuleHelper.cpp RuleHelper.hpp)
+add_library(quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
+ StarSchemaHashJoinOrderOptimization.cpp
+ StarSchemaHashJoinOrderOptimization.hpp)
add_library(quickstep_queryoptimizer_rules_TopDownRule ../../empty_src.cpp TopDownRule.hpp)
add_library(quickstep_queryoptimizer_rules_UpdateExpression UpdateExpression.cpp UpdateExpression.hpp)
add_library(quickstep_queryoptimizer_rules_UnnestSubqueries UnnestSubqueries.cpp UnnestSubqueries.hpp)
-
+
# Link dependencies:
target_link_libraries(quickstep_queryoptimizer_rules_BottomUpRule
glog
@@ -110,6 +113,20 @@ target_link_libraries(quickstep_queryoptimizer_rules_RuleHelper
quickstep_queryoptimizer_expressions_PatternMatcher
quickstep_queryoptimizer_expressions_Predicate
quickstep_queryoptimizer_rules_UpdateExpression)
+target_link_libraries(quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
+ quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_expressions_ExprId
+ quickstep_queryoptimizer_expressions_NamedExpression
+ quickstep_queryoptimizer_expressions_PatternMatcher
+ quickstep_queryoptimizer_expressions_Predicate
+ quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_PatternMatcher
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_queryoptimizer_physical_TopLevelPlan
+ quickstep_queryoptimizer_rules_Rule
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_rules_TopDownRule
quickstep_queryoptimizer_rules_Rule
quickstep_utility_Macros)
@@ -167,6 +184,7 @@ target_link_libraries(quickstep_queryoptimizer_rules
quickstep_queryoptimizer_rules_PushDownSemiAntiJoin
quickstep_queryoptimizer_rules_Rule
quickstep_queryoptimizer_rules_RuleHelper
+ quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
quickstep_queryoptimizer_rules_TopDownRule
quickstep_queryoptimizer_rules_UpdateExpression
quickstep_queryoptimizer_rules_UnnestSubqueries)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp
new file mode 100644
index 0000000..9770606
--- /dev/null
+++ b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp
@@ -0,0 +1,309 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp"
+
+#include <memory>
+#include <set>
+#include <unordered_map>
+#include <vector>
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/expressions/NamedExpression.hpp"
+#include "query_optimizer/expressions/PatternMatcher.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/PatternMatcher.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/PhysicalType.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+
+namespace E = ::quickstep::optimizer::expressions;
+namespace P = ::quickstep::optimizer::physical;
+
+P::PhysicalPtr StarSchemaHashJoinOrderOptimization::apply(const P::PhysicalPtr &input) {
+ DCHECK(input->getPhysicalType() == P::PhysicalType::kTopLevelPlan);
+ cost_model_.reset(
+ new cost::StarSchemaSimpleCostModel(
+ std::static_pointer_cast<const P::TopLevelPlan>(input)->shared_subplans()));
+
+ return applyInternal(input, nullptr);
+}
+
+P::PhysicalPtr StarSchemaHashJoinOrderOptimization::applyInternal(const P::PhysicalPtr &input,
+ JoinGroupInfo *parent_join_group) {
+ P::HashJoinPtr hash_join;
+ const bool is_hash_inner_join =
+ P::SomeHashJoin::MatchesWithConditionalCast(input, &hash_join)
+ && hash_join->join_type() == P::HashJoin::JoinType::kInnerJoin;
+
+ if (is_hash_inner_join) {
+ bool is_valid_cascading_hash_join = false;
+ if (hash_join->residual_predicate() == nullptr) {
+ is_valid_cascading_hash_join = true;
+ for (const E::NamedExpressionPtr expr : hash_join->project_expressions()) {
+ if (!E::SomeAttributeReference::Matches(expr)) {
+ is_valid_cascading_hash_join = false;
+ break;
+ }
+ }
+ }
+
+ std::unique_ptr<JoinGroupInfo> new_join_group;
+ JoinGroupInfo *join_group = nullptr;
+ if (parent_join_group == nullptr || !is_valid_cascading_hash_join) {
+ new_join_group.reset(new JoinGroupInfo());
+ join_group = new_join_group.get();
+ } else {
+ join_group = parent_join_group;
+ }
+
+ // Gather tables into the join group.
+ for (const P::PhysicalPtr &child : input->children()) {
+ applyInternal(child, join_group);
+ }
+
+ // Gather join attribute pairs.
+ for (std::size_t i = 0; i < hash_join->left_join_attributes().size(); ++i) {
+ const std::size_t left_attr_id = hash_join->left_join_attributes()[i]->id();
+ const std::size_t right_attr_id = hash_join->right_join_attributes()[i]->id();
+
+ join_group->join_attribute_pairs.emplace_back(left_attr_id, right_attr_id);
+ }
+
+ if (join_group != parent_join_group) {
+ // This node is the root node for a group of hash inner joins. Now plan the
+ // ordering of the joins.
+ P::PhysicalPtr output = generatePlan(*join_group,
+ hash_join->residual_predicate(),
+ hash_join->project_expressions());
+ if (parent_join_group == nullptr) {
+ return output;
+ } else {
+ parent_join_group->tables.emplace_back(output);
+ return nullptr;
+ }
+ } else {
+ return nullptr;
+ }
+ } else {
+ std::vector<P::PhysicalPtr> new_children;
+ bool has_changed_children = false;
+ for (const P::PhysicalPtr &child : input->children()) {
+ P::PhysicalPtr new_child = applyInternal(child, nullptr);
+ DCHECK(new_child != nullptr);
+ if (child != new_child && !has_changed_children) {
+ has_changed_children = true;
+ }
+ new_children.push_back(new_child);
+ }
+
+ P::PhysicalPtr output =
+ (has_changed_children ? input->copyWithNewChildren(new_children)
+ : input);
+
+ if (parent_join_group == nullptr) {
+ return output;
+ } else {
+ parent_join_group->tables.emplace_back(output);
+ return nullptr;
+ }
+ }
+}
+
+physical::PhysicalPtr StarSchemaHashJoinOrderOptimization::generatePlan(
+ const JoinGroupInfo &join_group,
+ const E::PredicatePtr &residual_predicate,
+ const std::vector<E::NamedExpressionPtr> &project_expressions) {
+ const std::size_t num_tables = join_group.tables.size();
+ DCHECK_GE(num_tables, 2u);
+
+ std::vector<TableInfo> table_info_storage;
+ const std::vector<P::PhysicalPtr> &tables = join_group.tables;
+ for (std::size_t i = 0; i < join_group.tables.size(); ++i) {
+ table_info_storage.emplace_back(
+ i,
+ tables[i],
+ cost_model_->estimateCardinality(tables[i]),
+ cost_model_->estimateSelectivity(tables[i]));
+ }
+
+ // Auxiliary mapping info.
+ std::unordered_map<E::ExprId, std::size_t> attribute_id_to_table_info_index_map;
+ std::unordered_map<E::ExprId, E::AttributeReferencePtr> attribute_id_to_reference_map;
+ for (std::size_t table_idx = 0; table_idx < num_tables; ++table_idx) {
+ for (const E::AttributeReferencePtr &attr :
+ table_info_storage[table_idx].table->getOutputAttributes()) {
+ DCHECK(attribute_id_to_table_info_index_map.find(attr->id())
+ == attribute_id_to_table_info_index_map.end());
+
+ attribute_id_to_table_info_index_map.emplace(attr->id(), table_idx);
+ attribute_id_to_reference_map.emplace(attr->id(), attr);
+ }
+ }
+
+ // Create a join graph where tables are vertices, and add an edge between vertices
+ // t1 and t2 for each join predicate t1.x = t2.y
+ std::vector<std::unordered_set<std::size_t>> join_graph(table_info_storage.size());
+ for (const auto &attr_id_pair : join_group.join_attribute_pairs) {
+ DCHECK(attribute_id_to_table_info_index_map.find(attr_id_pair.first)
+ != attribute_id_to_table_info_index_map.end());
+ DCHECK(attribute_id_to_table_info_index_map.find(attr_id_pair.second)
+ != attribute_id_to_table_info_index_map.end());
+
+ std::size_t first_table_idx =
+ attribute_id_to_table_info_index_map[attr_id_pair.first];
+ std::size_t second_table_idx =
+ attribute_id_to_table_info_index_map[attr_id_pair.second];
+ DCHECK_NE(first_table_idx, second_table_idx);
+
+ table_info_storage[first_table_idx].join_attribute_pairs.emplace(
+ attr_id_pair.first, attr_id_pair.second);
+ table_info_storage[second_table_idx].join_attribute_pairs.emplace(
+ attr_id_pair.second, attr_id_pair.first);
+
+ join_graph[first_table_idx].emplace(second_table_idx);
+ join_graph[second_table_idx].emplace(first_table_idx);
+ }
+
+ std::set<TableInfo*, TableInfoPtrLessComparator> table_info_ordered_by_priority;
+ for (std::size_t i = 0; i < table_info_storage.size(); ++i) {
+ table_info_ordered_by_priority.emplace(&table_info_storage[i]);
+ }
+
+ // Contruct hash join tree.
+ while (true) {
+ TableInfo *first_table_info = *table_info_ordered_by_priority.begin();
+ table_info_ordered_by_priority.erase(
+ table_info_ordered_by_priority.begin());
+ const std::size_t first_table_info_id = first_table_info->table_info_id;
+
+ TableInfo *second_table_info = nullptr;
+ std::set<TableInfo*, TableInfoPtrLessComparator>::iterator second_table_info_it;
+ for (auto candidate_table_info_it = table_info_ordered_by_priority.begin();
+ candidate_table_info_it != table_info_ordered_by_priority.end();
+ ++candidate_table_info_it) {
+ TableInfo *candidate_table_info = *candidate_table_info_it;
+ const std::size_t candidate_table_info_id = candidate_table_info->table_info_id;
+
+ if (join_graph[first_table_info_id].find(candidate_table_info_id)
+ == join_graph[first_table_info_id].end() &&
+ join_graph[candidate_table_info_id].find(first_table_info_id)
+ == join_graph[candidate_table_info_id].end()) {
+ continue;
+ } else if (second_table_info == nullptr) {
+ second_table_info = candidate_table_info;
+ second_table_info_it = candidate_table_info_it;
+ }
+
+ bool is_likely_many_to_many_join = false;
+ for (const auto join_attr_pair : first_table_info->join_attribute_pairs) {
+ if (candidate_table_info->joined_attribute_set.find(join_attr_pair.second)
+ != candidate_table_info->joined_attribute_set.end()) {
+ is_likely_many_to_many_join = true;
+ break;
+ }
+ }
+ for (const auto join_attr_pair : candidate_table_info->join_attribute_pairs) {
+ if (first_table_info->joined_attribute_set.find(join_attr_pair.second)
+ != first_table_info->joined_attribute_set.end()) {
+ is_likely_many_to_many_join = true;
+ break;
+ }
+ }
+ if (!is_likely_many_to_many_join) {
+ second_table_info = candidate_table_info;
+ second_table_info_it = candidate_table_info_it;
+ break;
+ }
+ }
+ DCHECK(second_table_info != nullptr);
+ table_info_ordered_by_priority.erase(second_table_info_it);
+
+ const P::PhysicalPtr &left_child = first_table_info->table;
+ const P::PhysicalPtr &right_child = second_table_info->table;
+ std::vector<E::NamedExpressionPtr> output_attributes;
+ for (const E::AttributeReferencePtr &left_attr : left_child->getOutputAttributes()) {
+ output_attributes.emplace_back(left_attr);
+ }
+ for (const E::AttributeReferencePtr &right_attr : right_child->getOutputAttributes()) {
+ output_attributes.emplace_back(right_attr);
+ }
+
+ std::vector<E::AttributeReferencePtr> left_join_attributes;
+ std::vector<E::AttributeReferencePtr> right_join_attributes;
+ std::unordered_set<expressions::ExprId> new_joined_attribute_set;
+ for (const auto &join_attr_pair : first_table_info->join_attribute_pairs) {
+ if (second_table_info->join_attribute_pairs.find(join_attr_pair.second)
+ != second_table_info->join_attribute_pairs.end()) {
+ left_join_attributes.emplace_back(
+ attribute_id_to_reference_map[join_attr_pair.first]);
+ right_join_attributes.emplace_back(
+ attribute_id_to_reference_map[join_attr_pair.second]);
+
+ new_joined_attribute_set.emplace(join_attr_pair.first);
+ new_joined_attribute_set.emplace(join_attr_pair.second);
+ }
+ }
+ DCHECK_GE(left_join_attributes.size(), static_cast<std::size_t>(1));
+
+ if (table_info_ordered_by_priority.size() > 0) {
+ P::PhysicalPtr output =
+ P::HashJoin::Create(left_child,
+ right_child,
+ left_join_attributes,
+ right_join_attributes,
+ nullptr,
+ output_attributes,
+ P::HashJoin::JoinType::kInnerJoin);
+
+ second_table_info->table = output;
+
+ // TODO(jianqiao): Cache the estimated cardinality for each plan in cost
+ // model to avoid duplicated estimation.
+ second_table_info->estimated_cardinality = cost_model_->estimateCardinality(output);
+
+ second_table_info->join_attribute_pairs.insert(first_table_info->join_attribute_pairs.begin(),
+ first_table_info->join_attribute_pairs.end());
+ second_table_info->joined_attribute_set.insert(first_table_info->joined_attribute_set.begin(),
+ first_table_info->joined_attribute_set.end());
+ second_table_info->joined_attribute_set.insert(new_joined_attribute_set.begin(),
+ new_joined_attribute_set.end());
+ table_info_ordered_by_priority.emplace(second_table_info);
+
+ join_graph[second_table_info->table_info_id].insert(join_graph[first_table_info_id].begin(),
+ join_graph[first_table_info_id].end());
+
+ } else {
+ return P::HashJoin::Create(left_child,
+ right_child,
+ left_join_attributes,
+ right_join_attributes,
+ residual_predicate,
+ project_expressions,
+ P::HashJoin::JoinType::kInnerJoin);
+ }
+ }
+}
+
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp
new file mode 100644
index 0000000..deddffd
--- /dev/null
+++ b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp
@@ -0,0 +1,136 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_OPTIMIZER_RULES_STAR_SCHEMA_HASH_JOIN_ORDER_OPTIMIZATION_HPP_
+#define QUICKSTEP_QUERY_OPTIMIZER_RULES_STAR_SCHEMA_HASH_JOIN_ORDER_OPTIMIZATION_HPP_
+
+#include <algorithm>
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/expressions/ExprId.hpp"
+#include "query_optimizer/expressions/NamedExpression.hpp"
+#include "query_optimizer/expressions/Predicate.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/rules/Rule.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+namespace optimizer {
+
+/** \addtogroup OptimizerRules
+ * @{
+ */
+
+/**
+ * @brief TODO
+ */
+class StarSchemaHashJoinOrderOptimization : public Rule<physical::Physical> {
+ public:
+ StarSchemaHashJoinOrderOptimization() {}
+
+ ~StarSchemaHashJoinOrderOptimization() override {}
+
+ std::string getName() const override {
+ return "StarSchemaHashJoinOrderOptimization";
+ }
+
+ physical::PhysicalPtr apply(const physical::PhysicalPtr &input) override;
+
+ private:
+ /**
+ * @brief A group of tables to form a hash join tree.
+ */
+ struct JoinGroupInfo {
+ std::vector<physical::PhysicalPtr> tables;
+ std::vector<std::pair<expressions::ExprId, expressions::ExprId>> join_attribute_pairs;
+ };
+
+ /**
+ * @brief Auxiliary information of a table for the optimizer.
+ */
+ struct TableInfo {
+ TableInfo(const std::size_t in_table_info_id,
+ const physical::PhysicalPtr &in_table,
+ const std::size_t in_estimated_cardinality,
+ const double in_estimated_selectivity)
+ : table_info_id(in_table_info_id),
+ table(in_table),
+ estimated_cardinality(in_estimated_cardinality),
+ estimated_selectivity(in_estimated_selectivity) {
+ }
+
+ const std::size_t table_info_id;
+ physical::PhysicalPtr table;
+ std::size_t estimated_cardinality;
+ double estimated_selectivity;
+ std::unordered_multimap<expressions::ExprId, expressions::ExprId> join_attribute_pairs;
+ std::unordered_set<expressions::ExprId> joined_attribute_set;
+ };
+
+ /**
+ * @brief Comparator that compares the join priorities between two tables.
+ */
+ struct TableInfoPtrLessComparator {
+ inline bool operator() (const TableInfo *lhs, const TableInfo *rhs) {
+ bool swapped = false;
+ if (lhs->estimated_cardinality > rhs->estimated_cardinality) {
+ std::swap(lhs, rhs);
+ swapped = true;
+ }
+
+ if (lhs->estimated_selectivity < rhs->estimated_selectivity) {
+ return !swapped;
+ } else if (lhs->estimated_cardinality < 1000u &&
+ rhs->estimated_cardinality > 10000u &&
+ lhs->estimated_selectivity < rhs->estimated_selectivity * 1.5) {
+ return !swapped;
+ } else if (lhs->estimated_selectivity > rhs->estimated_selectivity) {
+ return swapped;
+ } else if (lhs->estimated_cardinality != rhs->estimated_cardinality) {
+ return !swapped;
+ } else {
+ return swapped ^ (lhs->table < rhs->table);
+ }
+ }
+ };
+
+ physical::PhysicalPtr applyInternal(const physical::PhysicalPtr &input,
+ JoinGroupInfo *paret_join_group);
+
+ physical::PhysicalPtr generatePlan(
+ const JoinGroupInfo &join_group_info,
+ const expressions::PredicatePtr &residual_predicate,
+ const std::vector<expressions::NamedExpressionPtr> &project_expressions);
+
+ std::unique_ptr<cost::StarSchemaSimpleCostModel> cost_model_;
+
+ DISALLOW_COPY_AND_ASSIGN(StarSchemaHashJoinOrderOptimization);
+};
+
+/** @} */
+
+} // namespace optimizer
+} // namespace quickstep
+
+#endif /* QUICKSTEP_QUERY_OPTIMIZER_RULES_STAR_SCHEMA_HASH_JOIN_ORDER_OPTIMIZATION_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/CMakeLists.txt b/query_optimizer/tests/CMakeLists.txt
index 5647bfd..07af404 100644
--- a/query_optimizer/tests/CMakeLists.txt
+++ b/query_optimizer/tests/CMakeLists.txt
@@ -132,6 +132,7 @@ target_link_libraries(quickstep_queryoptimizer_tests_ExecutionGeneratorTest
tmb
${LIBS})
target_link_libraries(quickstep_queryoptimizer_tests_OptimizerTextTest
+ gflags_nothreads-static
glog
gtest
gtest_main
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/tests/OptimizerTextTest.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/OptimizerTextTest.cpp b/query_optimizer/tests/OptimizerTextTest.cpp
index c35cfa3..f3c243b 100644
--- a/query_optimizer/tests/OptimizerTextTest.cpp
+++ b/query_optimizer/tests/OptimizerTextTest.cpp
@@ -22,8 +22,18 @@
#include "query_optimizer/tests/OptimizerTextTestRunner.hpp"
#include "utility/textbased_test/TextBasedTestDriver.hpp"
+#include "gflags/gflags.h"
+
#include "glog/logging.h"
+namespace quickstep {
+namespace optimizer {
+
+DECLARE_bool(reorder_hash_joins);
+
+}
+}
+
using quickstep::TextBasedTest;
QUICKSTEP_GENERATE_TEXT_TEST(OPTIMIZER_TEST);
@@ -45,6 +55,10 @@ int main(int argc, char** argv) {
test_driver->registerOptions(
quickstep::optimizer::OptimizerTextTestRunner::kTestOptions);
+ // Turn off join order optimization for optimizer test since it is up to change
+ // and affects a large number of test cases.
+ quickstep::optimizer::FLAGS_reorder_hash_joins = false;
+
::testing::InitGoogleTest(&argc, argv);
int success = RUN_ALL_TESTS();
if (success != 0) {
[41/50] [abbrv] incubator-quickstep git commit: Groupby hashtable
pool (#236)
Posted by hb...@apache.org.
Groupby hashtable pool (#236)
- Created a HashTablePool class for group by clause.
- Each thread can checkout it's own hash table while doing group by
aggregation.
- AggregationOperationState uses one hash table pool per group by
clause.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/2ddb67bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/2ddb67bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/2ddb67bf
Branch: refs/heads/query-manager-used-in-foreman
Commit: 2ddb67bf438878b572e997caec2397e4c7ac9b8f
Parents: 5bda90e
Author: Harshad Deshmukh <d....@gmail.com>
Authored: Tue May 24 19:04:39 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:53 2016 -0700
----------------------------------------------------------------------
.../aggregation/AggregationConcreteHandle.hpp | 105 ++++++++++++
expressions/aggregation/AggregationHandle.hpp | 15 +-
.../aggregation/AggregationHandleAvg.cpp | 9 +
.../aggregation/AggregationHandleAvg.hpp | 4 +
.../aggregation/AggregationHandleCount.cpp | 11 ++
.../aggregation/AggregationHandleCount.hpp | 4 +
.../aggregation/AggregationHandleDistinct.hpp | 7 +
.../aggregation/AggregationHandleMax.cpp | 9 +
.../aggregation/AggregationHandleMax.hpp | 4 +
.../aggregation/AggregationHandleMin.cpp | 9 +
.../aggregation/AggregationHandleMin.hpp | 4 +
.../aggregation/AggregationHandleSum.cpp | 9 +
.../aggregation/AggregationHandleSum.hpp | 4 +
expressions/aggregation/CMakeLists.txt | 2 +
.../tests/AggregationHandleAvg_unittest.cpp | 109 ++++++++++++
.../tests/AggregationHandleCount_unittest.cpp | 126 +++++++++++++-
.../tests/AggregationHandleMax_unittest.cpp | 122 ++++++++++++++
.../tests/AggregationHandleMin_unittest.cpp | 121 ++++++++++++++
.../tests/AggregationHandleSum_unittest.cpp | 124 ++++++++++++++
query_execution/QueryContext.hpp | 2 +-
storage/AggregationOperationState.cpp | 84 ++++++++--
storage/AggregationOperationState.hpp | 4 +
storage/CMakeLists.txt | 10 ++
storage/HashTablePool.hpp | 166 +++++++++++++++++++
storage/StorageManager.cpp | 3 +-
25 files changed, 1045 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationConcreteHandle.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationConcreteHandle.hpp b/expressions/aggregation/AggregationConcreteHandle.hpp
index 52249f7..0267e17 100644
--- a/expressions/aggregation/AggregationConcreteHandle.hpp
+++ b/expressions/aggregation/AggregationConcreteHandle.hpp
@@ -44,6 +44,90 @@ class ValueAccessor;
* @{
*/
+/**
+ * @brief An upserter class for modifying the destination hash table while
+ * merging two group by hash tables.
+ **/
+template <typename HandleT, typename StateT>
+class HashTableStateUpserter {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param handle The aggregation handle being used.
+ * @param source_state The aggregation state in the source aggregation hash
+ * table. The corresponding state (for the same key) in the destination
+ * hash table will be upserted.
+ **/
+ HashTableStateUpserter(const HandleT &handle, const StateT &source_state)
+ : handle_(handle), source_state_(source_state) {}
+
+ /**
+ * @brief The operator for the functor required for the upsert.
+ *
+ * @param destination_state The aggregation state in the aggregation hash
+ * table that is being upserted.
+ **/
+ void operator()(StateT *destination_state) {
+ handle_.mergeStates(source_state_, destination_state);
+ }
+
+ private:
+ const HandleT &handle_;
+ const StateT &source_state_;
+
+ DISALLOW_COPY_AND_ASSIGN(HashTableStateUpserter);
+};
+
+/**
+ * @brief A class to support the functor for merging group by hash tables.
+ **/
+template <typename HandleT, typename StateT, typename HashTableT>
+class HashTableMerger {
+ public:
+ /**
+ * @brief Constructor
+ *
+ * @param handle The Aggregation handle being used.
+ * @param destination_hash_table The destination hash table to which other
+ * hash tables will be merged.
+ **/
+ HashTableMerger(const HandleT &handle,
+ AggregationStateHashTableBase *destination_hash_table)
+ : handle_(handle),
+ destination_hash_table_(
+ static_cast<HashTableT *>(destination_hash_table)) {}
+
+ /**
+ * @brief The operator for the functor.
+ *
+ * @param group_by_key The group by key being merged.
+ * @param source_state The aggregation state for the given key in the source
+ * aggregation hash table.
+ **/
+ inline void operator()(const std::vector<TypedValue> &group_by_key,
+ const StateT &source_state) {
+ const StateT *original_state =
+ destination_hash_table_->getSingleCompositeKey(group_by_key);
+ if (original_state != nullptr) {
+ HashTableStateUpserter<HandleT, StateT> upserter(
+ handle_, source_state);
+ // The CHECK is required as upsertCompositeKey can return false if the
+ // hash table runs out of space during the upsert process. The ideal
+ // solution will be to retry again if the upsert fails.
+ CHECK(destination_hash_table_->upsertCompositeKey(
+ group_by_key, *original_state, &upserter));
+ } else {
+ destination_hash_table_->putCompositeKey(group_by_key, source_state);
+ }
+ }
+
+ private:
+ const HandleT &handle_;
+ HashTableT *destination_hash_table_;
+
+ DISALLOW_COPY_AND_ASSIGN(HashTableMerger);
+};
/**
* @brief The helper intermediate subclass of AggregationHandle that provides
@@ -140,6 +224,11 @@ class AggregationConcreteHandle : public AggregationHandle {
return static_cast<const HandleT*>(this)->finalizeHashTableEntry(*group_state);
}
+ template <typename HandleT, typename StateT, typename HashTableT>
+ void mergeGroupByHashTablesHelper(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const;
+
private:
DISALLOW_COPY_AND_ASSIGN(AggregationConcreteHandle);
};
@@ -373,6 +462,22 @@ ColumnVector* AggregationConcreteHandle::finalizeHashTableHelper(
}
}
+template <typename HandleT,
+ typename StateT,
+ typename HashTableT>
+void AggregationConcreteHandle::mergeGroupByHashTablesHelper(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ const HandleT &handle = static_cast<const HandleT &>(*this);
+ const HashTableT &source_hash_table_concrete =
+ static_cast<const HashTableT &>(source_hash_table);
+
+ HashTableMerger<HandleT, StateT, HashTableT> merger(handle,
+ destination_hash_table);
+
+ source_hash_table_concrete.forEachCompositeKey(&merger);
+}
+
} // namespace quickstep
#endif // QUICKSTEP_EXPRESSIONS_AGGREGATION_AGGREGATION_CONCRETE_HANDLE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandle.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandle.hpp b/expressions/aggregation/AggregationHandle.hpp
index 625f334..cdebb03 100644
--- a/expressions/aggregation/AggregationHandle.hpp
+++ b/expressions/aggregation/AggregationHandle.hpp
@@ -276,7 +276,7 @@ class AggregationHandle {
* each GROUP BY group. Later, a second-round aggregation on the distinctify
* hash table will be performed to actually compute the aggregated result for
* each GROUP BY group.
- *
+ *
* In the case of single aggregation where there is no GROUP BY expressions,
* we simply treat it as a special GROUP BY case that the GROUP BY expression
* vector is empty.
@@ -349,6 +349,19 @@ class AggregationHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const = 0;
+ /**
+ * @brief Merge two GROUP BY hash tables in one.
+ *
+ * @note Both the hash tables should have the same structure.
+ *
+ * @param source_hash_table The hash table which will get merged.
+ * @param destination_hash_table The hash table to which we will merge the
+ * other hash table.
+ **/
+ virtual void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const = 0;
+
protected:
AggregationHandle() {
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleAvg.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleAvg.cpp b/expressions/aggregation/AggregationHandleAvg.cpp
index cb0d63d..42a2fb9 100644
--- a/expressions/aggregation/AggregationHandleAvg.cpp
+++ b/expressions/aggregation/AggregationHandleAvg.cpp
@@ -203,4 +203,13 @@ void AggregationHandleAvg::aggregateOnDistinctifyHashTableForGroupBy(
aggregation_hash_table);
}
+void AggregationHandleAvg::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<AggregationHandleAvg,
+ AggregationStateAvg,
+ AggregationStateHashTable<AggregationStateAvg>>(
+ source_hash_table, destination_hash_table);
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleAvg.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleAvg.hpp b/expressions/aggregation/AggregationHandleAvg.hpp
index 6a94ee6..4ad4b21 100644
--- a/expressions/aggregation/AggregationHandleAvg.hpp
+++ b/expressions/aggregation/AggregationHandleAvg.hpp
@@ -158,6 +158,10 @@ class AggregationHandleAvg : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionAvg;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleCount.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleCount.cpp b/expressions/aggregation/AggregationHandleCount.cpp
index 5ece8ba..964b7c2 100644
--- a/expressions/aggregation/AggregationHandleCount.cpp
+++ b/expressions/aggregation/AggregationHandleCount.cpp
@@ -206,6 +206,17 @@ void AggregationHandleCount<count_star, nullable_type>
aggregation_hash_table);
}
+template <bool count_star, bool nullable_type>
+void AggregationHandleCount<count_star, nullable_type>::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<
+ AggregationHandleCount,
+ AggregationStateCount,
+ AggregationStateHashTable<AggregationStateCount>>(source_hash_table,
+ destination_hash_table);
+}
+
// Explicitly instantiate and compile in the different versions of
// AggregationHandleCount we need. Note that we do not compile a version with
// 'count_star == true' and 'nullable_type == true', as that combination is
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleCount.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleCount.hpp b/expressions/aggregation/AggregationHandleCount.hpp
index 6bb4e65..50138b9 100644
--- a/expressions/aggregation/AggregationHandleCount.hpp
+++ b/expressions/aggregation/AggregationHandleCount.hpp
@@ -166,6 +166,10 @@ class AggregationHandleCount : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionCount;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleDistinct.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleDistinct.hpp b/expressions/aggregation/AggregationHandleDistinct.hpp
index 918fdf8..6342c2b 100644
--- a/expressions/aggregation/AggregationHandleDistinct.hpp
+++ b/expressions/aggregation/AggregationHandleDistinct.hpp
@@ -109,6 +109,13 @@ class AggregationHandleDistinct : public AggregationConcreteHandle {
const AggregationStateHashTableBase &hash_table,
std::vector<std::vector<TypedValue>> *group_by_keys) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override {
+ LOG(FATAL)
+ << "AggregationHandleDistinct does not support mergeGroupByHashTables";
+ }
+
private:
DISALLOW_COPY_AND_ASSIGN(AggregationHandleDistinct);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleMax.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMax.cpp b/expressions/aggregation/AggregationHandleMax.cpp
index 4703657..a7a4a52 100644
--- a/expressions/aggregation/AggregationHandleMax.cpp
+++ b/expressions/aggregation/AggregationHandleMax.cpp
@@ -139,4 +139,13 @@ void AggregationHandleMax::aggregateOnDistinctifyHashTableForGroupBy(
aggregation_hash_table);
}
+void AggregationHandleMax::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<AggregationHandleMax,
+ AggregationStateMax,
+ AggregationStateHashTable<AggregationStateMax>>(
+ source_hash_table, destination_hash_table);
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleMax.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMax.hpp b/expressions/aggregation/AggregationHandleMax.hpp
index 8932ef8..5af5a12 100644
--- a/expressions/aggregation/AggregationHandleMax.hpp
+++ b/expressions/aggregation/AggregationHandleMax.hpp
@@ -151,6 +151,10 @@ class AggregationHandleMax : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionMax;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleMin.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMin.cpp b/expressions/aggregation/AggregationHandleMin.cpp
index de2709a..ca9b163 100644
--- a/expressions/aggregation/AggregationHandleMin.cpp
+++ b/expressions/aggregation/AggregationHandleMin.cpp
@@ -141,4 +141,13 @@ void AggregationHandleMin::aggregateOnDistinctifyHashTableForGroupBy(
aggregation_hash_table);
}
+void AggregationHandleMin::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<AggregationHandleMin,
+ AggregationStateMin,
+ AggregationStateHashTable<AggregationStateMin>>(
+ source_hash_table, destination_hash_table);
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleMin.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMin.hpp b/expressions/aggregation/AggregationHandleMin.hpp
index 4e4c05d..f68bb9d 100644
--- a/expressions/aggregation/AggregationHandleMin.hpp
+++ b/expressions/aggregation/AggregationHandleMin.hpp
@@ -149,6 +149,10 @@ class AggregationHandleMin : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionMin;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleSum.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleSum.cpp b/expressions/aggregation/AggregationHandleSum.cpp
index 14421d2..691ff39 100644
--- a/expressions/aggregation/AggregationHandleSum.cpp
+++ b/expressions/aggregation/AggregationHandleSum.cpp
@@ -190,4 +190,13 @@ void AggregationHandleSum::aggregateOnDistinctifyHashTableForGroupBy(
aggregation_hash_table);
}
+void AggregationHandleSum::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<AggregationHandleSum,
+ AggregationStateSum,
+ AggregationStateHashTable<AggregationStateSum>>(
+ source_hash_table, destination_hash_table);
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleSum.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleSum.hpp b/expressions/aggregation/AggregationHandleSum.hpp
index b765243..fdc0884 100644
--- a/expressions/aggregation/AggregationHandleSum.hpp
+++ b/expressions/aggregation/AggregationHandleSum.hpp
@@ -148,6 +148,10 @@ class AggregationHandleSum : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionSum;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/expressions/aggregation/CMakeLists.txt b/expressions/aggregation/CMakeLists.txt
index 26cec7f..416c4c6 100644
--- a/expressions/aggregation/CMakeLists.txt
+++ b/expressions/aggregation/CMakeLists.txt
@@ -291,6 +291,8 @@ target_link_libraries(AggregationHandle_tests
quickstep_expressions_aggregation_AggregationHandleMin
quickstep_expressions_aggregation_AggregationHandleSum
quickstep_expressions_aggregation_AggregationID
+ quickstep_storage_HashTableBase
+ quickstep_storage_StorageManager
quickstep_types_CharType
quickstep_types_DateOperatorOverloads
quickstep_types_DatetimeIntervalType
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp b/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
index d27b54e..fd82cba 100644
--- a/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
@@ -26,6 +26,7 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleAvg.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DateOperatorOverloads.hpp"
#include "types/DatetimeIntervalType.hpp"
@@ -238,6 +239,7 @@ class AggregationHandleAvgTest : public::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_avg_;
std::unique_ptr<AggregationState> aggregation_handle_avg_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
const int AggregationHandleAvgTest::kNumSamples;
@@ -417,4 +419,111 @@ TEST_F(AggregationHandleAvgTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kYearMonthInterval, kYearMonthInterval));
}
+TEST_F(AggregationHandleAvgTest, GroupByTableMergeTestAvg) {
+ const Type &long_non_null_type = LongType::Instance(false);
+ initializeHandle(long_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_avg_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_avg_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_avg_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateAvg> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateAvg> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateAvg> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateAvg> *>(
+ source_hash_table.get());
+
+ AggregationHandleAvg *aggregation_handle_avg_derived =
+ static_cast<AggregationHandleAvg *>(aggregation_handle_avg_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(static_cast<std::int64_t>(0));
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(static_cast<std::int64_t>(1));
+ exclusive_destination_key.emplace_back(static_cast<std::int64_t>(2));
+
+ const std::int64_t common_key_source_avg = 355;
+ TypedValue common_key_source_avg_val(common_key_source_avg);
+
+ const std::int64_t common_key_destination_avg = 295;
+ TypedValue common_key_destination_avg_val(common_key_destination_avg);
+
+ const std::int64_t exclusive_key_source_avg = 1;
+ TypedValue exclusive_key_source_avg_val(exclusive_key_source_avg);
+
+ const std::int64_t exclusive_key_destination_avg = 1;
+ TypedValue exclusive_key_destination_avg_val(exclusive_key_destination_avg);
+
+ std::unique_ptr<AggregationStateAvg> common_key_source_state(
+ static_cast<AggregationStateAvg *>(
+ aggregation_handle_avg_->createInitialState()));
+ std::unique_ptr<AggregationStateAvg> common_key_destination_state(
+ static_cast<AggregationStateAvg *>(
+ aggregation_handle_avg_->createInitialState()));
+ std::unique_ptr<AggregationStateAvg> exclusive_key_source_state(
+ static_cast<AggregationStateAvg *>(
+ aggregation_handle_avg_->createInitialState()));
+ std::unique_ptr<AggregationStateAvg> exclusive_key_destination_state(
+ static_cast<AggregationStateAvg *>(
+ aggregation_handle_avg_->createInitialState()));
+
+ // Create avg value states for keys.
+ aggregation_handle_avg_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_avg_val);
+
+ aggregation_handle_avg_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_avg_val);
+
+ aggregation_handle_avg_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_avg_val);
+
+ aggregation_handle_avg_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_avg_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_avg_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckAvgValue<double>(
+ (common_key_destination_avg_val.getLiteral<std::int64_t>() +
+ common_key_source_avg_val.getLiteral<std::int64_t>()) / static_cast<double>(2),
+ *aggregation_handle_avg_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckAvgValue<double>(exclusive_key_destination_avg_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_avg_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckAvgValue<double>(exclusive_key_source_avg_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_avg_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp b/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
index 7bebf6a..bf02523 100644
--- a/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
@@ -27,6 +27,7 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleCount.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DoubleType.hpp"
#include "types/FloatType.hpp"
@@ -355,6 +356,7 @@ class AggregationHandleCountTest : public::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_count_;
std::unique_ptr<AggregationState> aggregation_handle_count_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
typedef AggregationHandleCountTest AggregationHandleCountDeathTest;
@@ -477,5 +479,127 @@ TEST_F(AggregationHandleCountTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kDouble, kLong));
}
-} // namespace quickstep
+TEST_F(AggregationHandleCountTest, GroupByTableMergeTestCount) {
+ const Type &long_non_null_type = LongType::Instance(false);
+ initializeHandle(&long_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_count_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_count_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_count_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateCount> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateCount> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateCount> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateCount> *>(
+ source_hash_table.get());
+
+ // TODO(harshad) - Use TemplateUtil::CreateBoolInstantiatedInstance to
+ // generate all the combinations of the bool template arguments and test them.
+ AggregationHandleCount<true, false> *aggregation_handle_count_derived =
+ static_cast<AggregationHandleCount<true, false> *>(
+ aggregation_handle_count_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(static_cast<std::int64_t>(0));
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(static_cast<std::int64_t>(1));
+ exclusive_destination_key.emplace_back(static_cast<std::int64_t>(2));
+
+ const std::int64_t common_key_source_count = 1;
+ TypedValue common_key_source_count_val(common_key_source_count);
+
+ const std::int64_t common_key_destination_count = 1;
+ TypedValue common_key_destination_count_val(common_key_destination_count);
+
+ const std::int64_t exclusive_key_source_count = 1;
+ TypedValue exclusive_key_source_count_val(exclusive_key_source_count);
+
+ const std::int64_t exclusive_key_destination_count = 1;
+ TypedValue exclusive_key_destination_count_val(exclusive_key_destination_count);
+
+ std::unique_ptr<AggregationStateCount> common_key_source_state(
+ static_cast<AggregationStateCount *>(
+ aggregation_handle_count_->createInitialState()));
+ std::unique_ptr<AggregationStateCount> common_key_destination_state(
+ static_cast<AggregationStateCount *>(
+ aggregation_handle_count_->createInitialState()));
+ std::unique_ptr<AggregationStateCount> exclusive_key_source_state(
+ static_cast<AggregationStateCount *>(
+ aggregation_handle_count_->createInitialState()));
+ std::unique_ptr<AggregationStateCount> exclusive_key_destination_state(
+ static_cast<AggregationStateCount *>(
+ aggregation_handle_count_->createInitialState()));
+
+ // Create count value states for keys.
+ aggregation_handle_count_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_count_val);
+ std::int64_t actual_val = aggregation_handle_count_->finalize(*common_key_source_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(common_key_source_count_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_count_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_count_val);
+ actual_val = aggregation_handle_count_->finalize(*common_key_destination_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(common_key_destination_count_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_count_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_count_val);
+ actual_val =
+ aggregation_handle_count_->finalize(*exclusive_key_destination_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(exclusive_key_destination_count_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_count_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_count_val);
+ actual_val = aggregation_handle_count_->finalize(*exclusive_key_source_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(exclusive_key_source_count_val.getLiteral<std::int64_t>(), actual_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_count_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckCountValue(
+ common_key_destination_count_val.getLiteral<std::int64_t>() +
+ common_key_source_count_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_count_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckCountValue(exclusive_key_destination_count_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_count_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckCountValue(exclusive_key_source_count_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_count_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp b/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
index 027f24b..fc25e91 100644
--- a/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
@@ -29,6 +29,8 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleMax.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/HashTableBase.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DatetimeIntervalType.hpp"
#include "types/DatetimeLit.hpp"
@@ -413,6 +415,7 @@ class AggregationHandleMaxTest : public ::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_max_;
std::unique_ptr<AggregationState> aggregation_handle_max_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
template <>
@@ -637,4 +640,123 @@ TEST_F(AggregationHandleMaxTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kDouble, kDouble));
}
+TEST_F(AggregationHandleMaxTest, GroupByTableMergeTest) {
+ const Type &int_non_null_type = IntType::Instance(false);
+ initializeHandle(int_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_max_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_max_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &int_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_max_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &int_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateMax> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateMax> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateMax> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateMax> *>(
+ source_hash_table.get());
+
+ AggregationHandleMax *aggregation_handle_max_derived =
+ static_cast<AggregationHandleMax *>(aggregation_handle_max_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(0);
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(1);
+ exclusive_destination_key.emplace_back(2);
+
+ const int common_key_source_max = 3000;
+ TypedValue common_key_source_max_val(common_key_source_max);
+
+ const int common_key_destination_max = 4000;
+ TypedValue common_key_destination_max_val(common_key_destination_max);
+
+ const int exclusive_key_source_max = 100;
+ TypedValue exclusive_key_source_max_val(exclusive_key_source_max);
+
+ const int exclusive_key_destination_max = 200;
+ TypedValue exclusive_key_destination_max_val(exclusive_key_destination_max);
+
+ std::unique_ptr<AggregationStateMax> common_key_source_state(
+ static_cast<AggregationStateMax *>(
+ aggregation_handle_max_->createInitialState()));
+ std::unique_ptr<AggregationStateMax> common_key_destination_state(
+ static_cast<AggregationStateMax *>(
+ aggregation_handle_max_->createInitialState()));
+ std::unique_ptr<AggregationStateMax> exclusive_key_source_state(
+ static_cast<AggregationStateMax *>(
+ aggregation_handle_max_->createInitialState()));
+ std::unique_ptr<AggregationStateMax> exclusive_key_destination_state(
+ static_cast<AggregationStateMax *>(
+ aggregation_handle_max_->createInitialState()));
+
+ // Create max value states for keys.
+ aggregation_handle_max_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_max_val);
+ int actual_val = aggregation_handle_max_->finalize(*common_key_source_state)
+ .getLiteral<int>();
+ EXPECT_EQ(common_key_source_max_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_max_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_max_val);
+ actual_val = aggregation_handle_max_->finalize(*common_key_destination_state)
+ .getLiteral<int>();
+ EXPECT_EQ(common_key_destination_max_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_max_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_max_val);
+ actual_val =
+ aggregation_handle_max_->finalize(*exclusive_key_destination_state)
+ .getLiteral<int>();
+ EXPECT_EQ(exclusive_key_destination_max_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_max_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_max_val);
+ actual_val = aggregation_handle_max_->finalize(*exclusive_key_source_state)
+ .getLiteral<int>();
+ EXPECT_EQ(exclusive_key_source_max_val.getLiteral<int>(), actual_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_max_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckMaxValue<int>(
+ common_key_destination_max_val.getLiteral<int>(),
+ *aggregation_handle_max_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckMaxValue<int>(exclusive_key_destination_max_val.getLiteral<int>(),
+ *aggregation_handle_max_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckMaxValue<int>(exclusive_key_source_max_val.getLiteral<int>(),
+ *aggregation_handle_max_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp b/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
index eb64472..a87ace9 100644
--- a/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
@@ -29,6 +29,7 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleMin.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DatetimeIntervalType.hpp"
#include "types/DatetimeLit.hpp"
@@ -411,6 +412,7 @@ class AggregationHandleMinTest : public ::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_min_;
std::unique_ptr<AggregationState> aggregation_handle_min_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
template <>
@@ -634,4 +636,123 @@ TEST_F(AggregationHandleMinTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kDouble, kDouble));
}
+TEST_F(AggregationHandleMinTest, GroupByTableMergeTest) {
+ const Type &int_non_null_type = IntType::Instance(false);
+ initializeHandle(int_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_min_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_min_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &int_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_min_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &int_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateMin> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateMin> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateMin> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateMin> *>(
+ source_hash_table.get());
+
+ AggregationHandleMin *aggregation_handle_min_derived =
+ static_cast<AggregationHandleMin *>(aggregation_handle_min_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(0);
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(1);
+ exclusive_destination_key.emplace_back(2);
+
+ const int common_key_source_min = 3000;
+ TypedValue common_key_source_min_val(common_key_source_min);
+
+ const int common_key_destination_min = 4000;
+ TypedValue common_key_destination_min_val(common_key_destination_min);
+
+ const int exclusive_key_source_min = 100;
+ TypedValue exclusive_key_source_min_val(exclusive_key_source_min);
+
+ const int exclusive_key_destination_min = 200;
+ TypedValue exclusive_key_destination_min_val(exclusive_key_destination_min);
+
+ std::unique_ptr<AggregationStateMin> common_key_source_state(
+ static_cast<AggregationStateMin *>(
+ aggregation_handle_min_->createInitialState()));
+ std::unique_ptr<AggregationStateMin> common_key_destination_state(
+ static_cast<AggregationStateMin *>(
+ aggregation_handle_min_->createInitialState()));
+ std::unique_ptr<AggregationStateMin> exclusive_key_source_state(
+ static_cast<AggregationStateMin *>(
+ aggregation_handle_min_->createInitialState()));
+ std::unique_ptr<AggregationStateMin> exclusive_key_destination_state(
+ static_cast<AggregationStateMin *>(
+ aggregation_handle_min_->createInitialState()));
+
+ // Create min value states for keys.
+ aggregation_handle_min_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_min_val);
+ int actual_val = aggregation_handle_min_->finalize(*common_key_source_state)
+ .getLiteral<int>();
+ EXPECT_EQ(common_key_source_min_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_min_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_min_val);
+ actual_val = aggregation_handle_min_->finalize(*common_key_destination_state)
+ .getLiteral<int>();
+ EXPECT_EQ(common_key_destination_min_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_min_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_min_val);
+ actual_val =
+ aggregation_handle_min_->finalize(*exclusive_key_destination_state)
+ .getLiteral<int>();
+ EXPECT_EQ(exclusive_key_destination_min_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_min_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_min_val);
+ actual_val = aggregation_handle_min_->finalize(*exclusive_key_source_state)
+ .getLiteral<int>();
+ EXPECT_EQ(exclusive_key_source_min_val.getLiteral<int>(), actual_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_min_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckMinValue<int>(
+ common_key_source_min_val.getLiteral<int>(),
+ *aggregation_handle_min_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckMinValue<int>(exclusive_key_destination_min_val.getLiteral<int>(),
+ *aggregation_handle_min_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckMinValue<int>(exclusive_key_source_min_val.getLiteral<int>(),
+ *aggregation_handle_min_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp b/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
index 7dbbeb3..abf8a89 100644
--- a/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
@@ -26,6 +26,7 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleSum.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DatetimeIntervalType.hpp"
#include "types/DoubleType.hpp"
@@ -237,6 +238,7 @@ class AggregationHandleSumTest : public::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_sum_;
std::unique_ptr<AggregationState> aggregation_handle_sum_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
const int AggregationHandleSumTest::kNumSamples;
@@ -425,4 +427,126 @@ TEST_F(AggregationHandleSumTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kYearMonthInterval, kYearMonthInterval));
}
+TEST_F(AggregationHandleSumTest, GroupByTableMergeTest) {
+ const Type &long_non_null_type = LongType::Instance(false);
+ initializeHandle(long_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_sum_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_sum_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_sum_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateSum> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateSum> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateSum> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateSum> *>(
+ source_hash_table.get());
+
+ AggregationHandleSum *aggregation_handle_sum_derived =
+ static_cast<AggregationHandleSum *>(aggregation_handle_sum_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(static_cast<std::int64_t>(0));
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(static_cast<std::int64_t>(1));
+ exclusive_destination_key.emplace_back(static_cast<std::int64_t>(2));
+
+ const std::int64_t common_key_source_sum = 3000;
+ TypedValue common_key_source_sum_val(common_key_source_sum);
+
+ const std::int64_t common_key_destination_sum = 4000;
+ TypedValue common_key_destination_sum_val(common_key_destination_sum);
+
+ const std::int64_t merged_common_key = common_key_source_sum + common_key_destination_sum;
+ TypedValue common_key_merged_val(merged_common_key);
+
+ const std::int64_t exclusive_key_source_sum = 100;
+ TypedValue exclusive_key_source_sum_val(exclusive_key_source_sum);
+
+ const std::int64_t exclusive_key_destination_sum = 200;
+ TypedValue exclusive_key_destination_sum_val(exclusive_key_destination_sum);
+
+ std::unique_ptr<AggregationStateSum> common_key_source_state(
+ static_cast<AggregationStateSum *>(
+ aggregation_handle_sum_->createInitialState()));
+ std::unique_ptr<AggregationStateSum> common_key_destination_state(
+ static_cast<AggregationStateSum *>(
+ aggregation_handle_sum_->createInitialState()));
+ std::unique_ptr<AggregationStateSum> exclusive_key_source_state(
+ static_cast<AggregationStateSum *>(
+ aggregation_handle_sum_->createInitialState()));
+ std::unique_ptr<AggregationStateSum> exclusive_key_destination_state(
+ static_cast<AggregationStateSum *>(
+ aggregation_handle_sum_->createInitialState()));
+
+ // Create sum value states for keys.
+ aggregation_handle_sum_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_sum_val);
+ std::int64_t actual_val = aggregation_handle_sum_->finalize(*common_key_source_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(common_key_source_sum_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_sum_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_sum_val);
+ actual_val = aggregation_handle_sum_->finalize(*common_key_destination_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(common_key_destination_sum_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_sum_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_sum_val);
+ actual_val =
+ aggregation_handle_sum_->finalize(*exclusive_key_destination_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(exclusive_key_destination_sum_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_sum_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_sum_val);
+ actual_val = aggregation_handle_sum_->finalize(*exclusive_key_source_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(exclusive_key_source_sum_val.getLiteral<std::int64_t>(), actual_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_sum_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckSumValue<std::int64_t>(
+ common_key_merged_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_sum_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckSumValue<std::int64_t>(exclusive_key_destination_sum_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_sum_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckSumValue<std::int64_t>(exclusive_key_source_sum_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_sum_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/query_execution/QueryContext.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp
index 9440fae..7d5628d 100644
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@ -216,7 +216,7 @@ class QueryContext {
*
* @param id The BloomFilter id.
*
- * @return The constant pointer to BloomFilter that is
+ * @return The constant pointer to BloomFilter that is
* already created in the constructor.
**/
inline const BloomFilter* getBloomFilter(const bloom_filter_id id) const {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/storage/AggregationOperationState.cpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.cpp b/storage/AggregationOperationState.cpp
index d209ceb..4878cf1 100644
--- a/storage/AggregationOperationState.cpp
+++ b/storage/AggregationOperationState.cpp
@@ -92,11 +92,12 @@ AggregationOperationState::AggregationOperationState(
arguments_.push_back({});
is_distinct_.emplace_back(false);
- group_by_hashtables_.emplace_back(handles_.back()->createGroupByHashTable(
- hash_table_impl_type,
- group_by_types,
- estimated_num_entries,
- storage_manager_));
+ group_by_hashtable_pools_.emplace_back(std::unique_ptr<HashTablePool>(
+ new HashTablePool(estimated_num_entries,
+ hash_table_impl_type,
+ group_by_types,
+ handles_.back().get(),
+ storage_manager)));
} else {
// Set up each individual aggregate in this operation.
std::vector<const AggregateFunction*>::const_iterator agg_func_it
@@ -124,12 +125,13 @@ AggregationOperationState::AggregationOperationState(
handles_.emplace_back((*agg_func_it)->createHandle(argument_types));
if (!group_by_list_.empty()) {
- // Aggregation with GROUP BY: create a HashTable for per-group states.
- group_by_hashtables_.emplace_back(handles_.back()->createGroupByHashTable(
- hash_table_impl_type,
- group_by_types,
- estimated_num_entries,
- storage_manager_));
+ // Aggregation with GROUP BY: create a HashTable pool for per-group states.
+ group_by_hashtable_pools_.emplace_back(std::unique_ptr<HashTablePool>(
+ new HashTablePool(estimated_num_entries,
+ hash_table_impl_type,
+ group_by_types,
+ handles_.back().get(),
+ storage_manager)));
} else {
// Aggregation without GROUP BY: create a single global state.
single_states_.emplace_back(handles_.back()->createInitialState());
@@ -408,17 +410,17 @@ void AggregationOperationState::aggregateBlockHashTable(const block_id input_blo
// Call StorageBlock::aggregateGroupBy() to aggregate this block's values
// directly into the (threadsafe) shared global HashTable for this
// aggregate.
- //
- // TODO(shoban): Implement optional code path for using local hash table per
- // block, which can be merged with global hash table for all blocks
- // aggregated on.
+ DCHECK(group_by_hashtable_pools_[agg_idx] != nullptr);
+ AggregationStateHashTableBase *agg_hash_table = group_by_hashtable_pools_[agg_idx]->getHashTable();
+ DCHECK(agg_hash_table != nullptr);
block->aggregateGroupBy(*handles_[agg_idx],
arguments_[agg_idx],
group_by_list_,
predicate_.get(),
- group_by_hashtables_[agg_idx].get(),
+ agg_hash_table,
&reuse_matches,
&reuse_group_by_vectors);
+ group_by_hashtable_pools_[agg_idx]->returnHashTable(agg_hash_table);
}
}
}
@@ -447,19 +449,65 @@ void AggregationOperationState::finalizeHashTable(InsertDestination *output_dest
// group (which is also the prefix of the finalized Tuple for that group).
std::vector<std::vector<TypedValue>> group_by_keys;
+ // TODO(harshad) - The merge phase may be slower when each hash table contains
+ // large number of entries. We should find ways in which we can perform a
+ // parallel merge.
+
+ // TODO(harshad) - Find heuristics for faster merge, even in a single thread.
+ // e.g. Keep merging entries from smaller hash tables to larger.
+ for (std::size_t agg_idx = 0; agg_idx < handles_.size(); ++agg_idx) {
+ auto *hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ if (hash_tables->size() > 1) {
+ for (int hash_table_index = 0;
+ hash_table_index < static_cast<int>(hash_tables->size() - 1);
+ ++hash_table_index) {
+ // Merge each hash table to the last hash table.
+ handles_[agg_idx]->mergeGroupByHashTables(
+ (*(*hash_tables)[hash_table_index]),
+ hash_tables->back().get());
+ }
+ }
+ }
+
// Collect per-aggregate finalized values.
std::vector<std::unique_ptr<ColumnVector>> final_values;
for (std::size_t agg_idx = 0;
agg_idx < handles_.size();
++agg_idx) {
if (is_distinct_[agg_idx]) {
+ DCHECK(group_by_hashtable_pools_[agg_idx] != nullptr);
+ auto *hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ DCHECK(hash_tables != nullptr);
+ if (hash_tables->empty()) {
+ // We may have a case where hash_tables is empty, e.g. no input blocks.
+ // However for aggregateOnDistinctifyHashTableForGroupBy to work
+ // correctly, we should create an empty group by hash table.
+ AggregationStateHashTableBase *new_hash_table = group_by_hashtable_pools_[agg_idx]->getHashTable();
+ group_by_hashtable_pools_[agg_idx]->returnHashTable(new_hash_table);
+ hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ }
+ DCHECK(hash_tables->back() != nullptr);
+ AggregationStateHashTableBase *agg_hash_table = hash_tables->back().get();
+ DCHECK(agg_hash_table != nullptr);
handles_[agg_idx]->aggregateOnDistinctifyHashTableForGroupBy(
*distinctify_hashtables_[agg_idx],
- group_by_hashtables_[agg_idx].get());
+ agg_hash_table);
}
+ auto *hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ DCHECK(hash_tables != nullptr);
+ if (hash_tables->empty()) {
+ // We may have a case where hash_tables is empty, e.g. no input blocks.
+ // However for aggregateOnDistinctifyHashTableForGroupBy to work
+ // correctly, we should create an empty group by hash table.
+ AggregationStateHashTableBase *new_hash_table = group_by_hashtable_pools_[agg_idx]->getHashTable();
+ group_by_hashtable_pools_[agg_idx]->returnHashTable(new_hash_table);
+ hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ }
+ AggregationStateHashTableBase *agg_hash_table = hash_tables->back().get();
+ DCHECK(agg_hash_table != nullptr);
ColumnVector* agg_result_col =
- handles_[agg_idx]->finalizeHashTable(*group_by_hashtables_[agg_idx],
+ handles_[agg_idx]->finalizeHashTable(*agg_hash_table,
&group_by_keys);
if (agg_result_col != nullptr) {
final_values.emplace_back(agg_result_col);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/storage/AggregationOperationState.hpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.hpp b/storage/AggregationOperationState.hpp
index c3a1278..0199749 100644
--- a/storage/AggregationOperationState.hpp
+++ b/storage/AggregationOperationState.hpp
@@ -31,6 +31,7 @@
#include "expressions/scalar/Scalar.hpp"
#include "storage/AggregationOperationState.pb.h"
#include "storage/HashTableBase.hpp"
+#include "storage/HashTablePool.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
@@ -209,6 +210,9 @@ class AggregationOperationState {
// hash table to prevent multiple lookups.
std::vector<std::unique_ptr<AggregationStateHashTableBase>> group_by_hashtables_;
+ // A vector of group by hash table pools, one for each group by clause.
+ std::vector<std::unique_ptr<HashTablePool>> group_by_hashtable_pools_;
+
StorageManager *storage_manager_;
DISALLOW_COPY_AND_ASSIGN(AggregationOperationState);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index a3093df..87a5e54 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -187,6 +187,7 @@ add_library(quickstep_storage_HashTable_proto ${storage_HashTable_proto_srcs})
add_library(quickstep_storage_HashTableBase ../empty_src.cpp HashTableBase.hpp)
add_library(quickstep_storage_HashTableFactory HashTableFactory.cpp HashTableFactory.hpp)
add_library(quickstep_storage_HashTableKeyManager ../empty_src.cpp HashTableKeyManager.hpp)
+add_library(quickstep_storage_HashTablePool ../empty_src.cpp HashTablePool.hpp)
add_library(quickstep_storage_IndexSubBlock ../empty_src.cpp IndexSubBlock.hpp)
add_library(quickstep_storage_IndexSubBlockDescriptionFactory ../empty_src.cpp IndexSubBlockDescriptionFactory.hpp)
add_library(quickstep_storage_InsertDestination InsertDestination.cpp InsertDestination.hpp)
@@ -252,6 +253,7 @@ target_link_libraries(quickstep_storage_AggregationOperationState
quickstep_storage_HashTable
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
+ quickstep_storage_HashTablePool
quickstep_storage_InsertDestination
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
@@ -662,6 +664,13 @@ target_link_libraries(quickstep_storage_HashTableKeyManager
quickstep_types_TypedValue
quickstep_types_operations_comparisons_ComparisonUtil
quickstep_utility_Macros)
+target_link_libraries(quickstep_storage_HashTablePool
+ glog
+ quickstep_expressions_aggregation_AggregationHandle
+ quickstep_storage_HashTableBase
+ quickstep_threading_SpinMutex
+ quickstep_utility_Macros
+ quickstep_utility_StringUtil)
target_link_libraries(quickstep_storage_IndexSubBlock
quickstep_catalog_CatalogTypedefs
quickstep_expressions_predicate_PredicateCost
@@ -1012,6 +1021,7 @@ target_link_libraries(quickstep_storage
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
quickstep_storage_HashTableKeyManager
+ quickstep_storage_HashTablePool
quickstep_storage_IndexSubBlock
quickstep_storage_IndexSubBlockDescriptionFactory
quickstep_storage_InsertDestination
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/storage/HashTablePool.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTablePool.hpp b/storage/HashTablePool.hpp
new file mode 100644
index 0000000..c16d0f1
--- /dev/null
+++ b/storage/HashTablePool.hpp
@@ -0,0 +1,166 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_HASH_TABLE_POOL_HPP_
+#define QUICKSTEP_STORAGE_HASH_TABLE_POOL_HPP_
+
+#include <chrono>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "expressions/aggregation/AggregationHandle.hpp"
+#include "storage/HashTableBase.hpp"
+#include "threading/SpinMutex.hpp"
+#include "utility/Macros.hpp"
+#include "utility/StringUtil.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class StorageManager;
+class Type;
+
+/** \addtogroup Storage
+ * @{
+ */
+
+/**
+ * @brief A pool of HashTables used for a single aggregation handle. This class
+ * has similar functionality as InsertDestination, but for checking out
+ * HashTables. A worker thread can check out a hash table for insertion,
+ * perform the insertions and return the hash table to the pool. While
+ * one thread is using a hash table, no other thread can access it.
+ **/
+class HashTablePool {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param estimated_num_entries The maximum number of entries in a hash table.
+ * @param hash_table_impl_type The type of hash table implementation.
+ * @param group_by_types A vector of pointer of types which form the group by
+ * key.
+ * @param agg_handle The aggregation handle.
+ * @param storage_manager A pointer to the storage manager.
+ *
+ * @note The estimate of number of entries is quite inaccurate at this time.
+ * If we go by the current estimate, each hash table demands much
+ * larger space than it actually needs, which causes the system to
+ * either trigger evictions or worse - run out of memory. To fix this
+ * issue, we divide the estimate by 100. The division will not affect
+ * correctness, however it may allocate some hash tables smaller space
+ * than their requirement, causing them to be resized during build
+ * phase, which has a performance penalty.
+ **/
+ HashTablePool(const std::size_t estimated_num_entries,
+ const HashTableImplType hash_table_impl_type,
+ const std::vector<const Type *> &group_by_types,
+ AggregationHandle *agg_handle,
+ StorageManager *storage_manager)
+ : estimated_num_entries_(reduceEstimatedCardinality(estimated_num_entries)),
+ hash_table_impl_type_(hash_table_impl_type),
+ group_by_types_(group_by_types),
+ agg_handle_(DCHECK_NOTNULL(agg_handle)),
+ storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
+
+ /**
+ * @brief Check out a hash table for insertion.
+ *
+ * @return A hash table pointer.
+ **/
+ AggregationStateHashTableBase* getHashTable() {
+ {
+ SpinMutexLock lock(mutex_);
+ if (!hash_tables_.empty()) {
+ std::unique_ptr<AggregationStateHashTableBase> ret_hash_table(
+ std::move(hash_tables_.back()));
+ hash_tables_.pop_back();
+ DCHECK(ret_hash_table != nullptr);
+ return ret_hash_table.release();
+ }
+ }
+ return createNewHashTable();
+ }
+
+ /**
+ * @brief Return a previously checked out hash table.
+ *
+ * @param hash_table A pointer to the checked out hash table.
+ **/
+ void returnHashTable(AggregationStateHashTableBase *hash_table) {
+ SpinMutexLock lock(mutex_);
+ hash_tables_.push_back(
+ std::unique_ptr<AggregationStateHashTableBase>(hash_table));
+ }
+
+ /**
+ * @brief Get all the hash tables from the pool.
+ *
+ * @warning The caller should ensure that this call is made when no hash table
+ * is being checked in or checked out from the pool. In other words
+ * the hash table pool is in read-only state.
+ *
+ * @param All the hash tables in the pool.
+ *
+ **/
+ const std::vector<std::unique_ptr<AggregationStateHashTableBase>>*
+ getAllHashTables() {
+ return &hash_tables_;
+ }
+
+ private:
+ AggregationStateHashTableBase* createNewHashTable() {
+ return agg_handle_->createGroupByHashTable(hash_table_impl_type_,
+ group_by_types_,
+ estimated_num_entries_,
+ storage_manager_);
+ }
+
+ inline std::size_t reduceEstimatedCardinality(
+ const std::size_t original_estimate) const {
+ if (original_estimate < kEstimateReductionFactor) {
+ return original_estimate;
+ } else {
+ DCHECK_GT(kEstimateReductionFactor, 0u);
+ return original_estimate / kEstimateReductionFactor;
+ }
+ }
+
+ static constexpr std::size_t kEstimateReductionFactor = 100;
+
+ std::vector<std::unique_ptr<AggregationStateHashTableBase>> hash_tables_;
+
+ const std::size_t estimated_num_entries_;
+ const HashTableImplType hash_table_impl_type_;
+
+ const std::vector<const Type *> group_by_types_;
+
+ AggregationHandle *agg_handle_;
+ StorageManager *storage_manager_;
+
+ SpinMutex mutex_;
+
+ DISALLOW_COPY_AND_ASSIGN(HashTablePool);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_STORAGE_HASH_TABLE_POOL_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index dfc95b8..5d91052 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -183,7 +183,8 @@ StorageManager::~StorageManager() {
it != blocks_.end();
++it) {
if (it->second.block->isDirty()) {
- LOG(WARNING) << "Block with ID " << BlockIdUtil::ToString(it->first)
+ LOG(WARNING) << (it->second.block->isBlob() ? "Blob " : "Block ")
+ << "with ID " << BlockIdUtil::ToString(it->first)
<< " is dirty during StorageManager shutdown";
}
delete it->second.block;
[05/50] [abbrv] incubator-quickstep git commit: Adds support for
multiple SharedSubplanReference. (#199)
Posted by hb...@apache.org.
Adds support for multiple SharedSubplanReference. (#199)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c5460f40
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c5460f40
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c5460f40
Branch: refs/heads/query-manager-used-in-foreman
Commit: c5460f40e82b9e2c81e14380f17609a6e7230eb4
Parents: 07ca1e7
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu Apr 28 16:09:32 2016 -0500
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Thu Apr 28 16:09:32 2016 -0500
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 13 ++
query_optimizer/ExecutionGenerator.hpp | 11 +-
query_optimizer/cost_model/CMakeLists.txt | 1 +
query_optimizer/cost_model/SimpleCostModel.cpp | 9 +
query_optimizer/cost_model/SimpleCostModel.hpp | 8 +-
.../logical/SharedSubplanReference.cpp | 7 +-
.../logical/SharedSubplanReference.hpp | 34 +++-
.../physical/SharedSubplanReference.cpp | 5 +
.../physical/SharedSubplanReference.hpp | 34 +++-
query_optimizer/resolver/Resolver.cpp | 25 ++-
query_optimizer/strategy/OneToOne.cpp | 1 +
.../tests/execution_generator/Select.test | 25 +++
.../tests/logical_generator/Select.test | 106 ++++++++++-
.../tests/physical_generator/Select.test | 165 ++++++++++++++++-
query_optimizer/tests/resolver/Select.test | 181 +++++++++++++++----
15 files changed, 542 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index abcdd6c..c34f084 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -154,6 +154,9 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) {
CHECK(P::SomeTopLevelPlan::MatchesWithConditionalCast(physical_plan, &top_level_physical_plan_))
<< "The physical plan must be rooted by a TopLevelPlan";
+ cost_model_.reset(
+ new cost::SimpleCostModel(top_level_physical_plan_->shared_subplans()));
+
const CatalogRelation *result_relation = nullptr;
try {
@@ -550,6 +553,16 @@ void ExecutionGenerator::convertSharedSubplanReference(const physical::SharedSub
top_level_physical_plan_->shared_subplan_at(physical_plan->subplan_id()));
if (found_it != physical_to_output_relation_map_.end()) {
physical_to_output_relation_map_.emplace(physical_plan, found_it->second);
+
+ // Propagate the (ExprId -> CatalogAttribute) mapping.
+ const std::vector<E::AttributeReferencePtr> &referenced_attributes =
+ physical_plan->referenced_attributes();
+ const std::vector<E::AttributeReferencePtr> &output_attributes =
+ physical_plan->output_attributes();
+ for (std::size_t i = 0; i < referenced_attributes.size(); ++i) {
+ attribute_substitution_map_[output_attributes[i]->id()] =
+ attribute_substitution_map_[referenced_attributes[i]->id()];
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/ExecutionGenerator.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.hpp b/query_optimizer/ExecutionGenerator.hpp
index df47b31..7c563d4 100644
--- a/query_optimizer/ExecutionGenerator.hpp
+++ b/query_optimizer/ExecutionGenerator.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -104,8 +106,6 @@ class ExecutionGenerator {
#ifdef QUICKSTEP_DISTRIBUTED
catalog_database_cache_proto_ = DCHECK_NOTNULL(query_handle->getCatalogDatabaseCacheProtoMutable());
#endif
-
- setupCostModel();
}
/**
@@ -163,13 +163,6 @@ class ExecutionGenerator {
void generatePlanInternal(const physical::PhysicalPtr &physical_plan);
/**
- * @brief Sets up the cost model.
- */
- void setupCostModel() {
- cost_model_.reset(new cost::SimpleCostModel());
- }
-
- /**
* @brief Finds the CatalogRelationInfo from <physical_to_output_relation_map_>
* by the physical node \p physical. Returns NULL if not found.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/cost_model/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/CMakeLists.txt b/query_optimizer/cost_model/CMakeLists.txt
index e561fc8..6697d52 100644
--- a/query_optimizer/cost_model/CMakeLists.txt
+++ b/query_optimizer/cost_model/CMakeLists.txt
@@ -31,6 +31,7 @@ target_link_libraries(quickstep_queryoptimizer_costmodel_SimpleCostModel
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_physical_PhysicalType
quickstep_queryoptimizer_physical_Selection
+ quickstep_queryoptimizer_physical_SharedSubplanReference
quickstep_queryoptimizer_physical_TableGenerator
quickstep_queryoptimizer_physical_TableReference
quickstep_queryoptimizer_physical_TopLevelPlan
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/cost_model/SimpleCostModel.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/SimpleCostModel.cpp b/query_optimizer/cost_model/SimpleCostModel.cpp
index 291fb9d..48f76fa 100644
--- a/query_optimizer/cost_model/SimpleCostModel.cpp
+++ b/query_optimizer/cost_model/SimpleCostModel.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -27,6 +29,7 @@
#include "query_optimizer/physical/Physical.hpp"
#include "query_optimizer/physical/PhysicalType.hpp"
#include "query_optimizer/physical/Selection.hpp"
+#include "query_optimizer/physical/SharedSubplanReference.hpp"
#include "query_optimizer/physical/TableGenerator.hpp"
#include "query_optimizer/physical/TableReference.hpp"
#include "query_optimizer/physical/TopLevelPlan.hpp"
@@ -63,6 +66,12 @@ std::size_t SimpleCostModel::estimateCardinality(
case P::PhysicalType::kAggregate:
return estimateCardinalityForAggregate(
std::static_pointer_cast<const P::Aggregate>(physical_plan));
+ case P::PhysicalType::kSharedSubplanReference: {
+ const P::SharedSubplanReferencePtr shared_subplan_reference =
+ std::static_pointer_cast<const P::SharedSubplanReference>(physical_plan);
+ return estimateCardinality(
+ shared_subplans_[shared_subplan_reference->subplan_id()]);
+ }
default:
LOG(FATAL) << "Unsupported physical plan:" << physical_plan->toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/cost_model/SimpleCostModel.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/SimpleCostModel.hpp b/query_optimizer/cost_model/SimpleCostModel.hpp
index e53887d..9862198 100644
--- a/query_optimizer/cost_model/SimpleCostModel.hpp
+++ b/query_optimizer/cost_model/SimpleCostModel.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,6 +21,7 @@
#define QUERY_OPTIMIZER_COST_MODEL_SIMPLE_COST_MODEL_HPP_
#include <cstddef>
+#include <vector>
#include "query_optimizer/cost_model/CostModel.hpp"
#include "query_optimizer/physical/Aggregate.hpp"
@@ -47,7 +50,8 @@ class SimpleCostModel : public CostModel {
/**
* @brief Constructor.
*/
- SimpleCostModel() {}
+ explicit SimpleCostModel(const std::vector<physical::PhysicalPtr> &shared_subplans)
+ : shared_subplans_(shared_subplans) {}
std::size_t estimateCardinality(
const physical::PhysicalPtr &physical_plan) override;
@@ -84,6 +88,8 @@ class SimpleCostModel : public CostModel {
std::size_t estimateCardinalityForAggregate(
const physical::AggregatePtr &physical_plan);
+ const std::vector<physical::PhysicalPtr> &shared_subplans_;
+
DISALLOW_COPY_AND_ASSIGN(SimpleCostModel);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/logical/SharedSubplanReference.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/logical/SharedSubplanReference.cpp b/query_optimizer/logical/SharedSubplanReference.cpp
index 8a5bec9..b5d3143 100644
--- a/query_optimizer/logical/SharedSubplanReference.cpp
+++ b/query_optimizer/logical/SharedSubplanReference.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -41,13 +43,16 @@ void SharedSubplanReference::getFieldStringItems(
inline_field_names->push_back("subplan_id");
inline_field_values->push_back(std::to_string(subplan_id_));
+ container_child_field_names->push_back("referenced_attributes");
+ container_child_fields->push_back(CastSharedPtrVector<OptimizerTreeBase>(referenced_attributes_));
+
container_child_field_names->push_back("output_attributes");
container_child_fields->push_back(CastSharedPtrVector<OptimizerTreeBase>(output_attributes_));
}
LogicalPtr SharedSubplanReference::copyWithNewChildren(const std::vector<LogicalPtr> &new_children) const {
DCHECK(new_children.empty());
- return Create(subplan_id_, output_attributes_);
+ return Create(subplan_id_, referenced_attributes_, output_attributes_);
}
} // namespace logical
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/logical/SharedSubplanReference.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/logical/SharedSubplanReference.hpp b/query_optimizer/logical/SharedSubplanReference.hpp
index 55035d1..8f0e37b 100644
--- a/query_optimizer/logical/SharedSubplanReference.hpp
+++ b/query_optimizer/logical/SharedSubplanReference.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,6 +30,8 @@
#include "query_optimizer/logical/LogicalType.hpp"
#include "utility/Macros.hpp"
+#include "glog/logging.h"
+
namespace quickstep {
namespace optimizer {
namespace logical {
@@ -60,7 +64,14 @@ class SharedSubplanReference : public Logical {
}
/**
- * @return The output attributes of the shared subplan.
+ * @return The attributes from the referenced shared subplan.
+ */
+ const std::vector<expressions::AttributeReferencePtr>& referenced_attributes() const {
+ return referenced_attributes_;
+ }
+
+ /**
+ * @return The output attributes of this shared subplan reference.
*/
const std::vector<expressions::AttributeReferencePtr>& output_attributes() const {
return output_attributes_;
@@ -81,7 +92,7 @@ class SharedSubplanReference : public Logical {
LogicalPtr copyWithNewChildren(const std::vector<LogicalPtr> &new_children) const override;
std::vector<expressions::AttributeReferencePtr> getReferencedAttributes() const override {
- return output_attributes_;
+ return referenced_attributes_;
}
/**
@@ -89,21 +100,30 @@ class SharedSubplanReference : public Logical {
*
* @param subplan_id The ID of the shared subplan, which is the index of the referenced
* shared subplan in <shared_subplans_> of the TopLevelPlan.
- * @param output_attributes The output attributes of the shared subplan.
+ * @param referenced_attributes The attributes from the referenced shared subplan.
+ * @param output_attributes The output attributes of this shared subplan reference.
* @return An immutable SharedSubplanReference.
*/
- static SharedSubplanReferencePtr Create(int subplan_id,
- const std::vector<expressions::AttributeReferencePtr> &output_attributes) {
- return SharedSubplanReferencePtr(new SharedSubplanReference(subplan_id, output_attributes));
+ static SharedSubplanReferencePtr Create(
+ int subplan_id,
+ const std::vector<expressions::AttributeReferencePtr> &referenced_attributes,
+ const std::vector<expressions::AttributeReferencePtr> &output_attributes) {
+ return SharedSubplanReferencePtr(
+ new SharedSubplanReference(subplan_id, referenced_attributes, output_attributes));
}
private:
SharedSubplanReference(int subplan_id,
+ const std::vector<expressions::AttributeReferencePtr> &referenced_attributes,
const std::vector<expressions::AttributeReferencePtr> &output_attributes)
: subplan_id_(subplan_id),
- output_attributes_(output_attributes) {}
+ referenced_attributes_(referenced_attributes),
+ output_attributes_(output_attributes) {
+ DCHECK_EQ(output_attributes_.size(), referenced_attributes_.size());
+ }
int subplan_id_;
+ std::vector<expressions::AttributeReferencePtr> referenced_attributes_;
std::vector<expressions::AttributeReferencePtr> output_attributes_;
DISALLOW_COPY_AND_ASSIGN(SharedSubplanReference);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/physical/SharedSubplanReference.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/SharedSubplanReference.cpp b/query_optimizer/physical/SharedSubplanReference.cpp
index ce851ac..eb33d32 100644
--- a/query_optimizer/physical/SharedSubplanReference.cpp
+++ b/query_optimizer/physical/SharedSubplanReference.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -40,6 +42,9 @@ void SharedSubplanReference::getFieldStringItems(
inline_field_names->push_back("subplan_id");
inline_field_values->push_back(std::to_string(subplan_id_));
+ container_child_field_names->push_back("referenced_attributes");
+ container_child_fields->push_back(CastSharedPtrVector<OptimizerTreeBase>(referenced_attributes_));
+
container_child_field_names->push_back("output_attributes");
container_child_fields->push_back(CastSharedPtrVector<OptimizerTreeBase>(output_attributes_));
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/physical/SharedSubplanReference.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/SharedSubplanReference.hpp b/query_optimizer/physical/SharedSubplanReference.hpp
index 5005598..4439256 100644
--- a/query_optimizer/physical/SharedSubplanReference.hpp
+++ b/query_optimizer/physical/SharedSubplanReference.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -63,7 +65,14 @@ class SharedSubplanReference : public physical::Physical {
}
/**
- * @return The output attributes of the shared subplan.
+ * @return The attributes from the referenced shared subplan.
+ */
+ const std::vector<expressions::AttributeReferencePtr>& referenced_attributes() const {
+ return referenced_attributes_;
+ }
+
+ /**
+ * @return The output attributes of this shared subplan reference.
*/
const std::vector<expressions::AttributeReferencePtr>& output_attributes() const {
return output_attributes_;
@@ -83,11 +92,11 @@ class SharedSubplanReference : public physical::Physical {
PhysicalPtr copyWithNewChildren(const std::vector<PhysicalPtr> &new_children) const override {
DCHECK(new_children.empty());
- return Create(subplan_id_, output_attributes_);
+ return Create(subplan_id_, referenced_attributes_, output_attributes_);
}
std::vector<expressions::AttributeReferencePtr> getReferencedAttributes() const override {
- return output_attributes_;
+ return referenced_attributes_;
}
bool maybeCopyWithPrunedExpressions(
@@ -101,21 +110,30 @@ class SharedSubplanReference : public physical::Physical {
*
* @param subplan_id The ID of the shared subplan, which is the index of
* the referenced shared subplan in <shared_subplans_> of the TopLevelPlan.
- * @param output_attributes The output attributes of the shared subplan.
+ * @param referenced_attributes The attributes from the referenced shared subplan.
+ * @param output_attributes The output attributes of this shared subplan reference.
* @return An immutable SharedSubplanReference.
*/
- static SharedSubplanReferencePtr Create(int subplan_id,
- const std::vector<expressions::AttributeReferencePtr> &output_attributes) {
- return SharedSubplanReferencePtr(new SharedSubplanReference(subplan_id, output_attributes));
+ static SharedSubplanReferencePtr Create(
+ int subplan_id,
+ const std::vector<expressions::AttributeReferencePtr> &referenced_attributes,
+ const std::vector<expressions::AttributeReferencePtr> &output_attributes) {
+ return SharedSubplanReferencePtr(
+ new SharedSubplanReference(subplan_id, referenced_attributes, output_attributes));
}
private:
SharedSubplanReference(int subplan_id,
+ const std::vector<expressions::AttributeReferencePtr> &referenced_attributes,
const std::vector<expressions::AttributeReferencePtr> &output_attributes)
: subplan_id_(subplan_id),
- output_attributes_(output_attributes) {}
+ referenced_attributes_(referenced_attributes),
+ output_attributes_(output_attributes) {
+ DCHECK_EQ(output_attributes_.size(), referenced_attributes_.size());
+ }
int subplan_id_;
+ std::vector<expressions::AttributeReferencePtr> referenced_attributes_;
std::vector<expressions::AttributeReferencePtr> output_attributes_;
DISALLOW_COPY_AND_ASSIGN(SharedSubplanReference);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index 1cf5c28..45ecf33 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -1541,9 +1541,28 @@ L::LogicalPtr Resolver::resolveSimpleTableReference(
with_queries_info_.with_query_name_to_vector_position.find(lower_table_name);
if (subplan_it != with_queries_info_.with_query_name_to_vector_position.end()) {
with_queries_info_.unreferenced_query_indexes.erase(subplan_it->second);
- return L::SharedSubplanReference::Create(
- subplan_it->second,
- with_queries_info_.with_query_plans[subplan_it->second]->getOutputAttributes());
+
+ const std::vector<E::AttributeReferencePtr> with_query_attributes =
+ with_queries_info_.with_query_plans[subplan_it->second]->getOutputAttributes();
+
+ // Create a vector of new attributes to delegate the original output attributes
+ // from the WITH query, to avoid (ExprId -> CatalogAttribute) mapping collision
+ // later in ExecutionGenerator when there are multiple SharedSubplanReference's
+ // referencing a same shared subplan.
+ std::vector<E::AttributeReferencePtr> delegator_attributes;
+ for (const E::AttributeReferencePtr &attribute : with_query_attributes) {
+ delegator_attributes.emplace_back(
+ E::AttributeReference::Create(context_->nextExprId(),
+ attribute->attribute_name(),
+ attribute->attribute_alias(),
+ attribute->relation_name(),
+ attribute->getValueType(),
+ attribute->scope()));
+ }
+
+ return L::SharedSubplanReference::Create(subplan_it->second,
+ with_query_attributes,
+ delegator_attributes);
}
// Then look up the name in the database.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/strategy/OneToOne.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/strategy/OneToOne.cpp b/query_optimizer/strategy/OneToOne.cpp
index d4d79f9..7f59151 100644
--- a/query_optimizer/strategy/OneToOne.cpp
+++ b/query_optimizer/strategy/OneToOne.cpp
@@ -84,6 +84,7 @@ bool OneToOne::generatePlan(const L::LogicalPtr &logical_input,
const L::SharedSubplanReferencePtr shared_subplan_reference =
std::static_pointer_cast<const L::SharedSubplanReference>(logical_input);
*physical_output = P::SharedSubplanReference::Create(shared_subplan_reference->subplan_id(),
+ shared_subplan_reference->referenced_attributes(),
shared_subplan_reference->output_attributes());
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index 3a64c9b..a08b012 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -891,6 +891,31 @@ FROM (
+-----------+--------------------+-----------+------------------------+
==
+# Same shared subplan referenced multiple times.
+WITH t(x, y) AS (
+ SELECT i % 5, i
+ FROM generate_series(1, 20) AS g(i)
+)
+SELECT *
+FROM t
+WHERE t.y = (
+ SELECT MAX(y)
+ FROM t t1
+ WHERE t.x = t1.x
+)
+ORDER BY x;
+--
++-----------+-----------+
+|x |y |
++-----------+-----------+
+| 0| 20|
+| 1| 16|
+| 2| 17|
+| 3| 18|
+| 4| 19|
++-----------+-----------+
+==
+
# TODO(team): Fix Issue #9 to enable COUNT(*).
SELECT COUNT(long_col)
FROM test,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/tests/logical_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/logical_generator/Select.test b/query_optimizer/tests/logical_generator/Select.test
index 6dff3e7..3c152e8 100644
--- a/query_optimizer/tests/logical_generator/Select.test
+++ b/query_optimizer/tests/logical_generator/Select.test
@@ -502,22 +502,29 @@ TopLevelPlan
| | | | type=VarChar(20) NULL]
| | | +-right=Filter
| | | | +-input=SharedSubplanReference[subplan_id=0]
+| | | | | +-referenced_attributes=
+| | | | | | +-AttributeReference[id=0,name=subquery_int0,relation=subquery,
+| | | | | | | type=Int NULL]
+| | | | | | +-AttributeReference[id=7,name=subquery_long,relation=subquery,
+| | | | | | | type=Long]
+| | | | | | +-AttributeReference[id=12,name=subquery_int1,relation=subquery,
+| | | | | | type=Int NULL]
| | | | | +-output_attributes=
-| | | | | +-AttributeReference[id=0,name=subquery_int0,relation=subquery,
+| | | | | +-AttributeReference[id=36,name=subquery_int0,relation=subquery,
| | | | | | type=Int NULL]
-| | | | | +-AttributeReference[id=7,name=subquery_long,relation=subquery,
+| | | | | +-AttributeReference[id=37,name=subquery_long,relation=subquery,
| | | | | | type=Long]
-| | | | | +-AttributeReference[id=12,name=subquery_int1,relation=subquery,
+| | | | | +-AttributeReference[id=38,name=subquery_int1,relation=subquery,
| | | | | type=Int NULL]
| | | | +-filter_predicate=Equal
-| | | | +-AttributeReference[id=12,name=subquery_int1,relation=subquery,
+| | | | +-AttributeReference[id=38,name=subquery_int1,relation=subquery,
| | | | | type=Int NULL]
-| | | | +-AttributeReference[id=7,name=subquery_long,relation=subquery,
+| | | | +-AttributeReference[id=37,name=subquery_long,relation=subquery,
| | | | type=Long]
| | | +-left_join_attributes=
| | | | +-AttributeReference[id=30,name=int_col,relation=c,type=Int NULL]
| | | +-right_join_attributes=
-| | | +-AttributeReference[id=0,name=subquery_int0,relation=subquery,
+| | | +-AttributeReference[id=36,name=subquery_int0,relation=subquery,
| | | type=Int NULL]
| | +-join_predicate=Literal[value=true]
| +-project_list=
@@ -572,11 +579,14 @@ select * from subquery
TopLevelPlan
+-plan=Project
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=7,name=,alias=(int_col+2),relation=subquery,
+| | | type=Int NULL]
| | +-output_attributes=
-| | +-AttributeReference[id=7,name=,alias=(int_col+2),relation=subquery,
+| | +-AttributeReference[id=8,name=,alias=(int_col+2),relation=subquery,
| | type=Int NULL]
| +-project_list=
-| +-AttributeReference[id=7,name=,alias=(int_col+2),relation=subquery,
+| +-AttributeReference[id=8,name=,alias=(int_col+2),relation=subquery,
| type=Int NULL]
+-shared_subplans=
| +-Project
@@ -596,7 +606,7 @@ TopLevelPlan
| | +-Literal[value=1,type=Int]
| +-Literal[value=2,type=Int]
+-output_attributes=
- +-AttributeReference[id=7,name=,alias=(int_col+2),relation=subquery,
+ +-AttributeReference[id=8,name=,alias=(int_col+2),relation=subquery,
type=Int NULL]
==
@@ -1266,3 +1276,81 @@ SELECT x + (
FROM b;
--
ERROR: Nested queries can only reference attributes in the outer query one level above
+==
+
+# Same shared subplan referenced multiple times.
+WITH t(x, y) AS (
+ SELECT i % 5, i
+ FROM generate_series(1, 20) AS g(i)
+)
+SELECT *
+FROM t
+WHERE t.y = (
+ SELECT MAX(y)
+ FROM t t1
+ WHERE t.x = t1.x
+)
+ORDER BY x;
+--
+TopLevelPlan
++-plan=Project
+| +-input=Sort[is_ascending=[true],nulls_first=[false]]
+| | +-input=Filter
+| | | +-input=HashJoin
+| | | | +-left=SharedSubplanReference[subplan_id=0]
+| | | | | +-referenced_attributes=
+| | | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | | +-output_attributes=
+| | | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | | +-right=Project
+| | | | | +-input=Aggregate
+| | | | | | +-input=SharedSubplanReference[subplan_id=0]
+| | | | | | | +-referenced_attributes=
+| | | | | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | | | | +-output_attributes=
+| | | | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | | | | +-grouping_expressions=
+| | | | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | | +-aggregate_expressions=
+| | | | | | +-Alias[id=9,name=,alias=$aggregate0,relation=$aggregate,
+| | | | | | type=Int NULL]
+| | | | | | +-AggregateFunction[function=MAX]
+| | | | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | | | +-project_list=
+| | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | +-Alias[id=9,name=,alias=MAX(y),relation=,type=Int NULL]
+| | | | | +-AttributeReference[id=9,name=,alias=$aggregate0,
+| | | | | relation=$aggregate,type=Int NULL]
+| | | | +-left_join_attributes=
+| | | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | +-right_join_attributes=
+| | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | +-filter_predicate=Equal
+| | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-AttributeReference[id=9,name=,alias=MAX(y),relation=,type=Int NULL]
+| | +-sort_expressions=
+| | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-project_list=
+| +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-AttributeReference[id=6,name=y,relation=,type=Int]
++-shared_subplans=
+| +-Project
+| +-input=TableGenerator[function_name=generate_series,table_alias=g]
+| | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | relation=generate_series,type=Int]
+| +-project_list=
+| +-Alias[id=3,name=x,relation=,type=Int]
+| | +-Modulo
+| | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | | relation=generate_series,type=Int]
+| | +-Literal[value=5,type=Int]
+| +-Alias[id=4,name=y,relation=,type=Int]
+| +-AttributeReference[id=0,name=generate_series,alias=g,
+| relation=generate_series,type=Int]
++-output_attributes=
+ +-AttributeReference[id=5,name=x,relation=,type=Int]
+ +-AttributeReference[id=6,name=y,relation=,type=Int]
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/tests/physical_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/physical_generator/Select.test b/query_optimizer/tests/physical_generator/Select.test
index 62d09f5..3365206 100644
--- a/query_optimizer/tests/physical_generator/Select.test
+++ b/query_optimizer/tests/physical_generator/Select.test
@@ -1542,12 +1542,16 @@ select int_col from subquery
TopLevelPlan
+-plan=Project
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
| | +-output_attributes=
-| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
-| | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
-| | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
+| | +-AttributeReference[id=7,name=long_col,relation=test,type=Long]
+| | +-AttributeReference[id=8,name=double_col,relation=test,type=Double NULL]
| +-project_list=
-| +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
+-shared_subplans=
| +-Project
| +-input=TableReference[relation_name=Test,relation_alias=test]
@@ -1563,17 +1567,21 @@ TopLevelPlan
| +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
| +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+-output_attributes=
- +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+ +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
[Physical Plan]
TopLevelPlan
+-plan=Selection
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
| | +-output_attributes=
-| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
-| | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
-| | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
+| | +-AttributeReference[id=7,name=long_col,relation=test,type=Long]
+| | +-AttributeReference[id=8,name=double_col,relation=test,type=Double NULL]
| +-project_expressions=
-| +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
+-shared_subplans=
| +-Selection
| +-input=TableReference[relation=Test,alias=test]
@@ -1589,7 +1597,7 @@ TopLevelPlan
| +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
| +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+-output_attributes=
- +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+ +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
==
SELECT COUNT(DISTINCT int_col), SUM(float_col)
@@ -2731,3 +2739,140 @@ TopLevelPlan
+-output_attributes=
+-AttributeReference[id=8,name=,alias=(x*SubqueryExpression),relation=,
type=Long NULL]
+==
+
+# Same shared subplan referenced multiple times.
+WITH t(x, y) AS (
+ SELECT i % 5, i
+ FROM generate_series(1, 20) AS g(i)
+)
+SELECT *
+FROM t
+WHERE t.y = (
+ SELECT MAX(y)
+ FROM t t1
+ WHERE t.x = t1.x
+)
+ORDER BY x;
+--
+[Optimized Logical Plan]
+TopLevelPlan
++-plan=Project
+| +-input=Sort[is_ascending=[true],nulls_first=[false]]
+| | +-input=Filter
+| | | +-input=HashJoin
+| | | | +-left=SharedSubplanReference[subplan_id=0]
+| | | | | +-referenced_attributes=
+| | | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | | +-output_attributes=
+| | | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | | +-right=Project
+| | | | | +-input=Aggregate
+| | | | | | +-input=SharedSubplanReference[subplan_id=0]
+| | | | | | | +-referenced_attributes=
+| | | | | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | | | | +-output_attributes=
+| | | | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | | | | +-grouping_expressions=
+| | | | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | | +-aggregate_expressions=
+| | | | | | +-Alias[id=9,name=,alias=$aggregate0,relation=$aggregate,
+| | | | | | type=Int NULL]
+| | | | | | +-AggregateFunction[function=MAX]
+| | | | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | | | +-project_list=
+| | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | +-Alias[id=9,name=,alias=MAX(y),relation=,type=Int NULL]
+| | | | | +-AttributeReference[id=9,name=,alias=$aggregate0,
+| | | | | relation=$aggregate,type=Int NULL]
+| | | | +-left_join_attributes=
+| | | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | +-right_join_attributes=
+| | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | +-filter_predicate=Equal
+| | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-AttributeReference[id=9,name=,alias=MAX(y),relation=,type=Int NULL]
+| | +-sort_expressions=
+| | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-project_list=
+| +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-AttributeReference[id=6,name=y,relation=,type=Int]
++-shared_subplans=
+| +-Project
+| +-input=TableGenerator[function_name=generate_series,table_alias=g]
+| | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | relation=generate_series,type=Int]
+| +-project_list=
+| +-Alias[id=3,name=x,relation=,type=Int]
+| | +-Modulo
+| | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | | relation=generate_series,type=Int]
+| | +-Literal[value=5,type=Int]
+| +-Alias[id=4,name=y,relation=,type=Int]
+| +-AttributeReference[id=0,name=generate_series,alias=g,
+| relation=generate_series,type=Int]
++-output_attributes=
+ +-AttributeReference[id=5,name=x,relation=,type=Int]
+ +-AttributeReference[id=6,name=y,relation=,type=Int]
+[Physical Plan]
+TopLevelPlan
++-plan=Selection
+| +-input=Sort[is_ascending=[true],nulls_first=[false]]
+| | +-input=HashJoin
+| | | +-left=SharedSubplanReference[subplan_id=0]
+| | | | +-referenced_attributes=
+| | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | +-output_attributes=
+| | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-right=Aggregate
+| | | | +-input=SharedSubplanReference[subplan_id=0]
+| | | | | +-referenced_attributes=
+| | | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | | +-output_attributes=
+| | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | | +-grouping_expressions=
+| | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | +-aggregate_expressions=
+| | | | +-Alias[id=9,name=,alias=$aggregate0,relation=$aggregate,type=Int NULL]
+| | | | +-AggregateFunction[function=MAX]
+| | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | +-project_expressions=
+| | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-left_join_attributes=
+| | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-right_join_attributes=
+| | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | +-AttributeReference[id=9,name=,alias=$aggregate0,relation=$aggregate,
+| | | type=Int NULL]
+| | +-sort_attributes=
+| | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-project_expressions=
+| +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-AttributeReference[id=6,name=y,relation=,type=Int]
++-shared_subplans=
+| +-Selection
+| +-input=TableGenerator[function_name=generate_series,table_alias=g]
+| | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | relation=generate_series,type=Int]
+| +-project_expressions=
+| +-Alias[id=3,name=x,relation=,type=Int]
+| | +-Modulo
+| | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | | relation=generate_series,type=Int]
+| | +-Literal[value=5,type=Int]
+| +-Alias[id=4,name=y,relation=,type=Int]
+| +-AttributeReference[id=0,name=generate_series,alias=g,
+| relation=generate_series,type=Int]
++-output_attributes=
+ +-AttributeReference[id=5,name=x,relation=,type=Int]
+ +-AttributeReference[id=6,name=y,relation=,type=Int]
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/tests/resolver/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/resolver/Select.test b/query_optimizer/tests/resolver/Select.test
index 00ff18a..9897934 100644
--- a/query_optimizer/tests/resolver/Select.test
+++ b/query_optimizer/tests/resolver/Select.test
@@ -1723,11 +1723,14 @@ select subquery_col from subquery
TopLevelPlan
+-plan=Project
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=0,name=subquery_col,relation=subquery,
+| | | type=Int NULL]
| | +-output_attributes=
-| | +-AttributeReference[id=0,name=subquery_col,relation=subquery,
+| | +-AttributeReference[id=6,name=subquery_col,relation=subquery,
| | type=Int NULL]
| +-project_list=
-| +-AttributeReference[id=0,name=subquery_col,relation=subquery,type=Int NULL]
+| +-AttributeReference[id=6,name=subquery_col,relation=subquery,type=Int NULL]
+-shared_subplans=
| +-Project
| +-input=TableReference[relation_name=Test,relation_alias=test]
@@ -1742,7 +1745,7 @@ TopLevelPlan
| +-Alias[id=0,name=subquery_col,relation=subquery,type=Int NULL]
| +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+-output_attributes=
- +-AttributeReference[id=0,name=subquery_col,relation=subquery,type=Int NULL]
+ +-AttributeReference[id=6,name=subquery_col,relation=subquery,type=Int NULL]
==
# Multiple WITH queries.
@@ -1755,14 +1758,18 @@ TopLevelPlan
+-plan=Project
| +-input=MultiwayCartesianJoin
| | +-SharedSubplanReference[subplan_id=2]
+| | | +-referenced_attributes=
+| | | | +-AttributeReference[id=10,name=int_col,relation=subquery3,type=Int NULL]
| | | +-output_attributes=
-| | | +-AttributeReference[id=7,name=int_col,relation=subquery3,type=Int NULL]
+| | | +-AttributeReference[id=11,name=int_col,relation=subquery3,type=Int NULL]
| | +-SharedSubplanReference[subplan_id=1]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=7,name=int_col,relation=subquery2,type=Int NULL]
| | +-output_attributes=
-| | +-AttributeReference[id=6,name=int_col,relation=subquery2,type=Int NULL]
+| | +-AttributeReference[id=12,name=int_col,relation=subquery2,type=Int NULL]
| +-project_list=
-| +-AttributeReference[id=7,name=int_col,relation=subquery3,type=Int NULL]
-| +-AttributeReference[id=6,name=int_col,relation=subquery2,type=Int NULL]
+| +-AttributeReference[id=11,name=int_col,relation=subquery3,type=Int NULL]
+| +-AttributeReference[id=12,name=int_col,relation=subquery2,type=Int NULL]
+-shared_subplans=
| +-Project
| | +-input=TableReference[relation_name=Test,relation_alias=test]
@@ -1777,31 +1784,38 @@ TopLevelPlan
| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
| +-Project
| | +-input=SharedSubplanReference[subplan_id=0]
+| | | +-referenced_attributes=
+| | | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
| | | +-output_attributes=
-| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
| | +-project_list=
-| | +-Alias[id=6,name=int_col,relation=subquery2,type=Int NULL]
+| | +-Alias[id=7,name=int_col,relation=subquery2,type=Int NULL]
| | +-Add
-| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
| | +-Literal[value=5,type=Int]
| +-Project
| +-input=MultiwayCartesianJoin
| | +-SharedSubplanReference[subplan_id=1]
+| | | +-referenced_attributes=
+| | | | +-AttributeReference[id=7,name=int_col,relation=subquery2,
+| | | | type=Int NULL]
| | | +-output_attributes=
-| | | +-AttributeReference[id=6,name=int_col,relation=subquery2,
+| | | +-AttributeReference[id=8,name=int_col,relation=subquery2,
| | | type=Int NULL]
| | +-SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
| | +-output_attributes=
-| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | +-AttributeReference[id=9,name=int_col,relation=test,type=Int NULL]
| +-project_list=
-| +-Alias[id=7,name=int_col,relation=subquery3,type=Int NULL]
+| +-Alias[id=10,name=int_col,relation=subquery3,type=Int NULL]
| +-Add
-| +-AttributeReference[id=6,name=int_col,relation=subquery2,
+| +-AttributeReference[id=8,name=int_col,relation=subquery2,
| | type=Int NULL]
| +-Literal[value=6,type=Int]
+-output_attributes=
- +-AttributeReference[id=7,name=int_col,relation=subquery3,type=Int NULL]
- +-AttributeReference[id=6,name=int_col,relation=subquery2,type=Int NULL]
+ +-AttributeReference[id=11,name=int_col,relation=subquery3,type=Int NULL]
+ +-AttributeReference[id=12,name=int_col,relation=subquery2,type=Int NULL]
==
with subquery(int_col_alias, long_col_alias) as (select int_col, long_col from test)
@@ -1810,11 +1824,14 @@ select int_col_alias from subquery
TopLevelPlan
+-plan=Project
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=6,name=int_col_alias,relation=,type=Int NULL]
+| | | +-AttributeReference[id=7,name=long_col_alias,relation=,type=Long]
| | +-output_attributes=
-| | +-AttributeReference[id=6,name=int_col_alias,relation=,type=Int NULL]
-| | +-AttributeReference[id=7,name=long_col_alias,relation=,type=Long]
+| | +-AttributeReference[id=8,name=int_col_alias,relation=,type=Int NULL]
+| | +-AttributeReference[id=9,name=long_col_alias,relation=,type=Long]
| +-project_list=
-| +-AttributeReference[id=6,name=int_col_alias,relation=,type=Int NULL]
+| +-AttributeReference[id=8,name=int_col_alias,relation=,type=Int NULL]
+-shared_subplans=
| +-Project
| +-input=Project
@@ -1835,7 +1852,7 @@ TopLevelPlan
| +-Alias[id=7,name=long_col_alias,relation=,type=Long]
| +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+-output_attributes=
- +-AttributeReference[id=6,name=int_col_alias,relation=,type=Int NULL]
+ +-AttributeReference[id=8,name=int_col_alias,relation=,type=Int NULL]
==
with subquery as (select int_col, long_col from test)
@@ -1845,16 +1862,19 @@ TopLevelPlan
+-plan=Project
| +-input=Project
| | +-input=SharedSubplanReference[subplan_id=0]
+| | | +-referenced_attributes=
+| | | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
| | | +-output_attributes=
-| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
-| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
+| | | +-AttributeReference[id=7,name=long_col,relation=test,type=Long]
| | +-project_list=
-| | +-Alias[id=6,name=int_col_alias,relation=,type=Int NULL]
-| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
-| | +-Alias[id=7,name=long_col_alias,relation=,type=Long]
-| | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | +-Alias[id=8,name=int_col_alias,relation=,type=Int NULL]
+| | | +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
+| | +-Alias[id=9,name=long_col_alias,relation=,type=Long]
+| | +-AttributeReference[id=7,name=long_col,relation=test,type=Long]
| +-project_list=
-| +-AttributeReference[id=6,name=int_col_alias,relation=,type=Int NULL]
+| +-AttributeReference[id=8,name=int_col_alias,relation=,type=Int NULL]
+-shared_subplans=
| +-Project
| +-input=TableReference[relation_name=Test,relation_alias=test]
@@ -1869,7 +1889,7 @@ TopLevelPlan
| +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
| +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+-output_attributes=
- +-AttributeReference[id=6,name=int_col_alias,relation=,type=Int NULL]
+ +-AttributeReference[id=8,name=int_col_alias,relation=,type=Int NULL]
==
with subquery as (select int_col int_col_alias, long_col from test)
@@ -1878,12 +1898,16 @@ select int_col_alias from subquery
TopLevelPlan
+-plan=Project
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=0,name=int_col_alias,relation=subquery,
+| | | | type=Int NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
| | +-output_attributes=
-| | +-AttributeReference[id=0,name=int_col_alias,relation=subquery,
+| | +-AttributeReference[id=6,name=int_col_alias,relation=subquery,
| | | type=Int NULL]
-| | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | +-AttributeReference[id=7,name=long_col,relation=test,type=Long]
| +-project_list=
-| +-AttributeReference[id=0,name=int_col_alias,relation=subquery,type=Int NULL]
+| +-AttributeReference[id=6,name=int_col_alias,relation=subquery,type=Int NULL]
+-shared_subplans=
| +-Project
| +-input=TableReference[relation_name=Test,relation_alias=test]
@@ -1899,7 +1923,7 @@ TopLevelPlan
| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
| +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+-output_attributes=
- +-AttributeReference[id=0,name=int_col_alias,relation=subquery,type=Int NULL]
+ +-AttributeReference[id=6,name=int_col_alias,relation=subquery,type=Int NULL]
==
# A WITH query cannot reference a subsequent WITH query.
@@ -1936,10 +1960,12 @@ select 1 from test
TopLevelPlan
+-plan=Project
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=6,name=,alias=1,relation=test,type=Int]
| | +-output_attributes=
-| | +-AttributeReference[id=6,name=,alias=1,relation=test,type=Int]
+| | +-AttributeReference[id=7,name=,alias=1,relation=test,type=Int]
| +-project_list=
-| +-Alias[id=7,name=,alias=1,relation=,type=Int]
+| +-Alias[id=8,name=,alias=1,relation=,type=Int]
| +-Literal[value=1,type=Int]
+-shared_subplans=
| +-Project
@@ -1955,7 +1981,7 @@ TopLevelPlan
| +-Alias[id=6,name=,alias=1,relation=test,type=Int]
| +-Literal[value=1,type=Int]
+-output_attributes=
- +-AttributeReference[id=7,name=,alias=1,relation=,type=Int]
+ +-AttributeReference[id=8,name=,alias=1,relation=,type=Int]
==
# Identifies with special characters.
@@ -2937,3 +2963,88 @@ TopLevelPlan
+-output_attributes=
+-AttributeReference[id=11,name=,alias=(x+SubqueryExpression),relation=,
type=Long NULL]
+==
+
+# Same shared subplan referenced multiple times.
+WITH t(x, y) AS (
+ SELECT i % 5, i
+ FROM generate_series(1, 20) AS g(i)
+)
+SELECT *
+FROM t
+WHERE t.y = (
+ SELECT MAX(y)
+ FROM t t1
+ WHERE t.x = t1.x
+)
+ORDER BY x;
+--
+TopLevelPlan
++-plan=Project
+| +-input=Sort[is_ascending=[true],nulls_first=[false]]
+| | +-input=Filter
+| | | +-input=SharedSubplanReference[subplan_id=0]
+| | | | +-referenced_attributes=
+| | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | +-output_attributes=
+| | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-filter_predicate=Equal
+| | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-SubqueryExpression
+| | | +-subquery=Project
+| | | +-input=Aggregate
+| | | | +-input=Filter
+| | | | | +-input=SharedSubplanReference[subplan_id=0]
+| | | | | | +-referenced_attributes=
+| | | | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | | | +-output_attributes=
+| | | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | | | +-filter_predicate=Equal
+| | | | | +-AttributeReference[id=5,name=x,relation=,type=Int,
+| | | | | | is_outer_reference=true]
+| | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | +-grouping_expressions=
+| | | | | +-[]
+| | | | +-aggregate_expressions=
+| | | | +-Alias[id=9,name=,alias=$aggregate0,relation=$aggregate,
+| | | | type=Int NULL]
+| | | | +-AggregateFunction[function=MAX]
+| | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | +-project_list=
+| | | +-Alias[id=9,name=,alias=MAX(y),relation=,type=Int NULL]
+| | | +-AttributeReference[id=9,name=,alias=$aggregate0,
+| | | relation=$aggregate,type=Int NULL]
+| | +-sort_expressions=
+| | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-project_list=
+| +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-AttributeReference[id=6,name=y,relation=,type=Int]
++-shared_subplans=
+| +-Project
+| +-input=Project
+| | +-input=Project
+| | | +-input=TableGenerator[function_name=generate_series,table_alias=g]
+| | | | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | | | relation=generate_series,type=Int]
+| | | +-project_list=
+| | | +-Alias[id=1,name=i,relation=,type=Int]
+| | | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | | relation=generate_series,type=Int]
+| | +-project_list=
+| | +-Alias[id=2,name=,alias=(i%5),relation=t,type=Int]
+| | | +-Modulo
+| | | +-AttributeReference[id=1,name=i,relation=,type=Int]
+| | | +-Literal[value=5,type=Int]
+| | +-AttributeReference[id=1,name=i,relation=,type=Int]
+| +-project_list=
+| +-Alias[id=3,name=x,relation=,type=Int]
+| | +-AttributeReference[id=2,name=,alias=(i%5),relation=t,type=Int]
+| +-Alias[id=4,name=y,relation=,type=Int]
+| +-AttributeReference[id=1,name=i,relation=,type=Int]
++-output_attributes=
+ +-AttributeReference[id=5,name=x,relation=,type=Int]
+ +-AttributeReference[id=6,name=y,relation=,type=Int]
[31/50] [abbrv] incubator-quickstep git commit: Minor improvements in
FileManagers. (#224)
Posted by hb...@apache.org.
Minor improvements in FileManagers. (#224)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/2c0722e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/2c0722e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/2c0722e1
Branch: refs/heads/query-manager-used-in-foreman
Commit: 2c0722e13afa6caabe2d281228b0e707b0ef28d9
Parents: 1fa81a8
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Tue May 17 14:26:19 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:45 2016 -0700
----------------------------------------------------------------------
storage/CMakeLists.txt | 21 ++++++++------
storage/FileManager.hpp | 9 ++----
storage/FileManagerHdfs.cpp | 57 ++++++++++++++++++-------------------
storage/FileManagerPosix.cpp | 55 +++++++++++++++++------------------
storage/FileManagerPosix.hpp | 6 ++--
storage/FileManagerWindows.cpp | 53 +++++++++++++++++-----------------
storage/FileManagerWindows.hpp | 4 +--
7 files changed, 103 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 115248c..26a3e32 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -585,20 +585,22 @@ target_link_libraries(quickstep_storage_FileManager
quickstep_utility_Macros
quickstep_utility_StringUtil)
if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
-target_link_libraries(quickstep_storage_FileManagerHdfs
- gflags_nothreads-static
- quickstep_storage_FileManager
- quickstep_storage_StorageBlockInfo
- quickstep_storage_StorageConstants
- quickstep_storage_StorageErrors
- quickstep_utility_Macros
- quickstep_utility_StringUtil
- ${LIBHDFS3_LIBRARIES})
+ target_link_libraries(quickstep_storage_FileManagerHdfs
+ glog
+ gflags_nothreads-static
+ quickstep_storage_FileManager
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_storage_StorageErrors
+ quickstep_utility_Macros
+ quickstep_utility_StringUtil
+ ${LIBHDFS3_LIBRARIES})
endif()
if (QUICKSTEP_HAVE_FILE_MANAGER_POSIX)
target_link_libraries(quickstep_storage_FileManagerLocal
quickstep_storage_FileManagerPosix)
target_link_libraries(quickstep_storage_FileManagerPosix
+ glog
quickstep_storage_FileManager
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageConstants
@@ -609,6 +611,7 @@ elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS)
target_link_libraries(quickstep_storage_FileManagerLocal
quickstep_storage_FileManagerWindows)
target_link_libraries(quickstep_storage_FileManagerWindows
+ glog
quickstep_storage_FileManager
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageConstants
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/FileManager.hpp
----------------------------------------------------------------------
diff --git a/storage/FileManager.hpp b/storage/FileManager.hpp
index d56c960..b179071 100644
--- a/storage/FileManager.hpp
+++ b/storage/FileManager.hpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,10 +19,7 @@
#define QUICKSTEP_STORAGE_FILE_MANAGER_HPP_
#include <cstddef>
-#include <cstdint>
-#include <cstdio>
#include <string>
-#include <utility>
#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
@@ -49,12 +46,12 @@ class FileManager {
* @param block_domain The domain of a block id.
**/
explicit FileManager(const std::string &storage_path)
- : storage_path_(storage_path) { }
+ : storage_path_(storage_path) {}
/**
* @brief Virtual destructor.
**/
- virtual ~FileManager() { }
+ virtual ~FileManager() {}
/**
* @brief Get a block or blob's relative filename, which uses storage_path_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/FileManagerHdfs.cpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerHdfs.cpp b/storage/FileManagerHdfs.cpp
index 5f9706e..e8f048b 100644
--- a/storage/FileManagerHdfs.cpp
+++ b/storage/FileManagerHdfs.cpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -32,10 +32,10 @@
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageConstants.hpp"
#include "storage/StorageErrors.hpp"
-#include "utility/Macros.hpp"
#include "utility/StringUtil.hpp"
#include "gflags/gflags.h"
+#include "glog/logging.h"
using std::size_t;
using std::sscanf;
@@ -76,20 +76,19 @@ static const bool hdfs_num_replications_dummy
FileManagerHdfs::FileManagerHdfs(const string &storage_path)
: FileManager(storage_path) {
- DEBUG_ASSERT(hdfs_namenode_port_dummy);
- DEBUG_ASSERT(hdfs_num_replications_dummy);
+ DCHECK(hdfs_namenode_port_dummy);
+ DCHECK(hdfs_num_replications_dummy);
struct hdfsBuilder *builder = hdfsNewBuilder();
hdfsBuilderSetNameNode(builder, FLAGS_hdfs_namenode_host.c_str());
hdfsBuilderSetNameNodePort(builder, FLAGS_hdfs_namenode_port);
// hdfsBuilderConnect releases builder.
hdfs_ = hdfsBuilderConnect(builder);
- DEBUG_ASSERT(hdfs_ != nullptr);
+ DCHECK(hdfs_ != nullptr);
}
FileManagerHdfs::~FileManagerHdfs() {
- int status = hdfsDisconnect(hdfs_);
- DEBUG_ASSERT(status == 0);
+ CHECK_EQ(0, hdfsDisconnect(hdfs_));
}
block_id_counter FileManagerHdfs::getMaxUsedBlockCounter(const block_id_domain block_domain) const {
@@ -97,7 +96,7 @@ block_id_counter FileManagerHdfs::getMaxUsedBlockCounter(const block_id_domain b
hdfsFileInfo *file_infos = hdfsListDirectory(hdfs_, storage_path_.c_str(), &num_files);
if (file_infos == nullptr) {
if (errno != ENOENT) {
- LOG_WARNING("Failed to list file info with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to list file info with error: " << strerror(errno);
}
return 0;
}
@@ -113,9 +112,9 @@ block_id_counter FileManagerHdfs::getMaxUsedBlockCounter(const block_id_domain b
// NOTE(zuyu): mName looks like
// "/user/<username>/<storage_path_>/qsblk_<block_domain>_[0-9]*.qsb".
const char *filename = std::strrchr(file_infos[i].mName, '/');
- if (filename != nullptr
- && sscanf(filename, filename_pattern.c_str(), &counter) == 1
- && counter > counter_max) {
+ if (filename != nullptr &&
+ sscanf(filename, filename_pattern.c_str(), &counter) == 1 &&
+ counter > counter_max) {
counter_max = counter;
}
}
@@ -126,12 +125,12 @@ block_id_counter FileManagerHdfs::getMaxUsedBlockCounter(const block_id_domain b
}
size_t FileManagerHdfs::numSlots(const block_id block) const {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
hdfsFileInfo *file_info = hdfsGetPathInfo(hdfs_, filename.c_str());
if (file_info == nullptr) {
if (errno != ENOENT) {
- LOG_WARNING("Failed to get size of file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to get size of file " << filename << " with error: " << strerror(errno);
}
return 0;
}
@@ -147,12 +146,12 @@ size_t FileManagerHdfs::numSlots(const block_id block) const {
}
bool FileManagerHdfs::deleteBlockOrBlob(const block_id block) {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
if ((hdfsDelete(hdfs_, filename.c_str(), 0) == 0) || (errno == ENOENT)) {
return true;
} else {
- LOG_WARNING("Failed to delete file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to delete file " << filename << " with error: " << strerror(errno);
return false;
}
}
@@ -160,10 +159,10 @@ bool FileManagerHdfs::deleteBlockOrBlob(const block_id block) {
bool FileManagerHdfs::readBlockOrBlob(const block_id block,
void *buffer,
const size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
hdfsFile file_handle = hdfsOpenFile(hdfs_,
filename.c_str(),
@@ -172,7 +171,7 @@ bool FileManagerHdfs::readBlockOrBlob(const block_id block,
FLAGS_hdfs_num_replications,
kSlotSizeBytes);
if (file_handle == nullptr) {
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
@@ -183,17 +182,17 @@ bool FileManagerHdfs::readBlockOrBlob(const block_id block,
bytes_total += bytes;
} else if (bytes == -1) {
if (errno != EINTR) {
- LOG_WARNING("Failed to read file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to read file " << filename << " with error: " << strerror(errno);
break;
}
} else {
- LOG_WARNING("Failed to read file " << filename << " since EOF was reached unexpectedly");
+ LOG(ERROR) << "Failed to read file " << filename << " since EOF was reached unexpectedly";
break;
}
}
if (hdfsCloseFile(hdfs_, file_handle) != 0) {
- LOG_WARNING("Failed to close file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to close file " << filename << " with error: " << strerror(errno);
}
return (bytes_total == length);
@@ -202,10 +201,10 @@ bool FileManagerHdfs::readBlockOrBlob(const block_id block,
bool FileManagerHdfs::writeBlockOrBlob(const block_id block,
const void *buffer,
const size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
hdfsFile file_handle = hdfsOpenFile(hdfs_,
filename.c_str(),
@@ -214,7 +213,7 @@ bool FileManagerHdfs::writeBlockOrBlob(const block_id block,
FLAGS_hdfs_num_replications,
kSlotSizeBytes);
if (file_handle == nullptr) {
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
@@ -224,17 +223,17 @@ bool FileManagerHdfs::writeBlockOrBlob(const block_id block,
if (bytes > 0) {
bytes_total += bytes;
} else if (bytes == -1) {
- LOG_WARNING("Failed to write file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to write file " << filename << " with error: " << strerror(errno);
break;
}
}
if (hdfsSync(hdfs_, file_handle) != 0) {
- LOG_WARNING("Failed to sync file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to sync file " << filename << " with error: " << strerror(errno);
}
if (hdfsCloseFile(hdfs_, file_handle) != 0) {
- LOG_WARNING("Failed to close file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to close file " << filename << " with error: " << strerror(errno);
}
return (bytes_total == length);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/FileManagerPosix.cpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerPosix.cpp b/storage/FileManagerPosix.cpp
index 3bfb69d..0346f0d 100644
--- a/storage/FileManagerPosix.cpp
+++ b/storage/FileManagerPosix.cpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -35,9 +35,10 @@
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageConstants.hpp"
#include "storage/StorageErrors.hpp"
-#include "utility/Macros.hpp"
#include "utility/StringUtil.hpp"
+#include "glog/logging.h"
+
using std::size_t;
using std::sscanf;
using std::strerror;
@@ -64,9 +65,9 @@ block_id_counter FileManagerPosix::getMaxUsedBlockCounter(const block_id_domain
filename_pattern.append(".qsb");
block_id_counter counter_max = 0, counter;
- if (glob_result.gl_pathc > 0
- && sscanf(glob_result.gl_pathv[glob_result.gl_pathc - 1], filename_pattern.c_str(), &counter) == 1
- && counter > counter_max) {
+ if (glob_result.gl_pathc > 0 &&
+ sscanf(glob_result.gl_pathv[glob_result.gl_pathc - 1], filename_pattern.c_str(), &counter) == 1 &&
+ counter > counter_max) {
counter_max = counter;
}
@@ -75,12 +76,12 @@ block_id_counter FileManagerPosix::getMaxUsedBlockCounter(const block_id_domain
}
size_t FileManagerPosix::numSlots(const block_id block) const {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
struct stat file_stat;
if (stat(filename.c_str(), &file_stat) == -1) {
if (errno != ENOENT) {
- LOG_WARNING("Failed to retrieve info about file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to retrieve info about file " << filename << " with error: " << strerror(errno);
}
return 0;
}
@@ -93,12 +94,12 @@ size_t FileManagerPosix::numSlots(const block_id block) const {
}
bool FileManagerPosix::deleteBlockOrBlob(const block_id block) {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
if ((unlink(filename.c_str()) == 0) || (errno == ENOENT)) {
return true;
} else {
- LOG_WARNING("Failed to delete file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to delete file " << filename << " with error: " << strerror(errno);
return false;
}
}
@@ -106,35 +107,35 @@ bool FileManagerPosix::deleteBlockOrBlob(const block_id block) {
bool FileManagerPosix::readBlockOrBlob(const block_id block,
void *buffer,
const std::size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
- int fd = open(filename.c_str(), O_RDONLY);
+ const int fd = open(filename.c_str(), O_RDONLY);
if (fd == -1) {
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
size_t bytes_total = 0;
while (bytes_total < length) {
- ssize_t bytes = read(fd, static_cast<char*>(buffer) + bytes_total, length - bytes_total);
+ const ssize_t bytes = read(fd, static_cast<char*>(buffer) + bytes_total, length - bytes_total);
if (bytes > 0) {
bytes_total += bytes;
} else if (bytes == -1) {
if (errno != EINTR) {
- LOG_WARNING("Failed to read file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to read file " << filename << " with error: " << strerror(errno);
break;
}
} else {
- LOG_WARNING("Failed to read file " << filename << " since EOF was reached unexpectedly");
+ LOG(ERROR) << "Failed to read file " << filename << " since EOF was reached unexpectedly";
break;
}
}
if (close(fd) != 0) {
- LOG_WARNING("Failed to close file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to close file " << filename << " with error: " << strerror(errno);
}
return (bytes_total == length);
@@ -143,34 +144,34 @@ bool FileManagerPosix::readBlockOrBlob(const block_id block,
bool FileManagerPosix::writeBlockOrBlob(const block_id block,
const void *buffer,
const std::size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
- int fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
+ const int fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
if (fd == -1) {
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
size_t bytes_total = 0;
while (bytes_total < length) {
- ssize_t bytes = write(fd, static_cast<const char*>(buffer) + bytes_total, length - bytes_total);
+ const ssize_t bytes = write(fd, static_cast<const char*>(buffer) + bytes_total, length - bytes_total);
if (bytes > 0) {
bytes_total += bytes;
} else if (bytes == -1 && errno != EINTR) {
- LOG_WARNING("Failed to write file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to write file " << filename << " with error: " << strerror(errno);
break;
}
}
if (fsync(fd) != 0) {
- LOG_WARNING("Failed to sync file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to sync file " << filename << " with error: " << strerror(errno);
}
if (close(fd) != 0) {
- LOG_WARNING("Failed to close file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to close file " << filename << " with error: " << strerror(errno);
}
return (bytes_total == length);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/FileManagerPosix.hpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerPosix.hpp b/storage/FileManagerPosix.hpp
index d8c1649..b2aea27 100644
--- a/storage/FileManagerPosix.hpp
+++ b/storage/FileManagerPosix.hpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -37,9 +37,9 @@ namespace quickstep {
class FileManagerPosix : public FileManager {
public:
explicit FileManagerPosix(const std::string &storage_path)
- : FileManager(storage_path) { }
+ : FileManager(storage_path) {}
- ~FileManagerPosix() override { }
+ ~FileManagerPosix() override {}
bool deleteBlockOrBlob(const block_id block) override;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/FileManagerWindows.cpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerWindows.cpp b/storage/FileManagerWindows.cpp
index 9e3d4c8..b1763ec 100644
--- a/storage/FileManagerWindows.cpp
+++ b/storage/FileManagerWindows.cpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -34,9 +34,10 @@
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageConstants.hpp"
#include "storage/StorageErrors.hpp"
-#include "utility/Macros.hpp"
#include "utility/StringUtil.hpp"
+#include "glog/logging.h"
+
using std::size_t;
using std::strerror;
using std::string;
@@ -60,7 +61,7 @@ block_id_counter FileManagerWindows::getMaxUsedBlockCounter(const block_id_domai
if (find_handle == INVALID_HANDLE_VALUE) {
error_code = GetLastError();
if (error_code != ERROR_FILE_NOT_FOUND) {
- LOG_WARNING("Failed to retrieve blockfiles with error_code: " << error_code);
+ LOG(ERROR) << "Failed to retrieve blockfiles with error_code: " << error_code;
}
return 0;
}
@@ -73,32 +74,32 @@ block_id_counter FileManagerWindows::getMaxUsedBlockCounter(const block_id_domai
block_id_counter counter_max = 0, counter;
do {
- if (sscanf(find_data.cFileName, filename_pattern.c_str(), &counter) == 1
- && counter > counter_max) {
+ if (sscanf(find_data.cFileName, filename_pattern.c_str(), &counter) == 1 &&
+ counter > counter_max) {
counter_max = counter;
}
} while (FindNextFile(find_handle, &find_data) != 0);
error_code = GetLastError();
if (error_code != ERROR_NO_MORE_FILES) {
- LOG_WARNING("Failed to FindNextFile with error_code: " << error_code);
+ LOG(ERROR) << "Failed to FindNextFile with error_code: " << error_code;
}
if (FindClose(find_handle) == 0) {
- LOG_WARNING("Failed to close the file with error_code: " << GetLastError());
+ LOG(ERROR) << "Failed to close the file with error_code: " << GetLastError();
}
return counter_max;
}
size_t FileManagerWindows::numSlots(const block_id block) const {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
WIN32_FILE_ATTRIBUTE_DATA file_stat;
if (!GetFileAttributesEx(filename.c_str(), GetFileExInfoStandard, &file_stat)) {
DWORD error_code = GetLastError();
if (error_code != ERROR_FILE_NOT_FOUND) {
- LOG_WARNING("Failed to retrieve info about file " << filename << " with error_code: " << error_code);
+ LOG(ERROR) << "Failed to retrieve info about file " << filename << " with error_code: " << error_code;
}
return 0;
}
@@ -113,28 +114,28 @@ size_t FileManagerWindows::numSlots(const block_id block) const {
}
bool FileManagerWindows::deleteBlockOrBlob(const block_id block) {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
if ((DeleteFile(filename.c_str()) != 0) || (GetLastError() == ERROR_FILE_NOT_FOUND)) {
return true;
} else {
- LOG_WARNING("Failed to delete file " << filename << " with error_code: " << GetLastError());
+ LOG(ERROR) << "Failed to delete file " << filename << " with error_code: " << GetLastError();
}
}
bool FileManagerWindows::readBlockOrBlob(const block_id block,
void *buffer,
const size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
FILE *file = fopen(filename.c_str(), "rb");
if (file == nullptr) {
// Note: On most, but not all, library implementations, the errno variable
// is set to a system-specific error code on failure.
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
@@ -142,16 +143,16 @@ bool FileManagerWindows::readBlockOrBlob(const block_id block,
const bool result_is_ok = (bytes == length);
if (!result_is_ok) {
if (std::feof(file)) {
- LOG_WARNING("Failed to read file " << filename << " since EOF was reached unexpectedly");
+ LOG(ERROR) << "Failed to read file " << filename << " since EOF was reached unexpectedly";
} else {
- LOG_WARNING("Failed to read file " << filename << " with error: " << strerror(ferror(file)));
+ LOG(ERROR) << "Failed to read file " << filename << " with error: " << strerror(ferror(file));
clearerr(file);
}
}
if (fclose(file)) {
// Note: fclose does not set errno on failure.
- LOG_WARNING("Failed to close file " << filename);
+ LOG(ERROR) << "Failed to close file " << filename;
}
return result_is_ok;
@@ -160,37 +161,37 @@ bool FileManagerWindows::readBlockOrBlob(const block_id block,
bool FileManagerWindows::writeBlockOrBlob(const block_id block,
const void *buffer,
const size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
FILE *file = fopen(filename.c_str(), "wb");
if (file == nullptr) {
// Note: On most, but not all, library implementations, the errno variable
// is set to a system-specific error code on failure.
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
const size_t bytes = std::fwrite(buffer, sizeof(char), length, file);
const bool result_is_ok = (bytes == length);
if (!result_is_ok) {
- LOG_WARNING("Failed to write file " << filename << " with error: " << strerror(ferror(file)));
+ LOG(ERROR) << "Failed to write file " << filename << " with error: " << strerror(ferror(file));
clearerr(file);
}
if (fflush(file)) {
- LOG_WARNING("Failed to flush file " << filename << " with error: " << strerror(ferror(file)));
+ LOG(ERROR) << "Failed to flush file " << filename << " with error: " << strerror(ferror(file));
}
if (!FlushFileBuffers(reinterpret_cast<HANDLE>(_get_osfhandle(_fileno(file))))) {
- LOG_WARNING("Failed to re-flush file " << filename << " with error: " << strerror(ferror(file)));
+ LOG(ERROR) << "Failed to re-flush file " << filename << " with error: " << strerror(ferror(file));
}
if (fclose(file)) {
// Note: fclose does not set errno on failure.
- LOG_WARNING("Failed to close file " << filename);
+ LOG(ERROR) << "Failed to close file " << filename;
}
return result_is_ok;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/FileManagerWindows.hpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerWindows.hpp b/storage/FileManagerWindows.hpp
index 085b418..e7b3aa8 100644
--- a/storage/FileManagerWindows.hpp
+++ b/storage/FileManagerWindows.hpp
@@ -37,9 +37,9 @@ namespace quickstep {
class FileManagerWindows : public FileManager {
public:
explicit FileManagerWindows(const std::string &storage_path)
- : FileManager(storage_path) { }
+ : FileManager(storage_path) {}
- ~FileManagerWindows() override { }
+ ~FileManagerWindows() override {}
bool deleteBlockOrBlob(const block_id block) override;
[18/50] [abbrv] incubator-quickstep git commit: Added support for the
substring function. (#211)
Posted by hb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/preprocessed/SqlParser_gen.hpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlParser_gen.hpp b/parser/preprocessed/SqlParser_gen.hpp
index 72fa9ef..71e4332 100644
--- a/parser/preprocessed/SqlParser_gen.hpp
+++ b/parser/preprocessed/SqlParser_gen.hpp
@@ -104,67 +104,69 @@ extern int quickstep_yydebug;
TOKEN_FALSE = 314,
TOKEN_FIRST = 315,
TOKEN_FLOAT = 316,
- TOKEN_FOREIGN = 317,
- TOKEN_FROM = 318,
- TOKEN_FULL = 319,
- TOKEN_GROUP = 320,
- TOKEN_HASH = 321,
- TOKEN_HAVING = 322,
- TOKEN_HOUR = 323,
- TOKEN_IN = 324,
- TOKEN_INDEX = 325,
- TOKEN_INNER = 326,
- TOKEN_INSERT = 327,
- TOKEN_INTEGER = 328,
- TOKEN_INTERVAL = 329,
- TOKEN_INTO = 330,
- TOKEN_JOIN = 331,
- TOKEN_KEY = 332,
- TOKEN_LAST = 333,
- TOKEN_LEFT = 334,
- TOKEN_LIMIT = 335,
- TOKEN_LONG = 336,
- TOKEN_MINUTE = 337,
- TOKEN_MONTH = 338,
- TOKEN_NULL = 339,
- TOKEN_NULLS = 340,
- TOKEN_OFF = 341,
- TOKEN_ON = 342,
- TOKEN_ORDER = 343,
- TOKEN_OUTER = 344,
- TOKEN_PARTITION = 345,
- TOKEN_PARTITIONS = 346,
- TOKEN_PERCENT = 347,
- TOKEN_PRIMARY = 348,
- TOKEN_QUIT = 349,
- TOKEN_RANGE = 350,
- TOKEN_REAL = 351,
- TOKEN_REFERENCES = 352,
- TOKEN_RIGHT = 353,
- TOKEN_ROW_DELIMITER = 354,
- TOKEN_SECOND = 355,
- TOKEN_SELECT = 356,
- TOKEN_SET = 357,
- TOKEN_SMA = 358,
- TOKEN_SMALLINT = 359,
- TOKEN_TABLE = 360,
- TOKEN_THEN = 361,
- TOKEN_TIME = 362,
- TOKEN_TIMESTAMP = 363,
- TOKEN_TRUE = 364,
- TOKEN_TUPLESAMPLE = 365,
- TOKEN_UNIQUE = 366,
- TOKEN_UPDATE = 367,
- TOKEN_USING = 368,
- TOKEN_VALUES = 369,
- TOKEN_VARCHAR = 370,
- TOKEN_WHEN = 371,
- TOKEN_WHERE = 372,
- TOKEN_WITH = 373,
- TOKEN_YEAR = 374,
- TOKEN_YEARMONTH = 375,
- TOKEN_EOF = 376,
- TOKEN_LEX_ERROR = 377
+ TOKEN_FOR = 317,
+ TOKEN_FOREIGN = 318,
+ TOKEN_FROM = 319,
+ TOKEN_FULL = 320,
+ TOKEN_GROUP = 321,
+ TOKEN_HASH = 322,
+ TOKEN_HAVING = 323,
+ TOKEN_HOUR = 324,
+ TOKEN_IN = 325,
+ TOKEN_INDEX = 326,
+ TOKEN_INNER = 327,
+ TOKEN_INSERT = 328,
+ TOKEN_INTEGER = 329,
+ TOKEN_INTERVAL = 330,
+ TOKEN_INTO = 331,
+ TOKEN_JOIN = 332,
+ TOKEN_KEY = 333,
+ TOKEN_LAST = 334,
+ TOKEN_LEFT = 335,
+ TOKEN_LIMIT = 336,
+ TOKEN_LONG = 337,
+ TOKEN_MINUTE = 338,
+ TOKEN_MONTH = 339,
+ TOKEN_NULL = 340,
+ TOKEN_NULLS = 341,
+ TOKEN_OFF = 342,
+ TOKEN_ON = 343,
+ TOKEN_ORDER = 344,
+ TOKEN_OUTER = 345,
+ TOKEN_PARTITION = 346,
+ TOKEN_PARTITIONS = 347,
+ TOKEN_PERCENT = 348,
+ TOKEN_PRIMARY = 349,
+ TOKEN_QUIT = 350,
+ TOKEN_RANGE = 351,
+ TOKEN_REAL = 352,
+ TOKEN_REFERENCES = 353,
+ TOKEN_RIGHT = 354,
+ TOKEN_ROW_DELIMITER = 355,
+ TOKEN_SECOND = 356,
+ TOKEN_SELECT = 357,
+ TOKEN_SET = 358,
+ TOKEN_SMA = 359,
+ TOKEN_SMALLINT = 360,
+ TOKEN_SUBSTRING = 361,
+ TOKEN_TABLE = 362,
+ TOKEN_THEN = 363,
+ TOKEN_TIME = 364,
+ TOKEN_TIMESTAMP = 365,
+ TOKEN_TRUE = 366,
+ TOKEN_TUPLESAMPLE = 367,
+ TOKEN_UNIQUE = 368,
+ TOKEN_UPDATE = 369,
+ TOKEN_USING = 370,
+ TOKEN_VALUES = 371,
+ TOKEN_VARCHAR = 372,
+ TOKEN_WHEN = 373,
+ TOKEN_WHERE = 374,
+ TOKEN_WITH = 375,
+ TOKEN_YEAR = 376,
+ TOKEN_YEARMONTH = 377,
+ TOKEN_EOF = 378,
+ TOKEN_LEX_ERROR = 379
};
#endif
@@ -265,7 +267,7 @@ union YYSTYPE
quickstep::PtrVector<quickstep::ParseSubqueryTableReference> *with_list_;
quickstep::ParseSubqueryTableReference *with_list_element_;
-#line 269 "SqlParser_gen.hpp" /* yacc.c:1915 */
+#line 271 "SqlParser_gen.hpp" /* yacc.c:1915 */
};
typedef union YYSTYPE YYSTYPE;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/query_optimizer/resolver/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/CMakeLists.txt b/query_optimizer/resolver/CMakeLists.txt
index db2a8af..5959879 100644
--- a/query_optimizer/resolver/CMakeLists.txt
+++ b/query_optimizer/resolver/CMakeLists.txt
@@ -120,6 +120,7 @@ target_link_libraries(quickstep_queryoptimizer_resolver_Resolver
quickstep_types_operations_comparisons_ComparisonFactory
quickstep_types_operations_comparisons_ComparisonID
quickstep_types_operations_unaryoperations_DateExtractOperation
+ quickstep_types_operations_unaryoperations_SubstringOperation
quickstep_types_operations_unaryoperations_UnaryOperation
quickstep_utility_Macros
quickstep_utility_PtrList
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index 2667ee9..78985a0 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -115,6 +115,7 @@
#include "types/operations/comparisons/ComparisonFactory.hpp"
#include "types/operations/comparisons/ComparisonID.hpp"
#include "types/operations/unary_operations/DateExtractOperation.hpp"
+#include "types/operations/unary_operations/SubstringOperation.hpp"
#include "types/operations/unary_operations/UnaryOperation.hpp"
#include "utility/PtrList.hpp"
#include "utility/PtrVector.hpp"
@@ -2068,6 +2069,37 @@ E::ScalarPtr Resolver::resolveExpression(
return E::UnaryExpression::Create(op, argument);
}
+ case ParseExpression::kSubstring: {
+ const ParseSubstringFunction &parse_substring =
+ static_cast<const ParseSubstringFunction&>(parse_expression);
+
+ // Validate start position and substring length.
+ if (parse_substring.start_position() <= 0) {
+ THROW_SQL_ERROR_AT(&parse_expression)
+ << "The start position must be greater than 0";
+ }
+ if (parse_substring.length() <= 0) {
+ THROW_SQL_ERROR_AT(&parse_expression)
+ << "The substring length must be greater than 0";
+ }
+
+ // Convert 1-base position to 0-base position
+ const std::size_t zero_base_start_position = parse_substring.start_position() - 1;
+ const SubstringOperation &op =
+ SubstringOperation::Instance(zero_base_start_position,
+ parse_substring.length());
+
+ const E::ScalarPtr argument =
+ resolveExpression(*parse_substring.operand(),
+ op.pushDownTypeHint(type_hint),
+ expression_resolution_info);
+ if (!op.canApplyToType(argument->getValueType())) {
+ THROW_SQL_ERROR_AT(&parse_substring)
+ << "Can not apply substring function to argument of type "
+ << argument->getValueType().getName();
+ }
+ return E::UnaryExpression::Create(op, argument);
+ }
default:
LOG(FATAL) << "Unknown scalar type: "
<< parse_expression.getExpressionType();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index 390b7b6..023ad2e 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -765,6 +765,30 @@ WHERE
+-----------+
==
+# SUBSTRING function
+SELECT char_col,
+ SUBSTRING(char_col FROM 1 FOR 2) AS negative_value
+FROM test
+WHERE SUBSTRING(char_col FROM 1 FOR 1) = '-'
+--
++--------------------+--------------+
+|char_col |negative_value|
++--------------------+--------------+
+| -1 1.000000| -1|
+| -3 1.732051| -3|
+| -5 2.236068| -5|
+| -7 2.645751| -7|
+| -9 3.000000| -9|
+| -11 3.316625| -1|
+| -13 3.605551| -1|
+| -15 3.872983| -1|
+| -17 4.123106| -1|
+| -19 4.358899| -1|
+| -21 4.582576| -2|
+| -23 4.795832| -2|
++--------------------+--------------+
+==
+
# IN predicate
SELECT *
FROM generate_series(1, 5) AS gs(i)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/query_optimizer/tests/resolver/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/resolver/Select.test b/query_optimizer/tests/resolver/Select.test
index 9897934..141bfa0 100644
--- a/query_optimizer/tests/resolver/Select.test
+++ b/query_optimizer/tests/resolver/Select.test
@@ -3048,3 +3048,81 @@ TopLevelPlan
+-output_attributes=
+-AttributeReference[id=5,name=x,relation=,type=Int]
+-AttributeReference[id=6,name=y,relation=,type=Int]
+==
+
+SELECT SUBSTRING(char_col FROM 1 FOR 2)
+FROM test;
+--
+TopLevelPlan
++-plan=Project
+| +-input=TableReference[relation_name=Test,relation_alias=test]
+| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | +-AttributeReference[id=5,name=vchar_col,relation=test,type=VarChar(20) NULL]
+| +-project_list=
+| +-Alias[id=6,name=,alias=SUBSTRING(char_col FROM 1 FOR 2),relation=,
+| type=Char(2)]
+| +-Substring
+| +-Operand=AttributeReference[id=4,name=char_col,relation=test,
+| type=Char(20)]
++-output_attributes=
+ +-AttributeReference[id=6,name=,alias=SUBSTRING(char_col FROM 1 FOR 2),
+ relation=,type=Char(2)]
+==
+
+SELECT *
+FROM test
+WHERE SUBSTRING(vchar_col FROM 1 FOR 2) IN ('12', '34', '56');
+--
+TopLevelPlan
++-plan=Project
+| +-input=Filter
+| | +-input=TableReference[relation_name=Test,relation_alias=test]
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | type=VarChar(20) NULL]
+| | +-filter_predicate=InValueList
+| | +-test_expression=Substring
+| | | +-Operand=AttributeReference[id=5,name=vchar_col,relation=test,
+| | | type=VarChar(20) NULL]
+| | +-match_expressions=
+| | +-Literal[value=12,type=Char(2)]
+| | +-Literal[value=34,type=Char(2)]
+| | +-Literal[value=56,type=Char(2)]
+| +-project_list=
+| +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| +-AttributeReference[id=5,name=vchar_col,relation=test,type=VarChar(20) NULL]
++-output_attributes=
+ +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+ +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+ +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+ +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+ +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+ +-AttributeReference[id=5,name=vchar_col,relation=test,type=VarChar(20) NULL]
+==
+
+SELECT SUBSTRING(char_col FROM 0 FOR 2)
+FROM test;
+--
+ERROR: The start position must be greater than 0 (1 : 8)
+SELECT SUBSTRING(char_col FROM 0 FOR ...
+ ^
+==
+
+SELECT SUBSTRING(char_col FROM 1 FOR 0)
+FROM test;
+--
+ERROR: The substring length must be greater than 0 (1 : 8)
+SELECT SUBSTRING(char_col FROM 1 FOR ...
+ ^
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/Operation.proto
----------------------------------------------------------------------
diff --git a/types/operations/Operation.proto b/types/operations/Operation.proto
index fb44b1d..33e6b09 100644
--- a/types/operations/Operation.proto
+++ b/types/operations/Operation.proto
@@ -43,6 +43,7 @@ message UnaryOperation {
NEGATE = 0;
CAST = 1;
DATE_EXTRACT = 2;
+ SUBSTRING = 3;
}
required UnaryOperationID operation_id = 1;
@@ -73,6 +74,14 @@ message DateExtractOperation {
}
}
+message SubstringOperation {
+ extend UnaryOperation {
+ // Required when operation_id = SUBSTRING.
+ optional int64 start_position = 100;
+ optional int64 substring_length = 101;
+ }
+}
+
message BinaryOperation {
enum BinaryOperationID {
ADD = 0;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/CMakeLists.txt b/types/operations/unary_operations/CMakeLists.txt
index 3a50445..5c54d9c 100644
--- a/types/operations/unary_operations/CMakeLists.txt
+++ b/types/operations/unary_operations/CMakeLists.txt
@@ -1,5 +1,7 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -18,6 +20,7 @@ add_library(quickstep_types_operations_unaryoperations_ArithmeticUnaryOperations
add_library(quickstep_types_operations_unaryoperations_ArithmeticUnaryOperators ../../../empty_src.cpp ArithmeticUnaryOperators.hpp)
add_library(quickstep_types_operations_unaryoperations_DateExtractOperation DateExtractOperation.cpp DateExtractOperation.hpp)
add_library(quickstep_types_operations_unaryoperations_NumericCastOperation ../../../empty_src.cpp NumericCastOperation.hpp)
+add_library(quickstep_types_operations_unaryoperations_SubstringOperation SubstringOperation.cpp SubstringOperation.hpp)
add_library(quickstep_types_operations_unaryoperations_UnaryOperation UnaryOperation.cpp UnaryOperation.hpp)
add_library(quickstep_types_operations_unaryoperations_UnaryOperationFactory UnaryOperationFactory.cpp UnaryOperationFactory.hpp)
add_library(quickstep_types_operations_unaryoperations_UnaryOperationID UnaryOperationID.cpp UnaryOperationID.hpp)
@@ -85,6 +88,23 @@ target_link_libraries(quickstep_types_operations_unaryoperations_NumericCastOper
quickstep_types_operations_unaryoperations_UnaryOperationID
quickstep_utility_Macros
quickstep_utility_PtrMap)
+target_link_libraries(quickstep_types_operations_unaryoperations_SubstringOperation
+ quickstep_catalog_CatalogTypedefs
+ quickstep_storage_ValueAccessor
+ quickstep_storage_ValueAccessorUtil
+ quickstep_types_Type
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ quickstep_types_TypedValue
+ quickstep_types_containers_ColumnVector
+ quickstep_types_containers_ColumnVectorUtil
+ quickstep_types_operations_Operation_proto
+ quickstep_types_operations_unaryoperations_UnaryOperation
+ quickstep_types_operations_unaryoperations_UnaryOperationID
+ quickstep_types_port_strnlen
+ quickstep_utility_HashPair
+ quickstep_utility_Macros
+ quickstep_utility_TemplateUtil)
target_link_libraries(quickstep_types_operations_unaryoperations_UnaryOperation
quickstep_catalog_CatalogTypedefs
quickstep_storage_StorageBlockInfo
@@ -100,6 +120,7 @@ target_link_libraries(quickstep_types_operations_unaryoperations_UnaryOperationF
quickstep_types_operations_unaryoperations_ArithmeticUnaryOperations
quickstep_types_operations_unaryoperations_DateExtractOperation
quickstep_types_operations_unaryoperations_NumericCastOperation
+ quickstep_types_operations_unaryoperations_SubstringOperation
quickstep_types_operations_unaryoperations_UnaryOperationID
quickstep_utility_Macros)
@@ -110,6 +131,7 @@ target_link_libraries(quickstep_types_operations_unaryoperations
quickstep_types_operations_unaryoperations_ArithmeticUnaryOperators
quickstep_types_operations_unaryoperations_DateExtractOperation
quickstep_types_operations_unaryoperations_NumericCastOperation
+ quickstep_types_operations_unaryoperations_SubstringOperation
quickstep_types_operations_unaryoperations_UnaryOperation
quickstep_types_operations_unaryoperations_UnaryOperationFactory
quickstep_types_operations_unaryoperations_UnaryOperationID)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/SubstringOperation.cpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/SubstringOperation.cpp b/types/operations/unary_operations/SubstringOperation.cpp
new file mode 100644
index 0000000..463cd33
--- /dev/null
+++ b/types/operations/unary_operations/SubstringOperation.cpp
@@ -0,0 +1,214 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "types/operations/unary_operations/SubstringOperation.hpp"
+
+#include <algorithm>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "storage/ValueAccessor.hpp"
+#include "storage/ValueAccessorUtil.hpp"
+#include "types/Type.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "types/containers/ColumnVector.hpp"
+#include "types/containers/ColumnVectorUtil.hpp"
+#include "types/operations/Operation.pb.h"
+#include "types/port/strnlen.hpp"
+#include "utility/TemplateUtil.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+serialization::UnaryOperation SubstringOperation::getProto() const {
+ serialization::UnaryOperation proto;
+ proto.set_operation_id(serialization::UnaryOperation::SUBSTRING);
+ proto.SetExtension(serialization::SubstringOperation::start_position,
+ start_position_);
+ proto.SetExtension(serialization::SubstringOperation::substring_length,
+ substring_length_);
+ return proto;
+}
+
+UncheckedUnaryOperator* SubstringOperation::makeUncheckedUnaryOperatorForType(
+ const Type &type) const {
+ DCHECK(type.getSuperTypeID() == Type::kAsciiString);
+
+ const std::size_t input_maximum_length =
+ static_cast<const AsciiStringSuperType&>(type).getStringLength();
+ const bool input_null_terminated = (type.getTypeID() == TypeID::kVarChar);
+
+ const Type *result_type = resultTypeForArgumentType(type);
+ DCHECK(result_type != nullptr);
+
+ return CreateBoolInstantiatedInstance<SubstringUncheckedOperator, UncheckedUnaryOperator>(
+ std::forward_as_tuple(start_position_,
+ computeMaximumSubstringLength(type),
+ input_maximum_length,
+ *result_type),
+ input_null_terminated, type.isNullable());
+}
+
+template <bool null_terminated, bool input_nullable>
+inline void SubstringUncheckedOperator<null_terminated, input_nullable>
+ ::computeSubstring(const char *input,
+ char *output) const {
+ std::size_t string_length =
+ (null_terminated ? strlen(input) : strnlen(input, maximum_input_length_));
+
+ if (start_position_ >= string_length) {
+ *output = '\0';
+ return;
+ }
+
+ const std::size_t actual_substring_length =
+ std::min(string_length - start_position_, substring_length_);
+ std::memcpy(output, input + start_position_, actual_substring_length);
+
+ if (actual_substring_length < substring_length_) {
+ output[actual_substring_length] = '\0';
+ }
+}
+
+template <bool null_terminated, bool input_nullable>
+TypedValue SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToTypedValue(const TypedValue& argument) const {
+ if (input_nullable && argument.isNull()) {
+ return TypedValue(result_type_.getTypeID());
+ }
+
+ char *output_ptr = static_cast<char*>(std::malloc(substring_length_));
+ computeSubstring(static_cast<const char*>(argument.getOutOfLineData()),
+ output_ptr);
+
+ return TypedValue::CreateWithOwnedData(result_type_.getTypeID(),
+ output_ptr,
+ substring_length_);
+}
+
+template <bool null_terminated, bool input_nullable>
+TypedValue SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToDataPtr(const void *argument) const {
+ if (input_nullable && argument == nullptr) {
+ return TypedValue(result_type_.getTypeID());
+ }
+
+ char *output_ptr = static_cast<char*>(std::malloc(substring_length_));
+ computeSubstring(static_cast<const char*>(argument),
+ output_ptr);
+
+ return TypedValue::CreateWithOwnedData(result_type_.getTypeID(),
+ output_ptr,
+ substring_length_);
+}
+
+template <bool null_terminated, bool input_nullable>
+ColumnVector* SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToColumnVector(const ColumnVector &argument) const {
+ return InvokeOnColumnVector(
+ argument,
+ [&](const auto &column_vector) -> ColumnVector* { // NOLINT(build/c++11)
+ NativeColumnVector *result =
+ new NativeColumnVector(result_type_, column_vector.size());
+
+ for (std::size_t cv_pos = 0;
+ cv_pos < column_vector.size();
+ ++cv_pos) {
+ const char *input_ptr = static_cast<const char *>(
+ column_vector.template getUntypedValue<input_nullable>(cv_pos));
+
+ if (input_nullable && input_ptr == nullptr) {
+ result->appendNullValue();
+ } else {
+ this->computeSubstring(input_ptr,
+ static_cast<char *>(result->getPtrForDirectWrite()));
+ }
+ }
+ return result;
+ });
+}
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
+template <bool null_terminated, bool input_nullable>
+ColumnVector* SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToValueAccessor(ValueAccessor *accessor,
+ const attribute_id argument_attr_id) const {
+ return InvokeOnValueAccessorMaybeTupleIdSequenceAdapter(
+ accessor,
+ [&](auto *accessor) -> ColumnVector* { // NOLINT(build/c++11)
+ NativeColumnVector *result =
+ new NativeColumnVector(result_type_, accessor->getNumTuples());
+
+ accessor->beginIteration();
+ while (accessor->next()) {
+ const char *input_ptr = static_cast<const char *>(
+ accessor->template getUntypedValue<input_nullable>(argument_attr_id));
+
+ if (input_nullable && (input_ptr == nullptr)) {
+ result->appendNullValue();
+ } else {
+ this->computeSubstring(input_ptr,
+ static_cast<char *>(result->getPtrForDirectWrite()));
+ }
+ }
+ return result;
+ });
+}
+#endif
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_JOIN
+template <bool null_terminated, bool input_nullable>
+ColumnVector* SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToValueAccessorForJoin(
+ ValueAccessor *accessor,
+ const bool use_left_relation,
+ const attribute_id argument_attr_id,
+ const std::vector<std::pair<tuple_id, tuple_id>> &joined_tuple_ids) const {
+ return InvokeOnValueAccessorNotAdapter(
+ accessor,
+ [&](auto *accessor) -> ColumnVector* { // NOLINT(build/c++11)
+ NativeColumnVector *result =
+ new NativeColumnVector(result_type_, accessor->getNumTuples());
+
+ for (const std::pair<tuple_id, tuple_id> &joined_pair : joined_tuple_ids) {
+ const char *input_ptr = static_cast<const char *>(
+ accessor->template getUntypedValueAtAbsolutePosition<input_nullable>(
+ argument_attr_id,
+ use_left_relation ? joined_pair.first : joined_pair.second));
+
+ if (input_nullable && input_ptr == nullptr) {
+ result->appendNullValue();
+ } else {
+ this->computeSubstring(input_ptr,
+ static_cast<char *>(result->getPtrForDirectWrite()));
+ }
+ }
+ return result;
+ });
+}
+#endif
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/SubstringOperation.hpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/SubstringOperation.hpp b/types/operations/unary_operations/SubstringOperation.hpp
new file mode 100644
index 0000000..d215eae
--- /dev/null
+++ b/types/operations/unary_operations/SubstringOperation.hpp
@@ -0,0 +1,234 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_TYPES_OPERATIONS_UNARY_OPERATIONS_SUBSTRING_OPERATION_HPP_
+#define QUICKSTEP_TYPES_OPERATIONS_UNARY_OPERATIONS_SUBSTRING_OPERATION_HPP_
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "types/Type.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "types/operations/Operation.pb.h"
+#include "types/operations/unary_operations/UnaryOperation.hpp"
+#include "types/operations/unary_operations/UnaryOperationID.hpp"
+#include "utility/HashPair.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class ColumnVector;
+class ValueAccessor;
+
+/**
+ * @brief Operation that extracts a number of characters from a string
+ * at a given starting position.
+ */
+class SubstringOperation : public UnaryOperation {
+ public:
+ /**
+ * @brief Get a reference to the singleton instance of this Operation for
+ * the given (start_position, substring_length) pair.
+ **/
+ static const SubstringOperation& Instance(const std::size_t start_position,
+ const std::size_t substring_length) {
+ // TODO(jianqiao): This is a temporary solution that creates a new instance
+ // for each distinct pair of start_position and substring_length arguments.
+ // The number of instances may be unbounded if quickstep continuously accepts
+ // queries that call SUBSTRING with different arguments. It still remains to
+ // design a better long-term solution.
+ const auto hash = [](const auto &pair) {
+ return hash_combine_detail::HashCombiner<std::size_t>::CombineHashes(pair.first, pair.second);
+ };
+ static std::unordered_map<std::pair<std::size_t, std::size_t>,
+ std::unique_ptr<const SubstringOperation>,
+ decltype(hash)> instance_map(10, hash);
+
+ const std::pair<std::size_t, std::size_t> key_pair =
+ std::make_pair(start_position, substring_length);
+ auto imit = instance_map.find(key_pair);
+ if (imit != instance_map.end()) {
+ return *imit->second;
+ } else {
+ const SubstringOperation *instance =
+ new SubstringOperation(start_position, substring_length);
+ instance_map.emplace(key_pair,
+ std::unique_ptr<const SubstringOperation>(instance));
+ return *instance;
+ }
+ }
+
+ serialization::UnaryOperation getProto() const override;
+
+ bool canApplyToType(const Type &type) const override {
+ return (type.getSuperTypeID() == Type::kAsciiString);
+ }
+
+ const Type *resultTypeForArgumentType(const Type &type) const override {
+ if (type.getSuperTypeID() == Type::kAsciiString) {
+ // Result is a Char string.
+ return &TypeFactory::GetType(TypeID::kChar,
+ computeMaximumSubstringLength(type),
+ type.isNullable());
+ }
+ return nullptr;
+ }
+
+ const Type* fixedNullableResultType() const override {
+ // Result type is not fixed (i.e. can have various lengths).
+ return nullptr;
+ }
+
+ bool resultTypeIsPlausible(const Type &result_type) const override {
+ // Result can be coerced to Char or VarChar.
+ return (result_type.getSuperTypeID() == Type::kAsciiString);
+ }
+
+ const Type* pushDownTypeHint(const Type *type_hint) const override {
+ // Input can only be a string, but we don't know the length.
+ return nullptr;
+ }
+
+ TypedValue applyToChecked(const TypedValue &argument,
+ const Type &argument_type) const override {
+ DCHECK(canApplyToType(argument_type));
+
+ const Type *result_type = resultTypeForArgumentType(argument_type);
+ DCHECK(result_type != nullptr);
+
+ if (argument_type.isNullable() && argument.isNull()) {
+ return result_type->makeNullValue();
+ } else {
+ const std::size_t result_length = computeMaximumSubstringLength(argument_type);
+ char *output_ptr = static_cast<char*>(std::malloc(result_length));
+ const char *input_ptr = static_cast<const char*>(argument.getOutOfLineData());
+
+ const std::size_t string_length = argument.getAsciiStringLength();
+ if (start_position_ >= string_length) {
+ *output_ptr = '\0';
+ } else {
+ const std::size_t actual_substring_length =
+ std::min(string_length - start_position_, substring_length_);
+ std::memcpy(output_ptr, input_ptr + start_position_, actual_substring_length);
+ if (actual_substring_length < result_length) {
+ output_ptr[actual_substring_length] = '\0';
+ }
+ }
+
+ return TypedValue::CreateWithOwnedData(result_type->getTypeID(),
+ output_ptr,
+ result_length);
+ }
+ }
+
+ UncheckedUnaryOperator* makeUncheckedUnaryOperatorForType(const Type &type) const override;
+
+ private:
+ /**
+ * @brief Constructor.
+ *
+ * @param input_type The data type of the input argument for substring.
+ * @param start_position The 0-base starting position of the substring.
+ * @param substring_length The substring length.
+ */
+ SubstringOperation(const std::size_t start_position,
+ const std::size_t substring_length)
+ : UnaryOperation(UnaryOperationID::kSubstring),
+ start_position_(start_position),
+ substring_length_(substring_length) {
+ }
+
+ /**
+ * @brief Compute an upper bound for the substring length regarding the input
+ * type and the substring_length_ field.
+ *
+ * @param type The type of the input, must be either CharType or VarCharType.
+ */
+ inline std::size_t computeMaximumSubstringLength(const Type& type) const {
+ DCHECK(type.getSuperTypeID() == Type::kAsciiString);
+
+ // Substring result should have length no greater than the minimum of
+ // (1) the input string length subtract the start position, and
+ // (2) the specified substring length.
+ return std::min(static_cast<const AsciiStringSuperType&>(type).getStringLength() - start_position_,
+ substring_length_);
+ }
+
+ const std::size_t start_position_;
+ const std::size_t substring_length_;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(SubstringOperation);
+};
+
+template <bool null_terminated, bool input_nullable>
+class SubstringUncheckedOperator : public UncheckedUnaryOperator {
+ public:
+ SubstringUncheckedOperator(const std::size_t start_position,
+ const std::size_t substring_length,
+ const std::size_t maximum_input_length,
+ const Type &result_type)
+ : start_position_(start_position),
+ substring_length_(substring_length),
+ maximum_input_length_(maximum_input_length),
+ result_type_(result_type) {
+ }
+
+ TypedValue applyToTypedValue(const TypedValue& argument) const override;
+
+ TypedValue applyToDataPtr(const void *argument) const override;
+
+ ColumnVector* applyToColumnVector(const ColumnVector &argument) const override;
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
+ ColumnVector* applyToValueAccessor(ValueAccessor *accessor,
+ const attribute_id argument_attr_id) const override;
+#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_JOIN
+ ColumnVector* applyToValueAccessorForJoin(
+ ValueAccessor *accessor,
+ const bool use_left_relation,
+ const attribute_id argument_attr_id,
+ const std::vector<std::pair<tuple_id, tuple_id>> &joined_tuple_ids) const override;
+#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_JOIN
+
+ private:
+ inline void computeSubstring(const char *input, char *output) const;
+
+ const std::size_t start_position_;
+ const std::size_t substring_length_;
+ const std::size_t maximum_input_length_;
+ const Type &result_type_;
+
+ DISALLOW_COPY_AND_ASSIGN(SubstringUncheckedOperator);
+};
+
+} // namespace quickstep
+
+#endif /* QUICKSTEP_TYPES_OPERATIONS_UNARY_OPERATIONS_SUBSTRING_OPERATION_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/UnaryOperation.cpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/UnaryOperation.cpp b/types/operations/unary_operations/UnaryOperation.cpp
index 0f1f2c7..6721bd6 100644
--- a/types/operations/unary_operations/UnaryOperation.cpp
+++ b/types/operations/unary_operations/UnaryOperation.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -33,6 +35,8 @@ serialization::UnaryOperation UnaryOperation::getProto() const {
FATAL_ERROR("Must use the overridden NumericCastOperation::getProto");
case UnaryOperationID::kDateExtract:
FATAL_ERROR("Must use the overridden DateExtractOperation::getProto");
+ case UnaryOperationID::kSubstring:
+ FATAL_ERROR("Must use the overridden SubstringOperation::getProto");
default:
FATAL_ERROR("Unrecognized UnaryOperationID in UnaryOperation::getProto");
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/UnaryOperationFactory.cpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/UnaryOperationFactory.cpp b/types/operations/unary_operations/UnaryOperationFactory.cpp
index 9d79cb0..13cb76e 100644
--- a/types/operations/unary_operations/UnaryOperationFactory.cpp
+++ b/types/operations/unary_operations/UnaryOperationFactory.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -24,6 +26,7 @@
#include "types/operations/unary_operations/ArithmeticUnaryOperations.hpp"
#include "types/operations/unary_operations/NumericCastOperation.hpp"
#include "types/operations/unary_operations/DateExtractOperation.hpp"
+#include "types/operations/unary_operations/SubstringOperation.hpp"
#include "types/operations/unary_operations/UnaryOperationID.hpp"
#include "utility/Macros.hpp"
@@ -39,6 +42,8 @@ const UnaryOperation& UnaryOperationFactory::GetUnaryOperation(const UnaryOperat
FATAL_ERROR("Getting a CastOperation through GetUnaryOperation is not supported");
case UnaryOperationID::kDateExtract:
FATAL_ERROR("Getting a DateExtractOperation through GetUnaryOperation is not supported");
+ case UnaryOperationID::kSubstring:
+ FATAL_ERROR("Getting a SubstringOperation through GetUnaryOperation is not supported");
default:
FATAL_ERROR("Unknown UnaryOperationID");
}
@@ -64,6 +69,9 @@ bool UnaryOperationFactory::ProtoIsValid(const serialization::UnaryOperation &pr
case serialization::UnaryOperation::DATE_EXTRACT:
return proto.HasExtension(serialization::DateExtractOperation::unit)
&& DateExtractOperation_Unit_IsValid(proto.GetExtension(serialization::DateExtractOperation::unit));
+ case serialization::UnaryOperation::SUBSTRING:
+ return proto.HasExtension(serialization::SubstringOperation::start_position)
+ && proto.HasExtension(serialization::SubstringOperation::substring_length);
default:
return false;
}
@@ -100,6 +108,10 @@ const UnaryOperation& UnaryOperationFactory::ReconstructFromProto(
default:
FATAL_ERROR("Unrecognized DateExtractOperation unit in UnaryOperation::ReconstructFromProto");
}
+ case serialization::UnaryOperation::SUBSTRING:
+ return SubstringOperation::Instance(
+ proto.GetExtension(serialization::SubstringOperation::start_position),
+ proto.GetExtension(serialization::SubstringOperation::substring_length));
default:
FATAL_ERROR("Unrecognized UnaryOperationID in UnaryOperation::ReconstructFromProto");
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/UnaryOperationID.cpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/UnaryOperationID.cpp b/types/operations/unary_operations/UnaryOperationID.cpp
index c45a0a9..dae64d3 100644
--- a/types/operations/unary_operations/UnaryOperationID.cpp
+++ b/types/operations/unary_operations/UnaryOperationID.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,11 +22,11 @@
namespace quickstep {
const char *kUnaryOperationNames[] = {
- "Negate", "Cast", "DateExtract"
+ "Negate", "Cast", "DateExtract", "Substring"
};
const char *kUnaryOperationShortNames[] = {
- "-", "Cast", "DateExtract"
+ "-", "Cast", "DateExtract", "Substring"
};
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/UnaryOperationID.hpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/UnaryOperationID.hpp b/types/operations/unary_operations/UnaryOperationID.hpp
index ba3e0ef..9cbb6e4 100644
--- a/types/operations/unary_operations/UnaryOperationID.hpp
+++ b/types/operations/unary_operations/UnaryOperationID.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -33,6 +35,7 @@ enum class UnaryOperationID {
kNegate = 0,
kCast,
kDateExtract,
+ kSubstring,
kNumUnaryOperationIDs // Not a real UnaryOperationID, exists for counting purposes.
};
[03/50] [abbrv] incubator-quickstep git commit: Refactored
makeRoomForBlock. (#192)
Posted by hb...@apache.org.
Refactored makeRoomForBlock. (#192)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3bd1586e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3bd1586e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3bd1586e
Branch: refs/heads/query-manager-used-in-foreman
Commit: 3bd1586ecfa343f467398892a8887b3ea3c7a5f5
Parents: 7ac1d22
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Wed Apr 27 23:00:23 2016 -0700
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Thu Apr 28 01:00:23 2016 -0500
----------------------------------------------------------------------
storage/StorageManager.cpp | 89 ++++++++++++++++++++---------------------
storage/StorageManager.hpp | 10 +++--
2 files changed, 50 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3bd1586e/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index b98a28c..b990a22 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -397,7 +397,7 @@ void* StorageManager::allocateSlots(const std::size_t num_slots,
= MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED_SUPER;
#endif
- makeRoomForBlock(num_slots);
+ makeRoomForBlockOrBlob(num_slots);
void *slots = nullptr;
#if defined(QUICKSTEP_HAVE_MMAP_LINUX_HUGETLB) || defined(QUICKSTEP_HAVE_MMAP_BSD_SUPERPAGE)
@@ -568,65 +568,62 @@ MutableBlobReference StorageManager::getBlobInternal(const block_id blob,
return ret;
}
-void StorageManager::makeRoomForBlock(const size_t slots) {
+void StorageManager::makeRoomForBlockOrBlob(const size_t slots) {
+ block_id block_to_evict;
while (total_memory_usage_ + slots > max_memory_usage_) {
- block_id block_index;
- EvictionPolicy::Status status = eviction_policy_->chooseBlockToEvict(&block_index);
-
- if (status == EvictionPolicy::Status::kOk) {
- bool has_collision = false;
- SpinSharedMutexExclusiveLock<false> eviction_lock(*lock_manager_.get(block_index, &has_collision));
- if (has_collision) {
- // We have a collision in the shared lock manager, where some callers
- // of this function (i.e., getBlockInternal or getBlobInternal) has
- // acquired an exclusive lock, and we are trying to evict a block that
- // hashes to the same location. This will cause a deadlock.
-
- // For now simply treat this situation as the case where there is not
- // enough memory and we temporarily go over the memory limit.
- break;
- }
+ const EvictionPolicy::Status status = eviction_policy_->chooseBlockToEvict(&block_to_evict);
+ if (status != EvictionPolicy::Status::kOk) {
+ // If status was not ok, then we must not have been able to evict enough
+ // blocks; therefore, we return anyway, temporarily going over the memory
+ // limit.
+ break;
+ }
- StorageBlockBase* block;
- {
- SpinSharedMutexSharedLock<false> read_lock(blocks_shared_mutex_);
- if (blocks_.find(block_index) == blocks_.end()) {
- // another thread must have jumped in and evicted it before us
-
- // NOTE(zuyu): It is ok to release the shard for a block or blob,
- // before 'eviction_lock' destructs, because we will never encounter a
- // self-deadlock in a single thread, and in multiple-thread case some
- // thread will block but not deadlock if there is a shard collision.
- lock_manager_.release(block_index);
- continue;
- }
- block = blocks_[block_index].block;
- }
- if (eviction_policy_->getRefCount(block->getID()) > 0) {
- // Someone sneaked in and referenced the block before we could evict it.
+ bool has_collision = false;
+ SpinSharedMutexExclusiveLock<false> eviction_lock(*lock_manager_.get(block_to_evict, &has_collision));
+ if (has_collision) {
+ // We have a collision in the shared lock manager, where some callers
+ // of this function (i.e., getBlockInternal or getBlobInternal) has
+ // acquired an exclusive lock, and we are trying to evict a block that
+ // hashes to the same location. This will cause a deadlock.
+
+ // For now simply treat this situation as the case where there is not
+ // enough memory and we temporarily go over the memory limit.
+ break;
+ }
- // NOTE(zuyu): It is ok to release the shard for a block or blob, before
+ {
+ SpinSharedMutexSharedLock<false> read_lock(blocks_shared_mutex_);
+ if (blocks_.find(block_to_evict) == blocks_.end()) {
+ // another thread must have jumped in and evicted it before us
+
+ // NOTE(zuyu): It is ok to release the shard for a block or blob,
// before 'eviction_lock' destructs, because we will never encounter a
// self-deadlock in a single thread, and in multiple-thread case some
// thread will block but not deadlock if there is a shard collision.
- lock_manager_.release(block_index);
+ lock_manager_.release(block_to_evict);
continue;
}
- if (saveBlockOrBlob(block->getID())) {
- evictBlockOrBlob(block->getID());
- } // else : Someone sneaked in and evicted the block before we could.
+ }
+ if (eviction_policy_->getRefCount(block_to_evict) > 0) {
+ // Someone sneaked in and referenced the block before we could evict it.
// NOTE(zuyu): It is ok to release the shard for a block or blob, before
// before 'eviction_lock' destructs, because we will never encounter a
// self-deadlock in a single thread, and in multiple-thread case some
// thread will block but not deadlock if there is a shard collision.
- lock_manager_.release(block_index);
- } else {
- // If status was not ok, then we must not have been able to evict enough
- // blocks; therefore, we return anyway, temporarily going over the memory
- // limit.
- break;
+ lock_manager_.release(block_to_evict);
+ continue;
}
+ if (saveBlockOrBlob(block_to_evict)) {
+ evictBlockOrBlob(block_to_evict);
+ } // else : Someone sneaked in and evicted the block before we could.
+
+ // NOTE(zuyu): It is ok to release the shard for a block or blob, before
+ // before 'eviction_lock' destructs, because we will never encounter a
+ // self-deadlock in a single thread, and in multiple-thread case some
+ // thread will block but not deadlock if there is a shard collision.
+ lock_manager_.release(block_to_evict);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3bd1586e/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index dab33f6..0b68b76 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -402,12 +402,16 @@ class StorageManager {
const int numa_node);
/**
- * @brief Evict blocks until there is enough space for a new block of the
- * requested size.
+ * @brief Evict blocks or blobs until there is enough space for a new block
+ * or blob of the requested size.
+ *
+ * @note This non-blocking method gives up evictions if there is a shard
+ * collision, and thus the buffer pool size may temporarily go beyond
+ * the memory limit.
*
* @param slots Number of slots to make room for.
*/
- void makeRoomForBlock(const std::size_t slots);
+ void makeRoomForBlockOrBlob(const std::size_t slots);
/**
* @brief Load a block from the persistent storage into memory.
[28/50] [abbrv] incubator-quickstep git commit: Fix bug in the SMA
code (#223)
Posted by hb...@apache.org.
Fix bug in the SMA code (#223)
* Fix bug in the SMA code so that the SMA predicate evaluation is only
applied if at least one of the operands in the predicate is a
static value.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1fa81a8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1fa81a8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1fa81a8b
Branch: refs/heads/query-manager-used-in-foreman
Commit: 1fa81a8bd2efc7e7638fb0952c422c20dd9ce2fa
Parents: ba25b13
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Mon May 16 12:04:32 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:27 2016 -0700
----------------------------------------------------------------------
storage/SMAIndexSubBlock.cpp | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1fa81a8b/storage/SMAIndexSubBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/SMAIndexSubBlock.cpp b/storage/SMAIndexSubBlock.cpp
index 2a0e4f9..aa9bc54 100644
--- a/storage/SMAIndexSubBlock.cpp
+++ b/storage/SMAIndexSubBlock.cpp
@@ -621,9 +621,14 @@ Selectivity SMAIndexSubBlock::getSelectivityForPredicate(const ComparisonPredica
predicate_cost_t SMAIndexSubBlock::estimatePredicateEvaluationCost(
const ComparisonPredicate &predicate) const {
DCHECK(initialized_);
- Selectivity selectivity = getSelectivityForPredicate(predicate);
- if (selectivity == Selectivity::kAll || selectivity == Selectivity::kNone) {
- return predicate_cost::kConstantTime;
+
+ // Check that at least one of the operands has a static value.
+ if (predicate.getLeftOperand().hasStaticValue() ||
+ predicate.getRightOperand().hasStaticValue()) {
+ Selectivity selectivity = getSelectivityForPredicate(predicate);
+ if (selectivity == Selectivity::kAll || selectivity == Selectivity::kNone) {
+ return predicate_cost::kConstantTime;
+ }
}
return predicate_cost::kInfinite;
}
[04/50] [abbrv] incubator-quickstep git commit: Fixes loss of alias
problem with PullUpProjectExpressions (#197)
Posted by hb...@apache.org.
Fixes loss of alias problem with PullUpProjectExpressions (#197)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/07ca1e73
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/07ca1e73
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/07ca1e73
Branch: refs/heads/query-manager-used-in-foreman
Commit: 07ca1e73ebce7b9df32af2b3c7721df5941f25a9
Parents: 3bd1586
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu Apr 28 01:00:50 2016 -0500
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Thu Apr 28 01:00:50 2016 -0500
----------------------------------------------------------------------
query_optimizer/rules/RuleHelper.cpp | 18 +++++++++++-------
.../tests/execution_generator/Select.test | 20 ++++++++++++++++++++
.../tests/physical_generator/Select.test | 16 ++++++++++------
3 files changed, 41 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/07ca1e73/query_optimizer/rules/RuleHelper.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/RuleHelper.cpp b/query_optimizer/rules/RuleHelper.cpp
index ce9ef9a..a273004 100644
--- a/query_optimizer/rules/RuleHelper.cpp
+++ b/query_optimizer/rules/RuleHelper.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -75,19 +77,21 @@ void PullUpProjectExpressions(
E::ExpressionPtr updated_expression = rule.apply(expression_to_update);
E::NamedExpressionPtr updated_named_expression;
- // If the updated expression is not a NamedExpression or we have changed
- // the ID, restore the ID by adding an Alias.
- if (!E::SomeNamedExpression::MatchesWithConditionalCast(updated_expression,
- &updated_named_expression) ||
- expression_to_update->id() != updated_named_expression->id()) {
+ if (E::SomeNamedExpression::MatchesWithConditionalCast(updated_expression,
+ &updated_named_expression) &&
+ expression_to_update->id() == updated_named_expression->id() &&
+ expression_to_update->attribute_alias() == updated_named_expression->attribute_alias()) {
+ // Pull up directly if the updated expression has the same ExprId and
+ // attribute_alias as the original project expression.
+ (*project_expressions)[i] = updated_named_expression;
+ } else {
+ // Otherwise create an Alias to wrap the updated expression.
(*project_expressions)[i] =
E::Alias::Create(expression_to_update->id(),
updated_expression,
expression_to_update->attribute_name(),
expression_to_update->attribute_alias(),
expression_to_update->relation_name());
- } else {
- (*project_expressions)[i] = updated_named_expression;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/07ca1e73/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index 438546c..3a64c9b 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -871,6 +871,26 @@ FROM generate_series(1, 5) AS gs(i);
+----------------------+
==
+# This query is to test that the output columns have the correct alias name as specified.
+SELECT *
+FROM (
+ SELECT i, SUM(i) AS sum
+ FROM generate_series(1, 2) AS gs(i)
+ GROUP BY i
+) t1 JOIN (
+ SELECT j, AVG(j) AS avg
+ FROM generate_series(1, 2) AS gs(j)
+ GROUP BY j
+) t2 ON i = j;
+--
++-----------+--------------------+-----------+------------------------+
+|i |sum |j |avg |
++-----------+--------------------+-----------+------------------------+
+| 1| 1| 1| 1|
+| 2| 2| 2| 2|
++-----------+--------------------+-----------+------------------------+
+==
+
# TODO(team): Fix Issue #9 to enable COUNT(*).
SELECT COUNT(long_col)
FROM test,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/07ca1e73/query_optimizer/tests/physical_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/physical_generator/Select.test b/query_optimizer/tests/physical_generator/Select.test
index b405bc9..62d09f5 100644
--- a/query_optimizer/tests/physical_generator/Select.test
+++ b/query_optimizer/tests/physical_generator/Select.test
@@ -1166,7 +1166,8 @@ TopLevelPlan
| | | | +-Add
| | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
| | | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
-| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-Alias[id=1,name=col1,relation=,type=Long]
+| | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
| | | +-Alias[id=8,name=,alias=$groupby2,relation=$groupby,type=Float]
| | | +-Add
| | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
@@ -1304,7 +1305,8 @@ TopLevelPlan
| | | +-Literal[value=2,type=Int]
| | +-grouping_expressions=
| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
-| | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | +-Alias[id=4,name=subquery_col3,relation=subquery,type=Char(20)]
+| | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
| | +-aggregate_expressions=
| | +-Alias[id=8,name=,alias=$aggregate0,relation=$aggregate,type=Long]
| | | +-AggregateFunction[function=COUNT]
@@ -2609,10 +2611,12 @@ TopLevelPlan
| | | +-AttributeReference[id=5,name=y,relation=d,type=Int]
| | +-join_predicate=Literal[value=true]
| | +-project_expressions=
-| | +-AttributeReference[id=4,name=,alias=$aggregate0,relation=$aggregate,
-| | | type=Long NULL]
-| | +-AttributeReference[id=7,name=,alias=$aggregate0,relation=$aggregate,
-| | type=Double NULL]
+| | +-Alias[id=4,name=,alias=SUM(y),relation=,type=Long NULL]
+| | | +-AttributeReference[id=4,name=,alias=$aggregate0,relation=$aggregate,
+| | | type=Long NULL]
+| | +-Alias[id=7,name=,alias=AVG(y),relation=,type=Double NULL]
+| | +-AttributeReference[id=7,name=,alias=$aggregate0,relation=$aggregate,
+| | type=Double NULL]
| +-join_predicate=Literal[value=true]
| +-project_expressions=
| +-Alias[id=8,name=,alias=((x*SubqueryExpression)+SubqueryExpression),
[24/50] [abbrv] incubator-quickstep git commit: Leave 10% more memory
for the OS. (#218)
Posted by hb...@apache.org.
Leave 10% more memory for the OS. (#218)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/6f184957
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/6f184957
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/6f184957
Branch: refs/heads/query-manager-used-in-foreman
Commit: 6f1849575e25a1ca4cc6d38898918cff7207ddf7
Parents: 4937e68
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Sun May 8 22:41:17 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:32 2016 -0700
----------------------------------------------------------------------
storage/StorageConstants.hpp | 4 ++--
storage/StorageManager.cpp | 6 +++---
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6f184957/storage/StorageConstants.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageConstants.hpp b/storage/StorageConstants.hpp
index 154d2f7..0609cba 100644
--- a/storage/StorageConstants.hpp
+++ b/storage/StorageConstants.hpp
@@ -58,8 +58,8 @@ const std::uint64_t kLargeMemorySystemThresholdInGB = 32;
// we grab a bigger fraction of the installed memory than for a "small" system.
// The two constants below define the percentages to grab in each case.
// TODO(jmp): May need to generalize this to more than two levels in the future.
-const std::uint64_t kPercentageToGrabForSmallSystems = 80;
-const std::uint64_t kPercentageToGrabForLargeSystems = 90;
+const std::uint64_t kPercentageToGrabForSmallSystems = 70;
+const std::uint64_t kPercentageToGrabForLargeSystems = 80;
// The default size of the buffer pool (in terms of the number of slots).
const std::uint64_t kDefaultBufferPoolSizeInSlots = 1024;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6f184957/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index b990a22..dfc95b8 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -100,8 +100,8 @@ static const volatile bool block_domain_dummy
/**
* @brief Set or validate the buffer pool slots. When automatically picking a
* default value, check if the system is "small" or "large." Set the
- * buffer pool space to 80% of the installed main memory for small
- * and 90% otherwise.
+ * buffer pool space to 70% of the installed main memory for small
+ * and 80% otherwise.
* This method follows the signature that is set by the gflags module.
* @param flagname The name of the buffer pool flag.
* @param value The value of this flag from the command line, or default (0)
@@ -142,7 +142,7 @@ static bool SetOrValidateBufferPoolSlots(const char *flagname,
DEFINE_uint64(buffer_pool_slots, 0,
"By default the value is 0 and the system automatically sets the "
- "buffer pool size/slots at 80-90% of the total installed memory. "
+ "buffer pool size/slots at 70-80% of the total installed memory. "
"The user can also explicity define the number of slots. "
"The units for this variable is the number of 2-megabyte slots "
"that is allocated in the buffer pool. This is a \"soft\" limit: "
[23/50] [abbrv] incubator-quickstep git commit: Enable semi-join
optimization for left-deep trees through bloom filters (#195)
Posted by hb...@apache.org.
Enable semi-join optimization for left-deep trees through bloom filters (#195)
Link: https://github.com/pivotalsoftware/quickstep/pull/195
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/21b85088
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/21b85088
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/21b85088
Branch: refs/heads/query-manager-used-in-foreman
Commit: 21b85088f5f2b177ade45d3bc86612501b7a92d4
Parents: 6f18495
Author: Saket Saurabh <sa...@users.noreply.github.com>
Authored: Mon May 9 12:29:47 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:32 2016 -0700
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 2 +
query_execution/QueryContext.cpp | 16 +-
query_execution/QueryContext.hpp | 57 +++-
query_execution/QueryContext.proto | 18 +-
query_optimizer/CMakeLists.txt | 11 +
query_optimizer/ExecutionGenerator.cpp | 55 +++-
query_optimizer/ExecutionGenerator.hpp | 5 +-
query_optimizer/ExecutionHeuristics.cpp | 127 ++++++++
query_optimizer/ExecutionHeuristics.hpp | 155 ++++++++++
query_optimizer/tests/CMakeLists.txt | 16 +
.../tests/ExecutionHeuristics_unittest.cpp | 301 +++++++++++++++++++
storage/CMakeLists.txt | 4 +-
storage/HashTable.hpp | 104 +++++++
storage/HashTable.proto | 8 +
storage/HashTableFactory.hpp | 44 ++-
utility/BloomFilter.hpp | 198 +++++++++++-
utility/BloomFilter.proto | 30 ++
utility/CMakeLists.txt | 15 +
18 files changed, 1135 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 5887237..04a0348 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -90,9 +90,11 @@ target_link_libraries(quickstep_queryexecution_QueryContext
quickstep_storage_InsertDestination_proto
quickstep_types_TypedValue
quickstep_types_containers_Tuple
+ quickstep_utility_BloomFilter
quickstep_utility_Macros
quickstep_utility_SortConfiguration)
target_link_libraries(quickstep_queryexecution_QueryContext_proto
+ quickstep_utility_BloomFilter_proto
quickstep_expressions_Expressions_proto
quickstep_expressions_tablegenerator_GeneratorFunction_proto
quickstep_storage_AggregationOperationState_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_execution/QueryContext.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.cpp b/query_execution/QueryContext.cpp
index b0e9cae..3bfce17 100644
--- a/query_execution/QueryContext.cpp
+++ b/query_execution/QueryContext.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -37,6 +39,7 @@
#include "storage/InsertDestination.pb.h"
#include "types/TypedValue.hpp"
#include "types/containers/Tuple.hpp"
+#include "utility/BloomFilter.hpp"
#include "utility/SortConfiguration.hpp"
#include "glog/logging.h"
@@ -65,6 +68,10 @@ QueryContext::QueryContext(const serialization::QueryContext &proto,
storage_manager));
}
+ for (int i = 0; i < proto.bloom_filters_size(); ++i) {
+ bloom_filters_.emplace_back(new BloomFilter(proto.bloom_filters(i)));
+ }
+
for (int i = 0; i < proto.generator_functions_size(); ++i) {
const GeneratorFunctionHandle *func_handle =
GeneratorFunctionFactory::Instance().reconstructFromProto(proto.generator_functions(i));
@@ -76,7 +83,8 @@ QueryContext::QueryContext(const serialization::QueryContext &proto,
for (int i = 0; i < proto.join_hash_tables_size(); ++i) {
join_hash_tables_.emplace_back(
JoinHashTableFactory::CreateResizableFromProto(proto.join_hash_tables(i),
- storage_manager));
+ storage_manager,
+ bloom_filters_));
}
for (int i = 0; i < proto.insert_destinations_size(); ++i) {
@@ -142,6 +150,12 @@ bool QueryContext::ProtoIsValid(const serialization::QueryContext &proto,
}
}
+ for (int i = 0; i < proto.bloom_filters_size(); ++i) {
+ if (!BloomFilter::ProtoIsValid(proto.bloom_filters(i))) {
+ return false;
+ }
+ }
+
// Each GeneratorFunctionHandle object is serialized as a function name with
// a list of arguments. Here checks that the arguments are valid TypedValue's.
for (int i = 0; i < proto.generator_functions_size(); ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_execution/QueryContext.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp
index 0e9e21c..9440fae 100644
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -32,6 +34,7 @@
#include "storage/HashTable.hpp"
#include "storage/InsertDestination.hpp"
#include "types/containers/Tuple.hpp"
+#include "utility/BloomFilter.hpp"
#include "utility/Macros.hpp"
#include "utility/SortConfiguration.hpp"
@@ -63,6 +66,11 @@ class QueryContext {
typedef std::uint32_t aggregation_state_id;
/**
+ * @brief A unique identifier for a BloomFilter per query.
+ **/
+ typedef std::uint32_t bloom_filter_id;
+
+ /**
* @brief A unique identifier for a GeneratorFunctionHandle per query.
**/
typedef std::uint32_t generator_function_id;
@@ -181,6 +189,52 @@ class QueryContext {
}
/**
+ * @brief Whether the given BloomFilter id is valid.
+ *
+ * @param id The BloomFilter id.
+ *
+ * @return True if valid, otherwise false.
+ **/
+ bool isValidBloomFilterId(const bloom_filter_id id) const {
+ return id < bloom_filters_.size();
+ }
+
+ /**
+ * @brief Get a mutable reference to the BloomFilter.
+ *
+ * @param id The BloomFilter id.
+ *
+ * @return The BloomFilter, already created in the constructor.
+ **/
+ inline BloomFilter* getBloomFilterMutable(const bloom_filter_id id) {
+ DCHECK_LT(id, bloom_filters_.size());
+ return bloom_filters_[id].get();
+ }
+
+ /**
+ * @brief Get a constant pointer to the BloomFilter.
+ *
+ * @param id The BloomFilter id.
+ *
+ * @return The constant pointer to BloomFilter that is
+ * already created in the constructor.
+ **/
+ inline const BloomFilter* getBloomFilter(const bloom_filter_id id) const {
+ DCHECK_LT(id, bloom_filters_.size());
+ return bloom_filters_[id].get();
+ }
+
+ /**
+ * @brief Destory the given BloomFilter.
+ *
+ * @param id The id of the BloomFilter to destroy.
+ **/
+ inline void destroyBloomFilter(const bloom_filter_id id) {
+ DCHECK_LT(id, bloom_filters_.size());
+ bloom_filters_[id].reset();
+ }
+
+ /**
* @brief Whether the given GeneratorFunctionHandle id is valid.
*
* @param id The GeneratorFunctionHandle id.
@@ -257,7 +311,7 @@ class QueryContext {
*
* @param id The JoinHashTable id in the query.
*
- * @return The JoinHashTable, alreadly created in the constructor.
+ * @return The JoinHashTable, already created in the constructor.
**/
inline JoinHashTable* getJoinHashTable(const join_hash_table_id id) {
DCHECK_LT(id, join_hash_tables_.size());
@@ -408,6 +462,7 @@ class QueryContext {
private:
std::vector<std::unique_ptr<AggregationOperationState>> aggregation_states_;
+ std::vector<std::unique_ptr<BloomFilter>> bloom_filters_;
std::vector<std::unique_ptr<const GeneratorFunctionHandle>> generator_functions_;
std::vector<std::unique_ptr<InsertDestination>> insert_destinations_;
std::vector<std::unique_ptr<JoinHashTable>> join_hash_tables_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_execution/QueryContext.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.proto b/query_execution/QueryContext.proto
index a7c2a25..b37286c 100644
--- a/query_execution/QueryContext.proto
+++ b/query_execution/QueryContext.proto
@@ -23,6 +23,7 @@ import "storage/AggregationOperationState.proto";
import "storage/HashTable.proto";
import "storage/InsertDestination.proto";
import "types/containers/Tuple.proto";
+import "utility/BloomFilter.proto";
import "utility/SortConfiguration.proto";
message QueryContext {
@@ -42,14 +43,15 @@ message QueryContext {
}
repeated AggregationOperationState aggregation_states = 1;
- repeated HashTable join_hash_tables = 2;
- repeated InsertDestination insert_destinations = 3;
- repeated Predicate predicates = 4;
- repeated ScalarGroup scalar_groups = 5;
- repeated SortConfiguration sort_configs = 6;
- repeated Tuple tuples = 7;
- repeated GeneratorFunctionHandle generator_functions = 8;
+ repeated BloomFilter bloom_filters = 2;
+ repeated GeneratorFunctionHandle generator_functions = 3;
+ repeated HashTable join_hash_tables = 4;
+ repeated InsertDestination insert_destinations = 5;
+ repeated Predicate predicates = 6;
+ repeated ScalarGroup scalar_groups = 7;
+ repeated SortConfiguration sort_configs = 8;
+ repeated Tuple tuples = 9;
// NOTE(zuyu): For UpdateWorkOrder only.
- repeated UpdateGroup update_groups = 9;
+ repeated UpdateGroup update_groups = 10;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 2d09bee..feaecb3 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -35,6 +35,7 @@ add_subdirectory(tests)
# Declare micro-libs:
add_library(quickstep_queryoptimizer_ExecutionGenerator ExecutionGenerator.cpp ExecutionGenerator.hpp)
+add_library(quickstep_queryoptimizer_ExecutionHeuristics ExecutionHeuristics.cpp ExecutionHeuristics.hpp)
add_library(quickstep_queryoptimizer_LogicalGenerator LogicalGenerator.cpp LogicalGenerator.hpp)
add_library(quickstep_queryoptimizer_LogicalToPhysicalMapper
../empty_src.cpp
@@ -64,6 +65,7 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_expressions_scalar_ScalarAttribute
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryContext_proto
+ quickstep_queryoptimizer_ExecutionHeuristics
quickstep_queryoptimizer_OptimizerContext
quickstep_queryoptimizer_QueryHandle
quickstep_queryoptimizer_QueryPlan
@@ -139,6 +141,14 @@ if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_catalog_Catalog_proto)
endif()
+target_link_libraries(quickstep_queryoptimizer_ExecutionHeuristics
+ glog
+ quickstep_catalog_CatalogRelation
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_QueryContext_proto
+ quickstep_queryoptimizer_QueryPlan
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_LogicalGenerator
glog
quickstep_parser_ParseStatement
@@ -211,6 +221,7 @@ target_link_libraries(quickstep_queryoptimizer_Validator
add_library(quickstep_queryoptimizer ../empty_src.cpp QueryOptimizerModule.hpp)
target_link_libraries(quickstep_queryoptimizer
quickstep_queryoptimizer_ExecutionGenerator
+ quickstep_queryoptimizer_ExecutionHeuristics
quickstep_queryoptimizer_LogicalGenerator
quickstep_queryoptimizer_LogicalToPhysicalMapper
quickstep_queryoptimizer_Optimizer
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 077d35d..7f26e85 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -50,6 +50,7 @@
#include "expressions/scalar/ScalarAttribute.hpp"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/ExecutionHeuristics.hpp"
#include "query_optimizer/OptimizerContext.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryPlan.hpp"
@@ -144,6 +145,9 @@ static const volatile bool aggregate_hashtable_type_dummy
DEFINE_bool(parallelize_load, true, "Parallelize loading data files.");
+DEFINE_bool(optimize_joins, false,
+ "Enable post execution plan generation optimizations for joins.");
+
namespace E = ::quickstep::optimizer::expressions;
namespace P = ::quickstep::optimizer::physical;
namespace S = ::quickstep::serialization;
@@ -198,6 +202,11 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) {
temporary_relation_info.producer_operator_index);
}
+ // Optimize execution plan based on heuristics captured during execution plan generation, if enabled.
+ if (FLAGS_optimize_joins) {
+ execution_heuristics_->optimizeExecutionPlan(execution_plan_, query_context_proto_);
+ }
+
#ifdef QUICKSTEP_DISTRIBUTED
catalog_database_cache_proto_->set_name(optimizer_context_->catalog_database()->getName());
@@ -576,12 +585,32 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::vector<attribute_id> probe_attribute_ids;
std::vector<attribute_id> build_attribute_ids;
+ std::vector<attribute_id> probe_original_attribute_ids;
+ std::vector<attribute_id> build_original_attribute_ids;
+
+ const CatalogRelation *referenced_stored_probe_relation;
+ const CatalogRelation *referenced_stored_build_relation;
+
bool any_probe_attributes_nullable = false;
bool any_build_attributes_nullable = false;
+ bool skip_hash_join_optimization = false;
+
const std::vector<E::AttributeReferencePtr> &left_join_attributes =
physical_plan->left_join_attributes();
for (const E::AttributeReferencePtr &left_join_attribute : left_join_attributes) {
+ // Try to determine the original stored relation referenced in the Hash Join.
+ referenced_stored_probe_relation =
+ optimizer_context_->catalog_database()->getRelationByName(left_join_attribute->relation_name());
+ if (referenced_stored_probe_relation == nullptr) {
+ // Hash Join optimizations are not possible, if the referenced relation cannot be determined.
+ skip_hash_join_optimization = true;
+ } else {
+ const attribute_id probe_operator_attribute_id =
+ referenced_stored_probe_relation->getAttributeByName(left_join_attribute->attribute_name())->getID();
+ probe_original_attribute_ids.emplace_back(probe_operator_attribute_id);
+ }
+
const CatalogAttribute *probe_catalog_attribute
= attribute_substitution_map_[left_join_attribute->id()];
probe_attribute_ids.emplace_back(probe_catalog_attribute->getID());
@@ -594,6 +623,18 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
const std::vector<E::AttributeReferencePtr> &right_join_attributes =
physical_plan->right_join_attributes();
for (const E::AttributeReferencePtr &right_join_attribute : right_join_attributes) {
+ // Try to determine the original stored relation referenced in the Hash Join.
+ referenced_stored_build_relation =
+ optimizer_context_->catalog_database()->getRelationByName(right_join_attribute->relation_name());
+ if (referenced_stored_build_relation == nullptr) {
+ // Hash Join optimizations are not possible, if the referenced relation cannot be determined.
+ skip_hash_join_optimization = true;
+ } else {
+ const attribute_id build_operator_attribute_id =
+ referenced_stored_build_relation->getAttributeByName(right_join_attribute->attribute_name())->getID();
+ build_original_attribute_ids.emplace_back(build_operator_attribute_id);
+ }
+
const CatalogAttribute *build_catalog_attribute
= attribute_substitution_map_[right_join_attribute->id()];
build_attribute_ids.emplace_back(build_catalog_attribute->getID());
@@ -629,6 +670,8 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::swap(probe_cardinality, build_cardinality);
std::swap(probe_attribute_ids, build_attribute_ids);
std::swap(any_probe_attributes_nullable, any_build_attributes_nullable);
+ std::swap(probe_original_attribute_ids, build_original_attribute_ids);
+ std::swap(referenced_stored_probe_relation, referenced_stored_build_relation);
}
}
@@ -783,6 +826,17 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::forward_as_tuple(join_operator_index,
output_relation));
temporary_relation_info_vec_.emplace_back(join_operator_index, output_relation);
+
+ // Add heuristics for the Hash Join, if enabled.
+ if (FLAGS_optimize_joins && !skip_hash_join_optimization) {
+ execution_heuristics_->addHashJoinInfo(build_operator_index,
+ join_operator_index,
+ referenced_stored_build_relation,
+ referenced_stored_probe_relation,
+ std::move(build_original_attribute_ids),
+ std::move(probe_original_attribute_ids),
+ join_hash_table_index);
+ }
}
void ExecutionGenerator::convertNestedLoopsJoin(
@@ -895,7 +949,6 @@ void ExecutionGenerator::convertCopyFrom(
false /* is_pipeline_breaker */);
}
-
void ExecutionGenerator::convertCreateIndex(
const P::CreateIndexPtr &physical_plan) {
// CreateIndex is converted to a CreateIndex operator.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/ExecutionGenerator.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.hpp b/query_optimizer/ExecutionGenerator.hpp
index 7c563d4..0630bca 100644
--- a/query_optimizer/ExecutionGenerator.hpp
+++ b/query_optimizer/ExecutionGenerator.hpp
@@ -33,6 +33,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/ExecutionHeuristics.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryPlan.hpp"
#include "query_optimizer/cost_model/CostModel.hpp"
@@ -102,7 +103,8 @@ class ExecutionGenerator {
: optimizer_context_(DCHECK_NOTNULL(optimizer_context)),
query_handle_(DCHECK_NOTNULL(query_handle)),
execution_plan_(DCHECK_NOTNULL(query_handle->getQueryPlanMutable())),
- query_context_proto_(DCHECK_NOTNULL(query_handle->getQueryContextProtoMutable())) {
+ query_context_proto_(DCHECK_NOTNULL(query_handle->getQueryContextProtoMutable())),
+ execution_heuristics_(new ExecutionHeuristics()) {
#ifdef QUICKSTEP_DISTRIBUTED
catalog_database_cache_proto_ = DCHECK_NOTNULL(query_handle->getCatalogDatabaseCacheProtoMutable());
#endif
@@ -376,6 +378,7 @@ class ExecutionGenerator {
QueryHandle *query_handle_;
QueryPlan *execution_plan_; // A part of QueryHandle.
serialization::QueryContext *query_context_proto_; // A part of QueryHandle.
+ std::unique_ptr<ExecutionHeuristics> execution_heuristics_;
#ifdef QUICKSTEP_DISTRIBUTED
serialization::CatalogDatabase *catalog_database_cache_proto_; // A part of QueryHandle.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/ExecutionHeuristics.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionHeuristics.cpp b/query_optimizer/ExecutionHeuristics.cpp
new file mode 100644
index 0000000..fc31c53
--- /dev/null
+++ b/query_optimizer/ExecutionHeuristics.cpp
@@ -0,0 +1,127 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_optimizer/ExecutionHeuristics.hpp"
+
+#include <cstddef>
+#include <utility>
+#include <unordered_map>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/QueryPlan.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+
+void ExecutionHeuristics::optimizeExecutionPlan(QueryPlan *query_plan,
+ serialization::QueryContext *query_context_proto) {
+ // Currently this only optimizes left deep joins using bloom filters.
+ // It uses a simple algorithm to discover the left deep joins.
+ // It starts with the first hash join in the plan and keeps on iterating
+ // over the next hash joins, till a probe on a different relation id is found.
+ // The set of hash joins found in this way forms a chain and can be recognized
+ // as a left deep join. It becomes a candidate for optimization.
+
+ // The optimization is done by modifying each of the build operators in the chain
+ // to generate a bloom filter on the build key during their hash table creation.
+ // The leaf-level probe operator is then modified to query all the bloom
+ // filters generated from all the build operators in the chain. These
+ // bloom filters are queried to test the membership of the probe key
+ // just prior to probing the hash table.
+
+ QueryPlan::DAGNodeIndex origin_node = 0;
+ while (origin_node < hash_joins_.size() - 1) {
+ std::vector<std::size_t> chained_nodes;
+ chained_nodes.push_back(origin_node);
+ for (std::size_t i = origin_node + 1; i < hash_joins_.size(); ++i) {
+ const relation_id checked_relation_id = hash_joins_[origin_node].referenced_stored_probe_relation_->getID();
+ const relation_id expected_relation_id = hash_joins_[i].referenced_stored_probe_relation_->getID();
+ if (checked_relation_id == expected_relation_id) {
+ chained_nodes.push_back(i);
+ } else {
+ break;
+ }
+ }
+
+ // Only chains of length greater than one are suitable candidates for semi-join optimization.
+ if (chained_nodes.size() > 1) {
+ std::unordered_map<QueryContext::bloom_filter_id, std::vector<attribute_id>> probe_bloom_filter_info;
+ for (const std::size_t node : chained_nodes) {
+ // Provision for a new bloom filter to be used by the build operator.
+ const QueryContext::bloom_filter_id bloom_filter_id = query_context_proto->bloom_filters_size();
+ serialization::BloomFilter *bloom_filter_proto = query_context_proto->add_bloom_filters();
+
+ // Modify the bloom filter properties based on the statistics of the relation.
+ setBloomFilterProperties(bloom_filter_proto, hash_joins_[node].referenced_stored_build_relation_);
+
+ // Add build-side bloom filter information to the corresponding hash table proto.
+ query_context_proto->mutable_join_hash_tables(hash_joins_[node].join_hash_table_id_)
+ ->add_build_side_bloom_filter_id(bloom_filter_id);
+
+ probe_bloom_filter_info.insert(std::make_pair(bloom_filter_id, hash_joins_[node].probe_attributes_));
+ }
+
+ // Add probe-side bloom filter information to the corresponding hash table proto for each build-side bloom filter.
+ for (const std::pair<QueryContext::bloom_filter_id, std::vector<attribute_id>>
+ &bloom_filter_info : probe_bloom_filter_info) {
+ auto *probe_side_bloom_filter =
+ query_context_proto->mutable_join_hash_tables(hash_joins_[origin_node].join_hash_table_id_)
+ ->add_probe_side_bloom_filters();
+ probe_side_bloom_filter->set_probe_side_bloom_filter_id(bloom_filter_info.first);
+ for (const attribute_id &probe_attribute_id : bloom_filter_info.second) {
+ probe_side_bloom_filter->add_probe_side_attr_ids(probe_attribute_id);
+ }
+ }
+
+ // Add node dependencies from chained build nodes to origin node probe.
+ for (std::size_t i = 1; i < chained_nodes.size(); ++i) { // Note: It starts from index 1.
+ query_plan->addDirectDependency(hash_joins_[origin_node].join_operator_index_,
+ hash_joins_[origin_node + i].build_operator_index_,
+ true /* is_pipeline_breaker */);
+ }
+ }
+
+ // Update the origin node.
+ origin_node = chained_nodes.back() + 1;
+ }
+}
+
+void ExecutionHeuristics::setBloomFilterProperties(serialization::BloomFilter *bloom_filter_proto,
+ const CatalogRelation *relation) {
+ const std::size_t cardinality = relation->estimateTupleCardinality();
+ if (cardinality < kOneThousand) {
+ bloom_filter_proto->set_bloom_filter_size(kOneThousand / kCompressionFactor);
+ bloom_filter_proto->set_number_of_hashes(kVeryLowSparsityHash);
+ } else if (cardinality < kTenThousand) {
+ bloom_filter_proto->set_bloom_filter_size(kTenThousand / kCompressionFactor);
+ bloom_filter_proto->set_number_of_hashes(kLowSparsityHash);
+ } else if (cardinality < kHundredThousand) {
+ bloom_filter_proto->set_bloom_filter_size(kHundredThousand / kCompressionFactor);
+ bloom_filter_proto->set_number_of_hashes(kMediumSparsityHash);
+ } else {
+ bloom_filter_proto->set_bloom_filter_size(kMillion / kCompressionFactor);
+ bloom_filter_proto->set_number_of_hashes(kHighSparsityHash);
+ }
+}
+
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/ExecutionHeuristics.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionHeuristics.hpp b/query_optimizer/ExecutionHeuristics.hpp
new file mode 100644
index 0000000..92a7fe8
--- /dev/null
+++ b/query_optimizer/ExecutionHeuristics.hpp
@@ -0,0 +1,155 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_HEURISTICS_HPP_
+#define QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_HEURISTICS_HPP_
+
+#include <vector>
+
+#include "catalog/CatalogRelation.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/QueryPlan.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+
+/** \addtogroup QueryOptimizer
+ * @{
+ */
+
+/**
+ * @brief The ExecutionHeuristics compiles certain heuristics for an execution plan
+ * as it is being converted to a physical plan. These heuristics can then be
+ * used to optimize the execution plan after it has been generated.
+ **/
+class ExecutionHeuristics {
+ public:
+ static const std::size_t kOneHundred = 100;
+ static const std::size_t kOneThousand = 1000;
+ static const std::size_t kTenThousand = 10000;
+ static const std::size_t kHundredThousand = 100000;
+ static const std::size_t kMillion = 1000000;
+
+ static const std::size_t kCompressionFactor = 10;
+
+ static const std::size_t kVeryLowSparsityHash = 1;
+ static const std::size_t kLowSparsityHash = 2;
+ static const std::size_t kMediumSparsityHash = 5;
+ static const std::size_t kHighSparsityHash = 10;
+
+ /**
+ * @brief A simple internal class that holds information about various
+ * hash joins within the execution plan for a query.
+ **/
+ struct HashJoinInfo {
+ HashJoinInfo(const QueryPlan::DAGNodeIndex build_operator_index,
+ const QueryPlan::DAGNodeIndex join_operator_index,
+ const CatalogRelation *referenced_stored_build_relation,
+ const CatalogRelation *referenced_stored_probe_relation,
+ std::vector<attribute_id> &&build_attributes,
+ std::vector<attribute_id> &&probe_attributes,
+ const QueryContext::join_hash_table_id join_hash_table_id)
+ : build_operator_index_(build_operator_index),
+ join_operator_index_(join_operator_index),
+ referenced_stored_build_relation_(referenced_stored_build_relation),
+ referenced_stored_probe_relation_(referenced_stored_probe_relation),
+ build_attributes_(std::move(build_attributes)),
+ probe_attributes_(std::move(probe_attributes)),
+ join_hash_table_id_(join_hash_table_id) {
+ }
+
+ const QueryPlan::DAGNodeIndex build_operator_index_;
+ const QueryPlan::DAGNodeIndex join_operator_index_;
+ const CatalogRelation *referenced_stored_build_relation_;
+ const CatalogRelation *referenced_stored_probe_relation_;
+ const std::vector<attribute_id> build_attributes_;
+ const std::vector<attribute_id> probe_attributes_;
+ const QueryContext::join_hash_table_id join_hash_table_id_;
+ };
+
+
+ /**
+ * @brief Constructor.
+ **/
+ ExecutionHeuristics() {}
+
+ /**
+ * @brief Saves information about a hash join used within the execution plan
+ * for a query.
+ *
+ * @param build_operator_index Index of the build operator of the hash join.
+ * @param join_operator_index Index of the join operator of the hash join.
+ * @param build_relation_id Id of the relation on which hash table is being built.
+ * @param probe_relation_id Id of the relation on which hash table is being probed.
+ * @param build_attributes List of attributes on which hash table is being built.
+ * @param probe_attributes List of attributes on which hash table is being probed.
+ * @param join_hash_table_id Id of the hash table which refers to the actual hash
+ * table within the query context.
+ **/
+ inline void addHashJoinInfo(const QueryPlan::DAGNodeIndex build_operator_index,
+ const QueryPlan::DAGNodeIndex join_operator_index,
+ const CatalogRelation *referenced_stored_build_relation,
+ const CatalogRelation *referenced_stored_probe_relation,
+ std::vector<attribute_id> &&build_attributes,
+ std::vector<attribute_id> &&probe_attributes,
+ const QueryContext::join_hash_table_id join_hash_table_id) {
+ hash_joins_.push_back(HashJoinInfo(build_operator_index,
+ join_operator_index,
+ referenced_stored_build_relation,
+ referenced_stored_probe_relation,
+ std::move(build_attributes),
+ std::move(probe_attributes),
+ join_hash_table_id));
+ }
+
+ /**
+ * @brief Optimize the execution plan based on heuristics generated
+ * during physical plan to execution plan conversion.
+ *
+ * @param query_plan A mutable reference to the query execution plan.
+ * @param query_context_proto A mutable reference to the protobuf representation
+ * of the query context.
+ **/
+ void optimizeExecutionPlan(QueryPlan *query_plan, serialization::QueryContext *query_context_proto);
+
+ /**
+ * @brief Set the properties of the bloom filter proto based on the statistics
+ * of the given relation.
+ *
+ * @param bloom_filter_proto A mutable reference to the bloom filter protobuf representation.
+ * @param relation The catalog relation on which bloom filter is being built.
+ **/
+ void setBloomFilterProperties(serialization::BloomFilter *bloom_filter_proto,
+ const CatalogRelation *relation);
+
+ private:
+ std::vector<HashJoinInfo> hash_joins_;
+
+ DISALLOW_COPY_AND_ASSIGN(ExecutionHeuristics);
+};
+
+/** @} */
+
+} // namespace optimizer
+} // namespace quickstep
+
+#endif /* QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_HEURISTICS_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/CMakeLists.txt b/query_optimizer/tests/CMakeLists.txt
index 1d2fa10..5647bfd 100644
--- a/query_optimizer/tests/CMakeLists.txt
+++ b/query_optimizer/tests/CMakeLists.txt
@@ -78,6 +78,22 @@ add_executable(quickstep_queryoptimizer_tests_ExecutionGeneratorTest
ExecutionGeneratorTestRunner.hpp
"${PROJECT_SOURCE_DIR}/utility/textbased_test/TextBasedTest.cpp"
"${PROJECT_SOURCE_DIR}/utility/textbased_test/TextBasedTest.hpp")
+add_executable(ExecutionHeuristics_unittest ExecutionHeuristics_unittest.cpp)
+target_link_libraries(ExecutionHeuristics_unittest
+ gtest
+ gtest_main
+ quickstep_catalog_Catalog
+ quickstep_catalog_CatalogDatabase
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_QueryContext_proto
+ quickstep_queryoptimizer_ExecutionHeuristics
+ quickstep_queryoptimizer_QueryPlan
+ quickstep_relationaloperators_BuildHashOperator
+ quickstep_relationaloperators_HashJoinOperator
+ quickstep_utility_Macros)
+add_test(ExecutionHeuristics_unittest ExecutionHeuristics_unittest)
+
add_executable(quickstep_queryoptimizer_tests_OptimizerTextTest
OptimizerTextTest.cpp
OptimizerTextTestRunner.cpp
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/ExecutionHeuristics_unittest.cpp b/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
new file mode 100644
index 0000000..12acaff
--- /dev/null
+++ b/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
@@ -0,0 +1,301 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "catalog/Catalog.hpp"
+#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/ExecutionHeuristics.hpp"
+#include "query_optimizer/QueryPlan.hpp"
+#include "relational_operators/BuildHashOperator.hpp"
+#include "relational_operators/HashJoinOperator.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+namespace quickstep {
+namespace optimizer {
+
+class ExecutionHeuristicsTest : public ::testing::Test {
+ protected:
+ virtual void SetUp() {
+ db_ = cat_.getDatabaseByIdMutable(cat_.addDatabase(new CatalogDatabase(nullptr, "db")));
+ execution_heuristics_.reset(new ExecutionHeuristics());
+ query_plan_.reset(new QueryPlan());
+ query_context_proto_.reset(new serialization::QueryContext());
+ }
+
+ CatalogRelation* createCatalogRelation(const std::string &name, bool temporary = false) {
+ return db_->getRelationByIdMutable(db_->addRelation(new CatalogRelation(nullptr, name, -1, temporary)));
+ }
+
+ void addDummyHashJoinInfo(ExecutionHeuristics *execution_heuristics,
+ const QueryPlan::DAGNodeIndex build_operator_index,
+ const QueryPlan::DAGNodeIndex join_operator_index,
+ const CatalogRelation *build_relation,
+ const CatalogRelation *probe_relation,
+ const attribute_id build_attribute_id,
+ const attribute_id probe_attribute_id,
+ const QueryContext::join_hash_table_id join_hash_table_id) {
+ std::vector<attribute_id> build_attribute_ids(1, build_attribute_id);
+ std::vector<attribute_id> probe_attribute_ids(1, probe_attribute_id);
+ execution_heuristics->addHashJoinInfo(build_operator_index,
+ join_operator_index,
+ build_relation,
+ probe_relation,
+ std::move(build_attribute_ids),
+ std::move(probe_attribute_ids),
+ join_hash_table_id);
+ }
+
+ QueryPlan::DAGNodeIndex createDummyBuildHashOperator(QueryPlan *query_plan,
+ const CatalogRelation *build_relation,
+ const attribute_id build_attribute_id,
+ const QueryContext::join_hash_table_id join_hash_table_index) {
+ std::vector<attribute_id> build_attribute_ids;
+ build_attribute_ids.push_back(build_attribute_id);
+ QueryPlan::DAGNodeIndex build_operator_index =
+ query_plan->addRelationalOperator(new BuildHashOperator(*build_relation,
+ true,
+ build_attribute_ids,
+ false,
+ join_hash_table_index));
+ return build_operator_index;
+ }
+
+ QueryPlan::DAGNodeIndex createDummyHashJoinOperator(QueryPlan *query_plan,
+ const CatalogRelation *build_relation,
+ const CatalogRelation *probe_relation,
+ const attribute_id probe_attribute_id,
+ const QueryContext::join_hash_table_id join_hash_table_index) {
+ std::vector<attribute_id> probe_attribute_ids;
+ probe_attribute_ids.push_back(probe_attribute_id);
+ QueryPlan::DAGNodeIndex join_operator_index =
+ query_plan->addRelationalOperator(new HashJoinOperator(*build_relation,
+ *probe_relation,
+ true,
+ probe_attribute_ids,
+ false,
+ *probe_relation,
+ 0,
+ join_hash_table_index,
+ 0,
+ 0));
+ return join_operator_index;
+ }
+
+ Catalog cat_;
+ CatalogDatabase *db_; // db_ is owned by cat_.
+ std::unique_ptr<QueryPlan> query_plan_;
+ std::unique_ptr<serialization::QueryContext> query_context_proto_;
+ std::unique_ptr<ExecutionHeuristics> execution_heuristics_;
+};
+
+TEST_F(ExecutionHeuristicsTest, HashJoinOptimizedTest) {
+ // This test case creates three hash joins, all of which are being probed on the same relation.
+ // Since the probe are being made on the same relation, ExecutionHeuristics should optimize
+ // these hash joins using bloom filters.
+
+ const CatalogRelation *build_relation_1 = createCatalogRelation("build_relation_1");
+ const CatalogRelation *build_relation_2 = createCatalogRelation("build_relation_2");
+ const CatalogRelation *build_relation_3 = createCatalogRelation("build_relation_3");
+ const CatalogRelation *probe_relation_1 = createCatalogRelation("probe_relation_1");
+
+ const attribute_id build_attribute_id_1 = 0;
+ const attribute_id build_attribute_id_2 = 0;
+ const attribute_id build_attribute_id_3 = 0;
+ const attribute_id probe_attribute_id_1 = 1;
+ const attribute_id probe_attribute_id_2 = 2;
+ const attribute_id probe_attribute_id_3 = 3;
+
+ const QueryContext::join_hash_table_id join_hash_table_index_1 = 0;
+ const QueryContext::join_hash_table_id join_hash_table_index_2 = 1;
+ const QueryContext::join_hash_table_id join_hash_table_index_3 = 2;
+ query_context_proto_->add_join_hash_tables();
+ query_context_proto_->add_join_hash_tables();
+ query_context_proto_->add_join_hash_tables();
+
+ const QueryPlan::DAGNodeIndex build_operator_index_1 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_1,
+ build_attribute_id_1,
+ join_hash_table_index_1);
+ const QueryPlan::DAGNodeIndex probe_operator_index_1 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_1,
+ probe_relation_1,
+ probe_attribute_id_1,
+ join_hash_table_index_1);
+ const QueryPlan::DAGNodeIndex build_operator_index_2 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_2,
+ build_attribute_id_2,
+ join_hash_table_index_2);
+ const QueryPlan::DAGNodeIndex probe_operator_index_2 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_2,
+ probe_relation_1,
+ probe_attribute_id_2,
+ join_hash_table_index_2);
+ const QueryPlan::DAGNodeIndex build_operator_index_3 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_3,
+ build_attribute_id_3,
+ join_hash_table_index_3);
+ const QueryPlan::DAGNodeIndex probe_operator_index_3 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_3,
+ probe_relation_1,
+ probe_attribute_id_3,
+ join_hash_table_index_3);
+
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_1,
+ probe_operator_index_1,
+ build_relation_1,
+ probe_relation_1,
+ build_attribute_id_1,
+ probe_attribute_id_1,
+ join_hash_table_index_1);
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_2,
+ probe_operator_index_2,
+ build_relation_2,
+ probe_relation_1,
+ build_attribute_id_2,
+ probe_attribute_id_2,
+ join_hash_table_index_2);
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_3,
+ probe_operator_index_3,
+ build_relation_3,
+ probe_relation_1,
+ build_attribute_id_3,
+ probe_attribute_id_3,
+ join_hash_table_index_3);
+
+ execution_heuristics_->optimizeExecutionPlan(query_plan_.get(), query_context_proto_.get());
+
+ // Test whether correct number of bloom filters were added.
+ EXPECT_EQ(1, query_context_proto_->join_hash_tables(0).build_side_bloom_filter_id_size());
+ EXPECT_EQ(1, query_context_proto_->join_hash_tables(1).build_side_bloom_filter_id_size());
+ EXPECT_EQ(1, query_context_proto_->join_hash_tables(2).build_side_bloom_filter_id_size());
+ EXPECT_EQ(3, query_context_proto_->join_hash_tables(0).probe_side_bloom_filters_size());
+
+ // Test that the DAG was modified correctly or not.
+ // Probe operator 1 should have now build operator 1 and build operator 2 added as dependencies.
+ auto const probe_node_dependencies = query_plan_->getQueryPlanDAG().getDependencies(probe_operator_index_1);
+ EXPECT_EQ(1u, probe_node_dependencies.count(build_operator_index_2));
+ EXPECT_EQ(1u, probe_node_dependencies.count(build_operator_index_3));
+}
+
+TEST_F(ExecutionHeuristicsTest, HashJoinNotOptimizedTest) {
+ // This test case creates three hash joins, all of which are being probed on different relations.
+ // Since the probe are being made on the different relations, ExecutionHeuristics should optimize
+ // these hash joins using bloom filters.
+
+ const CatalogRelation *build_relation_1 = createCatalogRelation("build_relation_1");
+ const CatalogRelation *build_relation_2 = createCatalogRelation("build_relation_2");
+ const CatalogRelation *build_relation_3 = createCatalogRelation("build_relation_3");
+ const CatalogRelation *probe_relation_1 = createCatalogRelation("probe_relation_1");
+ const CatalogRelation *probe_relation_2 = createCatalogRelation("probe_relation_2");
+ const CatalogRelation *probe_relation_3 = createCatalogRelation("probe_relation_3");
+
+ const attribute_id build_attribute_id_1 = 0;
+ const attribute_id build_attribute_id_2 = 0;
+ const attribute_id build_attribute_id_3 = 0;
+ const attribute_id probe_attribute_id_1 = 1;
+ const attribute_id probe_attribute_id_2 = 2;
+ const attribute_id probe_attribute_id_3 = 3;
+
+ const QueryContext::join_hash_table_id join_hash_table_index_1 = 0;
+ const QueryContext::join_hash_table_id join_hash_table_index_2 = 1;
+ const QueryContext::join_hash_table_id join_hash_table_index_3 = 2;
+ query_context_proto_->add_join_hash_tables();
+ query_context_proto_->add_join_hash_tables();
+ query_context_proto_->add_join_hash_tables();
+
+ const QueryPlan::DAGNodeIndex build_operator_index_1 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_1,
+ build_attribute_id_1,
+ join_hash_table_index_1);
+ const QueryPlan::DAGNodeIndex probe_operator_index_1 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_1,
+ probe_relation_1,
+ probe_attribute_id_1,
+ join_hash_table_index_1);
+ const QueryPlan::DAGNodeIndex build_operator_index_2 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_2,
+ build_attribute_id_2,
+ join_hash_table_index_2);
+ const QueryPlan::DAGNodeIndex probe_operator_index_2 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_2,
+ probe_relation_2,
+ probe_attribute_id_2,
+ join_hash_table_index_2);
+ const QueryPlan::DAGNodeIndex build_operator_index_3 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_3,
+ build_attribute_id_3,
+ join_hash_table_index_3);
+ const QueryPlan::DAGNodeIndex probe_operator_index_3 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_3,
+ probe_relation_3,
+ probe_attribute_id_3,
+ join_hash_table_index_3);
+
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_1,
+ probe_operator_index_1,
+ build_relation_1,
+ probe_relation_1,
+ build_attribute_id_1,
+ probe_attribute_id_1,
+ join_hash_table_index_1);
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_2,
+ probe_operator_index_2,
+ build_relation_2,
+ probe_relation_2,
+ build_attribute_id_2,
+ probe_attribute_id_2,
+ join_hash_table_index_2);
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_3,
+ probe_operator_index_3,
+ build_relation_3,
+ probe_relation_3,
+ build_attribute_id_3,
+ probe_attribute_id_3,
+ join_hash_table_index_3);
+
+ execution_heuristics_->optimizeExecutionPlan(query_plan_.get(), query_context_proto_.get());
+
+ // Test that no bloom filters were added.
+ EXPECT_EQ(0, query_context_proto_->join_hash_tables(0).build_side_bloom_filter_id_size());
+ EXPECT_EQ(0, query_context_proto_->join_hash_tables(1).build_side_bloom_filter_id_size());
+ EXPECT_EQ(0, query_context_proto_->join_hash_tables(2).build_side_bloom_filter_id_size());
+ EXPECT_EQ(0, query_context_proto_->join_hash_tables(0).probe_side_bloom_filters_size());
+
+ // Test that the DAG was not modified at all.
+ // Probe operator 1 should not have build operator 1 and build operator 2 added as dependencies.
+ auto probe_node_dependencies = query_plan_->getQueryPlanDAG().getDependencies(probe_operator_index_1);
+ EXPECT_EQ(0u, probe_node_dependencies.count(build_operator_index_2));
+ EXPECT_EQ(0u, probe_node_dependencies.count(build_operator_index_3));
+}
+
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index dacacfa..115248c 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -629,6 +629,7 @@ target_link_libraries(quickstep_storage_HashTable
quickstep_threading_SpinSharedMutex
quickstep_types_Type
quickstep_types_TypedValue
+ quickstep_utility_BloomFilter
quickstep_utility_HashPair
quickstep_utility_Macros)
target_link_libraries(quickstep_storage_HashTableBase
@@ -648,6 +649,7 @@ target_link_libraries(quickstep_storage_HashTableFactory
quickstep_types_Type
quickstep_types_TypeFactory
quickstep_types_TypedValue
+ quickstep_utility_BloomFilter
quickstep_utility_Macros)
target_link_libraries(quickstep_storage_HashTableKeyManager
glog
@@ -1196,7 +1198,7 @@ target_link_libraries(BloomFilterIndexSubBlock_unittest
add_test(BloomFilterIndexSubBlock_unittest BloomFilterIndexSubBlock_unittest)
if(QUICKSTEP_HAVE_BITWEAVING)
- add_executable(BitWeavingIndexSubBlock_unittest
+ add_executable(BitWeavingIndexSubBlock_unittest
"${CMAKE_CURRENT_SOURCE_DIR}/bitweaving/tests/BitWeavingIndexSubBlock_unittest.cpp")
target_link_libraries(BitWeavingIndexSubBlock_unittest
glog
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/storage/HashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTable.hpp b/storage/HashTable.hpp
index 667848e..be31fd9 100644
--- a/storage/HashTable.hpp
+++ b/storage/HashTable.hpp
@@ -38,6 +38,7 @@
#include "threading/SpinSharedMutex.hpp"
#include "types/Type.hpp"
#include "types/TypedValue.hpp"
+#include "utility/BloomFilter.hpp"
#include "utility/HashPair.hpp"
#include "utility/Macros.hpp"
@@ -990,6 +991,61 @@ class HashTable : public HashTableBase<resizable,
template <typename FunctorT>
std::size_t forEachCompositeKey(FunctorT *functor) const;
+ /**
+ * @brief A call to this function will cause a bloom filter to be built
+ * during the build phase of this hash table.
+ **/
+ inline void enableBuildSideBloomFilter() {
+ has_build_side_bloom_filter_ = true;
+ }
+
+ /**
+ * @brief A call to this function will cause a set of bloom filters to be
+ * probed during the probe phase of this hash table.
+ **/
+ inline void enableProbeSideBloomFilter() {
+ has_probe_side_bloom_filter_ = true;
+ }
+
+ /**
+ * @brief This function sets the pointer to the bloom filter to be
+ * used during the build phase of this hash table.
+ * @warning Should call enable_build_side_bloom_filter() first to enable
+ * bloom filter usage during build phase.
+ * @note The ownership of the bloom filter lies with the caller.
+ *
+ * @param bloom_filter The pointer to the bloom filter.
+ **/
+ inline void setBuildSideBloomFilter(BloomFilter *bloom_filter) {
+ build_bloom_filter_ = bloom_filter;
+ }
+
+ /**
+ * @brief This function adds a pointer to the list of bloom filters to be
+ * used during the probe phase of this hash table.
+ * @warning Should call enable_probe_side_bloom_filter() first to enable
+ * bloom filter usage during probe phase.
+ * @note The ownership of the bloom filter lies with the caller.
+ *
+ * @param bloom_filter The pointer to the bloom filter.
+ **/
+ inline void addProbeSideBloomFilter(const BloomFilter *bloom_filter) {
+ probe_bloom_filters_.emplace_back(bloom_filter);
+ }
+
+ /**
+ * @brief This function adds a vector of attribute ids corresponding to a
+ * bloom filter used during the probe phase of this hash table.
+ * @warning Should call enable_probe_side_bloom_filter() first to enable
+ * bloom filter usage during probe phase.
+ *
+ * @param probe_attribute_ids The vector of attribute ids to use for probing
+ * the bloom filter.
+ **/
+ inline void addProbeSideAttributeIds(std::vector<attribute_id> &&probe_attribute_ids) {
+ probe_attribute_ids_.push_back(probe_attribute_ids);
+ }
+
protected:
/**
* @brief Constructor for new resizable hash table.
@@ -1270,6 +1326,13 @@ class HashTable : public HashTableBase<resizable,
const attribute_id key_attr_id,
FunctorT *functor) const;
+ // Data structures used for bloom filter optimized semi-joins.
+ bool has_build_side_bloom_filter_ = false;
+ bool has_probe_side_bloom_filter_ = false;
+ BloomFilter *build_bloom_filter_;
+ std::vector<const BloomFilter*> probe_bloom_filters_;
+ std::vector<std::vector<attribute_id>> probe_attribute_ids_;
+
DISALLOW_COPY_AND_ASSIGN(HashTable);
};
@@ -1414,6 +1477,12 @@ HashTablePutResult HashTable<ValueT, resizable, serializable, force_key_copy, al
&prealloc_state);
}
}
+ std::unique_ptr<BloomFilter> thread_local_bloom_filter;
+ if (has_build_side_bloom_filter_) {
+ thread_local_bloom_filter.reset(new BloomFilter(build_bloom_filter_->getRandomSeed(),
+ build_bloom_filter_->getNumberOfHashes(),
+ build_bloom_filter_->getBitArraySize()));
+ }
if (resizable) {
while (result == HashTablePutResult::kOutOfSpace) {
{
@@ -1429,6 +1498,11 @@ HashTablePutResult HashTable<ValueT, resizable, serializable, force_key_copy, al
variable_size,
(*functor)(*accessor),
using_prealloc ? &prealloc_state : nullptr);
+ // Insert into bloom filter, if enabled.
+ if (has_build_side_bloom_filter_) {
+ thread_local_bloom_filter->insertUnSafe(static_cast<const std::uint8_t *>(key.getDataPtr()),
+ key.getDataSize());
+ }
if (result == HashTablePutResult::kDuplicateKey) {
DEBUG_ASSERT(!using_prealloc);
return result;
@@ -1454,11 +1528,20 @@ HashTablePutResult HashTable<ValueT, resizable, serializable, force_key_copy, al
variable_size,
(*functor)(*accessor),
using_prealloc ? &prealloc_state : nullptr);
+ // Insert into bloom filter, if enabled.
+ if (has_build_side_bloom_filter_) {
+ thread_local_bloom_filter->insertUnSafe(static_cast<const std::uint8_t *>(key.getDataPtr()),
+ key.getDataSize());
+ }
if (result != HashTablePutResult::kOK) {
return result;
}
}
}
+ // Update the build side bloom filter with thread local copy, if available.
+ if (has_build_side_bloom_filter_) {
+ build_bloom_filter_->bitwiseOr(thread_local_bloom_filter.get());
+ }
return HashTablePutResult::kOK;
});
@@ -2164,6 +2247,27 @@ void HashTable<ValueT, resizable, serializable, force_key_copy, allow_duplicate_
accessor,
[&](auto *accessor) -> void { // NOLINT(build/c++11)
while (accessor->next()) {
+ // Probe any bloom filters, if enabled.
+ if (has_probe_side_bloom_filter_) {
+ DCHECK_EQ(probe_bloom_filters_.size(), probe_attribute_ids_.size());
+ // Check if the key is contained in the BloomFilters or not.
+ bool bloom_miss = false;
+ for (std::size_t i = 0; i < probe_bloom_filters_.size() && !bloom_miss; ++i) {
+ const BloomFilter *bloom_filter = probe_bloom_filters_[i];
+ for (const attribute_id &attr_id : probe_attribute_ids_[i]) {
+ TypedValue bloom_key = accessor->getTypedValue(attr_id);
+ if (!bloom_filter->contains(static_cast<const std::uint8_t*>(bloom_key.getDataPtr()),
+ bloom_key.getDataSize())) {
+ bloom_miss = true;
+ break;
+ }
+ }
+ }
+ if (bloom_miss) {
+ continue; // On a bloom filter miss, probing the hash table can be skipped.
+ }
+ }
+
TypedValue key = accessor->getTypedValue(key_attr_id);
if (check_for_null_keys && key.isNull()) {
continue;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/storage/HashTable.proto
----------------------------------------------------------------------
diff --git a/storage/HashTable.proto b/storage/HashTable.proto
index 653c3a7..7f00f29 100644
--- a/storage/HashTable.proto
+++ b/storage/HashTable.proto
@@ -1,5 +1,7 @@
// Copyright 2011-2015 Quickstep Technologies LLC.
// Copyright 2015-2016 Pivotal Software, Inc.
+// Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+// University of Wisconsin\u2014Madison.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -32,4 +34,10 @@ message HashTable {
required HashTableImplType hash_table_impl_type = 1;
repeated Type key_types = 2;
required uint64 estimated_num_entries = 3;
+ repeated uint32 build_side_bloom_filter_id = 4;
+ message ProbeSideBloomFilter {
+ required uint32 probe_side_bloom_filter_id = 1;
+ repeated uint32 probe_side_attr_ids = 2;
+ }
+ repeated ProbeSideBloomFilter probe_side_bloom_filters = 6;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/storage/HashTableFactory.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTableFactory.hpp b/storage/HashTableFactory.hpp
index 94a0721..34baaeb 100644
--- a/storage/HashTableFactory.hpp
+++ b/storage/HashTableFactory.hpp
@@ -29,6 +29,7 @@
#include "storage/SimpleScalarSeparateChainingHashTable.hpp"
#include "storage/TupleReference.hpp"
#include "types/TypeFactory.hpp"
+#include "utility/BloomFilter.hpp"
#include "utility/Macros.hpp"
#include "glog/logging.h"
@@ -291,11 +292,14 @@ class HashTableFactory {
* @param proto A protobuf description of a resizable HashTable.
* @param storage_manager The StorageManager to use (a StorageBlob will be
* allocated to hold the HashTable's contents).
+ * @param bloom_filters A vector of pointers to bloom filters that may be used
+ * during hash table construction in build/probe phase.
* @return A new resizable HashTable with parameters specified by proto.
**/
static HashTable<ValueT, resizable, serializable, force_key_copy, allow_duplicate_keys>*
CreateResizableFromProto(const serialization::HashTable &proto,
- StorageManager *storage_manager) {
+ StorageManager *storage_manager,
+ const std::vector<std::unique_ptr<BloomFilter>> &bloom_filters) {
DCHECK(ProtoIsValid(proto))
<< "Attempted to create HashTable from invalid proto description:\n"
<< proto.DebugString();
@@ -305,10 +309,40 @@ class HashTableFactory {
key_types.emplace_back(&TypeFactory::ReconstructFromProto(proto.key_types(i)));
}
- return CreateResizable(HashTableImplTypeFromProto(proto.hash_table_impl_type()),
- key_types,
- proto.estimated_num_entries(),
- storage_manager);
+ auto hash_table = CreateResizable(HashTableImplTypeFromProto(proto.hash_table_impl_type()),
+ key_types,
+ proto.estimated_num_entries(),
+ storage_manager);
+
+ // TODO(ssaurabh): These lazy initializations can be moved from here and pushed to the
+ // individual implementations of the hash table constructors.
+
+ // Check if there are any build side bloom filter defined on the hash table.
+ if (proto.build_side_bloom_filter_id_size() > 0) {
+ hash_table->enableBuildSideBloomFilter();
+ hash_table->setBuildSideBloomFilter(bloom_filters[proto.build_side_bloom_filter_id(0)].get());
+ }
+
+ // Check if there are any probe side bloom filters defined on the hash table.
+ if (proto.probe_side_bloom_filters_size() > 0) {
+ hash_table->enableProbeSideBloomFilter();
+ // Add as many probe bloom filters as defined by the proto.
+ for (int j = 0; j < proto.probe_side_bloom_filters_size(); ++j) {
+ // Add the pointer to the probe bloom filter within the list of probe bloom filters to use.
+ const auto probe_side_bloom_filter = proto.probe_side_bloom_filters(j);
+ hash_table->addProbeSideBloomFilter(bloom_filters[probe_side_bloom_filter.probe_side_bloom_filter_id()].get());
+
+ // Add the attribute ids corresponding to this probe bloom filter.
+ std::vector<attribute_id> probe_attribute_ids;
+ for (int k = 0; k < probe_side_bloom_filter.probe_side_attr_ids_size(); ++k) {
+ const attribute_id probe_attribute_id = probe_side_bloom_filter.probe_side_attr_ids(k);
+ probe_attribute_ids.push_back(probe_attribute_id);
+ }
+ hash_table->addProbeSideAttributeIds(std::move(probe_attribute_ids));
+ }
+ }
+
+ return hash_table;
}
private:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/utility/BloomFilter.hpp
----------------------------------------------------------------------
diff --git a/utility/BloomFilter.hpp b/utility/BloomFilter.hpp
index 1d4fdc7..b93df84 100644
--- a/utility/BloomFilter.hpp
+++ b/utility/BloomFilter.hpp
@@ -26,8 +26,15 @@
#include <algorithm>
#include <cstddef>
#include <cstdint>
+#include <memory>
+#include <utility>
#include <vector>
+#include "storage/StorageConstants.hpp"
+#include "threading/Mutex.hpp"
+#include "threading/SharedMutex.hpp"
+#include "threading/SpinSharedMutex.hpp"
+#include "utility/BloomFilter.pb.h"
#include "utility/Macros.hpp"
#include "glog/logging.h"
@@ -47,7 +54,30 @@ class BloomFilter {
/**
* @brief Constructor.
- * @note The ownership of the bit array lies with the caller.
+ * @note When no bit_array is being passed to the constructor,
+ * then the bit_array is owned and managed by this class.
+ *
+ * @param random_seed A random_seed that generates unique hash functions.
+ * @param hash_fn_count The number of hash functions used by this bloom filter.
+ * @param bit_array_size_in_bytes Size of the bit array.
+ **/
+ BloomFilter(const std::uint64_t random_seed,
+ const std::size_t hash_fn_count,
+ const std::uint64_t bit_array_size_in_bytes)
+ : random_seed_(random_seed),
+ hash_fn_count_(hash_fn_count),
+ array_size_in_bytes_(bit_array_size_in_bytes),
+ array_size_(array_size_in_bytes_ * kNumBitsPerByte),
+ bit_array_(new std::uint8_t[array_size_in_bytes_]),
+ is_bit_array_owner_(true) {
+ reset();
+ generate_unique_hash_fn();
+ }
+
+ /**
+ * @brief Constructor.
+ * @note When a bit_array is passed as an argument to the constructor,
+ * then the ownership of the bit array lies with the caller.
*
* @param random_seed A random_seed that generates unique hash functions.
* @param hash_fn_count The number of hash functions used by this bloom filter.
@@ -61,11 +91,12 @@ class BloomFilter {
const std::uint64_t bit_array_size_in_bytes,
std::uint8_t *bit_array,
const bool is_initialized)
- : hash_fn_count_(hash_fn_count),
- random_seed_(random_seed) {
- array_size_ = bit_array_size_in_bytes * kNumBitsPerByte;
- array_size_in_bytes_ = bit_array_size_in_bytes;
- bit_array_ = bit_array; // Owned by the calling method.
+ : random_seed_(random_seed),
+ hash_fn_count_(hash_fn_count),
+ array_size_in_bytes_(bit_array_size_in_bytes),
+ array_size_(bit_array_size_in_bytes * kNumBitsPerByte),
+ bit_array_(bit_array), // Owned by the calling method.
+ is_bit_array_owner_(false) {
if (!is_initialized) {
reset();
}
@@ -73,27 +104,149 @@ class BloomFilter {
}
/**
+ * @brief Constructor.
+ * @note When a bloom filter proto is passed as an initializer,
+ * then the bit_array is owned and managed by this class.
+ *
+ * @param bloom_filter_proto The protobuf representation of a
+ * bloom filter configuration.
+ **/
+ explicit BloomFilter(const serialization::BloomFilter &bloom_filter_proto)
+ : random_seed_(bloom_filter_proto.bloom_filter_seed()),
+ hash_fn_count_(bloom_filter_proto.number_of_hashes()),
+ array_size_in_bytes_(bloom_filter_proto.bloom_filter_size()),
+ array_size_(array_size_in_bytes_ * kNumBitsPerByte),
+ bit_array_(new std::uint8_t[array_size_in_bytes_]),
+ is_bit_array_owner_(true) {
+ reset();
+ generate_unique_hash_fn();
+ }
+
+ /**
+ * @brief Destructor.
+ **/
+ ~BloomFilter() {
+ if (is_bit_array_owner_) {
+ bit_array_.reset();
+ } else {
+ bit_array_.release();
+ }
+ }
+
+ static bool ProtoIsValid(const serialization::BloomFilter &bloom_filter_proto) {
+ return bloom_filter_proto.IsInitialized();
+ }
+
+ /**
* @brief Zeros out the contents of the bit array.
**/
inline void reset() {
// Initialize the bit_array with all zeros.
- std::fill_n(bit_array_, array_size_in_bytes_, 0x00);
+ std::fill_n(bit_array_.get(), array_size_in_bytes_, 0x00);
inserted_element_count_ = 0;
}
/**
+ * @brief Get the random seed that was used to initialize this bloom filter.
+ *
+ * @return Returns the random seed.
+ **/
+ inline std::uint64_t getRandomSeed() const {
+ return random_seed_;
+ }
+
+ /**
+ * @brief Get the number of hash functions used in this bloom filter.
+ *
+ * @return Returns the number of hash functions.
+ **/
+ inline std::uint32_t getNumberOfHashes() const {
+ return hash_fn_count_;
+ }
+
+ /**
+ * @brief Get the size of the bit array in bytes for this bloom filter.
+ *
+ * @return Returns the bit array size (in bytes).
+ **/
+ inline std::uint64_t getBitArraySize() const {
+ return array_size_in_bytes_;
+ }
+
+ /**
+ * @brief Get the constant pointer to the bit array for this bloom filter
+ *
+ * @return Returns constant pointer to the bit array.
+ **/
+ inline const std::uint8_t* getBitArray() const {
+ return bit_array_.get();
+ }
+
+ /**
+ * @brief Inserts a given value into the bloom filter in a thread-safe manner.
+ *
+ * @param key_begin A pointer to the value being inserted.
+ * @param length Size of the value being inserted in bytes.
+ */
+ inline void insert(const std::uint8_t *key_begin, const std::size_t length) {
+ // Locks are needed during insertion, when multiple workers may be modifying the
+ // bloom filter concurrently. However, locks are not required during membership test.
+ std::size_t bit_index = 0;
+ std::size_t bit = 0;
+ std::vector<std::pair<std::size_t, std::size_t>> modified_bit_positions;
+ std::vector<bool> is_bit_position_correct;
+
+ // Determine all the bit positions that are required to be set.
+ for (std::size_t i = 0; i < hash_fn_count_; ++i) {
+ compute_indices(hash_ap(key_begin, length, hash_fn_[i]), &bit_index, &bit);
+ modified_bit_positions.push_back(std::make_pair(bit_index, bit));
+ }
+
+ // Acquire a reader lock and check which of the bit positions are already set.
+ {
+ SpinSharedMutexSharedLock<false> shared_reader_lock(bloom_filter_insert_mutex_);
+ for (std::size_t i = 0; i < hash_fn_count_; ++i) {
+ bit_index = modified_bit_positions[i].first;
+ bit = modified_bit_positions[i].second;
+ if (((bit_array_.get())[bit_index / kNumBitsPerByte] & (1 << bit)) != (1 << bit)) {
+ is_bit_position_correct.push_back(false);
+ } else {
+ is_bit_position_correct.push_back(true);
+ }
+ }
+ }
+
+ // Acquire a writer lock and set the bit positions are which are not set.
+ {
+ SpinSharedMutexExclusiveLock<false> exclusive_writer_lock(bloom_filter_insert_mutex_);
+ for (std::size_t i = 0; i < hash_fn_count_; ++i) {
+ if (!is_bit_position_correct[i]) {
+ bit_index = modified_bit_positions[i].first;
+ bit = modified_bit_positions[i].second;
+ (bit_array_.get())[bit_index / kNumBitsPerByte] |= (1 << bit);
+ }
+ }
+ }
+ ++inserted_element_count_;
+ }
+
+ /**
* @brief Inserts a given value into the bloom filter.
+ * @Warning This is a faster thread-unsafe version of the insert() function.
+ * The caller needs to ensure the thread safety.
*
* @param key_begin A pointer to the value being inserted.
* @param length Size of the value being inserted in bytes.
*/
- inline void insert(const std::uint8_t *key_begin, const std::size_t &length) {
+ inline void insertUnSafe(const std::uint8_t *key_begin, const std::size_t length) {
std::size_t bit_index = 0;
std::size_t bit = 0;
+
for (std::size_t i = 0; i < hash_fn_count_; ++i) {
compute_indices(hash_ap(key_begin, length, hash_fn_[i]), &bit_index, &bit);
- bit_array_[bit_index / kNumBitsPerByte] |= (1 << bit);
+ (bit_array_.get())[bit_index / kNumBitsPerByte] |= (1 << bit);
}
+
++inserted_element_count_;
}
@@ -102,6 +255,9 @@ class BloomFilter {
* If true is returned, then a value may or may not be present in the bloom filter.
* If false is returned, a value is certainly not present in the bloom filter.
*
+ * @note The membersip test does not require any locks, because the assumption is that
+ * the bloom filter will only be used after it has been built.
+ *
* @param key_begin A pointer to the value being tested for membership.
* @param length Size of the value being inserted in bytes.
*/
@@ -110,7 +266,7 @@ class BloomFilter {
std::size_t bit = 0;
for (std::size_t i = 0; i < hash_fn_count_; ++i) {
compute_indices(hash_ap(key_begin, length, hash_fn_[i]), &bit_index, &bit);
- if ((bit_array_[bit_index / kNumBitsPerByte] & (1 << bit)) != (1 << bit)) {
+ if (((bit_array_.get())[bit_index / kNumBitsPerByte] & (1 << bit)) != (1 << bit)) {
return false;
}
}
@@ -118,6 +274,19 @@ class BloomFilter {
}
/**
+ * @brief Perform a bitwise-OR of the given Bloom filter with this bloom filter.
+ * Essentially, it does a union of this bloom filter with the passed bloom filter.
+ *
+ * @param bloom_filter A const pointer to the bloom filter object to do bitwise-OR with.
+ */
+ inline void bitwiseOr(const BloomFilter *bloom_filter) {
+ SpinSharedMutexExclusiveLock<false> exclusive_writer_lock(bloom_filter_insert_mutex_);
+ for (std::size_t byte_index = 0; byte_index < bloom_filter->getBitArraySize(); ++byte_index) {
+ (bit_array_.get())[byte_index] |= bloom_filter->getBitArray()[byte_index];
+ }
+ }
+
+ /**
* @brief Return the number of elements currently inserted into bloom filter.
*
* @return The number of elements inserted into bloom filter.
@@ -219,13 +388,16 @@ class BloomFilter {
}
private:
+ const std::uint64_t random_seed_;
std::vector<std::uint32_t> hash_fn_;
- std::uint8_t *bit_array_;
const std::uint32_t hash_fn_count_;
- std::uint64_t array_size_;
std::uint64_t array_size_in_bytes_;
+ std::uint64_t array_size_;
+ std::unique_ptr<std::uint8_t> bit_array_;
std::uint32_t inserted_element_count_;
- const std::uint64_t random_seed_;
+ const bool is_bit_array_owner_;
+
+ alignas(kCacheLineBytes) mutable SpinSharedMutex<false> bloom_filter_insert_mutex_;
DISALLOW_COPY_AND_ASSIGN(BloomFilter);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/utility/BloomFilter.proto
----------------------------------------------------------------------
diff --git a/utility/BloomFilter.proto b/utility/BloomFilter.proto
new file mode 100644
index 0000000..8dd9163
--- /dev/null
+++ b/utility/BloomFilter.proto
@@ -0,0 +1,30 @@
+// Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+// University of Wisconsin\u2014Madison.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package quickstep.serialization;
+
+message BloomFilter {
+ // The default values were determined from empirical experiments.
+ // These values control the amount of false positivity that
+ // is expected from Bloom Filter.
+ // - Default seed for initializing family of hashes = 0xA5A5A5A55A5A5A5A.
+ // - Default bloom filter size = 10 KB.
+ // - Default number of hash functions used in bloom filter = 5.
+ optional fixed64 bloom_filter_seed = 1 [default = 0xA5A5A5A55A5A5A5A];
+ optional uint32 bloom_filter_size = 2 [default = 10000];
+ optional uint32 number_of_hashes = 3 [default = 5];
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index bb59f65..6d1eeab 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -146,6 +146,10 @@ configure_file (
"${CMAKE_CURRENT_BINARY_DIR}/UtilityConfig.h"
)
+QS_PROTOBUF_GENERATE_CPP(quickstep_utility_BloomFilter_proto_srcs
+ quickstep_utility_BloomFilter_proto_hdrs
+ BloomFilter.proto)
+
QS_PROTOBUF_GENERATE_CPP(quickstep_utility_SortConfiguration_proto_srcs
quickstep_utility_SortConfiguration_proto_hdrs
SortConfiguration.proto)
@@ -155,6 +159,9 @@ add_library(quickstep_utility_Alignment ../empty_src.cpp Alignment.hpp)
add_library(quickstep_utility_BitManipulation ../empty_src.cpp BitManipulation.hpp)
add_library(quickstep_utility_BitVector ../empty_src.cpp BitVector.hpp)
add_library(quickstep_utility_BloomFilter ../empty_src.cpp BloomFilter.hpp)
+add_library(quickstep_utility_BloomFilter_proto
+ ${quickstep_utility_BloomFilter_proto_srcs}
+ ${quickstep_utility_BloomFilter_proto_hdrs})
add_library(quickstep_utility_CalculateInstalledMemory CalculateInstalledMemory.cpp CalculateInstalledMemory.hpp)
add_library(quickstep_utility_Cast ../empty_src.cpp Cast.hpp)
add_library(quickstep_utility_CheckSnprintf ../empty_src.cpp CheckSnprintf.hpp)
@@ -202,7 +209,14 @@ target_link_libraries(quickstep_utility_BitVector
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_BloomFilter
glog
+ quickstep_storage_StorageConstants
+ quickstep_threading_Mutex
+ quickstep_threading_SharedMutex
+ quickstep_threading_SpinSharedMutex
+ quickstep_utility_BloomFilter_proto
quickstep_utility_Macros)
+target_link_libraries(quickstep_utility_BloomFilter_proto
+ ${PROTOBUF_LIBRARY})
target_link_libraries(quickstep_utility_CalculateInstalledMemory
glog)
target_link_libraries(quickstep_utility_CheckSnprintf
@@ -271,6 +285,7 @@ target_link_libraries(quickstep_utility
quickstep_utility_BitManipulation
quickstep_utility_BitVector
quickstep_utility_BloomFilter
+ quickstep_utility_BloomFilter_proto
quickstep_utility_CalculateInstalledMemory
quickstep_utility_Cast
quickstep_utility_CheckSnprintf
[11/50] [abbrv] incubator-quickstep git commit: Fixes a bug where
numa settings were not properly set (not compiled in) (#205)
Posted by hb...@apache.org.
Fixes a bug where numa settings were not properly set (not compiled in) (#205)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/456b4348
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/456b4348
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/456b4348
Branch: refs/heads/query-manager-used-in-foreman
Commit: 456b4348ddf8d6cb1f8b4c125e561da7fe205322
Parents: 29a71ac
Author: Marc S <cr...@users.noreply.github.com>
Authored: Tue May 3 12:06:38 2016 -0500
Committer: Harshad Deshmukh <d....@gmail.com>
Committed: Tue May 3 12:06:38 2016 -0500
----------------------------------------------------------------------
cli/DefaultsConfigurator.hpp | 16 ++++++++++++++++
cli/InputParserUtil.cpp | 1 +
cli/QuickstepCli.cpp | 6 +++---
3 files changed, 20 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/456b4348/cli/DefaultsConfigurator.hpp
----------------------------------------------------------------------
diff --git a/cli/DefaultsConfigurator.hpp b/cli/DefaultsConfigurator.hpp
index b40ef87..4da05b2 100644
--- a/cli/DefaultsConfigurator.hpp
+++ b/cli/DefaultsConfigurator.hpp
@@ -22,6 +22,7 @@
#include <unordered_map>
#include <vector>
+#include "storage/StorageConfig.h"
#include "utility/Macros.hpp"
#ifdef QUICKSTEP_HAVE_LIBNUMA
@@ -51,6 +52,21 @@ class DefaultsConfigurator {
}
/**
+ * @brief Get the number of available numa sockets.
+ *
+ * @return Number of available numa sockets. Always 1 if the system doesn't
+ * have libnuma.
+ **/
+ static std::size_t GetNumNUMANodes() {
+ #ifdef QUICKSTEP_HAVE_LIBNUMA
+ // Id of the maximum node.
+ return numa_max_node() + 1;
+ #else
+ return 1;
+ #endif
+ }
+
+ /**
* @brief Get the number of NUMA nodes covered by the given worker affinities
* to the CPU cores.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/456b4348/cli/InputParserUtil.cpp
----------------------------------------------------------------------
diff --git a/cli/InputParserUtil.cpp b/cli/InputParserUtil.cpp
index 328aaeb..352883e 100644
--- a/cli/InputParserUtil.cpp
+++ b/cli/InputParserUtil.cpp
@@ -24,6 +24,7 @@
#include <vector>
#include "catalog/CatalogConfig.h"
+#include "storage/StorageConfig.h"
#include "utility/StringUtil.hpp"
#include "glog/logging.h"
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/456b4348/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 8dee1f7..ec195f7 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -258,8 +258,7 @@ int main(int argc, char* argv[]) {
InputParserUtil::ParseWorkerAffinities(real_num_workers,
quickstep::FLAGS_worker_affinities);
- const std::size_t num_numa_nodes_covered =
- DefaultsConfigurator::GetNumNUMANodesCoveredByWorkers(worker_cpu_affinities);
+ const std::size_t num_numa_nodes_system = DefaultsConfigurator::GetNumNUMANodes();
if (quickstep::FLAGS_preload_buffer_pool) {
std::chrono::time_point<std::chrono::steady_clock> preload_start, preload_end;
@@ -280,7 +279,8 @@ int main(int argc, char* argv[]) {
Foreman foreman(&bus,
query_processor->getDefaultDatabase(),
query_processor->getStorageManager(),
- num_numa_nodes_covered);
+ -1, /* CPU id to bind foreman. -1 is unbound. */
+ num_numa_nodes_system);
// Get the NUMA affinities for workers.
vector<int> cpu_numa_nodes = InputParserUtil::GetNUMANodesForCPUs();
[49/50] [abbrv] incubator-quickstep git commit: Bug fixed in isFull()
function.
Posted by hb...@apache.org.
Bug fixed in isFull() function.
- Earlier check for isFull() was that whether there is at least one
empty bucket available for insertion.
- Now isFull() can check if there is enough space adding specified
number of buckets.
- To reproduce the bug behind this fix: Run TPC-H Q2 on SF100 dataset after
running the \analyze command. The optimizer allocates smaller than
necessary space for one of the hash joins. The resize() is triggered
but never gets executed because of the restrictive isFull() condition.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/69c65518
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/69c65518
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/69c65518
Branch: refs/heads/query-manager-used-in-foreman
Commit: 69c65518ff603081e724be581badffc7efeabead
Parents: 50b4e55
Author: Harshad Deshmukh <ha...@cs.wisc.edu>
Authored: Fri Jun 3 15:28:52 2016 -0500
Committer: Harshad Deshmukh <ha...@cs.wisc.edu>
Committed: Fri Jun 3 15:28:52 2016 -0500
----------------------------------------------------------------------
storage/SimpleScalarSeparateChainingHashTable.hpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/69c65518/storage/SimpleScalarSeparateChainingHashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/SimpleScalarSeparateChainingHashTable.hpp b/storage/SimpleScalarSeparateChainingHashTable.hpp
index eda6c86..962a66c 100644
--- a/storage/SimpleScalarSeparateChainingHashTable.hpp
+++ b/storage/SimpleScalarSeparateChainingHashTable.hpp
@@ -230,10 +230,10 @@ class SimpleScalarSeparateChainingHashTable : public HashTable<ValueT,
HashTablePreallocationState *prealloc_state);
// Determine whether it is actually necessary to resize this hash table.
- // Checks that there is at least one unallocated bucket.
- inline bool isFull() const {
- return header_->buckets_allocated.load(std::memory_order_relaxed)
- >= header_->num_buckets;
+ // Checks that there are at least ``extra_buckets`` unallocated buckets.
+ inline bool isFull(const std::size_t extra_buckets) const {
+ return (header_->buckets_allocated.load(std::memory_order_relaxed) +
+ extra_buckets) >= header_->num_buckets;
}
// Cache the TypeID of the key.
@@ -831,7 +831,7 @@ void SimpleScalarSeparateChainingHashTable<ValueT,
// Recheck whether the hash table is still full. Note that multiple threads
// might wait to rebuild this hash table simultaneously. Only the first one
// should do the rebuild.
- if (!isFull()) {
+ if (!isFull(extra_buckets)) {
return;
}