You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by hb...@apache.org on 2016/06/09 01:09:30 UTC
[01/48] incubator-quickstep git commit: Refactored HashJoinWorkOrder
protos. (#203)
Repository: incubator-quickstep
Updated Branches:
refs/heads/query-manager-used-in-foreman e8ead8610 -> d70ba5d1d
Refactored HashJoinWorkOrder protos. (#203)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/2ab9cd94
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/2ab9cd94
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/2ab9cd94
Branch: refs/heads/query-manager-used-in-foreman
Commit: 2ab9cd9446a585f095ffa0e2b4ace462facc38de
Parents: abfc5f2
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Fri Apr 29 21:11:23 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:39 2016 -0700
----------------------------------------------------------------------
relational_operators/CMakeLists.txt | 3 +
relational_operators/HashJoinOperator.cpp | 5 +-
relational_operators/WorkOrder.proto | 98 +++----
relational_operators/WorkOrderFactory.cpp | 362 +++++++++----------------
4 files changed, 162 insertions(+), 306 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ab9cd94/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index 759a233..a4600e6 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -181,6 +181,9 @@ target_link_libraries(quickstep_relationaloperators_HashJoinOperator
quickstep_storage_TupleReference
quickstep_storage_TupleStorageSubBlock
quickstep_storage_ValueAccessor
+ quickstep_types_Type
+ quickstep_types_TypedValue
+ quickstep_types_containers_ColumnVector
quickstep_types_containers_ColumnVectorsValueAccessor
quickstep_utility_Macros
tmb)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ab9cd94/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index 82f6b2a..104a02d 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -42,6 +42,9 @@
#include "storage/TupleReference.hpp"
#include "storage/TupleStorageSubBlock.hpp"
#include "storage/ValueAccessor.hpp"
+#include "types/Type.hpp"
+#include "types/TypedValue.hpp"
+#include "types/containers/ColumnVector.hpp"
#include "types/containers/ColumnVectorsValueAccessor.hpp"
#include "gflags/gflags.h"
@@ -820,7 +823,7 @@ void HashOuterJoinWorkOrder::execute() {
// where x is an attribute of the build relation.
// In that case, this HashOuterJoinWorkOrder needs to be updated to
// correctly handle the selections.
- const Type& column_type = selection_[i]->getType().getNullableVersion();
+ const Type &column_type = selection_[i]->getType().getNullableVersion();
if (NativeColumnVector::UsableForType(column_type)) {
NativeColumnVector *result = new NativeColumnVector(
column_type, num_tuples_without_matches);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ab9cd94/relational_operators/WorkOrder.proto
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrder.proto b/relational_operators/WorkOrder.proto
index 8ed2080..5d0619a 100644
--- a/relational_operators/WorkOrder.proto
+++ b/relational_operators/WorkOrder.proto
@@ -29,21 +29,18 @@ enum WorkOrderType {
DESTROY_HASH = 6;
DROP_TABLE = 7;
FINALIZE_AGGREGATION = 8;
- HASH_ANTI_JOIN = 9;
- HASH_INNER_JOIN = 10;
- HASH_OUTER_JOIN = 11;
- HASH_SEMI_JOIN = 12;
- INSERT = 13;
- NESTED_LOOP_JOIN = 14;
- SAMPLE = 15;
- SAVE_BLOCKS = 16;
- SELECT = 17;
- SORT_MERGE_RUN = 18;
- SORT_RUN_GENERATION = 19;
- TABLE_GENERATOR = 20;
- TEXT_SCAN = 21;
- TEXT_SPLIT = 22;
- UPDATE = 23;
+ HASH_JOIN = 9;
+ INSERT = 10;
+ NESTED_LOOP_JOIN = 11;
+ SAMPLE = 12;
+ SAVE_BLOCKS = 13;
+ SELECT = 14;
+ SORT_MERGE_RUN = 15;
+ SORT_RUN_GENERATION = 16;
+ TABLE_GENERATOR = 17;
+ TEXT_SCAN = 18;
+ TEXT_SPLIT = 19;
+ UPDATE = 20;
}
message WorkOrder {
@@ -107,63 +104,30 @@ message FinalizeAggregationWorkOrder {
}
}
-message HashInnerJoinWorkOrder {
- extend WorkOrder {
- // All required.
- optional int32 build_relation_id = 160;
- optional int32 probe_relation_id = 161;
- repeated int32 join_key_attributes = 162;
- optional bool any_join_key_attributes_nullable = 163;
- optional int32 insert_destination_index = 164;
- optional uint32 join_hash_table_index = 165;
- optional int32 residual_predicate_index = 166;
- optional int32 selection_index = 167;
- optional fixed64 block_id = 168;
- }
-}
-
-message HashAntiJoinWorkOrder {
- extend WorkOrder {
- // All required.
- optional int32 build_relation_id = 350;
- optional int32 probe_relation_id = 351;
- repeated int32 join_key_attributes = 352;
- optional bool any_join_key_attributes_nullable = 353;
- optional int32 insert_destination_index = 354;
- optional uint32 join_hash_table_index = 355;
- optional int32 residual_predicate_index = 356;
- optional int32 selection_index = 357;
- optional fixed64 block_id = 358;
+message HashJoinWorkOrder {
+ enum HashJoinWorkOrderType {
+ HASH_ANTI_JOIN = 0;
+ HASH_INNER_JOIN = 1;
+ HASH_OUTER_JOIN = 2;
+ HASH_SEMI_JOIN = 3;
}
-}
-message HashSemiJoinWorkOrder {
extend WorkOrder {
// All required.
- optional int32 build_relation_id = 360;
- optional int32 probe_relation_id = 361;
- repeated int32 join_key_attributes = 362;
- optional bool any_join_key_attributes_nullable = 363;
- optional int32 insert_destination_index = 364;
- optional uint32 join_hash_table_index = 365;
- optional int32 residual_predicate_index = 366;
- optional int32 selection_index = 367;
- optional fixed64 block_id = 368;
- }
-}
+ optional HashJoinWorkOrderType hash_join_work_order_type = 160;
+ optional int32 build_relation_id = 161;
+ optional int32 probe_relation_id = 162;
+ repeated int32 join_key_attributes = 163;
+ optional bool any_join_key_attributes_nullable = 164;
+ optional int32 insert_destination_index = 165;
+ optional uint32 join_hash_table_index = 166;
+ optional int32 selection_index = 167;
+ optional fixed64 block_id = 168;
-message HashOuterJoinWorkOrder {
- extend WorkOrder {
- // All required.
- optional int32 build_relation_id = 370;
- optional int32 probe_relation_id = 371;
- repeated int32 join_key_attributes = 372;
- optional bool any_join_key_attributes_nullable = 373;
- optional int32 insert_destination_index = 374;
- optional uint32 join_hash_table_index = 375;
- optional int32 selection_index = 376;
- repeated bool is_selection_on_build = 377;
- optional fixed64 block_id = 378;
+ // Used by all but HashOuterJoinWorkOrder.
+ optional int32 residual_predicate_index = 169;
+ // Used by HashOuterJoinWorkOrder only.
+ repeated bool is_selection_on_build = 170;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ab9cd94/relational_operators/WorkOrderFactory.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrderFactory.cpp b/relational_operators/WorkOrderFactory.cpp
index 964c11c..4157d0f 100644
--- a/relational_operators/WorkOrderFactory.cpp
+++ b/relational_operators/WorkOrderFactory.cpp
@@ -16,6 +16,7 @@
#include "relational_operators/WorkOrderFactory.hpp"
+#include <memory>
#include <utility>
#include <vector>
@@ -55,6 +56,10 @@ using std::vector;
namespace quickstep {
+class InsertDestination;
+class Predicate;
+class Scalar;
+
WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder &proto,
CatalogDatabaseLite *catalog_database,
QueryContext *query_context,
@@ -135,123 +140,115 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
query_context->getInsertDestination(
proto.GetExtension(serialization::FinalizeAggregationWorkOrder::insert_destination_index)));
}
- case serialization::HASH_ANTI_JOIN: {
- LOG(INFO) << "Creating HashAntiJoinWorkOrder";
- vector<attribute_id> join_key_attributes;
- const int join_key_attributes_size =
- proto.ExtensionSize(serialization::HashAntiJoinWorkOrder::join_key_attributes);
- for (int i = 0; i < join_key_attributes_size; ++i) {
- join_key_attributes.push_back(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::join_key_attributes, i));
- }
+ case serialization::HASH_JOIN: {
+ const auto hash_join_work_order_type =
+ proto.GetExtension(serialization::HashJoinWorkOrder::hash_join_work_order_type);
- return new HashAntiJoinWorkOrder(
+ const CatalogRelationSchema &build_relation =
catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::build_relation_id)),
+ proto.GetExtension(serialization::HashJoinWorkOrder::build_relation_id));
+ const CatalogRelationSchema &probe_relation =
catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::probe_relation_id)),
- move(join_key_attributes),
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::any_join_key_attributes_nullable),
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::block_id),
- query_context->getPredicate(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::residual_predicate_index)),
- query_context->getScalarGroup(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::selection_index)),
- *query_context->getJoinHashTable(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::join_hash_table_index)),
- query_context->getInsertDestination(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::insert_destination_index)),
- storage_manager);
- }
- case serialization::HASH_INNER_JOIN: {
- LOG(INFO) << "Creating HashInnerJoinWorkOrder";
- vector<attribute_id> join_key_attributes;
- const int join_key_attributes_size =
- proto.ExtensionSize(serialization::HashInnerJoinWorkOrder::join_key_attributes);
- for (int i = 0; i < join_key_attributes_size; ++i) {
- join_key_attributes.push_back(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::join_key_attributes, i));
- }
+ proto.GetExtension(serialization::HashJoinWorkOrder::probe_relation_id));
- return new HashInnerJoinWorkOrder(
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::build_relation_id)),
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::probe_relation_id)),
- move(join_key_attributes),
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::any_join_key_attributes_nullable),
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::block_id),
- query_context->getPredicate(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::residual_predicate_index)),
- query_context->getScalarGroup(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::selection_index)),
- *query_context->getJoinHashTable(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::join_hash_table_index)),
- query_context->getInsertDestination(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::insert_destination_index)),
- storage_manager);
- }
- case serialization::HASH_OUTER_JOIN: {
- LOG(INFO) << "Creating HashOuterJoinWorkOrder";
vector<attribute_id> join_key_attributes;
const int join_key_attributes_size =
- proto.ExtensionSize(serialization::HashOuterJoinWorkOrder::join_key_attributes);
+ proto.ExtensionSize(serialization::HashJoinWorkOrder::join_key_attributes);
for (int i = 0; i < join_key_attributes_size; ++i) {
join_key_attributes.push_back(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::join_key_attributes, i));
- }
- vector<bool> is_selection_on_build;
- const int is_selection_on_build_size =
- proto.ExtensionSize(serialization::HashOuterJoinWorkOrder::is_selection_on_build);
- for (int i = 0; i < is_selection_on_build_size; ++i) {
- is_selection_on_build.push_back(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::is_selection_on_build, i));
+ proto.GetExtension(serialization::HashJoinWorkOrder::join_key_attributes, i));
}
- return new HashOuterJoinWorkOrder(
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::build_relation_id)),
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::probe_relation_id)),
- move(join_key_attributes),
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::any_join_key_attributes_nullable),
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::block_id),
- query_context->getScalarGroup(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::selection_index)),
- move(is_selection_on_build),
- *query_context->getJoinHashTable(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::join_hash_table_index)),
- query_context->getInsertDestination(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::insert_destination_index)),
- storage_manager);
- }
- case serialization::HASH_SEMI_JOIN: {
- LOG(INFO) << "Creating HashSemiJoinWorkOrder";
- vector<attribute_id> join_key_attributes;
- const int join_key_attributes_size =
- proto.ExtensionSize(serialization::HashSemiJoinWorkOrder::join_key_attributes);
- for (int i = 0; i < join_key_attributes_size; ++i) {
- join_key_attributes.push_back(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::join_key_attributes, i));
+ const bool any_join_key_attributes_nullable =
+ proto.GetExtension(serialization::HashJoinWorkOrder::any_join_key_attributes_nullable);
+ const block_id lookup_block_id =
+ proto.GetExtension(serialization::HashJoinWorkOrder::block_id);
+
+ const Predicate *residual_predicate = nullptr;
+ if (hash_join_work_order_type != serialization::HashJoinWorkOrder::HASH_OUTER_JOIN) {
+ residual_predicate =
+ query_context->getPredicate(
+ proto.GetExtension(serialization::HashJoinWorkOrder::residual_predicate_index));
}
- return new HashSemiJoinWorkOrder(
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::build_relation_id)),
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::probe_relation_id)),
- move(join_key_attributes),
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::any_join_key_attributes_nullable),
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::block_id),
- query_context->getPredicate(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::residual_predicate_index)),
+ const std::vector<std::unique_ptr<const Scalar>> &selection =
query_context->getScalarGroup(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::selection_index)),
+ proto.GetExtension(serialization::HashJoinWorkOrder::selection_index));
+ const JoinHashTable &hash_table =
*query_context->getJoinHashTable(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::join_hash_table_index)),
+ proto.GetExtension(serialization::HashJoinWorkOrder::join_hash_table_index));
+ InsertDestination *output_destination =
query_context->getInsertDestination(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::insert_destination_index)),
- storage_manager);
+ proto.GetExtension(serialization::HashJoinWorkOrder::insert_destination_index));
+
+ switch (hash_join_work_order_type) {
+ case serialization::HashJoinWorkOrder::HASH_ANTI_JOIN: {
+ LOG(INFO) << "Creating HashAntiJoinWorkOrder";
+ return new HashAntiJoinWorkOrder(
+ build_relation,
+ probe_relation,
+ move(join_key_attributes),
+ any_join_key_attributes_nullable,
+ lookup_block_id,
+ residual_predicate,
+ selection,
+ hash_table,
+ output_destination,
+ storage_manager);
+ }
+ case serialization::HashJoinWorkOrder::HASH_INNER_JOIN: {
+ LOG(INFO) << "Creating HashInnerJoinWorkOrder";
+ return new HashInnerJoinWorkOrder(
+ build_relation,
+ probe_relation,
+ move(join_key_attributes),
+ any_join_key_attributes_nullable,
+ lookup_block_id,
+ residual_predicate,
+ selection,
+ hash_table,
+ output_destination,
+ storage_manager);
+ }
+ case serialization::HashJoinWorkOrder::HASH_OUTER_JOIN: {
+ vector<bool> is_selection_on_build;
+ const int is_selection_on_build_size =
+ proto.ExtensionSize(serialization::HashJoinWorkOrder::is_selection_on_build);
+ for (int i = 0; i < is_selection_on_build_size; ++i) {
+ is_selection_on_build.push_back(
+ proto.GetExtension(serialization::HashJoinWorkOrder::is_selection_on_build, i));
+ }
+
+ LOG(INFO) << "Creating HashOuterJoinWorkOrder";
+ return new HashOuterJoinWorkOrder(
+ build_relation,
+ probe_relation,
+ move(join_key_attributes),
+ any_join_key_attributes_nullable,
+ lookup_block_id,
+ selection,
+ move(is_selection_on_build),
+ hash_table,
+ output_destination,
+ storage_manager);
+ }
+ case serialization::HashJoinWorkOrder::HASH_SEMI_JOIN: {
+ LOG(INFO) << "Creating HashSemiJoinWorkOrder";
+ return new HashSemiJoinWorkOrder(
+ build_relation,
+ probe_relation,
+ move(join_key_attributes),
+ any_join_key_attributes_nullable,
+ lookup_block_id,
+ residual_predicate,
+ selection,
+ hash_table,
+ output_destination,
+ storage_manager);
+ }
+ default:
+ LOG(FATAL) << "Unknown HashJoinWorkOrder Type in WorkOrderFactory::ReconstructFromProto";
+ }
}
case serialization::INSERT: {
LOG(INFO) << "Creating InsertWorkOrder";
@@ -486,179 +483,68 @@ bool WorkOrderFactory::ProtoIsValid(const serialization::WorkOrder &proto,
query_context.isValidInsertDestinationId(
proto.GetExtension(serialization::FinalizeAggregationWorkOrder::insert_destination_index));
}
- case serialization::HASH_ANTI_JOIN: {
- if (!proto.HasExtension(serialization::HashAntiJoinWorkOrder::build_relation_id) ||
- !proto.HasExtension(serialization::HashAntiJoinWorkOrder::probe_relation_id)) {
+ case serialization::HASH_JOIN: {
+ if (!proto.HasExtension(serialization::HashJoinWorkOrder::hash_join_work_order_type)) {
return false;
}
- const relation_id build_relation_id =
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::build_relation_id);
- if (!catalog_database.hasRelationWithId(build_relation_id)) {
+ const auto hash_join_work_order_type =
+ proto.GetExtension(serialization::HashJoinWorkOrder::hash_join_work_order_type);
+ if (!serialization::HashJoinWorkOrder_HashJoinWorkOrderType_IsValid(hash_join_work_order_type)) {
return false;
}
- const relation_id probe_relation_id =
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::probe_relation_id);
- if (!catalog_database.hasRelationWithId(probe_relation_id)) {
- return false;
- }
-
- const CatalogRelationSchema &build_relation = catalog_database.getRelationSchemaById(build_relation_id);
- const CatalogRelationSchema &probe_relation = catalog_database.getRelationSchemaById(probe_relation_id);
- for (int i = 0; i < proto.ExtensionSize(serialization::HashAntiJoinWorkOrder::join_key_attributes); ++i) {
- const attribute_id attr_id =
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::join_key_attributes, i);
- if (!build_relation.hasAttributeWithId(attr_id) ||
- !probe_relation.hasAttributeWithId(attr_id)) {
- return false;
- }
- }
-
- return proto.HasExtension(serialization::HashAntiJoinWorkOrder::any_join_key_attributes_nullable) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::insert_destination_index) &&
- query_context.isValidInsertDestinationId(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::insert_destination_index)) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::join_hash_table_index) &&
- query_context.isValidJoinHashTableId(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::join_hash_table_index)) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::residual_predicate_index) &&
- query_context.isValidPredicate(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::residual_predicate_index)) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::selection_index) &&
- query_context.isValidScalarGroupId(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::selection_index)) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::block_id);
- }
- case serialization::HASH_INNER_JOIN: {
- if (!proto.HasExtension(serialization::HashInnerJoinWorkOrder::build_relation_id) ||
- !proto.HasExtension(serialization::HashInnerJoinWorkOrder::probe_relation_id)) {
+ if (!proto.HasExtension(serialization::HashJoinWorkOrder::build_relation_id) ||
+ !proto.HasExtension(serialization::HashJoinWorkOrder::probe_relation_id)) {
return false;
}
const relation_id build_relation_id =
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::build_relation_id);
+ proto.GetExtension(serialization::HashJoinWorkOrder::build_relation_id);
if (!catalog_database.hasRelationWithId(build_relation_id)) {
return false;
}
const relation_id probe_relation_id =
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::probe_relation_id);
+ proto.GetExtension(serialization::HashJoinWorkOrder::probe_relation_id);
if (!catalog_database.hasRelationWithId(probe_relation_id)) {
return false;
}
const CatalogRelationSchema &build_relation = catalog_database.getRelationSchemaById(build_relation_id);
const CatalogRelationSchema &probe_relation = catalog_database.getRelationSchemaById(probe_relation_id);
- for (int i = 0; i < proto.ExtensionSize(serialization::HashInnerJoinWorkOrder::join_key_attributes); ++i) {
+ for (int i = 0; i < proto.ExtensionSize(serialization::HashJoinWorkOrder::join_key_attributes); ++i) {
const attribute_id attr_id =
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::join_key_attributes, i);
+ proto.GetExtension(serialization::HashJoinWorkOrder::join_key_attributes, i);
if (!build_relation.hasAttributeWithId(attr_id) ||
!probe_relation.hasAttributeWithId(attr_id)) {
return false;
}
}
- return proto.HasExtension(serialization::HashInnerJoinWorkOrder::any_join_key_attributes_nullable) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::insert_destination_index) &&
- query_context.isValidInsertDestinationId(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::insert_destination_index)) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::join_hash_table_index) &&
- query_context.isValidJoinHashTableId(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::join_hash_table_index)) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::residual_predicate_index) &&
- query_context.isValidPredicate(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::residual_predicate_index)) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::selection_index) &&
- query_context.isValidScalarGroupId(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::selection_index)) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::block_id);
- }
- case serialization::HASH_OUTER_JOIN: {
- if (!proto.HasExtension(serialization::HashOuterJoinWorkOrder::build_relation_id) ||
- !proto.HasExtension(serialization::HashOuterJoinWorkOrder::probe_relation_id)) {
- return false;
- }
-
- const relation_id build_relation_id =
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::build_relation_id);
- if (!catalog_database.hasRelationWithId(build_relation_id)) {
- return false;
- }
-
- const relation_id probe_relation_id =
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::probe_relation_id);
- if (!catalog_database.hasRelationWithId(probe_relation_id)) {
- return false;
- }
-
- const CatalogRelationSchema &build_relation = catalog_database.getRelationSchemaById(build_relation_id);
- const CatalogRelationSchema &probe_relation = catalog_database.getRelationSchemaById(probe_relation_id);
- for (int i = 0; i < proto.ExtensionSize(serialization::HashOuterJoinWorkOrder::join_key_attributes); ++i) {
- const attribute_id attr_id =
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::join_key_attributes, i);
- if (!build_relation.hasAttributeWithId(attr_id) ||
- !probe_relation.hasAttributeWithId(attr_id)) {
+ if (hash_join_work_order_type == serialization::HashJoinWorkOrder::HASH_OUTER_JOIN) {
+ if (!proto.HasExtension(serialization::HashJoinWorkOrder::is_selection_on_build)) {
return false;
}
- }
-
- return proto.HasExtension(serialization::HashOuterJoinWorkOrder::any_join_key_attributes_nullable) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::insert_destination_index) &&
- query_context.isValidInsertDestinationId(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::insert_destination_index)) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::join_hash_table_index) &&
- query_context.isValidJoinHashTableId(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::join_hash_table_index)) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::selection_index) &&
- query_context.isValidScalarGroupId(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::selection_index)) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::is_selection_on_build) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::block_id);
- }
- case serialization::HASH_SEMI_JOIN: {
- if (!proto.HasExtension(serialization::HashSemiJoinWorkOrder::build_relation_id) ||
- !proto.HasExtension(serialization::HashSemiJoinWorkOrder::probe_relation_id)) {
- return false;
- }
-
- const relation_id build_relation_id =
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::build_relation_id);
- if (!catalog_database.hasRelationWithId(build_relation_id)) {
- return false;
- }
-
- const relation_id probe_relation_id =
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::probe_relation_id);
- if (!catalog_database.hasRelationWithId(probe_relation_id)) {
- return false;
- }
-
- const CatalogRelationSchema &build_relation = catalog_database.getRelationSchemaById(build_relation_id);
- const CatalogRelationSchema &probe_relation = catalog_database.getRelationSchemaById(probe_relation_id);
- for (int i = 0; i < proto.ExtensionSize(serialization::HashSemiJoinWorkOrder::join_key_attributes); ++i) {
- const attribute_id attr_id =
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::join_key_attributes, i);
- if (!build_relation.hasAttributeWithId(attr_id) ||
- !probe_relation.hasAttributeWithId(attr_id)) {
+ } else {
+ if (!proto.HasExtension(serialization::HashJoinWorkOrder::residual_predicate_index) ||
+ !query_context.isValidPredicate(
+ proto.GetExtension(serialization::HashJoinWorkOrder::residual_predicate_index))) {
return false;
}
}
- return proto.HasExtension(serialization::HashSemiJoinWorkOrder::any_join_key_attributes_nullable) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::insert_destination_index) &&
+ return proto.HasExtension(serialization::HashJoinWorkOrder::any_join_key_attributes_nullable) &&
+ proto.HasExtension(serialization::HashJoinWorkOrder::insert_destination_index) &&
query_context.isValidInsertDestinationId(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::insert_destination_index)) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::join_hash_table_index) &&
+ proto.GetExtension(serialization::HashJoinWorkOrder::insert_destination_index)) &&
+ proto.HasExtension(serialization::HashJoinWorkOrder::join_hash_table_index) &&
query_context.isValidJoinHashTableId(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::join_hash_table_index)) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::residual_predicate_index) &&
- query_context.isValidPredicate(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::residual_predicate_index)) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::selection_index) &&
+ proto.GetExtension(serialization::HashJoinWorkOrder::join_hash_table_index)) &&
+ proto.HasExtension(serialization::HashJoinWorkOrder::selection_index) &&
query_context.isValidScalarGroupId(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::selection_index)) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::block_id);
+ proto.GetExtension(serialization::HashJoinWorkOrder::selection_index)) &&
+ proto.HasExtension(serialization::HashJoinWorkOrder::block_id);
}
case serialization::INSERT: {
return proto.HasExtension(serialization::InsertWorkOrder::insert_destination_index) &&
[34/48] incubator-quickstep git commit: Reverting the PR that gets
the number of rows (#233)
Posted by hb...@apache.org.
Reverting the PR that gets the number of rows (#233)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0115e961
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0115e961
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0115e961
Branch: refs/heads/query-manager-used-in-foreman
Commit: 0115e961b214287f34ee14944e540462fb720918
Parents: 12d7928
Author: Rogers Jeffrey Leo John <ro...@gmail.com>
Authored: Sat May 21 07:36:06 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:45 2016 -0700
----------------------------------------------------------------------
cli/CommandExecutor.cpp | 23 +++--------------------
cli/tests/command_executor/D.test | 14 +++++++-------
cli/tests/command_executor/Dt.test | 20 ++++++++++----------
3 files changed, 20 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0115e961/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 3cb3f86..ddcd38f 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -72,7 +72,6 @@ void executeDescribeDatabase(
// Column width initialized to 6 to take into account the header name
// and the column value table
int max_column_width = C::kInitMaxColumnWidth;
- vector<std::size_t> num_tuples;
vector<std::size_t> num_blocks;
const CatalogRelation *relation = nullptr;
if (arguments->size() == 0) {
@@ -80,8 +79,6 @@ void executeDescribeDatabase(
max_column_width =
std::max(static_cast<int>(rel.getName().length()), max_column_width);
num_blocks.push_back(rel.size_blocks());
- num_tuples.push_back(
- PrintToScreen::GetNumTuplesInRelation(rel, storage_manager));
}
} else {
const ParseString &table_name = arguments->front();
@@ -94,49 +91,35 @@ void executeDescribeDatabase(
max_column_width = std::max(static_cast<int>(relation->getName().length()),
max_column_width);
num_blocks.push_back(relation->size_blocks());
- num_tuples.push_back(PrintToScreen::GetNumTuplesInRelation(
- *relation,
- storage_manager));
}
// Only if we have relations work on the printing logic.
if (catalog_database.size() > 0) {
const std::size_t max_num_blocks = *std::max_element(num_blocks.begin(), num_blocks.end());
- const std::size_t max_num_rows = *std::max_element(num_tuples.begin(), num_tuples.end());
- const int max_num_rows_digits = std::max(PrintToScreen::GetNumberOfDigits(max_num_rows),
- C::kInitMaxColumnWidth);
const int max_num_blocks_digits = std::max(PrintToScreen::GetNumberOfDigits(max_num_blocks),
C::kInitMaxColumnWidth+2);
-
vector<int> column_widths;
column_widths.push_back(max_column_width +1);
column_widths.push_back(C::kInitMaxColumnWidth + 1);
column_widths.push_back(max_num_blocks_digits + 1);
- column_widths.push_back(max_num_rows_digits + 1);
fputs(" List of relations\n\n", out);
fprintf(out, "%-*s |", max_column_width+1, " Name");
fprintf(out, "%-*s |", C::kInitMaxColumnWidth, " Type");
- fprintf(out, "%-*s |", max_num_blocks_digits, " Blocks");
- fprintf(out, "%-*s\n", max_num_rows_digits, " Rows");
+ fprintf(out, "%-*s\n", max_num_blocks_digits, " Blocks");
PrintToScreen::printHBar(column_widths, out);
// If there are no argument print the entire list of tables
// else print the particular table only.
- vector<std::size_t>::const_iterator num_tuples_it = num_tuples.begin();
vector<std::size_t>::const_iterator num_blocks_it = num_blocks.begin();
if (arguments->size() == 0) {
for (const CatalogRelation &rel : catalog_database) {
fprintf(out, " %-*s |", max_column_width, rel.getName().c_str());
fprintf(out, " %-*s |", C::kInitMaxColumnWidth - 1, "table");
- fprintf(out, " %-*lu |", max_num_blocks_digits - 1, *num_blocks_it);
- fprintf(out, " %-*lu\n", max_num_rows_digits - 1, *num_tuples_it);
- ++num_tuples_it;
+ fprintf(out, " %-*lu\n", max_num_blocks_digits - 1, *num_blocks_it);
++num_blocks_it;
}
} else {
fprintf(out, " %-*s |", max_column_width, relation->getName().c_str());
fprintf(out, " %-*s |", C::kInitMaxColumnWidth -1, "table");
- fprintf(out, " %-*lu |", max_num_blocks_digits - 1, *num_blocks_it);
- fprintf(out, " %-*lu\n", max_num_rows_digits - 1, *num_tuples_it);
- ++num_tuples_it;
+ fprintf(out, " %-*lu\n", max_num_blocks_digits - 1, *num_blocks_it);
++num_blocks_it;
}
fputc('\n', out);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0115e961/cli/tests/command_executor/D.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/D.test b/cli/tests/command_executor/D.test
index 45f8d8b..d18ab47 100644
--- a/cli/tests/command_executor/D.test
+++ b/cli/tests/command_executor/D.test
@@ -104,13 +104,13 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
--
List of relations
- Name | Type | Blocks | Rows
-+--------------------------------------+-------+---------+-------+
- foo | table | 1 | 5
- foo2 | table | 1 | 2
- foo3 | table | 1 | 1
- foo4 | table | 0 | 0
- averylongtablenamethatseemstoneverend | table | 1 | 3
+ Name | Type | Blocks
++--------------------------------------+-------+---------+
+ foo | table | 1
+ foo2 | table | 1
+ foo3 | table | 1
+ foo4 | table | 0
+ averylongtablenamethatseemstoneverend | table | 1
==
\d invalidtable
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0115e961/cli/tests/command_executor/Dt.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/Dt.test b/cli/tests/command_executor/Dt.test
index 1de6360..65c6be3 100644
--- a/cli/tests/command_executor/Dt.test
+++ b/cli/tests/command_executor/Dt.test
@@ -52,22 +52,22 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
--
List of relations
- Name | Type | Blocks | Rows
-+--------------------------------------+-------+---------+-------+
- foo | table | 1 | 5
- foo2 | table | 1 | 2
- foo3 | table | 1 | 1
- foo4 | table | 0 | 0
- averylongtablenamethatseemstoneverend | table | 1 | 3
+ Name | Type | Blocks
++--------------------------------------+-------+---------+
+ foo | table | 1
+ foo2 | table | 1
+ foo3 | table | 1
+ foo4 | table | 0
+ averylongtablenamethatseemstoneverend | table | 1
==
\dt foo
--
List of relations
- Name | Type | Blocks | Rows
-+-------+-------+---------+-------+
- foo | table | 1 | 5
+ Name | Type | Blocks
++-------+-------+---------+
+ foo | table | 1
==
\dt invalidtable
[07/48] incubator-quickstep git commit: Fixes a bug where numa
settings were not properly set (not compiled in) (#205)
Posted by hb...@apache.org.
Fixes a bug where numa settings were not properly set (not compiled in) (#205)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/7bb83d2e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/7bb83d2e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/7bb83d2e
Branch: refs/heads/query-manager-used-in-foreman
Commit: 7bb83d2ebac6d7c1f000d410829bf1635b4d6986
Parents: f18c8d1
Author: Marc S <cr...@users.noreply.github.com>
Authored: Tue May 3 12:06:38 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:40 2016 -0700
----------------------------------------------------------------------
cli/DefaultsConfigurator.hpp | 16 ++++++++++++++++
cli/InputParserUtil.cpp | 1 +
cli/QuickstepCli.cpp | 6 +++---
3 files changed, 20 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7bb83d2e/cli/DefaultsConfigurator.hpp
----------------------------------------------------------------------
diff --git a/cli/DefaultsConfigurator.hpp b/cli/DefaultsConfigurator.hpp
index b40ef87..4da05b2 100644
--- a/cli/DefaultsConfigurator.hpp
+++ b/cli/DefaultsConfigurator.hpp
@@ -22,6 +22,7 @@
#include <unordered_map>
#include <vector>
+#include "storage/StorageConfig.h"
#include "utility/Macros.hpp"
#ifdef QUICKSTEP_HAVE_LIBNUMA
@@ -51,6 +52,21 @@ class DefaultsConfigurator {
}
/**
+ * @brief Get the number of available numa sockets.
+ *
+ * @return Number of available numa sockets. Always 1 if the system doesn't
+ * have libnuma.
+ **/
+ static std::size_t GetNumNUMANodes() {
+ #ifdef QUICKSTEP_HAVE_LIBNUMA
+ // Id of the maximum node.
+ return numa_max_node() + 1;
+ #else
+ return 1;
+ #endif
+ }
+
+ /**
* @brief Get the number of NUMA nodes covered by the given worker affinities
* to the CPU cores.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7bb83d2e/cli/InputParserUtil.cpp
----------------------------------------------------------------------
diff --git a/cli/InputParserUtil.cpp b/cli/InputParserUtil.cpp
index 328aaeb..352883e 100644
--- a/cli/InputParserUtil.cpp
+++ b/cli/InputParserUtil.cpp
@@ -24,6 +24,7 @@
#include <vector>
#include "catalog/CatalogConfig.h"
+#include "storage/StorageConfig.h"
#include "utility/StringUtil.hpp"
#include "glog/logging.h"
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7bb83d2e/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 8dee1f7..ec195f7 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -258,8 +258,7 @@ int main(int argc, char* argv[]) {
InputParserUtil::ParseWorkerAffinities(real_num_workers,
quickstep::FLAGS_worker_affinities);
- const std::size_t num_numa_nodes_covered =
- DefaultsConfigurator::GetNumNUMANodesCoveredByWorkers(worker_cpu_affinities);
+ const std::size_t num_numa_nodes_system = DefaultsConfigurator::GetNumNUMANodes();
if (quickstep::FLAGS_preload_buffer_pool) {
std::chrono::time_point<std::chrono::steady_clock> preload_start, preload_end;
@@ -280,7 +279,8 @@ int main(int argc, char* argv[]) {
Foreman foreman(&bus,
query_processor->getDefaultDatabase(),
query_processor->getStorageManager(),
- num_numa_nodes_covered);
+ -1, /* CPU id to bind foreman. -1 is unbound. */
+ num_numa_nodes_system);
// Get the NUMA affinities for workers.
vector<int> cpu_numa_nodes = InputParserUtil::GetNUMANodesForCPUs();
[28/48] incubator-quickstep git commit: Change the default value of
Joined Tuple Collector (#226)
Posted by hb...@apache.org.
Change the default value of Joined Tuple Collector (#226)
On NUMA boxes, the vector based joined tuple collector slows down queries.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/aa76e482
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/aa76e482
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/aa76e482
Branch: refs/heads/query-manager-used-in-foreman
Commit: aa76e4827adea01c46502fdb3f1c5cec30a81b9f
Parents: 205aa4f
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Wed May 18 11:56:53 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:44 2016 -0700
----------------------------------------------------------------------
relational_operators/HashJoinOperator.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/aa76e482/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index 104a02d..aa03794 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -59,7 +59,7 @@ namespace quickstep {
namespace {
-DEFINE_bool(vector_based_joined_tuple_collector, true,
+DEFINE_bool(vector_based_joined_tuple_collector, false,
"If true, use simple vector-based joined tuple collector in "
"hash join, with a final sort pass to group joined tuple pairs "
"by inner block. If false, use unordered_map based collector that "
[46/48] incubator-quickstep git commit: Reordered Query ID in
operators and work orders.
Posted by hb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/NestedLoopsJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/NestedLoopsJoinOperator.hpp b/relational_operators/NestedLoopsJoinOperator.hpp
index f165442..0b13842 100644
--- a/relational_operators/NestedLoopsJoinOperator.hpp
+++ b/relational_operators/NestedLoopsJoinOperator.hpp
@@ -59,6 +59,7 @@ class NestedLoopsJoinOperator : public RelationalOperator {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param left_input_relation The first relation in the join (order is not
* actually important).
* @param right_input_relation The second relation in the join (order is not
@@ -76,17 +77,17 @@ class NestedLoopsJoinOperator : public RelationalOperator {
* @param left_relation_is_stored If left_input_relation is a stored relation.
* @param right_relation_is_stored If right_input_relation is a stored
* relation.
- * @param query_id The ID of the query to which this operator belongs.
**/
- NestedLoopsJoinOperator(const CatalogRelation &left_input_relation,
- const CatalogRelation &right_input_relation,
- const CatalogRelation &output_relation,
- const QueryContext::insert_destination_id output_destination_index,
- const QueryContext::predicate_id join_predicate_index,
- const QueryContext::scalar_group_id selection_index,
- bool left_relation_is_stored,
- bool right_relation_is_stored,
- const std::size_t query_id)
+ NestedLoopsJoinOperator(
+ const std::size_t query_id,
+ const CatalogRelation &left_input_relation,
+ const CatalogRelation &right_input_relation,
+ const CatalogRelation &output_relation,
+ const QueryContext::insert_destination_id output_destination_index,
+ const QueryContext::predicate_id join_predicate_index,
+ const QueryContext::scalar_group_id selection_index,
+ bool left_relation_is_stored,
+ bool right_relation_is_stored)
: RelationalOperator(query_id),
left_input_relation_(left_input_relation),
right_input_relation_(right_input_relation),
@@ -96,10 +97,12 @@ class NestedLoopsJoinOperator : public RelationalOperator {
selection_index_(selection_index),
left_relation_is_stored_(left_relation_is_stored),
right_relation_is_stored_(right_relation_is_stored),
- left_relation_block_ids_(left_relation_is_stored ? left_input_relation.getBlocksSnapshot()
- : std::vector<block_id>()),
- right_relation_block_ids_(right_relation_is_stored ? right_input_relation.getBlocksSnapshot()
- : std::vector<block_id>()),
+ left_relation_block_ids_(left_relation_is_stored
+ ? left_input_relation.getBlocksSnapshot()
+ : std::vector<block_id>()),
+ right_relation_block_ids_(right_relation_is_stored
+ ? right_input_relation.getBlocksSnapshot()
+ : std::vector<block_id>()),
num_left_workorders_generated_(0),
num_right_workorders_generated_(0),
done_feeding_left_relation_(false),
@@ -222,6 +225,7 @@ class NestedLoopsJoinWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param left_input_relation The first relation in the join (order is not
* actually important).
* @param right_input_relation The second relation in the join (order is not
@@ -233,19 +237,19 @@ class NestedLoopsJoinWorkOrder : public WorkOrder {
* @param selection A list of Scalars corresponding to the relation attributes
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
- * @param query_id The ID of the query to which this operator belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
- NestedLoopsJoinWorkOrder(const CatalogRelationSchema &left_input_relation,
- const CatalogRelationSchema &right_input_relation,
- const block_id left_block_id,
- const block_id right_block_id,
- const Predicate *join_predicate,
- const std::vector<std::unique_ptr<const Scalar>> &selection,
- const std::size_t query_id,
- InsertDestination *output_destination,
- StorageManager *storage_manager)
+ NestedLoopsJoinWorkOrder(
+ const std::size_t query_id,
+ const CatalogRelationSchema &left_input_relation,
+ const CatalogRelationSchema &right_input_relation,
+ const block_id left_block_id,
+ const block_id right_block_id,
+ const Predicate *join_predicate,
+ const std::vector<std::unique_ptr<const Scalar>> &selection,
+ InsertDestination *output_destination,
+ StorageManager *storage_manager)
: WorkOrder(query_id),
left_input_relation_(left_input_relation),
right_input_relation_(right_input_relation),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/RebuildWorkOrder.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/RebuildWorkOrder.hpp b/relational_operators/RebuildWorkOrder.hpp
index ae876ba..86f8eaf 100644
--- a/relational_operators/RebuildWorkOrder.hpp
+++ b/relational_operators/RebuildWorkOrder.hpp
@@ -48,6 +48,7 @@ class RebuildWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this RebuildWorkOrder belongs.
* @param block_ref A MutableBlockReference to the storage block which needs
* to be rebuilt.
* @param input_operator_index The index of the relational operator in the
@@ -55,15 +56,15 @@ class RebuildWorkOrder : public WorkOrder {
* @param input_relation_id The ID of the CatalogRelation to which the given
* storage block belongs to.
* @param scheduler_client_id The TMB client ID of the scheduler thread.
- * @param query_id The ID of the query to which this RebuildWorkOrder belongs.
* @param bus A pointer to the TMB.
**/
- RebuildWorkOrder(MutableBlockReference &&block_ref,
- const std::size_t input_operator_index,
- const relation_id input_relation_id,
- const client_id scheduler_client_id,
- const std::size_t query_id,
- MessageBus *bus)
+ RebuildWorkOrder(
+ const std::size_t query_id,
+ MutableBlockReference &&block_ref, // NOLINT(whitespace/operators)
+ const std::size_t input_operator_index,
+ const relation_id input_relation_id,
+ const client_id scheduler_client_id,
+ MessageBus *bus)
: WorkOrder(query_id),
block_ref_(std::move(block_ref)),
input_operator_index_(input_operator_index),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/RelationalOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/RelationalOperator.hpp b/relational_operators/RelationalOperator.hpp
index aa93018..c173a0a 100644
--- a/relational_operators/RelationalOperator.hpp
+++ b/relational_operators/RelationalOperator.hpp
@@ -212,16 +212,17 @@ class RelationalOperator {
* @param blocking_dependencies_met If those dependencies which break the
* pipeline have been met.
**/
- explicit RelationalOperator(const std::size_t query_id = 0,
+ explicit RelationalOperator(const std::size_t query_id,
const bool blocking_dependencies_met = false)
- : blocking_dependencies_met_(blocking_dependencies_met),
- done_feeding_input_relation_(false),
- query_id_(query_id) {}
+ : query_id_(query_id),
+ blocking_dependencies_met_(blocking_dependencies_met),
+ done_feeding_input_relation_(false) {}
+
+ const std::size_t query_id_;
bool blocking_dependencies_met_;
bool done_feeding_input_relation_;
std::size_t op_index_;
- const std::size_t query_id_;
private:
DISALLOW_COPY_AND_ASSIGN(RelationalOperator);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/SampleOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SampleOperator.cpp b/relational_operators/SampleOperator.cpp
index b318ce4..8d5fade 100644
--- a/relational_operators/SampleOperator.cpp
+++ b/relational_operators/SampleOperator.cpp
@@ -54,11 +54,11 @@ bool SampleOperator::getAllWorkOrders(
for (const block_id input_block_id : input_relation_block_ids_) {
if (distribution(generator) <= probability) {
container->addNormalWorkOrder(
- new SampleWorkOrder(input_relation_,
+ new SampleWorkOrder(query_id_,
+ input_relation_,
input_block_id,
is_block_sample_,
percentage_,
- query_id_,
output_destination,
storage_manager),
op_index_);
@@ -68,15 +68,14 @@ bool SampleOperator::getAllWorkOrders(
// Add all the blocks for tuple sampling which would handle
// the sampling from each block
for (const block_id input_block_id : input_relation_block_ids_) {
- container->addNormalWorkOrder(
- new SampleWorkOrder(input_relation_,
- input_block_id,
- is_block_sample_,
- percentage_,
- query_id_,
- output_destination,
- storage_manager),
- op_index_);
+ container->addNormalWorkOrder(new SampleWorkOrder(query_id_,
+ input_relation_,
+ input_block_id,
+ is_block_sample_,
+ percentage_,
+ output_destination,
+ storage_manager),
+ op_index_);
}
}
started_ = true;
@@ -87,13 +86,14 @@ bool SampleOperator::getAllWorkOrders(
while (num_workorders_generated_ < input_relation_block_ids_.size()) {
if (distribution(generator) <= probability) {
container->addNormalWorkOrder(
- new SampleWorkOrder(input_relation_,
- input_relation_block_ids_[num_workorders_generated_],
- is_block_sample_,
- percentage_,
- query_id_,
- output_destination,
- storage_manager),
+ new SampleWorkOrder(
+ query_id_,
+ input_relation_,
+ input_relation_block_ids_[num_workorders_generated_],
+ is_block_sample_,
+ percentage_,
+ output_destination,
+ storage_manager),
op_index_);
++num_workorders_generated_;
}
@@ -101,13 +101,14 @@ bool SampleOperator::getAllWorkOrders(
} else {
while (num_workorders_generated_ < input_relation_block_ids_.size()) {
container->addNormalWorkOrder(
- new SampleWorkOrder(input_relation_,
- input_relation_block_ids_[num_workorders_generated_],
- is_block_sample_,
- percentage_,
- query_id_,
- output_destination,
- storage_manager),
+ new SampleWorkOrder(
+ query_id_,
+ input_relation_,
+ input_relation_block_ids_[num_workorders_generated_],
+ is_block_sample_,
+ percentage_,
+ output_destination,
+ storage_manager),
op_index_);
++num_workorders_generated_;
}
@@ -115,6 +116,7 @@ bool SampleOperator::getAllWorkOrders(
return done_feeding_input_relation_;
}
}
+
void SampleWorkOrder::execute() {
BlockReference block(
storage_manager_->getBlock(input_block_id_, input_relation_));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/SampleOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SampleOperator.hpp b/relational_operators/SampleOperator.hpp
index f65f28a..505daa2 100644
--- a/relational_operators/SampleOperator.hpp
+++ b/relational_operators/SampleOperator.hpp
@@ -54,8 +54,9 @@ class WorkOrdersContainer;
class SampleOperator : public RelationalOperator {
public:
/**
- * @brief Constructor for SampleOperator with the sampling percentage and type of sampling.
+ * @brief Constructor for SampleOperator with the sampling percentage and type of sampling.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param input_relation The relation to perform sampling over.
* @param output_relation The output relation.
* @param output_destination_index The index of the InsertDestination in the
@@ -65,15 +66,15 @@ class SampleOperator : public RelationalOperator {
* workorders.
* @param is_block_sample Flag indicating whether the sample type is block or tuple.
* @param percentage The percentage of data to be sampled.
- * @param query_id The ID of the query to which this operator belongs.
**/
- SampleOperator(const CatalogRelation &input_relation,
- const CatalogRelationSchema &output_relation,
- const QueryContext::insert_destination_id output_destination_index,
- const bool input_relation_is_stored,
- const bool is_block_sample,
- const int percentage,
- const std::size_t query_id)
+ SampleOperator(
+ const std::size_t query_id,
+ const CatalogRelation &input_relation,
+ const CatalogRelationSchema &output_relation,
+ const QueryContext::insert_destination_id output_destination_index,
+ const bool input_relation_is_stored,
+ const bool is_block_sample,
+ const int percentage)
: RelationalOperator(query_id),
input_relation_(input_relation),
output_relation_(output_relation),
@@ -81,8 +82,9 @@ class SampleOperator : public RelationalOperator {
input_relation_is_stored_(input_relation_is_stored),
is_block_sample_(is_block_sample),
percentage_(percentage),
- input_relation_block_ids_(input_relation_is_stored ? input_relation.getBlocksSnapshot()
- : std::vector<block_id>()),
+ input_relation_block_ids_(input_relation_is_stored
+ ? input_relation.getBlocksSnapshot()
+ : std::vector<block_id>()),
num_workorders_generated_(0),
started_(false) {}
@@ -133,11 +135,22 @@ class SampleOperator : public RelationalOperator {
**/
class SampleWorkOrder : public WorkOrder {
public:
- SampleWorkOrder(const CatalogRelationSchema &input_relation,
+ /**
+ * @brief Constructor.
+ *
+ * @param query_id The ID of the query to which this WorkOrder belongs.
+ * @param input_relation The relation to perform sampling over.
+ * @param input_block_id The block to sample.
+ * @param is_block_sample Flag indicating whether the sample type is block or tuple.
+ * @param percentage The percentage of data to be sampled.
+ * @param output_destination The InsertDestination to insert the sample results.
+ * @param storage_manager The StorageManager to use.
+ **/
+ SampleWorkOrder(const std::size_t query_id,
+ const CatalogRelationSchema &input_relation,
const block_id input_block_id,
const bool is_block_sample,
const int percentage,
- const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
: WorkOrder(query_id),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/SaveBlocksOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SaveBlocksOperator.cpp b/relational_operators/SaveBlocksOperator.cpp
index 3581090..8127d88 100644
--- a/relational_operators/SaveBlocksOperator.cpp
+++ b/relational_operators/SaveBlocksOperator.cpp
@@ -36,9 +36,9 @@ bool SaveBlocksOperator::getAllWorkOrders(
while (num_workorders_generated_ < destination_block_ids_.size()) {
container->addNormalWorkOrder(
new SaveBlocksWorkOrder(
+ query_id_,
destination_block_ids_[num_workorders_generated_],
force_,
- query_id_,
storage_manager),
op_index_);
++num_workorders_generated_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/SaveBlocksOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SaveBlocksOperator.hpp b/relational_operators/SaveBlocksOperator.hpp
index f4650bb..6e2c72b 100644
--- a/relational_operators/SaveBlocksOperator.hpp
+++ b/relational_operators/SaveBlocksOperator.hpp
@@ -55,7 +55,8 @@ class SaveBlocksOperator : public RelationalOperator {
* @param force If true, force writing of all blocks to disk, otherwise only
* write dirty blocks.
**/
- explicit SaveBlocksOperator(const std::size_t query_id, bool force = false)
+ explicit SaveBlocksOperator(const std::size_t query_id,
+ const bool force = false)
: RelationalOperator(query_id),
force_(force),
num_workorders_generated_(0) {}
@@ -96,15 +97,15 @@ class SaveBlocksWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param save_block_id The id of the block to save.
* @param force If true, force writing of all blocks to disk, otherwise only
* write dirty blocks.
- * @param query_id The ID of the query to which this operator belongs.
* @param storage_manager The StorageManager to use.
**/
- SaveBlocksWorkOrder(const block_id save_block_id,
+ SaveBlocksWorkOrder(const std::size_t query_id,
+ const block_id save_block_id,
const bool force,
- const std::size_t query_id,
StorageManager *storage_manager)
: WorkOrder(query_id),
save_block_id_(save_block_id),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/SelectOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.cpp b/relational_operators/SelectOperator.cpp
index 350890d..e9a96f3 100644
--- a/relational_operators/SelectOperator.cpp
+++ b/relational_operators/SelectOperator.cpp
@@ -42,29 +42,28 @@ void SelectOperator::addWorkOrders(WorkOrdersContainer *container,
InsertDestination *output_destination) {
if (input_relation_is_stored_) {
for (const block_id input_block_id : input_relation_block_ids_) {
- container->addNormalWorkOrder(
- new SelectWorkOrder(input_relation_,
- input_block_id,
- predicate,
- simple_projection_,
- simple_selection_,
- selection,
- query_id_,
- output_destination,
- storage_manager),
- op_index_);
+ container->addNormalWorkOrder(new SelectWorkOrder(query_id_,
+ input_relation_,
+ input_block_id,
+ predicate,
+ simple_projection_,
+ simple_selection_,
+ selection,
+ output_destination,
+ storage_manager),
+ op_index_);
}
} else {
while (num_workorders_generated_ < input_relation_block_ids_.size()) {
container->addNormalWorkOrder(
new SelectWorkOrder(
+ query_id_,
input_relation_,
input_relation_block_ids_[num_workorders_generated_],
predicate,
simple_projection_,
simple_selection_,
selection,
- query_id_,
output_destination,
storage_manager),
op_index_);
@@ -87,13 +86,13 @@ void SelectOperator::addPartitionAwareWorkOrders(WorkOrdersContainer *container,
input_relation_block_ids_in_partition_[part_id]) {
container->addNormalWorkOrder(
new SelectWorkOrder(
+ query_id_,
input_relation_,
input_block_id,
predicate,
simple_projection_,
simple_selection_,
selection,
- query_id_,
output_destination,
storage_manager,
placement_scheme_->getNUMANodeForBlock(input_block_id)),
@@ -108,13 +107,13 @@ void SelectOperator::addPartitionAwareWorkOrders(WorkOrdersContainer *container,
= input_relation_block_ids_in_partition_[part_id][num_workorders_generated_in_partition_[part_id]];
container->addNormalWorkOrder(
new SelectWorkOrder(
+ query_id_,
input_relation_,
block_in_partition,
predicate,
simple_projection_,
simple_selection_,
selection,
- query_id_,
output_destination,
storage_manager,
placement_scheme_->getNUMANodeForBlock(block_in_partition)),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/SelectOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.hpp b/relational_operators/SelectOperator.hpp
index 4f5b8ca..ac7b038 100644
--- a/relational_operators/SelectOperator.hpp
+++ b/relational_operators/SelectOperator.hpp
@@ -64,6 +64,7 @@ class SelectOperator : public RelationalOperator {
* @brief Constructor for selection with arbitrary expressions in projection
* list.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param input_relation The relation to perform selection over.
* @param output_relation The output relation.
* @param output_destination_index The index of the InsertDestination in the
@@ -76,23 +77,24 @@ class SelectOperator : public RelationalOperator {
* @param input_relation_is_stored If input_relation is a stored relation and
* is fully available to the operator before it can start generating
* workorders.
- * @param query_id The ID of the query to which this operator belongs.
**/
- SelectOperator(const CatalogRelation &input_relation,
- const CatalogRelation &output_relation,
- const QueryContext::insert_destination_id output_destination_index,
- const QueryContext::predicate_id predicate_index,
- const QueryContext::scalar_group_id selection_index,
- const bool input_relation_is_stored,
- const std::size_t query_id)
+ SelectOperator(
+ const std::size_t query_id,
+ const CatalogRelation &input_relation,
+ const CatalogRelation &output_relation,
+ const QueryContext::insert_destination_id output_destination_index,
+ const QueryContext::predicate_id predicate_index,
+ const QueryContext::scalar_group_id selection_index,
+ const bool input_relation_is_stored)
: RelationalOperator(query_id),
input_relation_(input_relation),
output_relation_(output_relation),
output_destination_index_(output_destination_index),
predicate_index_(predicate_index),
selection_index_(selection_index),
- input_relation_block_ids_(input_relation_is_stored ? input_relation.getBlocksSnapshot()
- : std::vector<block_id>()),
+ input_relation_block_ids_(input_relation_is_stored
+ ? input_relation.getBlocksSnapshot()
+ : std::vector<block_id>()),
num_workorders_generated_(0),
simple_projection_(false),
input_relation_is_stored_(input_relation_is_stored),
@@ -124,6 +126,7 @@ class SelectOperator : public RelationalOperator {
*
* @note selection_index_ is invalid, and will not be used for projection.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param input_relation The relation to perform selection over.
* @param output_relation The output relation.
* @param output_destination_index The index of the InsertDestination in the
@@ -136,15 +139,15 @@ class SelectOperator : public RelationalOperator {
* @param input_relation_is_stored If input_relation is a stored relation and
* is fully available to the operator before it can start generating
* workorders.
- * @param query_id The ID of the query to which this operator belongs.
**/
- SelectOperator(const CatalogRelation &input_relation,
- const CatalogRelation &output_relation,
- const QueryContext::insert_destination_id output_destination_index,
- const QueryContext::predicate_id predicate_index,
- std::vector<attribute_id> &&selection,
- const bool input_relation_is_stored,
- const std::size_t query_id)
+ SelectOperator(
+ const std::size_t query_id,
+ const CatalogRelation &input_relation,
+ const CatalogRelation &output_relation,
+ const QueryContext::insert_destination_id output_destination_index,
+ const QueryContext::predicate_id predicate_index,
+ std::vector<attribute_id> &&selection,
+ const bool input_relation_is_stored)
: RelationalOperator(query_id),
input_relation_(input_relation),
output_relation_(output_relation),
@@ -152,8 +155,9 @@ class SelectOperator : public RelationalOperator {
predicate_index_(predicate_index),
selection_index_(QueryContext::kInvalidScalarGroupId),
simple_selection_(std::move(selection)),
- input_relation_block_ids_(input_relation_is_stored ? input_relation.getBlocksSnapshot()
- : std::vector<block_id>()),
+ input_relation_block_ids_(input_relation_is_stored
+ ? input_relation.getBlocksSnapshot()
+ : std::vector<block_id>()),
num_workorders_generated_(0),
simple_projection_(true),
input_relation_is_stored_(input_relation_is_stored),
@@ -278,6 +282,7 @@ class SelectWorkOrder : public WorkOrder {
* @note Reference parameter selection is NOT owned by this class and must
* remain valid until after execute() is called.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param input_relation The relation to perform selection over.
* @param input_block_id The block id.
* @param predicate All tuples matching \c predicate will be selected (or NULL
@@ -287,18 +292,17 @@ class SelectWorkOrder : public WorkOrder {
* simple_projection is true.
* @param selection A list of Scalars which will be evaluated to project
* input tuples, used if \c simple_projection is false.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the selection
* results.
* @param storage_manager The StorageManager to use.
**/
- SelectWorkOrder(const CatalogRelationSchema &input_relation,
+ SelectWorkOrder(const std::size_t query_id,
+ const CatalogRelationSchema &input_relation,
const block_id input_block_id,
const Predicate *predicate,
const bool simple_projection,
const std::vector<attribute_id> &simple_selection,
const std::vector<std::unique_ptr<const Scalar>> *selection,
- const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager,
const numa_node_id numa_node = 0)
@@ -320,6 +324,7 @@ class SelectWorkOrder : public WorkOrder {
* @note Reference parameter selection is NOT owned by this class and must
* remain valid until after execute() is called.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param input_relation The relation to perform selection over.
* @param input_block_id The block id.
* @param predicate All tuples matching \c predicate will be selected (or NULL
@@ -329,18 +334,17 @@ class SelectWorkOrder : public WorkOrder {
* simple_projection is true.
* @param selection A list of Scalars which will be evaluated to project
* input tuples, used if \c simple_projection is false.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the selection
* results.
* @param storage_manager The StorageManager to use.
**/
- SelectWorkOrder(const CatalogRelationSchema &input_relation,
+ SelectWorkOrder(const std::size_t query_id,
+ const CatalogRelationSchema &input_relation,
const block_id input_block_id,
const Predicate *predicate,
const bool simple_projection,
std::vector<attribute_id> &&simple_selection,
const std::vector<std::unique_ptr<const Scalar>> *selection,
- const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager,
const numa_node_id numa_node = 0)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/SortMergeRunOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortMergeRunOperator.cpp b/relational_operators/SortMergeRunOperator.cpp
index 9db8de1..6bf5719 100644
--- a/relational_operators/SortMergeRunOperator.cpp
+++ b/relational_operators/SortMergeRunOperator.cpp
@@ -85,12 +85,12 @@ WorkOrder *SortMergeRunOperator::createWorkOrder(
// Create a work order from the merge job from merge tree.
return new SortMergeRunWorkOrder(
+ query_id_,
query_context->getSortConfig(sort_config_index_),
job->level > 0 ? run_relation_ : input_relation_,
std::move(job->runs),
top_k_,
job->level,
- query_id_,
output_destination,
storage_manager,
op_index_,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/SortMergeRunOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortMergeRunOperator.hpp b/relational_operators/SortMergeRunOperator.hpp
index f54e925..cfff8b9 100644
--- a/relational_operators/SortMergeRunOperator.hpp
+++ b/relational_operators/SortMergeRunOperator.hpp
@@ -72,6 +72,7 @@ class SortMergeRunOperator : public RelationalOperator {
/**
* @brief Constructor for merging sorted runs to generate a sorted relation.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param input_relation The relation to merge sorted blocks.
* @param output_relation The output relation.
* @param output_destination_index The index of the InsertDestination in the
@@ -88,18 +89,18 @@ class SortMergeRunOperator : public RelationalOperator {
* \c top_k is 0.
* @param input_relation_is_stored Boolean to indicate is input relation is
* stored or streamed.
- * @param query_id The ID of the query to which this operator belongs.
**/
- SortMergeRunOperator(const CatalogRelation &input_relation,
- const CatalogRelation &output_relation,
- const QueryContext::insert_destination_id output_destination_index,
- const CatalogRelation &run_relation,
- const QueryContext::insert_destination_id run_block_destination_index,
- const QueryContext::sort_config_id sort_config_index,
- const std::size_t merge_factor,
- const std::size_t top_k,
- const bool input_relation_is_stored,
- const std::size_t query_id)
+ SortMergeRunOperator(
+ const std::size_t query_id,
+ const CatalogRelation &input_relation,
+ const CatalogRelation &output_relation,
+ const QueryContext::insert_destination_id output_destination_index,
+ const CatalogRelation &run_relation,
+ const QueryContext::insert_destination_id run_block_destination_index,
+ const QueryContext::sort_config_id sort_config_index,
+ const std::size_t merge_factor,
+ const std::size_t top_k,
+ const bool input_relation_is_stored)
: RelationalOperator(query_id),
input_relation_(input_relation),
output_relation_(output_relation),
@@ -214,12 +215,12 @@ class SortMergeRunWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param sort_config The Sort configuration.
* @param run_relation The relation to which the run blocks belong to.
* @param input_runs Input runs to merge.
* @param top_k If non-zero will merge only \c top_k tuples.
* @param merge_level Merge level in the merge tree.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to create new blocks.
* @param storage_manager The StorageManager to use.
* @param operator_index Merge-run operator index to send feedback messages
@@ -228,12 +229,12 @@ class SortMergeRunWorkOrder : public WorkOrder {
* @param bus TMB to send the feedback message on.
**/
SortMergeRunWorkOrder(
+ const std::size_t query_id,
const SortConfiguration &sort_config,
const CatalogRelationSchema &run_relation,
std::vector<merge_run_operator::Run> &&input_runs,
const std::size_t top_k,
const std::size_t merge_level,
- const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager,
const std::size_t operator_index,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/SortRunGenerationOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortRunGenerationOperator.cpp b/relational_operators/SortRunGenerationOperator.cpp
index e352f9e..37b8fb8 100644
--- a/relational_operators/SortRunGenerationOperator.cpp
+++ b/relational_operators/SortRunGenerationOperator.cpp
@@ -51,10 +51,10 @@ bool SortRunGenerationOperator::getAllWorkOrders(
if (!started_) {
for (const block_id input_block_id : input_relation_block_ids_) {
container->addNormalWorkOrder(
- new SortRunGenerationWorkOrder(input_relation_,
+ new SortRunGenerationWorkOrder(query_id_,
+ input_relation_,
input_block_id,
sort_config,
- query_id_,
output_destination,
storage_manager),
op_index_);
@@ -67,10 +67,10 @@ bool SortRunGenerationOperator::getAllWorkOrders(
while (num_workorders_generated_ < input_relation_block_ids_.size()) {
container->addNormalWorkOrder(
new SortRunGenerationWorkOrder(
+ query_id_,
input_relation_,
input_relation_block_ids_[num_workorders_generated_],
sort_config,
- query_id_,
output_destination,
storage_manager),
op_index_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/SortRunGenerationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortRunGenerationOperator.hpp b/relational_operators/SortRunGenerationOperator.hpp
index 3da9813..f96e6a6 100644
--- a/relational_operators/SortRunGenerationOperator.hpp
+++ b/relational_operators/SortRunGenerationOperator.hpp
@@ -73,6 +73,7 @@ class SortRunGenerationOperator : public RelationalOperator {
* @brief Constructor for sorting tuples in blocks based on the sort
* configuration and writing to output destination.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param input_relation The relation to generate sorted runs of.
* @param output_relation The output relation.
* @param output_destination_index The index of the InsertDestination in the
@@ -83,21 +84,22 @@ class SortRunGenerationOperator : public RelationalOperator {
* @param input_relation_is_stored Does the input relation contain the blocks
* to sort. If \c false, the blocks are
* streamed.
- * @param query_id The ID of the query to which this operator belongs.
**/
- SortRunGenerationOperator(const CatalogRelation &input_relation,
- const CatalogRelation &output_relation,
- const QueryContext::insert_destination_id output_destination_index,
- const QueryContext::sort_config_id sort_config_index,
- bool input_relation_is_stored,
- const std::size_t query_id)
+ SortRunGenerationOperator(
+ const std::size_t query_id,
+ const CatalogRelation &input_relation,
+ const CatalogRelation &output_relation,
+ const QueryContext::insert_destination_id output_destination_index,
+ const QueryContext::sort_config_id sort_config_index,
+ bool input_relation_is_stored)
: RelationalOperator(query_id),
input_relation_(input_relation),
output_relation_(output_relation),
output_destination_index_(output_destination_index),
sort_config_index_(sort_config_index),
- input_relation_block_ids_(input_relation_is_stored ? input_relation.getBlocksSnapshot()
- : std::vector<block_id>()),
+ input_relation_block_ids_(input_relation_is_stored
+ ? input_relation.getBlocksSnapshot()
+ : std::vector<block_id>()),
num_workorders_generated_(0),
started_(false),
input_relation_is_stored_(input_relation_is_stored) {}
@@ -151,19 +153,19 @@ class SortRunGenerationWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param input_relation The relation to generate sorted runs of.
* @param input_block_id The block id.
* @param sort_config The Sort configuration specifying ORDER BY, ordering,
* and null ordering.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to store the sorted blocks
* of runs.
* @param storage_manager The StorageManager to use.
**/
- SortRunGenerationWorkOrder(const CatalogRelationSchema &input_relation,
+ SortRunGenerationWorkOrder(const std::size_t query_id,
+ const CatalogRelationSchema &input_relation,
const block_id input_block_id,
const SortConfiguration &sort_config,
- const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
: WorkOrder(query_id),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/TableGeneratorOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/TableGeneratorOperator.cpp b/relational_operators/TableGeneratorOperator.cpp
index fb1f743..a3f9340 100644
--- a/relational_operators/TableGeneratorOperator.cpp
+++ b/relational_operators/TableGeneratorOperator.cpp
@@ -43,9 +43,9 @@ bool TableGeneratorOperator::getAllWorkOrders(
// so just produce one work order.
container->addNormalWorkOrder(
new TableGeneratorWorkOrder(
+ query_id_,
query_context->getGeneratorFunctionHandle(
generator_function_index_),
- query_id_,
query_context->getInsertDestination(output_destination_index_)),
op_index_);
started_ = true;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/TableGeneratorOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/TableGeneratorOperator.hpp b/relational_operators/TableGeneratorOperator.hpp
index bfc70c5..6a6af4b 100644
--- a/relational_operators/TableGeneratorOperator.hpp
+++ b/relational_operators/TableGeneratorOperator.hpp
@@ -55,23 +55,23 @@ class TableGeneratorOperator : public RelationalOperator {
/**
* @brief Constructor
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param output_relation The output relation.
* @param output_destination_index The index of the InsertDestination in the
* QueryContext to insert the generated output.
* @param generator_function_index The index of the GeneratorFunctionHandle in
* the QueryContext.
- * @param query_id The ID of the query to which this operator belongs.
**/
- TableGeneratorOperator(const CatalogRelation &output_relation,
- const QueryContext::insert_destination_id output_destination_index,
- const QueryContext::generator_function_id generator_function_index,
- const std::size_t query_id)
+ TableGeneratorOperator(
+ const std::size_t query_id,
+ const CatalogRelation &output_relation,
+ const QueryContext::insert_destination_id output_destination_index,
+ const QueryContext::generator_function_id generator_function_index)
: RelationalOperator(query_id),
output_relation_(output_relation),
output_destination_index_(output_destination_index),
generator_function_index_(generator_function_index),
- started_(false) {
- }
+ started_(false) {}
~TableGeneratorOperator() override {}
@@ -113,13 +113,13 @@ class TableGeneratorWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
- * @param generator_function The GeneratorFunctionHandle to use.
* @param query_id The ID of the query to which this WorkOrder belongs.
+ * @param generator_function The GeneratorFunctionHandle to use.
* @param output_destination The InsertDestination to insert the generated
* output.
**/
- TableGeneratorWorkOrder(const GeneratorFunctionHandle &function_handle,
- const std::size_t query_id,
+ TableGeneratorWorkOrder(const std::size_t query_id,
+ const GeneratorFunctionHandle &function_handle,
InsertDestination *output_destination)
: WorkOrder(query_id),
function_handle_(function_handle),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/TextScanOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/TextScanOperator.cpp b/relational_operators/TextScanOperator.cpp
index 8db5ef1..5acecbf 100644
--- a/relational_operators/TextScanOperator.cpp
+++ b/relational_operators/TextScanOperator.cpp
@@ -165,9 +165,9 @@ bool TextScanOperator::getAllWorkOrders(
// First, generate text-split work orders.
for (const auto &file : files) {
container->addNormalWorkOrder(
- new TextSplitWorkOrder(file,
+ new TextSplitWorkOrder(query_id_,
+ file,
process_escape_sequences_,
- query_id_,
storage_manager,
op_index_,
scheduler_client_id,
@@ -182,11 +182,11 @@ bool TextScanOperator::getAllWorkOrders(
while (!text_blob_queue_.empty()) {
const TextBlob blob_work = text_blob_queue_.popOne();
container->addNormalWorkOrder(
- new TextScanWorkOrder(blob_work.blob_id,
+ new TextScanWorkOrder(query_id_,
+ blob_work.blob_id,
blob_work.size,
field_terminator_,
process_escape_sequences_,
- query_id_,
output_destination,
storage_manager),
op_index_);
@@ -203,10 +203,10 @@ bool TextScanOperator::getAllWorkOrders(
if (blocking_dependencies_met_ && !work_generated_) {
for (const auto &file : files) {
container->addNormalWorkOrder(
- new TextScanWorkOrder(file,
+ new TextScanWorkOrder(query_id_,
+ file,
field_terminator_,
process_escape_sequences_,
- query_id_,
output_destination,
storage_manager),
op_index_);
@@ -234,11 +234,10 @@ void TextScanOperator::receiveFeedbackMessage(const WorkOrder::FeedbackMessage &
}
}
-
-TextScanWorkOrder::TextScanWorkOrder(const std::string &filename,
+TextScanWorkOrder::TextScanWorkOrder(const std::size_t query_id,
+ const std::string &filename,
const char field_terminator,
const bool process_escape_sequences,
- const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
: WorkOrder(query_id),
@@ -254,11 +253,11 @@ TextScanWorkOrder::TextScanWorkOrder(const std::string &filename,
DCHECK(storage_manager_ != nullptr);
}
-TextScanWorkOrder::TextScanWorkOrder(const block_id text_blob,
+TextScanWorkOrder::TextScanWorkOrder(const std::size_t query_id,
+ const block_id text_blob,
const std::size_t text_size,
const char field_terminator,
const bool process_escape_sequences,
- const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
: WorkOrder(query_id),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/TextScanOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/TextScanOperator.hpp b/relational_operators/TextScanOperator.hpp
index 1d0c04f..3cda65b 100644
--- a/relational_operators/TextScanOperator.hpp
+++ b/relational_operators/TextScanOperator.hpp
@@ -121,6 +121,7 @@ class TextScanOperator : public RelationalOperator {
/**
* @brief Constructor
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param file_pattern The glob-like file pattern of the sources to load. The
* pattern could include * (wildcard for multiple chars) and ?
* (wildcard for single char). It defaults to single file load, if a
@@ -134,15 +135,15 @@ class TextScanOperator : public RelationalOperator {
* @param output_relation The output relation.
* @param output_destination_index The index of the InsertDestination in the
* QueryContext to insert tuples.
- * @param query_id The ID of the query to which this operator belongs.
**/
- TextScanOperator(const std::string &file_pattern,
- const char field_terminator,
- const bool process_escape_sequences,
- const bool parallelize_load,
- const CatalogRelation &output_relation,
- const QueryContext::insert_destination_id output_destination_index,
- const std::size_t query_id)
+ TextScanOperator(
+ const std::size_t query_id,
+ const std::string &file_pattern,
+ const char field_terminator,
+ const bool process_escape_sequences,
+ const bool parallelize_load,
+ const CatalogRelation &output_relation,
+ const QueryContext::insert_destination_id output_destination_index)
: RelationalOperator(query_id),
file_pattern_(file_pattern),
field_terminator_(field_terminator),
@@ -200,42 +201,42 @@ class TextScanWorkOrder : public WorkOrder {
/**
* @brief Constructor
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param filename The name of the text file to bulk insert.
* @param field_terminator The string which separates attribute values in
* the text file.
* @param process_escape_sequences Whether to decode escape sequences in the
* text file.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert tuples.
* @param storage_manager The StorageManager to use.
**/
TextScanWorkOrder(
+ const std::size_t query_id,
const std::string &filename,
const char field_terminator,
const bool process_escape_sequences,
- const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager);
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param text_blob Blob ID containing the data to be scanned.
* @param text_size Size of the data in the blob.
* @param field_terminator The character which separates attribute values in
* the text file.
* @param process_escape_sequences Whether to decode escape sequences in the
* text file.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to write the read tuples.
* @param storage_manager The StorageManager to use.
*/
TextScanWorkOrder(
+ const std::size_t query_id,
const block_id text_blob,
const std::size_t text_size,
const char field_terminator,
const bool process_escape_sequences,
- const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager);
@@ -322,19 +323,20 @@ class TextSplitWorkOrder : public WorkOrder {
public:
/**
* @brief Constructor.
+ *
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param filename File to split into row-aligned blobs.
* @param process_escape_sequences Whether to decode escape sequences in the
* text file.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param storage_manager The StorageManager to use.
* @param operator_index Operator index of the current operator. This is used
* to send new-work available message to Foreman.
* @param scheduler_client_id The TMB client ID of the scheduler thread.
* @param bus A pointer to the TMB.
*/
- TextSplitWorkOrder(const std::string &filename,
+ TextSplitWorkOrder(const std::size_t query_id,
+ const std::string &filename,
const bool process_escape_sequences,
- const std::size_t query_id,
StorageManager *storage_manager,
const std::size_t operator_index,
const tmb::client_id scheduler_client_id,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/UpdateOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/UpdateOperator.cpp b/relational_operators/UpdateOperator.cpp
index b331a9c..2130563 100644
--- a/relational_operators/UpdateOperator.cpp
+++ b/relational_operators/UpdateOperator.cpp
@@ -53,16 +53,18 @@ bool UpdateOperator::getAllWorkOrders(
for (const block_id input_block_id : input_blocks_) {
container->addNormalWorkOrder(
- new UpdateWorkOrder(relation_,
- input_block_id,
- query_context->getPredicate(predicate_index_),
- query_context->getUpdateGroup(update_group_index_),
- query_id_,
- query_context->getInsertDestination(relocation_destination_index_),
- storage_manager,
- op_index_,
- scheduler_client_id,
- bus),
+ new UpdateWorkOrder(
+ query_id_,
+ relation_,
+ input_block_id,
+ query_context->getPredicate(predicate_index_),
+ query_context->getUpdateGroup(update_group_index_),
+ query_context->getInsertDestination(
+ relocation_destination_index_),
+ storage_manager,
+ op_index_,
+ scheduler_client_id,
+ bus),
op_index_);
}
started_ = true;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/UpdateOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/UpdateOperator.hpp b/relational_operators/UpdateOperator.hpp
index ba2d6cf..cebb9b5 100644
--- a/relational_operators/UpdateOperator.hpp
+++ b/relational_operators/UpdateOperator.hpp
@@ -62,6 +62,7 @@ class UpdateOperator : public RelationalOperator {
/**
* @brief Constructor
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param relation The relation to perform the UPDATE over.
* @param relocation_destination_index The index of the InsertDestination in
* the QueryContext to relocate tuples which can not be updated
@@ -72,16 +73,16 @@ class UpdateOperator : public RelationalOperator {
* @param update_group_index The index of a update group (the map of
* attribute_ids to Scalars) which should be evaluated to get the new
* value for the corresponding attribute.
- * @param query_id The ID of the query to which this operator belongs.
*
* @warning The constructed InsertDestination should belong to relation, but
* must NOT contain any pre-existing blocks.
**/
- UpdateOperator(const CatalogRelation &relation,
- const QueryContext::insert_destination_id relocation_destination_index,
- const QueryContext::predicate_id predicate_index,
- const QueryContext::update_group_id update_group_index,
- const std::size_t query_id)
+ UpdateOperator(
+ const std::size_t query_id,
+ const CatalogRelation &relation,
+ const QueryContext::insert_destination_id relocation_destination_index,
+ const QueryContext::predicate_id predicate_index,
+ const QueryContext::update_group_id update_group_index)
: RelationalOperator(query_id),
relation_(relation),
relocation_destination_index_(relocation_destination_index),
@@ -127,13 +128,13 @@ class UpdateWorkOrder : public WorkOrder {
/**
* @brief Constructor
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param relation The relation to perform the UPDATE over.
* @param predicate All tuples matching \c predicate will be updated (or NULL
* to update all tuples).
* @param assignments The assignments (the map of attribute_ids to Scalars)
* which should be evaluated to get the new value for the corresponding
* attribute.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param input_block_id The block id.
* @param relocation_destination The InsertDestination to relocate tuples
* which can not be updated in-place.
@@ -143,16 +144,18 @@ class UpdateWorkOrder : public WorkOrder {
* @param scheduler_client_id The TMB client ID of the scheduler thread.
* @param bus A pointer to the TMB.
**/
- UpdateWorkOrder(const CatalogRelationSchema &relation,
- const block_id input_block_id,
- const Predicate *predicate,
- const std::unordered_map<attribute_id, std::unique_ptr<const Scalar>> &assignments,
- const std::size_t query_id,
- InsertDestination *relocation_destination,
- StorageManager *storage_manager,
- const std::size_t update_operator_index,
- const tmb::client_id scheduler_client_id,
- MessageBus *bus)
+ UpdateWorkOrder(
+ const std::size_t query_id,
+ const CatalogRelationSchema &relation,
+ const block_id input_block_id,
+ const Predicate *predicate,
+ const std::unordered_map<attribute_id, std::unique_ptr<const Scalar>>
+ &assignments,
+ InsertDestination *relocation_destination,
+ StorageManager *storage_manager,
+ const std::size_t update_operator_index,
+ const tmb::client_id scheduler_client_id,
+ MessageBus *bus)
: WorkOrder(query_id),
relation_(relation),
input_block_id_(input_block_id),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/WorkOrderFactory.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrderFactory.cpp b/relational_operators/WorkOrderFactory.cpp
index fdd694f..489b666 100644
--- a/relational_operators/WorkOrderFactory.cpp
+++ b/relational_operators/WorkOrderFactory.cpp
@@ -75,8 +75,8 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
case serialization::AGGREGATION: {
LOG(INFO) << "Creating AggregationWorkOrder";
return new AggregationWorkOrder(
- proto.GetExtension(serialization::AggregationWorkOrder::block_id),
proto.query_id(),
+ proto.GetExtension(serialization::AggregationWorkOrder::block_id),
query_context->getAggregationState(
proto.GetExtension(serialization::AggregationWorkOrder::aggr_state_index)));
}
@@ -89,12 +89,12 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
}
return new BuildHashWorkOrder(
+ proto.query_id(),
catalog_database->getRelationSchemaById(
proto.GetExtension(serialization::BuildHashWorkOrder::relation_id)),
move(join_key_attributes),
proto.GetExtension(serialization::BuildHashWorkOrder::any_join_key_attributes_nullable),
proto.GetExtension(serialization::BuildHashWorkOrder::block_id),
- proto.query_id(),
query_context->getJoinHashTable(
proto.GetExtension(serialization::BuildHashWorkOrder::join_hash_table_index)),
storage_manager);
@@ -102,6 +102,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
case serialization::DELETE: {
LOG(INFO) << "Creating DeleteWorkOrder";
return new DeleteWorkOrder(
+ proto.query_id(),
catalog_database->getRelationSchemaById(
proto.GetExtension(serialization::DeleteWorkOrder::relation_id)),
proto.GetExtension(serialization::DeleteWorkOrder::block_id),
@@ -110,15 +111,14 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
storage_manager,
proto.GetExtension(serialization::DeleteWorkOrder::operator_index),
shiftboss_client_id,
- proto.query_id(),
bus);
}
case serialization::DESTROY_HASH: {
LOG(INFO) << "Creating DestroyHashWorkOrder";
return new DestroyHashWorkOrder(
+ proto.query_id(),
proto.GetExtension(
serialization::DestroyHashWorkOrder::join_hash_table_index),
- proto.query_id(),
query_context);
}
case serialization::DROP_TABLE: {
@@ -193,6 +193,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
case serialization::HashJoinWorkOrder::HASH_ANTI_JOIN: {
LOG(INFO) << "Creating HashAntiJoinWorkOrder";
return new HashAntiJoinWorkOrder(
+ proto.query_id(),
build_relation,
probe_relation,
move(join_key_attributes),
@@ -201,13 +202,13 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
residual_predicate,
selection,
hash_table,
- proto.query_id(),
output_destination,
storage_manager);
}
case serialization::HashJoinWorkOrder::HASH_INNER_JOIN: {
LOG(INFO) << "Creating HashInnerJoinWorkOrder";
return new HashInnerJoinWorkOrder(
+ proto.query_id(),
build_relation,
probe_relation,
move(join_key_attributes),
@@ -216,7 +217,6 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
residual_predicate,
selection,
hash_table,
- proto.query_id(),
output_destination,
storage_manager);
}
@@ -231,6 +231,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
LOG(INFO) << "Creating HashOuterJoinWorkOrder";
return new HashOuterJoinWorkOrder(
+ proto.query_id(),
build_relation,
probe_relation,
move(join_key_attributes),
@@ -239,13 +240,13 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
selection,
move(is_selection_on_build),
hash_table,
- proto.query_id(),
output_destination,
storage_manager);
}
case serialization::HashJoinWorkOrder::HASH_SEMI_JOIN: {
LOG(INFO) << "Creating HashSemiJoinWorkOrder";
return new HashSemiJoinWorkOrder(
+ proto.query_id(),
build_relation,
probe_relation,
move(join_key_attributes),
@@ -254,7 +255,6 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
residual_predicate,
selection,
hash_table,
- proto.query_id(),
output_destination,
storage_manager);
}
@@ -274,6 +274,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
case serialization::NESTED_LOOP_JOIN: {
LOG(INFO) << "Creating NestedLoopsJoinWorkOrder";
return new NestedLoopsJoinWorkOrder(
+ proto.query_id(),
catalog_database->getRelationSchemaById(
proto.GetExtension(serialization::NestedLoopsJoinWorkOrder::left_relation_id)),
catalog_database->getRelationSchemaById(
@@ -284,7 +285,6 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::NestedLoopsJoinWorkOrder::join_predicate_index)),
query_context->getScalarGroup(
proto.GetExtension(serialization::NestedLoopsJoinWorkOrder::selection_index)),
- proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::NestedLoopsJoinWorkOrder::insert_destination_index)),
storage_manager);
@@ -292,12 +292,12 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
case serialization::SAMPLE: {
LOG(INFO) << "Creating SampleWorkOrder";
return new SampleWorkOrder(
+ proto.query_id(),
catalog_database->getRelationSchemaById(
proto.GetExtension(serialization::SampleWorkOrder::relation_id)),
proto.GetExtension(serialization::SampleWorkOrder::block_id),
proto.GetExtension(serialization::SampleWorkOrder::is_block_sample),
proto.GetExtension(serialization::SampleWorkOrder::percentage),
- proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::SampleWorkOrder::insert_destination_index)),
storage_manager);
@@ -305,9 +305,9 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
case serialization::SAVE_BLOCKS: {
LOG(INFO) << "Creating SaveBlocksWorkOrder";
return new SaveBlocksWorkOrder(
+ proto.query_id(),
proto.GetExtension(serialization::SaveBlocksWorkOrder::block_id),
proto.GetExtension(serialization::SaveBlocksWorkOrder::force),
- proto.query_id(),
storage_manager);
}
case serialization::SELECT: {
@@ -321,6 +321,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
}
return new SelectWorkOrder(
+ proto.query_id(),
catalog_database->getRelationSchemaById(
proto.GetExtension(serialization::SelectWorkOrder::relation_id)),
proto.GetExtension(serialization::SelectWorkOrder::block_id),
@@ -331,7 +332,6 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
simple_projection ? nullptr
: &query_context->getScalarGroup(
proto.GetExtension(serialization::SelectWorkOrder::selection_index)),
- proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::SelectWorkOrder::insert_destination_index)),
storage_manager);
@@ -350,6 +350,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
}
return new SortMergeRunWorkOrder(
+ proto.query_id(),
query_context->getSortConfig(
proto.GetExtension(serialization::SortMergeRunWorkOrder::sort_config_index)),
catalog_database->getRelationSchemaById(
@@ -357,7 +358,6 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
move(runs),
proto.GetExtension(serialization::SortMergeRunWorkOrder::top_k),
proto.GetExtension(serialization::SortMergeRunWorkOrder::merge_level),
- proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::SortMergeRunWorkOrder::insert_destination_index)),
storage_manager,
@@ -368,12 +368,12 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
case serialization::SORT_RUN_GENERATION: {
LOG(INFO) << "Creating SortRunGenerationWorkOrder";
return new SortRunGenerationWorkOrder(
+ proto.query_id(),
catalog_database->getRelationSchemaById(
proto.GetExtension(serialization::SortRunGenerationWorkOrder::relation_id)),
proto.GetExtension(serialization::SortRunGenerationWorkOrder::block_id),
query_context->getSortConfig(
proto.GetExtension(serialization::SortRunGenerationWorkOrder::sort_config_index)),
- proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::SortRunGenerationWorkOrder::insert_destination_index)),
storage_manager);
@@ -381,9 +381,9 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
case serialization::TABLE_GENERATOR: {
LOG(INFO) << "Creating SortRunGenerationWorkOrder";
return new TableGeneratorWorkOrder(
+ proto.query_id(),
query_context->getGeneratorFunctionHandle(
proto.GetExtension(serialization::TableGeneratorWorkOrder::generator_function_index)),
- proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::TableGeneratorWorkOrder::insert_destination_index)));
}
@@ -391,10 +391,10 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
LOG(INFO) << "Creating TextScanWorkOrder";
if (proto.HasExtension(serialization::TextScanWorkOrder::filename)) {
return new TextScanWorkOrder(
+ proto.query_id(),
proto.GetExtension(serialization::TextScanWorkOrder::filename),
proto.GetExtension(serialization::TextScanWorkOrder::field_terminator),
proto.GetExtension(serialization::TextScanWorkOrder::process_escape_sequences),
- proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::TextScanWorkOrder::insert_destination_index)),
storage_manager);
@@ -403,11 +403,11 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::TextScanWorkOrder::text_blob);
return new TextScanWorkOrder(
+ proto.query_id(),
text_blob_proto.blob_id(),
text_blob_proto.size(),
proto.GetExtension(serialization::TextScanWorkOrder::field_terminator),
proto.GetExtension(serialization::TextScanWorkOrder::process_escape_sequences),
- proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::TextScanWorkOrder::insert_destination_index)),
storage_manager);
@@ -416,9 +416,9 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
case serialization::TEXT_SPLIT: {
LOG(INFO) << "Creating TextSplitWorkOrder";
return new TextSplitWorkOrder(
+ proto.query_id(),
proto.GetExtension(serialization::TextSplitWorkOrder::filename),
proto.GetExtension(serialization::TextSplitWorkOrder::process_escape_sequences),
- proto.query_id(),
storage_manager,
proto.GetExtension(serialization::TextSplitWorkOrder::operator_index),
shiftboss_client_id,
@@ -427,6 +427,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
case serialization::UPDATE: {
LOG(INFO) << "Creating UpdateWorkOrder";
return new UpdateWorkOrder(
+ proto.query_id(),
catalog_database->getRelationSchemaById(
proto.GetExtension(serialization::UpdateWorkOrder::relation_id)),
proto.GetExtension(serialization::UpdateWorkOrder::block_id),
@@ -434,7 +435,6 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::UpdateWorkOrder::predicate_index)),
query_context->getUpdateGroup(
proto.GetExtension(serialization::UpdateWorkOrder::update_group_index)),
- proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::UpdateWorkOrder::insert_destination_index)),
storage_manager,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/tests/AggregationOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/AggregationOperator_unittest.cpp b/relational_operators/tests/AggregationOperator_unittest.cpp
index ace7951..fdcc54f 100644
--- a/relational_operators/tests/AggregationOperator_unittest.cpp
+++ b/relational_operators/tests/AggregationOperator_unittest.cpp
@@ -77,6 +77,7 @@ using std::unique_ptr;
namespace quickstep {
namespace {
+constexpr std::size_t kQueryId = 0;
constexpr int kOpIndex = 0;
} // namespace
@@ -270,7 +271,7 @@ class AggregationOperatorTest : public ::testing::Test {
aggr_state_proto->set_estimated_num_entries(estimated_entries);
// Create Operators.
- op_.reset(new AggregationOperator(*table_, true, aggr_state_index, 0));
+ op_.reset(new AggregationOperator(0, *table_, true, aggr_state_index));
// Setup the InsertDestination proto in the query context proto.
const QueryContext::insert_destination_id insert_destination_index =
@@ -281,10 +282,11 @@ class AggregationOperatorTest : public ::testing::Test {
insert_destination_proto->set_relation_id(result_table_->getID());
insert_destination_proto->set_relational_op_index(kOpIndex);
- finalize_op_.reset(new FinalizeAggregationOperator(aggr_state_index,
- *result_table_,
- insert_destination_index,
- 0 /* dummy query ID */));
+ finalize_op_.reset(
+ new FinalizeAggregationOperator(kQueryId,
+ aggr_state_index,
+ *result_table_,
+ insert_destination_index));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -354,7 +356,7 @@ class AggregationOperatorTest : public ::testing::Test {
serialization::HashTableImplType::LINEAR_OPEN_ADDRESSING);
// Create Operators.
- op_.reset(new AggregationOperator(*table_, true, aggr_state_index, 0));
+ op_.reset(new AggregationOperator(0, *table_, true, aggr_state_index));
// Setup the InsertDestination proto in the query context proto.
const QueryContext::insert_destination_id insert_destination_index =
@@ -365,10 +367,11 @@ class AggregationOperatorTest : public ::testing::Test {
insert_destination_proto->set_relation_id(result_table_->getID());
insert_destination_proto->set_relational_op_index(kOpIndex);
- finalize_op_.reset(new FinalizeAggregationOperator(aggr_state_index,
- *result_table_,
- insert_destination_index,
- 0 /* dummy query ID */));
+ finalize_op_.reset(
+ new FinalizeAggregationOperator(kQueryId,
+ aggr_state_index,
+ *result_table_,
+ insert_destination_index));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
[04/48] incubator-quickstep git commit: Fixes bug which disallows the
creation of BITWEAVING_H indices. (#204)
Posted by hb...@apache.org.
Fixes bug which disallows the creation of BITWEAVING_H indices. (#204)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f18c8d1b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f18c8d1b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f18c8d1b
Branch: refs/heads/query-manager-used-in-foreman
Commit: f18c8d1b59fc87bf819ce547ee5929d9fce810c6
Parents: 4f7725f
Author: Marc S <cr...@users.noreply.github.com>
Authored: Sun May 1 17:14:20 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:40 2016 -0700
----------------------------------------------------------------------
parser/ParseIndexProperties.hpp | 2 +-
parser/tests/Index.test | 11 +++++++++++
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f18c8d1b/parser/ParseIndexProperties.hpp
----------------------------------------------------------------------
diff --git a/parser/ParseIndexProperties.hpp b/parser/ParseIndexProperties.hpp
index 9ca1419..2cb5df9 100644
--- a/parser/ParseIndexProperties.hpp
+++ b/parser/ParseIndexProperties.hpp
@@ -220,7 +220,7 @@ class BitWeavingIndexProperties : public IndexProperties {
}
const std::string key = ToLower(key_value.key()->value());
const std::string value = ToLower(
- static_cast<const ParseKeyStringValue&>(key_value).key()->value());
+ static_cast<const ParseKeyStringValue&>(key_value).value()->value());
if (key.compare(kBitWeavingType) == 0) {
if (value.compare("h") == 0) {
index_sub_block_description_->set_sub_block_type(IndexSubBlockDescription::BITWEAVING_H);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f18c8d1b/parser/tests/Index.test
----------------------------------------------------------------------
diff --git a/parser/tests/Index.test b/parser/tests/Index.test
index 605768b..39cd523 100644
--- a/parser/tests/Index.test
+++ b/parser/tests/Index.test
@@ -66,3 +66,14 @@ CREATE INDEX bwIndex ON test(int_col) USING bitweaving;
CreateIndexStatement[index_name=bwIndex,relation_name=test,index_type=bitweaving]
+-attribute_list=
+-AttributeReference[attribute_name=int_col]
+==
+CREATE INDEX bwhIndex ON test(int_col) USING bitweaving (TYPE H);
+--
+CreateIndexStatement[index_name=bwhIndex,relation_name=test,
+ index_type=bitweaving]
++-attribute_list=
+| +-AttributeReference[attribute_name=int_col]
++-index_property_list=
+ +-IndexProperties
+ +-index_property=KeyStringValue[key=TYPE]
+ +-value=String[value=H]
[11/48] incubator-quickstep git commit: Added support for the
substring function. (#211)
Posted by hb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/parser/preprocessed/SqlParser_gen.cpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlParser_gen.cpp b/parser/preprocessed/SqlParser_gen.cpp
index f2cb8ca..038db14 100644
--- a/parser/preprocessed/SqlParser_gen.cpp
+++ b/parser/preprocessed/SqlParser_gen.cpp
@@ -244,67 +244,69 @@ extern int quickstep_yydebug;
TOKEN_FALSE = 314,
TOKEN_FIRST = 315,
TOKEN_FLOAT = 316,
- TOKEN_FOREIGN = 317,
- TOKEN_FROM = 318,
- TOKEN_FULL = 319,
- TOKEN_GROUP = 320,
- TOKEN_HASH = 321,
- TOKEN_HAVING = 322,
- TOKEN_HOUR = 323,
- TOKEN_IN = 324,
- TOKEN_INDEX = 325,
- TOKEN_INNER = 326,
- TOKEN_INSERT = 327,
- TOKEN_INTEGER = 328,
- TOKEN_INTERVAL = 329,
- TOKEN_INTO = 330,
- TOKEN_JOIN = 331,
- TOKEN_KEY = 332,
- TOKEN_LAST = 333,
- TOKEN_LEFT = 334,
- TOKEN_LIMIT = 335,
- TOKEN_LONG = 336,
- TOKEN_MINUTE = 337,
- TOKEN_MONTH = 338,
- TOKEN_NULL = 339,
- TOKEN_NULLS = 340,
- TOKEN_OFF = 341,
- TOKEN_ON = 342,
- TOKEN_ORDER = 343,
- TOKEN_OUTER = 344,
- TOKEN_PARTITION = 345,
- TOKEN_PARTITIONS = 346,
- TOKEN_PERCENT = 347,
- TOKEN_PRIMARY = 348,
- TOKEN_QUIT = 349,
- TOKEN_RANGE = 350,
- TOKEN_REAL = 351,
- TOKEN_REFERENCES = 352,
- TOKEN_RIGHT = 353,
- TOKEN_ROW_DELIMITER = 354,
- TOKEN_SECOND = 355,
- TOKEN_SELECT = 356,
- TOKEN_SET = 357,
- TOKEN_SMA = 358,
- TOKEN_SMALLINT = 359,
- TOKEN_TABLE = 360,
- TOKEN_THEN = 361,
- TOKEN_TIME = 362,
- TOKEN_TIMESTAMP = 363,
- TOKEN_TRUE = 364,
- TOKEN_TUPLESAMPLE = 365,
- TOKEN_UNIQUE = 366,
- TOKEN_UPDATE = 367,
- TOKEN_USING = 368,
- TOKEN_VALUES = 369,
- TOKEN_VARCHAR = 370,
- TOKEN_WHEN = 371,
- TOKEN_WHERE = 372,
- TOKEN_WITH = 373,
- TOKEN_YEAR = 374,
- TOKEN_YEARMONTH = 375,
- TOKEN_EOF = 376,
- TOKEN_LEX_ERROR = 377
+ TOKEN_FOR = 317,
+ TOKEN_FOREIGN = 318,
+ TOKEN_FROM = 319,
+ TOKEN_FULL = 320,
+ TOKEN_GROUP = 321,
+ TOKEN_HASH = 322,
+ TOKEN_HAVING = 323,
+ TOKEN_HOUR = 324,
+ TOKEN_IN = 325,
+ TOKEN_INDEX = 326,
+ TOKEN_INNER = 327,
+ TOKEN_INSERT = 328,
+ TOKEN_INTEGER = 329,
+ TOKEN_INTERVAL = 330,
+ TOKEN_INTO = 331,
+ TOKEN_JOIN = 332,
+ TOKEN_KEY = 333,
+ TOKEN_LAST = 334,
+ TOKEN_LEFT = 335,
+ TOKEN_LIMIT = 336,
+ TOKEN_LONG = 337,
+ TOKEN_MINUTE = 338,
+ TOKEN_MONTH = 339,
+ TOKEN_NULL = 340,
+ TOKEN_NULLS = 341,
+ TOKEN_OFF = 342,
+ TOKEN_ON = 343,
+ TOKEN_ORDER = 344,
+ TOKEN_OUTER = 345,
+ TOKEN_PARTITION = 346,
+ TOKEN_PARTITIONS = 347,
+ TOKEN_PERCENT = 348,
+ TOKEN_PRIMARY = 349,
+ TOKEN_QUIT = 350,
+ TOKEN_RANGE = 351,
+ TOKEN_REAL = 352,
+ TOKEN_REFERENCES = 353,
+ TOKEN_RIGHT = 354,
+ TOKEN_ROW_DELIMITER = 355,
+ TOKEN_SECOND = 356,
+ TOKEN_SELECT = 357,
+ TOKEN_SET = 358,
+ TOKEN_SMA = 359,
+ TOKEN_SMALLINT = 360,
+ TOKEN_SUBSTRING = 361,
+ TOKEN_TABLE = 362,
+ TOKEN_THEN = 363,
+ TOKEN_TIME = 364,
+ TOKEN_TIMESTAMP = 365,
+ TOKEN_TRUE = 366,
+ TOKEN_TUPLESAMPLE = 367,
+ TOKEN_UNIQUE = 368,
+ TOKEN_UPDATE = 369,
+ TOKEN_USING = 370,
+ TOKEN_VALUES = 371,
+ TOKEN_VARCHAR = 372,
+ TOKEN_WHEN = 373,
+ TOKEN_WHERE = 374,
+ TOKEN_WITH = 375,
+ TOKEN_YEAR = 376,
+ TOKEN_YEARMONTH = 377,
+ TOKEN_EOF = 378,
+ TOKEN_LEX_ERROR = 379
};
#endif
@@ -405,7 +407,7 @@ union YYSTYPE
quickstep::PtrVector<quickstep::ParseSubqueryTableReference> *with_list_;
quickstep::ParseSubqueryTableReference *with_list_element_;
-#line 409 "SqlParser_gen.cpp" /* yacc.c:355 */
+#line 411 "SqlParser_gen.cpp" /* yacc.c:355 */
};
typedef union YYSTYPE YYSTYPE;
@@ -440,7 +442,7 @@ int quickstep_yyparse (yyscan_t yyscanner, quickstep::ParseStatement **parsedSta
#include "SqlLexer_gen.hpp"
void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const std::string &feature);
-#line 444 "SqlParser_gen.cpp" /* yacc.c:358 */
+#line 446 "SqlParser_gen.cpp" /* yacc.c:358 */
#ifdef short
# undef short
@@ -684,21 +686,21 @@ union yyalloc
/* YYFINAL -- State number of the termination state. */
#define YYFINAL 47
/* YYLAST -- Last index in YYTABLE. */
-#define YYLAST 1191
+#define YYLAST 1215
/* YYNTOKENS -- Number of terminals. */
-#define YYNTOKENS 134
+#define YYNTOKENS 136
/* YYNNTS -- Number of nonterminals. */
-#define YYNNTS 95
+#define YYNNTS 96
/* YYNRULES -- Number of rules. */
-#define YYNRULES 262
+#define YYNRULES 265
/* YYNSTATES -- Number of states. */
-#define YYNSTATES 478
+#define YYNSTATES 488
/* YYTRANSLATE[YYX] -- Symbol number corresponding to YYX as returned
by yylex, with out-of-bounds checking. */
#define YYUNDEFTOK 2
-#define YYMAXUTOK 377
+#define YYMAXUTOK 379
#define YYTRANSLATE(YYX) \
((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK)
@@ -708,11 +710,11 @@ union yyalloc
static const yytype_uint8 yytranslate[] =
{
0, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 129, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 131, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 133, 2, 2,
- 130, 131, 23, 21, 132, 22, 27, 24, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 128,
+ 2, 2, 2, 2, 2, 2, 2, 135, 2, 2,
+ 132, 133, 23, 21, 134, 22, 27, 24, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 130,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
@@ -744,40 +746,40 @@ static const yytype_uint8 yytranslate[] =
90, 91, 92, 93, 94, 95, 96, 97, 98, 99,
100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
- 120, 121, 122, 123, 124, 125, 126, 127
+ 120, 121, 122, 123, 124, 125, 126, 127, 128, 129
};
#if YYDEBUG
/* YYRLINE[YYN] -- Source line where rule number YYN was defined. */
static const yytype_uint16 yyrline[] =
{
- 0, 582, 582, 586, 590, 594, 598, 601, 608, 611,
- 614, 617, 620, 623, 626, 629, 632, 635, 641, 647,
- 654, 660, 667, 676, 681, 690, 695, 700, 704, 710,
- 715, 718, 721, 726, 729, 732, 735, 738, 741, 744,
- 747, 750, 753, 765, 768, 771, 789, 809, 812, 815,
- 820, 825, 831, 837, 846, 850, 856, 859, 864, 869,
- 874, 881, 888, 892, 898, 901, 906, 909, 914, 917,
- 922, 925, 944, 948, 954, 958, 964, 967, 970, 975,
- 978, 985, 990, 1001, 1006, 1010, 1014, 1020, 1023, 1029,
- 1037, 1040, 1043, 1049, 1054, 1057, 1062, 1066, 1070, 1074,
- 1080, 1085, 1090, 1094, 1100, 1106, 1109, 1114, 1119, 1123,
- 1129, 1135, 1141, 1144, 1148, 1154, 1157, 1162, 1166, 1172,
- 1175, 1178, 1183, 1188, 1193, 1196, 1199, 1204, 1207, 1210,
- 1213, 1216, 1219, 1222, 1225, 1230, 1233, 1238, 1242, 1246,
- 1249, 1253, 1256, 1261, 1264, 1269, 1272, 1277, 1281, 1287,
- 1290, 1295, 1298, 1303, 1306, 1311, 1314, 1333, 1337, 1343,
- 1350, 1353, 1356, 1361, 1364, 1367, 1373, 1376, 1381, 1386,
- 1395, 1400, 1409, 1414, 1417, 1422, 1425, 1430, 1436, 1442,
- 1445, 1448, 1451, 1454, 1457, 1463, 1472, 1475, 1480, 1483,
- 1488, 1491, 1496, 1499, 1502, 1505, 1508, 1511, 1514, 1519,
- 1523, 1527, 1530, 1535, 1540, 1543, 1548, 1552, 1558, 1563,
- 1567, 1573, 1578, 1581, 1586, 1590, 1596, 1599, 1602, 1605,
- 1617, 1621, 1640, 1653, 1668, 1671, 1674, 1677, 1680, 1683,
- 1688, 1692, 1698, 1701, 1706, 1710, 1717, 1720, 1723, 1726,
- 1729, 1732, 1735, 1738, 1741, 1744, 1749, 1760, 1763, 1768,
- 1771, 1774, 1780, 1784, 1790, 1793, 1801, 1804, 1807, 1810,
- 1816, 1821, 1826
+ 0, 585, 585, 589, 593, 597, 601, 604, 611, 614,
+ 617, 620, 623, 626, 629, 632, 635, 638, 644, 650,
+ 657, 663, 670, 679, 684, 693, 698, 703, 707, 713,
+ 718, 721, 724, 729, 732, 735, 738, 741, 744, 747,
+ 750, 753, 756, 768, 771, 774, 792, 812, 815, 818,
+ 823, 828, 834, 840, 849, 853, 859, 862, 867, 872,
+ 877, 884, 891, 895, 901, 904, 909, 912, 917, 920,
+ 925, 928, 947, 951, 957, 961, 967, 970, 973, 978,
+ 981, 988, 993, 1004, 1009, 1013, 1017, 1023, 1026, 1032,
+ 1040, 1043, 1046, 1052, 1057, 1060, 1065, 1069, 1073, 1077,
+ 1083, 1088, 1093, 1097, 1103, 1109, 1112, 1117, 1122, 1126,
+ 1132, 1138, 1144, 1147, 1151, 1157, 1160, 1165, 1169, 1175,
+ 1178, 1181, 1186, 1191, 1196, 1199, 1202, 1207, 1210, 1213,
+ 1216, 1219, 1222, 1225, 1228, 1233, 1236, 1241, 1245, 1249,
+ 1252, 1256, 1259, 1264, 1267, 1272, 1275, 1280, 1284, 1290,
+ 1293, 1298, 1301, 1306, 1309, 1314, 1317, 1336, 1340, 1346,
+ 1353, 1356, 1359, 1364, 1367, 1370, 1376, 1379, 1384, 1389,
+ 1398, 1403, 1412, 1417, 1420, 1425, 1428, 1433, 1439, 1445,
+ 1448, 1451, 1454, 1457, 1460, 1466, 1475, 1478, 1483, 1486,
+ 1491, 1494, 1499, 1502, 1505, 1508, 1511, 1514, 1517, 1520,
+ 1525, 1529, 1533, 1536, 1541, 1546, 1550, 1556, 1559, 1564,
+ 1568, 1574, 1579, 1583, 1589, 1594, 1597, 1602, 1606, 1612,
+ 1615, 1618, 1621, 1633, 1637, 1656, 1669, 1684, 1687, 1690,
+ 1693, 1696, 1699, 1704, 1708, 1714, 1717, 1722, 1726, 1733,
+ 1736, 1739, 1742, 1745, 1748, 1751, 1754, 1757, 1760, 1765,
+ 1776, 1779, 1784, 1787, 1790, 1796, 1800, 1806, 1809, 1817,
+ 1820, 1823, 1826, 1832, 1837, 1842
};
#endif
@@ -801,7 +803,7 @@ static const char *const yytname[] =
"TOKEN_DELETE", "TOKEN_DELIMITER", "TOKEN_DESC", "TOKEN_DISTINCT",
"TOKEN_DOUBLE", "TOKEN_DROP", "TOKEN_ELSE", "TOKEN_END",
"TOKEN_ESCAPE_STRINGS", "TOKEN_EXISTS", "TOKEN_EXTRACT", "TOKEN_FALSE",
- "TOKEN_FIRST", "TOKEN_FLOAT", "TOKEN_FOREIGN", "TOKEN_FROM",
+ "TOKEN_FIRST", "TOKEN_FLOAT", "TOKEN_FOR", "TOKEN_FOREIGN", "TOKEN_FROM",
"TOKEN_FULL", "TOKEN_GROUP", "TOKEN_HASH", "TOKEN_HAVING", "TOKEN_HOUR",
"TOKEN_IN", "TOKEN_INDEX", "TOKEN_INNER", "TOKEN_INSERT",
"TOKEN_INTEGER", "TOKEN_INTERVAL", "TOKEN_INTO", "TOKEN_JOIN",
@@ -811,26 +813,27 @@ static const char *const yytname[] =
"TOKEN_PARTITIONS", "TOKEN_PERCENT", "TOKEN_PRIMARY", "TOKEN_QUIT",
"TOKEN_RANGE", "TOKEN_REAL", "TOKEN_REFERENCES", "TOKEN_RIGHT",
"TOKEN_ROW_DELIMITER", "TOKEN_SECOND", "TOKEN_SELECT", "TOKEN_SET",
- "TOKEN_SMA", "TOKEN_SMALLINT", "TOKEN_TABLE", "TOKEN_THEN", "TOKEN_TIME",
- "TOKEN_TIMESTAMP", "TOKEN_TRUE", "TOKEN_TUPLESAMPLE", "TOKEN_UNIQUE",
- "TOKEN_UPDATE", "TOKEN_USING", "TOKEN_VALUES", "TOKEN_VARCHAR",
- "TOKEN_WHEN", "TOKEN_WHERE", "TOKEN_WITH", "TOKEN_YEAR",
- "TOKEN_YEARMONTH", "TOKEN_EOF", "TOKEN_LEX_ERROR", "';'", "'\\n'", "'('",
- "')'", "','", "'%'", "$accept", "start", "sql_statement",
- "quit_statement", "alter_table_statement", "create_table_statement",
- "create_index_statement", "drop_table_statement", "column_def",
- "column_def_commalist", "data_type", "column_constraint_def",
- "column_constraint_def_list", "opt_column_constraint_def_list",
- "table_constraint_def", "table_constraint_def_commalist",
- "opt_table_constraint_def_commalist", "opt_column_list",
- "opt_block_properties", "opt_partition_clause", "partition_type",
- "key_value_list", "key_value", "key_string_value", "key_string_list",
- "key_integer_value", "index_type", "opt_index_properties",
- "insert_statement", "copy_from_statement", "opt_copy_from_params",
- "copy_from_params", "update_statement", "delete_statement",
- "assignment_list", "assignment_item", "select_statement", "with_clause",
- "with_list", "with_list_element", "select_query", "opt_all_distinct",
- "selection", "selection_item_commalist", "selection_item", "from_clause",
+ "TOKEN_SMA", "TOKEN_SMALLINT", "TOKEN_SUBSTRING", "TOKEN_TABLE",
+ "TOKEN_THEN", "TOKEN_TIME", "TOKEN_TIMESTAMP", "TOKEN_TRUE",
+ "TOKEN_TUPLESAMPLE", "TOKEN_UNIQUE", "TOKEN_UPDATE", "TOKEN_USING",
+ "TOKEN_VALUES", "TOKEN_VARCHAR", "TOKEN_WHEN", "TOKEN_WHERE",
+ "TOKEN_WITH", "TOKEN_YEAR", "TOKEN_YEARMONTH", "TOKEN_EOF",
+ "TOKEN_LEX_ERROR", "';'", "'\\n'", "'('", "')'", "','", "'%'", "$accept",
+ "start", "sql_statement", "quit_statement", "alter_table_statement",
+ "create_table_statement", "create_index_statement",
+ "drop_table_statement", "column_def", "column_def_commalist",
+ "data_type", "column_constraint_def", "column_constraint_def_list",
+ "opt_column_constraint_def_list", "table_constraint_def",
+ "table_constraint_def_commalist", "opt_table_constraint_def_commalist",
+ "opt_column_list", "opt_block_properties", "opt_partition_clause",
+ "partition_type", "key_value_list", "key_value", "key_string_value",
+ "key_string_list", "key_integer_value", "index_type",
+ "opt_index_properties", "insert_statement", "copy_from_statement",
+ "opt_copy_from_params", "copy_from_params", "update_statement",
+ "delete_statement", "assignment_list", "assignment_item",
+ "select_statement", "with_clause", "with_list", "with_list_element",
+ "select_query", "opt_all_distinct", "selection",
+ "selection_item_commalist", "selection_item", "from_clause",
"subquery_expression", "opt_sample_clause", "join_type",
"joined_table_reference", "table_reference", "table_reference_signature",
"table_reference_signature_primary", "joined_table_reference_commalist",
@@ -840,14 +843,14 @@ static const char *const yytname[] =
"where_clause", "or_expression", "and_expression", "not_expression",
"predicate_expression_base", "add_expression", "multiply_expression",
"unary_expression", "expression_base", "function_call",
- "extract_function", "case_expression", "simple_when_clause_list",
- "simple_when_clause", "searched_when_clause_list",
- "searched_when_clause", "opt_else_clause", "expression_list",
- "literal_value", "datetime_unit", "literal_value_commalist",
- "attribute_ref", "attribute_ref_list", "comparison_operation",
- "unary_operation", "add_operation", "multiply_operation",
- "name_commalist", "any_name", "boolean_value", "command",
- "command_argument_list", YY_NULLPTR
+ "extract_function", "substr_function", "case_expression",
+ "simple_when_clause_list", "simple_when_clause",
+ "searched_when_clause_list", "searched_when_clause", "opt_else_clause",
+ "expression_list", "literal_value", "datetime_unit",
+ "literal_value_commalist", "attribute_ref", "attribute_ref_list",
+ "comparison_operation", "unary_operation", "add_operation",
+ "multiply_operation", "name_commalist", "any_name", "boolean_value",
+ "command", "command_argument_list", YY_NULLPTR
};
#endif
@@ -868,15 +871,15 @@ static const yytype_uint16 yytoknum[] =
340, 341, 342, 343, 344, 345, 346, 347, 348, 349,
350, 351, 352, 353, 354, 355, 356, 357, 358, 359,
360, 361, 362, 363, 364, 365, 366, 367, 368, 369,
- 370, 371, 372, 373, 374, 375, 376, 377, 59, 10,
- 40, 41, 44, 37
+ 370, 371, 372, 373, 374, 375, 376, 377, 378, 379,
+ 59, 10, 40, 41, 44, 37
};
# endif
-#define YYPACT_NINF -223
+#define YYPACT_NINF -230
#define yypact_value_is_default(Yystate) \
- (!!((Yystate) == (-223)))
+ (!!((Yystate) == (-230)))
#define YYTABLE_NINF -128
@@ -887,54 +890,55 @@ static const yytype_uint16 yytoknum[] =
STATE-NUM. */
static const yytype_int16 yypact[] =
{
- 921, -223, -223, -82, 231, -14, 56, 22, 74, -223,
- 70, 231, 231, -223, 135, 120, -223, -223, -223, -223,
- -223, -223, -223, -223, -223, -223, -35, -223, -73, 177,
- 231, -223, -223, 121, 231, 231, 231, 231, 231, -223,
- -223, 576, 85, 63, -223, 173, 77, -223, -223, -223,
- 140, -223, -223, -223, -223, 18, 218, 144, 97, 119,
- -223, 4, -223, -223, 240, 245, -223, -223, -223, 642,
- 134, -223, 187, -223, -223, 146, -223, -223, 265, -223,
- -223, -223, -223, -223, -223, 164, 203, 708, 290, 230,
- 176, -223, -223, 256, 20, -223, -223, -223, -223, -223,
- -223, -223, 840, -11, 231, 231, 182, 231, 231, 167,
- 206, 204, 231, 231, 483, -223, -223, 205, 231, -223,
- -223, -223, 483, 47, -10, -223, 329, -223, 129, 129,
- 330, -223, 207, 26, -223, 32, 119, 906, -223, -223,
- 231, 906, -223, -223, -223, -223, 906, 245, -223, 231,
- 324, -70, -223, 331, -223, 234, -223, 125, -223, 234,
- 231, 49, 231, 231, 211, -223, 213, -223, 137, 991,
- 774, 182, 390, 336, 339, -223, -223, 1117, 335, 1001,
- 142, 10, 906, -9, -223, 906, -223, 296, 228, -223,
- -223, -223, -223, -223, -223, 292, -223, 232, -223, -223,
- 21, 163, 122, -223, 229, 163, -13, 294, -223, -223,
- 20, -223, -223, 236, 906, -223, 259, 152, 231, -223,
- 906, -223, 231, -223, -223, 238, 287, 288, 241, -223,
- -223, -223, 61, 231, 258, 49, 231, -223, 156, -223,
- -223, -6, 69, 483, 483, 55, -223, -223, -223, -223,
- -223, -223, -223, -223, 906, 244, 906, 5, -223, 154,
- 260, 906, 37, -223, 317, 259, -223, -223, 906, -223,
- 102, 231, -223, -223, 284, -223, 291, 297, 301, 32,
- -223, 377, 379, 163, 348, 320, -223, 157, -223, 906,
- -223, 259, -223, 483, 263, 268, 231, 394, 151, 159,
- -223, 168, 378, 25, -223, 272, 283, -223, 318, 279,
- 1001, -223, 332, 231, -223, -223, 156, -223, -223, 339,
- -223, -223, -223, 906, 285, 161, 708, -223, 259, 327,
- -223, -223, 1001, 289, 259, 906, -223, 33, -223, -223,
- -223, -223, -223, 32, 122, 321, 323, -223, 906, 483,
- 328, -223, 259, 13, 231, 231, 170, -223, -223, -223,
- -223, -223, -223, -223, 76, -223, 231, -223, -223, -223,
- -223, 299, 49, 381, 340, -223, 483, -223, -223, 300,
- -223, 200, 708, -223, 906, 172, -223, -223, 1001, 259,
- -223, 342, -223, -223, 311, 336, 382, 361, -223, 175,
- 179, -223, 443, 151, -223, 231, -223, -223, 333, 410,
- -223, 29, 231, 906, 183, 259, -223, 185, 483, 906,
- 444, -223, 355, -223, -223, -223, 188, -223, -223, -223,
- -223, 16, 231, -5, -223, 334, 259, -223, -223, 336,
- 326, -223, 155, -223, 231, -223, 231, -223, -223, 231,
- -223, 190, -223, -223, 337, -223, 906, -223, -223, 369,
- 341, -223, 192, -223, 231, -223, 110, -223, 231, -223,
- 194, -223, -223, 201, 365, -223, 455, -223
+ 162, -230, -230, -75, 181, 7, 17, 23, 19, -230,
+ 41, 181, 181, -230, 151, 129, -230, -230, -230, -230,
+ -230, -230, -230, -230, -230, -230, -38, -230, -36, 166,
+ 181, -230, -230, 138, 181, 181, 181, 181, 181, -230,
+ -230, 644, 111, 93, -230, 220, 128, -230, -230, -230,
+ 202, -230, -230, -230, -230, 65, 273, 192, 163, 172,
+ -230, 11, -230, -230, 295, 300, -230, -230, -230, 699,
+ 207, -230, 233, -230, -230, 218, -230, -230, 310, -230,
+ -230, -230, -230, 219, -230, -230, 229, 245, 786, 322,
+ 261, 224, -230, -230, 182, 35, -230, -230, -230, -230,
+ -230, -230, -230, -230, 928, -12, 181, 181, 235, 181,
+ 181, 194, 226, 238, 181, 181, 542, -230, -230, 237,
+ 181, -230, -230, -230, 542, 53, -34, -230, 359, -230,
+ 105, 105, 983, 363, -230, 241, 40, -230, 18, 172,
+ 983, -230, -230, 181, 983, -230, -230, -230, -230, 983,
+ 300, -230, 181, 331, 82, -230, 360, -230, 268, -230,
+ -7, -230, 268, 181, 146, 181, 181, 244, -230, 246,
+ -230, 143, 1088, 841, 235, 455, 369, 372, -230, -230,
+ 624, 364, 1069, 155, 15, 983, 8, -230, 983, -230,
+ 323, 252, -230, -230, -230, -230, -230, -230, 317, -230,
+ 60, 256, -230, -230, 10, 90, 197, -230, 257, 90,
+ -10, 319, -230, -230, 35, -230, -230, 259, 983, -230,
+ 270, 160, 181, -230, 983, -230, 181, -230, -230, 263,
+ 313, 315, 267, -230, -230, -230, 174, 181, 281, 146,
+ 181, -230, 81, -230, -230, 20, 34, 542, 542, 61,
+ -230, -230, -230, -230, -230, -230, -230, -230, 983, 271,
+ 983, 47, -230, 165, 283, 983, 55, -230, 342, 270,
+ -230, -230, 983, 398, -230, 125, 181, -230, -230, 311,
+ -230, 312, 318, 326, 18, -230, 405, 407, 90, 375,
+ 344, -230, 171, -230, 983, -230, 270, -230, 542, 287,
+ 288, 181, 417, 157, 175, -230, 177, 396, 49, -230,
+ 290, 301, -230, 335, 296, 1069, -230, 346, 181, -230,
+ -230, 81, -230, -230, 372, -230, -230, -230, 983, 298,
+ 203, 786, -230, 270, 337, -230, -230, 1069, 299, 270,
+ 983, -230, 42, -25, -230, -230, -230, -230, -230, 18,
+ 197, 334, 336, -230, 983, 542, 341, -230, 270, 21,
+ 181, 181, 183, -230, -230, -230, -230, -230, -230, -230,
+ 193, -230, 181, -230, -230, -230, -230, 307, 146, 404,
+ 347, -230, 542, -230, -230, 316, -230, 234, 786, -230,
+ 983, 185, -230, -230, 1069, 270, -230, 437, -230, 354,
+ -230, -230, 320, 369, 409, 365, -230, 188, 190, -230,
+ 451, 157, -230, 181, -230, -230, 325, 426, -230, 31,
+ 181, 983, 195, 270, -230, 198, 338, 542, 983, 460,
+ -230, 366, -230, -230, -230, 200, -230, -230, -230, -230,
+ 14, 181, 13, -230, 339, 270, -230, -230, -230, 369,
+ 345, -230, 231, -230, 181, -230, 181, -230, -230, 181,
+ -230, 208, -230, -230, 343, -230, 983, -230, -230, 377,
+ 348, -230, 210, -230, 181, -230, 113, -230, 181, -230,
+ 213, -230, -230, 215, 376, -230, 471, -230
};
/* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM.
@@ -942,50 +946,51 @@ static const yytype_int16 yypact[] =
means the default is an error. */
static const yytype_uint16 yydefact[] =
{
- 0, 6, 262, 0, 0, 0, 0, 0, 0, 18,
+ 0, 6, 265, 0, 0, 0, 0, 0, 0, 18,
112, 0, 0, 7, 0, 0, 15, 8, 10, 11,
- 13, 14, 9, 17, 12, 16, 0, 105, 0, 260,
- 0, 254, 255, 0, 0, 0, 0, 0, 0, 113,
+ 13, 14, 9, 17, 12, 16, 0, 105, 0, 263,
+ 0, 257, 258, 0, 0, 0, 0, 0, 0, 113,
114, 0, 0, 107, 108, 0, 145, 1, 3, 2,
- 0, 106, 5, 4, 261, 0, 0, 0, 0, 166,
- 25, 0, 220, 217, 0, 246, 115, 40, 29, 0,
+ 0, 106, 5, 4, 264, 0, 0, 0, 0, 166,
+ 25, 0, 223, 220, 0, 249, 115, 40, 29, 0,
0, 30, 31, 34, 36, 0, 37, 39, 0, 41,
- 216, 35, 38, 32, 33, 0, 0, 0, 0, 0,
- 116, 117, 198, 121, 187, 189, 191, 194, 195, 196,
- 193, 192, 0, 232, 0, 0, 0, 0, 0, 0,
- 0, 94, 0, 0, 0, 101, 167, 0, 0, 91,
- 218, 219, 0, 0, 212, 209, 0, 43, 0, 221,
- 0, 44, 0, 0, 223, 0, 166, 0, 247, 248,
- 0, 0, 120, 250, 251, 249, 0, 0, 190, 0,
- 0, 166, 103, 0, 109, 0, 110, 0, 252, 0,
- 0, 0, 0, 0, 0, 93, 66, 27, 0, 0,
- 0, 0, 0, 168, 170, 172, 174, 0, 192, 0,
- 0, 0, 0, 212, 206, 0, 210, 0, 0, 226,
- 227, 228, 225, 229, 224, 0, 222, 0, 123, 197,
- 0, 0, 147, 136, 122, 141, 124, 149, 118, 119,
- 186, 188, 233, 0, 0, 199, 214, 0, 0, 100,
- 0, 146, 0, 92, 19, 0, 0, 0, 0, 20,
- 21, 22, 0, 0, 0, 64, 0, 42, 56, 173,
- 181, 0, 0, 0, 0, 0, 236, 238, 239, 240,
- 241, 237, 242, 244, 0, 0, 0, 0, 230, 0,
- 0, 0, 0, 207, 0, 213, 205, 45, 0, 46,
- 127, 0, 137, 143, 133, 128, 129, 131, 0, 0,
- 140, 0, 0, 139, 0, 151, 200, 0, 201, 0,
- 102, 104, 253, 0, 0, 0, 0, 0, 0, 0,
- 234, 0, 232, 0, 63, 65, 68, 28, 0, 0,
- 0, 47, 0, 0, 49, 55, 57, 26, 180, 169,
- 171, 243, 245, 0, 0, 0, 0, 182, 179, 0,
- 178, 90, 0, 0, 211, 0, 204, 0, 142, 144,
- 134, 130, 132, 0, 148, 0, 0, 138, 0, 0,
- 153, 202, 215, 0, 0, 0, 0, 96, 258, 259,
- 257, 256, 97, 95, 0, 67, 0, 83, 84, 85,
- 86, 87, 0, 0, 70, 48, 0, 51, 50, 0,
- 54, 0, 0, 184, 0, 0, 177, 231, 0, 208,
- 203, 0, 125, 126, 150, 152, 0, 155, 61, 0,
- 0, 58, 0, 0, 235, 0, 24, 62, 0, 0,
- 23, 0, 0, 0, 0, 175, 183, 0, 0, 0,
- 0, 111, 0, 59, 98, 99, 0, 74, 76, 77,
- 78, 0, 0, 0, 52, 0, 176, 185, 89, 135,
+ 219, 35, 38, 0, 32, 33, 0, 0, 0, 0,
+ 0, 116, 117, 199, 121, 187, 189, 191, 194, 195,
+ 196, 197, 193, 192, 0, 235, 0, 0, 0, 0,
+ 0, 0, 0, 94, 0, 0, 0, 101, 167, 0,
+ 0, 91, 221, 222, 0, 0, 215, 212, 0, 43,
+ 0, 224, 0, 0, 44, 0, 0, 226, 0, 166,
+ 0, 250, 251, 0, 0, 120, 253, 254, 252, 0,
+ 0, 190, 0, 0, 166, 103, 0, 109, 0, 110,
+ 0, 255, 0, 0, 0, 0, 0, 0, 93, 66,
+ 27, 0, 0, 0, 0, 0, 168, 170, 172, 174,
+ 0, 192, 0, 0, 0, 0, 215, 209, 0, 213,
+ 0, 0, 229, 230, 231, 228, 232, 227, 0, 225,
+ 0, 0, 123, 198, 0, 0, 147, 136, 122, 141,
+ 124, 149, 118, 119, 186, 188, 236, 0, 0, 200,
+ 217, 0, 0, 100, 0, 146, 0, 92, 19, 0,
+ 0, 0, 0, 20, 21, 22, 0, 0, 0, 64,
+ 0, 42, 56, 173, 181, 0, 0, 0, 0, 0,
+ 239, 241, 242, 243, 244, 240, 245, 247, 0, 0,
+ 0, 0, 233, 0, 0, 0, 0, 210, 0, 216,
+ 208, 45, 0, 0, 46, 127, 0, 137, 143, 133,
+ 128, 129, 131, 0, 0, 140, 0, 0, 139, 0,
+ 151, 201, 0, 202, 0, 102, 104, 256, 0, 0,
+ 0, 0, 0, 0, 0, 237, 0, 235, 0, 63,
+ 65, 68, 28, 0, 0, 0, 47, 0, 0, 49,
+ 55, 57, 26, 180, 169, 171, 246, 248, 0, 0,
+ 0, 0, 182, 179, 0, 178, 90, 0, 0, 214,
+ 0, 207, 0, 0, 142, 144, 134, 130, 132, 0,
+ 148, 0, 0, 138, 0, 0, 153, 203, 218, 0,
+ 0, 0, 0, 96, 261, 262, 260, 259, 97, 95,
+ 0, 67, 0, 83, 84, 85, 86, 87, 0, 0,
+ 70, 48, 0, 51, 50, 0, 54, 0, 0, 184,
+ 0, 0, 177, 234, 0, 211, 204, 0, 205, 0,
+ 125, 126, 150, 152, 0, 155, 61, 0, 0, 58,
+ 0, 0, 238, 0, 24, 62, 0, 0, 23, 0,
+ 0, 0, 0, 175, 183, 0, 0, 0, 0, 0,
+ 111, 0, 59, 98, 99, 0, 74, 76, 77, 78,
+ 0, 0, 0, 52, 0, 176, 185, 89, 206, 135,
154, 157, 160, 156, 0, 88, 0, 82, 80, 0,
79, 0, 72, 73, 0, 53, 0, 161, 162, 163,
0, 75, 0, 69, 0, 158, 0, 159, 0, 81,
@@ -995,31 +1000,31 @@ static const yytype_uint16 yydefact[] =
/* YYPGOTO[NTERM-NUM]. */
static const yytype_int16 yypgoto[] =
{
- -223, -223, -223, -223, -223, -223, -223, -223, -131, -223,
- 303, 150, -223, -223, -222, -223, -223, -223, -223, -223,
- -223, 38, 27, -223, -223, -223, -223, -223, -223, -223,
- -223, -223, -223, -223, -223, 257, -223, -223, -223, 372,
- 14, -223, -223, -223, 343, -223, -94, -223, -223, -181,
- 131, -170, -8, -223, -223, -223, -223, -223, -223, 28,
- -223, -223, -58, -223, -121, 235, 237, 312, -30, 344,
- 346, 384, -130, -223, -223, -223, 314, -223, 359, 315,
- -208, -169, 366, 106, -105, -223, -223, -223, -223, -223,
- -115, -4, 98, -223, -223
+ -230, -230, -230, -230, -230, -230, -230, -230, -127, -230,
+ 309, 161, -230, -230, -229, -230, -230, -230, -230, -230,
+ -230, 43, 27, -230, -230, -230, -230, -230, -230, -230,
+ -230, -230, -230, -230, -230, 264, -230, -230, -230, 378,
+ 12, -230, -230, -230, 350, -230, -103, -230, -230, -139,
+ 142, -190, -8, -230, -230, -230, -230, -230, -230, 26,
+ -230, -230, 44, -230, -121, 240, 247, 321, -28, 349,
+ 351, 394, -132, -230, -230, -230, -230, 324, -230, 373,
+ 327, -216, -171, 370, 108, -107, -230, -230, -230, -230,
+ -230, -119, -4, 94, -230, -230
};
/* YYDEFGOTO[NTERM-NUM]. */
static const yytype_int16 yydefgoto[] =
{
- -1, 14, 15, 16, 17, 18, 19, 20, 167, 168,
- 88, 315, 316, 317, 229, 305, 306, 234, 374, 410,
- 454, 426, 427, 428, 429, 430, 371, 406, 21, 22,
- 165, 299, 23, 24, 151, 152, 25, 26, 43, 44,
- 132, 41, 89, 90, 91, 136, 92, 283, 278, 202,
- 203, 272, 273, 204, 285, 350, 397, 421, 440, 441,
- 459, 467, 115, 116, 173, 174, 175, 176, 177, 94,
- 95, 96, 97, 98, 99, 183, 184, 124, 125, 187,
- 217, 100, 195, 259, 101, 301, 256, 102, 141, 146,
- 157, 103, 362, 28, 29
+ -1, 14, 15, 16, 17, 18, 19, 20, 170, 171,
+ 89, 320, 321, 322, 233, 310, 311, 238, 380, 418,
+ 464, 435, 436, 437, 438, 439, 377, 414, 21, 22,
+ 168, 304, 23, 24, 154, 155, 25, 26, 43, 44,
+ 135, 41, 90, 91, 92, 139, 93, 288, 283, 206,
+ 207, 277, 278, 208, 290, 356, 405, 430, 450, 451,
+ 469, 477, 117, 118, 176, 177, 178, 179, 180, 95,
+ 96, 97, 98, 99, 100, 101, 186, 187, 126, 127,
+ 190, 221, 102, 198, 263, 103, 306, 260, 104, 144,
+ 149, 160, 105, 368, 28, 29
};
/* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If
@@ -1027,336 +1032,341 @@ static const yytype_int16 yydefgoto[] =
number is the opposite. If YYTABLE_NINF, syntax error. */
static const yytype_int16 yytable[] =
{
- 33, 181, 243, 180, 45, 205, 287, 42, 46, 178,
- 258, 93, 156, 304, 27, 329, 149, 178, 243, 270,
- 31, 243, 32, 447, 281, 31, 55, 32, 30, 224,
- 57, 58, 59, 60, 61, 280, 31, 243, 32, 123,
- 51, 201, 50, 143, 144, 448, 109, 138, 139, 185,
- 185, 241, 114, 52, 138, 139, 53, 133, 138, 139,
- 367, 34, 218, 368, 369, 178, 452, 178, 138, 139,
- 205, 10, 321, 322, 323, 119, 110, 240, 207, 245,
- 246, 247, 248, 249, 250, 251, 252, 253, 254, 142,
- 138, 139, 225, 219, 330, 453, 35, 45, 344, 39,
- 153, 46, 282, 158, 159, 307, 201, 93, 166, 169,
- 10, 122, 182, 347, 158, 297, 226, 150, 385, 150,
- 216, 261, 298, 117, 36, 318, 40, 10, 300, 324,
- 402, 206, 37, 370, 118, 47, 209, 403, 178, 178,
- 394, 377, 242, 255, 398, 212, 449, 227, 335, 205,
- 407, 200, 262, 145, 38, 265, 169, 199, 230, 231,
- 434, 327, 200, 387, 390, 228, 308, 31, 182, 32,
- 384, 274, 353, 223, 414, 471, 138, 139, 275, 189,
- 54, 356, 138, 139, 216, 201, 276, 457, 178, 56,
- 291, 274, 104, 472, 271, 105, 206, 46, 275, 309,
- 199, 46, 190, -127, 106, 277, 276, 107, 310, 413,
- 458, 160, 161, 205, 153, 358, 191, 192, 292, 258,
- 108, 138, 139, 111, 325, 277, 328, 113, 395, 302,
- 383, 334, 169, 338, 193, 31, 112, 32, 337, 399,
- 400, 114, 359, 360, 178, 311, 48, 120, 49, 201,
- 162, 163, 121, 194, 312, 411, 221, 222, 313, 352,
- 31, 404, 32, 339, 126, 361, 127, 46, 235, 236,
- 129, 178, 314, 260, 222, 206, 128, 138, 139, 46,
- 138, 139, 131, 288, 289, 331, 332, 140, 351, 289,
- 363, 364, 158, 381, 130, 134, 216, 439, 135, 365,
- 366, 401, 222, 416, 289, 389, 422, 222, 137, 379,
- 423, 222, 155, 178, 437, 289, 438, 332, 216, 445,
- 446, 463, 446, 469, 222, 474, 222, 164, 31, 62,
- 32, 63, 475, 222, 462, 179, 188, 197, 198, 206,
- 10, 232, 220, 233, 243, 64, 65, 213, 244, 470,
- 158, 158, 216, 473, 415, 257, 266, 67, 68, 267,
- 268, 279, 302, 269, 284, 69, 70, 286, 293, 294,
- 295, 296, 71, 72, 326, 73, 303, 336, 340, 333,
- 214, 74, 343, 436, 345, 341, 346, 75, 348, 442,
- 76, 342, 349, 354, 31, 62, 32, 63, 355, 357,
- 170, 431, 77, 78, 372, 149, 373, 375, 435, 376,
- 79, 64, 65, 80, 378, 382, 386, 408, 392, 388,
- 393, 396, 419, 67, 68, 81, 442, 450, 431, 405,
- 412, 69, 70, 82, 418, 409, 83, 84, 71, 72,
- 460, 73, 431, 289, 85, 158, 420, 74, 424, 86,
- 433, 443, 171, 75, 87, 215, 76, 444, 456, 466,
- 158, 476, 477, 432, 158, 455, 380, 464, 77, 78,
- 451, 468, 238, 461, 391, 290, 79, 154, 319, 80,
- 208, 320, 239, 186, 465, 210, 148, 31, 62, 32,
- 63, 81, 211, 170, 417, 196, 10, 263, 264, 82,
- 0, 425, 83, 84, 64, 65, 0, 0, 0, 0,
- 85, 0, 0, 0, 0, 86, 67, 68, 0, 0,
- 172, 0, 0, 0, 69, 70, 0, 0, 0, 0,
- 0, 71, 72, 0, 73, 0, 0, 0, 0, 0,
- 74, 0, 0, 0, 0, 171, 75, 0, 0, 76,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 77, 78, 0, 0, 0, 0, 0, 0, 79,
- 0, 0, 80, 0, 0, 0, 0, 0, 0, 0,
- 31, 62, 32, 63, 81, 0, 0, 0, 0, 0,
- 0, 0, 82, 0, 0, 83, 84, 64, 65, 66,
- 0, 0, 0, 85, 0, 0, 0, 0, 86, 67,
- 68, 0, 0, 172, 0, 0, 0, 69, 70, 0,
- 0, 0, 0, 0, 71, 72, 0, 73, 0, 0,
- 0, 0, 0, 74, 0, 0, 0, 0, 0, 75,
- 0, 0, 76, 0, 0, 0, 31, 62, 32, 63,
+ 33, 183, 292, 184, 45, 159, 209, 42, 46, 181,
+ 309, 262, 27, 94, 31, 152, 32, 181, 31, 285,
+ 32, 457, 31, 247, 32, 188, 55, 286, 247, 247,
+ 57, 58, 59, 60, 61, 205, 228, 30, 51, 247,
+ 50, 125, 397, 458, 249, 250, 251, 252, 253, 254,
+ 255, 256, 257, 258, 245, 141, 142, 334, 146, 147,
+ 136, 141, 142, 141, 142, 275, 181, 188, 181, 10,
+ 39, 244, 209, 121, 141, 142, 141, 142, 326, 327,
+ 328, 141, 142, 34, 373, 462, 36, 374, 375, 124,
+ 145, 313, 52, 111, 31, 53, 32, 40, 353, 45,
+ 38, 205, 156, 46, 200, 161, 162, 287, 398, 259,
+ 169, 172, 94, 312, 463, 391, 161, 10, 10, 35,
+ 153, 276, 153, 112, 314, 220, 225, 226, 265, 273,
+ 305, 185, 119, 315, 210, 37, 329, 335, 402, 213,
+ 181, 181, 204, 120, 383, 350, 459, 246, 216, 415,
+ 204, 47, 209, 323, 406, 192, 332, 266, 376, 172,
+ 269, 234, 235, 1, 443, 2, 393, 203, 340, 54,
+ 148, 316, 422, 203, 227, 396, 185, 359, 481, 193,
+ 317, 205, 362, 211, 318, 31, 31, 32, 32, 229,
+ 220, 181, 3, 194, 195, 279, 296, 482, 223, 319,
+ 210, 46, 280, 141, 142, 46, 116, 56, 4, 5,
+ 281, 196, 390, 143, 230, 6, 222, 209, 156, 106,
+ 7, 364, 297, 262, 141, 142, 389, 107, 302, 282,
+ 330, 197, 333, 307, 403, 303, 172, 339, 163, 164,
+ 8, 407, 408, 421, 342, 231, 205, 410, 181, 365,
+ 366, 108, 141, 142, 411, 141, 142, 48, 344, 49,
+ 109, 419, 9, 467, 232, 412, 358, 279, 345, 10,
+ 165, 166, 46, 367, 280, 181, 239, 240, 113, -127,
+ 210, 11, 281, 110, 46, 114, 468, 12, 264, 226,
+ 13, 141, 142, 293, 294, 115, 116, 161, 336, 337,
+ 387, 282, 122, 220, 357, 294, 449, 123, 369, 370,
+ 371, 372, 395, 129, 385, 131, 409, 226, 424, 294,
+ 181, 431, 226, 432, 226, 134, 220, 137, 446, 294,
+ 138, 447, 337, 455, 456, 31, 62, 32, 63, 128,
+ 472, 473, 456, 479, 226, 210, 484, 226, 485, 226,
+ 130, 132, 64, 65, 217, 480, 161, 161, 140, 483,
+ 220, 133, 423, 167, 67, 68, 191, 158, 307, 182,
+ 201, 224, 69, 70, 202, 10, 236, 247, 237, 71,
+ 72, 248, 73, 270, 261, 271, 272, 218, 74, 274,
+ 289, 284, 291, 445, 75, 298, 299, 76, 300, 301,
+ 452, 308, 341, 331, 338, 343, 346, 347, 349, 440,
+ 77, 78, 351, 348, 352, 354, 444, 355, 79, 360,
+ 361, 80, 363, 152, 378, 381, 379, 392, 382, 384,
+ 388, 394, 400, 81, 401, 404, 460, 440, 452, 413,
+ 416, 82, 83, 417, 426, 84, 85, 427, 420, 428,
+ 470, 429, 440, 86, 294, 161, 433, 441, 87, 31,
+ 62, 32, 63, 88, 219, 173, 442, 453, 476, 454,
+ 161, 448, 465, 486, 161, 474, 64, 65, 487, 466,
+ 478, 242, 386, 471, 461, 157, 295, 324, 67, 68,
+ 212, 399, 475, 214, 243, 325, 69, 70, 151, 189,
+ 215, 199, 425, 71, 72, 434, 73, 0, 0, 0,
+ 267, 0, 74, 268, 0, 0, 0, 174, 75, 0,
+ 0, 76, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 77, 78, 0, 0, 0, 0,
- 0, 0, 79, 64, 65, 80, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 67, 68, 81, 0, 0,
- 0, 0, 0, 69, 70, 82, 0, 0, 83, 84,
- 71, 72, 0, 73, 0, 0, 85, 0, 0, 74,
- 0, 86, 0, 0, 0, 75, 87, 0, 76, 0,
- 0, 0, 31, 62, 32, 63, 0, 0, 0, 0,
- 77, 78, 0, 0, 0, 0, 0, 0, 79, 64,
- 65, 80, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 67, 68, 81, 0, 0, 0, 0, 0, 69,
- 70, 82, 0, 0, 83, 84, 71, 72, 0, 73,
- 0, 0, 85, 122, 0, 74, 0, 86, 0, 0,
- 0, 75, 87, 0, 76, 0, 0, 0, 31, 62,
- 32, 63, 0, 0, 0, 0, 77, 78, 0, 0,
- 0, 0, 0, 0, 79, 64, 65, 80, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 67, 68, 81,
- 0, 0, 0, 0, 10, 69, 70, 82, 0, 0,
- 83, 84, 71, 72, 0, 73, 0, 0, 85, 0,
- 0, 74, 0, 86, 0, 0, 171, 75, 87, 0,
- 76, 0, 0, 0, 31, 62, 32, 63, 0, 0,
- 0, 0, 77, 78, 0, 0, 0, 0, 0, 0,
- 79, 64, 147, 80, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 67, 68, 81, 0, 0, 0, 0,
- 0, 69, 70, 82, 0, 0, 83, 84, 71, 72,
- 0, 73, 0, 0, 85, 0, 0, 74, 0, 86,
- 0, 0, 0, 75, 172, 0, 76, 0, 0, 0,
- 31, 62, 32, 63, 0, 0, 0, 0, 77, 78,
- 0, 0, 1, 0, 2, 0, 79, 64, 65, 80,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 67,
- 68, 81, 0, 0, 0, 0, 0, 69, 70, 82,
- 0, 3, 83, 84, 71, 72, 0, 73, 0, 0,
- 85, 0, 0, 74, 0, 86, 0, 4, 5, 75,
- 87, 0, 76, 0, 6, 0, 0, 0, 0, 7,
- 0, 0, 0, 0, 77, 78, 0, 0, 0, 0,
- 0, 0, 79, 0, 0, 80, 0, 0, 8, 0,
- 0, 0, 0, 0, 0, 0, 62, 81, 63, 0,
- 0, 0, 0, 0, 0, 82, 0, 0, 83, 84,
- 9, 0, 64, 147, 67, 68, 85, 10, 0, 0,
- 0, 86, 0, 70, 67, 68, 87, 0, 11, 71,
- 72, 0, 73, 70, 12, 0, 0, 13, 74, 71,
- 72, 0, 73, 0, 0, 0, 0, 76, 74, 0,
- 0, 0, 0, 0, 0, 0, 0, 76, 0, 77,
- 237, 0, 0, 0, 0, 0, 0, 79, 0, 77,
- 78, 0, 0, 0, 0, 0, 0, 79, 0, 0,
- 80, 0, 81, 0, 0, 0, 0, 0, 0, 0,
- 82, 0, 81, 83, 84, 0, 0, 0, 0, 0,
- 82, 85, 0, 83, 84, 0, 86, 0, 0, 0,
- 0, 85, 0, 0, 0, 0, 86, 245, 246, 247,
- 248, 249, 250, 251, 252, 253, 254, 0, 138, 139,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 79, 0, 0, 80, 31, 62, 32, 63,
+ 0, 0, 173, 0, 0, 0, 0, 81, 0, 0,
+ 0, 0, 10, 64, 65, 82, 83, 0, 0, 84,
+ 85, 0, 0, 0, 0, 67, 68, 86, 0, 0,
+ 0, 0, 87, 69, 70, 0, 0, 175, 0, 0,
+ 71, 72, 0, 73, 0, 0, 0, 0, 0, 74,
+ 0, 0, 0, 0, 174, 75, 0, 0, 76, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 255
+ 0, 77, 78, 0, 0, 0, 0, 0, 0, 79,
+ 0, 0, 80, 0, 249, 250, 251, 252, 253, 254,
+ 255, 256, 257, 258, 81, 141, 142, 0, 31, 62,
+ 32, 63, 82, 83, 0, 0, 84, 85, 0, 0,
+ 0, 0, 0, 0, 86, 64, 65, 66, 0, 87,
+ 0, 0, 0, 0, 175, 0, 0, 67, 68, 0,
+ 0, 0, 0, 0, 0, 69, 70, 0, 0, 0,
+ 0, 0, 71, 72, 0, 73, 0, 0, 0, 259,
+ 0, 74, 0, 31, 62, 32, 63, 75, 0, 0,
+ 76, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 64, 65, 0, 77, 78, 0, 0, 0, 0, 0,
+ 0, 79, 67, 68, 80, 0, 0, 0, 0, 0,
+ 69, 70, 0, 0, 0, 0, 81, 71, 72, 0,
+ 73, 0, 0, 0, 82, 83, 74, 0, 84, 85,
+ 0, 0, 75, 0, 0, 76, 86, 0, 0, 0,
+ 0, 87, 0, 0, 0, 0, 88, 0, 77, 78,
+ 0, 0, 0, 0, 0, 0, 79, 0, 0, 80,
+ 31, 62, 32, 63, 0, 0, 0, 0, 0, 0,
+ 0, 81, 0, 0, 0, 0, 0, 64, 65, 82,
+ 83, 0, 0, 84, 85, 0, 0, 0, 0, 67,
+ 68, 86, 124, 0, 0, 0, 87, 69, 70, 0,
+ 0, 88, 0, 0, 71, 72, 0, 73, 0, 0,
+ 0, 0, 0, 74, 0, 31, 62, 32, 63, 75,
+ 0, 0, 76, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 64, 65, 0, 77, 78, 0, 0, 0,
+ 0, 0, 0, 79, 67, 68, 80, 0, 0, 0,
+ 0, 0, 69, 70, 0, 0, 0, 0, 81, 71,
+ 72, 0, 73, 10, 0, 0, 82, 83, 74, 0,
+ 84, 85, 0, 174, 75, 0, 0, 76, 86, 0,
+ 0, 0, 0, 87, 0, 0, 0, 0, 88, 0,
+ 77, 78, 0, 0, 0, 0, 0, 0, 79, 0,
+ 0, 80, 31, 62, 32, 63, 0, 0, 0, 0,
+ 0, 0, 0, 81, 0, 0, 0, 0, 0, 64,
+ 150, 82, 83, 0, 0, 84, 85, 0, 0, 0,
+ 0, 67, 68, 86, 0, 0, 0, 0, 87, 69,
+ 70, 0, 0, 175, 0, 0, 71, 72, 0, 73,
+ 0, 0, 0, 0, 0, 74, 0, 31, 62, 32,
+ 63, 75, 0, 0, 76, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 64, 65, 0, 77, 78, 0,
+ 0, 0, 0, 0, 0, 79, 67, 68, 80, 0,
+ 0, 0, 0, 0, 69, 70, 0, 0, 0, 0,
+ 81, 71, 72, 0, 73, 0, 0, 0, 82, 83,
+ 74, 0, 84, 85, 0, 0, 75, 0, 0, 76,
+ 86, 0, 0, 0, 0, 87, 0, 0, 0, 0,
+ 88, 0, 77, 78, 0, 0, 0, 0, 0, 0,
+ 79, 0, 0, 80, 62, 0, 63, 0, 0, 0,
+ 0, 0, 0, 0, 0, 81, 0, 0, 0, 0,
+ 64, 150, 0, 82, 83, 0, 0, 84, 85, 0,
+ 0, 0, 67, 68, 0, 86, 0, 0, 0, 0,
+ 87, 70, 0, 0, 0, 88, 0, 71, 72, 0,
+ 73, 67, 68, 0, 0, 0, 74, 0, 0, 0,
+ 70, 0, 0, 0, 0, 76, 71, 72, 0, 73,
+ 0, 0, 0, 0, 0, 74, 0, 0, 77, 78,
+ 0, 0, 0, 0, 76, 0, 79, 0, 0, 80,
+ 0, 0, 0, 0, 0, 0, 0, 77, 241, 0,
+ 0, 81, 0, 0, 0, 79, 0, 0, 0, 82,
+ 0, 0, 0, 84, 85, 0, 0, 0, 0, 0,
+ 81, 86, 0, 0, 0, 0, 87, 0, 82, 0,
+ 0, 0, 84, 85, 0, 0, 0, 0, 0, 0,
+ 86, 0, 0, 0, 0, 87
};
static const yytype_int16 yycheck[] =
{
- 4, 122, 8, 118, 12, 135, 214, 11, 12, 114,
- 179, 41, 106, 235, 0, 10, 27, 122, 8, 200,
- 4, 8, 6, 7, 37, 4, 30, 6, 110, 160,
- 34, 35, 36, 37, 38, 205, 4, 8, 6, 69,
- 26, 135, 77, 23, 24, 29, 28, 21, 22, 59,
- 59, 172, 122, 126, 21, 22, 129, 87, 21, 22,
- 35, 75, 132, 38, 39, 170, 71, 172, 21, 22,
- 200, 106, 17, 18, 19, 61, 58, 171, 136, 10,
- 11, 12, 13, 14, 15, 16, 17, 18, 19, 93,
- 21, 22, 43, 151, 89, 100, 110, 105, 279, 29,
- 104, 105, 115, 107, 108, 236, 200, 137, 112, 113,
- 106, 121, 121, 283, 118, 54, 67, 130, 326, 130,
- 150, 111, 61, 119, 68, 131, 56, 106, 233, 74,
- 54, 135, 110, 108, 130, 0, 140, 61, 243, 244,
- 348, 310, 172, 74, 131, 149, 130, 98, 111, 279,
- 372, 130, 182, 133, 80, 185, 160, 131, 162, 163,
- 131, 255, 130, 332, 131, 116, 10, 4, 121, 6,
- 9, 69, 293, 159, 382, 65, 21, 22, 76, 50,
- 3, 296, 21, 22, 214, 279, 84, 32, 293, 68,
- 220, 69, 107, 83, 31, 132, 200, 201, 76, 43,
- 131, 205, 73, 81, 31, 103, 84, 130, 52, 9,
- 55, 44, 45, 343, 218, 64, 87, 88, 222, 388,
- 80, 21, 22, 5, 254, 103, 256, 130, 349, 233,
- 324, 261, 236, 131, 105, 4, 92, 6, 268, 354,
- 355, 122, 91, 92, 349, 89, 126, 7, 128, 343,
- 44, 45, 7, 124, 98, 376, 131, 132, 102, 289,
- 4, 366, 6, 271, 130, 114, 79, 271, 131, 132,
- 5, 376, 116, 131, 132, 279, 130, 21, 22, 283,
- 21, 22, 79, 131, 132, 131, 132, 31, 131, 132,
- 131, 132, 296, 323, 130, 5, 326, 418, 68, 131,
- 132, 131, 132, 131, 132, 335, 131, 132, 132, 313,
- 131, 132, 130, 418, 131, 132, 131, 132, 348, 131,
- 132, 131, 132, 131, 132, 131, 132, 123, 4, 5,
- 6, 7, 131, 132, 449, 130, 7, 7, 131, 343,
- 106, 130, 11, 130, 8, 21, 22, 23, 9, 464,
- 354, 355, 382, 468, 384, 20, 60, 33, 34, 131,
- 68, 132, 366, 131, 70, 41, 42, 131, 130, 82,
- 82, 130, 48, 49, 130, 51, 118, 60, 94, 119,
- 56, 57, 81, 413, 7, 94, 7, 63, 40, 419,
- 66, 94, 72, 130, 4, 5, 6, 7, 130, 5,
- 10, 405, 78, 79, 132, 27, 123, 89, 412, 130,
- 86, 21, 22, 89, 82, 130, 89, 36, 97, 130,
- 97, 93, 40, 33, 34, 101, 456, 431, 432, 130,
- 130, 41, 42, 109, 92, 95, 112, 113, 48, 49,
- 444, 51, 446, 132, 120, 449, 85, 57, 5, 125,
- 40, 7, 62, 63, 130, 131, 66, 102, 132, 90,
- 464, 96, 7, 130, 468, 131, 316, 130, 78, 79,
- 432, 130, 169, 446, 343, 218, 86, 105, 243, 89,
- 137, 244, 170, 124, 456, 141, 102, 4, 5, 6,
- 7, 101, 146, 10, 388, 129, 106, 183, 183, 109,
- -1, 403, 112, 113, 21, 22, -1, -1, -1, -1,
- 120, -1, -1, -1, -1, 125, 33, 34, -1, -1,
- 130, -1, -1, -1, 41, 42, -1, -1, -1, -1,
- -1, 48, 49, -1, 51, -1, -1, -1, -1, -1,
- 57, -1, -1, -1, -1, 62, 63, -1, -1, 66,
+ 4, 120, 218, 124, 12, 108, 138, 11, 12, 116,
+ 239, 182, 0, 41, 4, 27, 6, 124, 4, 209,
+ 6, 7, 4, 8, 6, 59, 30, 37, 8, 8,
+ 34, 35, 36, 37, 38, 138, 163, 112, 26, 8,
+ 78, 69, 67, 29, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 175, 21, 22, 10, 23, 24,
+ 88, 21, 22, 21, 22, 204, 173, 59, 175, 107,
+ 29, 174, 204, 61, 21, 22, 21, 22, 17, 18,
+ 19, 21, 22, 76, 35, 72, 69, 38, 39, 123,
+ 94, 10, 128, 28, 4, 131, 6, 56, 288, 107,
+ 81, 204, 106, 107, 132, 109, 110, 117, 133, 75,
+ 114, 115, 140, 240, 101, 331, 120, 107, 107, 112,
+ 132, 31, 132, 58, 43, 153, 133, 134, 113, 69,
+ 237, 123, 121, 52, 138, 112, 75, 90, 354, 143,
+ 247, 248, 132, 132, 315, 284, 132, 175, 152, 378,
+ 132, 0, 284, 133, 133, 50, 259, 185, 109, 163,
+ 188, 165, 166, 1, 133, 3, 337, 133, 113, 3,
+ 135, 90, 388, 133, 162, 133, 123, 298, 65, 74,
+ 99, 284, 301, 139, 103, 4, 4, 6, 6, 43,
+ 218, 298, 30, 88, 89, 70, 224, 84, 154, 118,
+ 204, 205, 77, 21, 22, 209, 124, 69, 46, 47,
+ 85, 106, 9, 31, 68, 53, 134, 349, 222, 108,
+ 58, 64, 226, 394, 21, 22, 329, 134, 54, 104,
+ 258, 126, 260, 237, 355, 61, 240, 265, 44, 45,
+ 78, 360, 361, 9, 272, 99, 349, 54, 355, 92,
+ 93, 31, 21, 22, 61, 21, 22, 128, 133, 130,
+ 132, 382, 100, 32, 118, 372, 294, 70, 276, 107,
+ 44, 45, 276, 116, 77, 382, 133, 134, 5, 82,
+ 284, 119, 85, 81, 288, 93, 55, 125, 133, 134,
+ 128, 21, 22, 133, 134, 132, 124, 301, 133, 134,
+ 328, 104, 7, 331, 133, 134, 427, 7, 133, 134,
+ 133, 134, 340, 80, 318, 5, 133, 134, 133, 134,
+ 427, 133, 134, 133, 134, 80, 354, 5, 133, 134,
+ 69, 133, 134, 133, 134, 4, 5, 6, 7, 132,
+ 459, 133, 134, 133, 134, 349, 133, 134, 133, 134,
+ 132, 132, 21, 22, 23, 474, 360, 361, 134, 478,
+ 388, 132, 390, 125, 33, 34, 7, 132, 372, 132,
+ 7, 11, 41, 42, 133, 107, 132, 8, 132, 48,
+ 49, 9, 51, 60, 20, 133, 69, 56, 57, 133,
+ 71, 134, 133, 421, 63, 132, 83, 66, 83, 132,
+ 428, 120, 60, 132, 121, 7, 95, 95, 82, 413,
+ 79, 80, 7, 95, 7, 40, 420, 73, 87, 132,
+ 132, 90, 5, 27, 134, 90, 125, 90, 132, 83,
+ 132, 132, 98, 102, 98, 94, 440, 441, 466, 132,
+ 36, 110, 111, 96, 7, 114, 115, 93, 132, 40,
+ 454, 86, 456, 122, 134, 459, 5, 132, 127, 4,
+ 5, 6, 7, 132, 133, 10, 40, 7, 91, 103,
+ 474, 133, 133, 97, 478, 132, 21, 22, 7, 134,
+ 132, 172, 321, 456, 441, 107, 222, 247, 33, 34,
+ 140, 349, 466, 144, 173, 248, 41, 42, 104, 126,
+ 149, 131, 394, 48, 49, 411, 51, -1, -1, -1,
+ 186, -1, 57, 186, -1, -1, -1, 62, 63, -1,
+ -1, 66, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, 79, 80, -1, -1, -1, -1,
+ -1, -1, 87, -1, -1, 90, 4, 5, 6, 7,
+ -1, -1, 10, -1, -1, -1, -1, 102, -1, -1,
+ -1, -1, 107, 21, 22, 110, 111, -1, -1, 114,
+ 115, -1, -1, -1, -1, 33, 34, 122, -1, -1,
+ -1, -1, 127, 41, 42, -1, -1, 132, -1, -1,
+ 48, 49, -1, 51, -1, -1, -1, -1, -1, 57,
+ -1, -1, -1, -1, 62, 63, -1, -1, 66, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, 78, 79, -1, -1, -1, -1, -1, -1, 86,
- -1, -1, 89, -1, -1, -1, -1, -1, -1, -1,
- 4, 5, 6, 7, 101, -1, -1, -1, -1, -1,
- -1, -1, 109, -1, -1, 112, 113, 21, 22, 23,
- -1, -1, -1, 120, -1, -1, -1, -1, 125, 33,
- 34, -1, -1, 130, -1, -1, -1, 41, 42, -1,
- -1, -1, -1, -1, 48, 49, -1, 51, -1, -1,
- -1, -1, -1, 57, -1, -1, -1, -1, -1, 63,
- -1, -1, 66, -1, -1, -1, 4, 5, 6, 7,
- -1, -1, -1, -1, 78, 79, -1, -1, -1, -1,
- -1, -1, 86, 21, 22, 89, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, 33, 34, 101, -1, -1,
- -1, -1, -1, 41, 42, 109, -1, -1, 112, 113,
- 48, 49, -1, 51, -1, -1, 120, -1, -1, 57,
- -1, 125, -1, -1, -1, 63, 130, -1, 66, -1,
- -1, -1, 4, 5, 6, 7, -1, -1, -1, -1,
- 78, 79, -1, -1, -1, -1, -1, -1, 86, 21,
- 22, 89, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, 33, 34, 101, -1, -1, -1, -1, -1, 41,
- 42, 109, -1, -1, 112, 113, 48, 49, -1, 51,
- -1, -1, 120, 121, -1, 57, -1, 125, -1, -1,
- -1, 63, 130, -1, 66, -1, -1, -1, 4, 5,
- 6, 7, -1, -1, -1, -1, 78, 79, -1, -1,
- -1, -1, -1, -1, 86, 21, 22, 89, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, 33, 34, 101,
- -1, -1, -1, -1, 106, 41, 42, 109, -1, -1,
- 112, 113, 48, 49, -1, 51, -1, -1, 120, -1,
- -1, 57, -1, 125, -1, -1, 62, 63, 130, -1,
- 66, -1, -1, -1, 4, 5, 6, 7, -1, -1,
- -1, -1, 78, 79, -1, -1, -1, -1, -1, -1,
- 86, 21, 22, 89, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, 33, 34, 101, -1, -1, -1, -1,
- -1, 41, 42, 109, -1, -1, 112, 113, 48, 49,
- -1, 51, -1, -1, 120, -1, -1, 57, -1, 125,
- -1, -1, -1, 63, 130, -1, 66, -1, -1, -1,
- 4, 5, 6, 7, -1, -1, -1, -1, 78, 79,
- -1, -1, 1, -1, 3, -1, 86, 21, 22, 89,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, 33,
- 34, 101, -1, -1, -1, -1, -1, 41, 42, 109,
- -1, 30, 112, 113, 48, 49, -1, 51, -1, -1,
- 120, -1, -1, 57, -1, 125, -1, 46, 47, 63,
- 130, -1, 66, -1, 53, -1, -1, -1, -1, 58,
- -1, -1, -1, -1, 78, 79, -1, -1, -1, -1,
- -1, -1, 86, -1, -1, 89, -1, -1, 77, -1,
- -1, -1, -1, -1, -1, -1, 5, 101, 7, -1,
- -1, -1, -1, -1, -1, 109, -1, -1, 112, 113,
- 99, -1, 21, 22, 33, 34, 120, 106, -1, -1,
- -1, 125, -1, 42, 33, 34, 130, -1, 117, 48,
- 49, -1, 51, 42, 123, -1, -1, 126, 57, 48,
- 49, -1, 51, -1, -1, -1, -1, 66, 57, -1,
- -1, -1, -1, -1, -1, -1, -1, 66, -1, 78,
- 79, -1, -1, -1, -1, -1, -1, 86, -1, 78,
- 79, -1, -1, -1, -1, -1, -1, 86, -1, -1,
- 89, -1, 101, -1, -1, -1, -1, -1, -1, -1,
- 109, -1, 101, 112, 113, -1, -1, -1, -1, -1,
- 109, 120, -1, 112, 113, -1, 125, -1, -1, -1,
- -1, 120, -1, -1, -1, -1, 125, 10, 11, 12,
- 13, 14, 15, 16, 17, 18, 19, -1, 21, 22,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, 74
+ -1, 79, 80, -1, -1, -1, -1, -1, -1, 87,
+ -1, -1, 90, -1, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 102, 21, 22, -1, 4, 5,
+ 6, 7, 110, 111, -1, -1, 114, 115, -1, -1,
+ -1, -1, -1, -1, 122, 21, 22, 23, -1, 127,
+ -1, -1, -1, -1, 132, -1, -1, 33, 34, -1,
+ -1, -1, -1, -1, -1, 41, 42, -1, -1, -1,
+ -1, -1, 48, 49, -1, 51, -1, -1, -1, 75,
+ -1, 57, -1, 4, 5, 6, 7, 63, -1, -1,
+ 66, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ 21, 22, -1, 79, 80, -1, -1, -1, -1, -1,
+ -1, 87, 33, 34, 90, -1, -1, -1, -1, -1,
+ 41, 42, -1, -1, -1, -1, 102, 48, 49, -1,
+ 51, -1, -1, -1, 110, 111, 57, -1, 114, 115,
+ -1, -1, 63, -1, -1, 66, 122, -1, -1, -1,
+ -1, 127, -1, -1, -1, -1, 132, -1, 79, 80,
+ -1, -1, -1, -1, -1, -1, 87, -1, -1, 90,
+ 4, 5, 6, 7, -1, -1, -1, -1, -1, -1,
+ -1, 102, -1, -1, -1, -1, -1, 21, 22, 110,
+ 111, -1, -1, 114, 115, -1, -1, -1, -1, 33,
+ 34, 122, 123, -1, -1, -1, 127, 41, 42, -1,
+ -1, 132, -1, -1, 48, 49, -1, 51, -1, -1,
+ -1, -1, -1, 57, -1, 4, 5, 6, 7, 63,
+ -1, -1, 66, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, 21, 22, -1, 79, 80, -1, -1, -1,
+ -1, -1, -1, 87, 33, 34, 90, -1, -1, -1,
+ -1, -1, 41, 42, -1, -1, -1, -1, 102, 48,
+ 49, -1, 51, 107, -1, -1, 110, 111, 57, -1,
+ 114, 115, -1, 62, 63, -1, -1, 66, 122, -1,
+ -1, -1, -1, 127, -1, -1, -1, -1, 132, -1,
+ 79, 80, -1, -1, -1, -1, -1, -1, 87, -1,
+ -1, 90, 4, 5, 6, 7, -1, -1, -1, -1,
+ -1, -1, -1, 102, -1, -1, -1, -1, -1, 21,
+ 22, 110, 111, -1, -1, 114, 115, -1, -1, -1,
+ -1, 33, 34, 122, -1, -1, -1, -1, 127, 41,
+ 42, -1, -1, 132, -1, -1, 48, 49, -1, 51,
+ -1, -1, -1, -1, -1, 57, -1, 4, 5, 6,
+ 7, 63, -1, -1, 66, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, 21, 22, -1, 79, 80, -1,
+ -1, -1, -1, -1, -1, 87, 33, 34, 90, -1,
+ -1, -1, -1, -1, 41, 42, -1, -1, -1, -1,
+ 102, 48, 49, -1, 51, -1, -1, -1, 110, 111,
+ 57, -1, 114, 115, -1, -1, 63, -1, -1, 66,
+ 122, -1, -1, -1, -1, 127, -1, -1, -1, -1,
+ 132, -1, 79, 80, -1, -1, -1, -1, -1, -1,
+ 87, -1, -1, 90, 5, -1, 7, -1, -1, -1,
+ -1, -1, -1, -1, -1, 102, -1, -1, -1, -1,
+ 21, 22, -1, 110, 111, -1, -1, 114, 115, -1,
+ -1, -1, 33, 34, -1, 122, -1, -1, -1, -1,
+ 127, 42, -1, -1, -1, 132, -1, 48, 49, -1,
+ 51, 33, 34, -1, -1, -1, 57, -1, -1, -1,
+ 42, -1, -1, -1, -1, 66, 48, 49, -1, 51,
+ -1, -1, -1, -1, -1, 57, -1, -1, 79, 80,
+ -1, -1, -1, -1, 66, -1, 87, -1, -1, 90,
+ -1, -1, -1, -1, -1, -1, -1, 79, 80, -1,
+ -1, 102, -1, -1, -1, 87, -1, -1, -1, 110,
+ -1, -1, -1, 114, 115, -1, -1, -1, -1, -1,
+ 102, 122, -1, -1, -1, -1, 127, -1, 110, -1,
+ -1, -1, 114, 115, -1, -1, -1, -1, -1, -1,
+ 122, -1, -1, -1, -1, 127
};
/* YYSTOS[STATE-NUM] -- The (internal number of the) accessing
symbol of state STATE-NUM. */
static const yytype_uint8 yystos[] =
{
- 0, 1, 3, 30, 46, 47, 53, 58, 77, 99,
- 106, 117, 123, 126, 135, 136, 137, 138, 139, 140,
- 141, 162, 163, 166, 167, 170, 171, 174, 227, 228,
- 110, 4, 6, 225, 75, 110, 68, 110, 80, 29,
- 56, 175, 225, 172, 173, 186, 225, 0, 126, 128,
- 77, 174, 126, 129, 3, 225, 68, 225, 225, 225,
- 225, 225, 5, 7, 21, 22, 23, 33, 34, 41,
- 42, 48, 49, 51, 57, 63, 66, 78, 79, 86,
- 89, 101, 109, 112, 113, 120, 125, 130, 144, 176,
- 177, 178, 180, 202, 203, 204, 205, 206, 207, 208,
- 215, 218, 221, 225, 107, 132, 31, 130, 80, 28,
- 58, 5, 92, 130, 122, 196, 197, 119, 130, 174,
- 7, 7, 121, 202, 211, 212, 130, 79, 130, 5,
- 130, 79, 174, 202, 5, 68, 179, 132, 21, 22,
- 31, 222, 225, 23, 24, 133, 223, 22, 205, 27,
- 130, 168, 169, 225, 173, 130, 180, 224, 225, 225,
- 44, 45, 44, 45, 123, 164, 225, 142, 143, 225,
- 10, 62, 130, 198, 199, 200, 201, 202, 218, 130,
- 224, 198, 121, 209, 210, 59, 212, 213, 7, 50,
- 73, 87, 88, 105, 124, 216, 216, 7, 131, 131,
- 130, 180, 183, 184, 187, 206, 225, 196, 178, 225,
- 203, 204, 225, 23, 56, 131, 202, 214, 132, 196,
- 11, 131, 132, 174, 142, 43, 67, 98, 116, 148,
- 225, 225, 130, 130, 151, 131, 132, 79, 144, 201,
- 180, 198, 202, 8, 9, 10, 11, 12, 13, 14,
- 15, 16, 17, 18, 19, 74, 220, 20, 215, 217,
- 131, 111, 202, 210, 213, 202, 60, 131, 68, 131,
- 183, 31, 185, 186, 69, 76, 84, 103, 182, 132,
- 185, 37, 115, 181, 70, 188, 131, 214, 131, 132,
- 169, 202, 225, 130, 82, 82, 130, 54, 61, 165,
- 218, 219, 225, 118, 148, 149, 150, 142, 10, 43,
- 52, 89, 98, 102, 116, 145, 146, 147, 131, 199,
- 200, 17, 18, 19, 74, 202, 130, 180, 202, 10,
- 89, 131, 132, 119, 202, 111, 60, 202, 131, 186,
- 94, 94, 94, 81, 183, 7, 7, 185, 40, 72,
- 189, 131, 202, 198, 130, 130, 224, 5, 64, 91,
- 92, 114, 226, 131, 132, 131, 132, 35, 38, 39,
- 108, 160, 132, 123, 152, 89, 130, 215, 82, 225,
- 145, 202, 130, 180, 9, 214, 89, 215, 130, 202,
- 131, 184, 97, 97, 214, 198, 93, 190, 131, 224,
- 224, 131, 54, 61, 218, 130, 161, 148, 36, 95,
- 153, 198, 130, 9, 214, 202, 131, 217, 92, 40,
- 85, 191, 131, 131, 5, 226, 155, 156, 157, 158,
- 159, 225, 130, 40, 131, 225, 202, 131, 131, 198,
- 192, 193, 202, 7, 102, 131, 132, 7, 29, 130,
- 225, 155, 71, 100, 154, 131, 132, 32, 55, 194,
- 225, 156, 224, 131, 130, 193, 90, 195, 130, 131,
- 224, 65, 83, 224, 131, 131, 96, 7
+ 0, 1, 3, 30, 46, 47, 53, 58, 78, 100,
+ 107, 119, 125, 128, 137, 138, 139, 140, 141, 142,
+ 143, 164, 165, 168, 169, 172, 173, 176, 230, 231,
+ 112, 4, 6, 228, 76, 112, 69, 112, 81, 29,
+ 56, 177, 228, 174, 175, 188, 228, 0, 128, 130,
+ 78, 176, 128, 131, 3, 228, 69, 228, 228, 228,
+ 228, 228, 5, 7, 21, 22, 23, 33, 34, 41,
+ 42, 48, 49, 51, 57, 63, 66, 79, 80, 87,
+ 90, 102, 110, 111, 114, 115, 122, 127, 132, 146,
+ 178, 179, 180, 182, 204, 205, 206, 207, 208, 209,
+ 210, 211, 218, 221, 224, 228, 108, 134, 31, 132,
+ 81, 28, 58, 5, 93, 132, 124, 198, 199, 121,
+ 132, 176, 7, 7, 123, 204, 214, 215, 132, 80,
+ 132, 5, 132, 132, 80, 176, 204, 5, 69, 181,
+ 134, 21, 22, 31, 225, 228, 23, 24, 135, 226,
+ 22, 207, 27, 132, 170, 171, 228, 175, 132, 182,
+ 227, 228, 228, 44, 45, 44, 45, 125, 166, 228,
+ 144, 145, 228, 10, 62, 132, 200, 201, 202, 203,
+ 204, 221, 132, 227, 200, 123, 212, 213, 59, 215,
+ 216, 7, 50, 74, 88, 89, 106, 126, 219, 219,
+ 204, 7, 133, 133, 132, 182, 185, 186, 189, 208,
+ 228, 198, 180, 228, 205, 206, 228, 23, 56, 133,
+ 204, 217, 134, 198, 11, 133, 134, 176, 144, 43,
+ 68, 99, 118, 150, 228, 228, 132, 132, 153, 133,
+ 134, 80, 146, 203, 182, 200, 204, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18, 19, 75,
+ 223, 20, 218, 220, 133, 113, 204, 213, 216, 204,
+ 60, 133, 69, 69, 133, 185, 31, 187, 188, 70,
+ 77, 85, 104, 184, 134, 187, 37, 117, 183, 71,
+ 190, 133, 217, 133, 134, 171, 204, 228, 132, 83,
+ 83, 132, 54, 61, 167, 221, 222, 228, 120, 150,
+ 151, 152, 144, 10, 43, 52, 90, 99, 103, 118,
+ 147, 148, 149, 133, 201, 202, 17, 18, 19, 75,
+ 204, 132, 182, 204, 10, 90, 133, 134, 121, 204,
+ 113, 60, 204, 7, 133, 188, 95, 95, 95, 82,
+ 185, 7, 7, 187, 40, 73, 191, 133, 204, 200,
+ 132, 132, 227, 5, 64, 92, 93, 116, 229, 133,
+ 134, 133, 134, 35, 38, 39, 109, 162, 134, 125,
+ 154, 90, 132, 218, 83, 228, 147, 204, 132, 182,
+ 9, 217, 90, 218, 132, 204, 133, 67, 133, 186,
+ 98, 98, 217, 200, 94, 192, 133, 227, 227, 133,
+ 54, 61, 221, 132, 163, 150, 36, 96, 155, 200,
+ 132, 9, 217, 204, 133, 220, 7, 93, 40, 86,
+ 193, 133, 133, 5, 229, 157, 158, 159, 160, 161,
+ 228, 132, 40, 133, 228, 204, 133, 133, 133, 200,
+ 194, 195, 204, 7, 103, 133, 134, 7, 29, 132,
+ 228, 157, 72, 101, 156, 133, 134, 32, 55, 196,
+ 228, 158, 227, 133, 132, 195, 91, 197, 132, 133,
+ 227, 65, 84, 227, 133, 133, 97, 7
};
/* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */
static const yytype_uint8 yyr1[] =
{
- 0, 134, 135, 135, 135, 135, 135, 135, 136, 136,
- 136, 136, 136, 136, 136, 136, 136, 136, 137, 138,
- 138, 138, 138, 139, 140, 141, 142, 143, 143, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 145, 145, 145,
- 145, 145, 145, 145, 146, 146, 147, 147, 148, 148,
- 148, 148, 149, 149, 150, 150, 151, 151, 152, 152,
- 153, 153, 154, 154, 155, 155, 156, 156, 156, 157,
- 157, 158, 159, 160, 160, 160, 160, 161, 161, 162,
- 162, 162, 162, 163, 164, 164, 165, 165, 165, 165,
- 166, 167, 168, 168, 169, 170, 170, 171, 172, 172,
- 173, 174, 175, 175, 175, 176, 176, 177, 177, 178,
- 178, 178, 179, 180, 181, 181, 181, 182, 182, 182,
- 182, 182, 182, 182, 182, 183, 183, 184, 184, 184,
- 184, 184, 184, 185, 185, 186, 186, 187, 187, 188,
- 188, 189, 189, 190, 190, 191, 191, 192, 192, 193,
- 194, 194, 194, 195, 195, 195, 196, 196, 197, 198,
- 198, 199, 199, 200, 200, 201, 201, 201, 201, 201,
- 201, 201, 201, 201, 201, 201, 202, 202, 203, 203,
- 204, 204, 205, 205, 205, 205, 205, 205, 205, 206,
- 206, 206, 206, 207, 208, 208, 209, 209, 210, 211,
- 211, 212, 213, 213, 214, 214, 215, 215, 215, 215,
- 215, 215, 215, 215, 216, 216, 216, 216, 216, 216,
- 217, 217, 218, 218, 219, 219, 220, 220, 220, 220,
- 220, 220, 220, 220, 220, 220, 221, 222, 222, 223,
- 223, 223, 224, 224, 225, 225, 226, 226, 226, 226,
- 227, 228, 228
+ 0, 136, 137, 137, 137, 137, 137, 137, 138, 138,
+ 138, 138, 138, 138, 138, 138, 138, 138, 139, 140,
+ 140, 140, 140, 141, 142, 143, 144, 145, 145, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 147, 147, 147,
+ 147, 147, 147, 147, 148, 148, 149, 149, 150, 150,
+ 150, 150, 151, 151, 152, 152, 153, 153, 154, 154,
+ 155, 155, 156, 156, 157, 157, 158, 158, 158, 159,
+ 159, 160, 161, 162, 162, 162, 162, 163, 163, 164,
+ 164, 164, 164, 165, 166, 166, 167, 167, 167, 167,
+ 168, 169, 170, 170, 171, 172, 172, 173, 174, 174,
+ 175, 176, 177, 177, 177, 178, 178, 179, 179, 180,
+ 180, 180, 181, 182, 183, 183, 183, 184, 184, 184,
+ 184, 184, 184, 184, 184, 185, 185, 186, 186, 186,
+ 186, 186, 186, 187, 187, 188, 188, 189, 189, 190,
+ 190, 191, 191, 192, 192, 193, 193, 194, 194, 195,
+ 196, 196, 196, 197, 197, 197, 198, 198, 199, 200,
+ 200, 201, 201, 202, 202, 203, 203, 203, 203, 203,
+ 203, 203, 203, 203, 203, 203, 204, 204, 205, 205,
+ 206, 206, 207, 207, 207, 207, 207, 207, 207, 207,
+ 208, 208, 208, 208, 209, 210, 210, 211, 211, 212,
+ 212, 213, 214, 214, 215, 216, 216, 217, 217, 218,
+ 218, 218, 218, 218, 218, 218, 218, 219, 219, 219,
+ 219, 219, 219, 220, 220, 221, 221, 222, 222, 223,
+ 223, 223, 223, 223, 223, 223, 223, 223, 223, 224,
+ 225, 225, 226, 226, 226, 227, 227, 228, 228, 229,
+ 229, 229, 229, 230, 231, 231
};
/* YYR2[YYN] -- Number of symbols on the right hand side of rule YYN. */
@@ -1381,14 +1391,14 @@ static const yytype_uint8 yyr2[] =
0, 1, 1, 0, 2, 2, 0, 1, 2, 3,
1, 3, 1, 2, 1, 5, 6, 4, 3, 3,
3, 2, 3, 5, 4, 6, 3, 1, 3, 1,
- 2, 1, 1, 1, 1, 1, 1, 3, 1, 3,
- 4, 4, 5, 6, 5, 4, 1, 2, 4, 1,
- 2, 4, 0, 2, 1, 3, 1, 1, 2, 2,
- 1, 2, 3, 2, 1, 1, 1, 1, 1, 1,
- 1, 3, 1, 3, 1, 3, 1, 1, 1, 1,
- 1, 1, 1, 2, 1, 2, 1, 1, 1, 1,
- 1, 1, 1, 3, 1, 1, 1, 1, 1, 1,
- 2, 2, 0
+ 2, 1, 1, 1, 1, 1, 1, 1, 3, 1,
+ 3, 4, 4, 5, 6, 6, 8, 5, 4, 1,
+ 2, 4, 1, 2, 4, 0, 2, 1, 3, 1,
+ 1, 2, 2, 1, 2, 3, 2, 1, 1, 1,
+ 1, 1, 1, 1, 3, 1, 3, 1, 3, 1,
+ 1, 1, 1, 1, 1, 1, 2, 1, 2, 1,
+ 1, 1, 1, 1, 1, 1, 3, 1, 1, 1,
+ 1, 1, 1, 2, 2, 0
};
@@ -1885,603 +1895,593 @@ yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep, YYLTYPE *yylocatio
switch (yytype)
{
case 3: /* TOKEN_COMMAND */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
}
-#line 1895 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1905 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
case 4: /* TOKEN_NAME */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
}
-#line 1905 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1915 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
case 5: /* TOKEN_STRING_SINGLE_QUOTED */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
}
-#line 1915 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1925 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
case 6: /* TOKEN_STRING_DOUBLE_QUOTED */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
}
-#line 1925 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1935 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
case 7: /* TOKEN_UNSIGNED_NUMVAL */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).numeric_literal_value_) != nullptr) {
delete ((*yyvaluep).numeric_literal_value_);
}
}
-#line 1935 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1945 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 136: /* sql_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 138: /* sql_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).statement_) != nullptr) {
delete ((*yyvaluep).statement_);
}
}
-#line 1945 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1955 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 137: /* quit_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 139: /* quit_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).quit_statement_) != nullptr) {
delete ((*yyvaluep).quit_statement_);
}
}
-#line 1955 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1965 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 138: /* alter_table_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 140: /* alter_table_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).statement_) != nullptr) {
delete ((*yyvaluep).statement_);
}
}
-#line 1965 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1975 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 139: /* create_table_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 141: /* create_table_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).create_table_statement_) != nullptr) {
delete ((*yyvaluep).create_table_statement_);
}
}
-#line 1975 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1985 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 140: /* create_index_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 142: /* create_index_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).statement_) != nullptr) {
delete ((*yyvaluep).statement_);
}
}
-#line 1985 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1995 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 141: /* drop_table_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 143: /* drop_table_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).drop_table_statement_) != nullptr) {
delete ((*yyvaluep).drop_table_statement_);
}
}
-#line 1995 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2005 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 142: /* column_def */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 144: /* column_def */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).attribute_definition_) != nullptr) {
delete ((*yyvaluep).attribute_definition_);
}
}
-#line 2005 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2015 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 143: /* column_def_commalist */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 145: /* column_def_commalist */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).attribute_definition_list_) != nullptr) {
delete ((*yyvaluep).attribute_definition_list_);
}
}
-#line 2015 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2025 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 144: /* data_type */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 146: /* data_type */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).data_type_) != nullptr) {
delete ((*yyvaluep).data_type_);
}
}
-#line 2025 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2035 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 145: /* column_constraint_def */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 147: /* column_constraint_def */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).column_constraint_) != nullptr) {
delete ((*yyvaluep).column_constraint_);
}
}
-#line 2035 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2045 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 146: /* column_constraint_def_list */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 148: /* column_constraint_def_list */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).column_constraint_list_) != nullptr) {
delete ((*yyvaluep).column_constraint_list_);
}
}
-#line 2045 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2055 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 147: /* opt_column_constraint_def_list */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 149: /* opt_column_constraint_def_list */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).column_constraint_list_) != nullptr) {
delete ((*yyvaluep).column_constraint_list_);
}
}
-#line 2055 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2065 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 151: /* opt_column_list */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 153: /* opt_column_list */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).attribute_list_) != nullptr) {
delete ((*yyvaluep).attribute_list_);
}
}
-#line 2065 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2075 "SqlParser_gen.cpp" /*
<TRUNCATED>
[21/48] incubator-quickstep git commit: Quickstep print catalog (#222)
Posted by hb...@apache.org.
Quickstep print catalog (#222)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/205aa4fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/205aa4fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/205aa4fe
Branch: refs/heads/query-manager-used-in-foreman
Commit: 205aa4fefadbf28e8c3e7fbd0fc374e8f634c43a
Parents: 4608aa2
Author: Rogers Jeffrey Leo John <ro...@gmail.com>
Authored: Tue May 17 18:43:02 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:43 2016 -0700
----------------------------------------------------------------------
cli/CMakeLists.txt | 3 +-
cli/CommandExecutor.cpp | 50 +++++++++++++++++++++++-----
cli/CommandExecutor.hpp | 8 +++--
cli/PrintToScreen.cpp | 11 ++++++
cli/PrintToScreen.hpp | 8 +++++
cli/QuickstepCli.cpp | 4 ++-
cli/tests/CommandExecutorTestRunner.cpp | 1 +
cli/tests/command_executor/D.test | 35 ++++++++++++-------
cli/tests/command_executor/Dt.test | 36 +++++++++++++++-----
9 files changed, 124 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/205aa4fe/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index a1989d5..761b6d8 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -73,8 +73,9 @@ target_link_libraries(quickstep_cli_CommandExecutor
quickstep_catalog_CatalogDatabase
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogRelationSchema
+ quickstep_cli_PrintToScreen
quickstep_parser_ParseStatement
- quickstep_cli_PrintToScreen
+ quickstep_storage_StorageBlockInfo
quickstep_utility_Macros
quickstep_utility_PtrVector
quickstep_utility_SqlError)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/205aa4fe/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index f38121f..026922a 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -30,6 +30,7 @@
#include "catalog/CatalogRelationSchema.hpp"
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
+#include "storage/StorageBlockInfo.hpp"
#include "utility/PtrVector.hpp"
#include "utility/Macros.hpp"
#include "utility/SqlError.hpp"
@@ -52,15 +53,22 @@ namespace C = ::quickstep::cli;
void executeDescribeDatabase(
const PtrVector<ParseString> *arguments,
- const CatalogDatabase &catalog_database, FILE *out) {
+ const CatalogDatabase &catalog_database,
+ StorageManager *storage_manager,
+ FILE *out) {
// Column width initialized to 6 to take into account the header name
// and the column value table
int max_column_width = C::kInitMaxColumnWidth;
+ vector<std::size_t> num_tuples;
+ vector<std::size_t> num_blocks;
const CatalogRelation *relation = nullptr;
if (arguments->size() == 0) {
for (const CatalogRelation &rel : catalog_database) {
max_column_width =
std::max(static_cast<int>(rel.getName().length()), max_column_width);
+ num_blocks.push_back(rel.size_blocks());
+ num_tuples.push_back(
+ PrintToScreen::GetNumTuplesInRelation(rel, storage_manager));
}
} else {
const ParseString &table_name = arguments->front();
@@ -72,26 +80,51 @@ void executeDescribeDatabase(
}
max_column_width = std::max(static_cast<int>(relation->getName().length()),
max_column_width);
+ num_blocks.push_back(relation->size_blocks());
+ num_tuples.push_back(PrintToScreen::GetNumTuplesInRelation(
+ *relation,
+ storage_manager));
}
// Only if we have relations work on the printing logic.
if (catalog_database.size() > 0) {
+ const std::size_t max_num_blocks = *std::max_element(num_blocks.begin(), num_blocks.end());
+ const std::size_t max_num_rows = *std::max_element(num_tuples.begin(), num_tuples.end());
+ const int max_num_rows_digits = std::max(PrintToScreen::GetNumberOfDigits(max_num_rows),
+ C::kInitMaxColumnWidth);
+ const int max_num_blocks_digits = std::max(PrintToScreen::GetNumberOfDigits(max_num_blocks),
+ C::kInitMaxColumnWidth+2);
+
vector<int> column_widths;
- column_widths.push_back(max_column_width+1);
- column_widths.push_back(C::kInitMaxColumnWidth+1);
+ column_widths.push_back(max_column_width +1);
+ column_widths.push_back(C::kInitMaxColumnWidth + 1);
+ column_widths.push_back(max_num_blocks_digits + 1);
+ column_widths.push_back(max_num_rows_digits + 1);
fputs(" List of relations\n\n", out);
fprintf(out, "%-*s |", max_column_width+1, " Name");
- fprintf(out, "%-*s\n", C::kInitMaxColumnWidth, " Type");
+ fprintf(out, "%-*s |", C::kInitMaxColumnWidth, " Type");
+ fprintf(out, "%-*s |", max_num_blocks_digits, " Blocks");
+ fprintf(out, "%-*s\n", max_num_rows_digits, " Rows");
PrintToScreen::printHBar(column_widths, out);
// If there are no argument print the entire list of tables
// else print the particular table only.
+ vector<std::size_t>::const_iterator num_tuples_it = num_tuples.begin();
+ vector<std::size_t>::const_iterator num_blocks_it = num_blocks.begin();
if (arguments->size() == 0) {
for (const CatalogRelation &rel : catalog_database) {
fprintf(out, " %-*s |", max_column_width, rel.getName().c_str());
- fprintf(out, " %-*s\n", C::kInitMaxColumnWidth, "table");
+ fprintf(out, " %-*s |", C::kInitMaxColumnWidth - 1, "table");
+ fprintf(out, " %-*lu |", max_num_blocks_digits - 1, *num_blocks_it);
+ fprintf(out, " %-*lu\n", max_num_rows_digits - 1, *num_tuples_it);
+ ++num_tuples_it;
+ ++num_blocks_it;
}
} else {
fprintf(out, " %-*s |", max_column_width, relation->getName().c_str());
- fprintf(out, " %-*s\n", C::kInitMaxColumnWidth, "table");
+ fprintf(out, " %-*s |", C::kInitMaxColumnWidth -1, "table");
+ fprintf(out, " %-*lu |", max_num_blocks_digits - 1, *num_blocks_it);
+ fprintf(out, " %-*lu\n", max_num_rows_digits - 1, *num_tuples_it);
+ ++num_tuples_it;
+ ++num_blocks_it;
}
fputc('\n', out);
}
@@ -166,15 +199,16 @@ void executeDescribeTable(
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
+ StorageManager *storage_manager,
FILE *out) {
const ParseCommand &command = static_cast<const ParseCommand &>(statement);
const PtrVector<ParseString> *arguments = command.arguments();
const std::string &command_str = command.command()->value();
if (command_str == C::kDescribeDatabaseCommand) {
- executeDescribeDatabase(arguments, catalog_database, out);
+ executeDescribeDatabase(arguments, catalog_database, storage_manager, out);
} else if (command_str == C::kDescribeTableCommand) {
if (arguments->size() == 0) {
- executeDescribeDatabase(arguments, catalog_database, out);
+ executeDescribeDatabase(arguments, catalog_database, storage_manager, out);
} else {
executeDescribeTable(arguments, catalog_database, out);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/205aa4fe/cli/CommandExecutor.hpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.hpp b/cli/CommandExecutor.hpp
index 21eee6a..f367ca1 100644
--- a/cli/CommandExecutor.hpp
+++ b/cli/CommandExecutor.hpp
@@ -19,9 +19,11 @@
#define QUICKSTEP_CLI_COMMAND_COMMAND_EXECUTOR_HPP_
#include <cstdio>
+#include <limits>
#include <string>
#include "parser/ParseStatement.hpp"
+#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
using std::fprintf;
@@ -33,6 +35,7 @@ namespace quickstep {
class CatalogDatabase;
class CatalogAttribute;
class CatalogRelation;
+class StorageManager;
namespace cli {
/** \addtogroup CLI
@@ -49,13 +52,14 @@ constexpr char kDescribeTableCommand[] = "\\d";
/**
* @brief Executes the command by calling the command handler.
- *
+ *
* @param statement The parsed statement from the cli.
* @param catalog_database The catalog information about the current database.
- * @param out The stream where the output of the command has to be redirected to.
+ * @param out The stream where the output of the command has to be redirected to.
*/
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
+ StorageManager *storage_manager,
FILE *out);
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/205aa4fe/cli/PrintToScreen.cpp
----------------------------------------------------------------------
diff --git a/cli/PrintToScreen.cpp b/cli/PrintToScreen.cpp
index 227ff39..76e90eb 100644
--- a/cli/PrintToScreen.cpp
+++ b/cli/PrintToScreen.cpp
@@ -19,6 +19,7 @@
#include <cstddef>
#include <cstdio>
+#include <cmath>
#include <memory>
#include <vector>
@@ -47,6 +48,16 @@ DEFINE_bool(printing_enabled, true,
"If true, print query results to screen normally. If false, skip "
"printing output (e.g. for benchmarking).");
+int PrintToScreen::GetNumberOfDigits(int number) {
+ if (number > 0) {
+ return static_cast<int>(log10 (number)) + 1;
+ } else if (number < 0) {
+ return static_cast<int>(log10 ( abs(number) )) + 2;
+ } else {
+ return 1;
+ }
+}
+
void PrintToScreen::PrintRelation(const CatalogRelation &relation,
StorageManager *storage_manager,
FILE *out) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/205aa4fe/cli/PrintToScreen.hpp
----------------------------------------------------------------------
diff --git a/cli/PrintToScreen.hpp b/cli/PrintToScreen.hpp
index 0b57b4b..6a29426 100644
--- a/cli/PrintToScreen.hpp
+++ b/cli/PrintToScreen.hpp
@@ -69,6 +69,14 @@ class PrintToScreen {
StorageManager *storage_manager,
FILE *out);
+ /**
+ * @brief Return the number of digits in a number
+ *
+ * @param number The input number.
+ * @param out The number of digits in the input number.
+ **/
+ static int GetNumberOfDigits(int number);
+
private:
// Undefined default constructor. Class is all-static and should not be
// instantiated.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/205aa4fe/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 5881b3e..b7b28ba 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -365,7 +365,9 @@ int main(int argc, char* argv[]) {
try {
quickstep::cli::executeCommand(
*result.parsed_statement,
- *(query_processor->getDefaultDatabase()), stdout);
+ *(query_processor->getDefaultDatabase()),
+ query_processor->getStorageManager(),
+ stdout);
} catch (const quickstep::SqlError &sql_error) {
fprintf(stderr, "%s",
sql_error.formatMessage(*command_string).c_str());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/205aa4fe/cli/tests/CommandExecutorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/cli/tests/CommandExecutorTestRunner.cpp b/cli/tests/CommandExecutorTestRunner.cpp
index 49930e1..73d2092 100644
--- a/cli/tests/CommandExecutorTestRunner.cpp
+++ b/cli/tests/CommandExecutorTestRunner.cpp
@@ -87,6 +87,7 @@ void CommandExecutorTestRunner::runTestCase(
quickstep::cli::executeCommand(
*result.parsed_statement,
*(test_database_loader_.catalog_database()),
+ test_database_loader_.storage_manager(),
output_stream.file());
} else {
QueryHandle query_handle(optimizer_context.query_id());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/205aa4fe/cli/tests/command_executor/D.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/D.test b/cli/tests/command_executor/D.test
index 1d500df..45f8d8b 100644
--- a/cli/tests/command_executor/D.test
+++ b/cli/tests/command_executor/D.test
@@ -34,9 +34,21 @@ CREATE TABLE foo4 (col1 INT,
col3 DOUBLE,
col4 FLOAT,
col5 CHAR(5));
-CREATE INDEX foo4_index_1 ON foo4 (col1, col2) USING CSBTREE;
-CREATE INDEX foo4_index_2 ON foo4 (col3, col4) USING CSBTREE;
+CREATE INDEX foo4_index_1 ON foo4 (col1, col2) USING CSBTREE;
+CREATE INDEX foo4_index_2 ON foo4 (col3, col4) USING CSBTREE;
CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
+DROP TABLE TEST;
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (1);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (2);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (3);
+INSERT INTO foo values(1, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(2, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(3, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(4, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo2 values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo2 values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
--
==
\d foo
@@ -50,7 +62,7 @@ CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
col4 | Float
col5 | Char(5)
==
-\d foo2
+\d foo2
--
Table "foo2"
Column | Type
@@ -73,7 +85,7 @@ CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
col5 | Char(5)
Indexes
"foo3_index_1" CSB_TREE (col1)
-==
+==
\d foo4
--
Table "foo4"
@@ -92,14 +104,13 @@ CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
--
List of relations
- Name | Type
-+--------------------------------------+-------+
- Test | table
- foo | table
- foo2 | table
- foo3 | table
- foo4 | table
- averylongtablenamethatseemstoneverend | table
+ Name | Type | Blocks | Rows
++--------------------------------------+-------+---------+-------+
+ foo | table | 1 | 5
+ foo2 | table | 1 | 2
+ foo3 | table | 1 | 1
+ foo4 | table | 0 | 0
+ averylongtablenamethatseemstoneverend | table | 1 | 3
==
\d invalidtable
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/205aa4fe/cli/tests/command_executor/Dt.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/Dt.test b/cli/tests/command_executor/Dt.test
index 6458e15..1de6360 100644
--- a/cli/tests/command_executor/Dt.test
+++ b/cli/tests/command_executor/Dt.test
@@ -33,21 +33,41 @@ CREATE TABLE foo4 (col1 INT,
col3 DOUBLE,
col4 FLOAT,
col5 CHAR(5));
+DROP TABLE TEST;
CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (1);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (2);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (3);
+INSERT INTO foo values(1, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(2, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(3, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(4, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo2 values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo2 values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
--
==
\dt
--
List of relations
- Name | Type
-+--------------------------------------+-------+
- Test | table
- foo | table
- foo2 | table
- foo3 | table
- foo4 | table
- averylongtablenamethatseemstoneverend | table
+ Name | Type | Blocks | Rows
++--------------------------------------+-------+---------+-------+
+ foo | table | 1 | 5
+ foo2 | table | 1 | 2
+ foo3 | table | 1 | 1
+ foo4 | table | 0 | 0
+ averylongtablenamethatseemstoneverend | table | 1 | 3
+
+==
+\dt foo
+--
+ List of relations
+
+ Name | Type | Blocks | Rows
++-------+-------+---------+-------+
+ foo | table | 1 | 5
==
\dt invalidtable
[30/48] incubator-quickstep git commit: Added hash join order
optimization for star schema queries. (#229)
Posted by hb...@apache.org.
Added hash join order optimization for star schema queries. (#229)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/cd9db299
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/cd9db299
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/cd9db299
Branch: refs/heads/query-manager-used-in-foreman
Commit: cd9db299cbceb6cd850ac06a41cf53c310953046
Parents: c677259
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 19 18:19:28 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:44 2016 -0700
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 2 +
query_optimizer/PhysicalGenerator.cpp | 14 +
query_optimizer/cost_model/CMakeLists.txt | 32 +-
.../cost_model/StarSchemaSimpleCostModel.cpp | 258 ++++++++++++++++
.../cost_model/StarSchemaSimpleCostModel.hpp | 115 +++++++
query_optimizer/rules/CMakeLists.txt | 20 +-
.../StarSchemaHashJoinOrderOptimization.cpp | 309 +++++++++++++++++++
.../StarSchemaHashJoinOrderOptimization.hpp | 136 ++++++++
query_optimizer/tests/CMakeLists.txt | 1 +
query_optimizer/tests/OptimizerTextTest.cpp | 14 +
10 files changed, 899 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cd9db299/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index feaecb3..aa2873e 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -182,10 +182,12 @@ target_link_libraries(quickstep_queryoptimizer_OptimizerTree
quickstep_utility_Macros
quickstep_utility_TreeStringSerializable)
target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
+ gflags_nothreads-static
quickstep_queryoptimizer_LogicalToPhysicalMapper
quickstep_queryoptimizer_logical_Logical
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_rules_PruneColumns
+ quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
quickstep_queryoptimizer_strategy_Aggregate
quickstep_queryoptimizer_strategy_Join
quickstep_queryoptimizer_strategy_OneToOne
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cd9db299/query_optimizer/PhysicalGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/PhysicalGenerator.cpp b/query_optimizer/PhysicalGenerator.cpp
index ee52966..662236f 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -25,17 +27,26 @@
#include "query_optimizer/logical/Logical.hpp"
#include "query_optimizer/physical/Physical.hpp"
#include "query_optimizer/rules/PruneColumns.hpp"
+#include "query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp"
#include "query_optimizer/strategy/Aggregate.hpp"
#include "query_optimizer/strategy/Join.hpp"
#include "query_optimizer/strategy/OneToOne.hpp"
#include "query_optimizer/strategy/Selection.hpp"
#include "query_optimizer/strategy/Strategy.hpp"
+#include "gflags/gflags.h"
+
#include "glog/logging.h"
namespace quickstep {
namespace optimizer {
+DEFINE_bool(reorder_hash_joins, true,
+ "If true, apply hash join order optimization to each group of hash "
+ "joins. The optimization applies a greedy algorithm to favor smaller "
+ "cardinality and selective tables to be joined first, which is suitable "
+ "for queries on star-schema tables");
+
namespace L = ::quickstep::optimizer::logical;
namespace P = ::quickstep::optimizer::physical;
namespace S = ::quickstep::optimizer::strategy;
@@ -77,6 +88,9 @@ P::PhysicalPtr PhysicalGenerator::generateInitialPlan(
P::PhysicalPtr PhysicalGenerator::optimizePlan() {
std::vector<std::unique_ptr<Rule<P::Physical>>> rules;
+ if (FLAGS_reorder_hash_joins) {
+ rules.emplace_back(new StarSchemaHashJoinOrderOptimization());
+ }
rules.emplace_back(new PruneColumns());
for (std::unique_ptr<Rule<P::Physical>> &rule : rules) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cd9db299/query_optimizer/cost_model/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/CMakeLists.txt b/query_optimizer/cost_model/CMakeLists.txt
index 6697d52..6bf5240 100644
--- a/query_optimizer/cost_model/CMakeLists.txt
+++ b/query_optimizer/cost_model/CMakeLists.txt
@@ -1,5 +1,7 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -16,6 +18,9 @@
# Declare micro-libs:
add_library(quickstep_queryoptimizer_costmodel_CostModel ../../empty_src.cpp CostModel.hpp)
add_library(quickstep_queryoptimizer_costmodel_SimpleCostModel SimpleCostModel.cpp SimpleCostModel.hpp)
+add_library(quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ StarSchemaSimpleCostModel.cpp
+ StarSchemaSimpleCostModel.hpp)
# Link dependencies:
target_link_libraries(quickstep_queryoptimizer_costmodel_CostModel
@@ -36,9 +41,34 @@ target_link_libraries(quickstep_queryoptimizer_costmodel_SimpleCostModel
quickstep_queryoptimizer_physical_TableReference
quickstep_queryoptimizer_physical_TopLevelPlan
quickstep_utility_Macros)
+target_link_libraries(quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ glog
+ quickstep_catalog_CatalogRelation
+ quickstep_queryoptimizer_costmodel_CostModel
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_expressions_ComparisonExpression
+ quickstep_queryoptimizer_expressions_ExprId
+ quickstep_queryoptimizer_expressions_ExpressionType
+ quickstep_queryoptimizer_expressions_LogicalAnd
+ quickstep_queryoptimizer_expressions_LogicalOr
+ quickstep_queryoptimizer_expressions_PatternMatcher
+ quickstep_queryoptimizer_expressions_Predicate
+ quickstep_queryoptimizer_physical_Aggregate
+ quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_NestedLoopsJoin
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_queryoptimizer_physical_Selection
+ quickstep_queryoptimizer_physical_SharedSubplanReference
+ quickstep_queryoptimizer_physical_Sort
+ quickstep_queryoptimizer_physical_TableGenerator
+ quickstep_queryoptimizer_physical_TableReference
+ quickstep_queryoptimizer_physical_TopLevelPlan
+ quickstep_utility_Macros)
# Module all-in-one library:
add_library(quickstep_queryoptimizer_costmodel ../../empty_src.cpp CostModelModule.hpp)
target_link_libraries(quickstep_queryoptimizer_costmodel
quickstep_queryoptimizer_costmodel_CostModel
- quickstep_queryoptimizer_costmodel_SimpleCostModel)
+ quickstep_queryoptimizer_costmodel_SimpleCostModel
+ quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cd9db299/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
new file mode 100644
index 0000000..eb9fcc1
--- /dev/null
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -0,0 +1,258 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+
+#include <algorithm>
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "catalog/CatalogRelation.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/expressions/ComparisonExpression.hpp"
+#include "query_optimizer/expressions/ExprId.hpp"
+#include "query_optimizer/expressions/ExpressionType.hpp"
+#include "query_optimizer/expressions/LogicalAnd.hpp"
+#include "query_optimizer/expressions/LogicalOr.hpp"
+#include "query_optimizer/expressions/Predicate.hpp"
+#include "query_optimizer/expressions/PatternMatcher.hpp"
+#include "query_optimizer/physical/Aggregate.hpp"
+#include "query_optimizer/physical/NestedLoopsJoin.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/PhysicalType.hpp"
+#include "query_optimizer/physical/Selection.hpp"
+#include "query_optimizer/physical/SharedSubplanReference.hpp"
+#include "query_optimizer/physical/Sort.hpp"
+#include "query_optimizer/physical/TableGenerator.hpp"
+#include "query_optimizer/physical/TableReference.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+namespace cost {
+
+namespace E = ::quickstep::optimizer::expressions;
+namespace P = ::quickstep::optimizer::physical;
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinality(
+ const P::PhysicalPtr &physical_plan) {
+ switch (physical_plan->getPhysicalType()) {
+ case P::PhysicalType::kTopLevelPlan:
+ return estimateCardinalityForTopLevelPlan(
+ std::static_pointer_cast<const P::TopLevelPlan>(physical_plan));
+ case P::PhysicalType::kTableReference:
+ return estimateCardinalityForTableReference(
+ std::static_pointer_cast<const P::TableReference>(physical_plan));
+ case P::PhysicalType::kSelection:
+ return estimateCardinalityForSelection(
+ std::static_pointer_cast<const P::Selection>(physical_plan));
+ case P::PhysicalType::kTableGenerator:
+ return estimateCardinalityForTableGenerator(
+ std::static_pointer_cast<const P::TableGenerator>(physical_plan));
+ case P::PhysicalType::kHashJoin:
+ return estimateCardinalityForHashJoin(
+ std::static_pointer_cast<const P::HashJoin>(physical_plan));
+ case P::PhysicalType::kNestedLoopsJoin:
+ return estimateCardinalityForNestedLoopsJoin(
+ std::static_pointer_cast<const P::NestedLoopsJoin>(physical_plan));
+ case P::PhysicalType::kAggregate:
+ return estimateCardinalityForAggregate(
+ std::static_pointer_cast<const P::Aggregate>(physical_plan));
+ case P::PhysicalType::kSharedSubplanReference: {
+ const P::SharedSubplanReferencePtr shared_subplan_reference =
+ std::static_pointer_cast<const P::SharedSubplanReference>(physical_plan);
+ return estimateCardinality(
+ shared_subplans_[shared_subplan_reference->subplan_id()]);
+ }
+ case P::PhysicalType::kSort:
+ return estimateCardinality(
+ std::static_pointer_cast<const P::Sort>(physical_plan)->input());
+ default:
+ LOG(FATAL) << "Unsupported physical plan:" << physical_plan->toString();
+ }
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForTopLevelPlan(
+ const P::TopLevelPlanPtr &physical_plan) {
+ return estimateCardinality(physical_plan->plan());
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForTableReference(
+ const P::TableReferencePtr &physical_plan) {
+ std::size_t num_tuples = physical_plan->relation()->getStatistics().getNumTuples();
+ if (num_tuples == 0) {
+ num_tuples = physical_plan->relation()->estimateTupleCardinality();
+ }
+ return num_tuples;
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForSelection(
+ const P::SelectionPtr &physical_plan) {
+ double selectivity = estimateSelectivityForSelection(physical_plan);
+ return std::max(static_cast<std::size_t>(estimateCardinality(physical_plan->input()) * selectivity),
+ static_cast<std::size_t>(1));
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForTableGenerator(
+ const P::TableGeneratorPtr &physical_plan) {
+ return physical_plan->generator_function_handle()->getEstimatedCardinality();
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForHashJoin(
+ const P::HashJoinPtr &physical_plan) {
+ std::size_t left_cardinality = estimateCardinality(physical_plan->left());
+ std::size_t right_cardinality = estimateCardinality(physical_plan->right());
+ double left_selectivity = estimateSelectivity(physical_plan->left());
+ double right_selectivity = estimateSelectivity(physical_plan->right());
+ return std::max(static_cast<std::size_t>(left_cardinality * right_selectivity) + 1,
+ static_cast<std::size_t>(right_cardinality * left_selectivity) + 1);
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForNestedLoopsJoin(
+ const P::NestedLoopsJoinPtr &physical_plan) {
+ return std::max(estimateCardinality(physical_plan->left()),
+ estimateCardinality(physical_plan->right()));
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForAggregate(
+ const P::AggregatePtr &physical_plan) {
+ if (physical_plan->grouping_expressions().empty()) {
+ return 1;
+ }
+ return std::max(static_cast<std::size_t>(1),
+ estimateCardinality(physical_plan->input()) / 10);
+}
+
+double StarSchemaSimpleCostModel::estimateSelectivity(
+ const physical::PhysicalPtr &physical_plan) {
+ switch (physical_plan->getPhysicalType()) {
+ case P::PhysicalType::kSelection: {
+ return estimateSelectivityForSelection(
+ std::static_pointer_cast<const P::Selection>(physical_plan));
+ }
+ case P::PhysicalType::kHashJoin: {
+ const P::HashJoinPtr &hash_join =
+ std::static_pointer_cast<const P::HashJoin>(physical_plan);
+ return std::min(estimateSelectivity(hash_join->left()),
+ estimateSelectivity(hash_join->right()));
+ }
+ case P::PhysicalType::kNestedLoopsJoin: {
+ const P::NestedLoopsJoinPtr &nested_loop_join =
+ std::static_pointer_cast<const P::NestedLoopsJoin>(physical_plan);
+ return std::min(estimateSelectivity(nested_loop_join->left()),
+ estimateSelectivity(nested_loop_join->right()));
+ }
+ case P::PhysicalType::kSharedSubplanReference: {
+ const P::SharedSubplanReferencePtr shared_subplan_reference =
+ std::static_pointer_cast<const P::SharedSubplanReference>(physical_plan);
+ return estimateSelectivity(
+ shared_subplans_[shared_subplan_reference->subplan_id()]);
+ }
+ default:
+ return 1.0;
+ }
+}
+
+double StarSchemaSimpleCostModel::estimateSelectivityForSelection(
+ const physical::SelectionPtr &physical_plan) {
+ const E::PredicatePtr &filter_predicate = physical_plan->filter_predicate();
+
+ // If the subplan is a table reference, gather the number of distinct values
+ // statistics for each column (attribute).
+ std::unordered_map<E::ExprId, std::size_t> num_distinct_values_map;
+ if (physical_plan->input()->getPhysicalType() == P::PhysicalType::kTableReference) {
+ const P::TableReferencePtr &table_reference =
+ std::static_pointer_cast<const P::TableReference>(physical_plan->input());
+ const CatalogRelation &relation = *table_reference->relation();
+ const std::vector<E::AttributeReferencePtr> &attributes = table_reference->attribute_list();
+ for (std::size_t i = 0; i < attributes.size(); ++i) {
+ std::size_t num_distinct_values = relation.getStatistics().getNumDistinctValues(i);
+ if (num_distinct_values > 0) {
+ num_distinct_values_map[attributes[i]->id()] = num_distinct_values;
+ }
+ }
+ }
+
+ return estimateSelectivityForPredicate(num_distinct_values_map, filter_predicate);
+}
+
+double StarSchemaSimpleCostModel::estimateSelectivityForPredicate(
+ const std::unordered_map<expressions::ExprId, std::size_t> &num_distinct_values_map,
+ const expressions::PredicatePtr &filter_predicate) {
+ if (filter_predicate == nullptr) {
+ return 1.0;
+ }
+
+ switch (filter_predicate->getExpressionType()) {
+ case E::ExpressionType::kComparisonExpression: {
+ // Case 1 - Number of distinct values statistics available
+ // Case 1.1 - Equality comparison: 1.0 / num_distinct_values
+ // Case 1.2 - Otherwise: 5.0 / num_distinct_values
+ // Case 2 - Number of distinct values statistics not available
+ // Case 2.1 - Equality comparison: 0.1
+ // Case 2.2 - Otherwise: 0.5
+ const E::ComparisonExpressionPtr &comparison_expression =
+ std::static_pointer_cast<const E::ComparisonExpression>(filter_predicate);
+ E::AttributeReferencePtr attr;
+ if ((E::SomeAttributeReference::MatchesWithConditionalCast(comparison_expression->left(), &attr) &&
+ E::SomeScalarLiteral::Matches(comparison_expression->right())) ||
+ (E::SomeAttributeReference::MatchesWithConditionalCast(comparison_expression->right(), &attr) &&
+ E::SomeScalarLiteral::Matches(comparison_expression->left()))) {
+ const auto it = num_distinct_values_map.find(attr->id());
+ if (it != num_distinct_values_map.end() && it->second > 0) {
+ double unit_selectivity = 1.0 / it->second;
+ return comparison_expression->isEqualityComparisonPredicate()
+ ? unit_selectivity
+ : std::min(0.5, unit_selectivity * 5.0);
+ }
+ }
+
+ return comparison_expression->isEqualityComparisonPredicate() ? 0.1 : 0.5;
+ }
+ case E::ExpressionType::kLogicalAnd: {
+ const E::LogicalAndPtr &logical_and =
+ std::static_pointer_cast<const E::LogicalAnd>(filter_predicate);
+ double selectivity = 1.0;
+ for (const auto &predicate : logical_and->operands()) {
+ selectivity =
+ std::min(selectivity,
+ estimateSelectivityForPredicate(num_distinct_values_map, predicate));
+ }
+ return selectivity;
+ }
+ case E::ExpressionType::kLogicalOr: {
+ const E::LogicalOrPtr &logical_or =
+ std::static_pointer_cast<const E::LogicalOr>(filter_predicate);
+ double selectivity = 0;
+ for (const auto &predicate : logical_or->operands()) {
+ selectivity += estimateSelectivityForPredicate(num_distinct_values_map, predicate);
+ }
+ return std::min(selectivity, 1.0);
+ }
+ default:
+ break;
+ }
+ return 1.0;
+}
+
+} // namespace cost
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cd9db299/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp b/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp
new file mode 100644
index 0000000..c63e55a
--- /dev/null
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp
@@ -0,0 +1,115 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUERY_OPTIMIZER_COST_MODEL_STAR_SCHEMA_SIMPLE_COST_MODEL_HPP_
+#define QUERY_OPTIMIZER_COST_MODEL_STAR_SCHEMA_SIMPLE_COST_MODEL_HPP_
+
+#include <cstddef>
+#include <unordered_map>
+#include <vector>
+
+#include "query_optimizer/cost_model/CostModel.hpp"
+#include "query_optimizer/expressions/ExprId.hpp"
+#include "query_optimizer/expressions/Predicate.hpp"
+#include "query_optimizer/physical/Aggregate.hpp"
+#include "query_optimizer/physical/NestedLoopsJoin.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/Selection.hpp"
+#include "query_optimizer/physical/TableGenerator.hpp"
+#include "query_optimizer/physical/TableReference.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+namespace optimizer {
+namespace cost {
+
+/** \addtogroup CostModel
+ * @{
+ */
+
+/**
+ * @brief A simple cost model for hash join planning.
+ */
+class StarSchemaSimpleCostModel : public CostModel {
+ public:
+ /**
+ * @brief Constructor.
+ */
+ explicit StarSchemaSimpleCostModel(const std::vector<physical::PhysicalPtr> &shared_subplans)
+ : shared_subplans_(shared_subplans) {}
+
+ /**
+ * @brief Estimate the cardinality of a physical plan.
+ *
+ * @param physical_plan The physical plan.
+ * @return The estimated cardinality.
+ */
+ std::size_t estimateCardinality(
+ const physical::PhysicalPtr &physical_plan) override;
+
+ /**
+ * @brief Estimate the "selectivity" of a physical plan under the assumption
+ * that it acts as a filtered dimension table in a hash join.
+ *
+ * @param phyiscal_plan The physical plan.
+ * @return The estimated selectivity.
+ */
+ double estimateSelectivity(const physical::PhysicalPtr &physical_plan);
+
+ private:
+ std::size_t estimateCardinalityForTopLevelPlan(
+ const physical::TopLevelPlanPtr &physical_plan);
+
+ std::size_t estimateCardinalityForTableReference(
+ const physical::TableReferencePtr &physical_plan);
+
+ std::size_t estimateCardinalityForSelection(
+ const physical::SelectionPtr &physical_plan);
+
+ std::size_t estimateCardinalityForTableGenerator(
+ const physical::TableGeneratorPtr &physical_plan);
+
+ std::size_t estimateCardinalityForHashJoin(
+ const physical::HashJoinPtr &physical_plan);
+
+ std::size_t estimateCardinalityForNestedLoopsJoin(
+ const physical::NestedLoopsJoinPtr &physical_plan);
+
+ std::size_t estimateCardinalityForAggregate(
+ const physical::AggregatePtr &physical_plan);
+
+ double estimateSelectivityForSelection(
+ const physical::SelectionPtr &physical_plan);
+
+ double estimateSelectivityForPredicate(
+ const std::unordered_map<expressions::ExprId, std::size_t> &num_distinct_values_map,
+ const expressions::PredicatePtr &filter_predicate);
+
+ const std::vector<physical::PhysicalPtr> &shared_subplans_;
+
+ DISALLOW_COPY_AND_ASSIGN(StarSchemaSimpleCostModel);
+};
+
+/** @} */
+
+} // namespace cost
+} // namespace optimizer
+} // namespace quickstep
+
+#endif /* QUERY_OPTIMIZER_COST_MODEL_STAR_SCHEMA_SIMPLE_COST_MODEL_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cd9db299/query_optimizer/rules/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/CMakeLists.txt b/query_optimizer/rules/CMakeLists.txt
index 7032af5..1990174 100644
--- a/query_optimizer/rules/CMakeLists.txt
+++ b/query_optimizer/rules/CMakeLists.txt
@@ -26,11 +26,14 @@ add_library(quickstep_queryoptimizer_rules_PushDownFilter PushDownFilter.cpp Pus
add_library(quickstep_queryoptimizer_rules_PushDownSemiAntiJoin PushDownSemiAntiJoin.cpp PushDownSemiAntiJoin.hpp)
add_library(quickstep_queryoptimizer_rules_Rule ../../empty_src.cpp Rule.hpp)
add_library(quickstep_queryoptimizer_rules_RuleHelper RuleHelper.cpp RuleHelper.hpp)
+add_library(quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
+ StarSchemaHashJoinOrderOptimization.cpp
+ StarSchemaHashJoinOrderOptimization.hpp)
add_library(quickstep_queryoptimizer_rules_TopDownRule ../../empty_src.cpp TopDownRule.hpp)
add_library(quickstep_queryoptimizer_rules_UpdateExpression UpdateExpression.cpp UpdateExpression.hpp)
add_library(quickstep_queryoptimizer_rules_UnnestSubqueries UnnestSubqueries.cpp UnnestSubqueries.hpp)
-
+
# Link dependencies:
target_link_libraries(quickstep_queryoptimizer_rules_BottomUpRule
glog
@@ -110,6 +113,20 @@ target_link_libraries(quickstep_queryoptimizer_rules_RuleHelper
quickstep_queryoptimizer_expressions_PatternMatcher
quickstep_queryoptimizer_expressions_Predicate
quickstep_queryoptimizer_rules_UpdateExpression)
+target_link_libraries(quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
+ quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_expressions_ExprId
+ quickstep_queryoptimizer_expressions_NamedExpression
+ quickstep_queryoptimizer_expressions_PatternMatcher
+ quickstep_queryoptimizer_expressions_Predicate
+ quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_PatternMatcher
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_queryoptimizer_physical_TopLevelPlan
+ quickstep_queryoptimizer_rules_Rule
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_rules_TopDownRule
quickstep_queryoptimizer_rules_Rule
quickstep_utility_Macros)
@@ -167,6 +184,7 @@ target_link_libraries(quickstep_queryoptimizer_rules
quickstep_queryoptimizer_rules_PushDownSemiAntiJoin
quickstep_queryoptimizer_rules_Rule
quickstep_queryoptimizer_rules_RuleHelper
+ quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
quickstep_queryoptimizer_rules_TopDownRule
quickstep_queryoptimizer_rules_UpdateExpression
quickstep_queryoptimizer_rules_UnnestSubqueries)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cd9db299/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp
new file mode 100644
index 0000000..9770606
--- /dev/null
+++ b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp
@@ -0,0 +1,309 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp"
+
+#include <memory>
+#include <set>
+#include <unordered_map>
+#include <vector>
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/expressions/NamedExpression.hpp"
+#include "query_optimizer/expressions/PatternMatcher.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/PatternMatcher.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/PhysicalType.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+
+namespace E = ::quickstep::optimizer::expressions;
+namespace P = ::quickstep::optimizer::physical;
+
+P::PhysicalPtr StarSchemaHashJoinOrderOptimization::apply(const P::PhysicalPtr &input) {
+ DCHECK(input->getPhysicalType() == P::PhysicalType::kTopLevelPlan);
+ cost_model_.reset(
+ new cost::StarSchemaSimpleCostModel(
+ std::static_pointer_cast<const P::TopLevelPlan>(input)->shared_subplans()));
+
+ return applyInternal(input, nullptr);
+}
+
+P::PhysicalPtr StarSchemaHashJoinOrderOptimization::applyInternal(const P::PhysicalPtr &input,
+ JoinGroupInfo *parent_join_group) {
+ P::HashJoinPtr hash_join;
+ const bool is_hash_inner_join =
+ P::SomeHashJoin::MatchesWithConditionalCast(input, &hash_join)
+ && hash_join->join_type() == P::HashJoin::JoinType::kInnerJoin;
+
+ if (is_hash_inner_join) {
+ bool is_valid_cascading_hash_join = false;
+ if (hash_join->residual_predicate() == nullptr) {
+ is_valid_cascading_hash_join = true;
+ for (const E::NamedExpressionPtr expr : hash_join->project_expressions()) {
+ if (!E::SomeAttributeReference::Matches(expr)) {
+ is_valid_cascading_hash_join = false;
+ break;
+ }
+ }
+ }
+
+ std::unique_ptr<JoinGroupInfo> new_join_group;
+ JoinGroupInfo *join_group = nullptr;
+ if (parent_join_group == nullptr || !is_valid_cascading_hash_join) {
+ new_join_group.reset(new JoinGroupInfo());
+ join_group = new_join_group.get();
+ } else {
+ join_group = parent_join_group;
+ }
+
+ // Gather tables into the join group.
+ for (const P::PhysicalPtr &child : input->children()) {
+ applyInternal(child, join_group);
+ }
+
+ // Gather join attribute pairs.
+ for (std::size_t i = 0; i < hash_join->left_join_attributes().size(); ++i) {
+ const std::size_t left_attr_id = hash_join->left_join_attributes()[i]->id();
+ const std::size_t right_attr_id = hash_join->right_join_attributes()[i]->id();
+
+ join_group->join_attribute_pairs.emplace_back(left_attr_id, right_attr_id);
+ }
+
+ if (join_group != parent_join_group) {
+ // This node is the root node for a group of hash inner joins. Now plan the
+ // ordering of the joins.
+ P::PhysicalPtr output = generatePlan(*join_group,
+ hash_join->residual_predicate(),
+ hash_join->project_expressions());
+ if (parent_join_group == nullptr) {
+ return output;
+ } else {
+ parent_join_group->tables.emplace_back(output);
+ return nullptr;
+ }
+ } else {
+ return nullptr;
+ }
+ } else {
+ std::vector<P::PhysicalPtr> new_children;
+ bool has_changed_children = false;
+ for (const P::PhysicalPtr &child : input->children()) {
+ P::PhysicalPtr new_child = applyInternal(child, nullptr);
+ DCHECK(new_child != nullptr);
+ if (child != new_child && !has_changed_children) {
+ has_changed_children = true;
+ }
+ new_children.push_back(new_child);
+ }
+
+ P::PhysicalPtr output =
+ (has_changed_children ? input->copyWithNewChildren(new_children)
+ : input);
+
+ if (parent_join_group == nullptr) {
+ return output;
+ } else {
+ parent_join_group->tables.emplace_back(output);
+ return nullptr;
+ }
+ }
+}
+
+physical::PhysicalPtr StarSchemaHashJoinOrderOptimization::generatePlan(
+ const JoinGroupInfo &join_group,
+ const E::PredicatePtr &residual_predicate,
+ const std::vector<E::NamedExpressionPtr> &project_expressions) {
+ const std::size_t num_tables = join_group.tables.size();
+ DCHECK_GE(num_tables, 2u);
+
+ std::vector<TableInfo> table_info_storage;
+ const std::vector<P::PhysicalPtr> &tables = join_group.tables;
+ for (std::size_t i = 0; i < join_group.tables.size(); ++i) {
+ table_info_storage.emplace_back(
+ i,
+ tables[i],
+ cost_model_->estimateCardinality(tables[i]),
+ cost_model_->estimateSelectivity(tables[i]));
+ }
+
+ // Auxiliary mapping info.
+ std::unordered_map<E::ExprId, std::size_t> attribute_id_to_table_info_index_map;
+ std::unordered_map<E::ExprId, E::AttributeReferencePtr> attribute_id_to_reference_map;
+ for (std::size_t table_idx = 0; table_idx < num_tables; ++table_idx) {
+ for (const E::AttributeReferencePtr &attr :
+ table_info_storage[table_idx].table->getOutputAttributes()) {
+ DCHECK(attribute_id_to_table_info_index_map.find(attr->id())
+ == attribute_id_to_table_info_index_map.end());
+
+ attribute_id_to_table_info_index_map.emplace(attr->id(), table_idx);
+ attribute_id_to_reference_map.emplace(attr->id(), attr);
+ }
+ }
+
+ // Create a join graph where tables are vertices, and add an edge between vertices
+ // t1 and t2 for each join predicate t1.x = t2.y
+ std::vector<std::unordered_set<std::size_t>> join_graph(table_info_storage.size());
+ for (const auto &attr_id_pair : join_group.join_attribute_pairs) {
+ DCHECK(attribute_id_to_table_info_index_map.find(attr_id_pair.first)
+ != attribute_id_to_table_info_index_map.end());
+ DCHECK(attribute_id_to_table_info_index_map.find(attr_id_pair.second)
+ != attribute_id_to_table_info_index_map.end());
+
+ std::size_t first_table_idx =
+ attribute_id_to_table_info_index_map[attr_id_pair.first];
+ std::size_t second_table_idx =
+ attribute_id_to_table_info_index_map[attr_id_pair.second];
+ DCHECK_NE(first_table_idx, second_table_idx);
+
+ table_info_storage[first_table_idx].join_attribute_pairs.emplace(
+ attr_id_pair.first, attr_id_pair.second);
+ table_info_storage[second_table_idx].join_attribute_pairs.emplace(
+ attr_id_pair.second, attr_id_pair.first);
+
+ join_graph[first_table_idx].emplace(second_table_idx);
+ join_graph[second_table_idx].emplace(first_table_idx);
+ }
+
+ std::set<TableInfo*, TableInfoPtrLessComparator> table_info_ordered_by_priority;
+ for (std::size_t i = 0; i < table_info_storage.size(); ++i) {
+ table_info_ordered_by_priority.emplace(&table_info_storage[i]);
+ }
+
+ // Contruct hash join tree.
+ while (true) {
+ TableInfo *first_table_info = *table_info_ordered_by_priority.begin();
+ table_info_ordered_by_priority.erase(
+ table_info_ordered_by_priority.begin());
+ const std::size_t first_table_info_id = first_table_info->table_info_id;
+
+ TableInfo *second_table_info = nullptr;
+ std::set<TableInfo*, TableInfoPtrLessComparator>::iterator second_table_info_it;
+ for (auto candidate_table_info_it = table_info_ordered_by_priority.begin();
+ candidate_table_info_it != table_info_ordered_by_priority.end();
+ ++candidate_table_info_it) {
+ TableInfo *candidate_table_info = *candidate_table_info_it;
+ const std::size_t candidate_table_info_id = candidate_table_info->table_info_id;
+
+ if (join_graph[first_table_info_id].find(candidate_table_info_id)
+ == join_graph[first_table_info_id].end() &&
+ join_graph[candidate_table_info_id].find(first_table_info_id)
+ == join_graph[candidate_table_info_id].end()) {
+ continue;
+ } else if (second_table_info == nullptr) {
+ second_table_info = candidate_table_info;
+ second_table_info_it = candidate_table_info_it;
+ }
+
+ bool is_likely_many_to_many_join = false;
+ for (const auto join_attr_pair : first_table_info->join_attribute_pairs) {
+ if (candidate_table_info->joined_attribute_set.find(join_attr_pair.second)
+ != candidate_table_info->joined_attribute_set.end()) {
+ is_likely_many_to_many_join = true;
+ break;
+ }
+ }
+ for (const auto join_attr_pair : candidate_table_info->join_attribute_pairs) {
+ if (first_table_info->joined_attribute_set.find(join_attr_pair.second)
+ != first_table_info->joined_attribute_set.end()) {
+ is_likely_many_to_many_join = true;
+ break;
+ }
+ }
+ if (!is_likely_many_to_many_join) {
+ second_table_info = candidate_table_info;
+ second_table_info_it = candidate_table_info_it;
+ break;
+ }
+ }
+ DCHECK(second_table_info != nullptr);
+ table_info_ordered_by_priority.erase(second_table_info_it);
+
+ const P::PhysicalPtr &left_child = first_table_info->table;
+ const P::PhysicalPtr &right_child = second_table_info->table;
+ std::vector<E::NamedExpressionPtr> output_attributes;
+ for (const E::AttributeReferencePtr &left_attr : left_child->getOutputAttributes()) {
+ output_attributes.emplace_back(left_attr);
+ }
+ for (const E::AttributeReferencePtr &right_attr : right_child->getOutputAttributes()) {
+ output_attributes.emplace_back(right_attr);
+ }
+
+ std::vector<E::AttributeReferencePtr> left_join_attributes;
+ std::vector<E::AttributeReferencePtr> right_join_attributes;
+ std::unordered_set<expressions::ExprId> new_joined_attribute_set;
+ for (const auto &join_attr_pair : first_table_info->join_attribute_pairs) {
+ if (second_table_info->join_attribute_pairs.find(join_attr_pair.second)
+ != second_table_info->join_attribute_pairs.end()) {
+ left_join_attributes.emplace_back(
+ attribute_id_to_reference_map[join_attr_pair.first]);
+ right_join_attributes.emplace_back(
+ attribute_id_to_reference_map[join_attr_pair.second]);
+
+ new_joined_attribute_set.emplace(join_attr_pair.first);
+ new_joined_attribute_set.emplace(join_attr_pair.second);
+ }
+ }
+ DCHECK_GE(left_join_attributes.size(), static_cast<std::size_t>(1));
+
+ if (table_info_ordered_by_priority.size() > 0) {
+ P::PhysicalPtr output =
+ P::HashJoin::Create(left_child,
+ right_child,
+ left_join_attributes,
+ right_join_attributes,
+ nullptr,
+ output_attributes,
+ P::HashJoin::JoinType::kInnerJoin);
+
+ second_table_info->table = output;
+
+ // TODO(jianqiao): Cache the estimated cardinality for each plan in cost
+ // model to avoid duplicated estimation.
+ second_table_info->estimated_cardinality = cost_model_->estimateCardinality(output);
+
+ second_table_info->join_attribute_pairs.insert(first_table_info->join_attribute_pairs.begin(),
+ first_table_info->join_attribute_pairs.end());
+ second_table_info->joined_attribute_set.insert(first_table_info->joined_attribute_set.begin(),
+ first_table_info->joined_attribute_set.end());
+ second_table_info->joined_attribute_set.insert(new_joined_attribute_set.begin(),
+ new_joined_attribute_set.end());
+ table_info_ordered_by_priority.emplace(second_table_info);
+
+ join_graph[second_table_info->table_info_id].insert(join_graph[first_table_info_id].begin(),
+ join_graph[first_table_info_id].end());
+
+ } else {
+ return P::HashJoin::Create(left_child,
+ right_child,
+ left_join_attributes,
+ right_join_attributes,
+ residual_predicate,
+ project_expressions,
+ P::HashJoin::JoinType::kInnerJoin);
+ }
+ }
+}
+
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cd9db299/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp
new file mode 100644
index 0000000..deddffd
--- /dev/null
+++ b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp
@@ -0,0 +1,136 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_OPTIMIZER_RULES_STAR_SCHEMA_HASH_JOIN_ORDER_OPTIMIZATION_HPP_
+#define QUICKSTEP_QUERY_OPTIMIZER_RULES_STAR_SCHEMA_HASH_JOIN_ORDER_OPTIMIZATION_HPP_
+
+#include <algorithm>
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/expressions/ExprId.hpp"
+#include "query_optimizer/expressions/NamedExpression.hpp"
+#include "query_optimizer/expressions/Predicate.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/rules/Rule.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+namespace optimizer {
+
+/** \addtogroup OptimizerRules
+ * @{
+ */
+
+/**
+ * @brief TODO
+ */
+class StarSchemaHashJoinOrderOptimization : public Rule<physical::Physical> {
+ public:
+ StarSchemaHashJoinOrderOptimization() {}
+
+ ~StarSchemaHashJoinOrderOptimization() override {}
+
+ std::string getName() const override {
+ return "StarSchemaHashJoinOrderOptimization";
+ }
+
+ physical::PhysicalPtr apply(const physical::PhysicalPtr &input) override;
+
+ private:
+ /**
+ * @brief A group of tables to form a hash join tree.
+ */
+ struct JoinGroupInfo {
+ std::vector<physical::PhysicalPtr> tables;
+ std::vector<std::pair<expressions::ExprId, expressions::ExprId>> join_attribute_pairs;
+ };
+
+ /**
+ * @brief Auxiliary information of a table for the optimizer.
+ */
+ struct TableInfo {
+ TableInfo(const std::size_t in_table_info_id,
+ const physical::PhysicalPtr &in_table,
+ const std::size_t in_estimated_cardinality,
+ const double in_estimated_selectivity)
+ : table_info_id(in_table_info_id),
+ table(in_table),
+ estimated_cardinality(in_estimated_cardinality),
+ estimated_selectivity(in_estimated_selectivity) {
+ }
+
+ const std::size_t table_info_id;
+ physical::PhysicalPtr table;
+ std::size_t estimated_cardinality;
+ double estimated_selectivity;
+ std::unordered_multimap<expressions::ExprId, expressions::ExprId> join_attribute_pairs;
+ std::unordered_set<expressions::ExprId> joined_attribute_set;
+ };
+
+ /**
+ * @brief Comparator that compares the join priorities between two tables.
+ */
+ struct TableInfoPtrLessComparator {
+ inline bool operator() (const TableInfo *lhs, const TableInfo *rhs) {
+ bool swapped = false;
+ if (lhs->estimated_cardinality > rhs->estimated_cardinality) {
+ std::swap(lhs, rhs);
+ swapped = true;
+ }
+
+ if (lhs->estimated_selectivity < rhs->estimated_selectivity) {
+ return !swapped;
+ } else if (lhs->estimated_cardinality < 1000u &&
+ rhs->estimated_cardinality > 10000u &&
+ lhs->estimated_selectivity < rhs->estimated_selectivity * 1.5) {
+ return !swapped;
+ } else if (lhs->estimated_selectivity > rhs->estimated_selectivity) {
+ return swapped;
+ } else if (lhs->estimated_cardinality != rhs->estimated_cardinality) {
+ return !swapped;
+ } else {
+ return swapped ^ (lhs->table < rhs->table);
+ }
+ }
+ };
+
+ physical::PhysicalPtr applyInternal(const physical::PhysicalPtr &input,
+ JoinGroupInfo *paret_join_group);
+
+ physical::PhysicalPtr generatePlan(
+ const JoinGroupInfo &join_group_info,
+ const expressions::PredicatePtr &residual_predicate,
+ const std::vector<expressions::NamedExpressionPtr> &project_expressions);
+
+ std::unique_ptr<cost::StarSchemaSimpleCostModel> cost_model_;
+
+ DISALLOW_COPY_AND_ASSIGN(StarSchemaHashJoinOrderOptimization);
+};
+
+/** @} */
+
+} // namespace optimizer
+} // namespace quickstep
+
+#endif /* QUICKSTEP_QUERY_OPTIMIZER_RULES_STAR_SCHEMA_HASH_JOIN_ORDER_OPTIMIZATION_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cd9db299/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/CMakeLists.txt b/query_optimizer/tests/CMakeLists.txt
index 5647bfd..07af404 100644
--- a/query_optimizer/tests/CMakeLists.txt
+++ b/query_optimizer/tests/CMakeLists.txt
@@ -132,6 +132,7 @@ target_link_libraries(quickstep_queryoptimizer_tests_ExecutionGeneratorTest
tmb
${LIBS})
target_link_libraries(quickstep_queryoptimizer_tests_OptimizerTextTest
+ gflags_nothreads-static
glog
gtest
gtest_main
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cd9db299/query_optimizer/tests/OptimizerTextTest.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/OptimizerTextTest.cpp b/query_optimizer/tests/OptimizerTextTest.cpp
index c35cfa3..f3c243b 100644
--- a/query_optimizer/tests/OptimizerTextTest.cpp
+++ b/query_optimizer/tests/OptimizerTextTest.cpp
@@ -22,8 +22,18 @@
#include "query_optimizer/tests/OptimizerTextTestRunner.hpp"
#include "utility/textbased_test/TextBasedTestDriver.hpp"
+#include "gflags/gflags.h"
+
#include "glog/logging.h"
+namespace quickstep {
+namespace optimizer {
+
+DECLARE_bool(reorder_hash_joins);
+
+}
+}
+
using quickstep::TextBasedTest;
QUICKSTEP_GENERATE_TEXT_TEST(OPTIMIZER_TEST);
@@ -45,6 +55,10 @@ int main(int argc, char** argv) {
test_driver->registerOptions(
quickstep::optimizer::OptimizerTextTestRunner::kTestOptions);
+ // Turn off join order optimization for optimizer test since it is up to change
+ // and affects a large number of test cases.
+ quickstep::optimizer::FLAGS_reorder_hash_joins = false;
+
::testing::InitGoogleTest(&argc, argv);
int success = RUN_ALL_TESTS();
if (success != 0) {
[22/48] incubator-quickstep git commit: Fixed a g++ compiler warning
(#221)
Posted by hb...@apache.org.
Fixed a g++ compiler warning (#221)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/43f16262
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/43f16262
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/43f16262
Branch: refs/heads/query-manager-used-in-foreman
Commit: 43f16262f55126a0c2c27bf4785473dfc6e5bc46
Parents: 7131b0c
Author: Marc S <cr...@users.noreply.github.com>
Authored: Wed May 11 14:34:37 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:43 2016 -0700
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/43f16262/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 7f26e85..7209cfa 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -588,8 +588,8 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::vector<attribute_id> probe_original_attribute_ids;
std::vector<attribute_id> build_original_attribute_ids;
- const CatalogRelation *referenced_stored_probe_relation;
- const CatalogRelation *referenced_stored_build_relation;
+ const CatalogRelation *referenced_stored_probe_relation = nullptr;
+ const CatalogRelation *referenced_stored_build_relation = nullptr;
bool any_probe_attributes_nullable = false;
bool any_build_attributes_nullable = false;
[10/48] incubator-quickstep git commit: Added support for the
substring function. (#211)
Posted by hb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/parser/preprocessed/SqlParser_gen.hpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlParser_gen.hpp b/parser/preprocessed/SqlParser_gen.hpp
index 72fa9ef..71e4332 100644
--- a/parser/preprocessed/SqlParser_gen.hpp
+++ b/parser/preprocessed/SqlParser_gen.hpp
@@ -104,67 +104,69 @@ extern int quickstep_yydebug;
TOKEN_FALSE = 314,
TOKEN_FIRST = 315,
TOKEN_FLOAT = 316,
- TOKEN_FOREIGN = 317,
- TOKEN_FROM = 318,
- TOKEN_FULL = 319,
- TOKEN_GROUP = 320,
- TOKEN_HASH = 321,
- TOKEN_HAVING = 322,
- TOKEN_HOUR = 323,
- TOKEN_IN = 324,
- TOKEN_INDEX = 325,
- TOKEN_INNER = 326,
- TOKEN_INSERT = 327,
- TOKEN_INTEGER = 328,
- TOKEN_INTERVAL = 329,
- TOKEN_INTO = 330,
- TOKEN_JOIN = 331,
- TOKEN_KEY = 332,
- TOKEN_LAST = 333,
- TOKEN_LEFT = 334,
- TOKEN_LIMIT = 335,
- TOKEN_LONG = 336,
- TOKEN_MINUTE = 337,
- TOKEN_MONTH = 338,
- TOKEN_NULL = 339,
- TOKEN_NULLS = 340,
- TOKEN_OFF = 341,
- TOKEN_ON = 342,
- TOKEN_ORDER = 343,
- TOKEN_OUTER = 344,
- TOKEN_PARTITION = 345,
- TOKEN_PARTITIONS = 346,
- TOKEN_PERCENT = 347,
- TOKEN_PRIMARY = 348,
- TOKEN_QUIT = 349,
- TOKEN_RANGE = 350,
- TOKEN_REAL = 351,
- TOKEN_REFERENCES = 352,
- TOKEN_RIGHT = 353,
- TOKEN_ROW_DELIMITER = 354,
- TOKEN_SECOND = 355,
- TOKEN_SELECT = 356,
- TOKEN_SET = 357,
- TOKEN_SMA = 358,
- TOKEN_SMALLINT = 359,
- TOKEN_TABLE = 360,
- TOKEN_THEN = 361,
- TOKEN_TIME = 362,
- TOKEN_TIMESTAMP = 363,
- TOKEN_TRUE = 364,
- TOKEN_TUPLESAMPLE = 365,
- TOKEN_UNIQUE = 366,
- TOKEN_UPDATE = 367,
- TOKEN_USING = 368,
- TOKEN_VALUES = 369,
- TOKEN_VARCHAR = 370,
- TOKEN_WHEN = 371,
- TOKEN_WHERE = 372,
- TOKEN_WITH = 373,
- TOKEN_YEAR = 374,
- TOKEN_YEARMONTH = 375,
- TOKEN_EOF = 376,
- TOKEN_LEX_ERROR = 377
+ TOKEN_FOR = 317,
+ TOKEN_FOREIGN = 318,
+ TOKEN_FROM = 319,
+ TOKEN_FULL = 320,
+ TOKEN_GROUP = 321,
+ TOKEN_HASH = 322,
+ TOKEN_HAVING = 323,
+ TOKEN_HOUR = 324,
+ TOKEN_IN = 325,
+ TOKEN_INDEX = 326,
+ TOKEN_INNER = 327,
+ TOKEN_INSERT = 328,
+ TOKEN_INTEGER = 329,
+ TOKEN_INTERVAL = 330,
+ TOKEN_INTO = 331,
+ TOKEN_JOIN = 332,
+ TOKEN_KEY = 333,
+ TOKEN_LAST = 334,
+ TOKEN_LEFT = 335,
+ TOKEN_LIMIT = 336,
+ TOKEN_LONG = 337,
+ TOKEN_MINUTE = 338,
+ TOKEN_MONTH = 339,
+ TOKEN_NULL = 340,
+ TOKEN_NULLS = 341,
+ TOKEN_OFF = 342,
+ TOKEN_ON = 343,
+ TOKEN_ORDER = 344,
+ TOKEN_OUTER = 345,
+ TOKEN_PARTITION = 346,
+ TOKEN_PARTITIONS = 347,
+ TOKEN_PERCENT = 348,
+ TOKEN_PRIMARY = 349,
+ TOKEN_QUIT = 350,
+ TOKEN_RANGE = 351,
+ TOKEN_REAL = 352,
+ TOKEN_REFERENCES = 353,
+ TOKEN_RIGHT = 354,
+ TOKEN_ROW_DELIMITER = 355,
+ TOKEN_SECOND = 356,
+ TOKEN_SELECT = 357,
+ TOKEN_SET = 358,
+ TOKEN_SMA = 359,
+ TOKEN_SMALLINT = 360,
+ TOKEN_SUBSTRING = 361,
+ TOKEN_TABLE = 362,
+ TOKEN_THEN = 363,
+ TOKEN_TIME = 364,
+ TOKEN_TIMESTAMP = 365,
+ TOKEN_TRUE = 366,
+ TOKEN_TUPLESAMPLE = 367,
+ TOKEN_UNIQUE = 368,
+ TOKEN_UPDATE = 369,
+ TOKEN_USING = 370,
+ TOKEN_VALUES = 371,
+ TOKEN_VARCHAR = 372,
+ TOKEN_WHEN = 373,
+ TOKEN_WHERE = 374,
+ TOKEN_WITH = 375,
+ TOKEN_YEAR = 376,
+ TOKEN_YEARMONTH = 377,
+ TOKEN_EOF = 378,
+ TOKEN_LEX_ERROR = 379
};
#endif
@@ -265,7 +267,7 @@ union YYSTYPE
quickstep::PtrVector<quickstep::ParseSubqueryTableReference> *with_list_;
quickstep::ParseSubqueryTableReference *with_list_element_;
-#line 269 "SqlParser_gen.hpp" /* yacc.c:1915 */
+#line 271 "SqlParser_gen.hpp" /* yacc.c:1915 */
};
typedef union YYSTYPE YYSTYPE;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/query_optimizer/resolver/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/CMakeLists.txt b/query_optimizer/resolver/CMakeLists.txt
index db2a8af..5959879 100644
--- a/query_optimizer/resolver/CMakeLists.txt
+++ b/query_optimizer/resolver/CMakeLists.txt
@@ -120,6 +120,7 @@ target_link_libraries(quickstep_queryoptimizer_resolver_Resolver
quickstep_types_operations_comparisons_ComparisonFactory
quickstep_types_operations_comparisons_ComparisonID
quickstep_types_operations_unaryoperations_DateExtractOperation
+ quickstep_types_operations_unaryoperations_SubstringOperation
quickstep_types_operations_unaryoperations_UnaryOperation
quickstep_utility_Macros
quickstep_utility_PtrList
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index 2667ee9..78985a0 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -115,6 +115,7 @@
#include "types/operations/comparisons/ComparisonFactory.hpp"
#include "types/operations/comparisons/ComparisonID.hpp"
#include "types/operations/unary_operations/DateExtractOperation.hpp"
+#include "types/operations/unary_operations/SubstringOperation.hpp"
#include "types/operations/unary_operations/UnaryOperation.hpp"
#include "utility/PtrList.hpp"
#include "utility/PtrVector.hpp"
@@ -2068,6 +2069,37 @@ E::ScalarPtr Resolver::resolveExpression(
return E::UnaryExpression::Create(op, argument);
}
+ case ParseExpression::kSubstring: {
+ const ParseSubstringFunction &parse_substring =
+ static_cast<const ParseSubstringFunction&>(parse_expression);
+
+ // Validate start position and substring length.
+ if (parse_substring.start_position() <= 0) {
+ THROW_SQL_ERROR_AT(&parse_expression)
+ << "The start position must be greater than 0";
+ }
+ if (parse_substring.length() <= 0) {
+ THROW_SQL_ERROR_AT(&parse_expression)
+ << "The substring length must be greater than 0";
+ }
+
+ // Convert 1-base position to 0-base position
+ const std::size_t zero_base_start_position = parse_substring.start_position() - 1;
+ const SubstringOperation &op =
+ SubstringOperation::Instance(zero_base_start_position,
+ parse_substring.length());
+
+ const E::ScalarPtr argument =
+ resolveExpression(*parse_substring.operand(),
+ op.pushDownTypeHint(type_hint),
+ expression_resolution_info);
+ if (!op.canApplyToType(argument->getValueType())) {
+ THROW_SQL_ERROR_AT(&parse_substring)
+ << "Can not apply substring function to argument of type "
+ << argument->getValueType().getName();
+ }
+ return E::UnaryExpression::Create(op, argument);
+ }
default:
LOG(FATAL) << "Unknown scalar type: "
<< parse_expression.getExpressionType();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index 390b7b6..023ad2e 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -765,6 +765,30 @@ WHERE
+-----------+
==
+# SUBSTRING function
+SELECT char_col,
+ SUBSTRING(char_col FROM 1 FOR 2) AS negative_value
+FROM test
+WHERE SUBSTRING(char_col FROM 1 FOR 1) = '-'
+--
++--------------------+--------------+
+|char_col |negative_value|
++--------------------+--------------+
+| -1 1.000000| -1|
+| -3 1.732051| -3|
+| -5 2.236068| -5|
+| -7 2.645751| -7|
+| -9 3.000000| -9|
+| -11 3.316625| -1|
+| -13 3.605551| -1|
+| -15 3.872983| -1|
+| -17 4.123106| -1|
+| -19 4.358899| -1|
+| -21 4.582576| -2|
+| -23 4.795832| -2|
++--------------------+--------------+
+==
+
# IN predicate
SELECT *
FROM generate_series(1, 5) AS gs(i)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/query_optimizer/tests/resolver/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/resolver/Select.test b/query_optimizer/tests/resolver/Select.test
index 9897934..141bfa0 100644
--- a/query_optimizer/tests/resolver/Select.test
+++ b/query_optimizer/tests/resolver/Select.test
@@ -3048,3 +3048,81 @@ TopLevelPlan
+-output_attributes=
+-AttributeReference[id=5,name=x,relation=,type=Int]
+-AttributeReference[id=6,name=y,relation=,type=Int]
+==
+
+SELECT SUBSTRING(char_col FROM 1 FOR 2)
+FROM test;
+--
+TopLevelPlan
++-plan=Project
+| +-input=TableReference[relation_name=Test,relation_alias=test]
+| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | +-AttributeReference[id=5,name=vchar_col,relation=test,type=VarChar(20) NULL]
+| +-project_list=
+| +-Alias[id=6,name=,alias=SUBSTRING(char_col FROM 1 FOR 2),relation=,
+| type=Char(2)]
+| +-Substring
+| +-Operand=AttributeReference[id=4,name=char_col,relation=test,
+| type=Char(20)]
++-output_attributes=
+ +-AttributeReference[id=6,name=,alias=SUBSTRING(char_col FROM 1 FOR 2),
+ relation=,type=Char(2)]
+==
+
+SELECT *
+FROM test
+WHERE SUBSTRING(vchar_col FROM 1 FOR 2) IN ('12', '34', '56');
+--
+TopLevelPlan
++-plan=Project
+| +-input=Filter
+| | +-input=TableReference[relation_name=Test,relation_alias=test]
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | type=VarChar(20) NULL]
+| | +-filter_predicate=InValueList
+| | +-test_expression=Substring
+| | | +-Operand=AttributeReference[id=5,name=vchar_col,relation=test,
+| | | type=VarChar(20) NULL]
+| | +-match_expressions=
+| | +-Literal[value=12,type=Char(2)]
+| | +-Literal[value=34,type=Char(2)]
+| | +-Literal[value=56,type=Char(2)]
+| +-project_list=
+| +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| +-AttributeReference[id=5,name=vchar_col,relation=test,type=VarChar(20) NULL]
++-output_attributes=
+ +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+ +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+ +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+ +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+ +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+ +-AttributeReference[id=5,name=vchar_col,relation=test,type=VarChar(20) NULL]
+==
+
+SELECT SUBSTRING(char_col FROM 0 FOR 2)
+FROM test;
+--
+ERROR: The start position must be greater than 0 (1 : 8)
+SELECT SUBSTRING(char_col FROM 0 FOR ...
+ ^
+==
+
+SELECT SUBSTRING(char_col FROM 1 FOR 0)
+FROM test;
+--
+ERROR: The substring length must be greater than 0 (1 : 8)
+SELECT SUBSTRING(char_col FROM 1 FOR ...
+ ^
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/types/operations/Operation.proto
----------------------------------------------------------------------
diff --git a/types/operations/Operation.proto b/types/operations/Operation.proto
index fb44b1d..33e6b09 100644
--- a/types/operations/Operation.proto
+++ b/types/operations/Operation.proto
@@ -43,6 +43,7 @@ message UnaryOperation {
NEGATE = 0;
CAST = 1;
DATE_EXTRACT = 2;
+ SUBSTRING = 3;
}
required UnaryOperationID operation_id = 1;
@@ -73,6 +74,14 @@ message DateExtractOperation {
}
}
+message SubstringOperation {
+ extend UnaryOperation {
+ // Required when operation_id = SUBSTRING.
+ optional int64 start_position = 100;
+ optional int64 substring_length = 101;
+ }
+}
+
message BinaryOperation {
enum BinaryOperationID {
ADD = 0;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/types/operations/unary_operations/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/CMakeLists.txt b/types/operations/unary_operations/CMakeLists.txt
index 3a50445..5c54d9c 100644
--- a/types/operations/unary_operations/CMakeLists.txt
+++ b/types/operations/unary_operations/CMakeLists.txt
@@ -1,5 +1,7 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -18,6 +20,7 @@ add_library(quickstep_types_operations_unaryoperations_ArithmeticUnaryOperations
add_library(quickstep_types_operations_unaryoperations_ArithmeticUnaryOperators ../../../empty_src.cpp ArithmeticUnaryOperators.hpp)
add_library(quickstep_types_operations_unaryoperations_DateExtractOperation DateExtractOperation.cpp DateExtractOperation.hpp)
add_library(quickstep_types_operations_unaryoperations_NumericCastOperation ../../../empty_src.cpp NumericCastOperation.hpp)
+add_library(quickstep_types_operations_unaryoperations_SubstringOperation SubstringOperation.cpp SubstringOperation.hpp)
add_library(quickstep_types_operations_unaryoperations_UnaryOperation UnaryOperation.cpp UnaryOperation.hpp)
add_library(quickstep_types_operations_unaryoperations_UnaryOperationFactory UnaryOperationFactory.cpp UnaryOperationFactory.hpp)
add_library(quickstep_types_operations_unaryoperations_UnaryOperationID UnaryOperationID.cpp UnaryOperationID.hpp)
@@ -85,6 +88,23 @@ target_link_libraries(quickstep_types_operations_unaryoperations_NumericCastOper
quickstep_types_operations_unaryoperations_UnaryOperationID
quickstep_utility_Macros
quickstep_utility_PtrMap)
+target_link_libraries(quickstep_types_operations_unaryoperations_SubstringOperation
+ quickstep_catalog_CatalogTypedefs
+ quickstep_storage_ValueAccessor
+ quickstep_storage_ValueAccessorUtil
+ quickstep_types_Type
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ quickstep_types_TypedValue
+ quickstep_types_containers_ColumnVector
+ quickstep_types_containers_ColumnVectorUtil
+ quickstep_types_operations_Operation_proto
+ quickstep_types_operations_unaryoperations_UnaryOperation
+ quickstep_types_operations_unaryoperations_UnaryOperationID
+ quickstep_types_port_strnlen
+ quickstep_utility_HashPair
+ quickstep_utility_Macros
+ quickstep_utility_TemplateUtil)
target_link_libraries(quickstep_types_operations_unaryoperations_UnaryOperation
quickstep_catalog_CatalogTypedefs
quickstep_storage_StorageBlockInfo
@@ -100,6 +120,7 @@ target_link_libraries(quickstep_types_operations_unaryoperations_UnaryOperationF
quickstep_types_operations_unaryoperations_ArithmeticUnaryOperations
quickstep_types_operations_unaryoperations_DateExtractOperation
quickstep_types_operations_unaryoperations_NumericCastOperation
+ quickstep_types_operations_unaryoperations_SubstringOperation
quickstep_types_operations_unaryoperations_UnaryOperationID
quickstep_utility_Macros)
@@ -110,6 +131,7 @@ target_link_libraries(quickstep_types_operations_unaryoperations
quickstep_types_operations_unaryoperations_ArithmeticUnaryOperators
quickstep_types_operations_unaryoperations_DateExtractOperation
quickstep_types_operations_unaryoperations_NumericCastOperation
+ quickstep_types_operations_unaryoperations_SubstringOperation
quickstep_types_operations_unaryoperations_UnaryOperation
quickstep_types_operations_unaryoperations_UnaryOperationFactory
quickstep_types_operations_unaryoperations_UnaryOperationID)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/types/operations/unary_operations/SubstringOperation.cpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/SubstringOperation.cpp b/types/operations/unary_operations/SubstringOperation.cpp
new file mode 100644
index 0000000..463cd33
--- /dev/null
+++ b/types/operations/unary_operations/SubstringOperation.cpp
@@ -0,0 +1,214 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "types/operations/unary_operations/SubstringOperation.hpp"
+
+#include <algorithm>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "storage/ValueAccessor.hpp"
+#include "storage/ValueAccessorUtil.hpp"
+#include "types/Type.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "types/containers/ColumnVector.hpp"
+#include "types/containers/ColumnVectorUtil.hpp"
+#include "types/operations/Operation.pb.h"
+#include "types/port/strnlen.hpp"
+#include "utility/TemplateUtil.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+serialization::UnaryOperation SubstringOperation::getProto() const {
+ serialization::UnaryOperation proto;
+ proto.set_operation_id(serialization::UnaryOperation::SUBSTRING);
+ proto.SetExtension(serialization::SubstringOperation::start_position,
+ start_position_);
+ proto.SetExtension(serialization::SubstringOperation::substring_length,
+ substring_length_);
+ return proto;
+}
+
+UncheckedUnaryOperator* SubstringOperation::makeUncheckedUnaryOperatorForType(
+ const Type &type) const {
+ DCHECK(type.getSuperTypeID() == Type::kAsciiString);
+
+ const std::size_t input_maximum_length =
+ static_cast<const AsciiStringSuperType&>(type).getStringLength();
+ const bool input_null_terminated = (type.getTypeID() == TypeID::kVarChar);
+
+ const Type *result_type = resultTypeForArgumentType(type);
+ DCHECK(result_type != nullptr);
+
+ return CreateBoolInstantiatedInstance<SubstringUncheckedOperator, UncheckedUnaryOperator>(
+ std::forward_as_tuple(start_position_,
+ computeMaximumSubstringLength(type),
+ input_maximum_length,
+ *result_type),
+ input_null_terminated, type.isNullable());
+}
+
+template <bool null_terminated, bool input_nullable>
+inline void SubstringUncheckedOperator<null_terminated, input_nullable>
+ ::computeSubstring(const char *input,
+ char *output) const {
+ std::size_t string_length =
+ (null_terminated ? strlen(input) : strnlen(input, maximum_input_length_));
+
+ if (start_position_ >= string_length) {
+ *output = '\0';
+ return;
+ }
+
+ const std::size_t actual_substring_length =
+ std::min(string_length - start_position_, substring_length_);
+ std::memcpy(output, input + start_position_, actual_substring_length);
+
+ if (actual_substring_length < substring_length_) {
+ output[actual_substring_length] = '\0';
+ }
+}
+
+template <bool null_terminated, bool input_nullable>
+TypedValue SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToTypedValue(const TypedValue& argument) const {
+ if (input_nullable && argument.isNull()) {
+ return TypedValue(result_type_.getTypeID());
+ }
+
+ char *output_ptr = static_cast<char*>(std::malloc(substring_length_));
+ computeSubstring(static_cast<const char*>(argument.getOutOfLineData()),
+ output_ptr);
+
+ return TypedValue::CreateWithOwnedData(result_type_.getTypeID(),
+ output_ptr,
+ substring_length_);
+}
+
+template <bool null_terminated, bool input_nullable>
+TypedValue SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToDataPtr(const void *argument) const {
+ if (input_nullable && argument == nullptr) {
+ return TypedValue(result_type_.getTypeID());
+ }
+
+ char *output_ptr = static_cast<char*>(std::malloc(substring_length_));
+ computeSubstring(static_cast<const char*>(argument),
+ output_ptr);
+
+ return TypedValue::CreateWithOwnedData(result_type_.getTypeID(),
+ output_ptr,
+ substring_length_);
+}
+
+template <bool null_terminated, bool input_nullable>
+ColumnVector* SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToColumnVector(const ColumnVector &argument) const {
+ return InvokeOnColumnVector(
+ argument,
+ [&](const auto &column_vector) -> ColumnVector* { // NOLINT(build/c++11)
+ NativeColumnVector *result =
+ new NativeColumnVector(result_type_, column_vector.size());
+
+ for (std::size_t cv_pos = 0;
+ cv_pos < column_vector.size();
+ ++cv_pos) {
+ const char *input_ptr = static_cast<const char *>(
+ column_vector.template getUntypedValue<input_nullable>(cv_pos));
+
+ if (input_nullable && input_ptr == nullptr) {
+ result->appendNullValue();
+ } else {
+ this->computeSubstring(input_ptr,
+ static_cast<char *>(result->getPtrForDirectWrite()));
+ }
+ }
+ return result;
+ });
+}
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
+template <bool null_terminated, bool input_nullable>
+ColumnVector* SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToValueAccessor(ValueAccessor *accessor,
+ const attribute_id argument_attr_id) const {
+ return InvokeOnValueAccessorMaybeTupleIdSequenceAdapter(
+ accessor,
+ [&](auto *accessor) -> ColumnVector* { // NOLINT(build/c++11)
+ NativeColumnVector *result =
+ new NativeColumnVector(result_type_, accessor->getNumTuples());
+
+ accessor->beginIteration();
+ while (accessor->next()) {
+ const char *input_ptr = static_cast<const char *>(
+ accessor->template getUntypedValue<input_nullable>(argument_attr_id));
+
+ if (input_nullable && (input_ptr == nullptr)) {
+ result->appendNullValue();
+ } else {
+ this->computeSubstring(input_ptr,
+ static_cast<char *>(result->getPtrForDirectWrite()));
+ }
+ }
+ return result;
+ });
+}
+#endif
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_JOIN
+template <bool null_terminated, bool input_nullable>
+ColumnVector* SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToValueAccessorForJoin(
+ ValueAccessor *accessor,
+ const bool use_left_relation,
+ const attribute_id argument_attr_id,
+ const std::vector<std::pair<tuple_id, tuple_id>> &joined_tuple_ids) const {
+ return InvokeOnValueAccessorNotAdapter(
+ accessor,
+ [&](auto *accessor) -> ColumnVector* { // NOLINT(build/c++11)
+ NativeColumnVector *result =
+ new NativeColumnVector(result_type_, accessor->getNumTuples());
+
+ for (const std::pair<tuple_id, tuple_id> &joined_pair : joined_tuple_ids) {
+ const char *input_ptr = static_cast<const char *>(
+ accessor->template getUntypedValueAtAbsolutePosition<input_nullable>(
+ argument_attr_id,
+ use_left_relation ? joined_pair.first : joined_pair.second));
+
+ if (input_nullable && input_ptr == nullptr) {
+ result->appendNullValue();
+ } else {
+ this->computeSubstring(input_ptr,
+ static_cast<char *>(result->getPtrForDirectWrite()));
+ }
+ }
+ return result;
+ });
+}
+#endif
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/types/operations/unary_operations/SubstringOperation.hpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/SubstringOperation.hpp b/types/operations/unary_operations/SubstringOperation.hpp
new file mode 100644
index 0000000..d215eae
--- /dev/null
+++ b/types/operations/unary_operations/SubstringOperation.hpp
@@ -0,0 +1,234 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_TYPES_OPERATIONS_UNARY_OPERATIONS_SUBSTRING_OPERATION_HPP_
+#define QUICKSTEP_TYPES_OPERATIONS_UNARY_OPERATIONS_SUBSTRING_OPERATION_HPP_
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "types/Type.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "types/operations/Operation.pb.h"
+#include "types/operations/unary_operations/UnaryOperation.hpp"
+#include "types/operations/unary_operations/UnaryOperationID.hpp"
+#include "utility/HashPair.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class ColumnVector;
+class ValueAccessor;
+
+/**
+ * @brief Operation that extracts a number of characters from a string
+ * at a given starting position.
+ */
+class SubstringOperation : public UnaryOperation {
+ public:
+ /**
+ * @brief Get a reference to the singleton instance of this Operation for
+ * the given (start_position, substring_length) pair.
+ **/
+ static const SubstringOperation& Instance(const std::size_t start_position,
+ const std::size_t substring_length) {
+ // TODO(jianqiao): This is a temporary solution that creates a new instance
+ // for each distinct pair of start_position and substring_length arguments.
+ // The number of instances may be unbounded if quickstep continuously accepts
+ // queries that call SUBSTRING with different arguments. It still remains to
+ // design a better long-term solution.
+ const auto hash = [](const auto &pair) {
+ return hash_combine_detail::HashCombiner<std::size_t>::CombineHashes(pair.first, pair.second);
+ };
+ static std::unordered_map<std::pair<std::size_t, std::size_t>,
+ std::unique_ptr<const SubstringOperation>,
+ decltype(hash)> instance_map(10, hash);
+
+ const std::pair<std::size_t, std::size_t> key_pair =
+ std::make_pair(start_position, substring_length);
+ auto imit = instance_map.find(key_pair);
+ if (imit != instance_map.end()) {
+ return *imit->second;
+ } else {
+ const SubstringOperation *instance =
+ new SubstringOperation(start_position, substring_length);
+ instance_map.emplace(key_pair,
+ std::unique_ptr<const SubstringOperation>(instance));
+ return *instance;
+ }
+ }
+
+ serialization::UnaryOperation getProto() const override;
+
+ bool canApplyToType(const Type &type) const override {
+ return (type.getSuperTypeID() == Type::kAsciiString);
+ }
+
+ const Type *resultTypeForArgumentType(const Type &type) const override {
+ if (type.getSuperTypeID() == Type::kAsciiString) {
+ // Result is a Char string.
+ return &TypeFactory::GetType(TypeID::kChar,
+ computeMaximumSubstringLength(type),
+ type.isNullable());
+ }
+ return nullptr;
+ }
+
+ const Type* fixedNullableResultType() const override {
+ // Result type is not fixed (i.e. can have various lengths).
+ return nullptr;
+ }
+
+ bool resultTypeIsPlausible(const Type &result_type) const override {
+ // Result can be coerced to Char or VarChar.
+ return (result_type.getSuperTypeID() == Type::kAsciiString);
+ }
+
+ const Type* pushDownTypeHint(const Type *type_hint) const override {
+ // Input can only be a string, but we don't know the length.
+ return nullptr;
+ }
+
+ TypedValue applyToChecked(const TypedValue &argument,
+ const Type &argument_type) const override {
+ DCHECK(canApplyToType(argument_type));
+
+ const Type *result_type = resultTypeForArgumentType(argument_type);
+ DCHECK(result_type != nullptr);
+
+ if (argument_type.isNullable() && argument.isNull()) {
+ return result_type->makeNullValue();
+ } else {
+ const std::size_t result_length = computeMaximumSubstringLength(argument_type);
+ char *output_ptr = static_cast<char*>(std::malloc(result_length));
+ const char *input_ptr = static_cast<const char*>(argument.getOutOfLineData());
+
+ const std::size_t string_length = argument.getAsciiStringLength();
+ if (start_position_ >= string_length) {
+ *output_ptr = '\0';
+ } else {
+ const std::size_t actual_substring_length =
+ std::min(string_length - start_position_, substring_length_);
+ std::memcpy(output_ptr, input_ptr + start_position_, actual_substring_length);
+ if (actual_substring_length < result_length) {
+ output_ptr[actual_substring_length] = '\0';
+ }
+ }
+
+ return TypedValue::CreateWithOwnedData(result_type->getTypeID(),
+ output_ptr,
+ result_length);
+ }
+ }
+
+ UncheckedUnaryOperator* makeUncheckedUnaryOperatorForType(const Type &type) const override;
+
+ private:
+ /**
+ * @brief Constructor.
+ *
+ * @param input_type The data type of the input argument for substring.
+ * @param start_position The 0-base starting position of the substring.
+ * @param substring_length The substring length.
+ */
+ SubstringOperation(const std::size_t start_position,
+ const std::size_t substring_length)
+ : UnaryOperation(UnaryOperationID::kSubstring),
+ start_position_(start_position),
+ substring_length_(substring_length) {
+ }
+
+ /**
+ * @brief Compute an upper bound for the substring length regarding the input
+ * type and the substring_length_ field.
+ *
+ * @param type The type of the input, must be either CharType or VarCharType.
+ */
+ inline std::size_t computeMaximumSubstringLength(const Type& type) const {
+ DCHECK(type.getSuperTypeID() == Type::kAsciiString);
+
+ // Substring result should have length no greater than the minimum of
+ // (1) the input string length subtract the start position, and
+ // (2) the specified substring length.
+ return std::min(static_cast<const AsciiStringSuperType&>(type).getStringLength() - start_position_,
+ substring_length_);
+ }
+
+ const std::size_t start_position_;
+ const std::size_t substring_length_;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(SubstringOperation);
+};
+
+template <bool null_terminated, bool input_nullable>
+class SubstringUncheckedOperator : public UncheckedUnaryOperator {
+ public:
+ SubstringUncheckedOperator(const std::size_t start_position,
+ const std::size_t substring_length,
+ const std::size_t maximum_input_length,
+ const Type &result_type)
+ : start_position_(start_position),
+ substring_length_(substring_length),
+ maximum_input_length_(maximum_input_length),
+ result_type_(result_type) {
+ }
+
+ TypedValue applyToTypedValue(const TypedValue& argument) const override;
+
+ TypedValue applyToDataPtr(const void *argument) const override;
+
+ ColumnVector* applyToColumnVector(const ColumnVector &argument) const override;
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
+ ColumnVector* applyToValueAccessor(ValueAccessor *accessor,
+ const attribute_id argument_attr_id) const override;
+#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_JOIN
+ ColumnVector* applyToValueAccessorForJoin(
+ ValueAccessor *accessor,
+ const bool use_left_relation,
+ const attribute_id argument_attr_id,
+ const std::vector<std::pair<tuple_id, tuple_id>> &joined_tuple_ids) const override;
+#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_JOIN
+
+ private:
+ inline void computeSubstring(const char *input, char *output) const;
+
+ const std::size_t start_position_;
+ const std::size_t substring_length_;
+ const std::size_t maximum_input_length_;
+ const Type &result_type_;
+
+ DISALLOW_COPY_AND_ASSIGN(SubstringUncheckedOperator);
+};
+
+} // namespace quickstep
+
+#endif /* QUICKSTEP_TYPES_OPERATIONS_UNARY_OPERATIONS_SUBSTRING_OPERATION_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/types/operations/unary_operations/UnaryOperation.cpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/UnaryOperation.cpp b/types/operations/unary_operations/UnaryOperation.cpp
index 0f1f2c7..6721bd6 100644
--- a/types/operations/unary_operations/UnaryOperation.cpp
+++ b/types/operations/unary_operations/UnaryOperation.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -33,6 +35,8 @@ serialization::UnaryOperation UnaryOperation::getProto() const {
FATAL_ERROR("Must use the overridden NumericCastOperation::getProto");
case UnaryOperationID::kDateExtract:
FATAL_ERROR("Must use the overridden DateExtractOperation::getProto");
+ case UnaryOperationID::kSubstring:
+ FATAL_ERROR("Must use the overridden SubstringOperation::getProto");
default:
FATAL_ERROR("Unrecognized UnaryOperationID in UnaryOperation::getProto");
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/types/operations/unary_operations/UnaryOperationFactory.cpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/UnaryOperationFactory.cpp b/types/operations/unary_operations/UnaryOperationFactory.cpp
index 9d79cb0..13cb76e 100644
--- a/types/operations/unary_operations/UnaryOperationFactory.cpp
+++ b/types/operations/unary_operations/UnaryOperationFactory.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -24,6 +26,7 @@
#include "types/operations/unary_operations/ArithmeticUnaryOperations.hpp"
#include "types/operations/unary_operations/NumericCastOperation.hpp"
#include "types/operations/unary_operations/DateExtractOperation.hpp"
+#include "types/operations/unary_operations/SubstringOperation.hpp"
#include "types/operations/unary_operations/UnaryOperationID.hpp"
#include "utility/Macros.hpp"
@@ -39,6 +42,8 @@ const UnaryOperation& UnaryOperationFactory::GetUnaryOperation(const UnaryOperat
FATAL_ERROR("Getting a CastOperation through GetUnaryOperation is not supported");
case UnaryOperationID::kDateExtract:
FATAL_ERROR("Getting a DateExtractOperation through GetUnaryOperation is not supported");
+ case UnaryOperationID::kSubstring:
+ FATAL_ERROR("Getting a SubstringOperation through GetUnaryOperation is not supported");
default:
FATAL_ERROR("Unknown UnaryOperationID");
}
@@ -64,6 +69,9 @@ bool UnaryOperationFactory::ProtoIsValid(const serialization::UnaryOperation &pr
case serialization::UnaryOperation::DATE_EXTRACT:
return proto.HasExtension(serialization::DateExtractOperation::unit)
&& DateExtractOperation_Unit_IsValid(proto.GetExtension(serialization::DateExtractOperation::unit));
+ case serialization::UnaryOperation::SUBSTRING:
+ return proto.HasExtension(serialization::SubstringOperation::start_position)
+ && proto.HasExtension(serialization::SubstringOperation::substring_length);
default:
return false;
}
@@ -100,6 +108,10 @@ const UnaryOperation& UnaryOperationFactory::ReconstructFromProto(
default:
FATAL_ERROR("Unrecognized DateExtractOperation unit in UnaryOperation::ReconstructFromProto");
}
+ case serialization::UnaryOperation::SUBSTRING:
+ return SubstringOperation::Instance(
+ proto.GetExtension(serialization::SubstringOperation::start_position),
+ proto.GetExtension(serialization::SubstringOperation::substring_length));
default:
FATAL_ERROR("Unrecognized UnaryOperationID in UnaryOperation::ReconstructFromProto");
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/types/operations/unary_operations/UnaryOperationID.cpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/UnaryOperationID.cpp b/types/operations/unary_operations/UnaryOperationID.cpp
index c45a0a9..dae64d3 100644
--- a/types/operations/unary_operations/UnaryOperationID.cpp
+++ b/types/operations/unary_operations/UnaryOperationID.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,11 +22,11 @@
namespace quickstep {
const char *kUnaryOperationNames[] = {
- "Negate", "Cast", "DateExtract"
+ "Negate", "Cast", "DateExtract", "Substring"
};
const char *kUnaryOperationShortNames[] = {
- "-", "Cast", "DateExtract"
+ "-", "Cast", "DateExtract", "Substring"
};
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/types/operations/unary_operations/UnaryOperationID.hpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/UnaryOperationID.hpp b/types/operations/unary_operations/UnaryOperationID.hpp
index ba3e0ef..9cbb6e4 100644
--- a/types/operations/unary_operations/UnaryOperationID.hpp
+++ b/types/operations/unary_operations/UnaryOperationID.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -33,6 +35,7 @@ enum class UnaryOperationID {
kNegate = 0,
kCast,
kDateExtract,
+ kSubstring,
kNumUnaryOperationIDs // Not a real UnaryOperationID, exists for counting purposes.
};
[48/48] incubator-quickstep git commit: Merge branch 'master' into
query-manager-used-in-foreman
Posted by hb...@apache.org.
Merge branch 'master' into query-manager-used-in-foreman
Conflicts:
cli/CommandExecutor.cpp
cli/CommandExecutor.hpp
cli/QuickstepCli.cpp
cli/tests/CommandExecutorTestRunner.cpp
query_execution/CMakeLists.txt
query_execution/Foreman.cpp
query_execution/QueryContext.proto
query_execution/QueryExecutionTypedefs.hpp
query_optimizer/ExecutionGenerator.hpp
query_optimizer/tests/ExecutionHeuristics_unittest.cpp
relational_operators/RelationalOperator.hpp
relational_operators/SampleOperator.hpp
relational_operators/SaveBlocksOperator.hpp
relational_operators/tests/AggregationOperator_unittest.cpp
relational_operators/tests/HashJoinOperator_unittest.cpp
relational_operators/tests/SortMergeRunOperator_unittest.cpp
relational_operators/tests/SortRunGenerationOperator_unittest.cpp
relational_operators/tests/TextScanOperator_unittest.cpp
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d70ba5d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d70ba5d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d70ba5d1
Branch: refs/heads/query-manager-used-in-foreman
Commit: d70ba5d1df38297a22897c1135e589a696476749
Parents: e8ead86 1be47dc
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Wed Jun 8 20:08:00 2016 -0500
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Wed Jun 8 20:08:00 2016 -0500
----------------------------------------------------------------------
cli/CommandExecutor.cpp | 5 ---
query_execution/tests/Foreman_unittest.cpp | 3 +-
query_execution/tests/QueryManager_unittest.cpp | 3 +-
.../tests/ExecutionHeuristics_unittest.cpp | 9 +++--
relational_operators/HashJoinOperator.hpp | 1 +
relational_operators/RelationalOperator.hpp | 11 +++---
relational_operators/SampleOperator.cpp | 1 +
relational_operators/SampleOperator.hpp | 13 ++++++-
relational_operators/SaveBlocksOperator.hpp | 3 +-
.../tests/AggregationOperator_unittest.cpp | 5 +--
.../tests/HashJoinOperator_unittest.cpp | 37 ++++++++++----------
.../tests/SortMergeRunOperator_unittest.cpp | 5 +--
.../SortRunGenerationOperator_unittest.cpp | 3 +-
.../tests/TextScanOperator_unittest.cpp | 3 +-
14 files changed, 62 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d70ba5d1/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --cc cli/CommandExecutor.cpp
index cfb8c5b,ddcd38f..dc14741
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@@ -217,13 -210,11 +217,8 @@@ inline TypedValue executeQueryForSingle
DCHECK(query_handle->getQueryPlanMutable() != nullptr);
// Use foreman to execute the query plan.
- /*foreman->setQueryPlan(query_handle->getQueryPlanMutable()->getQueryPlanDAGMutable());
- foreman->setQueryPlan(query_handle->getQueryPlanMutable()->getQueryPlanDAGMutable());
-- foreman->reconstructQueryContextFromProto(query_handle->getQueryContextProto());
--
-- foreman->start();
- foreman->join();*/
- foreman->join();
+ QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
+ main_thread_client_id, foreman_client_id, query_handle.get(), bus);
// Retrieve the scalar result from the result relation.
const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d70ba5d1/query_execution/tests/QueryManager_unittest.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d70ba5d1/relational_operators/HashJoinOperator.hpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d70ba5d1/relational_operators/tests/AggregationOperator_unittest.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d70ba5d1/relational_operators/tests/HashJoinOperator_unittest.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d70ba5d1/relational_operators/tests/SortMergeRunOperator_unittest.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d70ba5d1/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d70ba5d1/relational_operators/tests/TextScanOperator_unittest.cpp
----------------------------------------------------------------------
[33/48] incubator-quickstep git commit: Resolves SMA bugs revealed by
TPCH. (#234)
Posted by hb...@apache.org.
Resolves SMA bugs revealed by TPCH. (#234)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/882cd386
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/882cd386
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/882cd386
Branch: refs/heads/query-manager-used-in-foreman
Commit: 882cd386ab184c69508a4d4ff71a64cca0053969
Parents: 0115e96
Author: Marc S <cr...@users.noreply.github.com>
Authored: Mon May 23 11:11:44 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:45 2016 -0700
----------------------------------------------------------------------
storage/CMakeLists.txt | 1 +
storage/SMAIndexSubBlock.cpp | 55 +++++++++++++++++++-----
storage/SMAIndexSubBlock.hpp | 3 +-
storage/tests/SMAIndexSubBlock_unittest.cpp | 52 ++++++++++++++++++++++
4 files changed, 99 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/882cd386/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 26a3e32..a3093df 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -1451,6 +1451,7 @@ target_link_libraries(SMAIndexSubBlock_unittest
quickstep_types_CharType
quickstep_types_DoubleType
quickstep_types_FloatType
+ quickstep_types_IntType
quickstep_types_LongType
quickstep_types_TypeFactory
quickstep_types_TypeID
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/882cd386/storage/SMAIndexSubBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/SMAIndexSubBlock.cpp b/storage/SMAIndexSubBlock.cpp
index aa9bc54..3a1ccc2 100644
--- a/storage/SMAIndexSubBlock.cpp
+++ b/storage/SMAIndexSubBlock.cpp
@@ -181,19 +181,22 @@ Selectivity getSelectivity(const TypedValue &literal,
}
SMAPredicate* SMAPredicate::ExtractSMAPredicate(const ComparisonPredicate &predicate) {
- if (predicate.getLeftOperand().hasStaticValue()) {
- DLOG_IF(FATAL, predicate.getRightOperand().getDataSource() != Scalar::kAttribute);
+ // Check that the predicate is contains an attribute and a literal value.
+ if (predicate.getLeftOperand().hasStaticValue() &&
+ predicate.getRightOperand().getDataSource() == Scalar::kAttribute) {
return new SMAPredicate(
static_cast<const ScalarAttribute&>(predicate.getRightOperand()).getAttribute().getID(),
flipComparisonID(predicate.getComparison().getComparisonID()),
predicate.getLeftOperand().getStaticValue().makeReferenceToThis());
- } else {
- DLOG_IF(FATAL, predicate.getLeftOperand().getDataSource() != Scalar::kAttribute);
+ } else if (predicate.getRightOperand().hasStaticValue() &&
+ predicate.getLeftOperand().getDataSource() == Scalar::kAttribute) {
return new SMAPredicate(
static_cast<const ScalarAttribute&>(predicate.getLeftOperand()).getAttribute().getID(),
predicate.getComparison().getComparisonID(),
predicate.getRightOperand().getStaticValue().makeReferenceToThis());
}
+ // Predicate is improper form, so return nullptr.
+ return nullptr;
}
/**
@@ -603,12 +606,47 @@ Selectivity SMAIndexSubBlock::getSelectivityForPredicate(const ComparisonPredica
}
std::unique_ptr<SMAPredicate> sma_predicate(SMAPredicate::ExtractSMAPredicate(predicate));
+ // The predicate did not contain a static value to compare against.
+ if (!sma_predicate) {
+ return Selectivity::kUnknown;
+ }
+
const SMAEntry *entry = getEntryChecked(sma_predicate->attribute);
// The attribute wasn't indexed.
if (entry == nullptr || !entry->min_entry_ref.valid || !entry->max_entry_ref.valid) {
return Selectivity::kUnknown;
}
+
+ // Check that the type of the comparison is the same or can be coerced the
+ // type of the underlying attribute.
+ if (sma_predicate->literal.getTypeID() != entry->type_id) {
+ // Try to coerce the literal type to the entry type.
+ // It may have to account for variable length attributes.
+ int literal_length = -1;
+ if (TypeFactory::TypeRequiresLengthParameter(sma_predicate->literal.getTypeID())) {
+ literal_length = sma_predicate->literal.getAsciiStringLength();
+ }
+
+ const Type &literal_type = literal_length == -1 ?
+ TypeFactory::GetType(sma_predicate->literal.getTypeID(), false) :
+ TypeFactory::GetType(sma_predicate->literal.getTypeID(), static_cast<std::size_t>(literal_length), false);
+ const Type &attribute_type = literal_length == -1 ?
+ TypeFactory::GetType(entry->type_id, false) :
+ TypeFactory::GetType(entry->type_id, static_cast<std::size_t>(literal_length), false);
+ if (attribute_type.isSafelyCoercibleFrom(literal_type)) {
+ // Convert the literal's type inside the predicate.
+ SMAPredicate *replacement = new SMAPredicate(
+ sma_predicate->attribute,
+ sma_predicate->comparison,
+ attribute_type.coerceValue(sma_predicate->literal, literal_type));
+ sma_predicate.reset(replacement);
+ } else {
+ // The literal type cannot be converted, so do not evaluate with the SMA.
+ return Selectivity::kUnknown;
+ }
+ }
+
return sma_internal::getSelectivity(
sma_predicate->literal,
sma_predicate->comparison,
@@ -623,12 +661,9 @@ predicate_cost_t SMAIndexSubBlock::estimatePredicateEvaluationCost(
DCHECK(initialized_);
// Check that at least one of the operands has a static value.
- if (predicate.getLeftOperand().hasStaticValue() ||
- predicate.getRightOperand().hasStaticValue()) {
- Selectivity selectivity = getSelectivityForPredicate(predicate);
- if (selectivity == Selectivity::kAll || selectivity == Selectivity::kNone) {
- return predicate_cost::kConstantTime;
- }
+ Selectivity selectivity = getSelectivityForPredicate(predicate);
+ if (selectivity == Selectivity::kAll || selectivity == Selectivity::kNone) {
+ return predicate_cost::kConstantTime;
}
return predicate_cost::kInfinite;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/882cd386/storage/SMAIndexSubBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/SMAIndexSubBlock.hpp b/storage/SMAIndexSubBlock.hpp
index 71ca75b..895add9 100644
--- a/storage/SMAIndexSubBlock.hpp
+++ b/storage/SMAIndexSubBlock.hpp
@@ -120,7 +120,6 @@ struct SMAPredicate {
// that the predicate had not been used previously.
Selectivity selectivity;
- private:
SMAPredicate(const attribute_id attr,
const ComparisonID comp,
const TypedValue lit)
@@ -376,7 +375,7 @@ class SMAIndexSubBlock : public IndexSubBlock {
private:
bool requiresRebuild() const;
- inline sma_internal::Selectivity getSelectivityForPredicate(const ComparisonPredicate &predicate) const;
+ sma_internal::Selectivity getSelectivityForPredicate(const ComparisonPredicate &predicate) const;
// Retrieves an entry, first checking if the given attribute is indexed.
inline const sma_internal::SMAEntry* getEntryChecked(attribute_id attribute) const {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/882cd386/storage/tests/SMAIndexSubBlock_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/SMAIndexSubBlock_unittest.cpp b/storage/tests/SMAIndexSubBlock_unittest.cpp
index b709712..ad842c2 100644
--- a/storage/tests/SMAIndexSubBlock_unittest.cpp
+++ b/storage/tests/SMAIndexSubBlock_unittest.cpp
@@ -41,6 +41,7 @@
#include "types/CharType.hpp"
#include "types/DoubleType.hpp"
#include "types/FloatType.hpp"
+#include "types/IntType.hpp"
#include "types/LongType.hpp"
#include "types/TypeFactory.hpp"
#include "types/TypeID.hpp"
@@ -301,6 +302,10 @@ class SMAIndexSubBlockTest : public ::testing::Test {
return !index_->header_->index_consistent;
}
+ Selectivity getSelectivityForPredicate(const ComparisonPredicate &predicate) const {
+ return index_->getSelectivityForPredicate(predicate);
+ }
+
std::unique_ptr<CatalogRelation> relation_;
std::unique_ptr<MockTupleStorageSubBlock> tuple_store_;
ScopedBuffer index_memory_;
@@ -359,6 +364,53 @@ TEST_F(SMAIndexSubBlockTest, TestConstructor) {
EXPECT_FALSE(requiresRebuild());
}
+TEST_F(SMAIndexSubBlockTest, TestGetSelectivityForPredicate) {
+ // Index long, float, bigchar, and varchar type.
+ createIndex({0, 2, 5, 6}, kIndexSubBlockSize);
+ int min = 0, max = 9010, step = 10;
+ for (std::size_t i = min; i <= static_cast<std::size_t>(max); i += step) {
+ generateAndInsertTuple(i, false, "suffix");
+ }
+ index_->rebuild();
+
+ // Tuple storage block now contains long values [0, 9010], and float values
+ // [0, 2252.5].
+ // Test several value types.
+ // Test with inline values, returning both possible value types.
+ std::unique_ptr<ComparisonPredicate> predicate(
+ generateNumericComparisonPredicate<LongType>(ComparisonID::kLess, 0, min));
+ Selectivity cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kNone, cost);
+
+ // Test on the float attribute with a type that will need coercion. In this
+ // case, a long type should not coerce to an integer type.
+ predicate.reset(generateNumericComparisonPredicate<LongType>(ComparisonID::kLess, 2, min));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kUnknown, cost);
+
+ predicate.reset(generateNumericComparisonPredicate<IntType>(ComparisonID::kLess, 2, min));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kNone, cost);
+
+ // This should be unknown because of the loss of precision on coercion.
+ predicate.reset(generateNumericComparisonPredicate<FloatType>(ComparisonID::kLess, 0, min + 0.5));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kUnknown, cost);
+
+ // Test coercion with String types as well.
+ predicate.reset(generateStringComparisonPredicate(ComparisonID::kLess, 6, ""));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kNone, cost);
+
+ predicate.reset(generateStringComparisonPredicate(ComparisonID::kLess, 5, ""));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kNone, cost);
+
+ predicate.reset(generateStringComparisonPredicate(ComparisonID::kLess, 5, "999999999999999"));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kAll, cost);
+}
+
TEST_F(SMAIndexSubBlockTest, TestRebuild) {
// This test is checking to see if the min/max values are set correctly on
// a rebuild of the index.
[37/48] incubator-quickstep git commit: Fixed Travis CI link in
Readme.
Posted by hb...@apache.org.
Fixed Travis CI link in Readme.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/45031985
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/45031985
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/45031985
Branch: refs/heads/query-manager-used-in-foreman
Commit: 45031985cdf32086c9f945cfc29dc4ce935c0209
Parents: 037ef4b
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Mon May 16 13:39:12 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:46 2016 -0700
----------------------------------------------------------------------
README.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/45031985/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 7cea80c..04d5d66 100644
--- a/README.md
+++ b/README.md
@@ -2,8 +2,8 @@
[![Travis Widget]][Travis]
-[Travis]: https://travis-ci.org/pivotalsoftware/quickstep
-[Travis Widget]: https://travis-ci.org/pivotalsoftware/quickstep.svg?branch=master
+[Travis]: https://travis-ci.org/apache/incubator-quickstep
+[Travis Widget]: https://travis-ci.org/apache/incubator-quickstep.svg?branch=master
Apache Quickstep is an experimental high-performance database engine designed with the
aim of Data at Bare-Metal Speed. It began life in 2011 as a
[06/48] incubator-quickstep git commit: Support for NUMA aware
preloading (#206)
Posted by hb...@apache.org.
Support for NUMA aware preloading (#206)
* Support for preloading of NUMA-partitioned relations.
- Stored relations which have been NUMA-partitioned, can now be
preloaded. The preloading can maintain the same NUMA placement of
storage blocks.
* Added missing glog library in CMakeLists
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3b33fa27
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3b33fa27
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3b33fa27
Branch: refs/heads/query-manager-used-in-foreman
Commit: 3b33fa2758a072eafd76e5f8c83e155ee560f947
Parents: 7bb83d2
Author: Harshad Deshmukh <d....@gmail.com>
Authored: Wed May 4 10:32:47 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:40 2016 -0700
----------------------------------------------------------------------
storage/CMakeLists.txt | 5 +++
storage/PreloaderThread.cpp | 86 +++++++++++++++++++++++++++++++++++-----
storage/PreloaderThread.hpp | 23 +++++++++++
3 files changed, 105 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3b33fa27/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index ed23802..dacacfa 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -741,6 +741,7 @@ target_link_libraries(quickstep_storage_PackedRowStoreValueAccessor
quickstep_utility_BitVector
quickstep_utility_Macros)
target_link_libraries(quickstep_storage_PreloaderThread
+ glog
quickstep_catalog_CatalogDatabase
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
@@ -750,6 +751,10 @@ target_link_libraries(quickstep_storage_PreloaderThread
quickstep_threading_Thread
quickstep_threading_ThreadUtil
quickstep_utility_Macros)
+if (QUICKSTEP_HAVE_LIBNUMA)
+ target_link_libraries(quickstep_storage_PreloaderThread
+ quickstep_catalog_NUMAPlacementScheme)
+endif()
target_link_libraries(quickstep_storage_SMAIndexSubBlock
glog
quickstep_catalog_CatalogAttribute
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3b33fa27/storage/PreloaderThread.cpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.cpp b/storage/PreloaderThread.cpp
index d5dc55b..8f600b8 100644
--- a/storage/PreloaderThread.cpp
+++ b/storage/PreloaderThread.cpp
@@ -17,8 +17,13 @@
#include "storage/PreloaderThread.hpp"
+#include <cstddef>
#include <vector>
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include <unordered_map>
+#endif
+
#include "catalog/CatalogDatabase.hpp"
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
@@ -27,6 +32,12 @@
#include "storage/StorageManager.hpp"
#include "threading/ThreadUtil.hpp"
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include "catalog/NUMAPlacementScheme.hpp"
+#endif
+
+#include "glog/logging.h"
+
namespace quickstep {
void PreloaderThread::run() {
@@ -38,24 +49,81 @@ void PreloaderThread::run() {
std::size_t blocks_loaded = 0;
for (const CatalogRelation &relation : database_) {
- std::vector<block_id> blocks = relation.getBlocksSnapshot();
- for (block_id current_block_id : blocks) {
+ if (relation.hasPartitionScheme()) {
+ blocks_loaded += preloadNUMAAware(relation, blocks_loaded, num_slots);
+ } else {
+ std::vector<block_id> blocks = relation.getBlocksSnapshot();
+ for (block_id current_block_id : blocks) {
+ try {
+ BlockReference current_block = storage_manager_->getBlock(current_block_id, relation);
+ } catch (...) {
+ LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks\n";
+ throw;
+ }
+ ++blocks_loaded;
+ if (blocks_loaded == num_slots) {
+ // The buffer pool has filled up. But, some database blocks are not loaded.
+ printf(" The database is larger than the buffer pool. Only %lu blocks were loaded ",
+ blocks_loaded);
+ return;
+ }
+ }
+ }
+ }
+ printf(" Loaded %lu blocks ", blocks_loaded);
+}
+
+std::size_t PreloaderThread::preloadNUMAAware(
+ const CatalogRelation &relation,
+ const std::size_t num_previously_loaded_blocks,
+ const std::size_t num_slots) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ std::size_t blocks_loaded = 0;
+ const NUMAPlacementScheme *placement_scheme = relation.getNUMAPlacementSchemePtr();
+ DCHECK(placement_scheme != nullptr);
+ DCHECK(relation.hasPartitionScheme());
+ const PartitionScheme &part_scheme = relation.getPartitionScheme();
+ const PartitionSchemeHeader &part_scheme_header =
+ part_scheme.getPartitionSchemeHeader();
+ const std::size_t num_partitions = part_scheme_header.getNumPartitions();
+ // Key = NUMA node ID, value = number of blocks loaded from that NUMA node.
+ std::unordered_map<numa_node_id, std::size_t> num_blocks_loaded;
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ const numa_node_id partition_numa_node_id =
+ placement_scheme->getNUMANodeForPartition(part_id);
+ for (block_id curr_block_id : part_scheme.getBlocksInPartition(part_id)) {
try {
- BlockReference current_block = storage_manager_->getBlock(current_block_id, relation);
+ BlockReference current_block = storage_manager_->getBlock(
+ curr_block_id, relation, partition_numa_node_id);
} catch (...) {
- LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks\n";
+ LOG(ERROR) << "Error after loading "
+ << blocks_loaded + num_previously_loaded_blocks
+ << " blocks\n";
throw;
}
++blocks_loaded;
- if (blocks_loaded == num_slots) {
+ num_blocks_loaded[partition_numa_node_id]++;
+ if ((blocks_loaded + num_previously_loaded_blocks) == num_slots) {
// The buffer pool has filled up. But, some database blocks are not loaded.
- printf(" The database is larger than the buffer pool. Only %lu blocks were loaded ",
- blocks_loaded);
- return;
+ printf(
+ " The database is larger than the buffer pool. Only %lu blocks "
+ "were loaded ",
+ blocks_loaded + num_previously_loaded_blocks);
+ return blocks_loaded;
}
}
}
- printf(" Loaded %lu blocks ", blocks_loaded);
+ LOG(INFO) << "For relation: " << relation.getName();
+ for (auto numa_block_loaded_info : num_blocks_loaded) {
+ LOG(INFO) << "NUMA node: " << numa_block_loaded_info.first
+ << " Number of loaded blocks: " << numa_block_loaded_info.second;
+ }
+ return blocks_loaded;
+#else
+ LOG(INFO) << "Relation: " << relation.getName()
+ << " has partition scheme but the system doesn't support NUMA";
+ return 0;
+#endif
}
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3b33fa27/storage/PreloaderThread.hpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.hpp b/storage/PreloaderThread.hpp
index ed866b4..f16fd50 100644
--- a/storage/PreloaderThread.hpp
+++ b/storage/PreloaderThread.hpp
@@ -18,12 +18,15 @@
#ifndef QUICKSTEP_STORAGE_PRELOADER_THREAD_HPP_
#define QUICKSTEP_STORAGE_PRELOADER_THREAD_HPP_
+#include <cstddef>
+
#include "threading/Thread.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
class CatalogDatabase;
+class CatalogRelation;
class StorageManager;
/** \addtogroup Storage
@@ -65,6 +68,26 @@ class PreloaderThread : public Thread {
void run() override;
private:
+ /**
+ * @brief Preload a relation which has a partition and a NUMA placement scheme.
+ *
+ * @param relation The relation to be preloaded.
+ * @param num_previously_loaded_blocks The number of blocks already preloaded.
+ * @param num_slots The maximum number of slots in the StorageManager.
+ *
+ * @warning This function may not detect skew on NUMA sockets, i.e. if a given
+ * NUMA socket has large number of blocks, preloading may cause the
+ * memory on that NUMA socket to be full. It is recommended to use
+ * this preloading when we are sure that each NUMA socket has been
+ * allocated sufficient amount of memory so as not to exceed that
+ * socket's memory limit.
+ *
+ * @return The number of blocks loaded during this function call.
+ **/
+ std::size_t preloadNUMAAware(const CatalogRelation &relation,
+ const std::size_t num_previously_loaded_blocks,
+ const std::size_t num_slots);
+
const CatalogDatabase &database_;
StorageManager *storage_manager_;
[42/48] incubator-quickstep git commit: Bug fixed in isFull()
function.
Posted by hb...@apache.org.
Bug fixed in isFull() function.
- Earlier check for isFull() was that whether there is at least one
empty bucket available for insertion.
- Now isFull() can check if there is enough space adding specified
number of buckets.
- To reproduce the bug behind this fix: Run TPC-H Q2 on SF100 dataset after
running the \analyze command. The optimizer allocates smaller than
necessary space for one of the hash joins. The resize() is triggered
but never gets executed because of the restrictive isFull() condition.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/83935e7d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/83935e7d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/83935e7d
Branch: refs/heads/query-manager-used-in-foreman
Commit: 83935e7d8c9ff95071906e4a6078d1ed096c7bae
Parents: 5ae5052
Author: Harshad Deshmukh <ha...@cs.wisc.edu>
Authored: Fri Jun 3 15:28:52 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:47 2016 -0700
----------------------------------------------------------------------
storage/SimpleScalarSeparateChainingHashTable.hpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/83935e7d/storage/SimpleScalarSeparateChainingHashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/SimpleScalarSeparateChainingHashTable.hpp b/storage/SimpleScalarSeparateChainingHashTable.hpp
index eda6c86..962a66c 100644
--- a/storage/SimpleScalarSeparateChainingHashTable.hpp
+++ b/storage/SimpleScalarSeparateChainingHashTable.hpp
@@ -230,10 +230,10 @@ class SimpleScalarSeparateChainingHashTable : public HashTable<ValueT,
HashTablePreallocationState *prealloc_state);
// Determine whether it is actually necessary to resize this hash table.
- // Checks that there is at least one unallocated bucket.
- inline bool isFull() const {
- return header_->buckets_allocated.load(std::memory_order_relaxed)
- >= header_->num_buckets;
+ // Checks that there are at least ``extra_buckets`` unallocated buckets.
+ inline bool isFull(const std::size_t extra_buckets) const {
+ return (header_->buckets_allocated.load(std::memory_order_relaxed) +
+ extra_buckets) >= header_->num_buckets;
}
// Cache the TypeID of the key.
@@ -831,7 +831,7 @@ void SimpleScalarSeparateChainingHashTable<ValueT,
// Recheck whether the hash table is still full. Note that multiple threads
// might wait to rebuild this hash table simultaneously. Only the first one
// should do the rebuild.
- if (!isFull()) {
+ if (!isFull(extra_buckets)) {
return;
}
[19/48] incubator-quickstep git commit: Refactor
hasNUMAPlacementScheme. (#215)
Posted by hb...@apache.org.
Refactor hasNUMAPlacementScheme. (#215)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d75ae429
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d75ae429
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d75ae429
Branch: refs/heads/query-manager-used-in-foreman
Commit: d75ae4293f35ca30cbb763968ea730e14aea4397
Parents: afcc9f0
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Sun May 8 06:21:51 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:42 2016 -0700
----------------------------------------------------------------------
catalog/CatalogRelation.hpp | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d75ae429/catalog/CatalogRelation.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.hpp b/catalog/CatalogRelation.hpp
index 3701090..312f3b4 100644
--- a/catalog/CatalogRelation.hpp
+++ b/catalog/CatalogRelation.hpp
@@ -31,6 +31,11 @@
#include "catalog/CatalogRelationSchema.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "catalog/IndexScheme.hpp"
+
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include "catalog/NUMAPlacementScheme.hpp"
+#endif // QUICKSTEP_HAVE_LIBNUMA
+
#include "catalog/PartitionScheme.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageBlockLayout.hpp"
@@ -40,10 +45,6 @@
#include "threading/SpinSharedMutex.hpp"
#include "utility/Macros.hpp"
-#ifdef QUICKSTEP_HAVE_LIBNUMA
-#include "catalog/NUMAPlacementScheme.hpp"
-#endif
-
namespace quickstep {
class CatalogDatabase;
@@ -135,16 +136,20 @@ class CatalogRelation : public CatalogRelationSchema {
return partition_scheme_.get();
}
-#ifdef QUICKSTEP_HAVE_LIBNUMA
/**
* @brief Check if a NUMA placement scheme is available for the relation.
*
* @return True if the relation has a NUMA placement scheme, false otherwise.
**/
bool hasNUMAPlacementScheme() const {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
return placement_scheme_ != nullptr;
+#else
+ return false;
+#endif // QUICKSTEP_HAVE_LIBNUMA
}
+#ifdef QUICKSTEP_HAVE_LIBNUMA
/**
* @brief Get the NUMA placement scheme of the catalog relation.
* @warning This is only safe if hasNUMAPlacementScheme() is true.
@@ -182,7 +187,7 @@ class CatalogRelation : public CatalogRelationSchema {
void setNUMAPlacementScheme(NUMAPlacementScheme *placement_scheme) {
placement_scheme_.reset(placement_scheme);
}
-#endif
+#endif // QUICKSTEP_HAVE_LIBNUMA
/**
* @brief Check if an index scheme is available for the relation.
@@ -397,7 +402,7 @@ class CatalogRelation : public CatalogRelationSchema {
// of the relation and the NUMA nodes/sockets. It also maintains a mapping
// between the blocks of the relation and the NUMA nodes..
std::unique_ptr<NUMAPlacementScheme> placement_scheme_;
-#endif
+#endif // QUICKSTEP_HAVE_LIBNUMA
friend class CatalogTest;
[02/48] incubator-quickstep git commit: Revert "Change default
aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining"
(#208)
Posted by hb...@apache.org.
Revert "Change default aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining" (#208)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/812773f9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/812773f9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/812773f9
Branch: refs/heads/query-manager-used-in-foreman
Commit: 812773f908dad2e0e5fb19e4b7713d924616dd14
Parents: b23341e
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Wed May 4 14:21:58 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:40 2016 -0700
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/812773f9/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 3698701..c34f084 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -135,7 +135,7 @@ static const volatile bool join_hashtable_type_dummy
= gflags::RegisterFlagValidator(&FLAGS_join_hashtable_type,
&ValidateHashTableImplTypeString);
-DEFINE_string(aggregate_hashtable_type, "SeparateChaining",
+DEFINE_string(aggregate_hashtable_type, "LinearOpenAddressing",
"HashTable implementation to use for aggregates with GROUP BY "
"(valid options are SeparateChaining or LinearOpenAddressing)");
static const volatile bool aggregate_hashtable_type_dummy
[41/48] incubator-quickstep git commit: Groupby hashtable pool (#236)
Posted by hb...@apache.org.
Groupby hashtable pool (#236)
- Created a HashTablePool class for group by clause.
- Each thread can checkout it's own hash table while doing group by
aggregation.
- AggregationOperationState uses one hash table pool per group by
clause.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/13281d86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/13281d86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/13281d86
Branch: refs/heads/query-manager-used-in-foreman
Commit: 13281d86d5232185afc021866b58c66181992a08
Parents: 44eff0d
Author: Harshad Deshmukh <d....@gmail.com>
Authored: Tue May 24 19:04:39 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:46 2016 -0700
----------------------------------------------------------------------
.../aggregation/AggregationConcreteHandle.hpp | 105 ++++++++++++
expressions/aggregation/AggregationHandle.hpp | 15 +-
.../aggregation/AggregationHandleAvg.cpp | 9 +
.../aggregation/AggregationHandleAvg.hpp | 4 +
.../aggregation/AggregationHandleCount.cpp | 11 ++
.../aggregation/AggregationHandleCount.hpp | 4 +
.../aggregation/AggregationHandleDistinct.hpp | 7 +
.../aggregation/AggregationHandleMax.cpp | 9 +
.../aggregation/AggregationHandleMax.hpp | 4 +
.../aggregation/AggregationHandleMin.cpp | 9 +
.../aggregation/AggregationHandleMin.hpp | 4 +
.../aggregation/AggregationHandleSum.cpp | 9 +
.../aggregation/AggregationHandleSum.hpp | 4 +
expressions/aggregation/CMakeLists.txt | 2 +
.../tests/AggregationHandleAvg_unittest.cpp | 109 ++++++++++++
.../tests/AggregationHandleCount_unittest.cpp | 126 +++++++++++++-
.../tests/AggregationHandleMax_unittest.cpp | 122 ++++++++++++++
.../tests/AggregationHandleMin_unittest.cpp | 121 ++++++++++++++
.../tests/AggregationHandleSum_unittest.cpp | 124 ++++++++++++++
query_execution/QueryContext.hpp | 2 +-
storage/AggregationOperationState.cpp | 84 ++++++++--
storage/AggregationOperationState.hpp | 4 +
storage/CMakeLists.txt | 10 ++
storage/HashTablePool.hpp | 166 +++++++++++++++++++
storage/StorageManager.cpp | 3 +-
25 files changed, 1045 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/AggregationConcreteHandle.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationConcreteHandle.hpp b/expressions/aggregation/AggregationConcreteHandle.hpp
index 52249f7..0267e17 100644
--- a/expressions/aggregation/AggregationConcreteHandle.hpp
+++ b/expressions/aggregation/AggregationConcreteHandle.hpp
@@ -44,6 +44,90 @@ class ValueAccessor;
* @{
*/
+/**
+ * @brief An upserter class for modifying the destination hash table while
+ * merging two group by hash tables.
+ **/
+template <typename HandleT, typename StateT>
+class HashTableStateUpserter {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param handle The aggregation handle being used.
+ * @param source_state The aggregation state in the source aggregation hash
+ * table. The corresponding state (for the same key) in the destination
+ * hash table will be upserted.
+ **/
+ HashTableStateUpserter(const HandleT &handle, const StateT &source_state)
+ : handle_(handle), source_state_(source_state) {}
+
+ /**
+ * @brief The operator for the functor required for the upsert.
+ *
+ * @param destination_state The aggregation state in the aggregation hash
+ * table that is being upserted.
+ **/
+ void operator()(StateT *destination_state) {
+ handle_.mergeStates(source_state_, destination_state);
+ }
+
+ private:
+ const HandleT &handle_;
+ const StateT &source_state_;
+
+ DISALLOW_COPY_AND_ASSIGN(HashTableStateUpserter);
+};
+
+/**
+ * @brief A class to support the functor for merging group by hash tables.
+ **/
+template <typename HandleT, typename StateT, typename HashTableT>
+class HashTableMerger {
+ public:
+ /**
+ * @brief Constructor
+ *
+ * @param handle The Aggregation handle being used.
+ * @param destination_hash_table The destination hash table to which other
+ * hash tables will be merged.
+ **/
+ HashTableMerger(const HandleT &handle,
+ AggregationStateHashTableBase *destination_hash_table)
+ : handle_(handle),
+ destination_hash_table_(
+ static_cast<HashTableT *>(destination_hash_table)) {}
+
+ /**
+ * @brief The operator for the functor.
+ *
+ * @param group_by_key The group by key being merged.
+ * @param source_state The aggregation state for the given key in the source
+ * aggregation hash table.
+ **/
+ inline void operator()(const std::vector<TypedValue> &group_by_key,
+ const StateT &source_state) {
+ const StateT *original_state =
+ destination_hash_table_->getSingleCompositeKey(group_by_key);
+ if (original_state != nullptr) {
+ HashTableStateUpserter<HandleT, StateT> upserter(
+ handle_, source_state);
+ // The CHECK is required as upsertCompositeKey can return false if the
+ // hash table runs out of space during the upsert process. The ideal
+ // solution will be to retry again if the upsert fails.
+ CHECK(destination_hash_table_->upsertCompositeKey(
+ group_by_key, *original_state, &upserter));
+ } else {
+ destination_hash_table_->putCompositeKey(group_by_key, source_state);
+ }
+ }
+
+ private:
+ const HandleT &handle_;
+ HashTableT *destination_hash_table_;
+
+ DISALLOW_COPY_AND_ASSIGN(HashTableMerger);
+};
/**
* @brief The helper intermediate subclass of AggregationHandle that provides
@@ -140,6 +224,11 @@ class AggregationConcreteHandle : public AggregationHandle {
return static_cast<const HandleT*>(this)->finalizeHashTableEntry(*group_state);
}
+ template <typename HandleT, typename StateT, typename HashTableT>
+ void mergeGroupByHashTablesHelper(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const;
+
private:
DISALLOW_COPY_AND_ASSIGN(AggregationConcreteHandle);
};
@@ -373,6 +462,22 @@ ColumnVector* AggregationConcreteHandle::finalizeHashTableHelper(
}
}
+template <typename HandleT,
+ typename StateT,
+ typename HashTableT>
+void AggregationConcreteHandle::mergeGroupByHashTablesHelper(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ const HandleT &handle = static_cast<const HandleT &>(*this);
+ const HashTableT &source_hash_table_concrete =
+ static_cast<const HashTableT &>(source_hash_table);
+
+ HashTableMerger<HandleT, StateT, HashTableT> merger(handle,
+ destination_hash_table);
+
+ source_hash_table_concrete.forEachCompositeKey(&merger);
+}
+
} // namespace quickstep
#endif // QUICKSTEP_EXPRESSIONS_AGGREGATION_AGGREGATION_CONCRETE_HANDLE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/AggregationHandle.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandle.hpp b/expressions/aggregation/AggregationHandle.hpp
index 625f334..cdebb03 100644
--- a/expressions/aggregation/AggregationHandle.hpp
+++ b/expressions/aggregation/AggregationHandle.hpp
@@ -276,7 +276,7 @@ class AggregationHandle {
* each GROUP BY group. Later, a second-round aggregation on the distinctify
* hash table will be performed to actually compute the aggregated result for
* each GROUP BY group.
- *
+ *
* In the case of single aggregation where there is no GROUP BY expressions,
* we simply treat it as a special GROUP BY case that the GROUP BY expression
* vector is empty.
@@ -349,6 +349,19 @@ class AggregationHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const = 0;
+ /**
+ * @brief Merge two GROUP BY hash tables in one.
+ *
+ * @note Both the hash tables should have the same structure.
+ *
+ * @param source_hash_table The hash table which will get merged.
+ * @param destination_hash_table The hash table to which we will merge the
+ * other hash table.
+ **/
+ virtual void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const = 0;
+
protected:
AggregationHandle() {
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/AggregationHandleAvg.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleAvg.cpp b/expressions/aggregation/AggregationHandleAvg.cpp
index cb0d63d..42a2fb9 100644
--- a/expressions/aggregation/AggregationHandleAvg.cpp
+++ b/expressions/aggregation/AggregationHandleAvg.cpp
@@ -203,4 +203,13 @@ void AggregationHandleAvg::aggregateOnDistinctifyHashTableForGroupBy(
aggregation_hash_table);
}
+void AggregationHandleAvg::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<AggregationHandleAvg,
+ AggregationStateAvg,
+ AggregationStateHashTable<AggregationStateAvg>>(
+ source_hash_table, destination_hash_table);
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/AggregationHandleAvg.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleAvg.hpp b/expressions/aggregation/AggregationHandleAvg.hpp
index 6a94ee6..4ad4b21 100644
--- a/expressions/aggregation/AggregationHandleAvg.hpp
+++ b/expressions/aggregation/AggregationHandleAvg.hpp
@@ -158,6 +158,10 @@ class AggregationHandleAvg : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionAvg;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/AggregationHandleCount.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleCount.cpp b/expressions/aggregation/AggregationHandleCount.cpp
index 5ece8ba..964b7c2 100644
--- a/expressions/aggregation/AggregationHandleCount.cpp
+++ b/expressions/aggregation/AggregationHandleCount.cpp
@@ -206,6 +206,17 @@ void AggregationHandleCount<count_star, nullable_type>
aggregation_hash_table);
}
+template <bool count_star, bool nullable_type>
+void AggregationHandleCount<count_star, nullable_type>::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<
+ AggregationHandleCount,
+ AggregationStateCount,
+ AggregationStateHashTable<AggregationStateCount>>(source_hash_table,
+ destination_hash_table);
+}
+
// Explicitly instantiate and compile in the different versions of
// AggregationHandleCount we need. Note that we do not compile a version with
// 'count_star == true' and 'nullable_type == true', as that combination is
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/AggregationHandleCount.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleCount.hpp b/expressions/aggregation/AggregationHandleCount.hpp
index 6bb4e65..50138b9 100644
--- a/expressions/aggregation/AggregationHandleCount.hpp
+++ b/expressions/aggregation/AggregationHandleCount.hpp
@@ -166,6 +166,10 @@ class AggregationHandleCount : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionCount;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/AggregationHandleDistinct.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleDistinct.hpp b/expressions/aggregation/AggregationHandleDistinct.hpp
index 918fdf8..6342c2b 100644
--- a/expressions/aggregation/AggregationHandleDistinct.hpp
+++ b/expressions/aggregation/AggregationHandleDistinct.hpp
@@ -109,6 +109,13 @@ class AggregationHandleDistinct : public AggregationConcreteHandle {
const AggregationStateHashTableBase &hash_table,
std::vector<std::vector<TypedValue>> *group_by_keys) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override {
+ LOG(FATAL)
+ << "AggregationHandleDistinct does not support mergeGroupByHashTables";
+ }
+
private:
DISALLOW_COPY_AND_ASSIGN(AggregationHandleDistinct);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/AggregationHandleMax.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMax.cpp b/expressions/aggregation/AggregationHandleMax.cpp
index 4703657..a7a4a52 100644
--- a/expressions/aggregation/AggregationHandleMax.cpp
+++ b/expressions/aggregation/AggregationHandleMax.cpp
@@ -139,4 +139,13 @@ void AggregationHandleMax::aggregateOnDistinctifyHashTableForGroupBy(
aggregation_hash_table);
}
+void AggregationHandleMax::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<AggregationHandleMax,
+ AggregationStateMax,
+ AggregationStateHashTable<AggregationStateMax>>(
+ source_hash_table, destination_hash_table);
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/AggregationHandleMax.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMax.hpp b/expressions/aggregation/AggregationHandleMax.hpp
index 8932ef8..5af5a12 100644
--- a/expressions/aggregation/AggregationHandleMax.hpp
+++ b/expressions/aggregation/AggregationHandleMax.hpp
@@ -151,6 +151,10 @@ class AggregationHandleMax : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionMax;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/AggregationHandleMin.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMin.cpp b/expressions/aggregation/AggregationHandleMin.cpp
index de2709a..ca9b163 100644
--- a/expressions/aggregation/AggregationHandleMin.cpp
+++ b/expressions/aggregation/AggregationHandleMin.cpp
@@ -141,4 +141,13 @@ void AggregationHandleMin::aggregateOnDistinctifyHashTableForGroupBy(
aggregation_hash_table);
}
+void AggregationHandleMin::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<AggregationHandleMin,
+ AggregationStateMin,
+ AggregationStateHashTable<AggregationStateMin>>(
+ source_hash_table, destination_hash_table);
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/AggregationHandleMin.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMin.hpp b/expressions/aggregation/AggregationHandleMin.hpp
index 4e4c05d..f68bb9d 100644
--- a/expressions/aggregation/AggregationHandleMin.hpp
+++ b/expressions/aggregation/AggregationHandleMin.hpp
@@ -149,6 +149,10 @@ class AggregationHandleMin : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionMin;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/AggregationHandleSum.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleSum.cpp b/expressions/aggregation/AggregationHandleSum.cpp
index 14421d2..691ff39 100644
--- a/expressions/aggregation/AggregationHandleSum.cpp
+++ b/expressions/aggregation/AggregationHandleSum.cpp
@@ -190,4 +190,13 @@ void AggregationHandleSum::aggregateOnDistinctifyHashTableForGroupBy(
aggregation_hash_table);
}
+void AggregationHandleSum::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<AggregationHandleSum,
+ AggregationStateSum,
+ AggregationStateHashTable<AggregationStateSum>>(
+ source_hash_table, destination_hash_table);
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/AggregationHandleSum.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleSum.hpp b/expressions/aggregation/AggregationHandleSum.hpp
index b765243..fdc0884 100644
--- a/expressions/aggregation/AggregationHandleSum.hpp
+++ b/expressions/aggregation/AggregationHandleSum.hpp
@@ -148,6 +148,10 @@ class AggregationHandleSum : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionSum;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/expressions/aggregation/CMakeLists.txt b/expressions/aggregation/CMakeLists.txt
index 26cec7f..416c4c6 100644
--- a/expressions/aggregation/CMakeLists.txt
+++ b/expressions/aggregation/CMakeLists.txt
@@ -291,6 +291,8 @@ target_link_libraries(AggregationHandle_tests
quickstep_expressions_aggregation_AggregationHandleMin
quickstep_expressions_aggregation_AggregationHandleSum
quickstep_expressions_aggregation_AggregationID
+ quickstep_storage_HashTableBase
+ quickstep_storage_StorageManager
quickstep_types_CharType
quickstep_types_DateOperatorOverloads
quickstep_types_DatetimeIntervalType
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp b/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
index d27b54e..fd82cba 100644
--- a/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
@@ -26,6 +26,7 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleAvg.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DateOperatorOverloads.hpp"
#include "types/DatetimeIntervalType.hpp"
@@ -238,6 +239,7 @@ class AggregationHandleAvgTest : public::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_avg_;
std::unique_ptr<AggregationState> aggregation_handle_avg_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
const int AggregationHandleAvgTest::kNumSamples;
@@ -417,4 +419,111 @@ TEST_F(AggregationHandleAvgTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kYearMonthInterval, kYearMonthInterval));
}
+TEST_F(AggregationHandleAvgTest, GroupByTableMergeTestAvg) {
+ const Type &long_non_null_type = LongType::Instance(false);
+ initializeHandle(long_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_avg_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_avg_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_avg_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateAvg> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateAvg> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateAvg> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateAvg> *>(
+ source_hash_table.get());
+
+ AggregationHandleAvg *aggregation_handle_avg_derived =
+ static_cast<AggregationHandleAvg *>(aggregation_handle_avg_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(static_cast<std::int64_t>(0));
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(static_cast<std::int64_t>(1));
+ exclusive_destination_key.emplace_back(static_cast<std::int64_t>(2));
+
+ const std::int64_t common_key_source_avg = 355;
+ TypedValue common_key_source_avg_val(common_key_source_avg);
+
+ const std::int64_t common_key_destination_avg = 295;
+ TypedValue common_key_destination_avg_val(common_key_destination_avg);
+
+ const std::int64_t exclusive_key_source_avg = 1;
+ TypedValue exclusive_key_source_avg_val(exclusive_key_source_avg);
+
+ const std::int64_t exclusive_key_destination_avg = 1;
+ TypedValue exclusive_key_destination_avg_val(exclusive_key_destination_avg);
+
+ std::unique_ptr<AggregationStateAvg> common_key_source_state(
+ static_cast<AggregationStateAvg *>(
+ aggregation_handle_avg_->createInitialState()));
+ std::unique_ptr<AggregationStateAvg> common_key_destination_state(
+ static_cast<AggregationStateAvg *>(
+ aggregation_handle_avg_->createInitialState()));
+ std::unique_ptr<AggregationStateAvg> exclusive_key_source_state(
+ static_cast<AggregationStateAvg *>(
+ aggregation_handle_avg_->createInitialState()));
+ std::unique_ptr<AggregationStateAvg> exclusive_key_destination_state(
+ static_cast<AggregationStateAvg *>(
+ aggregation_handle_avg_->createInitialState()));
+
+ // Create avg value states for keys.
+ aggregation_handle_avg_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_avg_val);
+
+ aggregation_handle_avg_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_avg_val);
+
+ aggregation_handle_avg_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_avg_val);
+
+ aggregation_handle_avg_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_avg_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_avg_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckAvgValue<double>(
+ (common_key_destination_avg_val.getLiteral<std::int64_t>() +
+ common_key_source_avg_val.getLiteral<std::int64_t>()) / static_cast<double>(2),
+ *aggregation_handle_avg_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckAvgValue<double>(exclusive_key_destination_avg_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_avg_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckAvgValue<double>(exclusive_key_source_avg_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_avg_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp b/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
index 7bebf6a..bf02523 100644
--- a/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
@@ -27,6 +27,7 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleCount.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DoubleType.hpp"
#include "types/FloatType.hpp"
@@ -355,6 +356,7 @@ class AggregationHandleCountTest : public::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_count_;
std::unique_ptr<AggregationState> aggregation_handle_count_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
typedef AggregationHandleCountTest AggregationHandleCountDeathTest;
@@ -477,5 +479,127 @@ TEST_F(AggregationHandleCountTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kDouble, kLong));
}
-} // namespace quickstep
+TEST_F(AggregationHandleCountTest, GroupByTableMergeTestCount) {
+ const Type &long_non_null_type = LongType::Instance(false);
+ initializeHandle(&long_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_count_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_count_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_count_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateCount> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateCount> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateCount> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateCount> *>(
+ source_hash_table.get());
+
+ // TODO(harshad) - Use TemplateUtil::CreateBoolInstantiatedInstance to
+ // generate all the combinations of the bool template arguments and test them.
+ AggregationHandleCount<true, false> *aggregation_handle_count_derived =
+ static_cast<AggregationHandleCount<true, false> *>(
+ aggregation_handle_count_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(static_cast<std::int64_t>(0));
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(static_cast<std::int64_t>(1));
+ exclusive_destination_key.emplace_back(static_cast<std::int64_t>(2));
+
+ const std::int64_t common_key_source_count = 1;
+ TypedValue common_key_source_count_val(common_key_source_count);
+
+ const std::int64_t common_key_destination_count = 1;
+ TypedValue common_key_destination_count_val(common_key_destination_count);
+
+ const std::int64_t exclusive_key_source_count = 1;
+ TypedValue exclusive_key_source_count_val(exclusive_key_source_count);
+
+ const std::int64_t exclusive_key_destination_count = 1;
+ TypedValue exclusive_key_destination_count_val(exclusive_key_destination_count);
+
+ std::unique_ptr<AggregationStateCount> common_key_source_state(
+ static_cast<AggregationStateCount *>(
+ aggregation_handle_count_->createInitialState()));
+ std::unique_ptr<AggregationStateCount> common_key_destination_state(
+ static_cast<AggregationStateCount *>(
+ aggregation_handle_count_->createInitialState()));
+ std::unique_ptr<AggregationStateCount> exclusive_key_source_state(
+ static_cast<AggregationStateCount *>(
+ aggregation_handle_count_->createInitialState()));
+ std::unique_ptr<AggregationStateCount> exclusive_key_destination_state(
+ static_cast<AggregationStateCount *>(
+ aggregation_handle_count_->createInitialState()));
+
+ // Create count value states for keys.
+ aggregation_handle_count_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_count_val);
+ std::int64_t actual_val = aggregation_handle_count_->finalize(*common_key_source_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(common_key_source_count_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_count_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_count_val);
+ actual_val = aggregation_handle_count_->finalize(*common_key_destination_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(common_key_destination_count_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_count_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_count_val);
+ actual_val =
+ aggregation_handle_count_->finalize(*exclusive_key_destination_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(exclusive_key_destination_count_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_count_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_count_val);
+ actual_val = aggregation_handle_count_->finalize(*exclusive_key_source_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(exclusive_key_source_count_val.getLiteral<std::int64_t>(), actual_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_count_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckCountValue(
+ common_key_destination_count_val.getLiteral<std::int64_t>() +
+ common_key_source_count_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_count_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckCountValue(exclusive_key_destination_count_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_count_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckCountValue(exclusive_key_source_count_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_count_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp b/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
index 027f24b..fc25e91 100644
--- a/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
@@ -29,6 +29,8 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleMax.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/HashTableBase.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DatetimeIntervalType.hpp"
#include "types/DatetimeLit.hpp"
@@ -413,6 +415,7 @@ class AggregationHandleMaxTest : public ::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_max_;
std::unique_ptr<AggregationState> aggregation_handle_max_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
template <>
@@ -637,4 +640,123 @@ TEST_F(AggregationHandleMaxTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kDouble, kDouble));
}
+TEST_F(AggregationHandleMaxTest, GroupByTableMergeTest) {
+ const Type &int_non_null_type = IntType::Instance(false);
+ initializeHandle(int_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_max_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_max_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &int_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_max_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &int_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateMax> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateMax> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateMax> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateMax> *>(
+ source_hash_table.get());
+
+ AggregationHandleMax *aggregation_handle_max_derived =
+ static_cast<AggregationHandleMax *>(aggregation_handle_max_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(0);
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(1);
+ exclusive_destination_key.emplace_back(2);
+
+ const int common_key_source_max = 3000;
+ TypedValue common_key_source_max_val(common_key_source_max);
+
+ const int common_key_destination_max = 4000;
+ TypedValue common_key_destination_max_val(common_key_destination_max);
+
+ const int exclusive_key_source_max = 100;
+ TypedValue exclusive_key_source_max_val(exclusive_key_source_max);
+
+ const int exclusive_key_destination_max = 200;
+ TypedValue exclusive_key_destination_max_val(exclusive_key_destination_max);
+
+ std::unique_ptr<AggregationStateMax> common_key_source_state(
+ static_cast<AggregationStateMax *>(
+ aggregation_handle_max_->createInitialState()));
+ std::unique_ptr<AggregationStateMax> common_key_destination_state(
+ static_cast<AggregationStateMax *>(
+ aggregation_handle_max_->createInitialState()));
+ std::unique_ptr<AggregationStateMax> exclusive_key_source_state(
+ static_cast<AggregationStateMax *>(
+ aggregation_handle_max_->createInitialState()));
+ std::unique_ptr<AggregationStateMax> exclusive_key_destination_state(
+ static_cast<AggregationStateMax *>(
+ aggregation_handle_max_->createInitialState()));
+
+ // Create max value states for keys.
+ aggregation_handle_max_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_max_val);
+ int actual_val = aggregation_handle_max_->finalize(*common_key_source_state)
+ .getLiteral<int>();
+ EXPECT_EQ(common_key_source_max_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_max_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_max_val);
+ actual_val = aggregation_handle_max_->finalize(*common_key_destination_state)
+ .getLiteral<int>();
+ EXPECT_EQ(common_key_destination_max_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_max_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_max_val);
+ actual_val =
+ aggregation_handle_max_->finalize(*exclusive_key_destination_state)
+ .getLiteral<int>();
+ EXPECT_EQ(exclusive_key_destination_max_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_max_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_max_val);
+ actual_val = aggregation_handle_max_->finalize(*exclusive_key_source_state)
+ .getLiteral<int>();
+ EXPECT_EQ(exclusive_key_source_max_val.getLiteral<int>(), actual_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_max_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckMaxValue<int>(
+ common_key_destination_max_val.getLiteral<int>(),
+ *aggregation_handle_max_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckMaxValue<int>(exclusive_key_destination_max_val.getLiteral<int>(),
+ *aggregation_handle_max_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckMaxValue<int>(exclusive_key_source_max_val.getLiteral<int>(),
+ *aggregation_handle_max_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp b/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
index eb64472..a87ace9 100644
--- a/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
@@ -29,6 +29,7 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleMin.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DatetimeIntervalType.hpp"
#include "types/DatetimeLit.hpp"
@@ -411,6 +412,7 @@ class AggregationHandleMinTest : public ::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_min_;
std::unique_ptr<AggregationState> aggregation_handle_min_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
template <>
@@ -634,4 +636,123 @@ TEST_F(AggregationHandleMinTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kDouble, kDouble));
}
+TEST_F(AggregationHandleMinTest, GroupByTableMergeTest) {
+ const Type &int_non_null_type = IntType::Instance(false);
+ initializeHandle(int_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_min_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_min_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &int_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_min_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &int_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateMin> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateMin> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateMin> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateMin> *>(
+ source_hash_table.get());
+
+ AggregationHandleMin *aggregation_handle_min_derived =
+ static_cast<AggregationHandleMin *>(aggregation_handle_min_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(0);
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(1);
+ exclusive_destination_key.emplace_back(2);
+
+ const int common_key_source_min = 3000;
+ TypedValue common_key_source_min_val(common_key_source_min);
+
+ const int common_key_destination_min = 4000;
+ TypedValue common_key_destination_min_val(common_key_destination_min);
+
+ const int exclusive_key_source_min = 100;
+ TypedValue exclusive_key_source_min_val(exclusive_key_source_min);
+
+ const int exclusive_key_destination_min = 200;
+ TypedValue exclusive_key_destination_min_val(exclusive_key_destination_min);
+
+ std::unique_ptr<AggregationStateMin> common_key_source_state(
+ static_cast<AggregationStateMin *>(
+ aggregation_handle_min_->createInitialState()));
+ std::unique_ptr<AggregationStateMin> common_key_destination_state(
+ static_cast<AggregationStateMin *>(
+ aggregation_handle_min_->createInitialState()));
+ std::unique_ptr<AggregationStateMin> exclusive_key_source_state(
+ static_cast<AggregationStateMin *>(
+ aggregation_handle_min_->createInitialState()));
+ std::unique_ptr<AggregationStateMin> exclusive_key_destination_state(
+ static_cast<AggregationStateMin *>(
+ aggregation_handle_min_->createInitialState()));
+
+ // Create min value states for keys.
+ aggregation_handle_min_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_min_val);
+ int actual_val = aggregation_handle_min_->finalize(*common_key_source_state)
+ .getLiteral<int>();
+ EXPECT_EQ(common_key_source_min_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_min_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_min_val);
+ actual_val = aggregation_handle_min_->finalize(*common_key_destination_state)
+ .getLiteral<int>();
+ EXPECT_EQ(common_key_destination_min_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_min_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_min_val);
+ actual_val =
+ aggregation_handle_min_->finalize(*exclusive_key_destination_state)
+ .getLiteral<int>();
+ EXPECT_EQ(exclusive_key_destination_min_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_min_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_min_val);
+ actual_val = aggregation_handle_min_->finalize(*exclusive_key_source_state)
+ .getLiteral<int>();
+ EXPECT_EQ(exclusive_key_source_min_val.getLiteral<int>(), actual_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_min_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckMinValue<int>(
+ common_key_source_min_val.getLiteral<int>(),
+ *aggregation_handle_min_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckMinValue<int>(exclusive_key_destination_min_val.getLiteral<int>(),
+ *aggregation_handle_min_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckMinValue<int>(exclusive_key_source_min_val.getLiteral<int>(),
+ *aggregation_handle_min_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp b/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
index 7dbbeb3..abf8a89 100644
--- a/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
@@ -26,6 +26,7 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleSum.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DatetimeIntervalType.hpp"
#include "types/DoubleType.hpp"
@@ -237,6 +238,7 @@ class AggregationHandleSumTest : public::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_sum_;
std::unique_ptr<AggregationState> aggregation_handle_sum_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
const int AggregationHandleSumTest::kNumSamples;
@@ -425,4 +427,126 @@ TEST_F(AggregationHandleSumTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kYearMonthInterval, kYearMonthInterval));
}
+TEST_F(AggregationHandleSumTest, GroupByTableMergeTest) {
+ const Type &long_non_null_type = LongType::Instance(false);
+ initializeHandle(long_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_sum_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_sum_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_sum_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateSum> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateSum> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateSum> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateSum> *>(
+ source_hash_table.get());
+
+ AggregationHandleSum *aggregation_handle_sum_derived =
+ static_cast<AggregationHandleSum *>(aggregation_handle_sum_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(static_cast<std::int64_t>(0));
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(static_cast<std::int64_t>(1));
+ exclusive_destination_key.emplace_back(static_cast<std::int64_t>(2));
+
+ const std::int64_t common_key_source_sum = 3000;
+ TypedValue common_key_source_sum_val(common_key_source_sum);
+
+ const std::int64_t common_key_destination_sum = 4000;
+ TypedValue common_key_destination_sum_val(common_key_destination_sum);
+
+ const std::int64_t merged_common_key = common_key_source_sum + common_key_destination_sum;
+ TypedValue common_key_merged_val(merged_common_key);
+
+ const std::int64_t exclusive_key_source_sum = 100;
+ TypedValue exclusive_key_source_sum_val(exclusive_key_source_sum);
+
+ const std::int64_t exclusive_key_destination_sum = 200;
+ TypedValue exclusive_key_destination_sum_val(exclusive_key_destination_sum);
+
+ std::unique_ptr<AggregationStateSum> common_key_source_state(
+ static_cast<AggregationStateSum *>(
+ aggregation_handle_sum_->createInitialState()));
+ std::unique_ptr<AggregationStateSum> common_key_destination_state(
+ static_cast<AggregationStateSum *>(
+ aggregation_handle_sum_->createInitialState()));
+ std::unique_ptr<AggregationStateSum> exclusive_key_source_state(
+ static_cast<AggregationStateSum *>(
+ aggregation_handle_sum_->createInitialState()));
+ std::unique_ptr<AggregationStateSum> exclusive_key_destination_state(
+ static_cast<AggregationStateSum *>(
+ aggregation_handle_sum_->createInitialState()));
+
+ // Create sum value states for keys.
+ aggregation_handle_sum_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_sum_val);
+ std::int64_t actual_val = aggregation_handle_sum_->finalize(*common_key_source_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(common_key_source_sum_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_sum_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_sum_val);
+ actual_val = aggregation_handle_sum_->finalize(*common_key_destination_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(common_key_destination_sum_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_sum_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_sum_val);
+ actual_val =
+ aggregation_handle_sum_->finalize(*exclusive_key_destination_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(exclusive_key_destination_sum_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_sum_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_sum_val);
+ actual_val = aggregation_handle_sum_->finalize(*exclusive_key_source_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(exclusive_key_source_sum_val.getLiteral<std::int64_t>(), actual_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_sum_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckSumValue<std::int64_t>(
+ common_key_merged_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_sum_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckSumValue<std::int64_t>(exclusive_key_destination_sum_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_sum_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckSumValue<std::int64_t>(exclusive_key_source_sum_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_sum_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/query_execution/QueryContext.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp
index 9440fae..7d5628d 100644
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@ -216,7 +216,7 @@ class QueryContext {
*
* @param id The BloomFilter id.
*
- * @return The constant pointer to BloomFilter that is
+ * @return The constant pointer to BloomFilter that is
* already created in the constructor.
**/
inline const BloomFilter* getBloomFilter(const bloom_filter_id id) const {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/storage/AggregationOperationState.cpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.cpp b/storage/AggregationOperationState.cpp
index d209ceb..4878cf1 100644
--- a/storage/AggregationOperationState.cpp
+++ b/storage/AggregationOperationState.cpp
@@ -92,11 +92,12 @@ AggregationOperationState::AggregationOperationState(
arguments_.push_back({});
is_distinct_.emplace_back(false);
- group_by_hashtables_.emplace_back(handles_.back()->createGroupByHashTable(
- hash_table_impl_type,
- group_by_types,
- estimated_num_entries,
- storage_manager_));
+ group_by_hashtable_pools_.emplace_back(std::unique_ptr<HashTablePool>(
+ new HashTablePool(estimated_num_entries,
+ hash_table_impl_type,
+ group_by_types,
+ handles_.back().get(),
+ storage_manager)));
} else {
// Set up each individual aggregate in this operation.
std::vector<const AggregateFunction*>::const_iterator agg_func_it
@@ -124,12 +125,13 @@ AggregationOperationState::AggregationOperationState(
handles_.emplace_back((*agg_func_it)->createHandle(argument_types));
if (!group_by_list_.empty()) {
- // Aggregation with GROUP BY: create a HashTable for per-group states.
- group_by_hashtables_.emplace_back(handles_.back()->createGroupByHashTable(
- hash_table_impl_type,
- group_by_types,
- estimated_num_entries,
- storage_manager_));
+ // Aggregation with GROUP BY: create a HashTable pool for per-group states.
+ group_by_hashtable_pools_.emplace_back(std::unique_ptr<HashTablePool>(
+ new HashTablePool(estimated_num_entries,
+ hash_table_impl_type,
+ group_by_types,
+ handles_.back().get(),
+ storage_manager)));
} else {
// Aggregation without GROUP BY: create a single global state.
single_states_.emplace_back(handles_.back()->createInitialState());
@@ -408,17 +410,17 @@ void AggregationOperationState::aggregateBlockHashTable(const block_id input_blo
// Call StorageBlock::aggregateGroupBy() to aggregate this block's values
// directly into the (threadsafe) shared global HashTable for this
// aggregate.
- //
- // TODO(shoban): Implement optional code path for using local hash table per
- // block, which can be merged with global hash table for all blocks
- // aggregated on.
+ DCHECK(group_by_hashtable_pools_[agg_idx] != nullptr);
+ AggregationStateHashTableBase *agg_hash_table = group_by_hashtable_pools_[agg_idx]->getHashTable();
+ DCHECK(agg_hash_table != nullptr);
block->aggregateGroupBy(*handles_[agg_idx],
arguments_[agg_idx],
group_by_list_,
predicate_.get(),
- group_by_hashtables_[agg_idx].get(),
+ agg_hash_table,
&reuse_matches,
&reuse_group_by_vectors);
+ group_by_hashtable_pools_[agg_idx]->returnHashTable(agg_hash_table);
}
}
}
@@ -447,19 +449,65 @@ void AggregationOperationState::finalizeHashTable(InsertDestination *output_dest
// group (which is also the prefix of the finalized Tuple for that group).
std::vector<std::vector<TypedValue>> group_by_keys;
+ // TODO(harshad) - The merge phase may be slower when each hash table contains
+ // large number of entries. We should find ways in which we can perform a
+ // parallel merge.
+
+ // TODO(harshad) - Find heuristics for faster merge, even in a single thread.
+ // e.g. Keep merging entries from smaller hash tables to larger.
+ for (std::size_t agg_idx = 0; agg_idx < handles_.size(); ++agg_idx) {
+ auto *hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ if (hash_tables->size() > 1) {
+ for (int hash_table_index = 0;
+ hash_table_index < static_cast<int>(hash_tables->size() - 1);
+ ++hash_table_index) {
+ // Merge each hash table to the last hash table.
+ handles_[agg_idx]->mergeGroupByHashTables(
+ (*(*hash_tables)[hash_table_index]),
+ hash_tables->back().get());
+ }
+ }
+ }
+
// Collect per-aggregate finalized values.
std::vector<std::unique_ptr<ColumnVector>> final_values;
for (std::size_t agg_idx = 0;
agg_idx < handles_.size();
++agg_idx) {
if (is_distinct_[agg_idx]) {
+ DCHECK(group_by_hashtable_pools_[agg_idx] != nullptr);
+ auto *hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ DCHECK(hash_tables != nullptr);
+ if (hash_tables->empty()) {
+ // We may have a case where hash_tables is empty, e.g. no input blocks.
+ // However for aggregateOnDistinctifyHashTableForGroupBy to work
+ // correctly, we should create an empty group by hash table.
+ AggregationStateHashTableBase *new_hash_table = group_by_hashtable_pools_[agg_idx]->getHashTable();
+ group_by_hashtable_pools_[agg_idx]->returnHashTable(new_hash_table);
+ hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ }
+ DCHECK(hash_tables->back() != nullptr);
+ AggregationStateHashTableBase *agg_hash_table = hash_tables->back().get();
+ DCHECK(agg_hash_table != nullptr);
handles_[agg_idx]->aggregateOnDistinctifyHashTableForGroupBy(
*distinctify_hashtables_[agg_idx],
- group_by_hashtables_[agg_idx].get());
+ agg_hash_table);
}
+ auto *hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ DCHECK(hash_tables != nullptr);
+ if (hash_tables->empty()) {
+ // We may have a case where hash_tables is empty, e.g. no input blocks.
+ // However for aggregateOnDistinctifyHashTableForGroupBy to work
+ // correctly, we should create an empty group by hash table.
+ AggregationStateHashTableBase *new_hash_table = group_by_hashtable_pools_[agg_idx]->getHashTable();
+ group_by_hashtable_pools_[agg_idx]->returnHashTable(new_hash_table);
+ hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ }
+ AggregationStateHashTableBase *agg_hash_table = hash_tables->back().get();
+ DCHECK(agg_hash_table != nullptr);
ColumnVector* agg_result_col =
- handles_[agg_idx]->finalizeHashTable(*group_by_hashtables_[agg_idx],
+ handles_[agg_idx]->finalizeHashTable(*agg_hash_table,
&group_by_keys);
if (agg_result_col != nullptr) {
final_values.emplace_back(agg_result_col);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/storage/AggregationOperationState.hpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.hpp b/storage/AggregationOperationState.hpp
index c3a1278..0199749 100644
--- a/storage/AggregationOperationState.hpp
+++ b/storage/AggregationOperationState.hpp
@@ -31,6 +31,7 @@
#include "expressions/scalar/Scalar.hpp"
#include "storage/AggregationOperationState.pb.h"
#include "storage/HashTableBase.hpp"
+#include "storage/HashTablePool.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
@@ -209,6 +210,9 @@ class AggregationOperationState {
// hash table to prevent multiple lookups.
std::vector<std::unique_ptr<AggregationStateHashTableBase>> group_by_hashtables_;
+ // A vector of group by hash table pools, one for each group by clause.
+ std::vector<std::unique_ptr<HashTablePool>> group_by_hashtable_pools_;
+
StorageManager *storage_manager_;
DISALLOW_COPY_AND_ASSIGN(AggregationOperationState);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index a3093df..87a5e54 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -187,6 +187,7 @@ add_library(quickstep_storage_HashTable_proto ${storage_HashTable_proto_srcs})
add_library(quickstep_storage_HashTableBase ../empty_src.cpp HashTableBase.hpp)
add_library(quickstep_storage_HashTableFactory HashTableFactory.cpp HashTableFactory.hpp)
add_library(quickstep_storage_HashTableKeyManager ../empty_src.cpp HashTableKeyManager.hpp)
+add_library(quickstep_storage_HashTablePool ../empty_src.cpp HashTablePool.hpp)
add_library(quickstep_storage_IndexSubBlock ../empty_src.cpp IndexSubBlock.hpp)
add_library(quickstep_storage_IndexSubBlockDescriptionFactory ../empty_src.cpp IndexSubBlockDescriptionFactory.hpp)
add_library(quickstep_storage_InsertDestination InsertDestination.cpp InsertDestination.hpp)
@@ -252,6 +253,7 @@ target_link_libraries(quickstep_storage_AggregationOperationState
quickstep_storage_HashTable
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
+ quickstep_storage_HashTablePool
quickstep_storage_InsertDestination
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
@@ -662,6 +664,13 @@ target_link_libraries(quickstep_storage_HashTableKeyManager
quickstep_types_TypedValue
quickstep_types_operations_comparisons_ComparisonUtil
quickstep_utility_Macros)
+target_link_libraries(quickstep_storage_HashTablePool
+ glog
+ quickstep_expressions_aggregation_AggregationHandle
+ quickstep_storage_HashTableBase
+ quickstep_threading_SpinMutex
+ quickstep_utility_Macros
+ quickstep_utility_StringUtil)
target_link_libraries(quickstep_storage_IndexSubBlock
quickstep_catalog_CatalogTypedefs
quickstep_expressions_predicate_PredicateCost
@@ -1012,6 +1021,7 @@ target_link_libraries(quickstep_storage
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
quickstep_storage_HashTableKeyManager
+ quickstep_storage_HashTablePool
quickstep_storage_IndexSubBlock
quickstep_storage_IndexSubBlockDescriptionFactory
quickstep_storage_InsertDestination
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/storage/HashTablePool.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTablePool.hpp b/storage/HashTablePool.hpp
new file mode 100644
index 0000000..c16d0f1
--- /dev/null
+++ b/storage/HashTablePool.hpp
@@ -0,0 +1,166 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_HASH_TABLE_POOL_HPP_
+#define QUICKSTEP_STORAGE_HASH_TABLE_POOL_HPP_
+
+#include <chrono>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "expressions/aggregation/AggregationHandle.hpp"
+#include "storage/HashTableBase.hpp"
+#include "threading/SpinMutex.hpp"
+#include "utility/Macros.hpp"
+#include "utility/StringUtil.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class StorageManager;
+class Type;
+
+/** \addtogroup Storage
+ * @{
+ */
+
+/**
+ * @brief A pool of HashTables used for a single aggregation handle. This class
+ * has similar functionality as InsertDestination, but for checking out
+ * HashTables. A worker thread can check out a hash table for insertion,
+ * perform the insertions and return the hash table to the pool. While
+ * one thread is using a hash table, no other thread can access it.
+ **/
+class HashTablePool {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param estimated_num_entries The maximum number of entries in a hash table.
+ * @param hash_table_impl_type The type of hash table implementation.
+ * @param group_by_types A vector of pointer of types which form the group by
+ * key.
+ * @param agg_handle The aggregation handle.
+ * @param storage_manager A pointer to the storage manager.
+ *
+ * @note The estimate of number of entries is quite inaccurate at this time.
+ * If we go by the current estimate, each hash table demands much
+ * larger space than it actually needs, which causes the system to
+ * either trigger evictions or worse - run out of memory. To fix this
+ * issue, we divide the estimate by 100. The division will not affect
+ * correctness, however it may allocate some hash tables smaller space
+ * than their requirement, causing them to be resized during build
+ * phase, which has a performance penalty.
+ **/
+ HashTablePool(const std::size_t estimated_num_entries,
+ const HashTableImplType hash_table_impl_type,
+ const std::vector<const Type *> &group_by_types,
+ AggregationHandle *agg_handle,
+ StorageManager *storage_manager)
+ : estimated_num_entries_(reduceEstimatedCardinality(estimated_num_entries)),
+ hash_table_impl_type_(hash_table_impl_type),
+ group_by_types_(group_by_types),
+ agg_handle_(DCHECK_NOTNULL(agg_handle)),
+ storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
+
+ /**
+ * @brief Check out a hash table for insertion.
+ *
+ * @return A hash table pointer.
+ **/
+ AggregationStateHashTableBase* getHashTable() {
+ {
+ SpinMutexLock lock(mutex_);
+ if (!hash_tables_.empty()) {
+ std::unique_ptr<AggregationStateHashTableBase> ret_hash_table(
+ std::move(hash_tables_.back()));
+ hash_tables_.pop_back();
+ DCHECK(ret_hash_table != nullptr);
+ return ret_hash_table.release();
+ }
+ }
+ return createNewHashTable();
+ }
+
+ /**
+ * @brief Return a previously checked out hash table.
+ *
+ * @param hash_table A pointer to the checked out hash table.
+ **/
+ void returnHashTable(AggregationStateHashTableBase *hash_table) {
+ SpinMutexLock lock(mutex_);
+ hash_tables_.push_back(
+ std::unique_ptr<AggregationStateHashTableBase>(hash_table));
+ }
+
+ /**
+ * @brief Get all the hash tables from the pool.
+ *
+ * @warning The caller should ensure that this call is made when no hash table
+ * is being checked in or checked out from the pool. In other words
+ * the hash table pool is in read-only state.
+ *
+ * @param All the hash tables in the pool.
+ *
+ **/
+ const std::vector<std::unique_ptr<AggregationStateHashTableBase>>*
+ getAllHashTables() {
+ return &hash_tables_;
+ }
+
+ private:
+ AggregationStateHashTableBase* createNewHashTable() {
+ return agg_handle_->createGroupByHashTable(hash_table_impl_type_,
+ group_by_types_,
+ estimated_num_entries_,
+ storage_manager_);
+ }
+
+ inline std::size_t reduceEstimatedCardinality(
+ const std::size_t original_estimate) const {
+ if (original_estimate < kEstimateReductionFactor) {
+ return original_estimate;
+ } else {
+ DCHECK_GT(kEstimateReductionFactor, 0u);
+ return original_estimate / kEstimateReductionFactor;
+ }
+ }
+
+ static constexpr std::size_t kEstimateReductionFactor = 100;
+
+ std::vector<std::unique_ptr<AggregationStateHashTableBase>> hash_tables_;
+
+ const std::size_t estimated_num_entries_;
+ const HashTableImplType hash_table_impl_type_;
+
+ const std::vector<const Type *> group_by_types_;
+
+ AggregationHandle *agg_handle_;
+ StorageManager *storage_manager_;
+
+ SpinMutex mutex_;
+
+ DISALLOW_COPY_AND_ASSIGN(HashTablePool);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_STORAGE_HASH_TABLE_POOL_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13281d86/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index dfc95b8..5d91052 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -183,7 +183,8 @@ StorageManager::~StorageManager() {
it != blocks_.end();
++it) {
if (it->second.block->isDirty()) {
- LOG(WARNING) << "Block with ID " << BlockIdUtil::ToString(it->first)
+ LOG(WARNING) << (it->second.block->isBlob() ? "Blob " : "Block ")
+ << "with ID " << BlockIdUtil::ToString(it->first)
<< " is dirty during StorageManager shutdown";
}
delete it->second.block;
[23/48] incubator-quickstep git commit: Added a quick fix to the
count(*) problem (#220)
Posted by hb...@apache.org.
Added a quick fix to the count(*) problem (#220)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/7131b0cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/7131b0cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/7131b0cd
Branch: refs/heads/query-manager-used-in-foreman
Commit: 7131b0cdecfd691f214f61dd1a9a02214ddc5b0a
Parents: 06f73ca
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Tue May 10 17:12:31 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:43 2016 -0700
----------------------------------------------------------------------
query_optimizer/physical/Aggregate.cpp | 14 ++++++++++++++
query_optimizer/tests/execution_generator/Select.test | 9 +++------
2 files changed, 17 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7131b0cd/query_optimizer/physical/Aggregate.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/Aggregate.cpp b/query_optimizer/physical/Aggregate.cpp
index 9bf606c..c582bba 100644
--- a/query_optimizer/physical/Aggregate.cpp
+++ b/query_optimizer/physical/Aggregate.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -70,6 +72,18 @@ std::vector<E::AttributeReferencePtr> Aggregate::getReferencedAttributes()
referenced_attributes_in_predicate.begin(),
referenced_attributes_in_predicate.end());
}
+
+ // TODO(jianqiao): This is a quick fix to the COUNT(*) problem that we retain
+ // at least one column from the child plan to survive the PruneColumns physical
+ // optimization. The comprehensive approach should also try to optimize the
+ // child plan with regard to the knowledge that the aggregation actually don't
+ // care about the values of the child plan's output but needs only the number
+ // of rows of the output.
+ if (referenced_attributes.size() == 0) {
+ DCHECK_GT(input_->getOutputAttributes().size(), static_cast<std::size_t>(0));
+ referenced_attributes.emplace_back(input_->getOutputAttributes()[0]);
+ }
+
return referenced_attributes;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7131b0cd/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index 023ad2e..05f7108 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -464,8 +464,6 @@ WHERE a.int_col = b.int_col
+-----------+-----------+---------------+-----------------------+---------------+------------------------+-----------+
==
-# FIXME(qzeng): Inner nested-loops join has no output column, causing query
-# result to be INCORRECT.
SELECT COUNT(*)
FROM
(SELECT a.float_col
@@ -476,7 +474,7 @@ FROM
+--------------------+
|COUNT(*) |
+--------------------+
-| 0|
+| 575|
+--------------------+
==
@@ -941,15 +939,14 @@ ORDER BY x;
+-----------+-----------+
==
-# TODO(team): Fix Issue #9 to enable COUNT(*).
-SELECT COUNT(long_col)
+SELECT COUNT(*)
FROM test,
(SELECT AVG(long_col) a FROM test) subquery
WHERE double_col < 0
AND long_col > subquery.a;
--
+--------------------+
-|COUNT(long_col) |
+|COUNT(*) |
+--------------------+
| 5|
+--------------------+
[25/48] incubator-quickstep git commit: Minor improvements in
FileManagers. (#224)
Posted by hb...@apache.org.
Minor improvements in FileManagers. (#224)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4608aa29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4608aa29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4608aa29
Branch: refs/heads/query-manager-used-in-foreman
Commit: 4608aa299ec2f59f03d4d4180fba91f8ec4967ad
Parents: 847941d
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Tue May 17 14:26:19 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:43 2016 -0700
----------------------------------------------------------------------
storage/CMakeLists.txt | 21 ++++++++------
storage/FileManager.hpp | 9 ++----
storage/FileManagerHdfs.cpp | 57 ++++++++++++++++++-------------------
storage/FileManagerPosix.cpp | 55 +++++++++++++++++------------------
storage/FileManagerPosix.hpp | 6 ++--
storage/FileManagerWindows.cpp | 53 +++++++++++++++++-----------------
storage/FileManagerWindows.hpp | 4 +--
7 files changed, 103 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4608aa29/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 115248c..26a3e32 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -585,20 +585,22 @@ target_link_libraries(quickstep_storage_FileManager
quickstep_utility_Macros
quickstep_utility_StringUtil)
if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
-target_link_libraries(quickstep_storage_FileManagerHdfs
- gflags_nothreads-static
- quickstep_storage_FileManager
- quickstep_storage_StorageBlockInfo
- quickstep_storage_StorageConstants
- quickstep_storage_StorageErrors
- quickstep_utility_Macros
- quickstep_utility_StringUtil
- ${LIBHDFS3_LIBRARIES})
+ target_link_libraries(quickstep_storage_FileManagerHdfs
+ glog
+ gflags_nothreads-static
+ quickstep_storage_FileManager
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_storage_StorageErrors
+ quickstep_utility_Macros
+ quickstep_utility_StringUtil
+ ${LIBHDFS3_LIBRARIES})
endif()
if (QUICKSTEP_HAVE_FILE_MANAGER_POSIX)
target_link_libraries(quickstep_storage_FileManagerLocal
quickstep_storage_FileManagerPosix)
target_link_libraries(quickstep_storage_FileManagerPosix
+ glog
quickstep_storage_FileManager
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageConstants
@@ -609,6 +611,7 @@ elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS)
target_link_libraries(quickstep_storage_FileManagerLocal
quickstep_storage_FileManagerWindows)
target_link_libraries(quickstep_storage_FileManagerWindows
+ glog
quickstep_storage_FileManager
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageConstants
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4608aa29/storage/FileManager.hpp
----------------------------------------------------------------------
diff --git a/storage/FileManager.hpp b/storage/FileManager.hpp
index d56c960..b179071 100644
--- a/storage/FileManager.hpp
+++ b/storage/FileManager.hpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,10 +19,7 @@
#define QUICKSTEP_STORAGE_FILE_MANAGER_HPP_
#include <cstddef>
-#include <cstdint>
-#include <cstdio>
#include <string>
-#include <utility>
#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
@@ -49,12 +46,12 @@ class FileManager {
* @param block_domain The domain of a block id.
**/
explicit FileManager(const std::string &storage_path)
- : storage_path_(storage_path) { }
+ : storage_path_(storage_path) {}
/**
* @brief Virtual destructor.
**/
- virtual ~FileManager() { }
+ virtual ~FileManager() {}
/**
* @brief Get a block or blob's relative filename, which uses storage_path_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4608aa29/storage/FileManagerHdfs.cpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerHdfs.cpp b/storage/FileManagerHdfs.cpp
index 5f9706e..e8f048b 100644
--- a/storage/FileManagerHdfs.cpp
+++ b/storage/FileManagerHdfs.cpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -32,10 +32,10 @@
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageConstants.hpp"
#include "storage/StorageErrors.hpp"
-#include "utility/Macros.hpp"
#include "utility/StringUtil.hpp"
#include "gflags/gflags.h"
+#include "glog/logging.h"
using std::size_t;
using std::sscanf;
@@ -76,20 +76,19 @@ static const bool hdfs_num_replications_dummy
FileManagerHdfs::FileManagerHdfs(const string &storage_path)
: FileManager(storage_path) {
- DEBUG_ASSERT(hdfs_namenode_port_dummy);
- DEBUG_ASSERT(hdfs_num_replications_dummy);
+ DCHECK(hdfs_namenode_port_dummy);
+ DCHECK(hdfs_num_replications_dummy);
struct hdfsBuilder *builder = hdfsNewBuilder();
hdfsBuilderSetNameNode(builder, FLAGS_hdfs_namenode_host.c_str());
hdfsBuilderSetNameNodePort(builder, FLAGS_hdfs_namenode_port);
// hdfsBuilderConnect releases builder.
hdfs_ = hdfsBuilderConnect(builder);
- DEBUG_ASSERT(hdfs_ != nullptr);
+ DCHECK(hdfs_ != nullptr);
}
FileManagerHdfs::~FileManagerHdfs() {
- int status = hdfsDisconnect(hdfs_);
- DEBUG_ASSERT(status == 0);
+ CHECK_EQ(0, hdfsDisconnect(hdfs_));
}
block_id_counter FileManagerHdfs::getMaxUsedBlockCounter(const block_id_domain block_domain) const {
@@ -97,7 +96,7 @@ block_id_counter FileManagerHdfs::getMaxUsedBlockCounter(const block_id_domain b
hdfsFileInfo *file_infos = hdfsListDirectory(hdfs_, storage_path_.c_str(), &num_files);
if (file_infos == nullptr) {
if (errno != ENOENT) {
- LOG_WARNING("Failed to list file info with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to list file info with error: " << strerror(errno);
}
return 0;
}
@@ -113,9 +112,9 @@ block_id_counter FileManagerHdfs::getMaxUsedBlockCounter(const block_id_domain b
// NOTE(zuyu): mName looks like
// "/user/<username>/<storage_path_>/qsblk_<block_domain>_[0-9]*.qsb".
const char *filename = std::strrchr(file_infos[i].mName, '/');
- if (filename != nullptr
- && sscanf(filename, filename_pattern.c_str(), &counter) == 1
- && counter > counter_max) {
+ if (filename != nullptr &&
+ sscanf(filename, filename_pattern.c_str(), &counter) == 1 &&
+ counter > counter_max) {
counter_max = counter;
}
}
@@ -126,12 +125,12 @@ block_id_counter FileManagerHdfs::getMaxUsedBlockCounter(const block_id_domain b
}
size_t FileManagerHdfs::numSlots(const block_id block) const {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
hdfsFileInfo *file_info = hdfsGetPathInfo(hdfs_, filename.c_str());
if (file_info == nullptr) {
if (errno != ENOENT) {
- LOG_WARNING("Failed to get size of file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to get size of file " << filename << " with error: " << strerror(errno);
}
return 0;
}
@@ -147,12 +146,12 @@ size_t FileManagerHdfs::numSlots(const block_id block) const {
}
bool FileManagerHdfs::deleteBlockOrBlob(const block_id block) {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
if ((hdfsDelete(hdfs_, filename.c_str(), 0) == 0) || (errno == ENOENT)) {
return true;
} else {
- LOG_WARNING("Failed to delete file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to delete file " << filename << " with error: " << strerror(errno);
return false;
}
}
@@ -160,10 +159,10 @@ bool FileManagerHdfs::deleteBlockOrBlob(const block_id block) {
bool FileManagerHdfs::readBlockOrBlob(const block_id block,
void *buffer,
const size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
hdfsFile file_handle = hdfsOpenFile(hdfs_,
filename.c_str(),
@@ -172,7 +171,7 @@ bool FileManagerHdfs::readBlockOrBlob(const block_id block,
FLAGS_hdfs_num_replications,
kSlotSizeBytes);
if (file_handle == nullptr) {
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
@@ -183,17 +182,17 @@ bool FileManagerHdfs::readBlockOrBlob(const block_id block,
bytes_total += bytes;
} else if (bytes == -1) {
if (errno != EINTR) {
- LOG_WARNING("Failed to read file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to read file " << filename << " with error: " << strerror(errno);
break;
}
} else {
- LOG_WARNING("Failed to read file " << filename << " since EOF was reached unexpectedly");
+ LOG(ERROR) << "Failed to read file " << filename << " since EOF was reached unexpectedly";
break;
}
}
if (hdfsCloseFile(hdfs_, file_handle) != 0) {
- LOG_WARNING("Failed to close file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to close file " << filename << " with error: " << strerror(errno);
}
return (bytes_total == length);
@@ -202,10 +201,10 @@ bool FileManagerHdfs::readBlockOrBlob(const block_id block,
bool FileManagerHdfs::writeBlockOrBlob(const block_id block,
const void *buffer,
const size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
hdfsFile file_handle = hdfsOpenFile(hdfs_,
filename.c_str(),
@@ -214,7 +213,7 @@ bool FileManagerHdfs::writeBlockOrBlob(const block_id block,
FLAGS_hdfs_num_replications,
kSlotSizeBytes);
if (file_handle == nullptr) {
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
@@ -224,17 +223,17 @@ bool FileManagerHdfs::writeBlockOrBlob(const block_id block,
if (bytes > 0) {
bytes_total += bytes;
} else if (bytes == -1) {
- LOG_WARNING("Failed to write file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to write file " << filename << " with error: " << strerror(errno);
break;
}
}
if (hdfsSync(hdfs_, file_handle) != 0) {
- LOG_WARNING("Failed to sync file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to sync file " << filename << " with error: " << strerror(errno);
}
if (hdfsCloseFile(hdfs_, file_handle) != 0) {
- LOG_WARNING("Failed to close file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to close file " << filename << " with error: " << strerror(errno);
}
return (bytes_total == length);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4608aa29/storage/FileManagerPosix.cpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerPosix.cpp b/storage/FileManagerPosix.cpp
index 3bfb69d..0346f0d 100644
--- a/storage/FileManagerPosix.cpp
+++ b/storage/FileManagerPosix.cpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -35,9 +35,10 @@
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageConstants.hpp"
#include "storage/StorageErrors.hpp"
-#include "utility/Macros.hpp"
#include "utility/StringUtil.hpp"
+#include "glog/logging.h"
+
using std::size_t;
using std::sscanf;
using std::strerror;
@@ -64,9 +65,9 @@ block_id_counter FileManagerPosix::getMaxUsedBlockCounter(const block_id_domain
filename_pattern.append(".qsb");
block_id_counter counter_max = 0, counter;
- if (glob_result.gl_pathc > 0
- && sscanf(glob_result.gl_pathv[glob_result.gl_pathc - 1], filename_pattern.c_str(), &counter) == 1
- && counter > counter_max) {
+ if (glob_result.gl_pathc > 0 &&
+ sscanf(glob_result.gl_pathv[glob_result.gl_pathc - 1], filename_pattern.c_str(), &counter) == 1 &&
+ counter > counter_max) {
counter_max = counter;
}
@@ -75,12 +76,12 @@ block_id_counter FileManagerPosix::getMaxUsedBlockCounter(const block_id_domain
}
size_t FileManagerPosix::numSlots(const block_id block) const {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
struct stat file_stat;
if (stat(filename.c_str(), &file_stat) == -1) {
if (errno != ENOENT) {
- LOG_WARNING("Failed to retrieve info about file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to retrieve info about file " << filename << " with error: " << strerror(errno);
}
return 0;
}
@@ -93,12 +94,12 @@ size_t FileManagerPosix::numSlots(const block_id block) const {
}
bool FileManagerPosix::deleteBlockOrBlob(const block_id block) {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
if ((unlink(filename.c_str()) == 0) || (errno == ENOENT)) {
return true;
} else {
- LOG_WARNING("Failed to delete file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to delete file " << filename << " with error: " << strerror(errno);
return false;
}
}
@@ -106,35 +107,35 @@ bool FileManagerPosix::deleteBlockOrBlob(const block_id block) {
bool FileManagerPosix::readBlockOrBlob(const block_id block,
void *buffer,
const std::size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
- int fd = open(filename.c_str(), O_RDONLY);
+ const int fd = open(filename.c_str(), O_RDONLY);
if (fd == -1) {
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
size_t bytes_total = 0;
while (bytes_total < length) {
- ssize_t bytes = read(fd, static_cast<char*>(buffer) + bytes_total, length - bytes_total);
+ const ssize_t bytes = read(fd, static_cast<char*>(buffer) + bytes_total, length - bytes_total);
if (bytes > 0) {
bytes_total += bytes;
} else if (bytes == -1) {
if (errno != EINTR) {
- LOG_WARNING("Failed to read file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to read file " << filename << " with error: " << strerror(errno);
break;
}
} else {
- LOG_WARNING("Failed to read file " << filename << " since EOF was reached unexpectedly");
+ LOG(ERROR) << "Failed to read file " << filename << " since EOF was reached unexpectedly";
break;
}
}
if (close(fd) != 0) {
- LOG_WARNING("Failed to close file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to close file " << filename << " with error: " << strerror(errno);
}
return (bytes_total == length);
@@ -143,34 +144,34 @@ bool FileManagerPosix::readBlockOrBlob(const block_id block,
bool FileManagerPosix::writeBlockOrBlob(const block_id block,
const void *buffer,
const std::size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
- int fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
+ const int fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
if (fd == -1) {
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
size_t bytes_total = 0;
while (bytes_total < length) {
- ssize_t bytes = write(fd, static_cast<const char*>(buffer) + bytes_total, length - bytes_total);
+ const ssize_t bytes = write(fd, static_cast<const char*>(buffer) + bytes_total, length - bytes_total);
if (bytes > 0) {
bytes_total += bytes;
} else if (bytes == -1 && errno != EINTR) {
- LOG_WARNING("Failed to write file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to write file " << filename << " with error: " << strerror(errno);
break;
}
}
if (fsync(fd) != 0) {
- LOG_WARNING("Failed to sync file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to sync file " << filename << " with error: " << strerror(errno);
}
if (close(fd) != 0) {
- LOG_WARNING("Failed to close file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to close file " << filename << " with error: " << strerror(errno);
}
return (bytes_total == length);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4608aa29/storage/FileManagerPosix.hpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerPosix.hpp b/storage/FileManagerPosix.hpp
index d8c1649..b2aea27 100644
--- a/storage/FileManagerPosix.hpp
+++ b/storage/FileManagerPosix.hpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -37,9 +37,9 @@ namespace quickstep {
class FileManagerPosix : public FileManager {
public:
explicit FileManagerPosix(const std::string &storage_path)
- : FileManager(storage_path) { }
+ : FileManager(storage_path) {}
- ~FileManagerPosix() override { }
+ ~FileManagerPosix() override {}
bool deleteBlockOrBlob(const block_id block) override;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4608aa29/storage/FileManagerWindows.cpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerWindows.cpp b/storage/FileManagerWindows.cpp
index 9e3d4c8..b1763ec 100644
--- a/storage/FileManagerWindows.cpp
+++ b/storage/FileManagerWindows.cpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -34,9 +34,10 @@
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageConstants.hpp"
#include "storage/StorageErrors.hpp"
-#include "utility/Macros.hpp"
#include "utility/StringUtil.hpp"
+#include "glog/logging.h"
+
using std::size_t;
using std::strerror;
using std::string;
@@ -60,7 +61,7 @@ block_id_counter FileManagerWindows::getMaxUsedBlockCounter(const block_id_domai
if (find_handle == INVALID_HANDLE_VALUE) {
error_code = GetLastError();
if (error_code != ERROR_FILE_NOT_FOUND) {
- LOG_WARNING("Failed to retrieve blockfiles with error_code: " << error_code);
+ LOG(ERROR) << "Failed to retrieve blockfiles with error_code: " << error_code;
}
return 0;
}
@@ -73,32 +74,32 @@ block_id_counter FileManagerWindows::getMaxUsedBlockCounter(const block_id_domai
block_id_counter counter_max = 0, counter;
do {
- if (sscanf(find_data.cFileName, filename_pattern.c_str(), &counter) == 1
- && counter > counter_max) {
+ if (sscanf(find_data.cFileName, filename_pattern.c_str(), &counter) == 1 &&
+ counter > counter_max) {
counter_max = counter;
}
} while (FindNextFile(find_handle, &find_data) != 0);
error_code = GetLastError();
if (error_code != ERROR_NO_MORE_FILES) {
- LOG_WARNING("Failed to FindNextFile with error_code: " << error_code);
+ LOG(ERROR) << "Failed to FindNextFile with error_code: " << error_code;
}
if (FindClose(find_handle) == 0) {
- LOG_WARNING("Failed to close the file with error_code: " << GetLastError());
+ LOG(ERROR) << "Failed to close the file with error_code: " << GetLastError();
}
return counter_max;
}
size_t FileManagerWindows::numSlots(const block_id block) const {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
WIN32_FILE_ATTRIBUTE_DATA file_stat;
if (!GetFileAttributesEx(filename.c_str(), GetFileExInfoStandard, &file_stat)) {
DWORD error_code = GetLastError();
if (error_code != ERROR_FILE_NOT_FOUND) {
- LOG_WARNING("Failed to retrieve info about file " << filename << " with error_code: " << error_code);
+ LOG(ERROR) << "Failed to retrieve info about file " << filename << " with error_code: " << error_code;
}
return 0;
}
@@ -113,28 +114,28 @@ size_t FileManagerWindows::numSlots(const block_id block) const {
}
bool FileManagerWindows::deleteBlockOrBlob(const block_id block) {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
if ((DeleteFile(filename.c_str()) != 0) || (GetLastError() == ERROR_FILE_NOT_FOUND)) {
return true;
} else {
- LOG_WARNING("Failed to delete file " << filename << " with error_code: " << GetLastError());
+ LOG(ERROR) << "Failed to delete file " << filename << " with error_code: " << GetLastError();
}
}
bool FileManagerWindows::readBlockOrBlob(const block_id block,
void *buffer,
const size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
FILE *file = fopen(filename.c_str(), "rb");
if (file == nullptr) {
// Note: On most, but not all, library implementations, the errno variable
// is set to a system-specific error code on failure.
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
@@ -142,16 +143,16 @@ bool FileManagerWindows::readBlockOrBlob(const block_id block,
const bool result_is_ok = (bytes == length);
if (!result_is_ok) {
if (std::feof(file)) {
- LOG_WARNING("Failed to read file " << filename << " since EOF was reached unexpectedly");
+ LOG(ERROR) << "Failed to read file " << filename << " since EOF was reached unexpectedly";
} else {
- LOG_WARNING("Failed to read file " << filename << " with error: " << strerror(ferror(file)));
+ LOG(ERROR) << "Failed to read file " << filename << " with error: " << strerror(ferror(file));
clearerr(file);
}
}
if (fclose(file)) {
// Note: fclose does not set errno on failure.
- LOG_WARNING("Failed to close file " << filename);
+ LOG(ERROR) << "Failed to close file " << filename;
}
return result_is_ok;
@@ -160,37 +161,37 @@ bool FileManagerWindows::readBlockOrBlob(const block_id block,
bool FileManagerWindows::writeBlockOrBlob(const block_id block,
const void *buffer,
const size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
FILE *file = fopen(filename.c_str(), "wb");
if (file == nullptr) {
// Note: On most, but not all, library implementations, the errno variable
// is set to a system-specific error code on failure.
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
const size_t bytes = std::fwrite(buffer, sizeof(char), length, file);
const bool result_is_ok = (bytes == length);
if (!result_is_ok) {
- LOG_WARNING("Failed to write file " << filename << " with error: " << strerror(ferror(file)));
+ LOG(ERROR) << "Failed to write file " << filename << " with error: " << strerror(ferror(file));
clearerr(file);
}
if (fflush(file)) {
- LOG_WARNING("Failed to flush file " << filename << " with error: " << strerror(ferror(file)));
+ LOG(ERROR) << "Failed to flush file " << filename << " with error: " << strerror(ferror(file));
}
if (!FlushFileBuffers(reinterpret_cast<HANDLE>(_get_osfhandle(_fileno(file))))) {
- LOG_WARNING("Failed to re-flush file " << filename << " with error: " << strerror(ferror(file)));
+ LOG(ERROR) << "Failed to re-flush file " << filename << " with error: " << strerror(ferror(file));
}
if (fclose(file)) {
// Note: fclose does not set errno on failure.
- LOG_WARNING("Failed to close file " << filename);
+ LOG(ERROR) << "Failed to close file " << filename;
}
return result_is_ok;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4608aa29/storage/FileManagerWindows.hpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerWindows.hpp b/storage/FileManagerWindows.hpp
index 085b418..e7b3aa8 100644
--- a/storage/FileManagerWindows.hpp
+++ b/storage/FileManagerWindows.hpp
@@ -37,9 +37,9 @@ namespace quickstep {
class FileManagerWindows : public FileManager {
public:
explicit FileManagerWindows(const std::string &storage_path)
- : FileManager(storage_path) { }
+ : FileManager(storage_path) {}
- ~FileManagerWindows() override { }
+ ~FileManagerWindows() override {}
bool deleteBlockOrBlob(const block_id block) override;
[09/48] incubator-quickstep git commit: Cleanup in Preloader class
(#209)
Posted by hb...@apache.org.
Cleanup in Preloader class (#209)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/cb763bab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/cb763bab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/cb763bab
Branch: refs/heads/query-manager-used-in-foreman
Commit: cb763bab42fdff5ba407490c60094b7679b04c8e
Parents: 1a685be
Author: Harshad Deshmukh <d....@gmail.com>
Authored: Fri May 6 22:29:41 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:41 2016 -0700
----------------------------------------------------------------------
storage/PreloaderThread.cpp | 44 ++++++++++++++++++++++++----------------
storage/PreloaderThread.hpp | 7 +++++++
2 files changed, 34 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cb763bab/storage/PreloaderThread.cpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.cpp b/storage/PreloaderThread.cpp
index 8f600b8..4bfa9c5 100644
--- a/storage/PreloaderThread.cpp
+++ b/storage/PreloaderThread.cpp
@@ -49,37 +49,46 @@ void PreloaderThread::run() {
std::size_t blocks_loaded = 0;
for (const CatalogRelation &relation : database_) {
- if (relation.hasPartitionScheme()) {
+ if (relation.hasPartitionScheme() && relation.hasNUMAPlacementScheme()) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
blocks_loaded += preloadNUMAAware(relation, blocks_loaded, num_slots);
+#endif
} else {
+ // NUMA agnostic preloading of relation.
std::vector<block_id> blocks = relation.getBlocksSnapshot();
for (block_id current_block_id : blocks) {
try {
- BlockReference current_block = storage_manager_->getBlock(current_block_id, relation);
+ BlockReference current_block =
+ storage_manager_->getBlock(current_block_id, relation);
} catch (...) {
- LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks\n";
+ LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks";
throw;
}
++blocks_loaded;
if (blocks_loaded == num_slots) {
- // The buffer pool has filled up. But, some database blocks are not loaded.
- printf(" The database is larger than the buffer pool. Only %lu blocks were loaded ",
- blocks_loaded);
+ // The buffer pool has filled up. But, some database blocks are not
+ // loaded.
+ printf(
+ " The database is larger than the buffer pool. Only %lu blocks "
+ "were loaded ", blocks_loaded);
return;
}
}
+ LOG(INFO) << "Relation " << relation.getName()
+ << " completely preloaded in buffer pool";
}
}
printf(" Loaded %lu blocks ", blocks_loaded);
}
+#ifdef QUICKSTEP_HAVE_LIBNUMA
std::size_t PreloaderThread::preloadNUMAAware(
const CatalogRelation &relation,
const std::size_t num_previously_loaded_blocks,
const std::size_t num_slots) {
-#ifdef QUICKSTEP_HAVE_LIBNUMA
std::size_t blocks_loaded = 0;
- const NUMAPlacementScheme *placement_scheme = relation.getNUMAPlacementSchemePtr();
+ const NUMAPlacementScheme *placement_scheme =
+ relation.getNUMAPlacementSchemePtr();
DCHECK(placement_scheme != nullptr);
DCHECK(relation.hasPartitionScheme());
const PartitionScheme &part_scheme = relation.getPartitionScheme();
@@ -96,15 +105,17 @@ std::size_t PreloaderThread::preloadNUMAAware(
BlockReference current_block = storage_manager_->getBlock(
curr_block_id, relation, partition_numa_node_id);
} catch (...) {
- LOG(ERROR) << "Error after loading "
+ LOG(ERROR) << "Error while preloading: After loading total "
<< blocks_loaded + num_previously_loaded_blocks
- << " blocks\n";
+ << " blocks and " << blocks_loaded
+ << " blocks of relation " << relation.getName();
throw;
}
++blocks_loaded;
num_blocks_loaded[partition_numa_node_id]++;
if ((blocks_loaded + num_previously_loaded_blocks) == num_slots) {
- // The buffer pool has filled up. But, some database blocks are not loaded.
+ // The buffer pool has filled up. But, some database blocks are not
+ // loaded.
printf(
" The database is larger than the buffer pool. Only %lu blocks "
"were loaded ",
@@ -116,14 +127,13 @@ std::size_t PreloaderThread::preloadNUMAAware(
LOG(INFO) << "For relation: " << relation.getName();
for (auto numa_block_loaded_info : num_blocks_loaded) {
LOG(INFO) << "NUMA node: " << numa_block_loaded_info.first
- << " Number of loaded blocks: " << numa_block_loaded_info.second;
+ << " Number of loaded blocks: "
+ << numa_block_loaded_info.second;
}
+ LOG(INFO) << "Relation " << relation.getName()
+ << " completely preloaded in buffer pool in a NUMA aware fashion";
return blocks_loaded;
-#else
- LOG(INFO) << "Relation: " << relation.getName()
- << " has partition scheme but the system doesn't support NUMA";
- return 0;
-#endif
}
+#endif
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cb763bab/storage/PreloaderThread.hpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.hpp b/storage/PreloaderThread.hpp
index f16fd50..16df34a 100644
--- a/storage/PreloaderThread.hpp
+++ b/storage/PreloaderThread.hpp
@@ -20,6 +20,7 @@
#include <cstddef>
+#include "storage/StorageConfig.h"
#include "threading/Thread.hpp"
#include "utility/Macros.hpp"
@@ -68,6 +69,7 @@ class PreloaderThread : public Thread {
void run() override;
private:
+#ifdef QUICKSTEP_HAVE_LIBNUMA
/**
* @brief Preload a relation which has a partition and a NUMA placement scheme.
*
@@ -82,11 +84,16 @@ class PreloaderThread : public Thread {
* allocated sufficient amount of memory so as not to exceed that
* socket's memory limit.
*
+ * TODO(harshad) - Allow multiple preloader threads, each pinned to a NUMA
+ * socket in the system. Each thread should preload only its share of
+ * storage blocks.
+ *
* @return The number of blocks loaded during this function call.
**/
std::size_t preloadNUMAAware(const CatalogRelation &relation,
const std::size_t num_previously_loaded_blocks,
const std::size_t num_slots);
+#endif
const CatalogDatabase &database_;
StorageManager *storage_manager_;
[32/48] incubator-quickstep git commit: Visualize optimized physical
plan in DOT format (#232)
Posted by hb...@apache.org.
Visualize optimized physical plan in DOT format (#232)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/12d7928b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/12d7928b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/12d7928b
Branch: refs/heads/query-manager-used-in-foreman
Commit: 12d7928b8d505c5993ee7b0224b51460a6a3f2b6
Parents: 5e2bb51
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Fri May 20 16:47:31 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:45 2016 -0700
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 3 +-
query_optimizer/PhysicalGenerator.cpp | 13 ++-
utility/CMakeLists.txt | 13 +++
utility/PlanVisualizer.cpp | 161 +++++++++++++++++++++++++++++
utility/PlanVisualizer.hpp | 94 +++++++++++++++++
utility/StringUtil.hpp | 30 +++++-
6 files changed, 310 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/12d7928b/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index aa2873e..5c9438d 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -194,7 +194,8 @@ target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
quickstep_queryoptimizer_strategy_Selection
quickstep_queryoptimizer_strategy_Strategy
quickstep_queryoptimizer_Validator
- quickstep_utility_Macros)
+ quickstep_utility_Macros
+ quickstep_utility_PlanVisualizer)
target_link_libraries(quickstep_queryoptimizer_QueryHandle
quickstep_catalog_Catalog_proto
quickstep_queryexecution_QueryContext_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/12d7928b/query_optimizer/PhysicalGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/PhysicalGenerator.cpp b/query_optimizer/PhysicalGenerator.cpp
index 662236f..75a7bc9 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -33,6 +33,7 @@
#include "query_optimizer/strategy/OneToOne.hpp"
#include "query_optimizer/strategy/Selection.hpp"
#include "query_optimizer/strategy/Strategy.hpp"
+#include "utility/PlanVisualizer.hpp"
#include "gflags/gflags.h"
@@ -45,7 +46,12 @@ DEFINE_bool(reorder_hash_joins, true,
"If true, apply hash join order optimization to each group of hash "
"joins. The optimization applies a greedy algorithm to favor smaller "
"cardinality and selective tables to be joined first, which is suitable "
- "for queries on star-schema tables");
+ "for queries on star-schema tables.");
+
+DEFINE_bool(visualize_plan, false,
+ "If true, visualize the final physical plan into a graph in DOT format "
+ "(DOT is a plain text graph description language). Then print the "
+ "generated graph through stderr.");
namespace L = ::quickstep::optimizer::logical;
namespace P = ::quickstep::optimizer::physical;
@@ -101,6 +107,11 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
DVLOG(4) << "Optimized physical plan:\n" << physical_plan_->toString();
+ if (FLAGS_visualize_plan) {
+ quickstep::PlanVisualizer plan_visualizer;
+ std::cerr << "\n" << plan_visualizer.visualize(physical_plan_) << "\n";
+ }
+
#ifdef QUICKSTEP_DEBUG
Validate(physical_plan_);
#endif
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/12d7928b/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index 6d1eeab..2d3db8f 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -171,6 +171,7 @@ add_library(quickstep_utility_Glob Glob.cpp Glob.hpp)
add_library(quickstep_utility_HashPair ../empty_src.cpp HashPair.hpp)
add_library(quickstep_utility_Macros ../empty_src.cpp Macros.hpp)
add_library(quickstep_utility_MemStream ../empty_src.cpp MemStream.hpp)
+add_library(quickstep_utility_PlanVisualizer PlanVisualizer.cpp PlanVisualizer.hpp)
add_library(quickstep_utility_PrimeNumber PrimeNumber.cpp PrimeNumber.hpp)
add_library(quickstep_utility_PtrList ../empty_src.cpp PtrList.hpp)
add_library(quickstep_utility_PtrMap ../empty_src.cpp PtrMap.hpp)
@@ -231,6 +232,17 @@ target_link_libraries(quickstep_utility_MemStream
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_PrimeNumber
glog)
+target_link_libraries(quickstep_utility_PlanVisualizer
+ quickstep_catalog_CatalogRelation
+ quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_queryoptimizer_physical_TableReference
+ quickstep_queryoptimizer_physical_TopLevelPlan
+ quickstep_utility_Macros
+ quickstep_utility_StringUtil)
target_link_libraries(quickstep_utility_PtrList
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_PtrMap
@@ -295,6 +307,7 @@ target_link_libraries(quickstep_utility
quickstep_utility_HashPair
quickstep_utility_Macros
quickstep_utility_MemStream
+ quickstep_utility_PlanVisualizer
quickstep_utility_PrimeNumber
quickstep_utility_PtrList
quickstep_utility_PtrMap
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/12d7928b/utility/PlanVisualizer.cpp
----------------------------------------------------------------------
diff --git a/utility/PlanVisualizer.cpp b/utility/PlanVisualizer.cpp
new file mode 100644
index 0000000..962d577
--- /dev/null
+++ b/utility/PlanVisualizer.cpp
@@ -0,0 +1,161 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "utility/PlanVisualizer.hpp"
+
+#include <cstddef>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "catalog/CatalogRelation.hpp"
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/PhysicalType.hpp"
+#include "query_optimizer/physical/TableReference.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+#include "utility/StringUtil.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+namespace E = ::quickstep::optimizer::expressions;
+namespace P = ::quickstep::optimizer::physical;
+namespace C = ::quickstep::optimizer::cost;
+
+std::string PlanVisualizer::visualize(const P::PhysicalPtr &input) {
+ DCHECK(input->getPhysicalType() == P::PhysicalType::kTopLevelPlan);
+ cost_model_.reset(
+ new C::StarSchemaSimpleCostModel(
+ std::static_pointer_cast<const P::TopLevelPlan>(input)->shared_subplans()));
+
+ color_map_["TableReference"] = "skyblue";
+ color_map_["Selection"] = "#90EE90";
+ color_map_["HashJoin"] = "red";
+ color_map_["HashLeftOuterJoin"] = "orange";
+ color_map_["HashLeftSemiJoin"] = "orange";
+ color_map_["HashLeftAntiJoin"] = "orange";
+
+ visit(input);
+
+ // Format output graph
+ std::ostringstream graph_oss;
+ graph_oss << "digraph g {\n";
+ graph_oss << " rankdir=BT\n";
+ graph_oss << " node [penwidth=2]\n";
+ graph_oss << " edge [fontsize=16 fontcolor=gray penwidth=2]\n\n";
+
+ // Format nodes
+ for (const NodeInfo &node_info : nodes_) {
+ graph_oss << " " << node_info.id << " [";
+ if (!node_info.labels.empty()) {
+ graph_oss << "label=\""
+ << EscapeSpecialChars(JoinToString(node_info.labels, " "))
+ << "\"";
+ }
+ if (!node_info.color.empty()) {
+ graph_oss << " style=filled fillcolor=\"" << node_info.color << "\"";
+ }
+ graph_oss << "]\n";
+ }
+ graph_oss << "\n";
+
+ // Format edges
+ for (const EdgeInfo &edge_info : edges_) {
+ graph_oss << " " << edge_info.src_node_id << " -> "
+ << edge_info.dst_node_id << " [";
+ if (!edge_info.labels.empty()) {
+ graph_oss << "label=\""
+ << EscapeSpecialChars(JoinToString(edge_info.labels, " "))
+ << "\"";
+ }
+ graph_oss << "]\n";
+ }
+
+ graph_oss << "}\n";
+
+ return graph_oss.str();
+}
+
+void PlanVisualizer::visit(const P::PhysicalPtr &input) {
+ int node_id = ++id_counter_;
+ node_id_map_.emplace(input, node_id);
+
+ for (const auto &child : input->children()) {
+ visit(child);
+
+ int child_id = node_id_map_[child];
+
+ edges_.emplace_back(EdgeInfo());
+ EdgeInfo &edge_info = edges_.back();
+ edge_info.src_node_id = child_id;
+ edge_info.dst_node_id = node_id;
+
+ // Print output attributes except for TableReference -- there are just too many
+ // attributes out of TableReference.
+ if (child->getPhysicalType() != P::PhysicalType::kTableReference) {
+ for (const auto &attr : child->getOutputAttributes()) {
+ edge_info.labels.emplace_back(attr->attribute_alias());
+ }
+ }
+ }
+
+ nodes_.emplace_back(NodeInfo());
+ NodeInfo &node_info = nodes_.back();
+ node_info.id = node_id;
+ if (color_map_.find(input->getName()) != color_map_.end()) {
+ node_info.color = color_map_[input->getName()];
+ }
+
+ switch (input->getPhysicalType()) {
+ case P::PhysicalType::kTableReference: {
+ const P::TableReferencePtr table_reference =
+ std::static_pointer_cast<const P::TableReference>(input);
+ node_info.labels.emplace_back(table_reference->relation()->getName());
+ break;
+ }
+ case P::PhysicalType::kHashJoin: {
+ const P::HashJoinPtr hash_join =
+ std::static_pointer_cast<const P::HashJoin>(input);
+ node_info.labels.emplace_back(input->getName());
+
+ const auto &left_attributes = hash_join->left_join_attributes();
+ const auto &right_attributes = hash_join->right_join_attributes();
+ for (std::size_t i = 0; i < left_attributes.size(); ++i) {
+ node_info.labels.emplace_back(
+ left_attributes[i]->attribute_alias() + " = " + right_attributes[i]->attribute_alias());
+ }
+ break;
+ }
+ default: {
+ node_info.labels.emplace_back(input->getName());
+ break;
+ }
+ }
+ node_info.labels.emplace_back(
+ "est. # = " + std::to_string(cost_model_->estimateCardinality(input)));
+ node_info.labels.emplace_back(
+ "est. Selectivity = " + std::to_string(cost_model_->estimateSelectivity(input)));
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/12d7928b/utility/PlanVisualizer.hpp
----------------------------------------------------------------------
diff --git a/utility/PlanVisualizer.hpp b/utility/PlanVisualizer.hpp
new file mode 100644
index 0000000..080b7de
--- /dev/null
+++ b/utility/PlanVisualizer.hpp
@@ -0,0 +1,94 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_UTILITY_PLAN_VISUALIZER_HPP_
+#define QUICKSTEP_UTILITY_PLAN_VISUALIZER_HPP_
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+/** \addtogroup Utility
+ * @{
+ */
+
+/**
+ * @brief A query plan visualizer that converts a physical plan into a graph in
+ * DOT format. Note that DOT is a plain text graph description language.
+ *
+ * @note This utility tool can be further extended to be more generic.
+ */
+class PlanVisualizer {
+ public:
+ PlanVisualizer()
+ : id_counter_(0) {}
+
+ ~PlanVisualizer() {}
+
+ /**
+ * @brief Visualize the query plan into a graph in DOT format (DOT is a plain
+ * text graph description language).
+ *
+ * @return The visualized query plan graph in DOT format.
+ */
+ std::string visualize(const optimizer::physical::PhysicalPtr &input);
+
+ private:
+ /**
+ * @brief Information of a graph node.
+ */
+ struct NodeInfo {
+ int id;
+ std::vector<std::string> labels;
+ std::string color;
+ };
+
+ /**
+ * @brief Information of a graph edge.
+ */
+ struct EdgeInfo {
+ int src_node_id;
+ int dst_node_id;
+ std::vector<std::string> labels;
+ };
+
+ void visit(const optimizer::physical::PhysicalPtr &input);
+
+ int id_counter_;
+ std::unordered_map<optimizer::physical::PhysicalPtr, int> node_id_map_;
+ std::unordered_map<std::string, std::string> color_map_;
+
+ std::vector<NodeInfo> nodes_;
+ std::vector<EdgeInfo> edges_;
+
+ std::unique_ptr<optimizer::cost::StarSchemaSimpleCostModel> cost_model_;
+
+ DISALLOW_COPY_AND_ASSIGN(PlanVisualizer);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif /* QUICKSTEP_UTILITY_PLAN_VISUALIZER_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/12d7928b/utility/StringUtil.hpp
----------------------------------------------------------------------
diff --git a/utility/StringUtil.hpp b/utility/StringUtil.hpp
index bd138bb..6477ded 100644
--- a/utility/StringUtil.hpp
+++ b/utility/StringUtil.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,6 +21,7 @@
#define QUICKSTEP_UTILITY_STRING_UTIL_HPP_
#include <cstdint>
+#include <sstream>
#include <string>
#include <vector>
@@ -34,7 +37,7 @@ namespace quickstep {
* @param str The string to be converted.
* @return The converted string with all lower case characters bing converted to upper case characters.
*/
-extern std::string ToLower(const std::string& str);
+extern std::string ToLower(const std::string &str);
/**
* @brief Converts special characters to escape characters.
@@ -42,7 +45,30 @@ extern std::string ToLower(const std::string& str);
* @param text The string to be unescaped.
* @return Unescaped string.
*/
-extern std::string EscapeSpecialChars(const std::string& text);
+extern std::string EscapeSpecialChars(const std::string &text);
+
+/**
+ * @brief Join all objects in a iterable container into a single string. The object
+ * must have implemented the operator<< overloading with std::stringstream.
+ *
+ * @param container The iterable container of objects.
+ * @param separator A string to separate each object.
+ */
+template <typename ContainerType>
+std::string JoinToString(const ContainerType &container,
+ const std::string &separator) {
+ std::ostringstream oss;
+ bool is_first = true;
+ for (const auto &item : container) {
+ if (is_first) {
+ is_first = false;
+ } else {
+ oss << separator;
+ }
+ oss << item;
+ }
+ return oss.str();
+}
/**
* @brief Parse a string of base-10 integers separated by delimiter characters
[13/48] incubator-quickstep git commit: Change default
aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining (#210)
Posted by hb...@apache.org.
Change default aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining (#210)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9e2fb82e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9e2fb82e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9e2fb82e
Branch: refs/heads/query-manager-used-in-foreman
Commit: 9e2fb82e647053e21d0c87444b8ca90c0b5f1adb
Parents: fc9c3bf
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 5 09:58:45 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:41 2016 -0700
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 69 ++++++++++++--------
.../tests/execution_generator/Select.test | 23 +++----
storage/AggregationOperationState.cpp | 28 +++++++-
storage/AggregationOperationState.hpp | 5 ++
storage/AggregationOperationState.proto | 5 ++
5 files changed, 90 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9e2fb82e/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index c34f084..077d35d 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -135,7 +135,7 @@ static const volatile bool join_hashtable_type_dummy
= gflags::RegisterFlagValidator(&FLAGS_join_hashtable_type,
&ValidateHashTableImplTypeString);
-DEFINE_string(aggregate_hashtable_type, "LinearOpenAddressing",
+DEFINE_string(aggregate_hashtable_type, "SeparateChaining",
"HashTable implementation to use for aggregates with GROUP BY "
"(valid options are SeparateChaining or LinearOpenAddressing)");
static const volatile bool aggregate_hashtable_type_dummy
@@ -1285,6 +1285,35 @@ void ExecutionGenerator::convertAggregate(
findRelationInfoOutputByPhysical(physical_plan->input());
aggr_state_proto->set_relation_id(input_relation_info->relation->getID());
+ std::vector<const Type*> group_by_types;
+ for (const E::NamedExpressionPtr &grouping_expression : physical_plan->grouping_expressions()) {
+ unique_ptr<const Scalar> execution_group_by_expression;
+ E::AliasPtr alias;
+ if (E::SomeAlias::MatchesWithConditionalCast(grouping_expression, &alias)) {
+ E::ScalarPtr scalar;
+ // NOTE(zuyu): For aggregate expressions, all child expressions of an
+ // Alias should be a Scalar.
+ CHECK(E::SomeScalar::MatchesWithConditionalCast(alias->expression(), &scalar))
+ << alias->toString();
+ execution_group_by_expression.reset(scalar->concretize(attribute_substitution_map_));
+ } else {
+ execution_group_by_expression.reset(
+ grouping_expression->concretize(attribute_substitution_map_));
+ }
+ aggr_state_proto->add_group_by_expressions()->CopyFrom(execution_group_by_expression->getProto());
+ group_by_types.push_back(&execution_group_by_expression->getType());
+ }
+
+ if (!group_by_types.empty()) {
+ // SimplifyHashTableImplTypeProto() switches the hash table implementation
+ // from SeparateChaining to SimpleScalarSeparateChaining when there is a
+ // single scalar key type with a reversible hash function.
+ aggr_state_proto->set_hash_table_impl_type(
+ SimplifyHashTableImplTypeProto(
+ HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
+ group_by_types));
+ }
+
for (const E::AliasPtr &named_aggregate_expression : physical_plan->aggregate_expressions()) {
const E::AggregateFunctionPtr unnamed_aggregate_expression =
std::static_pointer_cast<const E::AggregateFunction>(named_aggregate_expression->expression());
@@ -1304,25 +1333,19 @@ void ExecutionGenerator::convertAggregate(
// Set whether it is a DISTINCT aggregation.
aggr_proto->set_is_distinct(unnamed_aggregate_expression->is_distinct());
- }
- std::vector<const Type*> group_by_types;
- for (const E::NamedExpressionPtr &grouping_expression : physical_plan->grouping_expressions()) {
- unique_ptr<const Scalar> execution_group_by_expression;
- E::AliasPtr alias;
- if (E::SomeAlias::MatchesWithConditionalCast(grouping_expression, &alias)) {
- E::ScalarPtr scalar;
- // NOTE(zuyu): For aggregate expressions, all child expressions of an
- // Alias should be a Scalar.
- CHECK(E::SomeScalar::MatchesWithConditionalCast(alias->expression(), &scalar))
- << alias->toString();
- execution_group_by_expression.reset(scalar->concretize(attribute_substitution_map_));
- } else {
- execution_group_by_expression.reset(
- grouping_expression->concretize(attribute_substitution_map_));
+ // Add distinctify hash table impl type if it is a DISTINCT aggregation.
+ if (unnamed_aggregate_expression->is_distinct()) {
+ if (group_by_types.empty()) {
+ aggr_state_proto->add_distinctify_hash_table_impl_types(
+ SimplifyHashTableImplTypeProto(
+ HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
+ {&unnamed_aggregate_expression->getValueType()}));
+ } else {
+ aggr_state_proto->add_distinctify_hash_table_impl_types(
+ HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type));
+ }
}
- aggr_state_proto->add_group_by_expressions()->CopyFrom(execution_group_by_expression->getProto());
- group_by_types.push_back(&execution_group_by_expression->getType());
}
if (physical_plan->filter_predicate() != nullptr) {
@@ -1332,16 +1355,6 @@ void ExecutionGenerator::convertAggregate(
aggr_state_proto->set_estimated_num_entries(cost_model_->estimateCardinality(physical_plan));
- if (!group_by_types.empty()) {
- // SimplifyHashTableImplTypeProto() switches the hash table implementation
- // from SeparateChaining to SimpleScalarSeparateChaining when there is a
- // single scalar key type with a reversible hash function.
- aggr_state_proto->set_hash_table_impl_type(
- SimplifyHashTableImplTypeProto(
- HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
- group_by_types));
- }
-
const QueryPlan::DAGNodeIndex aggregation_operator_index =
execution_plan_->addRelationalOperator(
new AggregationOperator(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9e2fb82e/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index a08b012..390b7b6 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -535,21 +535,12 @@ WHERE int_col = 2;
SELECT int_col
FROM test
-GROUP BY int_col;
+GROUP BY int_col
+ORDER BY int_col;
--
+-----------+
|int_col |
+-----------+
-| 2|
-| 4|
-| 6|
-| 8|
-| 12|
-| 14|
-| 16|
-| 18|
-| 22|
-| 24|
| -23|
| -21|
| -19|
@@ -562,6 +553,16 @@ GROUP BY int_col;
| -5|
| -3|
| -1|
+| 2|
+| 4|
+| 6|
+| 8|
+| 12|
+| 14|
+| 16|
+| 18|
+| 22|
+| 24|
+-----------+
==
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9e2fb82e/storage/AggregationOperationState.cpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.cpp b/storage/AggregationOperationState.cpp
index a3a669c..d209ceb 100644
--- a/storage/AggregationOperationState.cpp
+++ b/storage/AggregationOperationState.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -64,6 +66,7 @@ AggregationOperationState::AggregationOperationState(
const Predicate *predicate,
const std::size_t estimated_num_entries,
const HashTableImplType hash_table_impl_type,
+ const std::vector<HashTableImplType> &distinctify_hash_table_impl_types,
StorageManager *storage_manager)
: input_relation_(input_relation),
predicate_(predicate),
@@ -101,6 +104,8 @@ AggregationOperationState::AggregationOperationState(
std::vector<std::vector<std::unique_ptr<const Scalar>>>::const_iterator args_it
= arguments_.begin();
std::vector<bool>::const_iterator is_distinct_it = is_distinct_.begin();
+ std::vector<HashTableImplType>::const_iterator distinctify_hash_table_impl_types_it
+ = distinctify_hash_table_impl_types.begin();
for (; agg_func_it != aggregate_functions.end(); ++agg_func_it, ++args_it, ++is_distinct_it) {
// Get the Types of this aggregate's arguments so that we can create an
// AggregationHandle.
@@ -161,10 +166,11 @@ AggregationOperationState::AggregationOperationState(
// query optimization, if it worths.
distinctify_hashtables_.emplace_back(
handles_.back()->createDistinctifyHashTable(
- hash_table_impl_type,
+ *distinctify_hash_table_impl_types_it,
key_types,
estimated_num_entries,
storage_manager));
+ ++distinctify_hash_table_impl_types_it;
} else {
distinctify_hashtables_.emplace_back(nullptr);
}
@@ -182,6 +188,8 @@ AggregationOperationState* AggregationOperationState::ReconstructFromProto(
std::vector<const AggregateFunction*> aggregate_functions;
std::vector<std::vector<std::unique_ptr<const Scalar>>> arguments;
std::vector<bool> is_distinct;
+ std::vector<HashTableImplType> distinctify_hash_table_impl_types;
+ std::size_t distinctify_hash_table_impl_type_index = 0;
for (int agg_idx = 0; agg_idx < proto.aggregates_size(); ++agg_idx) {
const serialization::Aggregate &agg_proto = proto.aggregates(agg_idx);
@@ -197,6 +205,13 @@ AggregationOperationState* AggregationOperationState::ReconstructFromProto(
}
is_distinct.emplace_back(agg_proto.is_distinct());
+
+ if (agg_proto.is_distinct()) {
+ distinctify_hash_table_impl_types.emplace_back(
+ HashTableImplTypeFromProto(
+ proto.distinctify_hash_table_impl_types(distinctify_hash_table_impl_type_index)));
+ ++distinctify_hash_table_impl_type_index;
+ }
}
std::vector<std::unique_ptr<const Scalar>> group_by_expressions;
@@ -223,6 +238,7 @@ AggregationOperationState* AggregationOperationState::ReconstructFromProto(
predicate.release(),
proto.estimated_num_entries(),
HashTableImplTypeFromProto(proto.hash_table_impl_type()),
+ distinctify_hash_table_impl_types,
storage_manager);
}
@@ -234,6 +250,8 @@ bool AggregationOperationState::ProtoIsValid(const serialization::AggregationOpe
return false;
}
+ std::size_t num_distinctify_hash_tables = proto.distinctify_hash_table_impl_types_size();
+ std::size_t distinctify_hash_table_impl_type_index = 0;
for (int i = 0; i < proto.aggregates_size(); ++i) {
if (!AggregateFunctionFactory::ProtoIsValid(proto.aggregates(i).function())) {
return false;
@@ -251,6 +269,14 @@ bool AggregationOperationState::ProtoIsValid(const serialization::AggregationOpe
return false;
}
}
+
+ if (proto.aggregates(i).is_distinct()) {
+ if (distinctify_hash_table_impl_type_index >= num_distinctify_hash_tables ||
+ !serialization::HashTableImplType_IsValid(
+ proto.distinctify_hash_table_impl_types(distinctify_hash_table_impl_type_index))) {
+ return false;
+ }
+ }
}
for (int i = 0; i < proto.group_by_expressions_size(); ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9e2fb82e/storage/AggregationOperationState.hpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.hpp b/storage/AggregationOperationState.hpp
index b883ed1..c3a1278 100644
--- a/storage/AggregationOperationState.hpp
+++ b/storage/AggregationOperationState.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -93,6 +95,8 @@ class AggregationOperationState {
* in the input relation.
* @param hash_table_impl_type The HashTable implementation to use for
* GROUP BY. Ignored if group_by is empty.
+ * @param distinctify_hash_table_impl_type The HashTable implementation to use
+ * for the distinctify phase of each DISTINCT aggregation.
* @param storage_manager The StorageManager to use for allocating hash
* tables. Single aggregation state (when GROUP BY list is not
* specified) is not allocated using memory from storage manager.
@@ -105,6 +109,7 @@ class AggregationOperationState {
const Predicate *predicate,
const std::size_t estimated_num_entries,
const HashTableImplType hash_table_impl_type,
+ const std::vector<HashTableImplType> &distinctify_hash_table_impl_types,
StorageManager *storage_manager);
~AggregationOperationState() {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9e2fb82e/storage/AggregationOperationState.proto
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.proto b/storage/AggregationOperationState.proto
index 031f782..bf78e3a 100644
--- a/storage/AggregationOperationState.proto
+++ b/storage/AggregationOperationState.proto
@@ -1,5 +1,7 @@
// Copyright 2011-2015 Quickstep Technologies LLC.
// Copyright 2015-2016 Pivotal Software, Inc.
+// Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+// University of Wisconsin\u2014Madison.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -37,4 +39,7 @@ message AggregationOperationState {
// NOTE(chasseur): 'hash_table_impl_type' is marked optional, but it is
// needed if 'group_by_expressions' is non-empty, and ignored otherwise.
optional HashTableImplType hash_table_impl_type = 6;
+
+ // Each DISTINCT aggregation has its distinctify hash table impl type.
+ repeated HashTableImplType distinctify_hash_table_impl_types = 7;
}
[36/48] incubator-quickstep git commit: Explicitly specify where
tcmalloc comes from (#235)
Posted by hb...@apache.org.
Explicitly specify where tcmalloc comes from (#235)
This change adds a declaration that `libtcmalloc_minimal.a` is generated
by the external project. This is needed by the Ninja generator to
properly build.
To reproduce:
run `cmake -GNinja path/to/quickstep`
then `cmake --build path/to/build/dir` and see it fails.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4403e6ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4403e6ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4403e6ce
Branch: refs/heads/query-manager-used-in-foreman
Commit: 4403e6ce9fef39eb3a017307d256b05d21df3d63
Parents: 882cd38
Author: Jesse Zhang <jz...@pivotallabs.com>
Authored: Mon May 23 13:38:59 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:45 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4403e6ce/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 87a8f7c..2ab0f57 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -490,6 +490,7 @@ if(USE_TCMALLOC)
CXXFLAGS=${THIRD_PARTY_CXX_FLAGS}
BUILD_COMMAND make
BUILD_IN_SOURCE 0
+ BUILD_BYPRODUCTS <INSTALL_DIR>/lib/libtcmalloc_minimal.a
)
# Static libtcmalloc_minimal.a
add_library(libtcmalloc_minimal STATIC IMPORTED)
[44/48] incubator-quickstep git commit: Added Query ID to Relational
operators and WorkOrders.
Posted by hb...@apache.org.
Added Query ID to Relational operators and WorkOrders.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/40542682
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/40542682
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/40542682
Branch: refs/heads/query-manager-used-in-foreman
Commit: 405426823d0e12c75cef447c130fb2c92cacb90f
Parents: 83935e7
Author: Harshad Deshmukh <ha...@cs.wisc.edu>
Authored: Sun May 1 23:01:39 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:49 2016 -0700
----------------------------------------------------------------------
query_execution/Foreman.cpp | 3 +
query_execution/QueryManager.cpp | 1 +
query_execution/tests/Foreman_unittest.cpp | 2 +-
query_execution/tests/QueryManager_unittest.cpp | 2 +-
.../tests/WorkOrdersContainer_unittest.cpp | 2 +-
query_optimizer/ExecutionGenerator.cpp | 68 +++++++++++++-------
.../tests/ExecutionHeuristics_unittest.cpp | 6 +-
relational_operators/AggregationOperator.cpp | 12 ++--
relational_operators/AggregationOperator.hpp | 12 +++-
relational_operators/BuildHashOperator.cpp | 2 +
relational_operators/BuildHashOperator.hpp | 17 +++--
relational_operators/CreateIndexOperator.hpp | 8 ++-
relational_operators/CreateTableOperator.hpp | 10 ++-
relational_operators/DeleteOperator.cpp | 2 +
relational_operators/DeleteOperator.hpp | 12 +++-
relational_operators/DestroyHashOperator.cpp | 5 +-
relational_operators/DestroyHashOperator.hpp | 12 +++-
relational_operators/DropTableOperator.cpp | 3 +-
relational_operators/DropTableOperator.hpp | 13 +++-
.../FinalizeAggregationOperator.cpp | 6 +-
.../FinalizeAggregationOperator.hpp | 15 +++--
relational_operators/HashJoinOperator.cpp | 4 ++
relational_operators/HashJoinOperator.hpp | 49 ++++++++++----
relational_operators/InsertOperator.cpp | 6 +-
relational_operators/InsertOperator.hpp | 15 +++--
.../NestedLoopsJoinOperator.cpp | 4 ++
.../NestedLoopsJoinOperator.hpp | 12 +++-
relational_operators/RebuildWorkOrder.hpp | 5 +-
relational_operators/RelationalOperator.hpp | 8 ++-
relational_operators/SampleOperator.cpp | 7 +-
relational_operators/SampleOperator.hpp | 13 ++--
relational_operators/SaveBlocksOperator.cpp | 1 +
relational_operators/SaveBlocksOperator.hpp | 12 +++-
relational_operators/SelectOperator.cpp | 4 ++
relational_operators/SelectOperator.hpp | 24 +++++--
relational_operators/SortMergeRunOperator.cpp | 1 +
relational_operators/SortMergeRunOperator.hpp | 12 +++-
.../SortRunGenerationOperator.cpp | 2 +
.../SortRunGenerationOperator.hpp | 12 +++-
relational_operators/TableGeneratorOperator.cpp | 7 +-
relational_operators/TableGeneratorOperator.hpp | 13 ++--
relational_operators/TextScanOperator.cpp | 11 +++-
relational_operators/TextScanOperator.hpp | 16 ++++-
relational_operators/UpdateOperator.cpp | 1 +
relational_operators/UpdateOperator.hpp | 12 +++-
relational_operators/WorkOrder.hpp | 9 ++-
relational_operators/WorkOrder.proto | 1 +
relational_operators/WorkOrderFactory.cpp | 32 +++++++--
.../tests/AggregationOperator_unittest.cpp | 16 +++--
.../tests/HashJoinOperator_unittest.cpp | 48 ++++++++------
.../tests/SortMergeRunOperator_unittest.cpp | 6 +-
.../SortRunGenerationOperator_unittest.cpp | 3 +-
.../tests/TextScanOperator_unittest.cpp | 3 +-
53 files changed, 430 insertions(+), 152 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/query_execution/Foreman.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Foreman.cpp b/query_execution/Foreman.cpp
index 304c429..b358f70 100644
--- a/query_execution/Foreman.cpp
+++ b/query_execution/Foreman.cpp
@@ -518,11 +518,14 @@ void Foreman::getRebuildWorkOrders(const dag_node_index index, WorkOrdersContain
for (vector<MutableBlockReference>::size_type i = 0;
i < partially_filled_block_refs.size();
++i) {
+ // Note: The query ID used below is dummy for now, it will be replaced with
+ // the true query ID when QueryManager gets used in Foreman.
container->addRebuildWorkOrder(
new RebuildWorkOrder(move(partially_filled_block_refs[i]),
index,
op.getOutputRelationID(),
foreman_client_id_,
+ 0,
bus_),
index);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/query_execution/QueryManager.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManager.cpp b/query_execution/QueryManager.cpp
index 02c5d4c..21f5820 100644
--- a/query_execution/QueryManager.cpp
+++ b/query_execution/QueryManager.cpp
@@ -461,6 +461,7 @@ void QueryManager::getRebuildWorkOrders(const dag_node_index index,
index,
op.getOutputRelationID(),
foreman_client_id_,
+ query_id_,
bus_),
index);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/query_execution/tests/Foreman_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/Foreman_unittest.cpp b/query_execution/tests/Foreman_unittest.cpp
index 47cc641..d2f43a4 100644
--- a/query_execution/tests/Foreman_unittest.cpp
+++ b/query_execution/tests/Foreman_unittest.cpp
@@ -61,7 +61,7 @@ namespace quickstep {
class MockWorkOrder : public WorkOrder {
public:
explicit MockWorkOrder(const int op_index)
- : op_index_(op_index) {}
+ : WorkOrder(0), op_index_(op_index) {}
void execute() override {
VLOG(3) << "WorkOrder[" << op_index_ << "] executing.";
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/query_execution/tests/QueryManager_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/QueryManager_unittest.cpp b/query_execution/tests/QueryManager_unittest.cpp
index 1b9be48..80876f2 100644
--- a/query_execution/tests/QueryManager_unittest.cpp
+++ b/query_execution/tests/QueryManager_unittest.cpp
@@ -62,7 +62,7 @@ namespace quickstep {
class MockWorkOrder : public WorkOrder {
public:
explicit MockWorkOrder(const int op_index)
- : op_index_(op_index) {}
+ : WorkOrder(0), op_index_(op_index) {}
void execute() override {
VLOG(3) << "WorkOrder[" << op_index_ << "] executing.";
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/query_execution/tests/WorkOrdersContainer_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/WorkOrdersContainer_unittest.cpp b/query_execution/tests/WorkOrdersContainer_unittest.cpp
index d7db9a6..cf133c4 100644
--- a/query_execution/tests/WorkOrdersContainer_unittest.cpp
+++ b/query_execution/tests/WorkOrdersContainer_unittest.cpp
@@ -30,7 +30,7 @@ namespace quickstep {
class MockNUMAWorkOrder : public WorkOrder {
public:
MockNUMAWorkOrder(const int id, const std::vector<int> &numa_nodes)
- : id_(id) {
+ : WorkOrder(0), id_(id) {
for (int numa_node : numa_nodes) {
preferred_numa_nodes_.push_back(numa_node);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index c590b6e..30dfa8e 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -194,6 +194,7 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) {
const QueryPlan::DAGNodeIndex drop_table_index =
execution_plan_->addRelationalOperator(
new DropTableOperator(*temporary_relation,
+ query_handle_->query_id(),
optimizer_context_->catalog_database(),
false /* only_drop_blocks */));
DCHECK(!temporary_relation_info.isStoredRelation());
@@ -415,7 +416,8 @@ void ExecutionGenerator::convertSample(const P::SamplePtr &physical_sample) {
insert_destination_index,
input_relation_info->isStoredRelation(),
physical_sample->is_block_sample(),
- physical_sample->percentage());
+ physical_sample->percentage(),
+ query_handle_->query_id());
const QueryPlan::DAGNodeIndex sample_index =
execution_plan_->addRelationalOperator(sample_op);
insert_destination_proto->set_relational_op_index(sample_index);
@@ -531,13 +533,15 @@ void ExecutionGenerator::convertSelection(
insert_destination_index,
execution_predicate_index,
move(attributes),
- input_relation_info->isStoredRelation())
+ input_relation_info->isStoredRelation(),
+ query_handle_->query_id())
: new SelectOperator(*input_relation_info->relation,
*output_relation,
insert_destination_index,
execution_predicate_index,
project_expressions_group_index,
- input_relation_info->isStoredRelation());
+ input_relation_info->isStoredRelation(),
+ query_handle_->query_id());
const QueryPlan::DAGNodeIndex select_index =
execution_plan_->addRelationalOperator(op);
@@ -741,7 +745,8 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
build_relation_info->isStoredRelation(),
build_attribute_ids,
any_build_attributes_nullable,
- join_hash_table_index));
+ join_hash_table_index,
+ query_handle_->query_id()));
// Create InsertDestination proto.
const CatalogRelation *output_relation = nullptr;
@@ -787,13 +792,14 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
join_hash_table_index,
residual_predicate_index,
project_expressions_group_index,
+ query_handle_->query_id(),
is_selection_on_build.get(),
join_type));
insert_destination_proto->set_relational_op_index(join_operator_index);
const QueryPlan::DAGNodeIndex destroy_operator_index =
- execution_plan_->addRelationalOperator(
- new DestroyHashOperator(join_hash_table_index));
+ execution_plan_->addRelationalOperator(new DestroyHashOperator(
+ join_hash_table_index, query_handle_->query_id()));
if (!build_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(build_operator_index,
@@ -887,7 +893,8 @@ void ExecutionGenerator::convertNestedLoopsJoin(
execution_join_predicate_index,
project_expressions_group_index,
left_relation_info->isStoredRelation(),
- right_relation_info->isStoredRelation()));
+ right_relation_info->isStoredRelation(),
+ query_handle_->query_id()));
insert_destination_proto->set_relational_op_index(join_operator_index);
if (!left_relation_info->isStoredRelation()) {
@@ -938,12 +945,13 @@ void ExecutionGenerator::convertCopyFrom(
physical_plan->escape_strings(),
FLAGS_parallelize_load,
*output_relation,
- insert_destination_index));
+ insert_destination_index,
+ query_handle_->query_id()));
insert_destination_proto->set_relational_op_index(scan_operator_index);
const QueryPlan::DAGNodeIndex save_blocks_operator_index =
execution_plan_->addRelationalOperator(
- new SaveBlocksOperator());
+ new SaveBlocksOperator(query_handle_->query_id()));
execution_plan_->addDirectDependency(save_blocks_operator_index,
scan_operator_index,
false /* is_pipeline_breaker */);
@@ -991,6 +999,7 @@ void ExecutionGenerator::convertCreateIndex(
}
execution_plan_->addRelationalOperator(new CreateIndexOperator(input_relation,
physical_plan->index_name(),
+ query_handle_->query_id(),
std::move(index_description)));
}
@@ -1031,7 +1040,8 @@ void ExecutionGenerator::convertCreateTable(
}
execution_plan_->addRelationalOperator(
- new CreateTableOperator(catalog_relation.release(),
+ new CreateTableOperator(query_handle_->query_id(),
+ catalog_relation.release(),
optimizer_context_->catalog_database()));
}
@@ -1058,6 +1068,7 @@ void ExecutionGenerator::convertDeleteTuples(
const QueryPlan::DAGNodeIndex drop_table_index =
execution_plan_->addRelationalOperator(
new DropTableOperator(*input_relation_info->relation,
+ query_handle_->query_id(),
optimizer_context_->catalog_database(),
true /* only_drop_blocks */));
if (!input_relation_info->isStoredRelation()) {
@@ -1073,7 +1084,8 @@ void ExecutionGenerator::convertDeleteTuples(
execution_plan_->addRelationalOperator(new DeleteOperator(
*input_relation_info->relation,
execution_predicate_index,
- input_relation_info->isStoredRelation()));
+ input_relation_info->isStoredRelation(),
+ query_handle_->query_id()));
if (!input_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(delete_tuples_index,
input_relation_info->producer_operator_index,
@@ -1082,7 +1094,7 @@ void ExecutionGenerator::convertDeleteTuples(
const QueryPlan::DAGNodeIndex save_blocks_index =
execution_plan_->addRelationalOperator(
- new SaveBlocksOperator());
+ new SaveBlocksOperator(query_handle_->query_id()));
execution_plan_->addDirectDependency(save_blocks_index,
delete_tuples_index,
false /* is_pipeline_breaker */);
@@ -1100,6 +1112,7 @@ void ExecutionGenerator::convertDropTable(
execution_plan_->addRelationalOperator(
new DropTableOperator(catalog_relation,
+ query_handle_->query_id(),
optimizer_context_->catalog_database()));
}
@@ -1153,12 +1166,13 @@ void ExecutionGenerator::convertInsertTuple(
execution_plan_->addRelationalOperator(
new InsertOperator(input_relation,
insert_destination_index,
- tuple_index));
+ tuple_index,
+ query_handle_->query_id()));
insert_destination_proto->set_relational_op_index(insert_operator_index);
const QueryPlan::DAGNodeIndex save_blocks_index =
execution_plan_->addRelationalOperator(
- new SaveBlocksOperator());
+ new SaveBlocksOperator(query_handle_->query_id()));
if (!input_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(insert_operator_index,
input_relation_info->producer_operator_index,
@@ -1231,14 +1245,15 @@ void ExecutionGenerator::convertInsertSelection(
insert_destination_index,
QueryContext::kInvalidPredicateId,
move(attributes),
- selection_relation_info->isStoredRelation());
+ selection_relation_info->isStoredRelation(),
+ query_handle_->query_id());
const QueryPlan::DAGNodeIndex insert_selection_index =
execution_plan_->addRelationalOperator(insert_selection_op);
insert_destination_proto->set_relational_op_index(insert_selection_index);
const QueryPlan::DAGNodeIndex save_blocks_index =
- execution_plan_->addRelationalOperator(new SaveBlocksOperator());
+ execution_plan_->addRelationalOperator(new SaveBlocksOperator(query_handle_->query_id()));
if (!selection_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(insert_selection_index,
@@ -1311,12 +1326,13 @@ void ExecutionGenerator::convertUpdateTable(
*optimizer_context_->catalog_database()->getRelationById(input_rel_id),
relocation_destination_index,
execution_predicate_index,
- update_group_index));
+ update_group_index,
+ query_handle_->query_id()));
relocation_destination_proto->set_relational_op_index(update_operator_index);
const QueryPlan::DAGNodeIndex save_blocks_index =
execution_plan_->addRelationalOperator(
- new SaveBlocksOperator());
+ new SaveBlocksOperator(query_handle_->query_id()));
if (!input_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(update_operator_index,
input_relation_info->producer_operator_index,
@@ -1415,7 +1431,8 @@ void ExecutionGenerator::convertAggregate(
new AggregationOperator(
*input_relation_info->relation,
input_relation_info->isStoredRelation(),
- aggr_state_index));
+ aggr_state_index,
+ query_handle_->query_id()));
if (!input_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(aggregation_operator_index,
input_relation_info->producer_operator_index,
@@ -1435,7 +1452,8 @@ void ExecutionGenerator::convertAggregate(
execution_plan_->addRelationalOperator(
new FinalizeAggregationOperator(aggr_state_index,
*output_relation,
- insert_destination_index));
+ insert_destination_index,
+ query_handle_->query_id()));
insert_destination_proto->set_relational_op_index(finalize_aggregation_operator_index);
execution_plan_->addDirectDependency(finalize_aggregation_operator_index,
@@ -1486,7 +1504,8 @@ void ExecutionGenerator::convertSort(const P::SortPtr &physical_sort) {
*initial_runs_relation,
initial_runs_destination_id,
sort_run_gen_config_id,
- input_relation_info->isStoredRelation()));
+ input_relation_info->isStoredRelation(),
+ query_handle_->query_id()));
if (!input_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(run_generator_index,
input_relation_info->producer_operator_index,
@@ -1543,7 +1562,8 @@ void ExecutionGenerator::convertSort(const P::SortPtr &physical_sort) {
sort_merge_run_config_id,
64 /* merge_factor */,
physical_sort->limit(),
- false /* input_relation_is_stored */));
+ false /* input_relation_is_stored */,
+ query_handle_->query_id()));
execution_plan_->addDirectDependency(merge_run_operator_index,
run_generator_index,
false /* is_pipeline_breaker */);
@@ -1557,6 +1577,7 @@ void ExecutionGenerator::convertSort(const P::SortPtr &physical_sort) {
execution_plan_->addRelationalOperator(
new DropTableOperator(
*merged_runs_relation,
+ query_handle_->query_id(),
optimizer_context_->catalog_database(),
false /* only_drop_blocks */));
execution_plan_->addDirectDependency(
@@ -1594,7 +1615,8 @@ void ExecutionGenerator::convertTableGenerator(
TableGeneratorOperator *op =
new TableGeneratorOperator(*output_relation,
insert_destination_index,
- generator_function_index);
+ generator_function_index,
+ query_handle_->query_id());
const QueryPlan::DAGNodeIndex tablegen_index =
execution_plan_->addRelationalOperator(op);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/ExecutionHeuristics_unittest.cpp b/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
index 12acaff..a08a476 100644
--- a/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
+++ b/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
@@ -79,7 +79,8 @@ class ExecutionHeuristicsTest : public ::testing::Test {
true,
build_attribute_ids,
false,
- join_hash_table_index));
+ join_hash_table_index,
+ 0 /* dummy query ID */));
return build_operator_index;
}
@@ -100,7 +101,8 @@ class ExecutionHeuristicsTest : public ::testing::Test {
0,
join_hash_table_index,
0,
- 0));
+ 0,
+ 0 /* dummy query ID */));
return join_operator_index;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/AggregationOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/AggregationOperator.cpp b/relational_operators/AggregationOperator.cpp
index 1b935ee..94ba901 100644
--- a/relational_operators/AggregationOperator.cpp
+++ b/relational_operators/AggregationOperator.cpp
@@ -38,8 +38,10 @@ bool AggregationOperator::getAllWorkOrders(
if (!started_) {
for (const block_id input_block_id : input_relation_block_ids_) {
container->addNormalWorkOrder(
- new AggregationWorkOrder(input_block_id,
- query_context->getAggregationState(aggr_state_index_)),
+ new AggregationWorkOrder(
+ input_block_id,
+ query_id_,
+ query_context->getAggregationState(aggr_state_index_)),
op_index_);
}
started_ = true;
@@ -48,8 +50,10 @@ bool AggregationOperator::getAllWorkOrders(
} else {
while (num_workorders_generated_ < input_relation_block_ids_.size()) {
container->addNormalWorkOrder(
- new AggregationWorkOrder(input_relation_block_ids_[num_workorders_generated_],
- query_context->getAggregationState(aggr_state_index_)),
+ new AggregationWorkOrder(
+ input_relation_block_ids_[num_workorders_generated_],
+ query_id_,
+ query_context->getAggregationState(aggr_state_index_)),
op_index_);
++num_workorders_generated_;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/AggregationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/AggregationOperator.hpp b/relational_operators/AggregationOperator.hpp
index 0e74dfc..0fbc381 100644
--- a/relational_operators/AggregationOperator.hpp
+++ b/relational_operators/AggregationOperator.hpp
@@ -58,11 +58,14 @@ class AggregationOperator : public RelationalOperator {
* is fully available to the operator before it can start generating
* workorders.
* @param aggr_state_index The index of the AggregationState in QueryContext.
+ * @param query_id The ID of this query.
**/
AggregationOperator(const CatalogRelation &input_relation,
bool input_relation_is_stored,
- const QueryContext::aggregation_state_id aggr_state_index)
- : input_relation_is_stored_(input_relation_is_stored),
+ const QueryContext::aggregation_state_id aggr_state_index,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ input_relation_is_stored_(input_relation_is_stored),
input_relation_block_ids_(input_relation_is_stored ? input_relation.getBlocksSnapshot()
: std::vector<block_id>()),
aggr_state_index_(aggr_state_index),
@@ -107,11 +110,14 @@ class AggregationWorkOrder : public WorkOrder {
* @brief Constructor
*
* @param input_block_id The block id.
+ * @param query_id The ID of this query.
* @param state The AggregationState to use.
**/
AggregationWorkOrder(const block_id input_block_id,
+ const std::size_t query_id,
AggregationOperationState *state)
- : input_block_id_(input_block_id),
+ : WorkOrder(query_id),
+ input_block_id_(input_block_id),
state_(DCHECK_NOTNULL(state)) {}
~AggregationWorkOrder() override {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/BuildHashOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/BuildHashOperator.cpp b/relational_operators/BuildHashOperator.cpp
index df92159..c6f6f96 100644
--- a/relational_operators/BuildHashOperator.cpp
+++ b/relational_operators/BuildHashOperator.cpp
@@ -73,6 +73,7 @@ bool BuildHashOperator::getAllWorkOrders(
join_key_attributes_,
any_join_key_attributes_nullable_,
input_block_id,
+ query_id_,
hash_table,
storage_manager),
op_index_);
@@ -88,6 +89,7 @@ bool BuildHashOperator::getAllWorkOrders(
join_key_attributes_,
any_join_key_attributes_nullable_,
input_relation_block_ids_[num_workorders_generated_],
+ query_id_,
hash_table,
storage_manager),
op_index_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/BuildHashOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/BuildHashOperator.hpp b/relational_operators/BuildHashOperator.hpp
index f9d830f..5a46d8b 100644
--- a/relational_operators/BuildHashOperator.hpp
+++ b/relational_operators/BuildHashOperator.hpp
@@ -69,13 +69,16 @@ class BuildHashOperator : public RelationalOperator {
* @param hash_table_index The index of the JoinHashTable in QueryContext.
* The HashTable's key Type(s) should be the Type(s) of the
* join_key_attributes in input_relation.
+ * @param query_id The ID of the query to which this operator belongs.
**/
BuildHashOperator(const CatalogRelation &input_relation,
const bool input_relation_is_stored,
const std::vector<attribute_id> &join_key_attributes,
const bool any_join_key_attributes_nullable,
- const QueryContext::join_hash_table_id hash_table_index)
- : input_relation_(input_relation),
+ const QueryContext::join_hash_table_id hash_table_index,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ input_relation_(input_relation),
input_relation_is_stored_(input_relation_is_stored),
join_key_attributes_(join_key_attributes),
any_join_key_attributes_nullable_(any_join_key_attributes_nullable),
@@ -133,6 +136,7 @@ class BuildHashWorkOrder : public WorkOrder {
* input_relation.
* @param any_join_key_attributes_nullable If any attribute is nullable.
* @param build_block_id The block id.
+ * @param query_id The ID of the query.
* @param hash_table The JoinHashTable to use.
* @param storage_manager The StorageManager to use.
**/
@@ -140,9 +144,11 @@ class BuildHashWorkOrder : public WorkOrder {
const std::vector<attribute_id> &join_key_attributes,
const bool any_join_key_attributes_nullable,
const block_id build_block_id,
+ const std::size_t query_id,
JoinHashTable *hash_table,
StorageManager *storage_manager)
- : input_relation_(input_relation),
+ : WorkOrder(query_id),
+ input_relation_(input_relation),
join_key_attributes_(join_key_attributes),
any_join_key_attributes_nullable_(any_join_key_attributes_nullable),
build_block_id_(build_block_id),
@@ -156,6 +162,7 @@ class BuildHashWorkOrder : public WorkOrder {
* @param join_key_attributes The IDs of equijoin attributes in
* input_relation.
* @param any_join_key_attributes_nullable If any attribute is nullable.
+ * @param query_id The ID of the query.
* @param build_block_id The block id.
* @param hash_table The JoinHashTable to use.
* @param storage_manager The StorageManager to use.
@@ -164,9 +171,11 @@ class BuildHashWorkOrder : public WorkOrder {
std::vector<attribute_id> &&join_key_attributes,
const bool any_join_key_attributes_nullable,
const block_id build_block_id,
+ const std::size_t query_id,
JoinHashTable *hash_table,
StorageManager *storage_manager)
- : input_relation_(input_relation),
+ : WorkOrder(query_id),
+ input_relation_(input_relation),
join_key_attributes_(std::move(join_key_attributes)),
any_join_key_attributes_nullable_(any_join_key_attributes_nullable),
build_block_id_(build_block_id),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/CreateIndexOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/CreateIndexOperator.hpp b/relational_operators/CreateIndexOperator.hpp
index 2bfacc4..ede3f02 100644
--- a/relational_operators/CreateIndexOperator.hpp
+++ b/relational_operators/CreateIndexOperator.hpp
@@ -18,6 +18,7 @@
#ifndef QUICKSTEP_RELATIONAL_OPERATORS_CREATE_INDEX_OPERATOR_HPP_
#define QUICKSTEP_RELATIONAL_OPERATORS_CREATE_INDEX_OPERATOR_HPP_
+#include <cstddef>
#include <string>
#include "catalog/CatalogRelation.hpp"
@@ -52,19 +53,22 @@ class CreateIndexOperator : public RelationalOperator {
*
* @param relation The relation to create index upon.
* @param index_name The index to create.
+ * @param query_id The ID of the query to which this operator belongs.
* @param index_description The index_description associated with this index.
**/
CreateIndexOperator(CatalogRelation *relation,
const std::string &index_name,
+ const std::size_t query_id,
IndexSubBlockDescription &&index_description) // NOLINT(whitespace/operators)
- : relation_(DCHECK_NOTNULL(relation)),
+ : RelationalOperator(query_id),
+ relation_(DCHECK_NOTNULL(relation)),
index_name_(index_name),
index_description_(index_description) {}
~CreateIndexOperator() override {}
/**
- * @note no WorkOrder generated for this operator.
+ * @note No WorkOrder generated for this operator.
**/
bool getAllWorkOrders(WorkOrdersContainer *container,
QueryContext *query_context,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/CreateTableOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/CreateTableOperator.hpp b/relational_operators/CreateTableOperator.hpp
index 98f3253..60bcef4 100644
--- a/relational_operators/CreateTableOperator.hpp
+++ b/relational_operators/CreateTableOperator.hpp
@@ -18,6 +18,7 @@
#ifndef QUICKSTEP_RELATIONAL_OPERATORS_CREATE_TABLE_OPERATOR_HPP_
#define QUICKSTEP_RELATIONAL_OPERATORS_CREATE_TABLE_OPERATOR_HPP_
+#include <cstddef>
#include <memory>
#include "catalog/CatalogRelation.hpp"
@@ -49,20 +50,23 @@ class CreateTableOperator : public RelationalOperator {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param relation The relation to add. This CreateTableOperator owns
* relation until the WorkOrder it produces is successfully executed,
* at which point it is owned by database.
* @param database The database to add a relation to.
**/
- CreateTableOperator(CatalogRelation *relation,
+ CreateTableOperator(const std::size_t query_id,
+ CatalogRelation *relation,
CatalogDatabase *database)
- : relation_(DCHECK_NOTNULL(relation)),
+ : RelationalOperator(query_id),
+ relation_(DCHECK_NOTNULL(relation)),
database_(DCHECK_NOTNULL(database)) {}
~CreateTableOperator() override {}
/**
- * @note no WorkOrder generated for this operator.
+ * @note No WorkOrder generated for this operator.
**/
bool getAllWorkOrders(WorkOrdersContainer *container,
QueryContext *query_context,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/DeleteOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/DeleteOperator.cpp b/relational_operators/DeleteOperator.cpp
index 2c2c6de..94169ed 100644
--- a/relational_operators/DeleteOperator.cpp
+++ b/relational_operators/DeleteOperator.cpp
@@ -60,6 +60,7 @@ bool DeleteOperator::getAllWorkOrders(
storage_manager,
op_index_,
scheduler_client_id,
+ query_id_,
bus),
op_index_);
}
@@ -75,6 +76,7 @@ bool DeleteOperator::getAllWorkOrders(
storage_manager,
op_index_,
scheduler_client_id,
+ query_id_,
bus),
op_index_);
++num_workorders_generated_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/DeleteOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/DeleteOperator.hpp b/relational_operators/DeleteOperator.hpp
index 1d44552..ba1f825 100644
--- a/relational_operators/DeleteOperator.hpp
+++ b/relational_operators/DeleteOperator.hpp
@@ -61,11 +61,14 @@ class DeleteOperator : public RelationalOperator {
* tuples will be deleted).
* @param relation_is_stored If relation is a stored relation and is fully
* available to the operator before it can start generating workorders.
+ * @param query_id The ID of the query to which this operator belongs.
**/
DeleteOperator(const CatalogRelation &relation,
const QueryContext::predicate_id predicate_index,
- const bool relation_is_stored)
- : relation_(relation),
+ const bool relation_is_stored,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ relation_(relation),
predicate_index_(predicate_index),
relation_is_stored_(relation_is_stored),
started_(false),
@@ -127,6 +130,7 @@ class DeleteWorkOrder : public WorkOrder {
* @param delete_operator_index The index of the Delete Operator in the query
* plan DAG.
* @param scheduler_client_id The TMB client ID of the scheduler thread.
+ * @param query_id The ID of the query to which this workorder belongs.
* @param bus A pointer to the TMB.
**/
DeleteWorkOrder(const CatalogRelationSchema &input_relation,
@@ -135,8 +139,10 @@ class DeleteWorkOrder : public WorkOrder {
StorageManager *storage_manager,
const std::size_t delete_operator_index,
const tmb::client_id scheduler_client_id,
+ const std::size_t query_id,
MessageBus *bus)
- : input_relation_(input_relation),
+ : WorkOrder(query_id),
+ input_relation_(input_relation),
input_block_id_(input_block_id),
predicate_(predicate),
storage_manager_(DCHECK_NOTNULL(storage_manager)),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/DestroyHashOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/DestroyHashOperator.cpp b/relational_operators/DestroyHashOperator.cpp
index c2220d5..c92117a 100644
--- a/relational_operators/DestroyHashOperator.cpp
+++ b/relational_operators/DestroyHashOperator.cpp
@@ -32,8 +32,9 @@ bool DestroyHashOperator::getAllWorkOrders(
tmb::MessageBus *bus) {
if (blocking_dependencies_met_ && !work_generated_) {
work_generated_ = true;
- container->addNormalWorkOrder(new DestroyHashWorkOrder(hash_table_index_, query_context),
- op_index_);
+ container->addNormalWorkOrder(
+ new DestroyHashWorkOrder(hash_table_index_, query_id_, query_context),
+ op_index_);
}
return work_generated_;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/DestroyHashOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/DestroyHashOperator.hpp b/relational_operators/DestroyHashOperator.hpp
index 46331ba..086c279 100644
--- a/relational_operators/DestroyHashOperator.hpp
+++ b/relational_operators/DestroyHashOperator.hpp
@@ -47,9 +47,12 @@ class DestroyHashOperator : public RelationalOperator {
* @brief Constructor.
*
* @param hash_table_index The index of the JoinHashTable in QueryContext.
+ * @param query_id The ID of the query to which this operator belongs.
**/
- explicit DestroyHashOperator(const QueryContext::join_hash_table_id hash_table_index)
- : hash_table_index_(hash_table_index),
+ DestroyHashOperator(const QueryContext::join_hash_table_id hash_table_index,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ hash_table_index_(hash_table_index),
work_generated_(false) {}
~DestroyHashOperator() override {}
@@ -76,11 +79,14 @@ class DestroyHashWorkOrder : public WorkOrder {
* @brief Constructor.
*
* @param hash_table_index The index of the JoinHashTable in QueryContext.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param query_context The QueryContext to use.
**/
DestroyHashWorkOrder(const QueryContext::join_hash_table_id hash_table_index,
+ const std::size_t query_id,
QueryContext *query_context)
- : hash_table_index_(hash_table_index),
+ : WorkOrder(query_id),
+ hash_table_index_(hash_table_index),
query_context_(DCHECK_NOTNULL(query_context)) {}
~DestroyHashWorkOrder() override {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/DropTableOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/DropTableOperator.cpp b/relational_operators/DropTableOperator.cpp
index f3a3a2c..256f6a1 100644
--- a/relational_operators/DropTableOperator.cpp
+++ b/relational_operators/DropTableOperator.cpp
@@ -45,7 +45,8 @@ bool DropTableOperator::getAllWorkOrders(
// DropTableWorkOrder only drops blocks, if any.
container->addNormalWorkOrder(
- new DropTableWorkOrder(std::move(relation_blocks), storage_manager),
+ new DropTableWorkOrder(
+ query_id_, std::move(relation_blocks), storage_manager),
op_index_);
database_->setStatus(CatalogDatabase::Status::kPendingBlockDeletions);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/DropTableOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/DropTableOperator.hpp b/relational_operators/DropTableOperator.hpp
index bf9b1b1..0bbb718 100644
--- a/relational_operators/DropTableOperator.hpp
+++ b/relational_operators/DropTableOperator.hpp
@@ -18,6 +18,7 @@
#ifndef QUICKSTEP_RELATIONAL_OPERATORS_DROP_TABLE_OPERATOR_HPP_
#define QUICKSTEP_RELATIONAL_OPERATORS_DROP_TABLE_OPERATOR_HPP_
+#include <cstddef>
#include <utility>
#include <vector>
@@ -55,14 +56,17 @@ class DropTableOperator : public RelationalOperator {
* @brief Constructor.
*
* @param relation The relation to drop.
+ * @param query_id The ID of the query to which this operator belongs.
* @param database The databse where to drop \c relation.
* @param only_drop_blocks If true, only drop the blocks belonging to \c
* relation, but leave \c relation in \c database.
**/
DropTableOperator(const CatalogRelation &relation,
+ const std::size_t query_id,
CatalogDatabase *database,
const bool only_drop_blocks = false)
- : relation_(relation),
+ : RelationalOperator(query_id),
+ relation_(relation),
database_(database),
only_drop_blocks_(only_drop_blocks),
work_generated_(false) {}
@@ -95,17 +99,20 @@ class DropTableWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param blocks The blocks to drop.
* @param storage_manager The StorageManager to use.
* @param rel_id The relation id to drop.
* @param catalog_database_cache The CatalogDatabaseCache in the distributed
* version.
**/
- DropTableWorkOrder(std::vector<block_id> &&blocks,
+ DropTableWorkOrder(const std::size_t query_id,
+ std::vector<block_id> &&blocks,
StorageManager *storage_manager,
const relation_id rel_id = kInvalidCatalogId,
CatalogDatabaseLite *catalog_database_cache = nullptr)
- : blocks_(std::move(blocks)),
+ : WorkOrder(query_id),
+ blocks_(std::move(blocks)),
storage_manager_(DCHECK_NOTNULL(storage_manager)),
rel_id_(rel_id),
catalog_database_cache_(catalog_database_cache) {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/FinalizeAggregationOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/FinalizeAggregationOperator.cpp b/relational_operators/FinalizeAggregationOperator.cpp
index 410ec69..1dc4188 100644
--- a/relational_operators/FinalizeAggregationOperator.cpp
+++ b/relational_operators/FinalizeAggregationOperator.cpp
@@ -38,8 +38,10 @@ bool FinalizeAggregationOperator::getAllWorkOrders(
if (blocking_dependencies_met_ && !started_) {
started_ = true;
container->addNormalWorkOrder(
- new FinalizeAggregationWorkOrder(query_context->releaseAggregationState(aggr_state_index_),
- query_context->getInsertDestination(output_destination_index_)),
+ new FinalizeAggregationWorkOrder(
+ query_id_,
+ query_context->releaseAggregationState(aggr_state_index_),
+ query_context->getInsertDestination(output_destination_index_)),
op_index_);
}
return started_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/FinalizeAggregationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/FinalizeAggregationOperator.hpp b/relational_operators/FinalizeAggregationOperator.hpp
index fb9608a..51e55dc 100644
--- a/relational_operators/FinalizeAggregationOperator.hpp
+++ b/relational_operators/FinalizeAggregationOperator.hpp
@@ -18,6 +18,7 @@
#ifndef QUICKSTEP_RELATIONAL_OPERATORS_FINALIZE_AGGREGATION_OPERATOR_HPP_
#define QUICKSTEP_RELATIONAL_OPERATORS_FINALIZE_AGGREGATION_OPERATOR_HPP_
+#include <cstddef>
#include <memory>
#include "catalog/CatalogRelation.hpp"
@@ -57,11 +58,14 @@ class FinalizeAggregationOperator : public RelationalOperator {
* @param output_relation The output relation.
* @param output_destination_index The index of the InsertDestination in the
* QueryContext to insert aggregation results.
+ * @param query_id The ID of the query to which this operator belongs.
*/
FinalizeAggregationOperator(const QueryContext::aggregation_state_id aggr_state_index,
const CatalogRelation &output_relation,
- const QueryContext::insert_destination_id output_destination_index)
- : aggr_state_index_(aggr_state_index),
+ const QueryContext::insert_destination_id output_destination_index,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ aggr_state_index_(aggr_state_index),
output_relation_(output_relation),
output_destination_index_(output_destination_index),
started_(false) {}
@@ -101,13 +105,16 @@ class FinalizeAggregationWorkOrder : public WorkOrder {
*
* @note InsertWorkOrder takes ownership of \c state.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param state The AggregationState to use.
* @param output_destination The InsertDestination to insert aggregation
* results.
*/
- FinalizeAggregationWorkOrder(AggregationOperationState *state,
+ FinalizeAggregationWorkOrder(const std::size_t query_id,
+ AggregationOperationState *state,
InsertDestination *output_destination)
- : state_(DCHECK_NOTNULL(state)),
+ : WorkOrder(query_id),
+ state_(DCHECK_NOTNULL(state)),
output_destination_(DCHECK_NOTNULL(output_destination)) {}
~FinalizeAggregationWorkOrder() override {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index aa03794..d8c7304 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -298,6 +298,7 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders(
residual_predicate,
selection,
hash_table,
+ query_id_,
output_destination,
storage_manager),
op_index_);
@@ -316,6 +317,7 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders(
residual_predicate,
selection,
hash_table,
+ query_id_,
output_destination,
storage_manager),
op_index_);
@@ -356,6 +358,7 @@ bool HashJoinOperator::getAllOuterJoinWorkOrders(
selection,
is_selection_on_build_,
hash_table,
+ query_id_,
output_destination,
storage_manager),
op_index_);
@@ -375,6 +378,7 @@ bool HashJoinOperator::getAllOuterJoinWorkOrders(
selection,
is_selection_on_build_,
hash_table,
+ query_id_,
output_destination,
storage_manager),
op_index_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/HashJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.hpp b/relational_operators/HashJoinOperator.hpp
index fcc087a..825f360 100644
--- a/relational_operators/HashJoinOperator.hpp
+++ b/relational_operators/HashJoinOperator.hpp
@@ -108,6 +108,7 @@ class HashJoinOperator : public RelationalOperator {
* corresponding to the attributes of the relation referred by
* output_relation_id. Each Scalar is evaluated for the joined tuples,
* and the resulting value is inserted into the join result.
+ * @param query_id The ID of the query.
* @param is_selection_on_build Whether each selection Scalar is using attributes
* from the build relation as input. Can be NULL for inner/semi/anti
* joins since this information is not utilized by these joins.
@@ -123,9 +124,11 @@ class HashJoinOperator : public RelationalOperator {
const QueryContext::join_hash_table_id hash_table_index,
const QueryContext::predicate_id residual_predicate_index,
const QueryContext::scalar_group_id selection_index,
+ const std::size_t query_id,
const std::vector<bool> *is_selection_on_build = nullptr,
const JoinType join_type = JoinType::kInnerJoin)
- : build_relation_(build_relation),
+ : RelationalOperator(query_id),
+ build_relation_(build_relation),
probe_relation_(probe_relation),
probe_relation_is_stored_(probe_relation_is_stored),
join_key_attributes_(join_key_attributes),
@@ -243,6 +246,7 @@ class HashInnerJoinWorkOrder : public WorkOrder {
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
* @param hash_table The JoinHashTable to use.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
@@ -254,9 +258,11 @@ class HashInnerJoinWorkOrder : public WorkOrder {
const Predicate *residual_predicate,
const std::vector<std::unique_ptr<const Scalar>> &selection,
const JoinHashTable &hash_table,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
- : build_relation_(build_relation),
+ : WorkOrder(query_id),
+ build_relation_(build_relation),
probe_relation_(probe_relation),
join_key_attributes_(join_key_attributes),
any_join_key_attributes_nullable_(any_join_key_attributes_nullable),
@@ -286,6 +292,7 @@ class HashInnerJoinWorkOrder : public WorkOrder {
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
* @param hash_table The JoinHashTable to use.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
@@ -297,9 +304,11 @@ class HashInnerJoinWorkOrder : public WorkOrder {
const Predicate *residual_predicate,
const std::vector<std::unique_ptr<const Scalar>> &selection,
const JoinHashTable &hash_table,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
- : build_relation_(build_relation),
+ : WorkOrder(query_id),
+ build_relation_(build_relation),
probe_relation_(probe_relation),
join_key_attributes_(std::move(join_key_attributes)),
any_join_key_attributes_nullable_(any_join_key_attributes_nullable),
@@ -366,6 +375,7 @@ class HashSemiJoinWorkOrder : public WorkOrder {
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
* @param hash_table The JoinHashTable to use.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
@@ -377,9 +387,11 @@ class HashSemiJoinWorkOrder : public WorkOrder {
const Predicate *residual_predicate,
const std::vector<std::unique_ptr<const Scalar>> &selection,
const JoinHashTable &hash_table,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
- : build_relation_(build_relation),
+ : WorkOrder(query_id),
+ build_relation_(build_relation),
probe_relation_(probe_relation),
join_key_attributes_(join_key_attributes),
any_join_key_attributes_nullable_(any_join_key_attributes_nullable),
@@ -409,6 +421,7 @@ class HashSemiJoinWorkOrder : public WorkOrder {
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
* @param hash_table The JoinHashTable to use.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
@@ -420,9 +433,11 @@ class HashSemiJoinWorkOrder : public WorkOrder {
const Predicate *residual_predicate,
const std::vector<std::unique_ptr<const Scalar>> &selection,
const JoinHashTable &hash_table,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
- : build_relation_(build_relation),
+ : WorkOrder(query_id),
+ build_relation_(build_relation),
probe_relation_(probe_relation),
join_key_attributes_(std::move(join_key_attributes)),
any_join_key_attributes_nullable_(any_join_key_attributes_nullable),
@@ -482,6 +497,7 @@ class HashAntiJoinWorkOrder : public WorkOrder {
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
* @param hash_table The JoinHashTable to use.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
@@ -493,9 +509,11 @@ class HashAntiJoinWorkOrder : public WorkOrder {
const Predicate *residual_predicate,
const std::vector<std::unique_ptr<const Scalar>> &selection,
const JoinHashTable &hash_table,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
- : build_relation_(build_relation),
+ : WorkOrder(query_id),
+ build_relation_(build_relation),
probe_relation_(probe_relation),
join_key_attributes_(join_key_attributes),
any_join_key_attributes_nullable_(any_join_key_attributes_nullable),
@@ -525,6 +543,7 @@ class HashAntiJoinWorkOrder : public WorkOrder {
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
* @param hash_table The JoinHashTable to use.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
@@ -536,9 +555,11 @@ class HashAntiJoinWorkOrder : public WorkOrder {
const Predicate *residual_predicate,
const std::vector<std::unique_ptr<const Scalar>> &selection,
const JoinHashTable &hash_table,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
- : build_relation_(build_relation),
+ : WorkOrder(query_id),
+ build_relation_(build_relation),
probe_relation_(probe_relation),
join_key_attributes_(std::move(join_key_attributes)),
any_join_key_attributes_nullable_(any_join_key_attributes_nullable),
@@ -602,6 +623,7 @@ class HashOuterJoinWorkOrder : public WorkOrder {
* is using attributes from the build relation as input. Note that the
* length of this vector should equal the length of \p selection.
* @param lookup_block_id The block id of the probe_relation.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
@@ -613,9 +635,11 @@ class HashOuterJoinWorkOrder : public WorkOrder {
const std::vector<std::unique_ptr<const Scalar>> &selection,
const std::vector<bool> &is_selection_on_build,
const JoinHashTable &hash_table,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
- : build_relation_(build_relation),
+ : WorkOrder(query_id),
+ build_relation_(build_relation),
probe_relation_(probe_relation),
join_key_attributes_(join_key_attributes),
any_join_key_attributes_nullable_(any_join_key_attributes_nullable),
@@ -636,14 +660,15 @@ class HashOuterJoinWorkOrder : public WorkOrder {
* @param join_key_attributes The IDs of equijoin attributes in \c
* probe_relation.
* @param any_join_key_attributes_nullable If any attribute is nullable.
- * @param hash_table The JoinHashTable to use.
+ * @param lookup_block_id The block id of the probe_relation.
* @param selection A list of Scalars corresponding to the relation attributes
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
* @param is_selection_on_build Whether each Scalar in the \p selection vector
* is using attributes from the build relation as input. Note that the
* length of this vector should equal the length of \p selection.
- * @param lookup_block_id The block id of the probe_relation.
+ * @param hash_table The JoinHashTable to use.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
@@ -655,9 +680,11 @@ class HashOuterJoinWorkOrder : public WorkOrder {
const std::vector<std::unique_ptr<const Scalar>> &selection,
std::vector<bool> &&is_selection_on_build,
const JoinHashTable &hash_table,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
- : build_relation_(build_relation),
+ : WorkOrder(query_id),
+ build_relation_(build_relation),
probe_relation_(probe_relation),
join_key_attributes_(std::move(join_key_attributes)),
any_join_key_attributes_nullable_(any_join_key_attributes_nullable),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/InsertOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/InsertOperator.cpp b/relational_operators/InsertOperator.cpp
index 8d083e5..3ec9933 100644
--- a/relational_operators/InsertOperator.cpp
+++ b/relational_operators/InsertOperator.cpp
@@ -40,8 +40,10 @@ bool InsertOperator::getAllWorkOrders(
work_generated_ = true;
container->addNormalWorkOrder(
- new InsertWorkOrder(query_context->getInsertDestination(output_destination_index_),
- query_context->releaseTuple(tuple_index_)),
+ new InsertWorkOrder(
+ query_id_,
+ query_context->getInsertDestination(output_destination_index_),
+ query_context->releaseTuple(tuple_index_)),
op_index_);
}
return work_generated_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/InsertOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/InsertOperator.hpp b/relational_operators/InsertOperator.hpp
index 8a06c94..ccef444 100644
--- a/relational_operators/InsertOperator.hpp
+++ b/relational_operators/InsertOperator.hpp
@@ -18,6 +18,7 @@
#ifndef QUICKSTEP_RELATIONAL_OPERATORS_INSERT_OPERATOR_HPP_
#define QUICKSTEP_RELATIONAL_OPERATORS_INSERT_OPERATOR_HPP_
+#include <cstddef>
#include <memory>
#include "catalog/CatalogRelation.hpp"
@@ -56,11 +57,14 @@ class InsertOperator : public RelationalOperator {
* @param output_destination_index The index of the InsertDestination in the
* QueryContext to insert the tuple.
* @param tuple_index The index of the tuple to insert in the QueryContext.
+ * @param query_id The ID of the query to which this operator belongs.
**/
InsertOperator(const CatalogRelation &output_relation,
const QueryContext::insert_destination_id output_destination_index,
- const QueryContext::tuple_id tuple_index)
- : output_relation_(output_relation),
+ const QueryContext::tuple_id tuple_index,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ output_relation_(output_relation),
output_destination_index_(output_destination_index),
tuple_index_(tuple_index),
work_generated_(false) {}
@@ -100,12 +104,15 @@ class InsertWorkOrder : public WorkOrder {
*
* @note InsertWorkOrder takes ownership of \c tuple.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the tuple.
* @param tuple The tuple to insert.
**/
- InsertWorkOrder(InsertDestination *output_destination,
+ InsertWorkOrder(const std::size_t query_id,
+ InsertDestination *output_destination,
Tuple *tuple)
- : output_destination_(DCHECK_NOTNULL(output_destination)),
+ : WorkOrder(query_id),
+ output_destination_(DCHECK_NOTNULL(output_destination)),
tuple_(DCHECK_NOTNULL(tuple)) {}
~InsertWorkOrder() override {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/NestedLoopsJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/NestedLoopsJoinOperator.cpp b/relational_operators/NestedLoopsJoinOperator.cpp
index 5cc498b..317cc5d 100644
--- a/relational_operators/NestedLoopsJoinOperator.cpp
+++ b/relational_operators/NestedLoopsJoinOperator.cpp
@@ -82,6 +82,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrders(
right_block_id,
query_context->getPredicate(join_predicate_index_),
query_context->getScalarGroup(selection_index_),
+ query_id_,
query_context->getInsertDestination(output_destination_index_),
storage_manager),
op_index_);
@@ -171,6 +172,7 @@ std::size_t NestedLoopsJoinOperator::getAllWorkOrdersHelperBothNotStored(WorkOrd
right_relation_block_ids_[right_index],
query_context->getPredicate(join_predicate_index_),
query_context->getScalarGroup(selection_index_),
+ query_id_,
query_context->getInsertDestination(output_destination_index_),
storage_manager),
op_index_);
@@ -205,6 +207,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrdersHelperOneStored(WorkOrdersContaine
right_relation_block_ids_[right_index],
join_predicate,
selection,
+ query_id_,
output_destination,
storage_manager),
op_index_);
@@ -224,6 +227,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrdersHelperOneStored(WorkOrdersContaine
right_block_id,
join_predicate,
selection,
+ query_id_,
output_destination,
storage_manager),
op_index_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/NestedLoopsJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/NestedLoopsJoinOperator.hpp b/relational_operators/NestedLoopsJoinOperator.hpp
index a52ca25..f165442 100644
--- a/relational_operators/NestedLoopsJoinOperator.hpp
+++ b/relational_operators/NestedLoopsJoinOperator.hpp
@@ -76,6 +76,7 @@ class NestedLoopsJoinOperator : public RelationalOperator {
* @param left_relation_is_stored If left_input_relation is a stored relation.
* @param right_relation_is_stored If right_input_relation is a stored
* relation.
+ * @param query_id The ID of the query to which this operator belongs.
**/
NestedLoopsJoinOperator(const CatalogRelation &left_input_relation,
const CatalogRelation &right_input_relation,
@@ -84,8 +85,10 @@ class NestedLoopsJoinOperator : public RelationalOperator {
const QueryContext::predicate_id join_predicate_index,
const QueryContext::scalar_group_id selection_index,
bool left_relation_is_stored,
- bool right_relation_is_stored)
- : left_input_relation_(left_input_relation),
+ bool right_relation_is_stored,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ left_input_relation_(left_input_relation),
right_input_relation_(right_input_relation),
output_relation_(output_relation),
output_destination_index_(output_destination_index),
@@ -230,6 +233,7 @@ class NestedLoopsJoinWorkOrder : public WorkOrder {
* @param selection A list of Scalars corresponding to the relation attributes
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
+ * @param query_id The ID of the query to which this operator belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
@@ -239,9 +243,11 @@ class NestedLoopsJoinWorkOrder : public WorkOrder {
const block_id right_block_id,
const Predicate *join_predicate,
const std::vector<std::unique_ptr<const Scalar>> &selection,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
- : left_input_relation_(left_input_relation),
+ : WorkOrder(query_id),
+ left_input_relation_(left_input_relation),
right_input_relation_(right_input_relation),
left_block_id_(left_block_id),
right_block_id_(right_block_id),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/RebuildWorkOrder.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/RebuildWorkOrder.hpp b/relational_operators/RebuildWorkOrder.hpp
index 5443d48..ae876ba 100644
--- a/relational_operators/RebuildWorkOrder.hpp
+++ b/relational_operators/RebuildWorkOrder.hpp
@@ -55,14 +55,17 @@ class RebuildWorkOrder : public WorkOrder {
* @param input_relation_id The ID of the CatalogRelation to which the given
* storage block belongs to.
* @param scheduler_client_id The TMB client ID of the scheduler thread.
+ * @param query_id The ID of the query to which this RebuildWorkOrder belongs.
* @param bus A pointer to the TMB.
**/
RebuildWorkOrder(MutableBlockReference &&block_ref,
const std::size_t input_operator_index,
const relation_id input_relation_id,
const client_id scheduler_client_id,
+ const std::size_t query_id,
MessageBus *bus)
- : block_ref_(std::move(block_ref)),
+ : WorkOrder(query_id),
+ block_ref_(std::move(block_ref)),
input_operator_index_(input_operator_index),
input_relation_id_(input_relation_id),
scheduler_client_id_(scheduler_client_id),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/RelationalOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/RelationalOperator.hpp b/relational_operators/RelationalOperator.hpp
index 75fde17..aa93018 100644
--- a/relational_operators/RelationalOperator.hpp
+++ b/relational_operators/RelationalOperator.hpp
@@ -208,16 +208,20 @@ class RelationalOperator {
/**
* @brief Constructor
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param blocking_dependencies_met If those dependencies which break the
* pipeline have been met.
**/
- explicit RelationalOperator(bool blocking_dependencies_met = false)
+ explicit RelationalOperator(const std::size_t query_id = 0,
+ const bool blocking_dependencies_met = false)
: blocking_dependencies_met_(blocking_dependencies_met),
- done_feeding_input_relation_(false) {}
+ done_feeding_input_relation_(false),
+ query_id_(query_id) {}
bool blocking_dependencies_met_;
bool done_feeding_input_relation_;
std::size_t op_index_;
+ const std::size_t query_id_;
private:
DISALLOW_COPY_AND_ASSIGN(RelationalOperator);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/SampleOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SampleOperator.cpp b/relational_operators/SampleOperator.cpp
index 6842b28..b318ce4 100644
--- a/relational_operators/SampleOperator.cpp
+++ b/relational_operators/SampleOperator.cpp
@@ -58,6 +58,7 @@ bool SampleOperator::getAllWorkOrders(
input_block_id,
is_block_sample_,
percentage_,
+ query_id_,
output_destination,
storage_manager),
op_index_);
@@ -72,6 +73,7 @@ bool SampleOperator::getAllWorkOrders(
input_block_id,
is_block_sample_,
percentage_,
+ query_id_,
output_destination,
storage_manager),
op_index_);
@@ -89,6 +91,7 @@ bool SampleOperator::getAllWorkOrders(
input_relation_block_ids_[num_workorders_generated_],
is_block_sample_,
percentage_,
+ query_id_,
output_destination,
storage_manager),
op_index_);
@@ -101,7 +104,9 @@ bool SampleOperator::getAllWorkOrders(
new SampleWorkOrder(input_relation_,
input_relation_block_ids_[num_workorders_generated_],
is_block_sample_,
- percentage_, output_destination,
+ percentage_,
+ query_id_,
+ output_destination,
storage_manager),
op_index_);
++num_workorders_generated_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/SampleOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SampleOperator.hpp b/relational_operators/SampleOperator.hpp
index 305de34..f65f28a 100644
--- a/relational_operators/SampleOperator.hpp
+++ b/relational_operators/SampleOperator.hpp
@@ -18,6 +18,7 @@
#ifndef QUICKSTEP_RELATIONAL_OPERATORS_SAMPLE_OPERATOR_HPP_
#define QUICKSTEP_RELATIONAL_OPERATORS_SAMPLE_OPERATOR_HPP_
+#include <cstddef>
#include <memory>
#include <vector>
@@ -64,15 +65,17 @@ class SampleOperator : public RelationalOperator {
* workorders.
* @param is_block_sample Flag indicating whether the sample type is block or tuple.
* @param percentage The percentage of data to be sampled.
- *
+ * @param query_id The ID of the query to which this operator belongs.
**/
SampleOperator(const CatalogRelation &input_relation,
const CatalogRelationSchema &output_relation,
const QueryContext::insert_destination_id output_destination_index,
const bool input_relation_is_stored,
const bool is_block_sample,
- const int percentage)
- : input_relation_(input_relation),
+ const int percentage,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ input_relation_(input_relation),
output_relation_(output_relation),
output_destination_index_(output_destination_index),
input_relation_is_stored_(input_relation_is_stored),
@@ -134,9 +137,11 @@ class SampleWorkOrder : public WorkOrder {
const block_id input_block_id,
const bool is_block_sample,
const int percentage,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
- : input_relation_(input_relation),
+ : WorkOrder(query_id),
+ input_relation_(input_relation),
input_block_id_(input_block_id),
is_block_sample_(is_block_sample),
percentage_(percentage),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/SaveBlocksOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SaveBlocksOperator.cpp b/relational_operators/SaveBlocksOperator.cpp
index ac61407..3581090 100644
--- a/relational_operators/SaveBlocksOperator.cpp
+++ b/relational_operators/SaveBlocksOperator.cpp
@@ -38,6 +38,7 @@ bool SaveBlocksOperator::getAllWorkOrders(
new SaveBlocksWorkOrder(
destination_block_ids_[num_workorders_generated_],
force_,
+ query_id_,
storage_manager),
op_index_);
++num_workorders_generated_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/SaveBlocksOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SaveBlocksOperator.hpp b/relational_operators/SaveBlocksOperator.hpp
index 49195ea..f4650bb 100644
--- a/relational_operators/SaveBlocksOperator.hpp
+++ b/relational_operators/SaveBlocksOperator.hpp
@@ -18,6 +18,7 @@
#ifndef QUICKSTEP_RELATIONAL_OPERATORS_SAVE_BLOCKS_OPERATOR_HPP_
#define QUICKSTEP_RELATIONAL_OPERATORS_SAVE_BLOCKS_OPERATOR_HPP_
+#include <cstddef>
#include <vector>
#include "catalog/CatalogTypedefs.hpp"
@@ -50,11 +51,13 @@ class SaveBlocksOperator : public RelationalOperator {
/**
* @brief Constructor for saving only modified blocks in a relation.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param force If true, force writing of all blocks to disk, otherwise only
* write dirty blocks.
**/
- explicit SaveBlocksOperator(bool force = false)
- : force_(force),
+ explicit SaveBlocksOperator(const std::size_t query_id, bool force = false)
+ : RelationalOperator(query_id),
+ force_(force),
num_workorders_generated_(0) {}
~SaveBlocksOperator() override {}
@@ -96,12 +99,15 @@ class SaveBlocksWorkOrder : public WorkOrder {
* @param save_block_id The id of the block to save.
* @param force If true, force writing of all blocks to disk, otherwise only
* write dirty blocks.
+ * @param query_id The ID of the query to which this operator belongs.
* @param storage_manager The StorageManager to use.
**/
SaveBlocksWorkOrder(const block_id save_block_id,
const bool force,
+ const std::size_t query_id,
StorageManager *storage_manager)
- : save_block_id_(save_block_id),
+ : WorkOrder(query_id),
+ save_block_id_(save_block_id),
force_(force),
storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/SelectOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.cpp b/relational_operators/SelectOperator.cpp
index 69bb434..350890d 100644
--- a/relational_operators/SelectOperator.cpp
+++ b/relational_operators/SelectOperator.cpp
@@ -49,6 +49,7 @@ void SelectOperator::addWorkOrders(WorkOrdersContainer *container,
simple_projection_,
simple_selection_,
selection,
+ query_id_,
output_destination,
storage_manager),
op_index_);
@@ -63,6 +64,7 @@ void SelectOperator::addWorkOrders(WorkOrdersContainer *container,
simple_projection_,
simple_selection_,
selection,
+ query_id_,
output_destination,
storage_manager),
op_index_);
@@ -91,6 +93,7 @@ void SelectOperator::addPartitionAwareWorkOrders(WorkOrdersContainer *container,
simple_projection_,
simple_selection_,
selection,
+ query_id_,
output_destination,
storage_manager,
placement_scheme_->getNUMANodeForBlock(input_block_id)),
@@ -111,6 +114,7 @@ void SelectOperator::addPartitionAwareWorkOrders(WorkOrdersContainer *container,
simple_projection_,
simple_selection_,
selection,
+ query_id_,
output_destination,
storage_manager,
placement_scheme_->getNUMANodeForBlock(block_in_partition)),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/SelectOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.hpp b/relational_operators/SelectOperator.hpp
index 76f4cb6..4f5b8ca 100644
--- a/relational_operators/SelectOperator.hpp
+++ b/relational_operators/SelectOperator.hpp
@@ -76,14 +76,17 @@ class SelectOperator : public RelationalOperator {
* @param input_relation_is_stored If input_relation is a stored relation and
* is fully available to the operator before it can start generating
* workorders.
+ * @param query_id The ID of the query to which this operator belongs.
**/
SelectOperator(const CatalogRelation &input_relation,
const CatalogRelation &output_relation,
const QueryContext::insert_destination_id output_destination_index,
const QueryContext::predicate_id predicate_index,
const QueryContext::scalar_group_id selection_index,
- const bool input_relation_is_stored)
- : input_relation_(input_relation),
+ const bool input_relation_is_stored,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ input_relation_(input_relation),
output_relation_(output_relation),
output_destination_index_(output_destination_index),
predicate_index_(predicate_index),
@@ -133,14 +136,17 @@ class SelectOperator : public RelationalOperator {
* @param input_relation_is_stored If input_relation is a stored relation and
* is fully available to the operator before it can start generating
* workorders.
+ * @param query_id The ID of the query to which this operator belongs.
**/
SelectOperator(const CatalogRelation &input_relation,
const CatalogRelation &output_relation,
const QueryContext::insert_destination_id output_destination_index,
const QueryContext::predicate_id predicate_index,
std::vector<attribute_id> &&selection,
- const bool input_relation_is_stored)
- : input_relation_(input_relation),
+ const bool input_relation_is_stored,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ input_relation_(input_relation),
output_relation_(output_relation),
output_destination_index_(output_destination_index),
predicate_index_(predicate_index),
@@ -281,6 +287,7 @@ class SelectWorkOrder : public WorkOrder {
* simple_projection is true.
* @param selection A list of Scalars which will be evaluated to project
* input tuples, used if \c simple_projection is false.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the selection
* results.
* @param storage_manager The StorageManager to use.
@@ -291,10 +298,12 @@ class SelectWorkOrder : public WorkOrder {
const bool simple_projection,
const std::vector<attribute_id> &simple_selection,
const std::vector<std::unique_ptr<const Scalar>> *selection,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager,
const numa_node_id numa_node = 0)
- : input_relation_(input_relation),
+ : WorkOrder(query_id),
+ input_relation_(input_relation),
input_block_id_(input_block_id),
predicate_(predicate),
simple_projection_(simple_projection),
@@ -320,6 +329,7 @@ class SelectWorkOrder : public WorkOrder {
* simple_projection is true.
* @param selection A list of Scalars which will be evaluated to project
* input tuples, used if \c simple_projection is false.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the selection
* results.
* @param storage_manager The StorageManager to use.
@@ -330,10 +340,12 @@ class SelectWorkOrder : public WorkOrder {
const bool simple_projection,
std::vector<attribute_id> &&simple_selection,
const std::vector<std::unique_ptr<const Scalar>> *selection,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager,
const numa_node_id numa_node = 0)
- : input_relation_(input_relation),
+ : WorkOrder(query_id),
+ input_relation_(input_relation),
input_block_id_(input_block_id),
predicate_(predicate),
simple_projection_(simple_projection),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/SortMergeRunOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortMergeRunOperator.cpp b/relational_operators/SortMergeRunOperator.cpp
index 7427d44..9db8de1 100644
--- a/relational_operators/SortMergeRunOperator.cpp
+++ b/relational_operators/SortMergeRunOperator.cpp
@@ -90,6 +90,7 @@ WorkOrder *SortMergeRunOperator::createWorkOrder(
std::move(job->runs),
top_k_,
job->level,
+ query_id_,
output_destination,
storage_manager,
op_index_,
[14/48] incubator-quickstep git commit: Transaction Part 4:
LockManager, CycleDetector and DeadLockDetector. (#187)
Posted by hb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/tests/StronglyConnectedComponents_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/StronglyConnectedComponents_unittest.cpp b/transaction/tests/StronglyConnectedComponents_unittest.cpp
index 79d6881..35ef842 100644
--- a/transaction/tests/StronglyConnectedComponents_unittest.cpp
+++ b/transaction/tests/StronglyConnectedComponents_unittest.cpp
@@ -35,21 +35,20 @@ namespace transaction {
class GraphConfiguration {
public:
GraphConfiguration(DirectedGraph *graph,
- std::size_t no_transaction,
+ const std::size_t num_transactions,
const std::vector<std::pair<transaction_id,
transaction_id>> &mapping)
: graph_(graph) {
- for (std::size_t index = 0; index < no_transaction; ++index) {
- std::unique_ptr<transaction_id> tid =
- std::make_unique<transaction_id>(transaction_id(index));
- transaction_list_.push_back(*tid);
- DirectedGraph::node_id nid = graph->addNodeUnchecked(tid.release());
+ for (std::size_t index = 0; index < num_transactions; ++index) {
+ const transaction_id transaction = static_cast<transaction_id>(index);
+ transaction_list_.push_back(transaction);
+ const DirectedGraph::node_id nid = graph->addNodeUnchecked(transaction);
node_id_list_.push_back(nid);
}
for (const std::pair<transaction_id, transaction_id> &edge : mapping) {
- transaction_id pending = edge.first;
- transaction_id owner = edge.second;
+ const transaction_id pending = edge.first;
+ const transaction_id owner = edge.second;
graph_->addEdgeUnchecked(pending, owner);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/tests/TransactionTable_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/TransactionTable_unittest.cpp b/transaction/tests/TransactionTable_unittest.cpp
index f5b5bc9..cd47a2b 100644
--- a/transaction/tests/TransactionTable_unittest.cpp
+++ b/transaction/tests/TransactionTable_unittest.cpp
@@ -41,91 +41,101 @@ class TransactionTableTest : public ::testing::Test {
};
TEST_F(TransactionTableTest, NormalOperations) {
- EXPECT_EQ(transaction_table_.putOwnEntry(tid_1_,
+ const AccessMode is_lock_mode = AccessMode::IsLockMode();
+ const AccessMode x_lock_mode = AccessMode::XLockMode();
+
+ EXPECT_EQ(TransactionTableResult::kPlacedInOwned,
+ transaction_table_.putOwnEntry(tid_1_,
ResourceId(3),
- AccessMode(AccessModeType::kIsLock)),
- TransactionTableResult::kPlacedInOwned);
+ is_lock_mode));
+
- EXPECT_EQ(transaction_table_.putPendingEntry(tid_1_,
+ EXPECT_EQ(TransactionTableResult::kPlacedInPending,
+ transaction_table_.putPendingEntry(tid_1_,
ResourceId(5),
- AccessMode(AccessModeType::kXLock)),
- TransactionTableResult::kPlacedInPending);
+ x_lock_mode));
}
TEST_F(TransactionTableTest, DeleteEntryOperations) {
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_2_,
+ const AccessMode s_lock_mode = AccessMode::SLockMode();
+ const AccessMode x_lock_mode = AccessMode::XLockMode();
+
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deleteOwnEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.putOwnEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kPlacedInOwned,
+ transaction_table_.putOwnEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kPlacedInOwned);
+ s_lock_mode));
// Tring to delete a lock with different acces mode on same resource id
// will result in an error.
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deleteOwnEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kXLock)),
- TransactionTableResult::kDelError);
+ x_lock_mode));
// Transaction 3 does not have a lock on this resource id.
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_3_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deleteOwnEntry(tid_3_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
// This will result in success since transaction 2 have acquired the lock on
// this resource with the corresponding mode.
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kDelFromOwned,
+ transaction_table_.deleteOwnEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelFromOwned);
+ s_lock_mode));
// Repeat the previous sequence, with pending list.
- EXPECT_EQ(transaction_table_.deletePendingEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deletePendingEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.putPendingEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kPlacedInPending,
+ transaction_table_.putPendingEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kPlacedInPending);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.deletePendingEntry(tid_2_,
+
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deletePendingEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kXLock)),
- TransactionTableResult::kDelError);
+ x_lock_mode));
- EXPECT_EQ(transaction_table_.deletePendingEntry(tid_3_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deletePendingEntry(tid_3_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.deletePendingEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kDelFromPending,
+ transaction_table_.deletePendingEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelFromPending);
+ s_lock_mode));
}
TEST_F(TransactionTableTest, TransactionEntries) {
- EXPECT_EQ(transaction_table_.deleteTransaction(tid_1_),
- TransactionTableResult::kTransactionDeleteError);
+ const AccessMode s_lock_mode = AccessMode::SLockMode();
+
+ EXPECT_EQ(TransactionTableResult::kTransactionDeleteError,
+ transaction_table_.deleteTransaction(tid_1_));
- EXPECT_EQ(transaction_table_.putOwnEntry(tid_1_,
+ EXPECT_EQ(TransactionTableResult::kPlacedInOwned,
+ transaction_table_.putOwnEntry(tid_1_,
ResourceId(4),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kPlacedInOwned);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.deleteTransaction(tid_1_),
- TransactionTableResult::kTransactionDeleteOk);
+ EXPECT_EQ(TransactionTableResult::kTransactionDeleteOk,
+ transaction_table_.deleteTransaction(tid_1_));
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_1_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deleteOwnEntry(tid_1_,
ResourceId(4),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
}
} // namespace transaction
[39/48] incubator-quickstep git commit: QUICKSTEP-1: Add incubation
disclaimer to README.md.
Posted by hb...@apache.org.
QUICKSTEP-1: Add incubation disclaimer to README.md.
This closes #2
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/037ef4ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/037ef4ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/037ef4ba
Branch: refs/heads/query-manager-used-in-foreman
Commit: 037ef4ba77f88067627a7f581101af0701dcf5a8
Parents: 13281d8
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Tue Apr 19 12:23:51 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:46 2016 -0700
----------------------------------------------------------------------
DISCLAIMER | 11 +++++++++++
README.md | 21 ++++++++++++++++++---
2 files changed, 29 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/037ef4ba/DISCLAIMER
----------------------------------------------------------------------
diff --git a/DISCLAIMER b/DISCLAIMER
new file mode 100644
index 0000000..09d629d
--- /dev/null
+++ b/DISCLAIMER
@@ -0,0 +1,11 @@
+Apache Quickstep is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further
+review indicates that the infrastructure, communications, and decision
+making process have stabilized in a manner consistent with other
+successful ASF projects.
+
+While incubation status is not necessarily a reflection of the
+completeness or stability of the code, it does indicate that the
+project has yet to be fully endorsed by the ASF.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/037ef4ba/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index c44e0b9..7cea80c 100644
--- a/README.md
+++ b/README.md
@@ -1,14 +1,16 @@
-# Quickstep
+# Apache Quickstep (Incubating)
[![Travis Widget]][Travis]
[Travis]: https://travis-ci.org/pivotalsoftware/quickstep
[Travis Widget]: https://travis-ci.org/pivotalsoftware/quickstep.svg?branch=master
-Quickstep is an experimental high-performance database engine designed with the
-aim of Data at Bare-Metal Speed. It began life in 2011 as a
+Apache Quickstep is an experimental high-performance database engine designed with the
+aim of Data at Bare-Metal Speed. It began life in 2011 as a
[research project at the University of Wisconsin](https://quickstep.cs.wisc.edu)
and was acquired by [Pivotal](https://pivotal.io) in 2015.
+Quickstep entered incubation at the
+[Apache Software Foundation](https://www.apache.org) in April, 2016.
## Getting Started (Building)
@@ -91,3 +93,16 @@ concerns of a database system. The main modules are:
## Licensing
Quickstep is licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/pivotalsoftware/quickstep/blob/master/LICENSE) for the full license text.
+
+## Disclaimer
+Apache Quickstep is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further
+review indicates that the infrastructure, communications, and decision
+making process have stabilized in a manner consistent with other
+successful ASF projects.
+
+While incubation status is not necessarily a reflection of the
+completeness or stability of the code, it does indicate that the
+project has yet to be fully endorsed by the ASF.
[47/48] incubator-quickstep git commit: Reordered Query ID in
operators and work orders.
Posted by hb...@apache.org.
Reordered Query ID in operators and work orders.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1be47dcb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1be47dcb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1be47dcb
Branch: refs/heads/query-manager-used-in-foreman
Commit: 1be47dcbd79e6e2ee01f237d0dbaa8a97e562519
Parents: 4054268
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Fri Jun 3 15:40:34 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 14:00:13 2016 -0700
----------------------------------------------------------------------
query_execution/Foreman.cpp | 12 +-
query_execution/QueryManager.cpp | 12 +-
query_execution/tests/Foreman_unittest.cpp | 3 +-
query_execution/tests/QueryManager_unittest.cpp | 3 +-
query_optimizer/ExecutionGenerator.cpp | 194 ++++++++--------
.../tests/ExecutionHeuristics_unittest.cpp | 34 +--
relational_operators/AggregationOperator.cpp | 4 +-
relational_operators/AggregationOperator.hpp | 14 +-
relational_operators/BuildHashOperator.cpp | 6 +-
relational_operators/BuildHashOperator.hpp | 20 +-
relational_operators/CreateIndexOperator.hpp | 6 +-
relational_operators/DeleteOperator.cpp | 8 +-
relational_operators/DeleteOperator.hpp | 14 +-
relational_operators/DestroyHashOperator.cpp | 2 +-
relational_operators/DestroyHashOperator.hpp | 12 +-
relational_operators/DropTableOperator.hpp | 6 +-
.../FinalizeAggregationOperator.hpp | 11 +-
relational_operators/HashJoinOperator.cpp | 31 +--
relational_operators/HashJoinOperator.hpp | 229 ++++++++++---------
relational_operators/InsertOperator.hpp | 11 +-
.../NestedLoopsJoinOperator.cpp | 45 ++--
.../NestedLoopsJoinOperator.hpp | 52 +++--
relational_operators/RebuildWorkOrder.hpp | 15 +-
relational_operators/RelationalOperator.hpp | 11 +-
relational_operators/SampleOperator.cpp | 52 +++--
relational_operators/SampleOperator.hpp | 39 ++--
relational_operators/SaveBlocksOperator.cpp | 2 +-
relational_operators/SaveBlocksOperator.hpp | 9 +-
relational_operators/SelectOperator.cpp | 27 ++-
relational_operators/SelectOperator.hpp | 56 ++---
relational_operators/SortMergeRunOperator.cpp | 2 +-
relational_operators/SortMergeRunOperator.hpp | 27 +--
.../SortRunGenerationOperator.cpp | 6 +-
.../SortRunGenerationOperator.hpp | 26 ++-
relational_operators/TableGeneratorOperator.cpp | 2 +-
relational_operators/TableGeneratorOperator.hpp | 20 +-
relational_operators/TextScanOperator.cpp | 21 +-
relational_operators/TextScanOperator.hpp | 32 +--
relational_operators/UpdateOperator.cpp | 22 +-
relational_operators/UpdateOperator.hpp | 37 +--
relational_operators/WorkOrderFactory.cpp | 38 +--
.../tests/AggregationOperator_unittest.cpp | 23 +-
.../tests/HashJoinOperator_unittest.cpp | 181 ++++++++-------
.../tests/SortMergeRunOperator_unittest.cpp | 14 +-
.../SortRunGenerationOperator_unittest.cpp | 16 +-
.../tests/TextScanOperator_unittest.cpp | 7 +-
46 files changed, 745 insertions(+), 669 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/query_execution/Foreman.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Foreman.cpp b/query_execution/Foreman.cpp
index b358f70..7705819 100644
--- a/query_execution/Foreman.cpp
+++ b/query_execution/Foreman.cpp
@@ -521,12 +521,12 @@ void Foreman::getRebuildWorkOrders(const dag_node_index index, WorkOrdersContain
// Note: The query ID used below is dummy for now, it will be replaced with
// the true query ID when QueryManager gets used in Foreman.
container->addRebuildWorkOrder(
- new RebuildWorkOrder(move(partially_filled_block_refs[i]),
- index,
- op.getOutputRelationID(),
- foreman_client_id_,
- 0,
- bus_),
+ new RebuildWorkOrder(0,
+ move(partially_filled_block_refs[i]),
+ index,
+ op.getOutputRelationID(),
+ foreman_client_id_,
+ bus_),
index);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/query_execution/QueryManager.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManager.cpp b/query_execution/QueryManager.cpp
index 21f5820..e4e4c9d 100644
--- a/query_execution/QueryManager.cpp
+++ b/query_execution/QueryManager.cpp
@@ -457,12 +457,12 @@ void QueryManager::getRebuildWorkOrders(const dag_node_index index,
i < partially_filled_block_refs.size();
++i) {
container->addRebuildWorkOrder(
- new RebuildWorkOrder(std::move(partially_filled_block_refs[i]),
- index,
- op.getOutputRelationID(),
- foreman_client_id_,
- query_id_,
- bus_),
+ new RebuildWorkOrder(query_id_,
+ std::move(partially_filled_block_refs[i]),
+ index,
+ op.getOutputRelationID(),
+ foreman_client_id_,
+ bus_),
index);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/query_execution/tests/Foreman_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/Foreman_unittest.cpp b/query_execution/tests/Foreman_unittest.cpp
index d2f43a4..79f8f4a 100644
--- a/query_execution/tests/Foreman_unittest.cpp
+++ b/query_execution/tests/Foreman_unittest.cpp
@@ -90,7 +90,8 @@ class MockOperator: public RelationalOperator {
const bool has_streaming_input,
const int max_getworkorder_iters = 1,
const int max_workorders = INT_MAX)
- : produce_workorders_(produce_workorders),
+ : RelationalOperator(0 /* Query Id */),
+ produce_workorders_(produce_workorders),
has_streaming_input_(has_streaming_input),
max_workorders_(max_workorders),
max_getworkorder_iters_(max_getworkorder_iters),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/query_execution/tests/QueryManager_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/QueryManager_unittest.cpp b/query_execution/tests/QueryManager_unittest.cpp
index 80876f2..308d5ca 100644
--- a/query_execution/tests/QueryManager_unittest.cpp
+++ b/query_execution/tests/QueryManager_unittest.cpp
@@ -91,7 +91,8 @@ class MockOperator: public RelationalOperator {
const bool has_streaming_input,
const int max_getworkorder_iters = 1,
const int max_workorders = INT_MAX)
- : produce_workorders_(produce_workorders),
+ : RelationalOperator(0 /* Query Id */),
+ produce_workorders_(produce_workorders),
has_streaming_input_(has_streaming_input),
max_workorders_(max_workorders),
max_getworkorder_iters_(max_getworkorder_iters),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 30dfa8e..99c2a21 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -193,8 +193,8 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) {
}
const QueryPlan::DAGNodeIndex drop_table_index =
execution_plan_->addRelationalOperator(
- new DropTableOperator(*temporary_relation,
- query_handle_->query_id(),
+ new DropTableOperator(query_handle_->query_id(),
+ *temporary_relation,
optimizer_context_->catalog_database(),
false /* only_drop_blocks */));
DCHECK(!temporary_relation_info.isStoredRelation());
@@ -411,13 +411,14 @@ void ExecutionGenerator::convertSample(const P::SamplePtr &physical_sample) {
findRelationInfoOutputByPhysical(physical_sample->input());
DCHECK(input_relation_info != nullptr);
- SampleOperator *sample_op = new SampleOperator(*input_relation_info->relation,
- *output_relation,
- insert_destination_index,
- input_relation_info->isStoredRelation(),
- physical_sample->is_block_sample(),
- physical_sample->percentage(),
- query_handle_->query_id());
+ SampleOperator *sample_op =
+ new SampleOperator(query_handle_->query_id(),
+ *input_relation_info->relation,
+ *output_relation,
+ insert_destination_index,
+ input_relation_info->isStoredRelation(),
+ physical_sample->is_block_sample(),
+ physical_sample->percentage());
const QueryPlan::DAGNodeIndex sample_index =
execution_plan_->addRelationalOperator(sample_op);
insert_destination_proto->set_relational_op_index(sample_index);
@@ -526,22 +527,22 @@ void ExecutionGenerator::convertSelection(
// doesn't require any expression evaluation or intermediate copies) if
// possible.
std::vector<attribute_id> attributes;
- SelectOperator *op
- = convertSimpleProjection(project_expressions_group_index, &attributes)
- ? new SelectOperator(*input_relation_info->relation,
- *output_relation,
- insert_destination_index,
- execution_predicate_index,
- move(attributes),
- input_relation_info->isStoredRelation(),
- query_handle_->query_id())
- : new SelectOperator(*input_relation_info->relation,
- *output_relation,
- insert_destination_index,
- execution_predicate_index,
- project_expressions_group_index,
- input_relation_info->isStoredRelation(),
- query_handle_->query_id());
+ SelectOperator *op =
+ convertSimpleProjection(project_expressions_group_index, &attributes)
+ ? new SelectOperator(query_handle_->query_id(),
+ *input_relation_info->relation,
+ *output_relation,
+ insert_destination_index,
+ execution_predicate_index,
+ move(attributes),
+ input_relation_info->isStoredRelation())
+ : new SelectOperator(query_handle_->query_id(),
+ *input_relation_info->relation,
+ *output_relation,
+ insert_destination_index,
+ execution_predicate_index,
+ project_expressions_group_index,
+ input_relation_info->isStoredRelation());
const QueryPlan::DAGNodeIndex select_index =
execution_plan_->addRelationalOperator(op);
@@ -741,12 +742,12 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
const QueryPlan::DAGNodeIndex build_operator_index =
execution_plan_->addRelationalOperator(
new BuildHashOperator(
+ query_handle_->query_id(),
*build_relation_info->relation,
build_relation_info->isStoredRelation(),
build_attribute_ids,
any_build_attributes_nullable,
- join_hash_table_index,
- query_handle_->query_id()));
+ join_hash_table_index));
// Create InsertDestination proto.
const CatalogRelation *output_relation = nullptr;
@@ -782,6 +783,7 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
const QueryPlan::DAGNodeIndex join_operator_index =
execution_plan_->addRelationalOperator(
new HashJoinOperator(
+ query_handle_->query_id(),
*build_relation_info->relation,
*probe_operator_info->relation,
probe_operator_info->isStoredRelation(),
@@ -792,14 +794,13 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
join_hash_table_index,
residual_predicate_index,
project_expressions_group_index,
- query_handle_->query_id(),
is_selection_on_build.get(),
join_type));
insert_destination_proto->set_relational_op_index(join_operator_index);
const QueryPlan::DAGNodeIndex destroy_operator_index =
execution_plan_->addRelationalOperator(new DestroyHashOperator(
- join_hash_table_index, query_handle_->query_id()));
+ query_handle_->query_id(), join_hash_table_index));
if (!build_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(build_operator_index,
@@ -885,16 +886,16 @@ void ExecutionGenerator::convertNestedLoopsJoin(
// Create and add a NestedLoopsJoin operator.
const QueryPlan::DAGNodeIndex join_operator_index =
- execution_plan_->addRelationalOperator(new NestedLoopsJoinOperator(
- *left_relation_info->relation,
- *right_relation_info->relation,
- *output_relation,
- insert_destination_index,
- execution_join_predicate_index,
- project_expressions_group_index,
- left_relation_info->isStoredRelation(),
- right_relation_info->isStoredRelation(),
- query_handle_->query_id()));
+ execution_plan_->addRelationalOperator(
+ new NestedLoopsJoinOperator(query_handle_->query_id(),
+ *left_relation_info->relation,
+ *right_relation_info->relation,
+ *output_relation,
+ insert_destination_index,
+ execution_join_predicate_index,
+ project_expressions_group_index,
+ left_relation_info->isStoredRelation(),
+ right_relation_info->isStoredRelation()));
insert_destination_proto->set_relational_op_index(join_operator_index);
if (!left_relation_info->isStoredRelation()) {
@@ -940,13 +941,13 @@ void ExecutionGenerator::convertCopyFrom(
const QueryPlan::DAGNodeIndex scan_operator_index =
execution_plan_->addRelationalOperator(
new TextScanOperator(
+ query_handle_->query_id(),
physical_plan->file_name(),
physical_plan->column_delimiter(),
physical_plan->escape_strings(),
FLAGS_parallelize_load,
*output_relation,
- insert_destination_index,
- query_handle_->query_id()));
+ insert_destination_index));
insert_destination_proto->set_relational_op_index(scan_operator_index);
const QueryPlan::DAGNodeIndex save_blocks_operator_index =
@@ -997,10 +998,11 @@ void ExecutionGenerator::convertCreateIndex(
// Check if the given index description is valid.
THROW_SQL_ERROR() << "The index with given properties cannot be created.";
}
- execution_plan_->addRelationalOperator(new CreateIndexOperator(input_relation,
- physical_plan->index_name(),
- query_handle_->query_id(),
- std::move(index_description)));
+ execution_plan_->addRelationalOperator(
+ new CreateIndexOperator(query_handle_->query_id(),
+ input_relation,
+ physical_plan->index_name(),
+ std::move(index_description)));
}
void ExecutionGenerator::convertCreateTable(
@@ -1067,8 +1069,8 @@ void ExecutionGenerator::convertDeleteTuples(
execution_predicate->getStaticResult())) {
const QueryPlan::DAGNodeIndex drop_table_index =
execution_plan_->addRelationalOperator(
- new DropTableOperator(*input_relation_info->relation,
- query_handle_->query_id(),
+ new DropTableOperator(query_handle_->query_id(),
+ *input_relation_info->relation,
optimizer_context_->catalog_database(),
true /* only_drop_blocks */));
if (!input_relation_info->isStoredRelation()) {
@@ -1081,11 +1083,12 @@ void ExecutionGenerator::convertDeleteTuples(
query_context_proto_->add_predicates()->CopyFrom(execution_predicate->getProto());
const QueryPlan::DAGNodeIndex delete_tuples_index =
- execution_plan_->addRelationalOperator(new DeleteOperator(
- *input_relation_info->relation,
- execution_predicate_index,
- input_relation_info->isStoredRelation(),
- query_handle_->query_id()));
+ execution_plan_->addRelationalOperator(
+ new DeleteOperator(query_handle_->query_id(),
+ *input_relation_info->relation,
+ execution_predicate_index,
+ input_relation_info->isStoredRelation()));
+
if (!input_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(delete_tuples_index,
input_relation_info->producer_operator_index,
@@ -1111,8 +1114,8 @@ void ExecutionGenerator::convertDropTable(
#endif
execution_plan_->addRelationalOperator(
- new DropTableOperator(catalog_relation,
- query_handle_->query_id(),
+ new DropTableOperator(query_handle_->query_id(),
+ catalog_relation,
optimizer_context_->catalog_database()));
}
@@ -1164,10 +1167,10 @@ void ExecutionGenerator::convertInsertTuple(
const QueryPlan::DAGNodeIndex insert_operator_index =
execution_plan_->addRelationalOperator(
- new InsertOperator(input_relation,
+ new InsertOperator(query_handle_->query_id(),
+ input_relation,
insert_destination_index,
- tuple_index,
- query_handle_->query_id()));
+ tuple_index));
insert_destination_proto->set_relational_op_index(insert_operator_index);
const QueryPlan::DAGNodeIndex save_blocks_index =
@@ -1240,13 +1243,13 @@ void ExecutionGenerator::convertInsertSelection(
// optimization is to enable specifying a specific output relation for each
// physical plan by modifying class Physical.
SelectOperator *insert_selection_op =
- new SelectOperator(*selection_relation_info->relation,
+ new SelectOperator(query_handle_->query_id(),
+ *selection_relation_info->relation,
destination_relation,
insert_destination_index,
QueryContext::kInvalidPredicateId,
move(attributes),
- selection_relation_info->isStoredRelation(),
- query_handle_->query_id());
+ selection_relation_info->isStoredRelation());
const QueryPlan::DAGNodeIndex insert_selection_index =
execution_plan_->addRelationalOperator(insert_selection_op);
@@ -1321,13 +1324,13 @@ void ExecutionGenerator::convertUpdateTable(
}
const QueryPlan::DAGNodeIndex update_operator_index =
- execution_plan_->addRelationalOperator(
- new UpdateOperator(
- *optimizer_context_->catalog_database()->getRelationById(input_rel_id),
- relocation_destination_index,
- execution_predicate_index,
- update_group_index,
- query_handle_->query_id()));
+ execution_plan_->addRelationalOperator(new UpdateOperator(
+ query_handle_->query_id(),
+ *optimizer_context_->catalog_database()->getRelationById(
+ input_rel_id),
+ relocation_destination_index,
+ execution_predicate_index,
+ update_group_index));
relocation_destination_proto->set_relational_op_index(update_operator_index);
const QueryPlan::DAGNodeIndex save_blocks_index =
@@ -1429,10 +1432,11 @@ void ExecutionGenerator::convertAggregate(
const QueryPlan::DAGNodeIndex aggregation_operator_index =
execution_plan_->addRelationalOperator(
new AggregationOperator(
+ query_handle_->query_id(),
*input_relation_info->relation,
input_relation_info->isStoredRelation(),
- aggr_state_index,
- query_handle_->query_id()));
+ aggr_state_index));
+
if (!input_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(aggregation_operator_index,
input_relation_info->producer_operator_index,
@@ -1450,10 +1454,11 @@ void ExecutionGenerator::convertAggregate(
const QueryPlan::DAGNodeIndex finalize_aggregation_operator_index =
execution_plan_->addRelationalOperator(
- new FinalizeAggregationOperator(aggr_state_index,
+ new FinalizeAggregationOperator(query_handle_->query_id(),
+ aggr_state_index,
*output_relation,
- insert_destination_index,
- query_handle_->query_id()));
+ insert_destination_index));
+
insert_destination_proto->set_relational_op_index(finalize_aggregation_operator_index);
execution_plan_->addDirectDependency(finalize_aggregation_operator_index,
@@ -1499,13 +1504,13 @@ void ExecutionGenerator::convertSort(const P::SortPtr &physical_sort) {
const CatalogRelationInfo *input_relation_info =
findRelationInfoOutputByPhysical(physical_sort->input());
const QueryPlan::DAGNodeIndex run_generator_index =
- execution_plan_->addRelationalOperator(
- new SortRunGenerationOperator(*input_relation_info->relation,
- *initial_runs_relation,
- initial_runs_destination_id,
- sort_run_gen_config_id,
- input_relation_info->isStoredRelation(),
- query_handle_->query_id()));
+ execution_plan_->addRelationalOperator(new SortRunGenerationOperator(
+ query_handle_->query_id(),
+ *input_relation_info->relation,
+ *initial_runs_relation,
+ initial_runs_destination_id,
+ sort_run_gen_config_id,
+ input_relation_info->isStoredRelation()));
if (!input_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(run_generator_index,
input_relation_info->producer_operator_index,
@@ -1553,17 +1558,18 @@ void ExecutionGenerator::convertSort(const P::SortPtr &physical_sort) {
// TODO(qzeng): Make the merge factor configurable.
const QueryPlan::DAGNodeIndex merge_run_operator_index =
- execution_plan_->addRelationalOperator(
- new SortMergeRunOperator(*initial_runs_relation,
- *sorted_relation,
- sorted_output_destination_id,
- *merged_runs_relation,
- merged_runs_destination_id,
- sort_merge_run_config_id,
- 64 /* merge_factor */,
- physical_sort->limit(),
- false /* input_relation_is_stored */,
- query_handle_->query_id()));
+ execution_plan_->addRelationalOperator(new SortMergeRunOperator(
+ query_handle_->query_id(),
+ *initial_runs_relation,
+ *sorted_relation,
+ sorted_output_destination_id,
+ *merged_runs_relation,
+ merged_runs_destination_id,
+ sort_merge_run_config_id,
+ 64 /* merge_factor */,
+ physical_sort->limit(),
+ false /* input_relation_is_stored */));
+
execution_plan_->addDirectDependency(merge_run_operator_index,
run_generator_index,
false /* is_pipeline_breaker */);
@@ -1576,8 +1582,8 @@ void ExecutionGenerator::convertSort(const P::SortPtr &physical_sort) {
const QueryPlan::DAGNodeIndex drop_merged_runs_index =
execution_plan_->addRelationalOperator(
new DropTableOperator(
- *merged_runs_relation,
query_handle_->query_id(),
+ *merged_runs_relation,
optimizer_context_->catalog_database(),
false /* only_drop_blocks */));
execution_plan_->addDirectDependency(
@@ -1613,10 +1619,10 @@ void ExecutionGenerator::convertTableGenerator(
physical_tablegen->generator_function_handle()->getProto());
TableGeneratorOperator *op =
- new TableGeneratorOperator(*output_relation,
+ new TableGeneratorOperator(query_handle_->query_id(),
+ *output_relation,
insert_destination_index,
- generator_function_index,
- query_handle_->query_id());
+ generator_function_index);
const QueryPlan::DAGNodeIndex tablegen_index =
execution_plan_->addRelationalOperator(op);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/ExecutionHeuristics_unittest.cpp b/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
index a08a476..815c13e 100644
--- a/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
+++ b/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
@@ -15,6 +15,7 @@
* limitations under the License.
**/
+#include <cstddef>
#include <memory>
#include <string>
#include <vector>
@@ -36,6 +37,10 @@
namespace quickstep {
namespace optimizer {
+namespace {
+constexpr std::size_t kQueryId = 0;
+}
+
class ExecutionHeuristicsTest : public ::testing::Test {
protected:
virtual void SetUp() {
@@ -75,12 +80,12 @@ class ExecutionHeuristicsTest : public ::testing::Test {
std::vector<attribute_id> build_attribute_ids;
build_attribute_ids.push_back(build_attribute_id);
QueryPlan::DAGNodeIndex build_operator_index =
- query_plan->addRelationalOperator(new BuildHashOperator(*build_relation,
+ query_plan->addRelationalOperator(new BuildHashOperator(kQueryId,
+ *build_relation,
true,
build_attribute_ids,
false,
- join_hash_table_index,
- 0 /* dummy query ID */));
+ join_hash_table_index));
return build_operator_index;
}
@@ -92,17 +97,18 @@ class ExecutionHeuristicsTest : public ::testing::Test {
std::vector<attribute_id> probe_attribute_ids;
probe_attribute_ids.push_back(probe_attribute_id);
QueryPlan::DAGNodeIndex join_operator_index =
- query_plan->addRelationalOperator(new HashJoinOperator(*build_relation,
- *probe_relation,
- true,
- probe_attribute_ids,
- false,
- *probe_relation,
- 0,
- join_hash_table_index,
- 0,
- 0,
- 0 /* dummy query ID */));
+ query_plan->addRelationalOperator(
+ new HashJoinOperator(kQueryId,
+ *build_relation,
+ *probe_relation,
+ true,
+ probe_attribute_ids,
+ false,
+ *probe_relation,
+ 0,
+ join_hash_table_index,
+ 0,
+ 0));
return join_operator_index;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/AggregationOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/AggregationOperator.cpp b/relational_operators/AggregationOperator.cpp
index 94ba901..7252541 100644
--- a/relational_operators/AggregationOperator.cpp
+++ b/relational_operators/AggregationOperator.cpp
@@ -39,8 +39,8 @@ bool AggregationOperator::getAllWorkOrders(
for (const block_id input_block_id : input_relation_block_ids_) {
container->addNormalWorkOrder(
new AggregationWorkOrder(
- input_block_id,
query_id_,
+ input_block_id,
query_context->getAggregationState(aggr_state_index_)),
op_index_);
}
@@ -51,8 +51,8 @@ bool AggregationOperator::getAllWorkOrders(
while (num_workorders_generated_ < input_relation_block_ids_.size()) {
container->addNormalWorkOrder(
new AggregationWorkOrder(
- input_relation_block_ids_[num_workorders_generated_],
query_id_,
+ input_relation_block_ids_[num_workorders_generated_],
query_context->getAggregationState(aggr_state_index_)),
op_index_);
++num_workorders_generated_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/AggregationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/AggregationOperator.hpp b/relational_operators/AggregationOperator.hpp
index 0fbc381..f340d4e 100644
--- a/relational_operators/AggregationOperator.hpp
+++ b/relational_operators/AggregationOperator.hpp
@@ -53,17 +53,17 @@ class AggregationOperator : public RelationalOperator {
* @brief Constructor for aggregating with arbitrary expressions in projection
* list.
*
+ * @param query_id The ID of this query.
* @param input_relation The relation to perform aggregation over.
* @param input_relation_is_stored If input_relation is a stored relation and
* is fully available to the operator before it can start generating
* workorders.
* @param aggr_state_index The index of the AggregationState in QueryContext.
- * @param query_id The ID of this query.
**/
- AggregationOperator(const CatalogRelation &input_relation,
+ AggregationOperator(const std::size_t query_id,
+ const CatalogRelation &input_relation,
bool input_relation_is_stored,
- const QueryContext::aggregation_state_id aggr_state_index,
- const std::size_t query_id)
+ const QueryContext::aggregation_state_id aggr_state_index)
: RelationalOperator(query_id),
input_relation_is_stored_(input_relation_is_stored),
input_relation_block_ids_(input_relation_is_stored ? input_relation.getBlocksSnapshot()
@@ -109,12 +109,12 @@ class AggregationWorkOrder : public WorkOrder {
/**
* @brief Constructor
*
- * @param input_block_id The block id.
* @param query_id The ID of this query.
+ * @param input_block_id The block id.
* @param state The AggregationState to use.
**/
- AggregationWorkOrder(const block_id input_block_id,
- const std::size_t query_id,
+ AggregationWorkOrder(const std::size_t query_id,
+ const block_id input_block_id,
AggregationOperationState *state)
: WorkOrder(query_id),
input_block_id_(input_block_id),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/BuildHashOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/BuildHashOperator.cpp b/relational_operators/BuildHashOperator.cpp
index c6f6f96..9dc4afe 100644
--- a/relational_operators/BuildHashOperator.cpp
+++ b/relational_operators/BuildHashOperator.cpp
@@ -69,11 +69,11 @@ bool BuildHashOperator::getAllWorkOrders(
if (!started_) {
for (const block_id input_block_id : input_relation_block_ids_) {
container->addNormalWorkOrder(
- new BuildHashWorkOrder(input_relation_,
+ new BuildHashWorkOrder(query_id_,
+ input_relation_,
join_key_attributes_,
any_join_key_attributes_nullable_,
input_block_id,
- query_id_,
hash_table,
storage_manager),
op_index_);
@@ -85,11 +85,11 @@ bool BuildHashOperator::getAllWorkOrders(
while (num_workorders_generated_ < input_relation_block_ids_.size()) {
container->addNormalWorkOrder(
new BuildHashWorkOrder(
+ query_id_,
input_relation_,
join_key_attributes_,
any_join_key_attributes_nullable_,
input_relation_block_ids_[num_workorders_generated_],
- query_id_,
hash_table,
storage_manager),
op_index_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/BuildHashOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/BuildHashOperator.hpp b/relational_operators/BuildHashOperator.hpp
index 5a46d8b..50dd7d6 100644
--- a/relational_operators/BuildHashOperator.hpp
+++ b/relational_operators/BuildHashOperator.hpp
@@ -59,6 +59,7 @@ class BuildHashOperator : public RelationalOperator {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param input_relation The relation to build hash table on.
* @param input_relation_is_stored If input_relation is a stored relation and
* is fully available to the operator before it can start generating
@@ -69,14 +70,13 @@ class BuildHashOperator : public RelationalOperator {
* @param hash_table_index The index of the JoinHashTable in QueryContext.
* The HashTable's key Type(s) should be the Type(s) of the
* join_key_attributes in input_relation.
- * @param query_id The ID of the query to which this operator belongs.
**/
- BuildHashOperator(const CatalogRelation &input_relation,
+ BuildHashOperator(const std::size_t query_id,
+ const CatalogRelation &input_relation,
const bool input_relation_is_stored,
const std::vector<attribute_id> &join_key_attributes,
const bool any_join_key_attributes_nullable,
- const QueryContext::join_hash_table_id hash_table_index,
- const std::size_t query_id)
+ const QueryContext::join_hash_table_id hash_table_index)
: RelationalOperator(query_id),
input_relation_(input_relation),
input_relation_is_stored_(input_relation_is_stored),
@@ -131,20 +131,20 @@ class BuildHashWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query.
* @param input_relation The relation to build hash table on.
* @param join_key_attributes The IDs of equijoin attributes in
* input_relation.
* @param any_join_key_attributes_nullable If any attribute is nullable.
* @param build_block_id The block id.
- * @param query_id The ID of the query.
* @param hash_table The JoinHashTable to use.
* @param storage_manager The StorageManager to use.
**/
- BuildHashWorkOrder(const CatalogRelationSchema &input_relation,
+ BuildHashWorkOrder(const std::size_t query_id,
+ const CatalogRelationSchema &input_relation,
const std::vector<attribute_id> &join_key_attributes,
const bool any_join_key_attributes_nullable,
const block_id build_block_id,
- const std::size_t query_id,
JoinHashTable *hash_table,
StorageManager *storage_manager)
: WorkOrder(query_id),
@@ -158,20 +158,20 @@ class BuildHashWorkOrder : public WorkOrder {
/**
* @brief Constructor for the distributed version.
*
+ * @param query_id The ID of the query.
* @param input_relation The relation to build hash table on.
* @param join_key_attributes The IDs of equijoin attributes in
* input_relation.
* @param any_join_key_attributes_nullable If any attribute is nullable.
- * @param query_id The ID of the query.
* @param build_block_id The block id.
* @param hash_table The JoinHashTable to use.
* @param storage_manager The StorageManager to use.
**/
- BuildHashWorkOrder(const CatalogRelationSchema &input_relation,
+ BuildHashWorkOrder(const std::size_t query_id,
+ const CatalogRelationSchema &input_relation,
std::vector<attribute_id> &&join_key_attributes,
const bool any_join_key_attributes_nullable,
const block_id build_block_id,
- const std::size_t query_id,
JoinHashTable *hash_table,
StorageManager *storage_manager)
: WorkOrder(query_id),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/CreateIndexOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/CreateIndexOperator.hpp b/relational_operators/CreateIndexOperator.hpp
index ede3f02..11a01ae 100644
--- a/relational_operators/CreateIndexOperator.hpp
+++ b/relational_operators/CreateIndexOperator.hpp
@@ -51,14 +51,14 @@ class CreateIndexOperator : public RelationalOperator {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param relation The relation to create index upon.
* @param index_name The index to create.
- * @param query_id The ID of the query to which this operator belongs.
* @param index_description The index_description associated with this index.
**/
- CreateIndexOperator(CatalogRelation *relation,
+ CreateIndexOperator(const std::size_t query_id,
+ CatalogRelation *relation,
const std::string &index_name,
- const std::size_t query_id,
IndexSubBlockDescription &&index_description) // NOLINT(whitespace/operators)
: RelationalOperator(query_id),
relation_(DCHECK_NOTNULL(relation)),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/DeleteOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/DeleteOperator.cpp b/relational_operators/DeleteOperator.cpp
index 94169ed..15dc9e3 100644
--- a/relational_operators/DeleteOperator.cpp
+++ b/relational_operators/DeleteOperator.cpp
@@ -54,13 +54,13 @@ bool DeleteOperator::getAllWorkOrders(
if (!started_) {
for (const block_id input_block_id : relation_block_ids_) {
container->addNormalWorkOrder(
- new DeleteWorkOrder(relation_,
+ new DeleteWorkOrder(query_id_,
+ relation_,
input_block_id,
predicate,
storage_manager,
op_index_,
scheduler_client_id,
- query_id_,
bus),
op_index_);
}
@@ -70,13 +70,13 @@ bool DeleteOperator::getAllWorkOrders(
} else {
while (num_workorders_generated_ < relation_block_ids_.size()) {
container->addNormalWorkOrder(
- new DeleteWorkOrder(relation_,
+ new DeleteWorkOrder(query_id_,
+ relation_,
relation_block_ids_[num_workorders_generated_],
predicate,
storage_manager,
op_index_,
scheduler_client_id,
- query_id_,
bus),
op_index_);
++num_workorders_generated_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/DeleteOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/DeleteOperator.hpp b/relational_operators/DeleteOperator.hpp
index ba1f825..c55f585 100644
--- a/relational_operators/DeleteOperator.hpp
+++ b/relational_operators/DeleteOperator.hpp
@@ -55,18 +55,18 @@ class DeleteOperator : public RelationalOperator {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param relation The relation to perform the DELETE over.
* @param predicate_index The index of Predicate in QueryContext. All tuples
* matching pred will be deleted (If kInvalidPredicateId, then all
* tuples will be deleted).
* @param relation_is_stored If relation is a stored relation and is fully
* available to the operator before it can start generating workorders.
- * @param query_id The ID of the query to which this operator belongs.
**/
- DeleteOperator(const CatalogRelation &relation,
+ DeleteOperator(const std::size_t query_id,
+ const CatalogRelation &relation,
const QueryContext::predicate_id predicate_index,
- const bool relation_is_stored,
- const std::size_t query_id)
+ const bool relation_is_stored)
: RelationalOperator(query_id),
relation_(relation),
predicate_index_(predicate_index),
@@ -122,6 +122,7 @@ class DeleteWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this workorder belongs.
* @param input_relation The relation to perform the DELETE over.
* @param input_block_id The block Id.
* @param predicate All tuples matching \c predicate will be deleted (If
@@ -130,16 +131,15 @@ class DeleteWorkOrder : public WorkOrder {
* @param delete_operator_index The index of the Delete Operator in the query
* plan DAG.
* @param scheduler_client_id The TMB client ID of the scheduler thread.
- * @param query_id The ID of the query to which this workorder belongs.
* @param bus A pointer to the TMB.
**/
- DeleteWorkOrder(const CatalogRelationSchema &input_relation,
+ DeleteWorkOrder(const std::size_t query_id,
+ const CatalogRelationSchema &input_relation,
const block_id input_block_id,
const Predicate *predicate,
StorageManager *storage_manager,
const std::size_t delete_operator_index,
const tmb::client_id scheduler_client_id,
- const std::size_t query_id,
MessageBus *bus)
: WorkOrder(query_id),
input_relation_(input_relation),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/DestroyHashOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/DestroyHashOperator.cpp b/relational_operators/DestroyHashOperator.cpp
index c92117a..8aa40b4 100644
--- a/relational_operators/DestroyHashOperator.cpp
+++ b/relational_operators/DestroyHashOperator.cpp
@@ -33,7 +33,7 @@ bool DestroyHashOperator::getAllWorkOrders(
if (blocking_dependencies_met_ && !work_generated_) {
work_generated_ = true;
container->addNormalWorkOrder(
- new DestroyHashWorkOrder(hash_table_index_, query_id_, query_context),
+ new DestroyHashWorkOrder(query_id_, hash_table_index_, query_context),
op_index_);
}
return work_generated_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/DestroyHashOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/DestroyHashOperator.hpp b/relational_operators/DestroyHashOperator.hpp
index 086c279..7d8acb7 100644
--- a/relational_operators/DestroyHashOperator.hpp
+++ b/relational_operators/DestroyHashOperator.hpp
@@ -46,11 +46,11 @@ class DestroyHashOperator : public RelationalOperator {
/**
* @brief Constructor.
*
- * @param hash_table_index The index of the JoinHashTable in QueryContext.
* @param query_id The ID of the query to which this operator belongs.
+ * @param hash_table_index The index of the JoinHashTable in QueryContext.
**/
- DestroyHashOperator(const QueryContext::join_hash_table_id hash_table_index,
- const std::size_t query_id)
+ DestroyHashOperator(const std::size_t query_id,
+ const QueryContext::join_hash_table_id hash_table_index)
: RelationalOperator(query_id),
hash_table_index_(hash_table_index),
work_generated_(false) {}
@@ -78,12 +78,12 @@ class DestroyHashWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
- * @param hash_table_index The index of the JoinHashTable in QueryContext.
* @param query_id The ID of the query to which this WorkOrder belongs.
+ * @param hash_table_index The index of the JoinHashTable in QueryContext.
* @param query_context The QueryContext to use.
**/
- DestroyHashWorkOrder(const QueryContext::join_hash_table_id hash_table_index,
- const std::size_t query_id,
+ DestroyHashWorkOrder(const std::size_t query_id,
+ const QueryContext::join_hash_table_id hash_table_index,
QueryContext *query_context)
: WorkOrder(query_id),
hash_table_index_(hash_table_index),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/DropTableOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/DropTableOperator.hpp b/relational_operators/DropTableOperator.hpp
index 0bbb718..a0a8d6e 100644
--- a/relational_operators/DropTableOperator.hpp
+++ b/relational_operators/DropTableOperator.hpp
@@ -55,14 +55,14 @@ class DropTableOperator : public RelationalOperator {
/**
* @brief Constructor.
*
- * @param relation The relation to drop.
* @param query_id The ID of the query to which this operator belongs.
+ * @param relation The relation to drop.
* @param database The databse where to drop \c relation.
* @param only_drop_blocks If true, only drop the blocks belonging to \c
* relation, but leave \c relation in \c database.
**/
- DropTableOperator(const CatalogRelation &relation,
- const std::size_t query_id,
+ DropTableOperator(const std::size_t query_id,
+ const CatalogRelation &relation,
CatalogDatabase *database,
const bool only_drop_blocks = false)
: RelationalOperator(query_id),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/FinalizeAggregationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/FinalizeAggregationOperator.hpp b/relational_operators/FinalizeAggregationOperator.hpp
index 51e55dc..e8a403f 100644
--- a/relational_operators/FinalizeAggregationOperator.hpp
+++ b/relational_operators/FinalizeAggregationOperator.hpp
@@ -54,16 +54,17 @@ class FinalizeAggregationOperator : public RelationalOperator {
* @brief Constructor for finalizing aggregation state and writing output
* tuples. The actual aggregation is computed by the AggregationOperator.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param aggr_state_index The index of the AggregationState in QueryContext.
* @param output_relation The output relation.
* @param output_destination_index The index of the InsertDestination in the
* QueryContext to insert aggregation results.
- * @param query_id The ID of the query to which this operator belongs.
*/
- FinalizeAggregationOperator(const QueryContext::aggregation_state_id aggr_state_index,
- const CatalogRelation &output_relation,
- const QueryContext::insert_destination_id output_destination_index,
- const std::size_t query_id)
+ FinalizeAggregationOperator(
+ const std::size_t query_id,
+ const QueryContext::aggregation_state_id aggr_state_index,
+ const CatalogRelation &output_relation,
+ const QueryContext::insert_destination_id output_destination_index)
: RelationalOperator(query_id),
aggr_state_index_(aggr_state_index),
output_relation_(output_relation),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index d8c7304..b89cfb3 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -290,7 +290,8 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders(
if (!started_) {
for (const block_id probe_block_id : probe_relation_block_ids_) {
container->addNormalWorkOrder(
- new JoinWorkOrderClass(build_relation_,
+ new JoinWorkOrderClass(query_id_,
+ build_relation_,
probe_relation_,
join_key_attributes_,
any_join_key_attributes_nullable_,
@@ -298,7 +299,6 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders(
residual_predicate,
selection,
hash_table,
- query_id_,
output_destination,
storage_manager),
op_index_);
@@ -309,17 +309,18 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders(
} else {
while (num_workorders_generated_ < probe_relation_block_ids_.size()) {
container->addNormalWorkOrder(
- new JoinWorkOrderClass(build_relation_,
- probe_relation_,
- join_key_attributes_,
- any_join_key_attributes_nullable_,
- probe_relation_block_ids_[num_workorders_generated_],
- residual_predicate,
- selection,
- hash_table,
- query_id_,
- output_destination,
- storage_manager),
+ new JoinWorkOrderClass(
+ query_id_,
+ build_relation_,
+ probe_relation_,
+ join_key_attributes_,
+ any_join_key_attributes_nullable_,
+ probe_relation_block_ids_[num_workorders_generated_],
+ residual_predicate,
+ selection,
+ hash_table,
+ output_destination,
+ storage_manager),
op_index_);
++num_workorders_generated_;
} // end while
@@ -350,6 +351,7 @@ bool HashJoinOperator::getAllOuterJoinWorkOrders(
for (const block_id probe_block_id : probe_relation_block_ids_) {
container->addNormalWorkOrder(
new HashOuterJoinWorkOrder(
+ query_id_,
build_relation_,
probe_relation_,
join_key_attributes_,
@@ -358,7 +360,6 @@ bool HashJoinOperator::getAllOuterJoinWorkOrders(
selection,
is_selection_on_build_,
hash_table,
- query_id_,
output_destination,
storage_manager),
op_index_);
@@ -370,6 +371,7 @@ bool HashJoinOperator::getAllOuterJoinWorkOrders(
while (num_workorders_generated_ < probe_relation_block_ids_.size()) {
container->addNormalWorkOrder(
new HashOuterJoinWorkOrder(
+ query_id_,
build_relation_,
probe_relation_,
join_key_attributes_,
@@ -378,7 +380,6 @@ bool HashJoinOperator::getAllOuterJoinWorkOrders(
selection,
is_selection_on_build_,
hash_table,
- query_id_,
output_destination,
storage_manager),
op_index_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/HashJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.hpp b/relational_operators/HashJoinOperator.hpp
index 825f360..1d5d4e3 100644
--- a/relational_operators/HashJoinOperator.hpp
+++ b/relational_operators/HashJoinOperator.hpp
@@ -85,6 +85,7 @@ class HashJoinOperator : public RelationalOperator {
* dependent on the selectivity of the predicates involved. The decision
* is left to the query optimizer.
*
+ * @param query_id The ID of the query.
* @param build_relation The relation that the hash table was originally
* built on (i.e. the inner relation in the join).
* @param probe_relation The relation to probe the hash table with (i.e. the
@@ -108,25 +109,25 @@ class HashJoinOperator : public RelationalOperator {
* corresponding to the attributes of the relation referred by
* output_relation_id. Each Scalar is evaluated for the joined tuples,
* and the resulting value is inserted into the join result.
- * @param query_id The ID of the query.
* @param is_selection_on_build Whether each selection Scalar is using attributes
* from the build relation as input. Can be NULL for inner/semi/anti
* joins since this information is not utilized by these joins.
* @param join_type The type of join corresponding to this operator.
**/
- HashJoinOperator(const CatalogRelation &build_relation,
- const CatalogRelation &probe_relation,
- const bool probe_relation_is_stored,
- const std::vector<attribute_id> &join_key_attributes,
- const bool any_join_key_attributes_nullable,
- const CatalogRelation &output_relation,
- const QueryContext::insert_destination_id output_destination_index,
- const QueryContext::join_hash_table_id hash_table_index,
- const QueryContext::predicate_id residual_predicate_index,
- const QueryContext::scalar_group_id selection_index,
- const std::size_t query_id,
- const std::vector<bool> *is_selection_on_build = nullptr,
- const JoinType join_type = JoinType::kInnerJoin)
+ HashJoinOperator(
+ const std::size_t query_id,
+ const CatalogRelation &build_relation,
+ const CatalogRelation &probe_relation,
+ const bool probe_relation_is_stored,
+ const std::vector<attribute_id> &join_key_attributes,
+ const bool any_join_key_attributes_nullable,
+ const CatalogRelation &output_relation,
+ const QueryContext::insert_destination_id output_destination_index,
+ const QueryContext::join_hash_table_id hash_table_index,
+ const QueryContext::predicate_id residual_predicate_index,
+ const QueryContext::scalar_group_id selection_index,
+ const std::vector<bool> *is_selection_on_build = nullptr,
+ const JoinType join_type = JoinType::kInnerJoin)
: RelationalOperator(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -230,6 +231,7 @@ class HashInnerJoinWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param build_relation The relation that the hash table was originally built
* on (i.e. the inner relation in the join).
* @param probe_relation The relation to probe the hash table with (i.e. the
@@ -246,21 +248,21 @@ class HashInnerJoinWorkOrder : public WorkOrder {
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
* @param hash_table The JoinHashTable to use.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
- HashInnerJoinWorkOrder(const CatalogRelationSchema &build_relation,
- const CatalogRelationSchema &probe_relation,
- const std::vector<attribute_id> &join_key_attributes,
- const bool any_join_key_attributes_nullable,
- const block_id lookup_block_id,
- const Predicate *residual_predicate,
- const std::vector<std::unique_ptr<const Scalar>> &selection,
- const JoinHashTable &hash_table,
- const std::size_t query_id,
- InsertDestination *output_destination,
- StorageManager *storage_manager)
+ HashInnerJoinWorkOrder(
+ const std::size_t query_id,
+ const CatalogRelationSchema &build_relation,
+ const CatalogRelationSchema &probe_relation,
+ const std::vector<attribute_id> &join_key_attributes,
+ const bool any_join_key_attributes_nullable,
+ const block_id lookup_block_id,
+ const Predicate *residual_predicate,
+ const std::vector<std::unique_ptr<const Scalar>> &selection,
+ const JoinHashTable &hash_table,
+ InsertDestination *output_destination,
+ StorageManager *storage_manager)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -276,6 +278,7 @@ class HashInnerJoinWorkOrder : public WorkOrder {
/**
* @brief Constructor for the distributed version.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param build_relation The relation that the hash table was originally built
* on (i.e. the inner relation in the join).
* @param probe_relation The relation to probe the hash table with (i.e. the
@@ -292,21 +295,21 @@ class HashInnerJoinWorkOrder : public WorkOrder {
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
* @param hash_table The JoinHashTable to use.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
- HashInnerJoinWorkOrder(const CatalogRelationSchema &build_relation,
- const CatalogRelationSchema &probe_relation,
- std::vector<attribute_id> &&join_key_attributes,
- const bool any_join_key_attributes_nullable,
- const block_id lookup_block_id,
- const Predicate *residual_predicate,
- const std::vector<std::unique_ptr<const Scalar>> &selection,
- const JoinHashTable &hash_table,
- const std::size_t query_id,
- InsertDestination *output_destination,
- StorageManager *storage_manager)
+ HashInnerJoinWorkOrder(
+ const std::size_t query_id,
+ const CatalogRelationSchema &build_relation,
+ const CatalogRelationSchema &probe_relation,
+ std::vector<attribute_id> &&join_key_attributes,
+ const bool any_join_key_attributes_nullable,
+ const block_id lookup_block_id,
+ const Predicate *residual_predicate,
+ const std::vector<std::unique_ptr<const Scalar>> &selection,
+ const JoinHashTable &hash_table,
+ InsertDestination *output_destination,
+ StorageManager *storage_manager)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -359,6 +362,7 @@ class HashSemiJoinWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param build_relation The relation that the hash table was originally built
* on (i.e. the inner relation in the join).
* @param probe_relation The relation to probe the hash table with (i.e. the
@@ -375,21 +379,21 @@ class HashSemiJoinWorkOrder : public WorkOrder {
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
* @param hash_table The JoinHashTable to use.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
- HashSemiJoinWorkOrder(const CatalogRelationSchema &build_relation,
- const CatalogRelationSchema &probe_relation,
- const std::vector<attribute_id> &join_key_attributes,
- const bool any_join_key_attributes_nullable,
- const block_id lookup_block_id,
- const Predicate *residual_predicate,
- const std::vector<std::unique_ptr<const Scalar>> &selection,
- const JoinHashTable &hash_table,
- const std::size_t query_id,
- InsertDestination *output_destination,
- StorageManager *storage_manager)
+ HashSemiJoinWorkOrder(
+ const std::size_t query_id,
+ const CatalogRelationSchema &build_relation,
+ const CatalogRelationSchema &probe_relation,
+ const std::vector<attribute_id> &join_key_attributes,
+ const bool any_join_key_attributes_nullable,
+ const block_id lookup_block_id,
+ const Predicate *residual_predicate,
+ const std::vector<std::unique_ptr<const Scalar>> &selection,
+ const JoinHashTable &hash_table,
+ InsertDestination *output_destination,
+ StorageManager *storage_manager)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -405,6 +409,7 @@ class HashSemiJoinWorkOrder : public WorkOrder {
/**
* @brief Constructor for the distributed version.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param build_relation The relation that the hash table was originally built
* on (i.e. the inner relation in the join).
* @param probe_relation The relation to probe the hash table with (i.e. the
@@ -421,21 +426,21 @@ class HashSemiJoinWorkOrder : public WorkOrder {
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
* @param hash_table The JoinHashTable to use.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
- HashSemiJoinWorkOrder(const CatalogRelationSchema &build_relation,
- const CatalogRelationSchema &probe_relation,
- std::vector<attribute_id> &&join_key_attributes,
- const bool any_join_key_attributes_nullable,
- const block_id lookup_block_id,
- const Predicate *residual_predicate,
- const std::vector<std::unique_ptr<const Scalar>> &selection,
- const JoinHashTable &hash_table,
- const std::size_t query_id,
- InsertDestination *output_destination,
- StorageManager *storage_manager)
+ HashSemiJoinWorkOrder(
+ const std::size_t query_id,
+ const CatalogRelationSchema &build_relation,
+ const CatalogRelationSchema &probe_relation,
+ std::vector<attribute_id> &&join_key_attributes,
+ const bool any_join_key_attributes_nullable,
+ const block_id lookup_block_id,
+ const Predicate *residual_predicate,
+ const std::vector<std::unique_ptr<const Scalar>> &selection,
+ const JoinHashTable &hash_table,
+ InsertDestination *output_destination,
+ StorageManager *storage_manager)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -481,6 +486,7 @@ class HashAntiJoinWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param build_relation The relation that the hash table was originally built
* on (i.e. the inner relation in the join).
* @param probe_relation The relation to probe the hash table with (i.e. the
@@ -497,21 +503,21 @@ class HashAntiJoinWorkOrder : public WorkOrder {
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
* @param hash_table The JoinHashTable to use.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
- HashAntiJoinWorkOrder(const CatalogRelationSchema &build_relation,
- const CatalogRelationSchema &probe_relation,
- const std::vector<attribute_id> &join_key_attributes,
- const bool any_join_key_attributes_nullable,
- const block_id lookup_block_id,
- const Predicate *residual_predicate,
- const std::vector<std::unique_ptr<const Scalar>> &selection,
- const JoinHashTable &hash_table,
- const std::size_t query_id,
- InsertDestination *output_destination,
- StorageManager *storage_manager)
+ HashAntiJoinWorkOrder(
+ const std::size_t query_id,
+ const CatalogRelationSchema &build_relation,
+ const CatalogRelationSchema &probe_relation,
+ const std::vector<attribute_id> &join_key_attributes,
+ const bool any_join_key_attributes_nullable,
+ const block_id lookup_block_id,
+ const Predicate *residual_predicate,
+ const std::vector<std::unique_ptr<const Scalar>> &selection,
+ const JoinHashTable &hash_table,
+ InsertDestination *output_destination,
+ StorageManager *storage_manager)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -527,6 +533,7 @@ class HashAntiJoinWorkOrder : public WorkOrder {
/**
* @brief Constructor for the distributed version.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param build_relation The relation that the hash table was originally built
* on (i.e. the inner relation in the join).
* @param probe_relation The relation to probe the hash table with (i.e. the
@@ -543,21 +550,21 @@ class HashAntiJoinWorkOrder : public WorkOrder {
* in \c output_destination. Each Scalar is evaluated for the joined
* tuples, and the resulting value is inserted into the join result.
* @param hash_table The JoinHashTable to use.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
- HashAntiJoinWorkOrder(const CatalogRelationSchema &build_relation,
- const CatalogRelationSchema &probe_relation,
- std::vector<attribute_id> &&join_key_attributes,
- const bool any_join_key_attributes_nullable,
- const block_id lookup_block_id,
- const Predicate *residual_predicate,
- const std::vector<std::unique_ptr<const Scalar>> &selection,
- const JoinHashTable &hash_table,
- const std::size_t query_id,
- InsertDestination *output_destination,
- StorageManager *storage_manager)
+ HashAntiJoinWorkOrder(
+ const std::size_t query_id,
+ const CatalogRelationSchema &build_relation,
+ const CatalogRelationSchema &probe_relation,
+ std::vector<attribute_id> &&join_key_attributes,
+ const bool any_join_key_attributes_nullable,
+ const block_id lookup_block_id,
+ const Predicate *residual_predicate,
+ const std::vector<std::unique_ptr<const Scalar>> &selection,
+ const JoinHashTable &hash_table,
+ InsertDestination *output_destination,
+ StorageManager *storage_manager)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -608,6 +615,7 @@ class HashOuterJoinWorkOrder : public WorkOrder {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param build_relation The relation that the hash table was originally built
* on (i.e. the inner relation in the join).
* @param probe_relation The relation to probe the hash table with (i.e. the
@@ -623,21 +631,21 @@ class HashOuterJoinWorkOrder : public WorkOrder {
* is using attributes from the build relation as input. Note that the
* length of this vector should equal the length of \p selection.
* @param lookup_block_id The block id of the probe_relation.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
- HashOuterJoinWorkOrder(const CatalogRelationSchema &build_relation,
- const CatalogRelationSchema &probe_relation,
- const std::vector<attribute_id> &join_key_attributes,
- const bool any_join_key_attributes_nullable,
- const block_id lookup_block_id,
- const std::vector<std::unique_ptr<const Scalar>> &selection,
- const std::vector<bool> &is_selection_on_build,
- const JoinHashTable &hash_table,
- const std::size_t query_id,
- InsertDestination *output_destination,
- StorageManager *storage_manager)
+ HashOuterJoinWorkOrder(
+ const std::size_t query_id,
+ const CatalogRelationSchema &build_relation,
+ const CatalogRelationSchema &probe_relation,
+ const std::vector<attribute_id> &join_key_attributes,
+ const bool any_join_key_attributes_nullable,
+ const block_id lookup_block_id,
+ const std::vector<std::unique_ptr<const Scalar>> &selection,
+ const std::vector<bool> &is_selection_on_build,
+ const JoinHashTable &hash_table,
+ InsertDestination *output_destination,
+ StorageManager *storage_manager)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
@@ -653,6 +661,7 @@ class HashOuterJoinWorkOrder : public WorkOrder {
/**
* @brief Constructor for the distributed version.
*
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param build_relation The relation that the hash table was originally built
* on (i.e. the inner relation in the join).
* @param probe_relation The relation to probe the hash table with (i.e. the
@@ -668,21 +677,21 @@ class HashOuterJoinWorkOrder : public WorkOrder {
* is using attributes from the build relation as input. Note that the
* length of this vector should equal the length of \p selection.
* @param hash_table The JoinHashTable to use.
- * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the join results.
* @param storage_manager The StorageManager to use.
**/
- HashOuterJoinWorkOrder(const CatalogRelationSchema &build_relation,
- const CatalogRelationSchema &probe_relation,
- std::vector<attribute_id> &&join_key_attributes,
- const bool any_join_key_attributes_nullable,
- const block_id lookup_block_id,
- const std::vector<std::unique_ptr<const Scalar>> &selection,
- std::vector<bool> &&is_selection_on_build,
- const JoinHashTable &hash_table,
- const std::size_t query_id,
- InsertDestination *output_destination,
- StorageManager *storage_manager)
+ HashOuterJoinWorkOrder(
+ const std::size_t query_id,
+ const CatalogRelationSchema &build_relation,
+ const CatalogRelationSchema &probe_relation,
+ std::vector<attribute_id> &&join_key_attributes,
+ const bool any_join_key_attributes_nullable,
+ const block_id lookup_block_id,
+ const std::vector<std::unique_ptr<const Scalar>> &selection,
+ std::vector<bool> &&is_selection_on_build,
+ const JoinHashTable &hash_table,
+ InsertDestination *output_destination,
+ StorageManager *storage_manager)
: WorkOrder(query_id),
build_relation_(build_relation),
probe_relation_(probe_relation),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/InsertOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/InsertOperator.hpp b/relational_operators/InsertOperator.hpp
index ccef444..51c606d 100644
--- a/relational_operators/InsertOperator.hpp
+++ b/relational_operators/InsertOperator.hpp
@@ -53,16 +53,17 @@ class InsertOperator : public RelationalOperator {
/**
* @brief Constructor.
*
+ * @param query_id The ID of the query to which this operator belongs.
* @param output_relation_id The output relation.
* @param output_destination_index The index of the InsertDestination in the
* QueryContext to insert the tuple.
* @param tuple_index The index of the tuple to insert in the QueryContext.
- * @param query_id The ID of the query to which this operator belongs.
**/
- InsertOperator(const CatalogRelation &output_relation,
- const QueryContext::insert_destination_id output_destination_index,
- const QueryContext::tuple_id tuple_index,
- const std::size_t query_id)
+ InsertOperator(
+ const std::size_t query_id,
+ const CatalogRelation &output_relation,
+ const QueryContext::insert_destination_id output_destination_index,
+ const QueryContext::tuple_id tuple_index)
: RelationalOperator(query_id),
output_relation_(output_relation),
output_destination_index_(output_destination_index),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/NestedLoopsJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/NestedLoopsJoinOperator.cpp b/relational_operators/NestedLoopsJoinOperator.cpp
index 317cc5d..5a47fca 100644
--- a/relational_operators/NestedLoopsJoinOperator.cpp
+++ b/relational_operators/NestedLoopsJoinOperator.cpp
@@ -76,15 +76,17 @@ bool NestedLoopsJoinOperator::getAllWorkOrders(
for (const block_id left_block_id : left_relation_block_ids_) {
for (const block_id right_block_id : right_relation_block_ids_) {
container->addNormalWorkOrder(
- new NestedLoopsJoinWorkOrder(left_input_relation_,
- right_input_relation_,
- left_block_id,
- right_block_id,
- query_context->getPredicate(join_predicate_index_),
- query_context->getScalarGroup(selection_index_),
- query_id_,
- query_context->getInsertDestination(output_destination_index_),
- storage_manager),
+ new NestedLoopsJoinWorkOrder(
+ query_id_,
+ left_input_relation_,
+ right_input_relation_,
+ left_block_id,
+ right_block_id,
+ query_context->getPredicate(join_predicate_index_),
+ query_context->getScalarGroup(selection_index_),
+ query_context->getInsertDestination(
+ output_destination_index_),
+ storage_manager),
op_index_);
}
}
@@ -166,15 +168,16 @@ std::size_t NestedLoopsJoinOperator::getAllWorkOrdersHelperBothNotStored(WorkOrd
right_index < right_max;
++right_index) {
container->addNormalWorkOrder(
- new NestedLoopsJoinWorkOrder(left_input_relation_,
- right_input_relation_,
- left_relation_block_ids_[left_index],
- right_relation_block_ids_[right_index],
- query_context->getPredicate(join_predicate_index_),
- query_context->getScalarGroup(selection_index_),
- query_id_,
- query_context->getInsertDestination(output_destination_index_),
- storage_manager),
+ new NestedLoopsJoinWorkOrder(
+ query_id_,
+ left_input_relation_,
+ right_input_relation_,
+ left_relation_block_ids_[left_index],
+ right_relation_block_ids_[right_index],
+ query_context->getPredicate(join_predicate_index_),
+ query_context->getScalarGroup(selection_index_),
+ query_context->getInsertDestination(output_destination_index_),
+ storage_manager),
op_index_);
}
}
@@ -201,13 +204,13 @@ bool NestedLoopsJoinOperator::getAllWorkOrdersHelperOneStored(WorkOrdersContaine
for (const block_id left_block_id : left_relation_block_ids_) {
container->addNormalWorkOrder(
new NestedLoopsJoinWorkOrder(
+ query_id_,
left_input_relation_,
right_input_relation_,
left_block_id,
right_relation_block_ids_[right_index],
join_predicate,
selection,
- query_id_,
output_destination,
storage_manager),
op_index_);
@@ -221,13 +224,13 @@ bool NestedLoopsJoinOperator::getAllWorkOrdersHelperOneStored(WorkOrdersContaine
++left_index) {
for (const block_id right_block_id : right_relation_block_ids_) {
container->addNormalWorkOrder(
- new NestedLoopsJoinWorkOrder(left_input_relation_,
+ new NestedLoopsJoinWorkOrder(query_id_,
+ left_input_relation_,
right_input_relation_,
left_relation_block_ids_[left_index],
right_block_id,
join_predicate,
selection,
- query_id_,
output_destination,
storage_manager),
op_index_);
[29/48] incubator-quickstep git commit: Initial support for
collecting table statistics: number of distinct values (#227)
Posted by hb...@apache.org.
Initial support for collecting table statistics: number of distinct values (#227)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c6772597
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c6772597
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c6772597
Branch: refs/heads/query-manager-used-in-foreman
Commit: c677259754249038670ec003dfc24de6d430e8e9
Parents: aa76e48
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 19 10:58:43 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:44 2016 -0700
----------------------------------------------------------------------
catalog/CMakeLists.txt | 11 +++
catalog/Catalog.proto | 13 ++-
catalog/CatalogRelation.cpp | 11 +++
catalog/CatalogRelation.hpp | 24 ++++-
catalog/CatalogRelationStatistics.cpp | 49 ++++++++++
catalog/CatalogRelationStatistics.hpp | 122 +++++++++++++++++++++++
cli/CMakeLists.txt | 21 +++-
cli/CommandExecutor.cpp | 138 ++++++++++++++++++++++++++-
cli/CommandExecutor.hpp | 17 ++--
cli/QuickstepCli.cpp | 2 +
cli/tests/CommandExecutorTestRunner.cpp | 2 +
query_optimizer/ExecutionGenerator.cpp | 6 +-
query_optimizer/QueryProcessor.hpp | 10 ++
13 files changed, 410 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c6772597/catalog/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/catalog/CMakeLists.txt b/catalog/CMakeLists.txt
index 94da838..64b4f16 100644
--- a/catalog/CMakeLists.txt
+++ b/catalog/CMakeLists.txt
@@ -1,5 +1,7 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015-2016 Pivotal Software, Inc.
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -36,6 +38,9 @@ add_library(quickstep_catalog_CatalogRelation CatalogRelation.cpp CatalogRelatio
add_library(quickstep_catalog_CatalogRelationSchema
CatalogRelationSchema.cpp
CatalogRelationSchema.hpp)
+add_library(quickstep_catalog_CatalogRelationStatistics
+ CatalogRelationStatistics.cpp
+ CatalogRelationStatistics.hpp)
add_library(quickstep_catalog_CatalogTypedefs ../empty_src.cpp CatalogTypedefs.hpp)
add_library(quickstep_catalog_IndexScheme IndexScheme.cpp IndexScheme.hpp)
if(QUICKSTEP_HAVE_LIBNUMA)
@@ -98,6 +103,7 @@ target_link_libraries(quickstep_catalog_CatalogRelation
glog
quickstep_catalog_CatalogAttribute
quickstep_catalog_CatalogRelationSchema
+ quickstep_catalog_CatalogRelationStatistics
quickstep_catalog_CatalogTypedefs
quickstep_catalog_Catalog_proto
quickstep_catalog_IndexScheme
@@ -111,6 +117,10 @@ target_link_libraries(quickstep_catalog_CatalogRelation
quickstep_threading_SpinSharedMutex
quickstep_utility_Macros
quickstep_utility_PtrVector)
+target_link_libraries(quickstep_catalog_CatalogRelationStatistics
+ quickstep_catalog_CatalogTypedefs
+ quickstep_catalog_Catalog_proto
+ quickstep_utility_Macros)
target_link_libraries(quickstep_catalog_IndexScheme
glog
quickstep_catalog_Catalog_proto
@@ -173,6 +183,7 @@ target_link_libraries(quickstep_catalog
quickstep_catalog_CatalogErrors
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogRelationSchema
+ quickstep_catalog_CatalogRelationStatistics
quickstep_catalog_CatalogTypedefs
quickstep_catalog_IndexScheme
quickstep_catalog_PartitionScheme
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c6772597/catalog/Catalog.proto
----------------------------------------------------------------------
diff --git a/catalog/Catalog.proto b/catalog/Catalog.proto
index 81e28cf..ce4bc2e 100644
--- a/catalog/Catalog.proto
+++ b/catalog/Catalog.proto
@@ -1,7 +1,7 @@
// Copyright 2011-2015 Quickstep Technologies LLC.
// Copyright 2015-2016 Pivotal Software, Inc.
// Copyright 2016, Quickstep Research Group, Computer Sciences Department,
-// University of Wisconsin\u2014Madison.
+// University of Wisconsin\u2014Madison.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -80,6 +80,16 @@ message IndexScheme {
repeated IndexEntry index_entries = 1;
}
+message CatalogRelationStatistics {
+ optional fixed64 num_tuples = 1;
+
+ message NumDistinctValuesEntry {
+ required int32 attr_id = 1;
+ required fixed64 num_distinct_values = 2;
+ }
+ repeated NumDistinctValuesEntry num_distinct_values_map = 2;
+}
+
message CatalogRelationSchema {
required int32 relation_id = 1;
required string name = 2;
@@ -99,6 +109,7 @@ message CatalogRelation {
optional IndexScheme index_scheme = 18;
optional PartitionScheme partition_scheme = 19;
optional NUMAPlacementScheme placement_scheme = 20;
+ optional CatalogRelationStatistics statistics = 21;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c6772597/catalog/CatalogRelation.cpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.cpp b/catalog/CatalogRelation.cpp
index 36f82d9..01aebb5 100644
--- a/catalog/CatalogRelation.cpp
+++ b/catalog/CatalogRelation.cpp
@@ -132,6 +132,14 @@ CatalogRelation::CatalogRelation(const serialization::CatalogRelationSchema &pro
}
default_layout_.reset(new StorageBlockLayout(*this, proto_default_layout));
+
+ if (proto.HasExtension(serialization::CatalogRelation::statistics)) {
+ statistics_.reset(
+ new CatalogRelationStatistics(
+ proto.GetExtension(serialization::CatalogRelation::statistics)));
+ } else {
+ statistics_.reset(new CatalogRelationStatistics());
+ }
}
serialization::CatalogRelationSchema CatalogRelation::getProto() const {
@@ -177,6 +185,9 @@ serialization::CatalogRelationSchema CatalogRelation::getProto() const {
#endif
}
+ proto.MutableExtension(serialization::CatalogRelation::statistics)
+ ->MergeFrom(statistics_->getProto());
+
return proto;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c6772597/catalog/CatalogRelation.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.hpp b/catalog/CatalogRelation.hpp
index 312f3b4..e0d5350 100644
--- a/catalog/CatalogRelation.hpp
+++ b/catalog/CatalogRelation.hpp
@@ -29,6 +29,7 @@
#include "catalog/Catalog.pb.h"
#include "catalog/CatalogConfig.h"
#include "catalog/CatalogRelationSchema.hpp"
+#include "catalog/CatalogRelationStatistics.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "catalog/IndexScheme.hpp"
@@ -79,7 +80,8 @@ class CatalogRelation : public CatalogRelationSchema {
const relation_id id = -1,
bool temporary = false)
: CatalogRelationSchema(parent, name, id, temporary),
- default_layout_(nullptr) {
+ default_layout_(nullptr),
+ statistics_(new CatalogRelationStatistics()) {
}
/**
@@ -377,6 +379,24 @@ class CatalogRelation : public CatalogRelationSchema {
* getDefaultStorageBlockLayout().estimateTuplesPerBlock();
}
+ /**
+ * @brief Get an immutable reference to the statistics of this catalog relation.
+ *
+ * @return A reference to the statistics of this catalog relation.
+ */
+ const CatalogRelationStatistics& getStatistics() const {
+ return *statistics_;
+ }
+
+ /**
+ * @brief Get a mutable pointer to the statistics of this catalog relation.
+ *
+ * @return A pointer to the statistics of this catalog relation.
+ */
+ CatalogRelationStatistics* getStatisticsMutable() {
+ return statistics_.get();
+ }
+
private:
// A list of blocks belonged to the relation.
std::vector<block_id> blocks_;
@@ -397,6 +417,8 @@ class CatalogRelation : public CatalogRelationSchema {
// Mutex for locking the index scheme.
alignas(kCacheLineBytes) mutable SpinSharedMutex<false> index_scheme_mutex_;
+ std::unique_ptr<CatalogRelationStatistics> statistics_;
+
#ifdef QUICKSTEP_HAVE_LIBNUMA
// NUMA placement scheme object which has the mapping between the partitions
// of the relation and the NUMA nodes/sockets. It also maintains a mapping
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c6772597/catalog/CatalogRelationStatistics.cpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelationStatistics.cpp b/catalog/CatalogRelationStatistics.cpp
new file mode 100644
index 0000000..2bd92b4
--- /dev/null
+++ b/catalog/CatalogRelationStatistics.cpp
@@ -0,0 +1,49 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "catalog/CatalogRelationStatistics.hpp"
+
+#include "catalog/Catalog.pb.h"
+
+namespace quickstep {
+
+CatalogRelationStatistics::CatalogRelationStatistics(
+ const serialization::CatalogRelationStatistics &proto) {
+ if (proto.has_num_tuples()) {
+ num_tuples_ = proto.num_tuples();
+ }
+ for (int i = 0; i < proto.num_distinct_values_map_size(); ++i) {
+ const auto &entry = proto.num_distinct_values_map(i);
+ num_distinct_values_map_.emplace(entry.attr_id(),
+ entry.num_distinct_values());
+ }
+}
+
+serialization::CatalogRelationStatistics CatalogRelationStatistics::getProto() const {
+ serialization::CatalogRelationStatistics proto;
+ if (num_tuples_ != 0) {
+ proto.set_num_tuples(num_tuples_);
+ }
+ for (const auto &pair : num_distinct_values_map_) {
+ auto entry = proto.add_num_distinct_values_map();
+ entry->set_attr_id(pair.first);
+ entry->set_num_distinct_values(pair.second);
+ }
+ return proto;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c6772597/catalog/CatalogRelationStatistics.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelationStatistics.hpp b/catalog/CatalogRelationStatistics.hpp
new file mode 100644
index 0000000..572d141
--- /dev/null
+++ b/catalog/CatalogRelationStatistics.hpp
@@ -0,0 +1,122 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_CATALOG_CATALOG_RELATION_STATISTICS_HPP_
+#define QUICKSTEP_CATALOG_CATALOG_RELATION_STATISTICS_HPP_
+
+#include <cstddef>
+#include <unordered_map>
+#include <utility>
+
+#include "catalog/Catalog.pb.h"
+#include "catalog/CatalogTypedefs.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+/** \addtogroup Catalog
+ * @{
+ */
+
+/**
+ * @brief Statistics of a catalog relation. E.g. total number of tuples,
+ * number of distinct values for each column.
+ **/
+class CatalogRelationStatistics {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ CatalogRelationStatistics()
+ : num_tuples_(0) {}
+
+ /**
+ * @brief Reconstruct a CatalogRelationStatistics object from its serialized
+ * Protocol Buffer form.
+ *
+ * @param proto The Protocol Buffer serialization of a CatalogRelationStatistics
+ * object, previously produced by getProto().
+ **/
+ explicit CatalogRelationStatistics(const serialization::CatalogRelationStatistics &proto);
+
+ /**
+ * @brief Serialize the CatalogRelationStatistics object as Protocol Buffer.
+ *
+ * @return The Protocol Buffer representation of the CatalogRelationStatistics
+ * object.
+ **/
+ serialization::CatalogRelationStatistics getProto() const;
+
+ /**
+ * @brief Set the number of tuples statistic.
+ *
+ * @param num_tuples The number of tuples statistic.
+ */
+ void setNumTuples(std::size_t num_tuples) {
+ num_tuples_ = num_tuples;
+ }
+
+ /**
+ * @brief Get the number of tuples statistic.
+ *
+ * @return The number of tuples. Returns 0 if the statistic is not set.
+ */
+ std::size_t getNumTuples() const {
+ return num_tuples_;
+ }
+
+ /**
+ * @brief Set the number of distinct values statistic for a column (catalog attribute).
+ *
+ * @param attr_id The id of the column.
+ * @param num_distinct_values The number of distinct values statistic.
+ */
+ void setNumDistinctValues(attribute_id attr_id, std::size_t num_distinct_values) {
+ num_distinct_values_map_[attr_id] = num_distinct_values;
+ }
+
+ /**
+ * @brief Get the number of distinct values statistic for a column (catalog attribute).
+ *
+ * @param The id of the column.
+ * @return The number of distinct values statistic for the column. Returns 0
+ * if the statistic is not set.
+ */
+ std::size_t getNumDistinctValues(attribute_id attr_id) const {
+ const auto it = num_distinct_values_map_.find(attr_id);
+ if (it == num_distinct_values_map_.end()) {
+ return static_cast<std::size_t>(0);
+ } else {
+ return it->second;
+ }
+ }
+
+ private:
+ // Total number of tuples in the relation.
+ std::size_t num_tuples_;
+
+ // Number of distinct values for each column.
+ std::unordered_map<attribute_id, std::size_t> num_distinct_values_map_;
+
+ DISALLOW_COPY_AND_ASSIGN(CatalogRelationStatistics);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_CATALOG_CATALOG_RELATION_STATISTICS_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c6772597/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 761b6d8..8fee7a4 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -1,5 +1,7 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -73,11 +75,24 @@ target_link_libraries(quickstep_cli_CommandExecutor
quickstep_catalog_CatalogDatabase
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogRelationSchema
- quickstep_cli_PrintToScreen
+ quickstep_cli_DropRelation
+ quickstep_cli_PrintToScreen
quickstep_parser_ParseStatement
+ quickstep_parser_SqlParserWrapper
+ quickstep_queryexecution_Foreman
+ quickstep_queryoptimizer_QueryHandle
+ quickstep_queryoptimizer_QueryPlan
+ quickstep_queryoptimizer_QueryProcessor
+ quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
- quickstep_utility_Macros
- quickstep_utility_PtrVector
+ quickstep_storage_StorageManager
+ quickstep_storage_TupleIdSequence
+ quickstep_storage_TupleStorageSubBlock
+ quickstep_parser_ParseString
+ quickstep_types_Type
+ quickstep_types_TypeID
+ quickstep_types_TypedValue
+ quickstep_utility_PtrVector
quickstep_utility_SqlError)
target_link_libraries(quickstep_cli_DefaultsConfigurator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c6772597/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 026922a..3cb3f86 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -19,6 +19,7 @@
#include <algorithm>
#include <cstddef>
+#include <cstdint>
#include <cstdio>
#include <memory>
#include <string>
@@ -28,14 +29,26 @@
#include "catalog/CatalogDatabase.hpp"
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogRelationSchema.hpp"
+#include "cli/DropRelation.hpp"
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
+#include "parser/ParseString.hpp"
+#include "parser/SqlParserWrapper.hpp"
+#include "query_execution/Foreman.hpp"
+#include "query_optimizer/QueryHandle.hpp"
+#include "query_optimizer/QueryPlan.hpp"
+#include "query_optimizer/QueryProcessor.hpp"
+#include "storage/StorageBlock.hpp"
#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageManager.hpp"
+#include "storage/TupleIdSequence.hpp"
+#include "storage/TupleStorageSubBlock.hpp"
+#include "types/Type.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
#include "utility/PtrVector.hpp"
-#include "utility/Macros.hpp"
#include "utility/SqlError.hpp"
-#include "gflags/gflags.h"
#include "glog/logging.h"
using std::fprintf;
@@ -195,11 +208,130 @@ void executeDescribeTable(
}
}
+/**
+ * @brief A helper function that executes a SQL query to obtain a scalar result.
+ */
+inline TypedValue executeQueryForSingleResult(const std::string &query_string,
+ StorageManager *storage_manager,
+ QueryProcessor *query_processor,
+ SqlParserWrapper *parser_wrapper,
+ Foreman *foreman) {
+ parser_wrapper->feedNextBuffer(new std::string(query_string));
+
+ ParseResult result = parser_wrapper->getNextStatement();
+ DCHECK(result.condition == ParseResult::kSuccess);
+
+ // Generate the query plan.
+ std::unique_ptr<QueryHandle> query_handle(
+ query_processor->generateQueryHandle(*result.parsed_statement));
+ DCHECK(query_handle->getQueryPlanMutable() != nullptr);
+
+ // Use foreman to execute the query plan.
+ foreman->setQueryPlan(query_handle->getQueryPlanMutable()->getQueryPlanDAGMutable());
+ foreman->reconstructQueryContextFromProto(query_handle->getQueryContextProto());
+
+ foreman->start();
+ foreman->join();
+
+ // Retrieve the scalar result from the result relation.
+ const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
+ DCHECK(query_result_relation != nullptr);
+
+ TypedValue value;
+ {
+ std::vector<block_id> blocks = query_result_relation->getBlocksSnapshot();
+ DCHECK_EQ(1u, blocks.size());
+ BlockReference block = storage_manager->getBlock(blocks[0], *query_result_relation);
+ const TupleStorageSubBlock &tuple_store = block->getTupleStorageSubBlock();
+ DCHECK_EQ(1, tuple_store.numTuples());
+ DCHECK_EQ(1u, tuple_store.getRelation().size());
+
+ if (tuple_store.isPacked()) {
+ value = tuple_store.getAttributeValueTyped(0, 0);
+ } else {
+ std::unique_ptr<TupleIdSequence> existence_map(tuple_store.getExistenceMap());
+ value = tuple_store.getAttributeValueTyped(*existence_map->begin(), 0);
+ }
+ value.ensureNotReference();
+ }
+
+ // Drop the result relation.
+ DropRelation::Drop(*query_result_relation,
+ query_processor->getDefaultDatabase(),
+ query_processor->getStorageManager());
+
+ return value;
+}
+
+void executeAnalyze(QueryProcessor *query_processor,
+ Foreman *foreman,
+ FILE *out) {
+ const CatalogDatabase &database = *query_processor->getDefaultDatabase();
+ StorageManager *storage_manager = query_processor->getStorageManager();
+
+ std::unique_ptr<SqlParserWrapper> parser_wrapper(new SqlParserWrapper());
+ std::vector<std::reference_wrapper<const CatalogRelation>> relations(
+ database.begin(), database.end());
+
+ // Analyze each relation in the database.
+ for (const CatalogRelation &relation : relations) {
+ fprintf(out, "Analyzing %s ... ", relation.getName().c_str());
+ fflush(out);
+
+ CatalogRelation *mutable_relation =
+ query_processor->getDefaultDatabase()->getRelationByIdMutable(relation.getID());
+
+ // Get the number of distinct values for each column.
+ for (const CatalogAttribute &attribute : relation) {
+ std::string query_string = "SELECT COUNT(DISTINCT ";
+ query_string.append(attribute.getName());
+ query_string.append(") FROM ");
+ query_string.append(relation.getName());
+ query_string.append(";");
+
+ TypedValue num_distinct_values =
+ executeQueryForSingleResult(query_string,
+ storage_manager,
+ query_processor,
+ parser_wrapper.get(),
+ foreman);
+
+ DCHECK(num_distinct_values.getTypeID() == TypeID::kLong);
+ mutable_relation->getStatisticsMutable()->setNumDistinctValues(
+ attribute.getID(),
+ num_distinct_values.getLiteral<std::int64_t>());
+ }
+
+ // Get the number of tuples for the relation.
+ std::string query_string = "SELECT COUNT(*) FROM ";
+ query_string.append(relation.getName());
+ query_string.append(";");
+
+ TypedValue num_tuples =
+ executeQueryForSingleResult(query_string,
+ storage_manager,
+ query_processor,
+ parser_wrapper.get(),
+ foreman);
+
+ DCHECK(num_tuples.getTypeID() == TypeID::kLong);
+ mutable_relation->getStatisticsMutable()->setNumTuples(
+ num_tuples.getLiteral<std::int64_t>());
+
+ fprintf(out, "done\n");
+ fflush(out);
+ }
+ query_processor->markCatalogAltered();
+ query_processor->saveCatalog();
+}
+
} // namespace
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
StorageManager *storage_manager,
+ QueryProcessor *query_processor,
+ Foreman *foreman,
FILE *out) {
const ParseCommand &command = static_cast<const ParseCommand &>(statement);
const PtrVector<ParseString> *arguments = command.arguments();
@@ -212,6 +344,8 @@ void executeCommand(const ParseStatement &statement,
} else {
executeDescribeTable(arguments, catalog_database, out);
}
+ } else if (command_str == C::kAnalyzeCommand) {
+ executeAnalyze(query_processor, foreman, out);
} else {
THROW_SQL_ERROR_AT(command.command()) << "Invalid Command";
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c6772597/cli/CommandExecutor.hpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.hpp b/cli/CommandExecutor.hpp
index f367ca1..c819981 100644
--- a/cli/CommandExecutor.hpp
+++ b/cli/CommandExecutor.hpp
@@ -19,13 +19,8 @@
#define QUICKSTEP_CLI_COMMAND_COMMAND_EXECUTOR_HPP_
#include <cstdio>
-#include <limits>
#include <string>
-#include "parser/ParseStatement.hpp"
-#include "storage/StorageBlockInfo.hpp"
-#include "utility/Macros.hpp"
-
using std::fprintf;
using std::fputc;
using std::string;
@@ -33,11 +28,13 @@ using std::string;
namespace quickstep {
class CatalogDatabase;
-class CatalogAttribute;
-class CatalogRelation;
+class Foreman;
+class ParseStatement;
+class QueryProcessor;
class StorageManager;
namespace cli {
+
/** \addtogroup CLI
* @{
*/
@@ -49,17 +46,23 @@ constexpr int kInitMaxColumnWidth = 6;
constexpr char kDescribeDatabaseCommand[] = "\\dt";
constexpr char kDescribeTableCommand[] = "\\d";
+constexpr char kAnalyzeCommand[] = "\\analyze";
/**
* @brief Executes the command by calling the command handler.
*
* @param statement The parsed statement from the cli.
* @param catalog_database The catalog information about the current database.
+ * @param storage_manager The current StorageManager.
+ * @param query_processor The query processor to generate plans for SQL queries.
+ * @param foreman The foreman to execute query plans.
* @param out The stream where the output of the command has to be redirected to.
*/
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
StorageManager *storage_manager,
+ QueryProcessor *query_processor,
+ Foreman *foreman,
FILE *out);
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c6772597/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index b7b28ba..558d6eb 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -367,6 +367,8 @@ int main(int argc, char* argv[]) {
*result.parsed_statement,
*(query_processor->getDefaultDatabase()),
query_processor->getStorageManager(),
+ query_processor.get(),
+ &foreman,
stdout);
} catch (const quickstep::SqlError &sql_error) {
fprintf(stderr, "%s",
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c6772597/cli/tests/CommandExecutorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/cli/tests/CommandExecutorTestRunner.cpp b/cli/tests/CommandExecutorTestRunner.cpp
index 73d2092..9cd493e 100644
--- a/cli/tests/CommandExecutorTestRunner.cpp
+++ b/cli/tests/CommandExecutorTestRunner.cpp
@@ -88,6 +88,8 @@ void CommandExecutorTestRunner::runTestCase(
*result.parsed_statement,
*(test_database_loader_.catalog_database()),
test_database_loader_.storage_manager(),
+ nullptr,
+ nullptr,
output_stream.file());
} else {
QueryHandle query_handle(optimizer_context.query_id());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c6772597/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 7209cfa..c590b6e 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1389,11 +1389,13 @@ void ExecutionGenerator::convertAggregate(
// Add distinctify hash table impl type if it is a DISTINCT aggregation.
if (unnamed_aggregate_expression->is_distinct()) {
- if (group_by_types.empty()) {
+ const std::vector<E::ScalarPtr> &arguments = unnamed_aggregate_expression->getArguments();
+ DCHECK_GE(arguments.size(), 1u);
+ if (group_by_types.empty() && arguments.size() == 1) {
aggr_state_proto->add_distinctify_hash_table_impl_types(
SimplifyHashTableImplTypeProto(
HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
- {&unnamed_aggregate_expression->getValueType()}));
+ {&arguments[0]->getValueType()}));
} else {
aggr_state_proto->add_distinctify_hash_table_impl_types(
HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c6772597/query_optimizer/QueryProcessor.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryProcessor.hpp b/query_optimizer/QueryProcessor.hpp
index 4514f45..32739dc 100644
--- a/query_optimizer/QueryProcessor.hpp
+++ b/query_optimizer/QueryProcessor.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -159,6 +161,14 @@ class QueryProcessor {
void saveCatalog();
/**
+ * @brief Set \p catalog_altered_ to true to indicate that the catalog
+ * has been altered.
+ */
+ void markCatalogAltered() {
+ catalog_altered_ = true;
+ }
+
+ /**
* @brief Get the default database in the Catalog held by this
* QueryProcessor.
**/
[45/48] incubator-quickstep git commit: Reordered Query ID in
operators and work orders.
Posted by hb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/tests/HashJoinOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/HashJoinOperator_unittest.cpp b/relational_operators/tests/HashJoinOperator_unittest.cpp
index 4ef5a5c..074b603 100644
--- a/relational_operators/tests/HashJoinOperator_unittest.cpp
+++ b/relational_operators/tests/HashJoinOperator_unittest.cpp
@@ -92,6 +92,7 @@ constexpr tuple_id kNumDimTuples = 200;
constexpr tuple_id kNumFactTuples = 300;
constexpr tuple_id kBlockSize = 10;
+constexpr std::size_t kQueryId = 0;
constexpr int kOpIndex = 0;
} // namespace
@@ -332,12 +333,12 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) {
// Create the builder operator.
unique_ptr<BuildHashOperator> builder(
- new BuildHashOperator(*dim_table_,
+ new BuildHashOperator(kQueryId,
+ *dim_table_,
true /* is_stored */,
std::vector<attribute_id>(1, dim_col_long.getID()),
dim_col_long.getType().isNullable(),
- join_hash_table_index,
- 0)); // dummy query ID
+ join_hash_table_index));
// Create the prober operator with one selection attribute.
const QueryContext::scalar_group_id selection_index = query_context_proto.scalar_groups_size();
@@ -359,18 +360,18 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) {
insert_destination_proto->set_relation_id(output_relation_id);
insert_destination_proto->set_relational_op_index(kOpIndex);
- unique_ptr<HashJoinOperator> prober(
- new HashJoinOperator(*dim_table_,
- *fact_table_,
- true /* is_stored */,
- std::vector<attribute_id>(1, fact_col_long.getID()),
- fact_col_long.getType().isNullable(),
- *result_table,
- output_destination_index,
- join_hash_table_index,
- QueryContext::kInvalidPredicateId /* residual_predicate_index */,
- selection_index,
- 0 /* dummy query ID */));
+ unique_ptr<HashJoinOperator> prober(new HashJoinOperator(
+ kQueryId,
+ *dim_table_,
+ *fact_table_,
+ true /* is_stored */,
+ std::vector<attribute_id>(1, fact_col_long.getID()),
+ fact_col_long.getType().isNullable(),
+ *result_table,
+ output_destination_index,
+ join_hash_table_index,
+ QueryContext::kInvalidPredicateId /* residual_predicate_index */,
+ selection_index));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -423,7 +424,7 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) {
}
// Create cleaner operator.
- unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index, 0 /* dummy query ID */));
+ unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(kQueryId, join_hash_table_index));
cleaner->informAllBlockingDependenciesMet();
fetchAndExecuteWorkOrders(cleaner.get());
@@ -475,12 +476,12 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) {
// Create the builder operator.
unique_ptr<BuildHashOperator> builder(
- new BuildHashOperator(*dim_table_,
+ new BuildHashOperator(kQueryId,
+ *dim_table_,
true /* is_stored */,
std::vector<attribute_id>(1, dim_col_int.getID()),
dim_col_int.getType().isNullable(),
- join_hash_table_index,
- 0)); // dummy query ID
+ join_hash_table_index));
// Create the prober operator with two selection attributes.
const QueryContext::scalar_group_id selection_index = query_context_proto.scalar_groups_size();
@@ -507,18 +508,19 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) {
insert_destination_proto->set_relation_id(output_relation_id);
insert_destination_proto->set_relational_op_index(kOpIndex);
- unique_ptr<HashJoinOperator> prober(
- new HashJoinOperator(*dim_table_,
- *fact_table_,
- true /* is_stored */,
- std::vector<attribute_id>(1, fact_col_int.getID()),
- fact_col_int.getType().isNullable(),
- *result_table,
- output_destination_index,
- join_hash_table_index,
- QueryContext::kInvalidPredicateId /* residual_predicate_index */,
- selection_index,
- 0 /* dummy query ID */));
+ unique_ptr<HashJoinOperator> prober(new HashJoinOperator(
+ kQueryId,
+ *dim_table_,
+ *fact_table_,
+ true /* is_stored */,
+ std::vector<attribute_id>(1, fact_col_int.getID()),
+ fact_col_int.getType().isNullable(),
+ *result_table,
+ output_destination_index,
+ join_hash_table_index,
+ QueryContext::kInvalidPredicateId /* residual_predicate_index */,
+ selection_index));
+
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -592,7 +594,7 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) {
}
// Create cleaner operator.
- unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index, 0 /* dummy query ID */));
+ unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(kQueryId, join_hash_table_index));
cleaner->informAllBlockingDependenciesMet();
fetchAndExecuteWorkOrders(cleaner.get());
@@ -636,12 +638,12 @@ TEST_P(HashJoinOperatorTest, CharKeyCartesianProductHashJoinTest) {
// Create builder operator.
unique_ptr<BuildHashOperator> builder(
- new BuildHashOperator(*dim_table_,
+ new BuildHashOperator(kQueryId,
+ *dim_table_,
true /* is_stored */,
std::vector<attribute_id>(1, dim_col_char.getID()),
dim_col_char.getType().isNullable(),
- join_hash_table_index,
- 0)); // dummy query ID.
+ join_hash_table_index));
// Create prober operator with one selection attribute.
const QueryContext::scalar_group_id selection_index = query_context_proto.scalar_groups_size();
@@ -663,18 +665,18 @@ TEST_P(HashJoinOperatorTest, CharKeyCartesianProductHashJoinTest) {
insert_destination_proto->set_relation_id(output_relation_id);
insert_destination_proto->set_relational_op_index(kOpIndex);
- unique_ptr<HashJoinOperator> prober(
- new HashJoinOperator(*dim_table_,
- *fact_table_,
- true /* is_stored */,
- std::vector<attribute_id>(1, fact_col_char.getID()),
- fact_col_char.getType().isNullable(),
- *result_table,
- output_destination_index,
- join_hash_table_index,
- QueryContext::kInvalidPredicateId /* residual_predicate_index */,
- selection_index,
- 0 /* dummy query ID */));
+ unique_ptr<HashJoinOperator> prober(new HashJoinOperator(
+ kQueryId,
+ *dim_table_,
+ *fact_table_,
+ true /* is_stored */,
+ std::vector<attribute_id>(1, fact_col_char.getID()),
+ fact_col_char.getType().isNullable(),
+ *result_table,
+ output_destination_index,
+ join_hash_table_index,
+ QueryContext::kInvalidPredicateId /* residual_predicate_index */,
+ selection_index));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -727,7 +729,7 @@ TEST_P(HashJoinOperatorTest, CharKeyCartesianProductHashJoinTest) {
}
// Create cleaner operator.
- unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index, 0 /* dummy query ID */));
+ unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(kQueryId, join_hash_table_index));
cleaner->informAllBlockingDependenciesMet();
fetchAndExecuteWorkOrders(cleaner.get());
@@ -772,12 +774,12 @@ TEST_P(HashJoinOperatorTest, VarCharDuplicateKeyHashJoinTest) {
// Create builder operator.
unique_ptr<BuildHashOperator> builder(
- new BuildHashOperator(*dim_table_,
+ new BuildHashOperator(kQueryId,
+ *dim_table_,
true /* is_stored */,
std::vector<attribute_id>(1, dim_col_varchar.getID()),
dim_col_varchar.getType().isNullable(),
- join_hash_table_index,
- 0)); // dummy query ID.
+ join_hash_table_index));
// Create prober operator with two selection attributes.
const QueryContext::scalar_group_id selection_index = query_context_proto.scalar_groups_size();
@@ -804,18 +806,19 @@ TEST_P(HashJoinOperatorTest, VarCharDuplicateKeyHashJoinTest) {
insert_destination_proto->set_relation_id(output_relation_id);
insert_destination_proto->set_relational_op_index(kOpIndex);
- unique_ptr<HashJoinOperator> prober(
- new HashJoinOperator(*dim_table_,
- *fact_table_,
- true /* is_stored */,
- std::vector<attribute_id>(1, fact_col_varchar.getID()),
- fact_col_varchar.getType().isNullable(),
- *result_table,
- output_destination_index,
- join_hash_table_index,
- QueryContext::kInvalidPredicateId /* residual_predicate_index */,
- selection_index,
- 0 /* dummy query ID */));
+ unique_ptr<HashJoinOperator> prober(new HashJoinOperator(
+ kQueryId,
+ *dim_table_,
+ *fact_table_,
+ true /* is_stored */,
+ std::vector<attribute_id>(1, fact_col_varchar.getID()),
+ fact_col_varchar.getType().isNullable(),
+ *result_table,
+ output_destination_index,
+ join_hash_table_index,
+ QueryContext::kInvalidPredicateId /* residual_predicate_index */,
+ selection_index));
+
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -893,7 +896,7 @@ TEST_P(HashJoinOperatorTest, VarCharDuplicateKeyHashJoinTest) {
}
// Create the cleaner operator.
- unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index, 0 /* dummy query ID */));
+ unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(kQueryId, join_hash_table_index));
cleaner->informAllBlockingDependenciesMet();
fetchAndExecuteWorkOrders(cleaner.get());
@@ -943,12 +946,12 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) {
dim_key_attrs.push_back(dim_col_varchar.getID());
unique_ptr<BuildHashOperator> builder(
- new BuildHashOperator(*dim_table_,
+ new BuildHashOperator(kQueryId,
+ *dim_table_,
true /* is_stored */,
dim_key_attrs,
dim_col_long.getType().isNullable() || dim_col_varchar.getType().isNullable(),
- join_hash_table_index,
- 0)); // dummy query ID.
+ join_hash_table_index));
// Create the prober operator with two selection attributes.
const QueryContext::scalar_group_id selection_index = query_context_proto.scalar_groups_size();
@@ -979,18 +982,19 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) {
fact_key_attrs.push_back(fact_col_long.getID());
fact_key_attrs.push_back(fact_col_varchar.getID());
- unique_ptr<HashJoinOperator> prober(
- new HashJoinOperator(*dim_table_,
- *fact_table_,
- true /* is_stored */,
- fact_key_attrs,
- fact_col_long.getType().isNullable() || fact_col_varchar.getType().isNullable(),
- *result_table,
- output_destination_index,
- join_hash_table_index,
- QueryContext::kInvalidPredicateId /* residual_predicate_index */,
- selection_index,
- 0 /* dummy query ID */));
+ unique_ptr<HashJoinOperator> prober(new HashJoinOperator(
+ kQueryId,
+ *dim_table_,
+ *fact_table_,
+ true /* is_stored */,
+ fact_key_attrs,
+ fact_col_long.getType().isNullable() ||
+ fact_col_varchar.getType().isNullable(),
+ *result_table,
+ output_destination_index,
+ join_hash_table_index,
+ QueryContext::kInvalidPredicateId /* residual_predicate_index */,
+ selection_index));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -1068,7 +1072,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) {
}
// Create cleaner operator.
- unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index, 0 /* dummy query ID */));
+ unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(kQueryId, join_hash_table_index));
cleaner->informAllBlockingDependenciesMet();
fetchAndExecuteWorkOrders(cleaner.get());
@@ -1119,12 +1123,12 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinWithResidualPredicateTest) {
dim_key_attrs.push_back(dim_col_varchar.getID());
unique_ptr<BuildHashOperator> builder(
- new BuildHashOperator(*dim_table_,
+ new BuildHashOperator(kQueryId,
+ *dim_table_,
true /* is_stored */,
dim_key_attrs,
dim_col_long.getType().isNullable() || dim_col_varchar.getType().isNullable(),
- join_hash_table_index,
- 0)); // dummy query ID.
+ join_hash_table_index));
// Create prober operator with two selection attributes.
const QueryContext::scalar_group_id selection_index = query_context_proto.scalar_groups_size();
@@ -1166,17 +1170,18 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinWithResidualPredicateTest) {
query_context_proto.add_predicates()->CopyFrom(residual_pred->getProto());
unique_ptr<HashJoinOperator> prober(
- new HashJoinOperator(*dim_table_,
+ new HashJoinOperator(kQueryId,
+ *dim_table_,
*fact_table_,
true /* is_stored */,
fact_key_attrs,
- fact_col_long.getType().isNullable() || fact_col_varchar.getType().isNullable(),
+ fact_col_long.getType().isNullable() ||
+ fact_col_varchar.getType().isNullable(),
*result_table,
output_destination_index,
join_hash_table_index,
residual_pred_index,
- selection_index,
- 0 /* dummy query ID */));
+ selection_index));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -1254,7 +1259,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinWithResidualPredicateTest) {
}
// Create cleaner operator.
- unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index, 0 /* dummy query ID */));
+ unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(kQueryId, join_hash_table_index));
cleaner->informAllBlockingDependenciesMet();
fetchAndExecuteWorkOrders(cleaner.get());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/tests/SortMergeRunOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/SortMergeRunOperator_unittest.cpp b/relational_operators/tests/SortMergeRunOperator_unittest.cpp
index 244091f..fc10671 100644
--- a/relational_operators/tests/SortMergeRunOperator_unittest.cpp
+++ b/relational_operators/tests/SortMergeRunOperator_unittest.cpp
@@ -84,6 +84,7 @@ namespace quickstep {
namespace {
+constexpr std::size_t kQueryId = 0;
constexpr const std::size_t kOpIndex = 0;
// Helper struct for test tuple that will that will be inserted and sorted.
@@ -1565,7 +1566,8 @@ class SortMergeRunOperatorTest : public ::testing::Test {
const std::size_t top_k = 0) {
const QueryContext::sort_config_id sort_config_index = createSortConfigProto(attrs, ordering, null_ordering);
- merge_op_.reset(new SortMergeRunOperator(*input_table_,
+ merge_op_.reset(new SortMergeRunOperator(kQueryId,
+ *input_table_,
*result_table_,
insert_destination_index_,
*run_table_,
@@ -1573,8 +1575,8 @@ class SortMergeRunOperatorTest : public ::testing::Test {
sort_config_index,
merge_factor,
top_k,
- true,
- 0 /* dummy query ID */));
+ true));
+
merge_op_->setOperatorIndex(kOpIndex);
// Set up the QueryContext.
@@ -1609,7 +1611,8 @@ class SortMergeRunOperatorTest : public ::testing::Test {
const std::size_t top_k = 0) {
const QueryContext::sort_config_id sort_config_index = createSortConfigProto(attrs, ordering, null_ordering);
- merge_op_.reset(new SortMergeRunOperator(*input_table_,
+ merge_op_.reset(new SortMergeRunOperator(kQueryId,
+ *input_table_,
*result_table_,
insert_destination_index_,
*run_table_,
@@ -1617,8 +1620,7 @@ class SortMergeRunOperatorTest : public ::testing::Test {
sort_config_index,
merge_factor,
top_k,
- false,
- 0 /* dummy query ID */));
+ false));
merge_op_->setOperatorIndex(kOpIndex);
// Set up the QueryContext.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/SortRunGenerationOperator_unittest.cpp b/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
index 6f24b92..71a80e4 100644
--- a/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
+++ b/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
@@ -77,6 +77,7 @@ namespace quickstep {
namespace {
+constexpr std::size_t kQueryId = 0;
constexpr int kOpIndex = 0;
// Helper struct for test tuple that will that will be inserted and sorted.
@@ -353,13 +354,14 @@ class SortRunGenerationOperatorTest : public ::testing::Test {
order_by_proto->set_null_first(null_ordering[i]);
}
- std::unique_ptr<RelationalOperator> run_gen(
- new SortRunGenerationOperator(*input_table_,
- *result_table_,
- insert_destination_index,
- sort_config_index,
- true /* is_stored */,
- 0 /* dummy query ID */));
+ std::unique_ptr<RelationalOperator> run_gen(new SortRunGenerationOperator(
+ kQueryId,
+ *input_table_,
+ *result_table_,
+ insert_destination_index,
+ sort_config_index,
+ true /* is_stored */));
+
run_gen->setOperatorIndex(kOpIndex);
// Set up the QueryContext.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1be47dcb/relational_operators/tests/TextScanOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/TextScanOperator_unittest.cpp b/relational_operators/tests/TextScanOperator_unittest.cpp
index 7626686..ef6fc2d 100644
--- a/relational_operators/tests/TextScanOperator_unittest.cpp
+++ b/relational_operators/tests/TextScanOperator_unittest.cpp
@@ -55,6 +55,7 @@ const char *failure_output_filename;
namespace quickstep {
namespace {
+constexpr std::size_t kQueryId = 0;
constexpr int kOpIndex = 0;
} // namespace
@@ -188,13 +189,13 @@ TEST_F(TextScanOperatorTest, ScanTest) {
output_destination_proto->set_relational_op_index(kOpIndex);
std::unique_ptr<TextScanOperator> text_scan_op(
- new TextScanOperator(input_filename,
+ new TextScanOperator(kQueryId,
+ input_filename,
'\t',
true,
false,
*relation_,
- output_destination_index,
- 0 /* dummy query ID */));
+ output_destination_index));
// Setup query_context_.
query_context_.reset(new QueryContext(query_context_proto,
[38/48] incubator-quickstep git commit: Added BlockLocator.
Posted by hb...@apache.org.
Added BlockLocator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3789da72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3789da72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3789da72
Branch: refs/heads/query-manager-used-in-foreman
Commit: 3789da728e95b91bd97587f5e34da6ff1b55ea5f
Parents: 4503198
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Sat May 28 14:24:13 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:46 2016 -0700
----------------------------------------------------------------------
query_execution/BlockLocator.cpp | 223 +++++++++++++++
query_execution/BlockLocator.hpp | 125 +++++++++
query_execution/CMakeLists.txt | 48 ++++
query_execution/QueryExecutionMessages.proto | 34 +++
query_execution/QueryExecutionTypedefs.hpp | 16 ++
query_execution/tests/BlockLocator_unittest.cpp | 270 +++++++++++++++++++
storage/CMakeLists.txt | 9 +-
storage/StorageManager.cpp | 190 ++++++++++++-
storage/StorageManager.hpp | 76 +++++-
9 files changed, 984 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3789da72/query_execution/BlockLocator.cpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocator.cpp b/query_execution/BlockLocator.cpp
new file mode 100644
index 0000000..6cf5249
--- /dev/null
+++ b/query_execution/BlockLocator.cpp
@@ -0,0 +1,223 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_execution/BlockLocator.hpp"
+
+#include <cstdlib>
+#include <string>
+#include <utility>
+
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "threading/ThreadUtil.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+
+using std::free;
+using std::malloc;
+using std::move;
+
+using tmb::TaggedMessage;
+using tmb::client_id;
+
+namespace quickstep {
+
+void BlockLocator::run() {
+ if (cpu_id_ >= 0) {
+ ThreadUtil::BindToCPU(cpu_id_);
+ }
+
+ for (;;) {
+ // Receive() is a blocking call, causing this thread to sleep until next
+ // message is received.
+ const tmb::AnnotatedMessage annotated_message = bus_->Receive(locator_client_id_, 0, true);
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ const client_id sender = annotated_message.sender;
+ LOG(INFO) << "BlockLocator received the typed '" << tagged_message.message_type()
+ << "' message from TMB Client " << sender;
+ switch (tagged_message.message_type()) {
+ case kBlockDomainRegistrationMessage: {
+ serialization::BlockDomainRegistrationMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processBlockDomainRegistrationMessage(sender, proto.domain_network_address());
+ break;
+ }
+ case kAddBlockLocationMessage: {
+ serialization::BlockLocationMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ const block_id block = proto.block_id();
+ const block_id_domain domain = proto.block_domain();
+
+ const auto result_block_locations = block_locations_[block].insert(domain);
+ const auto result_domain_blocks = domain_blocks_[domain].insert(block);
+ DCHECK_EQ(result_block_locations.second, result_domain_blocks.second);
+
+ if (result_domain_blocks.second) {
+ LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " loaded in Domain " << domain;
+ } else {
+ LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " existed in Domain " << domain;
+ }
+ break;
+ }
+ case kDeleteBlockLocationMessage: {
+ serialization::BlockLocationMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ const block_id block = proto.block_id();
+ const block_id_domain domain = proto.block_domain();
+
+ const auto cit = block_locations_[block].find(domain);
+ if (cit != block_locations_[block].end()) {
+ block_locations_[block].erase(domain);
+ domain_blocks_[domain].erase(block);
+
+ LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " evicted in Domain " << domain;
+ } else {
+ LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " not found in Domain " << domain;
+ }
+ break;
+ }
+ case kLocateBlockMessage: {
+ serialization::BlockMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processLocateBlockMessage(sender, proto.block_id());
+ break;
+ }
+ case kGetPeerDomainNetworkAddressesMessage: {
+ serialization::BlockMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processGetPeerDomainNetworkAddressesMessage(sender, proto.block_id());
+ break;
+ }
+ case kBlockDomainUnregistrationMessage: {
+ serialization::BlockDomainMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ const block_id_domain domain = proto.block_domain();
+
+ domain_network_addresses_.erase(domain);
+
+ for (const block_id block : domain_blocks_[domain]) {
+ block_locations_[block].erase(domain);
+ }
+ domain_blocks_.erase(domain);
+
+ LOG(INFO) << "Unregistered Domain " << domain;
+ break;
+ }
+ case kPoisonMessage: {
+ return;
+ }
+ }
+ }
+}
+
+void BlockLocator::processBlockDomainRegistrationMessage(const client_id receiver,
+ const std::string &network_address) {
+ DCHECK_LT(block_domain_, kMaxDomain);
+
+ domain_network_addresses_.emplace(++block_domain_, network_address);
+ domain_blocks_[block_domain_];
+
+ serialization::BlockDomainMessage proto;
+ proto.set_block_domain(block_domain_);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainRegistrationResponseMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent BlockDomainRegistrationResponseMessage (typed '"
+ << kBlockDomainRegistrationResponseMessage
+ << "') to Worker (id '" << receiver << "')";
+ CHECK(tmb::MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ locator_client_id_,
+ receiver,
+ move(message)));
+}
+
+void BlockLocator::processLocateBlockMessage(const client_id receiver,
+ const block_id block) {
+ serialization::LocateBlockResponseMessage proto;
+
+ for (const block_id_domain domain : block_locations_[block]) {
+ proto.add_block_domains(domain);
+ }
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kLocateBlockResponseMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent LocateBlockResponseMessage (typed '" << kLocateBlockResponseMessage
+ << "') to StorageManager (id '" << receiver << "')";
+ CHECK(tmb::MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ locator_client_id_,
+ receiver,
+ move(message)));
+}
+
+void BlockLocator::processGetPeerDomainNetworkAddressesMessage(const client_id receiver,
+ const block_id block) {
+ serialization::GetPeerDomainNetworkAddressesResponseMessage proto;
+
+ for (const block_id_domain domain : block_locations_[block]) {
+ proto.add_domain_network_addresses(domain_network_addresses_[domain]);
+ }
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kGetPeerDomainNetworkAddressesResponseMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent GetPeerDomainNetworkAddressesResponseMessage (typed '"
+ << kGetPeerDomainNetworkAddressesResponseMessage
+ << "') to StorageManager (id '" << receiver << "')";
+ CHECK(tmb::MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ locator_client_id_,
+ receiver,
+ move(message)));
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3789da72/query_execution/BlockLocator.hpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocator.hpp b/query_execution/BlockLocator.hpp
new file mode 100644
index 0000000..bbd9b8f
--- /dev/null
+++ b/query_execution/BlockLocator.hpp
@@ -0,0 +1,125 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_HPP_
+
+#include <atomic>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageConstants.hpp"
+#include "threading/Thread.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+
+namespace quickstep {
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief A class for keeping trace of blocks loaded in a Worker's buffer pool
+ * in the distributed version.
+ **/
+class BlockLocator : public Thread {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param bus A pointer to the TMB.
+ * @param cpu_id The ID of the CPU to which the BlockLocator thread can be pinned.
+ *
+ * @note If cpu_id is not specified, BlockLocator thread can be possibly moved
+ * around on different CPUs by the OS.
+ **/
+ BlockLocator(tmb::MessageBus *bus,
+ const int cpu_id = -1)
+ : bus_(DCHECK_NOTNULL(bus)),
+ cpu_id_(cpu_id),
+ block_domain_(0) {
+ locator_client_id_ = bus_->Connect();
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kBlockDomainRegistrationMessage);
+ bus_->RegisterClientAsSender(locator_client_id_, kBlockDomainRegistrationResponseMessage);
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kAddBlockLocationMessage);
+ bus_->RegisterClientAsReceiver(locator_client_id_, kDeleteBlockLocationMessage);
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kLocateBlockMessage);
+ bus_->RegisterClientAsSender(locator_client_id_, kLocateBlockResponseMessage);
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kGetPeerDomainNetworkAddressesMessage);
+ bus_->RegisterClientAsSender(locator_client_id_, kGetPeerDomainNetworkAddressesResponseMessage);
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kBlockDomainUnregistrationMessage);
+ bus_->RegisterClientAsReceiver(locator_client_id_, kPoisonMessage);
+ }
+
+ ~BlockLocator() override {}
+
+ /**
+ * @brief Get the TMB client ID of BlockLocator thread.
+ *
+ * @return TMB client ID of BlockLocator thread.
+ **/
+ tmb::client_id getBusClientID() const {
+ return locator_client_id_;
+ }
+
+ protected:
+ void run() override;
+
+ private:
+ void processBlockDomainRegistrationMessage(const tmb::client_id receiver, const std::string &network_address);
+ void processLocateBlockMessage(const tmb::client_id receiver, const block_id block);
+ void processGetPeerDomainNetworkAddressesMessage(const tmb::client_id receiver, const block_id block);
+
+ tmb::MessageBus *bus_;
+
+ // The ID of the CPU that the BlockLocator thread can optionally be pinned to.
+ const int cpu_id_;
+
+ alignas(kCacheLineBytes) std::atomic<block_id_domain> block_domain_;
+
+ // From a block domain to its network info in the ip:port format, i.e.,
+ // "0.0.0.0:0".
+ std::unordered_map<block_id_domain, const std::string> domain_network_addresses_;
+
+ // From a block to its domains.
+ std::unordered_map<block_id, std::unordered_set<block_id_domain>> block_locations_;
+
+ // From a block domain to all blocks loaded in its buffer pool.
+ std::unordered_map<block_id_domain, std::unordered_set<block_id>> domain_blocks_;
+
+ tmb::client_id locator_client_id_;
+
+ DISALLOW_COPY_AND_ASSIGN(BlockLocator);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3789da72/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 04a0348..7d9d601 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -20,6 +20,9 @@ QS_PROTOBUF_GENERATE_CPP(queryexecution_QueryExecutionMessages_proto_srcs
QueryExecutionMessages.proto)
# Declare micro-libs:
+if (ENABLE_DISTRIBUTED)
+ add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp BlockLocator.hpp)
+endif()
add_library(quickstep_queryexecution_Foreman Foreman.cpp Foreman.hpp)
add_library(quickstep_queryexecution_ForemanLite ../empty_src.cpp ForemanLite.hpp)
add_library(quickstep_queryexecution_QueryContext QueryContext.cpp QueryContext.hpp)
@@ -40,6 +43,19 @@ add_library(quickstep_queryexecution_WorkerMessage ../empty_src.cpp WorkerMessag
add_library(quickstep_queryexecution_WorkerSelectionPolicy ../empty_src.cpp WorkerSelectionPolicy.hpp)
# Link dependencies:
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryexecution_BlockLocator
+ glog
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_threading_Thread
+ quickstep_threading_ThreadUtil
+ quickstep_utility_Macros
+ tmb)
+endif()
target_link_libraries(quickstep_queryexecution_Foreman
glog
gtest
@@ -176,7 +192,37 @@ target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_WorkerDirectory
quickstep_queryexecution_WorkerMessage
quickstep_queryexecution_WorkerSelectionPolicy)
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryexecution
+ quickstep_queryexecution_BlockLocator)
+endif()
+
# Tests:
+if (ENABLE_DISTRIBUTED)
+ add_executable(BlockLocator_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/BlockLocator_unittest.cpp")
+ target_link_libraries(BlockLocator_unittest
+ gflags_nothreads-static
+ glog
+ gtest
+ quickstep_catalog_CatalogAttribute
+ quickstep_catalog_CatalogRelation
+ quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_StorageBlob
+ quickstep_storage_StorageBlock
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_storage_StorageManager
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ tmb
+ ${LIBS})
+ add_test(BlockLocator_unittest BlockLocator_unittest)
+endif()
+
add_executable(Foreman_unittest
"${CMAKE_CURRENT_SOURCE_DIR}/tests/Foreman_unittest.cpp")
target_link_libraries(Foreman_unittest
@@ -269,3 +315,5 @@ target_link_libraries(WorkerSelectionPolicy_unittest
quickstep_queryexecution_WorkerDirectory
quickstep_queryexecution_WorkerSelectionPolicy)
add_test(WorkerSelectionPolicy_unittest WorkerSelectionPolicy_unittest)
+
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/block_locator_test_data/)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3789da72/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 8d2efd0..15803cf 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -16,6 +16,10 @@ syntax = "proto2";
package quickstep.serialization;
+// Used for any messages that do not carry payloads.
+message EmptyMessage {
+}
+
// Used for both Normal WorkOrders and RebuildWorkOrders.
// NOTE(zuyu): we might need to seperate the completion messages to contain
// run-time information for Foreman to make better decisions on scheduling
@@ -42,3 +46,33 @@ message DataPipelineMessage {
message WorkOrdersAvailableMessage {
required uint64 operator_index = 1;
}
+
+// BlockLocator related messages.
+message BlockDomainRegistrationMessage {
+ // Format IP:Port, i.e., "0.0.0.0:0".
+ required string domain_network_address = 1;
+}
+
+// Used for RegistrationResponse, Unregistration, and FailureReport.
+message BlockDomainMessage {
+ required uint32 block_domain = 1;
+}
+
+// Used when StorageManager loads or evicts a block or a blob from its buffer
+// pool.
+message BlockLocationMessage {
+ required fixed64 block_id = 1;
+ required uint32 block_domain = 2;
+}
+
+message BlockMessage {
+ required fixed64 block_id = 1;
+}
+
+message LocateBlockResponseMessage {
+ repeated uint32 block_domains = 1;
+}
+
+message GetPeerDomainNetworkAddressesResponseMessage {
+ repeated string domain_network_addresses = 1;
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3789da72/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionTypedefs.hpp b/query_execution/QueryExecutionTypedefs.hpp
index 36cfd82..fc253bc 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -18,6 +18,7 @@
#ifndef QUICKSTEP_QUERY_EXECUTION_QUERY_EXECUTION_TYPEDEFS_HPP_
#define QUICKSTEP_QUERY_EXECUTION_QUERY_EXECUTION_TYPEDEFS_HPP_
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED
#include "threading/ThreadIDBasedMap.hpp"
#include "tmb/address.h"
@@ -55,6 +56,7 @@ using ClientIDMap = ThreadIDBasedMap<client_id,
'a',
'p'>;
+// We sort the following message types in the order of a life cycle of a query.
enum QueryExecutionMessageType : message_type_id {
kWorkOrderMessage, // From Foreman to Worker.
kWorkOrderCompleteMessage, // From Worker to Foreman.
@@ -66,6 +68,20 @@ enum QueryExecutionMessageType : message_type_id {
kRebuildWorkOrderMessage, // From Foreman to Worker.
kRebuildWorkOrderCompleteMessage, // From Worker to Foreman.
kPoisonMessage, // From the CLI shell to Foreman, then from Foreman to Workers.
+
+#ifdef QUICKSTEP_DISTRIBUTED
+ // BlockLocator related messages, sorted in a life cycle of StorageManager
+ // with a unique block domain.
+ kBlockDomainRegistrationMessage, // From Worker to BlockLocator.
+ kBlockDomainRegistrationResponseMessage, // From BlockLocator to Worker.
+ kAddBlockLocationMessage, // From StorageManager to BlockLocator.
+ kDeleteBlockLocationMessage, // From StorageManager to BlockLocator.
+ kLocateBlockMessage, // From StorageManager to BlockLocator.
+ kLocateBlockResponseMessage, // From BlockLocator to StorageManager.
+ kGetPeerDomainNetworkAddressesMessage, // From StorageManager to BlockLocator.
+ kGetPeerDomainNetworkAddressesResponseMessage, // From BlockLocator to StorageManager.
+ kBlockDomainUnregistrationMessage, // From StorageManager to BlockLocator.
+#endif
};
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3789da72/query_execution/tests/BlockLocator_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/BlockLocator_unittest.cpp b/query_execution/tests/BlockLocator_unittest.cpp
new file mode 100644
index 0000000..fe7b86b
--- /dev/null
+++ b/query_execution/tests/BlockLocator_unittest.cpp
@@ -0,0 +1,270 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogAttribute.hpp"
+#include "catalog/CatalogRelation.hpp"
+#include "query_execution/BlockLocator.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "storage/StorageBlob.hpp"
+#include "storage/StorageBlock.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageConstants.hpp"
+#include "storage/StorageManager.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+
+using std::free;
+using std::malloc;
+using std::move;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+using tmb::AnnotatedMessage;
+using tmb::MessageBus;
+using tmb::TaggedMessage;
+
+namespace quickstep {
+
+class BlockLocatorTest : public ::testing::Test {
+ protected:
+ static const char kStoragePath[];
+ static const char kDomainNetworkAddress[];
+
+ ~BlockLocatorTest() {
+ locator_->join();
+ }
+
+ virtual void SetUp() {
+ bus_.Initialize();
+
+ locator_.reset(new BlockLocator(&bus_));
+ locator_client_id_ = locator_->getBusClientID();
+ locator_->start();
+
+ worker_client_id_ = bus_.Connect();
+
+ bus_.RegisterClientAsSender(worker_client_id_, kBlockDomainRegistrationMessage);
+ bus_.RegisterClientAsReceiver(worker_client_id_, kBlockDomainRegistrationResponseMessage);
+
+ bus_.RegisterClientAsSender(worker_client_id_, kLocateBlockMessage);
+ bus_.RegisterClientAsReceiver(worker_client_id_, kLocateBlockResponseMessage);
+
+ bus_.RegisterClientAsSender(worker_client_id_, kPoisonMessage);
+
+ block_domain_ = getBlockDomain(kDomainNetworkAddress);
+
+ storage_manager_.reset(
+ new StorageManager(kStoragePath, block_domain_, locator_client_id_, &bus_));
+ }
+
+ virtual void TearDown() {
+ storage_manager_.reset();
+
+ serialization::EmptyMessage proto;
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kPoisonMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent PoisonMessage (typed '" << kPoisonMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+ }
+
+ vector<block_id_domain> getPeerDomains(const block_id block) {
+ serialization::BlockMessage proto;
+ proto.set_block_id(block);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kLocateBlockMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent LocateBlockMessage (typed '" << kLocateBlockMessage
+ << "') to BlockLocator";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+
+ const AnnotatedMessage annotated_message(bus_.Receive(worker_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ CHECK_EQ(kLocateBlockResponseMessage, tagged_message.message_type());
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') received LocateBlockResponseMessage from BlockLocator";
+
+ serialization::LocateBlockResponseMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ vector<block_id_domain> domains;
+ for (int i = 0; i < response_proto.block_domains_size(); ++i) {
+ domains.push_back(response_proto.block_domains(i));
+ }
+
+ return domains;
+ }
+
+ void checkLoaded(const block_id block) {
+ const vector<string> peer_domain_network_addresses = storage_manager_->getPeerDomainNetworkAddresses(block);
+ EXPECT_EQ(1u, peer_domain_network_addresses.size());
+ EXPECT_STREQ(kDomainNetworkAddress, peer_domain_network_addresses[0].data());
+
+ const vector<block_id_domain> domains = getPeerDomains(block);
+ EXPECT_EQ(1u, domains.size());
+ EXPECT_EQ(block_domain_, domains[0]);
+ }
+
+ void checkEvicted(const block_id block) {
+ const vector<string> peer_domain_network_addresses = storage_manager_->getPeerDomainNetworkAddresses(block);
+ EXPECT_TRUE(peer_domain_network_addresses.empty());
+
+ const vector<block_id_domain> domains = getPeerDomains(block);
+ EXPECT_TRUE(domains.empty());
+ }
+
+ tmb::client_id worker_client_id_;
+
+ block_id_domain block_domain_;
+ unique_ptr<StorageManager> storage_manager_;
+
+ private:
+ block_id_domain getBlockDomain(const string &network_address) {
+ serialization::BlockDomainRegistrationMessage proto;
+ proto.set_domain_network_address(network_address);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainRegistrationMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+
+ const AnnotatedMessage annotated_message(bus_.Receive(worker_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ EXPECT_EQ(locator_client_id_, annotated_message.sender);
+ EXPECT_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') received BlockDomainRegistrationResponseMessage from BlockLocator";
+
+ serialization::BlockDomainMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ return static_cast<block_id_domain>(response_proto.block_domain());
+ }
+
+ MessageBusImpl bus_;
+
+ unique_ptr<BlockLocator> locator_;
+ tmb::client_id locator_client_id_;
+};
+
+const char BlockLocatorTest::kStoragePath[] = "./block_locator_test_data/";
+const char BlockLocatorTest::kDomainNetworkAddress[] = "ip:port";
+
+TEST_F(BlockLocatorTest, BlockTest) {
+ CatalogRelation relation(nullptr, "rel");
+ relation.addAttribute(new CatalogAttribute(nullptr, "attr_int", TypeFactory::GetType(kInt)));
+
+ const block_id block =
+ storage_manager_->createBlock(relation, relation.getDefaultStorageBlockLayout());
+ checkLoaded(block);
+
+ ASSERT_TRUE(storage_manager_->saveBlockOrBlob(block));
+ storage_manager_->evictBlockOrBlob(block);
+ checkEvicted(block);
+
+ {
+ const BlockReference block_ref = storage_manager_->getBlock(block, relation);
+ }
+ checkLoaded(block);
+
+ storage_manager_->deleteBlockOrBlobFile(block);
+ checkEvicted(block);
+}
+
+TEST_F(BlockLocatorTest, BlobTest) {
+ const block_id blob = storage_manager_->createBlob(kDefaultBlockSizeInSlots);
+ checkLoaded(blob);
+
+ ASSERT_TRUE(storage_manager_->saveBlockOrBlob(blob));
+ storage_manager_->evictBlockOrBlob(blob);
+ checkEvicted(blob);
+
+ {
+ const BlobReference blob_ref = storage_manager_->getBlob(blob);
+ }
+ checkLoaded(blob);
+
+ storage_manager_->deleteBlockOrBlobFile(blob);
+ checkEvicted(blob);
+}
+
+} // namespace quickstep
+
+int main(int argc, char **argv) {
+ google::InitGoogleLogging(argv[0]);
+ // Honor FLAGS_buffer_pool_slots in StorageManager.
+ gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+ ::testing::InitGoogleTest(&argc, argv);
+
+ return RUN_ALL_TESTS();
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3789da72/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 87a5e54..4da16ea 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -941,7 +941,8 @@ target_link_libraries(quickstep_storage_StorageManager
quickstep_utility_Alignment
quickstep_utility_CalculateInstalledMemory
quickstep_utility_Macros
- quickstep_utility_ShardedLockManager)
+ quickstep_utility_ShardedLockManager
+ tmb)
if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
target_link_libraries(quickstep_storage_StorageManager
quickstep_storage_FileManagerHdfs)
@@ -950,6 +951,12 @@ if (QUICKSTEP_HAVE_LIBNUMA)
target_link_libraries(quickstep_storage_StorageManager
${LIBNUMA_LIBRARY})
endif()
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_storage_StorageManager
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil)
+endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_storage_SubBlockTypeRegistry
glog
quickstep_storage_StorageBlockLayout_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3789da72/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 5d91052..15e2503 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -18,6 +18,7 @@
// This is included before other files so that we can conditionally determine
// what else to include.
#include "catalog/CatalogConfig.h"
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED
#include "storage/StorageConfig.h"
// Define feature test macros to enable large page support for mmap.
@@ -52,6 +53,12 @@
#include <unordered_map>
#include <vector>
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#endif
+
#include "storage/CountedReference.hpp"
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManagerLocal.hpp"
@@ -74,6 +81,13 @@
#include "gflags/gflags.h"
#include "glog/logging.h"
+#include "tmb/id_typedefs.h"
+
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+#endif
+
using std::free;
using std::int32_t;
using std::memset;
@@ -81,6 +95,15 @@ using std::size_t;
using std::string;
using std::vector;
+#ifdef QUICKSTEP_DISTRIBUTED
+using std::malloc;
+using std::move;
+using std::unique_ptr;
+
+using tmb::MessageBus;
+using tmb::TaggedMessage;
+#endif
+
namespace quickstep {
static bool ValidateBlockDomain(const char *flagname,
@@ -157,14 +180,21 @@ DEFINE_bool(use_hdfs, false, "Use HDFS as the persistent storage, instead of the
#endif
StorageManager::StorageManager(
- const std::string &storage_path,
- const block_id_domain block_domain,
- const size_t max_memory_usage,
- EvictionPolicy *eviction_policy)
+ const std::string &storage_path,
+ const block_id_domain block_domain,
+ const size_t max_memory_usage,
+ EvictionPolicy *eviction_policy,
+ const tmb::client_id block_locator_client_id,
+ tmb::MessageBus *bus)
: storage_path_(storage_path),
total_memory_usage_(0),
max_memory_usage_(max_memory_usage),
- eviction_policy_(eviction_policy) {
+ eviction_policy_(eviction_policy),
+#ifdef QUICKSTEP_DISTRIBUTED
+ block_domain_(block_domain),
+#endif
+ block_locator_client_id_(block_locator_client_id),
+ bus_(bus) {
#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
if (FLAGS_use_hdfs) {
file_manager_.reset(new FileManagerHdfs(storage_path));
@@ -175,10 +205,55 @@ StorageManager::StorageManager(
file_manager_.reset(new FileManagerLocal(storage_path));
#endif
+#ifdef QUICKSTEP_DISTRIBUTED
+ // NOTE(zuyu): The following if-condition is a workaround to bypass code for
+ // the distributed version in some unittests that does not use TMB. The
+ // end-to-end functional tests for the distributed version, however, would not
+ // be affected.
+ if (bus_) {
+ storage_manager_client_id_ = bus_->Connect();
+
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kGetPeerDomainNetworkAddressesMessage);
+ bus_->RegisterClientAsReceiver(storage_manager_client_id_, kGetPeerDomainNetworkAddressesResponseMessage);
+
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kAddBlockLocationMessage);
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kDeleteBlockLocationMessage);
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kBlockDomainUnregistrationMessage);
+
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') starts with Domain " << block_domain;
+ }
+#endif
+
block_index_ = BlockIdUtil::GetBlockId(block_domain, file_manager_->getMaxUsedBlockCounter(block_domain));
}
StorageManager::~StorageManager() {
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ serialization::BlockDomainMessage proto;
+ proto.set_block_domain(block_domain_);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainUnregistrationMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent BlockDomainUnregistrationMessage (typed '" << kBlockDomainUnregistrationMessage
+ << "') to BlockLocator";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ storage_manager_client_id_,
+ block_locator_client_id_,
+ move(message)));
+ }
+#endif
+
for (std::unordered_map<block_id, BlockHandle>::iterator it = blocks_.begin();
it != blocks_.end();
++it) {
@@ -222,6 +297,12 @@ block_id StorageManager::createBlock(const CatalogRelationSchema &relation,
// Make '*eviction_policy_' aware of the new block's existence.
eviction_policy_->blockCreated(new_block_id);
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ sendBlockLocationMessage(new_block_id, kAddBlockLocationMessage);
+ }
+#endif
+
return new_block_id;
}
@@ -249,6 +330,12 @@ block_id StorageManager::createBlob(const std::size_t num_slots,
// Make '*eviction_policy_' aware of the new blob's existence.
eviction_policy_->blockCreated(new_block_id);
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ sendBlockLocationMessage(new_block_id, kAddBlockLocationMessage);
+ }
+#endif
+
return new_block_id;
}
@@ -315,6 +402,12 @@ bool StorageManager::saveBlockOrBlob(const block_id block, const bool force) {
}
void StorageManager::evictBlockOrBlob(const block_id block) {
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ sendBlockLocationMessage(block, kDeleteBlockLocationMessage);
+ }
+#endif
+
BlockHandle handle;
{
SpinSharedMutexExclusiveLock<false> write_lock(blocks_shared_mutex_);
@@ -362,6 +455,87 @@ block_id StorageManager::allocateNewBlockOrBlob(const std::size_t num_slots,
return ++block_index_;
}
+#ifdef QUICKSTEP_DISTRIBUTED
+vector<string> StorageManager::getPeerDomainNetworkAddresses(const block_id block) {
+ serialization::BlockMessage proto;
+ proto.set_block_id(block);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kGetPeerDomainNetworkAddressesMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent GetPeerDomainNetworkAddressesMessage (typed '" << kGetPeerDomainNetworkAddressesMessage
+ << "') to BlockLocator";
+
+ DCHECK_NE(block_locator_client_id_, tmb::kClientIdNone);
+ DCHECK(bus_ != nullptr);
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ storage_manager_client_id_,
+ block_locator_client_id_,
+ move(message)));
+
+ const tmb::AnnotatedMessage annotated_message(bus_->Receive(storage_manager_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ CHECK_EQ(block_locator_client_id_, annotated_message.sender);
+ CHECK_EQ(kGetPeerDomainNetworkAddressesResponseMessage, tagged_message.message_type());
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') received GetPeerDomainNetworkAddressesResponseMessage from BlockLocator";
+
+ serialization::GetPeerDomainNetworkAddressesResponseMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ vector<string> domain_network_addresses;
+ for (int i = 0; i < response_proto.domain_network_addresses_size(); ++i) {
+ domain_network_addresses.push_back(response_proto.domain_network_addresses(i));
+ }
+
+ return domain_network_addresses;
+}
+
+void StorageManager::sendBlockLocationMessage(const block_id block,
+ const tmb::message_type_id message_type) {
+ switch (message_type) {
+ case kAddBlockLocationMessage:
+ LOG(INFO) << "Loaded Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
+ break;
+ case kDeleteBlockLocationMessage:
+ LOG(INFO) << "Evicted Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
+ break;
+ default:
+ LOG(FATAL) << "Unknown message type " << message_type;
+ }
+
+ serialization::BlockLocationMessage proto;
+ proto.set_block_id(block);
+ proto.set_block_domain(block_domain_);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ message_type);
+ free(proto_bytes);
+
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent BlockLocationMessage (typed '" << message_type
+ << "') to BlockLocator";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ storage_manager_client_id_,
+ block_locator_client_id_,
+ move(message)));
+}
+#endif
+
StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
const block_id block, const int numa_node) {
// The caller of this function holds an exclusive lock on this block/blob's
@@ -378,6 +552,12 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
loaded_handle.block_memory = block_buffer;
loaded_handle.block_memory_size = num_slots;
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ sendBlockLocationMessage(block, kAddBlockLocationMessage);
+ }
+#endif
+
return loaded_handle;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3789da72/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index 52326c2..55a011e 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -20,11 +20,14 @@
#include <atomic>
#include <chrono>
+#include <cstddef>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED
+
#include "storage/CountedReference.hpp"
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManager.hpp"
@@ -40,6 +43,10 @@
#include "gflags/gflags.h"
#include "gtest/gtest_prod.h"
+#include "tmb/id_typedefs.h"
+
+namespace tmb { class MessageBus; }
+
namespace quickstep {
DECLARE_int32(block_domain);
@@ -50,6 +57,7 @@ DECLARE_bool(use_hdfs);
#endif
class CatalogRelationSchema;
+
class StorageBlockLayout;
/** \addtogroup Storage
@@ -104,6 +112,33 @@ class StorageManager {
std::chrono::milliseconds(200))) {
}
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Constructor.
+ * @param storage_path The filesystem directory where blocks have persistent
+ * storage.
+ * @param block_domain The unique block domain.
+ * @param block_locator_client_id The TMB client ID of the block locator.
+ * @param bus A pointer to the TMB.
+ *
+ * @exception CorruptPersistentStorage The storage directory layout is not
+ * in the expected format.
+ **/
+ StorageManager(const std::string &storage_path,
+ const block_id_domain block_domain,
+ const tmb::client_id block_locator_client_id,
+ tmb::MessageBus *bus)
+ : StorageManager(storage_path,
+ block_domain,
+ FLAGS_buffer_pool_slots,
+ LRUKEvictionPolicyFactory::ConstructLRUKEvictionPolicy(
+ 2,
+ std::chrono::milliseconds(200)),
+ block_locator_client_id,
+ bus) {
+ }
+#endif
+
/**
* @brief Constructor.
* @param storage_path The filesystem directory where blocks have persistent
@@ -121,13 +156,18 @@ class StorageManager {
* @param eviction_policy The eviction policy that the storage manager should
* use to manage the cache. The storage manager takes
* ownership of *eviction_policy.
+ * @param block_locator_client_id The TMB client ID of the block locator.
+ * @param bus A pointer to the TMB.
+ *
* @exception CorruptPersistentStorage The storage directory layout is not
* in the expected format.
**/
StorageManager(const std::string &storage_path,
const block_id_domain block_domain,
const size_t max_memory_usage,
- EvictionPolicy *eviction_policy);
+ EvictionPolicy *eviction_policy,
+ const tmb::client_id block_locator_client_id = tmb::kClientIdNone,
+ tmb::MessageBus *bus = nullptr);
/**
* @brief Destructor which also destroys all managed blocks.
@@ -332,6 +372,27 @@ class StorageManager {
StorageBlockBase *block;
};
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Get the network info of all the remote StorageManagers which may
+ * load the given block in the buffer pool.
+ *
+ * @param block The block or blob to pull.
+ *
+ * @return The network info of all the possible peers to pull.
+ **/
+ std::vector<std::string> getPeerDomainNetworkAddresses(const block_id block);
+
+ /**
+ * @brief Update the block location info in BlockLocator.
+ *
+ * @param block The given block or blob.
+ * @param message_type Indicate whether to add or delete a block location.
+ **/
+ void sendBlockLocationMessage(const block_id block,
+ const tmb::message_type_id message_type);
+#endif
+
// Helper for createBlock() and createBlob(). Allocates a block ID and memory
// slots for a new StorageBlock or StorageBlob. Returns the allocated ID and
// writes the allocated slot range into 'handle->slot_index_low' and
@@ -459,6 +520,15 @@ class StorageManager {
std::unique_ptr<EvictionPolicy> eviction_policy_;
+#ifdef QUICKSTEP_DISTRIBUTED
+ const block_id_domain block_domain_;
+
+ tmb::client_id storage_manager_client_id_;
+#endif
+
+ const tmb::client_id block_locator_client_id_;
+ tmb::MessageBus *bus_;
+
std::unique_ptr<FileManager> file_manager_;
// Used to generate unique IDs in allocateNewBlockOrBlob().
@@ -486,6 +556,10 @@ class StorageManager {
static constexpr std::size_t kLockManagerNumShards = 0x2000-1;
ShardedLockManager<block_id, kLockManagerNumShards, SpinSharedMutex<false>> lock_manager_;
+ friend class BlockLocatorTest;
+ FRIEND_TEST(BlockLocatorTest, BlockTest);
+ FRIEND_TEST(BlockLocatorTest, BlobTest);
+
FRIEND_TEST(StorageManagerTest, DifferentNUMANodeBlobTestWithEviction);
FRIEND_TEST(StorageManagerTest, EvictFromSameShardTest);
[12/48] incubator-quickstep git commit: Added support for the
substring function. (#211)
Posted by hb...@apache.org.
Added support for the substring function. (#211)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1a685be4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1a685be4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1a685be4
Branch: refs/heads/query-manager-used-in-foreman
Commit: 1a685be4029d9c48e55ac508d5fc24fda245667a
Parents: 9e2fb82
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 5 11:17:37 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:41 2016 -0700
----------------------------------------------------------------------
parser/ParseBasicExpressions.cpp | 31 +
parser/ParseBasicExpressions.hpp | 77 +
parser/ParseExpression.hpp | 1 +
parser/SqlLexer.lpp | 2 +
parser/SqlParser.ypp | 16 +
parser/preprocessed/SqlLexer_gen.cpp | 1173 +++----
parser/preprocessed/SqlLexer_gen.hpp | 2 +-
parser/preprocessed/SqlParser_gen.cpp | 2920 +++++++++---------
parser/preprocessed/SqlParser_gen.hpp | 126 +-
query_optimizer/resolver/CMakeLists.txt | 1 +
query_optimizer/resolver/Resolver.cpp | 32 +
.../tests/execution_generator/Select.test | 24 +
query_optimizer/tests/resolver/Select.test | 78 +
types/operations/Operation.proto | 9 +
.../operations/unary_operations/CMakeLists.txt | 22 +
.../unary_operations/SubstringOperation.cpp | 214 ++
.../unary_operations/SubstringOperation.hpp | 234 ++
.../unary_operations/UnaryOperation.cpp | 4 +
.../unary_operations/UnaryOperationFactory.cpp | 12 +
.../unary_operations/UnaryOperationID.cpp | 6 +-
.../unary_operations/UnaryOperationID.hpp | 3 +
21 files changed, 2905 insertions(+), 2082 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/parser/ParseBasicExpressions.cpp
----------------------------------------------------------------------
diff --git a/parser/ParseBasicExpressions.cpp b/parser/ParseBasicExpressions.cpp
index 2b1d7e0..a9d84ea 100644
--- a/parser/ParseBasicExpressions.cpp
+++ b/parser/ParseBasicExpressions.cpp
@@ -189,4 +189,35 @@ void ParseExtractFunction::getFieldStringItems(
non_container_child_fields->push_back(date_expression_.get());
}
+std::string ParseSubstringFunction::generateName() const {
+ std::string name;
+ name.append("SUBSTRING(");
+ name.append(operand_->generateName());
+ name.append(" FROM ");
+ name.append(std::to_string(start_position_));
+ if (length_ != kDefaultLength) {
+ name.append(" FOR ");
+ name.append(std::to_string(length_));
+ }
+ name.push_back(')');
+ return name;
+}
+
+void ParseSubstringFunction::getFieldStringItems(
+ std::vector<std::string> *inline_field_names,
+ std::vector<std::string> *inline_field_values,
+ std::vector<std::string> *non_container_child_field_names,
+ std::vector<const ParseTreeNode*> *non_container_child_fields,
+ std::vector<std::string> *container_child_field_names,
+ std::vector<std::vector<const ParseTreeNode*>> *container_child_fields) const {
+ inline_field_names->push_back("start_position");
+ inline_field_values->push_back(std::to_string(start_position_));
+
+ inline_field_names->push_back("length");
+ inline_field_values->push_back(std::to_string(length_));
+
+ non_container_child_field_names->push_back("operand");
+ non_container_child_fields->push_back(operand_.get());
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/parser/ParseBasicExpressions.hpp
----------------------------------------------------------------------
diff --git a/parser/ParseBasicExpressions.hpp b/parser/ParseBasicExpressions.hpp
index 1886c8b..dea25d7 100644
--- a/parser/ParseBasicExpressions.hpp
+++ b/parser/ParseBasicExpressions.hpp
@@ -20,6 +20,8 @@
#ifndef QUICKSTEP_PARSER_PARSE_BASIC_EXPRESSIONS_HPP_
#define QUICKSTEP_PARSER_PARSE_BASIC_EXPRESSIONS_HPP_
+#include <cstddef>
+#include <limits>
#include <memory>
#include <string>
#include <vector>
@@ -511,6 +513,81 @@ class ParseExtractFunction : public ParseExpression {
DISALLOW_COPY_AND_ASSIGN(ParseExtractFunction);
};
+
+/**
+ * @brief Parsed representation of the substring function.
+ */
+class ParseSubstringFunction : public ParseExpression {
+ public:
+ static constexpr std::size_t kDefaultLength = std::numeric_limits<std::size_t>::max();
+
+ /**
+ * @brief Constructor.
+ *
+ * @param line_number The line number of the first token of the function call.
+ * @param column_number The column number of the first token of the function call.
+ * @param operand The operand of the substring.
+ * @param start_position The 1-based starting position of the substring.
+ * @param length Optional substring length.
+ */
+ ParseSubstringFunction(const int line_number,
+ const int column_number,
+ ParseExpression *operand,
+ const std::size_t start_position,
+ const std::size_t length = kDefaultLength)
+ : ParseExpression(line_number, column_number),
+ operand_(operand),
+ start_position_(start_position),
+ length_(length) {}
+
+ ExpressionType getExpressionType() const override {
+ return kSubstring;
+ }
+
+ std::string getName() const override {
+ return "Substring";
+ }
+
+ /**
+ * @return The operand of the substring.
+ */
+ const ParseExpression* operand() const {
+ return operand_.get();
+ }
+
+ /**
+ * @return The 1-based starting position of the substring.
+ */
+ std::size_t start_position() const {
+ return start_position_;
+ }
+
+ /**
+ * @return Then substring length.
+ */
+ std::size_t length() const {
+ return length_;
+ }
+
+ std::string generateName() const override;
+
+ protected:
+ void getFieldStringItems(
+ std::vector<std::string> *inline_field_names,
+ std::vector<std::string> *inline_field_values,
+ std::vector<std::string> *non_container_child_field_names,
+ std::vector<const ParseTreeNode*> *non_container_child_fields,
+ std::vector<std::string> *container_child_field_names,
+ std::vector<std::vector<const ParseTreeNode*>> *container_child_fields) const override;
+
+ private:
+ std::unique_ptr<ParseExpression> operand_;
+ const std::size_t start_position_;
+ const std::size_t length_;
+
+ DISALLOW_COPY_AND_ASSIGN(ParseSubstringFunction);
+};
+
/** @} */
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/parser/ParseExpression.hpp
----------------------------------------------------------------------
diff --git a/parser/ParseExpression.hpp b/parser/ParseExpression.hpp
index e959e72..3541f83 100644
--- a/parser/ParseExpression.hpp
+++ b/parser/ParseExpression.hpp
@@ -45,6 +45,7 @@ class ParseExpression : public ParseTreeNode {
kSearchedCaseExpression,
kSimpleCaseExpression,
kSubqueryExpression,
+ kSubstring,
kUnaryExpression,
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/parser/SqlLexer.lpp
----------------------------------------------------------------------
diff --git a/parser/SqlLexer.lpp b/parser/SqlLexer.lpp
index a399723..ac1c708 100644
--- a/parser/SqlLexer.lpp
+++ b/parser/SqlLexer.lpp
@@ -209,6 +209,7 @@ unsigned_numeric_literal {exact_numeric_literal}|{approximate_numeric_literal}
"false" return TOKEN_FALSE;
"first" return TOKEN_FIRST;
"float" return TOKEN_FLOAT;
+ "for" return TOKEN_FOR;
"foreign" return TOKEN_FOREIGN;
"from" return TOKEN_FROM;
"full" return TOKEN_FULL;
@@ -258,6 +259,7 @@ unsigned_numeric_literal {exact_numeric_literal}|{approximate_numeric_literal}
"set" return TOKEN_SET;
"sma" return TOKEN_SMA;
"smallint" return TOKEN_SMALLINT;
+ "substring" return TOKEN_SUBSTRING;
"table" return TOKEN_TABLE;
"then" return TOKEN_THEN;
"time" return TOKEN_TIME;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/parser/SqlParser.ypp
----------------------------------------------------------------------
diff --git a/parser/SqlParser.ypp b/parser/SqlParser.ypp
index 1202d66..b07c48e 100644
--- a/parser/SqlParser.ypp
+++ b/parser/SqlParser.ypp
@@ -273,6 +273,7 @@ void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const std::string
%token TOKEN_FALSE;
%token TOKEN_FIRST;
%token TOKEN_FLOAT;
+%token TOKEN_FOR;
%token TOKEN_FOREIGN;
%token TOKEN_FROM;
%token TOKEN_FULL;
@@ -319,6 +320,7 @@ void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const std::string
%token TOKEN_SET;
%token TOKEN_SMA;
%token TOKEN_SMALLINT;
+%token TOKEN_SUBSTRING;
%token TOKEN_TABLE;
%token TOKEN_THEN;
%token TOKEN_TIME;
@@ -361,6 +363,7 @@ void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const std::string
case_expression
opt_else_clause
extract_function
+ substr_function
%type <attribute_>
attribute_ref
@@ -1505,6 +1508,9 @@ expression_base:
| extract_function {
$$ = $1;
}
+ | substr_function {
+ $$ = $1;
+ }
| case_expression {
$$ = $1;
}
@@ -1536,6 +1542,16 @@ extract_function:
$$ = new quickstep::ParseExtractFunction(@1.first_line, @1.first_column, $3, $5);
};
+substr_function:
+ TOKEN_SUBSTRING '(' add_expression TOKEN_FROM TOKEN_UNSIGNED_NUMVAL ')' {
+ $$ = new quickstep::ParseSubstringFunction(
+ @1.first_line, @1.first_column, $3, $5->long_value());
+ }
+ | TOKEN_SUBSTRING '(' add_expression TOKEN_FROM TOKEN_UNSIGNED_NUMVAL TOKEN_FOR TOKEN_UNSIGNED_NUMVAL ')' {
+ $$ = new quickstep::ParseSubstringFunction(
+ @1.first_line, @1.first_column, $3, $5->long_value(), $7->long_value());
+ };
+
case_expression:
TOKEN_CASE add_expression simple_when_clause_list opt_else_clause TOKEN_END {
$$ = new quickstep::ParseSimpleCaseExpression(@1.first_line, @1.first_column, $2, $3, $4);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/parser/preprocessed/SqlLexer_gen.cpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlLexer_gen.cpp b/parser/preprocessed/SqlLexer_gen.cpp
index d836988..db20491 100644
--- a/parser/preprocessed/SqlLexer_gen.cpp
+++ b/parser/preprocessed/SqlLexer_gen.cpp
@@ -381,8 +381,8 @@ static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner );
*yy_cp = '\0'; \
yyg->yy_c_buf_p = yy_cp;
-#define YY_NUM_RULES 150
-#define YY_END_OF_BUFFER 151
+#define YY_NUM_RULES 152
+#define YY_END_OF_BUFFER 153
/* This struct is not used in this scanner,
but its presence is necessary. */
struct yy_trans_info
@@ -390,68 +390,69 @@ struct yy_trans_info
flex_int32_t yy_verify;
flex_int32_t yy_nxt;
};
-static yyconst flex_int16_t yy_accept[545] =
+static yyconst flex_int16_t yy_accept[553] =
{ 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 151, 2, 2, 149, 149, 148, 147, 149,
- 126, 122, 125, 122, 122, 145, 118, 115, 119, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 123, 4, 5, 5, 3, 141,
- 141, 138, 142, 142, 136, 143, 143, 140, 1, 148,
- 116, 146, 145, 145, 145, 0, 120, 117, 121, 144,
- 144, 144, 144, 10, 144, 144, 144, 22, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 124, 144,
-
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 57, 65, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 79, 80, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 4, 5, 3, 141, 137, 142, 135, 135, 127, 129,
- 130, 131, 132, 133, 134, 135, 143, 139, 146, 145,
- 0, 145, 6, 7, 144, 9, 11, 144, 144, 15,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 32, 144, 144, 144, 144, 144, 144, 144, 144, 43,
-
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 61, 144, 67, 144,
- 144, 144, 144, 144, 144, 144, 75, 144, 78, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 96, 97, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 127, 129,
- 128, 144, 144, 144, 144, 144, 144, 144, 20, 23,
- 144, 144, 144, 28, 144, 144, 30, 144, 144, 144,
- 144, 37, 144, 144, 41, 42, 144, 144, 144, 144,
- 144, 144, 144, 51, 52, 144, 54, 144, 56, 144,
-
- 144, 144, 144, 64, 66, 68, 69, 70, 144, 72,
- 144, 144, 76, 144, 144, 144, 144, 144, 87, 144,
- 89, 144, 144, 144, 144, 144, 144, 144, 144, 100,
- 101, 103, 144, 144, 144, 144, 144, 144, 110, 144,
- 112, 113, 127, 128, 8, 144, 144, 144, 144, 144,
- 144, 144, 25, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 47, 48,
- 49, 144, 53, 144, 58, 59, 144, 144, 144, 71,
- 144, 74, 77, 81, 82, 144, 144, 144, 88, 144,
- 144, 92, 144, 144, 144, 144, 99, 144, 144, 144,
-
- 144, 107, 144, 144, 111, 144, 144, 144, 14, 144,
- 144, 144, 144, 144, 26, 144, 29, 144, 144, 144,
- 144, 35, 144, 144, 144, 40, 144, 45, 144, 144,
- 55, 60, 144, 144, 73, 144, 144, 144, 144, 91,
- 144, 94, 95, 144, 144, 144, 105, 106, 108, 144,
- 144, 144, 13, 144, 144, 144, 144, 144, 144, 21,
- 144, 33, 34, 144, 144, 144, 144, 46, 50, 62,
- 144, 144, 85, 86, 144, 144, 144, 144, 144, 109,
- 144, 144, 144, 144, 144, 144, 144, 144, 31, 144,
- 144, 39, 144, 63, 144, 144, 144, 98, 144, 144,
-
- 144, 12, 144, 144, 144, 144, 24, 144, 36, 144,
- 144, 83, 144, 144, 102, 144, 114, 16, 144, 144,
- 144, 27, 38, 144, 84, 90, 144, 144, 144, 18,
- 19, 144, 144, 104, 144, 144, 144, 144, 144, 93,
- 144, 44, 17, 0
+ 0, 0, 153, 2, 2, 151, 151, 150, 149, 151,
+ 128, 124, 127, 124, 124, 147, 120, 117, 121, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 125, 4, 5, 5, 3, 143,
+ 143, 140, 144, 144, 138, 145, 145, 142, 1, 150,
+ 118, 148, 147, 147, 147, 0, 122, 119, 123, 146,
+ 146, 146, 146, 10, 146, 146, 146, 22, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 126, 146,
+
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 58, 66, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 80, 81, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 4, 5, 3, 143, 139, 144, 137, 137, 129,
+ 131, 132, 133, 134, 135, 136, 137, 145, 141, 148,
+ 147, 0, 147, 6, 7, 146, 9, 11, 146, 146,
+ 15, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 32, 146, 146, 146, 146, 146, 146, 146, 146,
+
+ 43, 146, 146, 146, 146, 146, 146, 50, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 62, 146, 68,
+ 146, 146, 146, 146, 146, 146, 146, 76, 146, 79,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 97, 98, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 129, 131, 130, 146, 146, 146, 146, 146, 146, 146,
+ 20, 23, 146, 146, 146, 28, 146, 146, 30, 146,
+ 146, 146, 146, 37, 146, 146, 41, 42, 146, 146,
+ 146, 146, 146, 146, 146, 52, 53, 146, 55, 146,
+
+ 57, 146, 146, 146, 146, 65, 67, 69, 70, 71,
+ 146, 73, 146, 146, 77, 146, 146, 146, 146, 146,
+ 88, 146, 90, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 102, 103, 105, 146, 146, 146, 146, 146,
+ 146, 112, 146, 114, 115, 129, 130, 8, 146, 146,
+ 146, 146, 146, 146, 146, 25, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 47, 48, 49, 146, 54, 146, 59, 60, 146,
+ 146, 146, 72, 146, 75, 78, 82, 83, 146, 146,
+ 146, 89, 146, 146, 93, 146, 146, 146, 146, 146,
+
+ 101, 146, 146, 146, 146, 109, 146, 146, 113, 146,
+ 146, 146, 14, 146, 146, 146, 146, 146, 26, 146,
+ 29, 146, 146, 146, 146, 35, 146, 146, 146, 40,
+ 146, 45, 146, 146, 56, 61, 146, 146, 74, 146,
+ 146, 146, 146, 92, 146, 95, 96, 146, 146, 146,
+ 146, 107, 108, 110, 146, 146, 146, 13, 146, 146,
+ 146, 146, 146, 146, 21, 146, 33, 34, 146, 146,
+ 146, 146, 46, 51, 63, 146, 146, 86, 87, 146,
+ 146, 146, 146, 146, 146, 111, 146, 146, 146, 146,
+ 146, 146, 146, 146, 31, 146, 146, 39, 146, 64,
+
+ 146, 146, 146, 99, 146, 146, 146, 146, 12, 146,
+ 146, 146, 146, 24, 146, 36, 146, 146, 84, 146,
+ 146, 100, 104, 146, 116, 16, 146, 146, 146, 27,
+ 38, 146, 85, 91, 146, 146, 146, 18, 19, 146,
+ 146, 106, 146, 146, 146, 146, 146, 94, 146, 44,
+ 17, 0
} ;
static yyconst YY_CHAR yy_ec[256] =
@@ -498,143 +499,145 @@ static yyconst YY_CHAR yy_meta[72] =
8
} ;
-static yyconst flex_uint16_t yy_base[560] =
+static yyconst flex_uint16_t yy_base[568] =
{ 0,
0, 1, 46, 0, 117, 163, 2, 3, 128, 132,
- 6, 10, 257, 1212, 1212, 0, 1212, 13, 1212, 233,
- 1212, 1212, 1212, 208, 6, 130, 4, 1212, 195, 124,
+ 6, 10, 257, 1219, 1219, 0, 1219, 13, 1219, 233,
+ 1219, 1219, 1219, 208, 6, 130, 4, 1219, 195, 124,
161, 170, 178, 207, 260, 92, 167, 161, 96, 107,
- 219, 214, 212, 224, 236, 92, 279, 171, 278, 281,
- 128, 227, 0, 125, 1212, 184, 4, 19, 0, 0,
+ 219, 214, 212, 224, 236, 92, 279, 272, 278, 281,
+ 128, 168, 0, 125, 1219, 184, 4, 19, 0, 0,
0, 146, 0, 0, 343, 0, 0, 145, 0, 22,
- 1212, 0, 297, 316, 338, 18, 1212, 1212, 1212, 0,
- 170, 227, 173, 178, 224, 299, 270, 0, 270, 335,
- 330, 286, 320, 327, 376, 308, 316, 326, 1212, 335,
-
- 351, 355, 371, 348, 346, 353, 359, 370, 382, 383,
- 380, 379, 399, 0, 392, 379, 386, 401, 399, 401,
- 402, 407, 402, 413, 420, 0, 431, 417, 420, 422,
- 434, 437, 435, 451, 446, 433, 456, 459, 459, 457,
- 450, 444, 454, 462, 469, 465, 465, 474, 460, 483,
- 148, 29, 0, 0, 1212, 0, 1212, 1212, 22, 24,
- 1212, 1212, 1212, 1212, 1212, 0, 0, 1212, 0, 515,
- 26, 28, 0, 0, 488, 0, 490, 473, 489, 478,
- 501, 502, 496, 512, 496, 499, 494, 520, 503, 521,
- 0, 519, 528, 526, 529, 514, 535, 522, 534, 0,
-
- 539, 522, 524, 532, 534, 553, 551, 546, 550, 544,
- 564, 564, 556, 570, 571, 572, 574, 564, 0, 561,
- 564, 581, 578, 583, 571, 573, 0, 583, 0, 591,
- 592, 578, 596, 587, 589, 604, 600, 609, 612, 612,
- 98, 608, 625, 0, 619, 620, 619, 629, 630, 624,
- 620, 638, 628, 623, 642, 633, 640, 632, 30, 125,
- 0, 635, 640, 650, 642, 652, 647, 654, 0, 668,
- 659, 659, 655, 0, 658, 663, 668, 676, 669, 671,
- 679, 688, 685, 683, 0, 0, 681, 680, 701, 698,
- 685, 686, 699, 0, 0, 693, 0, 697, 0, 688,
-
- 695, 696, 709, 0, 0, 0, 0, 0, 695, 0,
- 705, 720, 711, 715, 718, 730, 741, 746, 0, 743,
- 0, 731, 726, 731, 748, 739, 752, 746, 755, 0,
- 742, 0, 758, 743, 746, 760, 764, 762, 0, 766,
- 0, 759, 136, 1212, 0, 769, 769, 763, 784, 772,
- 780, 791, 0, 783, 786, 800, 801, 798, 807, 797,
- 805, 802, 799, 802, 813, 814, 802, 819, 0, 0,
- 0, 817, 0, 818, 0, 0, 807, 823, 807, 0,
- 825, 0, 0, 0, 0, 811, 818, 823, 0, 838,
- 828, 0, 841, 845, 832, 846, 0, 842, 844, 859,
-
- 860, 0, 847, 866, 0, 853, 860, 857, 0, 852,
- 858, 876, 870, 860, 0, 881, 0, 878, 872, 874,
- 867, 0, 868, 885, 887, 0, 93, 0, 879, 887,
- 0, 0, 884, 903, 0, 898, 890, 888, 906, 0,
- 909, 0, 0, 908, 922, 923, 0, 0, 0, 907,
- 912, 913, 0, 920, 917, 921, 923, 932, 929, 0,
- 935, 0, 0, 936, 934, 924, 926, 0, 0, 0,
- 934, 932, 0, 0, 945, 948, 939, 947, 949, 0,
- 945, 961, 957, 962, 963, 960, 963, 968, 0, 965,
- 970, 0, 965, 0, 973, 985, 979, 0, 977, 979,
-
- 988, 0, 991, 994, 989, 997, 0, 983, 0, 997,
- 987, 987, 996, 1008, 0, 1006, 0, 0, 1002, 1018,
- 1008, 0, 0, 1020, 0, 0, 1016, 1032, 1018, 0,
- 0, 1025, 1035, 0, 1032, 1035, 1025, 1040, 1029, 0,
- 1031, 0, 0, 1212, 1096, 1106, 1116, 1126, 1136, 1140,
- 1143, 1149, 1157, 1167, 1177, 1187, 1197, 1202, 1204
+ 1219, 0, 307, 337, 341, 18, 1219, 1219, 1219, 0,
+ 170, 224, 177, 181, 215, 269, 224, 0, 263, 326,
+ 336, 286, 290, 333, 382, 321, 322, 329, 1219, 328,
+
+ 347, 351, 346, 350, 346, 353, 352, 374, 386, 385,
+ 383, 382, 402, 0, 395, 382, 389, 403, 399, 397,
+ 399, 401, 399, 408, 418, 0, 421, 406, 410, 425,
+ 437, 438, 438, 455, 451, 436, 457, 462, 462, 463,
+ 461, 454, 447, 454, 462, 469, 465, 466, 474, 460,
+ 481, 148, 29, 0, 0, 1219, 0, 1219, 1219, 22,
+ 24, 1219, 1219, 1219, 1219, 1219, 0, 0, 1219, 0,
+ 509, 26, 28, 0, 0, 478, 0, 481, 478, 501,
+ 493, 514, 513, 501, 517, 500, 503, 498, 523, 505,
+ 521, 0, 518, 527, 526, 529, 513, 532, 520, 532,
+
+ 0, 537, 521, 523, 523, 525, 558, 563, 559, 562,
+ 554, 568, 568, 560, 574, 575, 576, 577, 569, 0,
+ 564, 565, 581, 578, 582, 569, 571, 0, 581, 0,
+ 589, 590, 576, 595, 586, 580, 595, 605, 621, 625,
+ 624, 98, 618, 629, 0, 623, 617, 625, 624, 634,
+ 635, 629, 625, 642, 630, 624, 644, 634, 641, 632,
+ 30, 125, 0, 633, 639, 649, 642, 652, 647, 647,
+ 0, 674, 673, 675, 670, 0, 671, 674, 673, 681,
+ 674, 676, 684, 693, 690, 688, 0, 0, 685, 682,
+ 702, 700, 686, 687, 699, 0, 0, 693, 0, 696,
+
+ 0, 687, 694, 696, 726, 0, 0, 0, 0, 0,
+ 696, 0, 698, 734, 727, 730, 731, 741, 746, 751,
+ 0, 748, 0, 736, 731, 736, 753, 744, 756, 748,
+ 741, 758, 0, 745, 0, 760, 745, 747, 761, 764,
+ 762, 0, 767, 0, 761, 136, 1219, 0, 772, 786,
+ 780, 801, 788, 800, 805, 0, 795, 792, 806, 807,
+ 804, 813, 803, 811, 808, 804, 805, 816, 817, 804,
+ 821, 0, 0, 0, 818, 0, 819, 0, 0, 807,
+ 823, 808, 0, 827, 0, 0, 0, 0, 814, 835,
+ 840, 0, 855, 846, 0, 859, 861, 846, 858, 850,
+
+ 0, 849, 851, 866, 867, 0, 854, 873, 0, 860,
+ 867, 863, 0, 857, 862, 880, 873, 863, 0, 883,
+ 0, 880, 873, 876, 870, 0, 872, 903, 913, 0,
+ 93, 0, 898, 906, 0, 0, 904, 922, 0, 915,
+ 905, 901, 913, 0, 916, 0, 0, 915, 921, 930,
+ 931, 0, 0, 0, 915, 920, 921, 0, 928, 923,
+ 926, 928, 936, 933, 0, 938, 0, 0, 939, 937,
+ 928, 931, 0, 0, 0, 953, 959, 0, 0, 973,
+ 969, 960, 967, 969, 970, 0, 964, 978, 972, 971,
+ 972, 969, 972, 977, 0, 974, 979, 0, 975, 0,
+
+ 982, 992, 985, 0, 992, 984, 985, 994, 0, 997,
+ 1001, 996, 1018, 0, 1012, 0, 1027, 1018, 1019, 1020,
+ 1031, 0, 0, 1029, 0, 0, 1024, 1038, 1026, 0,
+ 0, 1036, 0, 0, 1026, 1042, 1028, 0, 0, 1035,
+ 1045, 0, 1042, 1046, 1036, 1050, 1037, 0, 1038, 0,
+ 0, 1219, 1103, 1113, 1123, 1133, 1143, 1147, 1150, 1156,
+ 1164, 1174, 1184, 1194, 1204, 1209, 1211
} ;
-static yyconst flex_int16_t yy_def[560] =
+static yyconst flex_int16_t yy_def[568] =
{ 0,
- 545, 545, 544, 3, 546, 546, 547, 547, 548, 548,
- 549, 549, 544, 544, 544, 550, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 544, 544, 544, 544, 552, 553,
- 553, 544, 554, 554, 555, 556, 556, 544, 550, 544,
- 544, 557, 544, 544, 544, 544, 544, 544, 544, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 544, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 544, 544, 552, 553, 544, 554, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 558, 556, 544, 557, 544,
- 544, 544, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 544, 544,
- 559, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 544, 544, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 0, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544
+ 553, 553, 552, 3, 554, 554, 555, 555, 556, 556,
+ 557, 557, 552, 552, 552, 558, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 552, 552, 552, 552, 560, 561,
+ 561, 552, 562, 562, 563, 564, 564, 552, 558, 552,
+ 552, 565, 552, 552, 552, 552, 552, 552, 552, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 552, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 552, 552, 560, 561, 552, 562, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 566, 564, 552, 565,
+ 552, 552, 552, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 552, 552, 567, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 552, 552, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 0, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552
} ;
-static yyconst flex_uint16_t yy_nxt[1284] =
+static yyconst flex_uint16_t yy_nxt[1291] =
{ 0,
- 544, 544, 15, 15, 61, 61, 152, 152, 67, 62,
- 62, 68, 67, 544, 70, 68, 70, 73, 73, 77,
- 78, 152, 152, 70, 544, 70, 171, 171, 544, 172,
- 172, 152, 152, 259, 260, 260, 260, 172, 172, 172,
- 172, 343, 260, 544, 16, 16, 17, 18, 19, 18,
+ 552, 552, 15, 15, 61, 61, 153, 153, 67, 62,
+ 62, 68, 67, 552, 70, 68, 70, 73, 73, 77,
+ 78, 153, 153, 70, 552, 70, 172, 172, 552, 173,
+ 173, 153, 153, 261, 262, 262, 262, 173, 173, 173,
+ 173, 346, 262, 552, 16, 16, 17, 18, 19, 18,
20, 21, 22, 23, 22, 24, 25, 26, 26, 17,
27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
37, 38, 39, 40, 41, 42, 43, 44, 45, 46,
@@ -644,142 +647,142 @@ static yyconst flex_uint16_t yy_nxt[1284] =
38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
48, 49, 50, 51, 52, 53, 54, 17, 56, 57,
58, 17, 17, 17, 17, 17, 110, 115, 116, 132,
- 64, 17, 17, 17, 64, 62, 260, 260, 467, 62,
- 74, 75, 75, 325, 81, 147, 150, 260, 260, 151,
- 168, 76, 82, 155, 83, 110, 115, 116, 132, 84,
+ 64, 17, 17, 17, 64, 62, 262, 262, 472, 62,
+ 74, 75, 75, 327, 81, 148, 151, 262, 262, 152,
+ 169, 76, 82, 156, 83, 110, 115, 116, 132, 84,
17, 17, 17, 17, 56, 57, 58, 17, 17, 17,
- 17, 17, 65, 81, 147, 150, 65, 17, 17, 17,
- 76, 82, 85, 83, 111, 151, 86, 89, 84, 87,
- 173, 113, 137, 176, 90, 94, 114, 177, 112, 95,
+ 17, 17, 65, 81, 148, 151, 65, 17, 17, 17,
+ 76, 82, 85, 83, 111, 152, 86, 89, 84, 87,
+ 174, 113, 149, 150, 90, 94, 114, 177, 112, 95,
- 138, 91, 88, 96, 92, 93, 17, 17, 17, 97,
- 79, 85, 98, 111, 99, 86, 89, 72, 87, 173,
- 113, 137, 176, 90, 94, 114, 177, 112, 95, 138,
+ 178, 91, 88, 96, 92, 93, 17, 17, 17, 97,
+ 79, 85, 98, 111, 99, 86, 89, 72, 87, 174,
+ 113, 149, 150, 90, 94, 114, 177, 112, 95, 178,
91, 88, 96, 92, 93, 100, 117, 101, 97, 121,
118, 98, 102, 123, 119, 122, 125, 103, 71, 124,
- 120, 148, 149, 129, 126, 174, 544, 130, 127, 544,
- 178, 128, 544, 175, 100, 117, 101, 544, 121, 118,
+ 120, 179, 175, 129, 126, 182, 552, 130, 127, 552,
+ 176, 128, 552, 552, 100, 117, 101, 552, 121, 118,
131, 102, 123, 119, 122, 125, 103, 104, 124, 120,
- 148, 149, 129, 126, 174, 105, 130, 127, 106, 178,
- 128, 107, 175, 544, 108, 139, 133, 109, 544, 131,
-
- 134, 181, 140, 141, 135, 182, 104, 188, 73, 73,
- 136, 144, 142, 145, 105, 143, 146, 106, 76, 544,
- 107, 544, 179, 108, 139, 133, 109, 170, 170, 134,
- 181, 140, 141, 135, 182, 180, 188, 76, 189, 136,
- 144, 142, 145, 196, 143, 146, 158, 76, 74, 75,
- 75, 179, 183, 197, 159, 160, 184, 198, 185, 76,
- 186, 161, 187, 190, 180, 162, 76, 189, 191, 544,
- 199, 200, 196, 163, 201, 544, 204, 164, 544, 165,
- 205, 183, 197, 166, 206, 184, 198, 185, 76, 186,
- 161, 187, 190, 207, 162, 192, 202, 191, 193, 199,
-
- 200, 208, 163, 201, 194, 204, 164, 203, 165, 205,
- 209, 195, 166, 206, 210, 211, 213, 218, 212, 214,
- 219, 220, 207, 221, 192, 202, 222, 193, 223, 215,
- 208, 224, 225, 194, 216, 217, 203, 226, 227, 209,
- 195, 228, 229, 210, 211, 213, 218, 212, 214, 219,
- 220, 230, 221, 231, 232, 222, 233, 223, 215, 234,
- 224, 225, 235, 216, 217, 236, 226, 227, 237, 240,
- 228, 229, 241, 238, 239, 242, 245, 246, 247, 248,
- 230, 249, 231, 232, 243, 233, 250, 251, 234, 252,
- 253, 235, 244, 254, 236, 256, 257, 237, 240, 255,
-
- 258, 241, 238, 239, 242, 245, 246, 247, 248, 262,
- 249, 263, 264, 243, 265, 250, 251, 266, 252, 253,
- 267, 244, 254, 269, 256, 257, 170, 170, 255, 258,
- 270, 271, 268, 272, 273, 274, 76, 275, 262, 276,
- 263, 264, 277, 265, 278, 279, 266, 280, 282, 267,
- 283, 281, 269, 284, 285, 286, 287, 288, 289, 270,
- 271, 268, 272, 273, 274, 76, 275, 290, 276, 291,
- 292, 277, 293, 278, 279, 294, 280, 282, 295, 283,
- 281, 296, 284, 285, 286, 287, 288, 289, 297, 298,
- 299, 300, 301, 302, 305, 303, 290, 306, 291, 292,
-
- 307, 293, 308, 309, 294, 304, 310, 295, 311, 312,
- 296, 313, 314, 315, 316, 317, 318, 297, 298, 299,
- 300, 301, 302, 305, 303, 319, 306, 320, 321, 307,
- 322, 308, 309, 323, 304, 310, 324, 311, 312, 326,
- 313, 314, 315, 316, 317, 318, 327, 328, 329, 330,
- 331, 332, 333, 334, 319, 335, 320, 321, 336, 322,
- 337, 338, 323, 339, 341, 324, 342, 340, 326, 345,
- 346, 347, 348, 349, 350, 327, 328, 329, 330, 331,
- 332, 333, 334, 351, 335, 352, 353, 336, 354, 337,
- 338, 355, 339, 341, 356, 342, 340, 357, 345, 346,
-
- 347, 348, 349, 350, 358, 359, 360, 361, 362, 363,
- 364, 365, 351, 366, 352, 353, 367, 354, 368, 369,
- 355, 370, 371, 356, 372, 373, 357, 374, 375, 376,
- 377, 380, 378, 358, 359, 360, 361, 362, 363, 364,
- 365, 381, 366, 379, 382, 367, 383, 368, 369, 384,
- 370, 371, 385, 372, 373, 386, 374, 375, 376, 377,
- 380, 378, 387, 388, 389, 390, 391, 392, 393, 394,
- 381, 395, 379, 382, 396, 383, 397, 398, 384, 399,
- 400, 385, 401, 402, 386, 403, 404, 405, 406, 407,
- 408, 387, 388, 389, 390, 391, 392, 393, 394, 409,
-
- 395, 410, 413, 396, 411, 397, 398, 412, 399, 400,
- 414, 401, 402, 415, 403, 404, 405, 406, 407, 408,
- 416, 417, 418, 419, 420, 421, 422, 423, 409, 424,
- 410, 413, 425, 411, 426, 427, 412, 428, 429, 414,
- 430, 431, 415, 432, 433, 434, 435, 436, 437, 416,
- 417, 418, 419, 420, 421, 422, 423, 438, 424, 439,
- 440, 425, 441, 426, 427, 442, 428, 429, 443, 430,
- 431, 444, 432, 433, 434, 435, 436, 437, 445, 446,
- 447, 448, 449, 450, 451, 452, 438, 453, 439, 440,
- 454, 441, 455, 456, 442, 457, 458, 443, 459, 460,
-
- 444, 461, 462, 463, 464, 465, 466, 445, 446, 447,
- 448, 449, 450, 451, 452, 468, 453, 469, 470, 454,
- 471, 455, 456, 472, 457, 458, 473, 459, 460, 474,
- 461, 462, 463, 464, 465, 466, 475, 476, 477, 478,
- 479, 480, 481, 482, 468, 483, 469, 470, 484, 471,
- 485, 486, 472, 487, 488, 473, 489, 490, 474, 491,
- 492, 493, 494, 495, 496, 475, 476, 477, 478, 479,
- 480, 481, 482, 497, 483, 498, 499, 484, 500, 485,
- 486, 501, 487, 488, 502, 489, 490, 503, 491, 492,
- 493, 494, 495, 496, 504, 505, 506, 507, 508, 509,
-
- 510, 511, 497, 512, 498, 499, 513, 500, 514, 515,
- 501, 516, 517, 502, 518, 519, 503, 520, 521, 522,
- 523, 524, 525, 504, 505, 506, 507, 508, 509, 510,
- 511, 526, 512, 527, 528, 513, 529, 514, 515, 530,
- 516, 517, 531, 518, 519, 532, 520, 521, 522, 523,
- 524, 525, 533, 534, 535, 536, 537, 538, 539, 540,
- 526, 541, 527, 528, 542, 529, 543, 544, 530, 544,
- 544, 531, 544, 544, 532, 544, 544, 544, 544, 544,
- 544, 533, 534, 535, 536, 537, 538, 539, 540, 544,
- 541, 544, 544, 542, 544, 543, 14, 14, 14, 14,
-
- 14, 14, 14, 14, 14, 14, 59, 59, 59, 59,
- 59, 59, 59, 59, 59, 59, 60, 60, 60, 60,
- 60, 60, 60, 60, 60, 60, 63, 63, 63, 63,
- 63, 63, 63, 63, 63, 63, 66, 66, 66, 66,
- 66, 66, 66, 66, 66, 66, 69, 69, 80, 80,
- 80, 544, 80, 153, 153, 153, 153, 154, 154, 154,
- 544, 154, 154, 154, 154, 154, 154, 156, 156, 156,
- 544, 156, 156, 156, 156, 544, 156, 157, 157, 157,
- 157, 157, 157, 157, 157, 157, 157, 167, 167, 544,
- 167, 167, 167, 167, 167, 167, 167, 169, 544, 169,
-
- 169, 169, 169, 169, 169, 169, 169, 261, 261, 344,
- 344, 13, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544
+ 179, 175, 129, 126, 182, 105, 130, 127, 106, 176,
+ 128, 107, 180, 137, 108, 140, 133, 109, 183, 131,
+
+ 134, 138, 141, 142, 135, 181, 104, 189, 190, 139,
+ 136, 145, 143, 146, 105, 144, 147, 106, 73, 73,
+ 107, 180, 137, 108, 140, 133, 109, 183, 76, 134,
+ 138, 141, 142, 135, 181, 552, 189, 190, 139, 136,
+ 145, 143, 146, 184, 144, 147, 159, 185, 171, 171,
+ 552, 74, 75, 75, 160, 161, 197, 76, 76, 198,
+ 199, 162, 76, 200, 186, 163, 187, 201, 188, 191,
+ 202, 203, 184, 164, 192, 552, 185, 165, 205, 166,
+ 206, 552, 204, 167, 207, 197, 208, 76, 198, 199,
+ 162, 76, 200, 186, 163, 187, 201, 188, 191, 202,
+
+ 203, 193, 164, 192, 194, 209, 165, 205, 166, 206,
+ 195, 204, 167, 207, 210, 208, 211, 196, 212, 214,
+ 219, 213, 215, 220, 221, 222, 223, 225, 224, 226,
+ 193, 227, 216, 194, 209, 228, 229, 217, 218, 195,
+ 230, 231, 232, 210, 233, 211, 196, 212, 214, 219,
+ 213, 215, 220, 221, 222, 223, 225, 224, 226, 234,
+ 227, 216, 235, 236, 228, 229, 217, 218, 237, 230,
+ 231, 232, 238, 233, 241, 242, 243, 239, 240, 246,
+ 247, 248, 249, 250, 251, 244, 252, 253, 234, 254,
+ 255, 235, 236, 245, 256, 258, 259, 237, 260, 264,
+
+ 257, 238, 265, 241, 242, 243, 239, 240, 246, 247,
+ 248, 249, 250, 251, 244, 252, 253, 266, 254, 255,
+ 171, 171, 245, 256, 258, 259, 267, 260, 264, 257,
+ 76, 265, 268, 269, 271, 272, 273, 274, 275, 276,
+ 277, 278, 279, 280, 281, 270, 266, 282, 284, 285,
+ 286, 283, 287, 288, 289, 267, 290, 291, 292, 76,
+ 293, 268, 269, 271, 272, 273, 274, 275, 276, 277,
+ 278, 279, 280, 281, 270, 294, 282, 284, 285, 286,
+ 283, 287, 288, 289, 295, 290, 291, 292, 296, 293,
+ 297, 298, 299, 300, 301, 302, 303, 304, 305, 307,
+
+ 308, 309, 310, 311, 294, 312, 313, 314, 306, 315,
+ 316, 317, 318, 295, 319, 320, 321, 296, 322, 297,
+ 298, 299, 300, 301, 302, 303, 304, 305, 307, 308,
+ 309, 310, 311, 323, 312, 313, 314, 306, 315, 316,
+ 317, 318, 324, 319, 320, 321, 325, 322, 326, 328,
+ 329, 330, 331, 332, 333, 334, 335, 336, 337, 338,
+ 339, 340, 323, 341, 342, 344, 345, 348, 343, 349,
+ 350, 324, 351, 352, 353, 325, 354, 326, 328, 329,
+ 330, 331, 332, 333, 334, 335, 336, 337, 338, 339,
+ 340, 355, 341, 342, 344, 345, 348, 343, 349, 350,
+
+ 356, 351, 352, 353, 357, 354, 358, 359, 360, 361,
+ 362, 363, 364, 365, 366, 367, 368, 369, 370, 371,
+ 355, 372, 373, 374, 375, 376, 377, 378, 379, 356,
+ 380, 552, 383, 357, 384, 358, 359, 360, 361, 362,
+ 363, 364, 365, 366, 367, 368, 369, 370, 371, 381,
+ 372, 373, 374, 375, 376, 377, 378, 379, 385, 380,
+ 382, 383, 386, 384, 387, 388, 389, 390, 391, 392,
+ 393, 394, 395, 396, 397, 398, 399, 400, 381, 401,
+ 402, 403, 404, 405, 406, 407, 408, 385, 409, 382,
+ 410, 386, 411, 387, 388, 389, 390, 391, 392, 393,
+
+ 394, 395, 396, 397, 398, 399, 400, 412, 401, 402,
+ 403, 404, 405, 406, 407, 408, 413, 409, 414, 410,
+ 415, 411, 417, 416, 418, 419, 420, 421, 422, 423,
+ 424, 425, 426, 427, 428, 429, 412, 430, 431, 432,
+ 433, 434, 435, 436, 437, 413, 438, 414, 439, 415,
+ 440, 417, 416, 418, 419, 420, 421, 422, 423, 424,
+ 425, 426, 427, 428, 429, 441, 430, 431, 432, 433,
+ 434, 435, 436, 437, 442, 438, 443, 439, 444, 440,
+ 445, 446, 447, 448, 449, 450, 451, 452, 453, 454,
+ 455, 456, 457, 458, 441, 459, 460, 461, 462, 463,
+
+ 464, 465, 466, 442, 467, 443, 468, 444, 469, 445,
+ 446, 447, 448, 449, 450, 451, 452, 453, 454, 455,
+ 456, 457, 458, 470, 459, 460, 461, 462, 463, 464,
+ 465, 466, 471, 467, 473, 468, 474, 469, 475, 476,
+ 477, 478, 479, 480, 481, 482, 483, 484, 485, 486,
+ 487, 488, 470, 489, 490, 491, 492, 493, 494, 495,
+ 496, 471, 497, 473, 498, 474, 499, 475, 476, 477,
+ 478, 479, 480, 481, 482, 483, 484, 485, 486, 487,
+ 488, 500, 489, 490, 491, 492, 493, 494, 495, 496,
+ 501, 497, 502, 498, 503, 499, 504, 505, 506, 507,
+
+ 508, 509, 510, 511, 512, 513, 514, 515, 516, 517,
+ 500, 518, 519, 520, 521, 522, 523, 524, 525, 501,
+ 526, 502, 527, 503, 528, 504, 505, 506, 507, 508,
+ 509, 510, 511, 512, 513, 514, 515, 516, 517, 529,
+ 518, 519, 520, 521, 522, 523, 524, 525, 530, 526,
+ 531, 527, 532, 528, 533, 534, 535, 536, 537, 538,
+ 539, 540, 541, 542, 543, 544, 545, 546, 529, 547,
+ 548, 549, 550, 551, 552, 552, 552, 530, 552, 531,
+ 552, 532, 552, 533, 534, 535, 536, 537, 538, 539,
+ 540, 541, 542, 543, 544, 545, 546, 552, 547, 548,
+
+ 549, 550, 551, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 59, 59, 59, 59, 59, 59, 59,
+ 59, 59, 59, 60, 60, 60, 60, 60, 60, 60,
+ 60, 60, 60, 63, 63, 63, 63, 63, 63, 63,
+ 63, 63, 63, 66, 66, 66, 66, 66, 66, 66,
+ 66, 66, 66, 69, 69, 80, 80, 80, 552, 80,
+ 154, 154, 154, 154, 155, 155, 155, 552, 155, 155,
+ 155, 155, 155, 155, 157, 157, 157, 552, 157, 157,
+ 157, 157, 552, 157, 158, 158, 158, 158, 158, 158,
+ 158, 158, 158, 158, 168, 168, 552, 168, 168, 168,
+
+ 168, 168, 168, 168, 170, 552, 170, 170, 170, 170,
+ 170, 170, 170, 170, 263, 263, 347, 347, 13, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552
} ;
-static yyconst flex_int16_t yy_chk[1284] =
+static yyconst flex_int16_t yy_chk[1291] =
{ 0,
0, 0, 1, 2, 7, 8, 57, 57, 11, 7,
8, 11, 12, 0, 18, 12, 18, 25, 25, 27,
27, 58, 58, 70, 0, 70, 76, 76, 0, 76,
- 76, 152, 152, 159, 159, 160, 160, 171, 171, 172,
- 172, 259, 259, 0, 1, 2, 3, 3, 3, 3,
+ 76, 153, 153, 160, 160, 161, 161, 172, 172, 173,
+ 173, 261, 261, 0, 1, 2, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
@@ -789,137 +792,137 @@ static yyconst flex_int16_t yy_chk[1284] =
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 5, 5, 5,
5, 5, 5, 5, 5, 5, 36, 39, 40, 46,
- 9, 5, 5, 5, 10, 9, 260, 260, 427, 10,
- 26, 26, 26, 241, 30, 51, 54, 343, 343, 151,
+ 9, 5, 5, 5, 10, 9, 262, 262, 431, 10,
+ 26, 26, 26, 242, 30, 51, 54, 346, 346, 152,
68, 26, 30, 62, 30, 36, 39, 40, 46, 30,
5, 5, 5, 6, 6, 6, 6, 6, 6, 6,
6, 6, 9, 30, 51, 54, 10, 6, 6, 6,
26, 30, 31, 30, 37, 56, 31, 32, 30, 31,
- 81, 38, 48, 83, 32, 33, 38, 84, 37, 33,
+ 81, 38, 52, 52, 32, 33, 38, 83, 37, 33,
- 48, 32, 31, 33, 32, 32, 6, 6, 6, 33,
+ 84, 32, 31, 33, 32, 32, 6, 6, 6, 33,
29, 31, 33, 37, 34, 31, 32, 24, 31, 81,
- 38, 48, 83, 32, 33, 38, 84, 37, 33, 48,
+ 38, 52, 52, 32, 33, 38, 83, 37, 33, 84,
32, 31, 33, 32, 32, 34, 41, 34, 33, 42,
41, 33, 34, 43, 41, 42, 44, 34, 20, 43,
- 41, 52, 52, 45, 44, 82, 13, 45, 44, 0,
- 85, 44, 0, 82, 34, 41, 34, 0, 42, 41,
+ 41, 85, 82, 45, 44, 87, 13, 45, 44, 0,
+ 82, 44, 0, 0, 34, 41, 34, 0, 42, 41,
45, 34, 43, 41, 42, 44, 34, 35, 43, 41,
- 52, 52, 45, 44, 82, 35, 45, 44, 35, 85,
- 44, 35, 82, 0, 35, 49, 47, 35, 0, 45,
-
- 47, 87, 49, 49, 47, 89, 35, 92, 73, 73,
- 47, 50, 49, 50, 35, 49, 50, 35, 73, 0,
- 35, 0, 86, 35, 49, 47, 35, 74, 74, 47,
- 87, 49, 49, 47, 89, 86, 92, 74, 93, 47,
- 50, 49, 50, 96, 49, 50, 65, 73, 75, 75,
- 75, 86, 90, 97, 65, 65, 90, 98, 91, 75,
- 91, 65, 91, 94, 86, 65, 74, 93, 94, 0,
- 100, 101, 96, 65, 102, 0, 104, 65, 0, 65,
- 105, 90, 97, 65, 106, 90, 98, 91, 75, 91,
- 65, 91, 94, 107, 65, 95, 103, 94, 95, 100,
-
- 101, 108, 65, 102, 95, 104, 65, 103, 65, 105,
- 109, 95, 65, 106, 110, 111, 112, 115, 111, 113,
- 116, 117, 107, 118, 95, 103, 119, 95, 119, 113,
- 108, 120, 121, 95, 113, 113, 103, 122, 123, 109,
- 95, 124, 125, 110, 111, 112, 115, 111, 113, 116,
- 117, 127, 118, 128, 129, 119, 130, 119, 113, 131,
- 120, 121, 132, 113, 113, 133, 122, 123, 134, 135,
- 124, 125, 136, 134, 134, 137, 138, 139, 140, 141,
- 127, 142, 128, 129, 137, 130, 143, 144, 131, 145,
- 146, 132, 137, 147, 133, 148, 149, 134, 135, 147,
-
- 150, 136, 134, 134, 137, 138, 139, 140, 141, 175,
- 142, 177, 178, 137, 179, 143, 144, 180, 145, 146,
- 181, 137, 147, 182, 148, 149, 170, 170, 147, 150,
- 183, 184, 181, 185, 186, 187, 170, 188, 175, 189,
- 177, 178, 190, 179, 192, 193, 180, 194, 195, 181,
- 196, 194, 182, 197, 198, 199, 201, 202, 203, 183,
- 184, 181, 185, 186, 187, 170, 188, 204, 189, 205,
- 206, 190, 207, 192, 193, 208, 194, 195, 209, 196,
- 194, 210, 197, 198, 199, 201, 202, 203, 211, 212,
- 213, 214, 215, 216, 218, 217, 204, 220, 205, 206,
-
- 221, 207, 222, 223, 208, 217, 224, 209, 225, 226,
- 210, 228, 230, 231, 232, 233, 234, 211, 212, 213,
- 214, 215, 216, 218, 217, 235, 220, 236, 237, 221,
- 238, 222, 223, 239, 217, 224, 240, 225, 226, 242,
- 228, 230, 231, 232, 233, 234, 243, 245, 246, 247,
- 248, 249, 250, 251, 235, 252, 236, 237, 253, 238,
- 254, 255, 239, 256, 257, 240, 258, 256, 242, 262,
- 263, 264, 265, 266, 267, 243, 245, 246, 247, 248,
- 249, 250, 251, 268, 252, 270, 271, 253, 272, 254,
- 255, 273, 256, 257, 275, 258, 256, 276, 262, 263,
-
- 264, 265, 266, 267, 277, 278, 279, 280, 281, 282,
- 283, 284, 268, 287, 270, 271, 288, 272, 289, 290,
- 273, 291, 292, 275, 293, 296, 276, 298, 300, 301,
- 302, 309, 303, 277, 278, 279, 280, 281, 282, 283,
- 284, 311, 287, 303, 312, 288, 313, 289, 290, 314,
- 291, 292, 315, 293, 296, 316, 298, 300, 301, 302,
- 309, 303, 317, 318, 320, 322, 323, 324, 325, 326,
- 311, 327, 303, 312, 328, 313, 329, 331, 314, 333,
- 334, 315, 335, 336, 316, 337, 338, 340, 342, 346,
- 347, 317, 318, 320, 322, 323, 324, 325, 326, 348,
-
- 327, 349, 351, 328, 350, 329, 331, 350, 333, 334,
- 352, 335, 336, 354, 337, 338, 340, 342, 346, 347,
- 355, 356, 357, 358, 359, 360, 361, 362, 348, 363,
- 349, 351, 364, 350, 365, 366, 350, 367, 368, 352,
- 372, 374, 354, 377, 378, 379, 381, 386, 387, 355,
- 356, 357, 358, 359, 360, 361, 362, 388, 363, 390,
- 391, 364, 393, 365, 366, 394, 367, 368, 395, 372,
- 374, 396, 377, 378, 379, 381, 386, 387, 398, 399,
- 400, 401, 403, 404, 406, 407, 388, 408, 390, 391,
- 410, 393, 411, 412, 394, 413, 414, 395, 416, 418,
-
- 396, 419, 420, 421, 423, 424, 425, 398, 399, 400,
- 401, 403, 404, 406, 407, 429, 408, 430, 433, 410,
- 434, 411, 412, 436, 413, 414, 437, 416, 418, 438,
- 419, 420, 421, 423, 424, 425, 439, 441, 444, 445,
- 446, 450, 451, 452, 429, 454, 430, 433, 455, 434,
- 456, 457, 436, 458, 459, 437, 461, 464, 438, 465,
- 466, 467, 471, 472, 475, 439, 441, 444, 445, 446,
- 450, 451, 452, 476, 454, 477, 478, 455, 479, 456,
- 457, 481, 458, 459, 482, 461, 464, 483, 465, 466,
- 467, 471, 472, 475, 484, 485, 486, 487, 488, 490,
-
- 491, 493, 476, 495, 477, 478, 496, 479, 497, 499,
- 481, 500, 501, 482, 503, 504, 483, 505, 506, 508,
- 510, 511, 512, 484, 485, 486, 487, 488, 490, 491,
- 493, 513, 495, 514, 516, 496, 519, 497, 499, 520,
- 500, 501, 521, 503, 504, 524, 505, 506, 508, 510,
- 511, 512, 527, 528, 529, 532, 533, 535, 536, 537,
- 513, 538, 514, 516, 539, 519, 541, 0, 520, 0,
- 0, 521, 0, 0, 524, 0, 0, 0, 0, 0,
- 0, 527, 528, 529, 532, 533, 535, 536, 537, 0,
- 538, 0, 0, 539, 0, 541, 545, 545, 545, 545,
-
- 545, 545, 545, 545, 545, 545, 546, 546, 546, 546,
- 546, 546, 546, 546, 546, 546, 547, 547, 547, 547,
- 547, 547, 547, 547, 547, 547, 548, 548, 548, 548,
- 548, 548, 548, 548, 548, 548, 549, 549, 549, 549,
- 549, 549, 549, 549, 549, 549, 550, 550, 551, 551,
- 551, 0, 551, 552, 552, 552, 552, 553, 553, 553,
- 0, 553, 553, 553, 553, 553, 553, 554, 554, 554,
- 0, 554, 554, 554, 554, 0, 554, 555, 555, 555,
- 555, 555, 555, 555, 555, 555, 555, 556, 556, 0,
- 556, 556, 556, 556, 556, 556, 556, 557, 0, 557,
-
- 557, 557, 557, 557, 557, 557, 557, 558, 558, 559,
- 559, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544
+ 85, 82, 45, 44, 87, 35, 45, 44, 35, 82,
+ 44, 35, 86, 48, 35, 49, 47, 35, 89, 45,
+
+ 47, 48, 49, 49, 47, 86, 35, 92, 93, 48,
+ 47, 50, 49, 50, 35, 49, 50, 35, 73, 73,
+ 35, 86, 48, 35, 49, 47, 35, 89, 73, 47,
+ 48, 49, 49, 47, 86, 0, 92, 93, 48, 47,
+ 50, 49, 50, 90, 49, 50, 65, 90, 74, 74,
+ 0, 75, 75, 75, 65, 65, 96, 73, 74, 97,
+ 98, 65, 75, 100, 91, 65, 91, 101, 91, 94,
+ 102, 103, 90, 65, 94, 0, 90, 65, 104, 65,
+ 105, 0, 103, 65, 106, 96, 107, 74, 97, 98,
+ 65, 75, 100, 91, 65, 91, 101, 91, 94, 102,
+
+ 103, 95, 65, 94, 95, 108, 65, 104, 65, 105,
+ 95, 103, 65, 106, 109, 107, 110, 95, 111, 112,
+ 115, 111, 113, 116, 117, 118, 119, 120, 119, 121,
+ 95, 122, 113, 95, 108, 123, 124, 113, 113, 95,
+ 125, 127, 128, 109, 129, 110, 95, 111, 112, 115,
+ 111, 113, 116, 117, 118, 119, 120, 119, 121, 130,
+ 122, 113, 131, 132, 123, 124, 113, 113, 133, 125,
+ 127, 128, 134, 129, 135, 136, 137, 134, 134, 138,
+ 139, 140, 141, 142, 143, 137, 144, 145, 130, 146,
+ 147, 131, 132, 137, 148, 149, 150, 133, 151, 176,
+
+ 148, 134, 178, 135, 136, 137, 134, 134, 138, 139,
+ 140, 141, 142, 143, 137, 144, 145, 179, 146, 147,
+ 171, 171, 137, 148, 149, 150, 180, 151, 176, 148,
+ 171, 178, 181, 182, 183, 184, 185, 186, 187, 188,
+ 189, 190, 191, 193, 194, 182, 179, 195, 196, 197,
+ 198, 195, 199, 200, 202, 180, 203, 204, 205, 171,
+ 206, 181, 182, 183, 184, 185, 186, 187, 188, 189,
+ 190, 191, 193, 194, 182, 207, 195, 196, 197, 198,
+ 195, 199, 200, 202, 208, 203, 204, 205, 209, 206,
+ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219,
+
+ 221, 222, 223, 224, 207, 225, 226, 227, 218, 229,
+ 231, 232, 233, 208, 234, 235, 236, 209, 237, 210,
+ 211, 212, 213, 214, 215, 216, 217, 218, 219, 221,
+ 222, 223, 224, 238, 225, 226, 227, 218, 229, 231,
+ 232, 233, 239, 234, 235, 236, 240, 237, 241, 243,
+ 244, 246, 247, 248, 249, 250, 251, 252, 253, 254,
+ 255, 256, 238, 257, 258, 259, 260, 264, 258, 265,
+ 266, 239, 267, 268, 269, 240, 270, 241, 243, 244,
+ 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
+ 256, 272, 257, 258, 259, 260, 264, 258, 265, 266,
+
+ 273, 267, 268, 269, 274, 270, 275, 277, 278, 279,
+ 280, 281, 282, 283, 284, 285, 286, 289, 290, 291,
+ 272, 292, 293, 294, 295, 298, 300, 302, 303, 273,
+ 304, 0, 311, 274, 313, 275, 277, 278, 279, 280,
+ 281, 282, 283, 284, 285, 286, 289, 290, 291, 305,
+ 292, 293, 294, 295, 298, 300, 302, 303, 314, 304,
+ 305, 311, 315, 313, 316, 317, 318, 319, 320, 322,
+ 324, 325, 326, 327, 328, 329, 330, 331, 305, 332,
+ 334, 336, 337, 338, 339, 340, 341, 314, 343, 305,
+ 345, 315, 349, 316, 317, 318, 319, 320, 322, 324,
+
+ 325, 326, 327, 328, 329, 330, 331, 350, 332, 334,
+ 336, 337, 338, 339, 340, 341, 351, 343, 352, 345,
+ 353, 349, 354, 353, 355, 357, 358, 359, 360, 361,
+ 362, 363, 364, 365, 366, 367, 350, 368, 369, 370,
+ 371, 375, 377, 380, 381, 351, 382, 352, 384, 353,
+ 389, 354, 353, 355, 357, 358, 359, 360, 361, 362,
+ 363, 364, 365, 366, 367, 390, 368, 369, 370, 371,
+ 375, 377, 380, 381, 391, 382, 393, 384, 394, 389,
+ 396, 397, 398, 399, 400, 402, 403, 404, 405, 407,
+ 408, 410, 411, 412, 390, 414, 415, 416, 417, 418,
+
+ 420, 422, 423, 391, 424, 393, 425, 394, 427, 396,
+ 397, 398, 399, 400, 402, 403, 404, 405, 407, 408,
+ 410, 411, 412, 428, 414, 415, 416, 417, 418, 420,
+ 422, 423, 429, 424, 433, 425, 434, 427, 437, 438,
+ 440, 441, 442, 443, 445, 448, 449, 450, 451, 455,
+ 456, 457, 428, 459, 460, 461, 462, 463, 464, 466,
+ 469, 429, 470, 433, 471, 434, 472, 437, 438, 440,
+ 441, 442, 443, 445, 448, 449, 450, 451, 455, 456,
+ 457, 476, 459, 460, 461, 462, 463, 464, 466, 469,
+ 477, 470, 480, 471, 481, 472, 482, 483, 484, 485,
+
+ 487, 488, 489, 490, 491, 492, 493, 494, 496, 497,
+ 476, 499, 501, 502, 503, 505, 506, 507, 508, 477,
+ 510, 480, 511, 481, 512, 482, 483, 484, 485, 487,
+ 488, 489, 490, 491, 492, 493, 494, 496, 497, 513,
+ 499, 501, 502, 503, 505, 506, 507, 508, 515, 510,
+ 517, 511, 518, 512, 519, 520, 521, 524, 527, 528,
+ 529, 532, 535, 536, 537, 540, 541, 543, 513, 544,
+ 545, 546, 547, 549, 0, 0, 0, 515, 0, 517,
+ 0, 518, 0, 519, 520, 521, 524, 527, 528, 529,
+ 532, 535, 536, 537, 540, 541, 543, 0, 544, 545,
+
+ 546, 547, 549, 553, 553, 553, 553, 553, 553, 553,
+ 553, 553, 553, 554, 554, 554, 554, 554, 554, 554,
+ 554, 554, 554, 555, 555, 555, 555, 555, 555, 555,
+ 555, 555, 555, 556, 556, 556, 556, 556, 556, 556,
+ 556, 556, 556, 557, 557, 557, 557, 557, 557, 557,
+ 557, 557, 557, 558, 558, 559, 559, 559, 0, 559,
+ 560, 560, 560, 560, 561, 561, 561, 0, 561, 561,
+ 561, 561, 561, 561, 562, 562, 562, 0, 562, 562,
+ 562, 562, 0, 562, 563, 563, 563, 563, 563, 563,
+ 563, 563, 563, 563, 564, 564, 0, 564, 564, 564,
+
+ 564, 564, 564, 564, 565, 0, 565, 565, 565, 565,
+ 565, 565, 565, 565, 566, 566, 567, 567, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552
} ;
/* Table of booleans, true if rule could match eol. */
-static yyconst flex_int32_t yy_rule_can_match_eol[151] =
+static yyconst flex_int32_t yy_rule_can_match_eol[153] =
{ 0,
0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -927,8 +930,8 @@ static yyconst flex_int32_t yy_rule_can_match_eol[151] =
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,
- 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, };
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,
+ 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, };
/* The intent behind this definition is that it'll catch
* any uses of REJECT which flex missed.
@@ -1045,7 +1048,7 @@ class UnaryOperation;
-#line 1049 "SqlLexer_gen.cpp"
+#line 1052 "SqlLexer_gen.cpp"
#define INITIAL 0
#define CONDITION_SQL 1
@@ -1336,7 +1339,7 @@ YY_DECL
#line 128 "../SqlLexer.lpp"
-#line 1340 "SqlLexer_gen.cpp"
+#line 1343 "SqlLexer_gen.cpp"
while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */
{
@@ -1363,13 +1366,13 @@ yy_match:
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 545 )
+ if ( yy_current_state >= 553 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
++yy_cp;
}
- while ( yy_current_state != 544 );
+ while ( yy_current_state != 552 );
yy_cp = yyg->yy_last_accepting_cpos;
yy_current_state = yyg->yy_last_accepting_state;
@@ -1678,62 +1681,62 @@ return TOKEN_FLOAT;
case 50:
YY_RULE_SETUP
#line 212 "../SqlLexer.lpp"
-return TOKEN_FOREIGN;
+return TOKEN_FOR;
YY_BREAK
case 51:
YY_RULE_SETUP
#line 213 "../SqlLexer.lpp"
-return TOKEN_FROM;
+return TOKEN_FOREIGN;
YY_BREAK
case 52:
YY_RULE_SETUP
#line 214 "../SqlLexer.lpp"
-return TOKEN_FULL;
+return TOKEN_FROM;
YY_BREAK
case 53:
YY_RULE_SETUP
#line 215 "../SqlLexer.lpp"
-return TOKEN_GROUP;
+return TOKEN_FULL;
YY_BREAK
case 54:
YY_RULE_SETUP
#line 216 "../SqlLexer.lpp"
-return TOKEN_HASH;
+return TOKEN_GROUP;
YY_BREAK
case 55:
YY_RULE_SETUP
#line 217 "../SqlLexer.lpp"
-return TOKEN_HAVING;
+return TOKEN_HASH;
YY_BREAK
case 56:
YY_RULE_SETUP
#line 218 "../SqlLexer.lpp"
-return TOKEN_HOUR;
+return TOKEN_HAVING;
YY_BREAK
case 57:
YY_RULE_SETUP
#line 219 "../SqlLexer.lpp"
-return TOKEN_IN;
+return TOKEN_HOUR;
YY_BREAK
case 58:
YY_RULE_SETUP
#line 220 "../SqlLexer.lpp"
-return TOKEN_INDEX;
+return TOKEN_IN;
YY_BREAK
case 59:
YY_RULE_SETUP
#line 221 "../SqlLexer.lpp"
-return TOKEN_INNER;
+return TOKEN_INDEX;
YY_BREAK
case 60:
YY_RULE_SETUP
#line 222 "../SqlLexer.lpp"
-return TOKEN_INSERT;
+return TOKEN_INNER;
YY_BREAK
case 61:
YY_RULE_SETUP
#line 223 "../SqlLexer.lpp"
-return TOKEN_INTEGER;
+return TOKEN_INSERT;
YY_BREAK
case 62:
YY_RULE_SETUP
@@ -1743,331 +1746,341 @@ return TOKEN_INTEGER;
case 63:
YY_RULE_SETUP
#line 225 "../SqlLexer.lpp"
-return TOKEN_INTERVAL;
+return TOKEN_INTEGER;
YY_BREAK
case 64:
YY_RULE_SETUP
#line 226 "../SqlLexer.lpp"
-return TOKEN_INTO;
+return TOKEN_INTERVAL;
YY_BREAK
case 65:
YY_RULE_SETUP
#line 227 "../SqlLexer.lpp"
-return TOKEN_IS;
+return TOKEN_INTO;
YY_BREAK
case 66:
YY_RULE_SETUP
#line 228 "../SqlLexer.lpp"
-return TOKEN_JOIN;
+return TOKEN_IS;
YY_BREAK
case 67:
YY_RULE_SETUP
#line 229 "../SqlLexer.lpp"
-return TOKEN_KEY;
+return TOKEN_JOIN;
YY_BREAK
case 68:
YY_RULE_SETUP
#line 230 "../SqlLexer.lpp"
-return TOKEN_LAST;
+return TOKEN_KEY;
YY_BREAK
case 69:
YY_RULE_SETUP
#line 231 "../SqlLexer.lpp"
-return TOKEN_LEFT;
+return TOKEN_LAST;
YY_BREAK
case 70:
YY_RULE_SETUP
#line 232 "../SqlLexer.lpp"
-return TOKEN_LIKE;
+return TOKEN_LEFT;
YY_BREAK
case 71:
YY_RULE_SETUP
#line 233 "../SqlLexer.lpp"
-return TOKEN_LIMIT;
+return TOKEN_LIKE;
YY_BREAK
case 72:
YY_RULE_SETUP
#line 234 "../SqlLexer.lpp"
-return TOKEN_LONG;
+return TOKEN_LIMIT;
YY_BREAK
case 73:
YY_RULE_SETUP
#line 235 "../SqlLexer.lpp"
-return TOKEN_MINUTE;
+return TOKEN_LONG;
YY_BREAK
case 74:
YY_RULE_SETUP
#line 236 "../SqlLexer.lpp"
-return TOKEN_MONTH;
+return TOKEN_MINUTE;
YY_BREAK
case 75:
YY_RULE_SETUP
#line 237 "../SqlLexer.lpp"
-return TOKEN_NOT;
+return TOKEN_MONTH;
YY_BREAK
case 76:
YY_RULE_SETUP
#line 238 "../SqlLexer.lpp"
-return TOKEN_NULL;
+return TOKEN_NOT;
YY_BREAK
case 77:
YY_RULE_SETUP
#line 239 "../SqlLexer.lpp"
-return TOKEN_NULLS;
+return TOKEN_NULL;
YY_BREAK
case 78:
YY_RULE_SETUP
#line 240 "../SqlLexer.lpp"
-return TOKEN_OFF;
+return TOKEN_NULLS;
YY_BREAK
case 79:
YY_RULE_SETUP
#line 241 "../SqlLexer.lpp"
-return TOKEN_ON;
+return TOKEN_OFF;
YY_BREAK
case 80:
YY_RULE_SETUP
#line 242 "../SqlLexer.lpp"
-return TOKEN_OR;
+return TOKEN_ON;
YY_BREAK
case 81:
YY_RULE_SETUP
#line 243 "../SqlLexer.lpp"
-return TOKEN_ORDER;
+return TOKEN_OR;
YY_BREAK
case 82:
YY_RULE_SETUP
#line 244 "../SqlLexer.lpp"
-return TOKEN_OUTER;
+return TOKEN_ORDER;
YY_BREAK
case 83:
YY_RULE_SETUP
#line 245 "../SqlLexer.lpp"
-return TOKEN_PARTITION;
+return TOKEN_OUTER;
YY_BREAK
case 84:
YY_RULE_SETUP
#line 246 "../SqlLexer.lpp"
-return TOKEN_PARTITIONS;
+return TOKEN_PARTITION;
YY_BREAK
case 85:
YY_RULE_SETUP
#line 247 "../SqlLexer.lpp"
-return TOKEN_PERCENT;
+return TOKEN_PARTITIONS;
YY_BREAK
case 86:
YY_RULE_SETUP
#line 248 "../SqlLexer.lpp"
-return TOKEN_PRIMARY;
+return TOKEN_PERCENT;
YY_BREAK
case 87:
YY_RULE_SETUP
#line 249 "../SqlLexer.lpp"
-return TOKEN_QUIT;
+return TOKEN_PRIMARY;
YY_BREAK
case 88:
YY_RULE_SETUP
#line 250 "../SqlLexer.lpp"
-return TOKEN_RANGE;
+return TOKEN_QUIT;
YY_BREAK
case 89:
YY_RULE_SETUP
#line 251 "../SqlLexer.lpp"
-return TOKEN_REAL;
+return TOKEN_RANGE;
YY_BREAK
case 90:
YY_RULE_SETUP
#line 252 "../SqlLexer.lpp"
-return TOKEN_REFERENCES;
+return TOKEN_REAL;
YY_BREAK
case 91:
YY_RULE_SETUP
#line 253 "../SqlLexer.lpp"
-return TOKEN_REGEXP;
+return TOKEN_REFERENCES;
YY_BREAK
case 92:
YY_RULE_SETUP
#line 254 "../SqlLexer.lpp"
-return TOKEN_RIGHT;
+return TOKEN_REGEXP;
YY_BREAK
case 93:
YY_RULE_SETUP
#line 255 "../SqlLexer.lpp"
-return TOKEN_ROW_DELIMITER;
+return TOKEN_RIGHT;
YY_BREAK
case 94:
YY_RULE_SETUP
#line 256 "../SqlLexer.lpp"
-return TOKEN_SECOND;
+return TOKEN_ROW_DELIMITER;
YY_BREAK
case 95:
YY_RULE_SETUP
#line 257 "../SqlLexer.lpp"
-return TOKEN_SELECT;
+return TOKEN_SECOND;
YY_BREAK
case 96:
YY_RULE_SETUP
#line 258 "../SqlLexer.lpp"
-return TOKEN_SET;
+return TOKEN_SELECT;
YY_BREAK
case 97:
YY_RULE_SETUP
#line 259 "../SqlLexer.lpp"
-return TOKEN_SMA;
+return TOKEN_SET;
YY_BREAK
case 98:
YY_RULE_SETUP
#line 260 "../SqlLexer.lpp"
-return TOKEN_SMALLINT;
+return TOKEN_SMA;
YY_BREAK
case 99:
YY_RULE_SETUP
#line 261 "../SqlLexer.lpp"
-return TOKEN_TABLE;
+return TOKEN_SMALLINT;
YY_BREAK
case 100:
YY_RULE_SETUP
#line 262 "../SqlLexer.lpp"
-return TOKEN_THEN;
+return TOKEN_SUBSTRING;
YY_BREAK
case 101:
YY_RULE_SETUP
#line 263 "../SqlLexer.lpp"
-return TOKEN_TIME;
+return TOKEN_TABLE;
YY_BREAK
case 102:
YY_RULE_SETUP
#line 264 "../SqlLexer.lpp"
-return TOKEN_TIMESTAMP;
+return TOKEN_THEN;
YY_BREAK
case 103:
YY_RULE_SETUP
#line 265 "../SqlLexer.lpp"
-return TOKEN_TRUE;
+return TOKEN_TIME;
YY_BREAK
case 104:
YY_RULE_SETUP
#line 266 "../SqlLexer.lpp"
-return TOKEN_TUPLESAMPLE;
+return TOKEN_TIMESTAMP;
YY_BREAK
case 105:
YY_RULE_SETUP
#line 267 "../SqlLexer.lpp"
-return TOKEN_UNIQUE;
+return TOKEN_TRUE;
YY_BREAK
case 106:
YY_RULE_SETUP
#line 268 "../SqlLexer.lpp"
-return TOKEN_UPDATE;
+return TOKEN_TUPLESAMPLE;
YY_BREAK
case 107:
YY_RULE_SETUP
#line 269 "../SqlLexer.lpp"
-return TOKEN_USING;
+return TOKEN_UNIQUE;
YY_BREAK
case 108:
YY_RULE_SETUP
#line 270 "../SqlLexer.lpp"
-return TOKEN_VALUES;
+return TOKEN_UPDATE;
YY_BREAK
case 109:
YY_RULE_SETUP
#line 271 "../SqlLexer.lpp"
-return TOKEN_VARCHAR;
+return TOKEN_USING;
YY_BREAK
case 110:
YY_RULE_SETUP
#line 272 "../SqlLexer.lpp"
-return TOKEN_WHEN;
+return TOKEN_VALUES;
YY_BREAK
case 111:
YY_RULE_SETUP
#line 273 "../SqlLexer.lpp"
-return TOKEN_WHERE;
+return TOKEN_VARCHAR;
YY_BREAK
case 112:
YY_RULE_SETUP
#line 274 "../SqlLexer.lpp"
-return TOKEN_WITH;
+return TOKEN_WHEN;
YY_BREAK
case 113:
YY_RULE_SETUP
#line 275 "../SqlLexer.lpp"
-return TOKEN_YEAR;
+return TOKEN_WHERE;
YY_BREAK
case 114:
YY_RULE_SETUP
#line 276 "../SqlLexer.lpp"
-return TOKEN_YEARMONTH;
+return TOKEN_WITH;
YY_BREAK
case 115:
YY_RULE_SETUP
-#line 278 "../SqlLexer.lpp"
-return TOKEN_EQ;
+#line 277 "../SqlLexer.lpp"
+return TOKEN_YEAR;
YY_BREAK
case 116:
YY_RULE_SETUP
-#line 279 "../SqlLexer.lpp"
-return TOKEN_NEQ;
+#line 278 "../SqlLexer.lpp"
+return TOKEN_YEARMONTH;
YY_BREAK
case 117:
YY_RULE_SETUP
#line 280 "../SqlLexer.lpp"
-return TOKEN_NEQ;
+return TOKEN_EQ;
YY_BREAK
case 118:
YY_RULE_SETUP
#line 281 "../SqlLexer.lpp"
-return TOKEN_LT;
+return TOKEN_NEQ;
YY_BREAK
case 119:
YY_RULE_SETUP
#line 282 "../SqlLexer.lpp"
-return TOKEN_GT;
+return TOKEN_NEQ;
YY_BREAK
case 120:
YY_RULE_SETUP
#line 283 "../SqlLexer.lpp"
-return TOKEN_LEQ;
+return TOKEN_LT;
YY_BREAK
case 121:
YY_RULE_SETUP
#line 284 "../SqlLexer.lpp"
-return TOKEN_GEQ;
+return TOKEN_GT;
YY_BREAK
case 122:
YY_RULE_SETUP
+#line 285 "../SqlLexer.lpp"
+return TOKEN_LEQ;
+ YY_BREAK
+case 123:
+YY_RULE_SETUP
#line 286 "../SqlLexer.lpp"
+return TOKEN_GEQ;
+ YY_BREAK
+case 124:
+YY_RULE_SETUP
+#line 288 "../SqlLexer.lpp"
return yytext[0];
YY_BREAK
-case 123:
+case 125:
YY_RULE_SETUP
-#line 287 "../SqlLexer.lpp"
+#line 289 "../SqlLexer.lpp"
return yytext[0];
YY_BREAK
/**
* Quoted strings. Prefacing a string with an 'e' or 'E' causes escape
* sequences to be processed (as in PostgreSQL).
**/
-case 124:
+case 126:
YY_RULE_SETUP
-#line 293 "../SqlLexer.lpp"
+#line 295 "../SqlLexer.lpp"
{
yylval->string_value_ = new quickstep::ParseString(yylloc->first_line, yylloc->first_column);
BEGIN(CONDITION_STRING_SINGLE_QUOTED_ESCAPED);
}
YY_BREAK
-case 125:
+case 127:
YY_RULE_SETUP
-#line 298 "../SqlLexer.lpp"
+#line 300 "../SqlLexer.lpp"
{
yylval->string_value_ = new quickstep::ParseString(yylloc->first_line, yylloc->first_column);
BEGIN(CONDITION_STRING_SINGLE_QUOTED);
}
YY_BREAK
-case 126:
+case 128:
YY_RULE_SETUP
-#line 303 "../SqlLexer.lpp"
+#line 305 "../SqlLexer.lpp"
{
yylval->string_value_ = new quickstep::ParseString(yylloc->first_line, yylloc->first_column);
BEGIN(CONDITION_STRING_DOUBLE_QUOTED);
@@ -2079,7 +2092,7 @@ YY_RULE_SETUP
case YY_STATE_EOF(CONDITION_STRING_SINGLE_QUOTED):
case YY_STATE_EOF(CONDITION_STRING_SINGLE_QUOTED_ESCAPED):
case YY_STATE_EOF(CONDITION_STRING_DOUBLE_QUOTED):
-#line 312 "../SqlLexer.lpp"
+#line 314 "../SqlLexer.lpp"
{
delete yylval->string_value_;
BEGIN(INITIAL);
@@ -2090,9 +2103,9 @@ case YY_STATE_EOF(CONDITION_STRING_DOUBLE_QUOTED):
/* Process escape sequences. */
-case 127:
+case 129:
YY_RULE_SETUP
-#line 322 "../SqlLexer.lpp"
+#line 324 "../SqlLexer.lpp"
{
/* Octal code */
unsigned int code;
@@ -2106,9 +2119,9 @@ YY_RULE_SETUP
yylval->string_value_->push_back(code);
}
YY_BREAK
-case 128:
+case 130:
YY_RULE_SETUP
-#line 334 "../SqlLexer.lpp"
+#line 336 "../SqlLexer.lpp"
{
/* Hexadecimal code */
unsigned int code;
@@ -2116,9 +2129,9 @@ YY_RULE_SETUP
yylval->string_value_->push_back(code);
}
YY_BREAK
-case 129:
+case 131:
YY_RULE_SETUP
-#line 340 "../SqlLexer.lpp"
+#line 342 "../SqlLexer.lpp"
{
/* A numeric escape sequence that isn't correctly specified. */
delete yylval->string_value_;
@@ -2127,58 +2140,58 @@ YY_RULE_SETUP
return TOKEN_LEX_ERROR;
}
YY_BREAK
-case 130:
+case 132:
YY_RULE_SETUP
-#line 347 "../SqlLexer.lpp"
+#line 349 "../SqlLexer.lpp"
{
/* Backspace */
yylval->string_value_->push_back('\b');
}
YY_BREAK
-case 131:
+case 133:
YY_RULE_SETUP
-#line 351 "../SqlLexer.lpp"
+#line 353 "../SqlLexer.lpp"
{
/* Form-feed */
yylval->string_value_->push_back('\f');
}
YY_BREAK
-case 132:
+case 134:
YY_RULE_SETUP
-#line 355 "../SqlLexer.lpp"
+#line 357 "../SqlLexer.lpp"
{
/* Newline */
yylval->string_value_->push_back('\n');
}
YY_BREAK
-case 133:
+case 135:
YY_RULE_SETUP
-#line 359 "../SqlLexer.lpp"
+#line 361 "../SqlLexer.lpp"
{
/* Carriage-return */
yylval->string_value_->push_back('\r');
}
YY_BREAK
-case 134:
+case 136:
YY_RULE_SETUP
-#line 363 "../SqlLexer.lpp"
+#line 365 "../SqlLexer.lpp"
{
/* Horizontal Tab */
yylval->string_value_->push_back('\t');
}
YY_BREAK
-case 135:
-/* rule 135 can match eol */
+case 137:
+/* rule 137 can match eol */
YY_RULE_SETUP
-#line 367 "../SqlLexer.lpp"
+#line 369 "../SqlLexer.lpp"
{
/* Any other character (including actual newline or carriage return) */
yylval->string_value_->push_back(yytext[1]);
}
YY_BREAK
-case 136:
+case 138:
YY_RULE_SETUP
-#line 371 "../SqlLexer.lpp"
+#line 373 "../SqlLexer.lpp"
{
/* This should only be encountered right before an EOF. */
delete yylval->string_value_;
@@ -2189,17 +2202,17 @@ YY_RULE_SETUP
YY_BREAK
-case 137:
+case 139:
YY_RULE_SETUP
-#line 381 "../SqlLexer.lpp"
+#line 383 "../SqlLexer.lpp"
{
/* Two quotes in a row become a single quote (this is specified by the SQL standard). */
yylval->string_value_->push_back('\'');
}
YY_BREAK
-case 138:
+case 140:
YY_RULE_SETUP
-#line 385 "../SqlLexer.lpp"
+#line 387 "../SqlLexer.lpp"
{
/* End string */
BEGIN(CONDITION_SQL);
@@ -2208,17 +2221,17 @@ YY_RULE_SETUP
YY_BREAK
-case 139:
+case 141:
YY_RULE_SETUP
-#line 393 "../SqlLexer.lpp"
+#line 395 "../SqlLexer.lpp"
{
/* Two quotes in a row become a single quote (this is specified by the SQL standard). */
yylval->string_value_->push_back('"');
}
YY_BREAK
-case 140:
+case 142:
YY_RULE_SETUP
-#line 397 "../SqlLexer.lpp"
+#line 399 "../SqlLexer.lpp"
{
/* End string */
BEGIN(CONDITION_SQL);
@@ -2226,94 +2239,94 @@ YY_RULE_SETUP
}
YY_BREAK
-case 141:
-/* rule 141 can match eol */
+case 143:
+/* rule 143 can match eol */
YY_RULE_SETUP
-#line 404 "../SqlLexer.lpp"
+#line 406 "../SqlLexer.lpp"
{
/* Scan up to a quote. */
yylval->string_value_->append(yytext, yyleng);
}
YY_BREAK
-case 142:
-/* rule 142 can match eol */
+case 144:
+/* rule 144 can match eol */
YY_RULE_SETUP
-#line 409 "../SqlLexer.lpp"
+#line 411 "../SqlLexer.lpp"
{
/* Scan up to a quote or escape sequence. */
yylval->string_value_->append(yytext, yyleng);
}
YY_BREAK
-case 143:
-/* rule 143 can match eol */
+case 145:
+/* rule 145 can match eol */
YY_RULE_SETUP
-#line 414 "../SqlLexer.lpp"
+#line 416 "../SqlLexer.lpp"
{
/* Scan up to a quote. */
yylval->string_value_->append(yytext, yyleng);
}
YY_BREAK
-case 144:
+case 146:
YY_RULE_SETUP
-#line 420 "../SqlLexer.lpp"
+#line 422 "../SqlLexer.lpp"
{
yylval->string_value_ = new quickstep::ParseString(
yylloc->first_line, yylloc->first_column, std::string(yytext, yyleng));
return TOKEN_NAME;
}
YY_BREAK
-case 145:
+case 147:
YY_RULE_SETUP
-#line 426 "../SqlLexer.lpp"
+#line 428 "../SqlLexer.lpp"
{
yylval->numeric_literal_value_ = new quickstep::NumericParseLiteralValue(
yylloc->first_line, yylloc->first_column, yytext);
return TOKEN_UNSIGNED_NUMVAL;
}
YY_BREAK
-case 146:
+case 148:
YY_RULE_SETUP
-#line 432 "../SqlLexer.lpp"
+#line 434 "../SqlLexer.lpp"
/* comment */
YY_BREAK
-case 147:
-/* rule 147 can match eol */
+case 149:
+/* rule 149 can match eol */
YY_RULE_SETUP
-#line 434 "../SqlLexer.lpp"
+#line 436 "../SqlLexer.lpp"
{ yycolumn = 0; }
YY_BREAK
-case 148:
+case 150:
YY_RULE_SETUP
-#line 436 "../SqlLexer.lpp"
+#line 438 "../SqlLexer.lpp"
; /* ignore white space */
YY_BREAK
/* CONDITION_SQL */
case YY_STATE_EOF(INITIAL):
case YY_STATE_EOF(CONDITION_COMMAND):
case YY_STATE_EOF(CONDITION_SQL):
-#line 440 "../SqlLexer.lpp"
+#line 442 "../SqlLexer.lpp"
{
/* All conditions except for mutli-state string extracting conditions. */
BEGIN(INITIAL);
return TOKEN_EOF;
}
YY_BREAK
-case 149:
+case 151:
YY_RULE_SETUP
-#line 446 "../SqlLexer.lpp"
+#line 448 "../SqlLexer.lpp"
{
BEGIN(INITIAL);
quickstep_yyerror(NULL, yyscanner, NULL, "illegal character");
return TOKEN_LEX_ERROR;
}
YY_BREAK
-case 150:
+case 152:
YY_RULE_SETUP
-#line 452 "../SqlLexer.lpp"
+#line 454 "../SqlLexer.lpp"
YY_FATAL_ERROR( "flex scanner jammed" );
YY_BREAK
-#line 2317 "SqlLexer_gen.cpp"
+#line 2330 "SqlLexer_gen.cpp"
case YY_END_OF_BUFFER:
{
@@ -2607,7 +2620,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 545 )
+ if ( yy_current_state >= 553 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
@@ -2636,11 +2649,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 545 )
+ if ( yy_current_state >= 553 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
- yy_is_jam = (yy_current_state == 544);
+ yy_is_jam = (yy_current_state == 552);
(void)yyg;
return yy_is_jam ? 0 : yy_current_state;
@@ -3474,7 +3487,7 @@ void quickstep_yyfree (void * ptr , yyscan_t yyscanner)
#define YYTABLES_NAME "yytables"
-#line 452 "../SqlLexer.lpp"
+#line 454 "../SqlLexer.lpp"
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1a685be4/parser/preprocessed/SqlLexer_gen.hpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlLexer_gen.hpp b/parser/preprocessed/SqlLexer_gen.hpp
index d629f04..c14559b 100644
--- a/parser/preprocessed/SqlLexer_gen.hpp
+++ b/parser/preprocessed/SqlLexer_gen.hpp
@@ -360,7 +360,7 @@ extern int quickstep_yylex \
#undef YY_DECL
#endif
-#line 452 "../SqlLexer.lpp"
+#line 454 "../SqlLexer.lpp"
#line 367 "SqlLexer_gen.hpp"
[35/48] incubator-quickstep git commit: Revert "Explicitly specify
where tcmalloc comes from" (#237)
Posted by hb...@apache.org.
Revert "Explicitly specify where tcmalloc comes from" (#237)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/44eff0d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/44eff0d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/44eff0d0
Branch: refs/heads/query-manager-used-in-foreman
Commit: 44eff0d0561e330141e1d88fb2e41509c33d4035
Parents: 4403e6c
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Mon May 23 22:56:22 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:45 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/44eff0d0/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2ab0f57..dc51ca6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -490,7 +490,9 @@ if(USE_TCMALLOC)
CXXFLAGS=${THIRD_PARTY_CXX_FLAGS}
BUILD_COMMAND make
BUILD_IN_SOURCE 0
- BUILD_BYPRODUCTS <INSTALL_DIR>/lib/libtcmalloc_minimal.a
+ # Uncomment the next line to change the path of the build by products
+ # as some generators, e.g. Ninja, may need it to build properly
+ # BUILD_BYPRODUCTS <INSTALL_DIR>/lib/libtcmalloc_minimal.a
)
# Static libtcmalloc_minimal.a
add_library(libtcmalloc_minimal STATIC IMPORTED)
[20/48] incubator-quickstep git commit: Added a method for double
parsing with significant number argument. (#216)
Posted by hb...@apache.org.
Added a method for double parsing with significant number argument. (#216)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/19aeb6c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/19aeb6c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/19aeb6c5
Branch: refs/heads/query-manager-used-in-foreman
Commit: 19aeb6c5d5985312978a0dfdca24777bd8d552cd
Parents: d75ae42
Author: Hakan Memisoglu <ha...@gmail.com>
Authored: Sun May 8 08:22:17 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:42 2016 -0700
----------------------------------------------------------------------
utility/StringUtil.cpp | 14 ++++++++++++++
utility/StringUtil.hpp | 18 ++++++++++++++++++
2 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/19aeb6c5/utility/StringUtil.cpp
----------------------------------------------------------------------
diff --git a/utility/StringUtil.cpp b/utility/StringUtil.cpp
index 77b0a3a..b38e8f4 100644
--- a/utility/StringUtil.cpp
+++ b/utility/StringUtil.cpp
@@ -22,10 +22,13 @@
#include <algorithm>
#include <cctype>
#include <cinttypes>
+#include <cmath>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
+#include <iomanip>
+#include <sstream>
#include <string>
#include <vector>
@@ -128,4 +131,15 @@ string ToZeroPaddedString(std::uint64_t val, int pad_width) {
return result;
}
+std::string
+DoubleToStringWithSignificantDigits(double val,
+ std::uint64_t significant_digits) {
+ std::uint64_t precision_needed =
+ (val >= 1.0) ? significant_digits
+ : significant_digits + -(std::trunc(std::log10(val)));
+ std::stringstream string_buffer;
+ string_buffer << std::fixed << std::setprecision(precision_needed) << val;
+ return string_buffer.str();
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/19aeb6c5/utility/StringUtil.hpp
----------------------------------------------------------------------
diff --git a/utility/StringUtil.hpp b/utility/StringUtil.hpp
index 26c04a4..bd138bb 100644
--- a/utility/StringUtil.hpp
+++ b/utility/StringUtil.hpp
@@ -70,6 +70,24 @@ extern bool ParseIntString(const std::string &int_string,
*/
extern std::string ToZeroPaddedString(std::uint64_t val, int pad_width);
+
+/**
+ * @brief Parse double with significant digits.
+ *
+ * @param val The value that will be parsed.
+ * @param significant_digit The number of non-zero digits that will be parsed
+ * in the fraction part of double.
+ * @return Corresponding string representation of parsed double.
+ * Ex:
+ * DoubleToStringWithSignificantDigits(3.124355123, 3)
+ * => 3.124
+ * DoubleToStringWithSignificantDigits(0.00000000323411, 3)
+ * => 0.00000000323
+ **/
+extern std::string
+DoubleToStringWithSignificantDigits(double val,
+ std::uint64_t significant_digits);
+
/** @} */
} // namespace quickstep
[26/48] incubator-quickstep git commit: Quickstep gen stats (#225)
Posted by hb...@apache.org.
Quickstep gen stats (#225)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4b849cd2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4b849cd2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4b849cd2
Branch: refs/heads/query-manager-used-in-foreman
Commit: 4b849cd2b040538042f21f21d0c0b118158226d0
Parents: ebfe952
Author: Rogers Jeffrey Leo John <ro...@gmail.com>
Authored: Thu May 19 20:31:37 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:44 2016 -0700
----------------------------------------------------------------------
catalog/Catalog.proto | 2 +-
query_optimizer/CMakeLists.txt | 1 +
query_optimizer/ExecutionGenerator.cpp | 10 +++
relational_operators/CMakeLists.txt | 11 +++
.../GenerateNumRowsStatsOperator.cpp | 42 +++++++++++
.../GenerateNumRowsStatsOperator.hpp | 79 ++++++++++++++++++++
6 files changed, 144 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b849cd2/catalog/Catalog.proto
----------------------------------------------------------------------
diff --git a/catalog/Catalog.proto b/catalog/Catalog.proto
index ce4bc2e..8e44181 100644
--- a/catalog/Catalog.proto
+++ b/catalog/Catalog.proto
@@ -82,7 +82,7 @@ message IndexScheme {
message CatalogRelationStatistics {
optional fixed64 num_tuples = 1;
-
+
message NumDistinctValuesEntry {
required int32 attr_id = 1;
required fixed64 num_distinct_values = 2;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b849cd2/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index aa2873e..1cc38d1 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -111,6 +111,7 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_relationaloperators_DestroyHashOperator
quickstep_relationaloperators_DropTableOperator
quickstep_relationaloperators_FinalizeAggregationOperator
+ quickstep_relationaloperators_GenerateNumRowsStatsOperator
quickstep_relationaloperators_HashJoinOperator
quickstep_relationaloperators_InsertOperator
quickstep_relationaloperators_NestedLoopsJoinOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b849cd2/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index c590b6e..612efd9 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -91,6 +91,7 @@
#include "relational_operators/DestroyHashOperator.hpp"
#include "relational_operators/DropTableOperator.hpp"
#include "relational_operators/FinalizeAggregationOperator.hpp"
+#include "relational_operators/GenerateNumRowsStatsOperator.hpp"
#include "relational_operators/HashJoinOperator.hpp"
#include "relational_operators/InsertOperator.hpp"
#include "relational_operators/NestedLoopsJoinOperator.hpp"
@@ -947,6 +948,15 @@ void ExecutionGenerator::convertCopyFrom(
execution_plan_->addDirectDependency(save_blocks_operator_index,
scan_operator_index,
false /* is_pipeline_breaker */);
+
+ const QueryPlan::DAGNodeIndex num_rows_operator_index =
+ execution_plan_->addRelationalOperator(new GenerateNumRowsStatsOperator(
+ optimizer_context_->catalog_database()->getRelationByIdMutable(
+ output_relation->getID())));
+ insert_destination_proto->set_relational_op_index(num_rows_operator_index);
+ execution_plan_->addDirectDependency(num_rows_operator_index,
+ scan_operator_index,
+ true /* is_pipeline_breaker */);
}
void ExecutionGenerator::convertCreateIndex(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b849cd2/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index eec5300..e211630 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -34,6 +34,9 @@ add_library(quickstep_relationaloperators_DropTableOperator DropTableOperator.cp
add_library(quickstep_relationaloperators_FinalizeAggregationOperator
FinalizeAggregationOperator.cpp
FinalizeAggregationOperator.hpp)
+add_library(quickstep_relationaloperators_GenerateNumRowsStatsOperator
+ GenerateNumRowsStatsOperator.cpp
+ GenerateNumRowsStatsOperator.hpp)
add_library(quickstep_relationaloperators_HashJoinOperator HashJoinOperator.cpp HashJoinOperator.hpp)
add_library(quickstep_relationaloperators_InsertOperator InsertOperator.cpp InsertOperator.hpp)
add_library(quickstep_relationaloperators_NestedLoopsJoinOperator
@@ -159,6 +162,13 @@ target_link_libraries(quickstep_relationaloperators_FinalizeAggregationOperator
quickstep_storage_AggregationOperationState
quickstep_utility_Macros
tmb)
+target_link_libraries(quickstep_relationaloperators_GenerateNumRowsStatsOperator
+ glog
+ quickstep_catalog_CatalogRelation
+ quickstep_cli_PrintToScreen
+ quickstep_relationaloperators_RelationalOperator
+ quickstep_utility_Macros
+ tmb)
target_link_libraries(quickstep_relationaloperators_HashJoinOperator
gflags_nothreads-static
glog
@@ -446,6 +456,7 @@ target_link_libraries(quickstep_relationaloperators
quickstep_relationaloperators_DestroyHashOperator
quickstep_relationaloperators_DropTableOperator
quickstep_relationaloperators_FinalizeAggregationOperator
+ quickstep_relationaloperators_GenerateNumRowsStatsOperator
quickstep_relationaloperators_HashJoinOperator
quickstep_relationaloperators_InsertOperator
quickstep_relationaloperators_NestedLoopsJoinOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b849cd2/relational_operators/GenerateNumRowsStatsOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/GenerateNumRowsStatsOperator.cpp b/relational_operators/GenerateNumRowsStatsOperator.cpp
new file mode 100644
index 0000000..074e1ca
--- /dev/null
+++ b/relational_operators/GenerateNumRowsStatsOperator.cpp
@@ -0,0 +1,42 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "relational_operators/GenerateNumRowsStatsOperator.hpp"
+
+#include <memory>
+
+#include "catalog/CatalogRelation.hpp"
+#include "cli/PrintToScreen.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace quickstep {
+
+bool GenerateNumRowsStatsOperator::getAllWorkOrders(
+ WorkOrdersContainer *container,
+ QueryContext *query_context,
+ StorageManager *storage_manager,
+ const tmb::client_id scheduler_client_id,
+ tmb::MessageBus *bus) {
+ std::size_t num_tuples =
+ PrintToScreen::GetNumTuplesInRelation(*relation_, storage_manager);
+ relation_->getStatisticsMutable()->setNumTuples(num_tuples);
+ return true;
+}
+
+} // namespace quickstep
+
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b849cd2/relational_operators/GenerateNumRowsStatsOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/GenerateNumRowsStatsOperator.hpp b/relational_operators/GenerateNumRowsStatsOperator.hpp
new file mode 100644
index 0000000..8622a63
--- /dev/null
+++ b/relational_operators/GenerateNumRowsStatsOperator.hpp
@@ -0,0 +1,79 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
+#define QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
+
+#include <memory>
+
+#include "catalog/CatalogRelation.hpp"
+#include "relational_operators/RelationalOperator.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+
+namespace tmb { class MessageBus; }
+
+namespace quickstep {
+
+class CatalogRelation;
+class QueryContext;
+class StorageManager;
+class WorkOrdersContainer;
+
+/** \addtogroup RelationalOperators
+ * @{
+ */
+
+/**
+ * @brief An operator that gets the number of rows after loading a relation.
+ **/
+class GenerateNumRowsStatsOperator : public RelationalOperator {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param relation The relation to get the number of rows from.
+ * This GenNumRowStatsOperator owns relation until
+ * the WorkOrder it produces is successfully executed.
+ **/
+ explicit GenerateNumRowsStatsOperator(CatalogRelation *relation)
+ : relation_(relation) {}
+ ~GenerateNumRowsStatsOperator() override {}
+
+ /**
+ * @note no WorkOrder is generated for this operator.
+ **/
+ bool getAllWorkOrders(WorkOrdersContainer *container,
+ QueryContext *query_context,
+ StorageManager *storage_manager,
+ const tmb::client_id scheduler_client_id,
+ tmb::MessageBus *bus) override;
+
+ private:
+ CatalogRelation *relation_;
+
+ DISALLOW_COPY_AND_ASSIGN(GenerateNumRowsStatsOperator);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
[40/48] incubator-quickstep git commit: Added Async DataExchange
Service.
Posted by hb...@apache.org.
Added Async DataExchange Service.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5ae50520
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5ae50520
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5ae50520
Branch: refs/heads/query-manager-used-in-foreman
Commit: 5ae50520bdf701153c2390f601900493c7410d08
Parents: 3789da7
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Sat May 28 22:55:05 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:46 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 11 +-
storage/CMakeLists.txt | 69 +++++++-
storage/DataExchange.proto | 31 ++++
storage/DataExchangerAsync.cpp | 165 ++++++++++++++++++
storage/DataExchangerAsync.hpp | 97 +++++++++++
storage/StorageManager.cpp | 113 ++++++++++++-
storage/StorageManager.hpp | 61 ++++++-
storage/tests/DataExchange_unittest.cpp | 240 +++++++++++++++++++++++++++
third_party/iwyu/iwyu_helper.py | 3 +-
validate_cmakelists.py | 5 +-
10 files changed, 787 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae50520/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index dc51ca6..ef7fd50 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -662,7 +662,12 @@ set(ENABLE_PUREMEMORY ON CACHE BOOL "Enable PureMemory TMB")
set(ENABLE_LEVELDB OFF CACHE BOOL "Enable LevelDB TMB")
set(ENABLE_MEMORYMIRROR OFF CACHE BOOL "Enable MemoryMirror TMB")
set(ENABLE_NATIVELOG OFF CACHE BOOL "Enable NativeLog TMB")
-set(ENABLE_NATIVENET OFF CACHE BOOL "Enable NativeNet TMB")
+
+# The distributed version requires to use the NativeNet implementation.
+if (NOT ENABLE_DISTRIBUTED)
+ set(ENABLE_NATIVENET OFF CACHE BOOL "Enable NativeNet TMB")
+endif()
+
set(ENABLE ENABLE_SQLITE OFF CACHE BOOL "Enable SQLite TMB")
set(ENABLE_VOLTDB OFF CACHE BOOL "Enable VoltDB TMB")
set(ENABLE_ZOOKEEPER OFF CACHE BOOL "Enable Zookeeper TMB")
@@ -670,6 +675,10 @@ set(ENABLE_ZOOKEEPER OFF CACHE BOOL "Enable Zookeeper TMB")
add_subdirectory("${THIRD_PARTY_SOURCE_DIR}/tmb" "${CMAKE_CURRENT_BINARY_DIR}/third_party/tmb")
include_directories(${TMB_INCLUDE_DIRS})
+if (ENABLE_DISTRIBUTED)
+ include_directories(${CMAKE_CURRENT_BINARY_DIR}/third_party/tmb/include)
+endif()
+
# Add all of the module subdirectories. CMakeLists.txt in each of the subdirectories
# defines how to build that module's libraries.
add_subdirectory(catalog)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae50520/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 4da16ea..a77976a 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -126,6 +126,13 @@ QS_PROTOBUF_GENERATE_CPP(storage_StorageBlockLayout_proto_srcs
storage_StorageBlockLayout_proto_hdrs
StorageBlockLayout.proto)
+if (ENABLE_DISTRIBUTED)
+ GRPC_GENERATE_CPP(storage_DataExchange_proto_srcs
+ storage_DataExchange_proto_hdrs
+ .
+ DataExchange.proto)
+endif()
+
# Declare micro-libs:
add_library(quickstep_storage_AggregationOperationState
AggregationOperationState.cpp
@@ -171,6 +178,14 @@ add_library(quickstep_storage_CompressedTupleStorageSubBlock
CompressedTupleStorageSubBlock.hpp)
add_library(quickstep_storage_CountedReference ../empty_src.cpp CountedReference.hpp)
add_library(quickstep_storage_CSBTreeIndexSubBlock CSBTreeIndexSubBlock.cpp CSBTreeIndexSubBlock.hpp)
+
+if (ENABLE_DISTRIBUTED)
+ add_library(quickstep_storage_DataExchange_proto
+ ${storage_DataExchange_proto_srcs}
+ ${storage_DataExchange_proto_hdrs})
+ add_library(quickstep_storage_DataExchangerAsync DataExchangerAsync.cpp DataExchangerAsync.hpp)
+endif()
+
add_library(quickstep_storage_EvictionPolicy EvictionPolicy.cpp EvictionPolicy.hpp)
add_library(quickstep_storage_FileManager ../empty_src.cpp FileManager.hpp)
if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
@@ -575,6 +590,19 @@ target_link_libraries(quickstep_storage_CSBTreeIndexSubBlock
quickstep_utility_Macros
quickstep_utility_PtrVector
quickstep_utility_ScopedBuffer)
+
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_storage_DataExchange_proto
+ ${PROTOBUF3_LIBRARY})
+ target_link_libraries(quickstep_storage_DataExchangerAsync
+ glog
+ quickstep_storage_DataExchange_proto
+ quickstep_storage_StorageManager
+ quickstep_threading_Thread
+ quickstep_utility_Macros
+ ${GRPCPLUSPLUS_LIBRARIES})
+endif()
+
target_link_libraries(quickstep_storage_EvictionPolicy
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageConstants
@@ -925,6 +953,7 @@ target_link_libraries(quickstep_storage_StorageManager
gflags_nothreads-static
glog
gtest
+ quickstep_catalog_CatalogTypedefs
quickstep_storage_CountedReference
quickstep_storage_EvictionPolicy
quickstep_storage_FileManager
@@ -955,7 +984,9 @@ if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_storage_StorageManager
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
- quickstep_queryexecution_QueryExecutionUtil)
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_DataExchange_proto
+ ${GRPCPLUSPLUS_LIBRARIES})
endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_storage_SubBlockTypeRegistry
glog
@@ -1071,6 +1102,11 @@ elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS)
target_link_libraries(quickstep_storage
quickstep_storage_FileManagerWindows)
endif()
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_storage
+ quickstep_storage_DataExchange_proto
+ quickstep_storage_DataExchangerAsync)
+endif()
# CMAKE_VALIDATE_IGNORE_BEGIN
if(QUICKSTEP_HAVE_BITWEAVING)
target_link_libraries(quickstep_storage
@@ -1340,6 +1376,37 @@ target_link_libraries(CompressedPackedRowStoreTupleStorageSubBlock_unittest
${LIBS})
add_test(CompressedPackedRowStoreTupleStorageSubBlock_unittest CompressedPackedRowStoreTupleStorageSubBlock_unittest)
+if (ENABLE_DISTRIBUTED)
+ add_executable(DataExchange_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/DataExchange_unittest.cpp")
+ target_link_libraries(DataExchange_unittest
+ gflags_nothreads-static
+ glog
+ gtest
+ quickstep_catalog_CatalogAttribute
+ quickstep_catalog_CatalogRelation
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_CountedReference
+ quickstep_storage_DataExchangerAsync
+ quickstep_storage_StorageBlob
+ quickstep_storage_StorageBlock
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_storage_StorageManager
+ quickstep_storage_TupleStorageSubBlock
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ quickstep_types_TypedValue
+ quickstep_types_containers_Tuple
+ tmb
+ ${LIBS})
+ add_test(DataExchange_unittest DataExchange_unittest)
+endif(ENABLE_DISTRIBUTED)
+
add_executable(CSBTreeIndexSubBlock_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/CSBTreeIndexSubBlock_unittest.cpp")
target_link_libraries(CSBTreeIndexSubBlock_unittest
gtest
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae50520/storage/DataExchange.proto
----------------------------------------------------------------------
diff --git a/storage/DataExchange.proto b/storage/DataExchange.proto
new file mode 100644
index 0000000..a2636e5
--- /dev/null
+++ b/storage/DataExchange.proto
@@ -0,0 +1,31 @@
+// Copyright 2016 Pivotal Software, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package quickstep;
+
+service DataExchange {
+ rpc Pull (PullRequest) returns (PullResponse) {}
+}
+
+message PullRequest {
+ fixed64 block_id = 1;
+}
+
+message PullResponse {
+ bool is_valid = 1;
+ uint64 num_slots = 2;
+ bytes block = 3;
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae50520/storage/DataExchangerAsync.cpp
----------------------------------------------------------------------
diff --git a/storage/DataExchangerAsync.cpp b/storage/DataExchangerAsync.cpp
new file mode 100644
index 0000000..78c6565
--- /dev/null
+++ b/storage/DataExchangerAsync.cpp
@@ -0,0 +1,165 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "storage/DataExchangerAsync.hpp"
+
+#include <grpc++/grpc++.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+
+#include "storage/DataExchange.grpc.pb.h"
+#include "storage/DataExchange.pb.h"
+#include "storage/StorageManager.hpp"
+
+#include "glog/logging.h"
+
+using grpc::ServerCompletionQueue;
+
+namespace quickstep {
+namespace {
+
+/**
+ * @brief RPC Request Context Instance.
+ **/
+class CallContext {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param service The async service.
+ * @param queue The RPC request queue.
+ * @param storage_manager The StorageManager to use.
+ **/
+ CallContext(DataExchange::AsyncService *service,
+ ServerCompletionQueue *queue,
+ StorageManager *storage_manager)
+ : service_(service),
+ queue_(queue),
+ storage_manager_(DCHECK_NOTNULL(storage_manager)),
+ responder_(&context_),
+ status_(CallStatus::CREATE) {
+ Proceed();
+ }
+
+ /**
+ * @brief Process the RPC request.
+ **/
+ void Proceed();
+
+ private:
+ DataExchange::AsyncService *service_;
+ ServerCompletionQueue *queue_;
+
+ StorageManager *storage_manager_;
+
+ grpc::ServerContext context_;
+
+ PullRequest request_;
+ PullResponse response_;
+
+ grpc::ServerAsyncResponseWriter<PullResponse> responder_;
+
+ enum class CallStatus {
+ CREATE = 0,
+ PROCESS,
+ FINISH,
+ };
+ CallStatus status_;
+};
+
+void CallContext::Proceed() {
+ switch (status_) {
+ case CallStatus::CREATE: {
+ // Change this instance progress to the PROCESS state.
+ status_ = CallStatus::PROCESS;
+
+ // As part of the initial CREATE state, we *request* that the system
+ // start processing Pull requests. In this request, "this" acts are
+ // the tag uniquely identifying the request (so that different CallContext
+ // instances can serve different requests concurrently), in this case
+ // the memory address of this CallContext instance.
+ service_->RequestPull(&context_, &request_, &responder_, queue_, queue_, this);
+ break;
+ }
+ case CallStatus::PROCESS: {
+ // Spawn a new CallContext instance to serve new clients while we process
+ // the one for this CallContext. The instance will deallocate itself as
+ // part of its FINISH state.
+ new CallContext(service_, queue_, storage_manager_);
+
+ // The actual processing.
+ storage_manager_->pullBlockOrBlob(request_.block_id(), &response_);
+
+ // And we are done! Let the gRPC runtime know we've finished, using the
+ // memory address of this instance as the uniquely identifying tag for
+ // the event.
+ status_ = CallStatus::FINISH;
+ responder_.Finish(response_, grpc::Status::OK, this);
+ break;
+ }
+ case CallStatus::FINISH: {
+ // Once in the FINISH state, deallocate ourselves (CallContext).
+ delete this;
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown call status.";
+ }
+}
+
+} // namespace
+
+const char *DataExchangerAsync::kLocalNetworkAddress = "0.0.0.0:";
+
+DataExchangerAsync::DataExchangerAsync() {
+ grpc::ServerBuilder builder;
+ builder.AddListeningPort(kLocalNetworkAddress, grpc::InsecureServerCredentials(), &port_);
+ builder.RegisterService(&service_);
+
+ queue_ = builder.AddCompletionQueue();
+ server_ = builder.BuildAndStart();
+
+ DCHECK_GT(port_, 0);
+ server_address_ = kLocalNetworkAddress + std::to_string(port_);
+ LOG(INFO) << "DataExchangerAsync Service listening on " << server_address_;
+}
+
+void DataExchangerAsync::run() {
+ // Self-destruct upon success.
+ new CallContext(&service_, queue_.get(), storage_manager_);
+
+ void *tag = nullptr; // Uniquely identify a request.
+ bool ok = false;
+
+ while (true) {
+ if (queue_->Next(&tag, &ok)) {
+ CallContext *call_context = static_cast<CallContext*>(tag);
+ if (ok) {
+ call_context->Proceed();
+ } else {
+ LOG(WARNING) << "Not ok\n";
+ delete call_context;
+ }
+ } else {
+ LOG(INFO) << "Shutdown\n";
+ return;
+ }
+ }
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae50520/storage/DataExchangerAsync.hpp
----------------------------------------------------------------------
diff --git a/storage/DataExchangerAsync.hpp b/storage/DataExchangerAsync.hpp
new file mode 100644
index 0000000..75a4e4d
--- /dev/null
+++ b/storage/DataExchangerAsync.hpp
@@ -0,0 +1,97 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_DATA_EXCHANGER_ASYNC_HPP_
+#define QUICKSTEP_STORAGE_DATA_EXCHANGER_ASYNC_HPP_
+
+#include <grpc++/grpc++.h>
+
+#include <memory>
+#include <string>
+
+#include "storage/DataExchange.grpc.pb.h"
+#include "threading/Thread.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class StorageManager;
+
+/**
+ * @brief A class which exchanges data from a StorageManager to its peer.
+ **/
+class DataExchangerAsync final : public Thread {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ DataExchangerAsync();
+
+ ~DataExchangerAsync() override {}
+
+ /**
+ * @brief Set the local StorageManager.
+ *
+ * @param storage_manager The StorageManager to use.
+ **/
+ void set_storage_manager(StorageManager *storage_manager) {
+ storage_manager_ = storage_manager;
+ }
+
+ /**
+ * @brief Return its network address for peers to connect.
+ *
+ * @return Its network address.
+ **/
+ std::string network_address() const {
+ DCHECK(!server_address_.empty());
+ return server_address_;
+ }
+
+ /**
+ * @brief Shutdown itself.
+ **/
+ void shutdown() {
+ server_->Shutdown();
+ // Always shutdown the completion queue after the server.
+ queue_->Shutdown();
+ }
+
+ protected:
+ void run() override;
+
+ private:
+ static const char *kLocalNetworkAddress;
+
+ DataExchange::AsyncService service_;
+
+ int port_ = -1;
+ // Format IP:port, i.e., "0.0.0.0:0".
+ std::string server_address_;
+
+ std::unique_ptr<grpc::ServerCompletionQueue> queue_;
+ std::unique_ptr<grpc::Server> server_;
+
+ StorageManager *storage_manager_ = nullptr;
+
+ DISALLOW_COPY_AND_ASSIGN(DataExchangerAsync);
+};
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_STORAGE_DATA_EXCHANGER_ASYNC_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae50520/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 15e2503..8fc1224 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -41,6 +41,10 @@
#include <numaif.h>
#endif
+#ifdef QUICKSTEP_DISTRIBUTED
+#include <grpc++/grpc++.h>
+#endif
+
#include <atomic>
#include <cerrno>
#include <cstddef>
@@ -53,6 +57,8 @@
#include <unordered_map>
#include <vector>
+#include "catalog/CatalogTypedefs.hpp"
+
#ifdef QUICKSTEP_DISTRIBUTED
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionTypedefs.hpp"
@@ -60,6 +66,12 @@
#endif
#include "storage/CountedReference.hpp"
+
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "storage/DataExchange.grpc.pb.h"
+#include "storage/DataExchange.pb.h"
+#endif
+
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManagerLocal.hpp"
#include "storage/StorageBlob.hpp"
@@ -456,6 +468,80 @@ block_id StorageManager::allocateNewBlockOrBlob(const std::size_t num_slots,
}
#ifdef QUICKSTEP_DISTRIBUTED
+void StorageManager::pullBlockOrBlob(const block_id block,
+ PullResponse *response) const {
+ SpinSharedMutexSharedLock<false> read_lock(blocks_shared_mutex_);
+ std::unordered_map<block_id, BlockHandle>::const_iterator cit = blocks_.find(block);
+ if (cit != blocks_.end()) {
+ response->set_is_valid(true);
+
+ const BlockHandle &block_handle = cit->second;
+ const std::size_t num_slots = block_handle.block_memory_size;
+
+ response->set_num_slots(num_slots);
+ response->set_block(block_handle.block_memory,
+ num_slots * kSlotSizeBytes);
+ } else {
+ response->set_is_valid(false);
+ }
+}
+
+StorageManager::DataExchangerClientAsync::DataExchangerClientAsync(const std::shared_ptr<grpc::Channel> &channel,
+ StorageManager *storage_manager)
+ : stub_(DataExchange::NewStub(channel)),
+ storage_manager_(storage_manager) {
+}
+
+bool StorageManager::DataExchangerClientAsync::Pull(const block_id block,
+ const numa_node_id numa_node,
+ BlockHandle *block_handle) {
+ grpc::ClientContext context;
+
+ PullRequest request;
+ request.set_block_id(block);
+
+ grpc::CompletionQueue queue;
+
+ unique_ptr<grpc::ClientAsyncResponseReader<PullResponse>> rpc(
+ stub_->AsyncPull(&context, request, &queue));
+
+ PullResponse response;
+ grpc::Status status;
+
+ rpc->Finish(&response, &status, reinterpret_cast<void*>(1));
+
+ void *got_tag;
+ bool ok = false;
+
+ queue.Next(&got_tag, &ok);
+ CHECK(got_tag == reinterpret_cast<void*>(1));
+ CHECK(ok);
+
+ if (!status.ok()) {
+ LOG(ERROR) << "DataExchangerClientAsync Pull error: RPC failed";
+ return false;
+ }
+
+ if (!response.is_valid()) {
+ LOG(INFO) << "The pulling block not found in all the peers";
+ return false;
+ }
+
+ const size_t num_slots = response.num_slots();
+ DCHECK_NE(num_slots, 0u);
+
+ const string &block_content = response.block();
+ DCHECK_EQ(kSlotSizeBytes * num_slots, block_content.size());
+
+ void *block_buffer = storage_manager_->allocateSlots(num_slots, numa_node);
+
+ block_handle->block_memory =
+ std::memcpy(block_buffer, block_content.c_str(), block_content.size());
+ block_handle->block_memory_size = num_slots;
+
+ return true;
+}
+
vector<string> StorageManager::getPeerDomainNetworkAddresses(const block_id block) {
serialization::BlockMessage proto;
proto.set_block_id(block);
@@ -541,14 +627,37 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
// The caller of this function holds an exclusive lock on this block/blob's
// mutex in the lock manager. The caller has ensured that the block is not
// already loaded before this function gets called.
- size_t num_slots = file_manager_->numSlots(block);
+ BlockHandle loaded_handle;
+
+#ifdef QUICKSTEP_DISTRIBUTED
+ // TODO(quickstep-team): Use a cost model to determine whether to load from
+ // a remote peer or the disk.
+ if (BlockIdUtil::Domain(block) != block_domain_) {
+ LOG(INFO) << "Pulling Block " << BlockIdUtil::ToString(block) << " from a remote peer";
+ const vector<string> peer_domain_network_addresses = getPeerDomainNetworkAddresses(block);
+ for (const string &peer_domain_network_address : peer_domain_network_addresses) {
+ DataExchangerClientAsync client(
+ grpc::CreateChannel(peer_domain_network_address, grpc::InsecureChannelCredentials()),
+ this);
+
+ if (client.Pull(block, numa_node, &loaded_handle)) {
+ sendBlockLocationMessage(block, kAddBlockLocationMessage);
+ return loaded_handle;
+ }
+ }
+
+ LOG(INFO) << "Failed to pull Block " << BlockIdUtil::ToString(block)
+ << " from remote peers, so try to load from disk.";
+ }
+#endif
+
+ const size_t num_slots = file_manager_->numSlots(block);
DEBUG_ASSERT(num_slots != 0);
void *block_buffer = allocateSlots(num_slots, numa_node);
const bool status = file_manager_->readBlockOrBlob(block, block_buffer, kSlotSizeBytes * num_slots);
CHECK(status) << "Failed to read block from persistent storage: " << block;
- BlockHandle loaded_handle;
loaded_handle.block_memory = block_buffer;
loaded_handle.block_memory_size = num_slots;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae50520/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index 55a011e..50ddb0f 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -26,9 +26,14 @@
#include <unordered_map>
#include <vector>
+#include "catalog/CatalogTypedefs.hpp"
#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED
-
#include "storage/CountedReference.hpp"
+
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "storage/DataExchange.grpc.pb.h"
+#endif
+
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManager.hpp"
#include "storage/StorageBlob.hpp"
@@ -45,6 +50,10 @@
#include "tmb/id_typedefs.h"
+#ifdef QUICKSTEP_DISTRIBUTED
+namespace grpc { class Channel; }
+#endif
+
namespace tmb { class MessageBus; }
namespace quickstep {
@@ -58,6 +67,10 @@ DECLARE_bool(use_hdfs);
class CatalogRelationSchema;
+#ifdef QUICKSTEP_DISTRIBUTED
+class PullResponse;
+#endif
+
class StorageBlockLayout;
/** \addtogroup Storage
@@ -365,6 +378,16 @@ class StorageManager {
**/
bool blockOrBlobIsLoadedAndDirty(const block_id block);
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Pull a block or a blob. Used by DataExchangerAsync.
+ *
+ * @param block The id of the block or blob.
+ * @param response Where to store the pulled block content.
+ **/
+ void pullBlockOrBlob(const block_id block, PullResponse *response) const;
+#endif
+
private:
struct BlockHandle {
void *block_memory;
@@ -374,6 +397,42 @@ class StorageManager {
#ifdef QUICKSTEP_DISTRIBUTED
/**
+ * @brief A class which connects to DataExchangerAsync to exchange data from
+ * remote peers.
+ **/
+ class DataExchangerClientAsync {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param channel The RPC channel to connect DataExchangerAsync.
+ * @param storage_manager The StorageManager to use.
+ */
+ DataExchangerClientAsync(const std::shared_ptr<grpc::Channel> &channel,
+ StorageManager *storage_manager);
+
+ /**
+ * @brief Pull a block or blob from a remote StorageManager.
+ *
+ * @param block The block or blob to pull.
+ * @param numa_node The NUMA node for placing this block.
+ * @param block_handle Where the pulled block or blob stores.
+ *
+ * @return Whether the pull operation is successful or not.
+ */
+ bool Pull(const block_id block,
+ const numa_node_id numa_node,
+ BlockHandle *block_handle);
+
+ private:
+ std::unique_ptr<DataExchange::Stub> stub_;
+
+ StorageManager *storage_manager_;
+
+ DISALLOW_COPY_AND_ASSIGN(DataExchangerClientAsync);
+ };
+
+ /**
* @brief Get the network info of all the remote StorageManagers which may
* load the given block in the buffer pool.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae50520/storage/tests/DataExchange_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/DataExchange_unittest.cpp b/storage/tests/DataExchange_unittest.cpp
new file mode 100644
index 0000000..38d12f6
--- /dev/null
+++ b/storage/tests/DataExchange_unittest.cpp
@@ -0,0 +1,240 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogAttribute.hpp"
+#include "catalog/CatalogRelation.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/BlockLocator.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "storage/CountedReference.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageBlob.hpp"
+#include "storage/StorageBlock.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageConstants.hpp"
+#include "storage/StorageManager.hpp"
+#include "storage/TupleStorageSubBlock.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "types/containers/Tuple.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+
+using std::free;
+using std::malloc;
+using std::move;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+using tmb::MessageBus;
+using tmb::TaggedMessage;
+
+namespace quickstep {
+
+class DataExchangeTest : public ::testing::Test {
+ protected:
+ static const char kStoragePath[];
+ static const char kCheckedDomainNetworkAddress[];
+
+ ~DataExchangeTest() {
+ data_exchanger_expected_.join();
+ locator_->join();
+ }
+
+ virtual void SetUp() {
+ bus_.Initialize();
+
+ locator_.reset(new BlockLocator(&bus_));
+ locator_client_id_ = locator_->getBusClientID();
+ locator_->start();
+
+ worker_client_id_ = bus_.Connect();
+ bus_.RegisterClientAsSender(worker_client_id_, kBlockDomainRegistrationMessage);
+ bus_.RegisterClientAsReceiver(worker_client_id_, kBlockDomainRegistrationResponseMessage);
+
+ bus_.RegisterClientAsSender(worker_client_id_, kPoisonMessage);
+
+ storage_manager_expected_.reset(new StorageManager(
+ kStoragePath,
+ getBlockDomain(data_exchanger_expected_.network_address()),
+ locator_client_id_,
+ &bus_));
+
+ data_exchanger_expected_.set_storage_manager(storage_manager_expected_.get());
+ data_exchanger_expected_.start();
+
+ storage_manager_checked_.reset(new StorageManager(
+ kStoragePath,
+ getBlockDomain(kCheckedDomainNetworkAddress),
+ locator_client_id_,
+ &bus_));
+ }
+
+ virtual void TearDown() {
+ storage_manager_checked_.reset();
+
+ data_exchanger_expected_.shutdown();
+ storage_manager_expected_.reset();
+
+ serialization::EmptyMessage proto;
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kPoisonMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent PoisonMessage (typed '" << kPoisonMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+ }
+
+ unique_ptr<StorageManager> storage_manager_expected_, storage_manager_checked_;
+
+ private:
+ block_id_domain getBlockDomain(const string &network_address) {
+ serialization::BlockDomainRegistrationMessage proto;
+ proto.set_domain_network_address(network_address);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainRegistrationMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+
+ const tmb::AnnotatedMessage annotated_message(bus_.Receive(worker_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ EXPECT_EQ(locator_client_id_, annotated_message.sender);
+ EXPECT_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') received BlockDomainRegistrationResponseMessage (typed '"
+ << kBlockDomainRegistrationResponseMessage
+ << "') from BlockLocator";
+
+ serialization::BlockDomainMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ return static_cast<block_id_domain>(response_proto.block_domain());
+ }
+
+ MessageBusImpl bus_;
+
+ unique_ptr<BlockLocator> locator_;
+ tmb::client_id locator_client_id_;
+
+ tmb::client_id worker_client_id_;
+
+ DataExchangerAsync data_exchanger_expected_;
+};
+
+const char DataExchangeTest::kStoragePath[] = "./data_exchange_test_data/";
+const char DataExchangeTest::kCheckedDomainNetworkAddress[] = "ip:port";
+
+TEST_F(DataExchangeTest, BlockPull) {
+ CatalogRelation relation(nullptr, "rel");
+ const attribute_id attr_id =
+ relation.addAttribute(new CatalogAttribute(nullptr, "attr_int", TypeFactory::GetType(kInt)));
+
+ const block_id block =
+ storage_manager_expected_->createBlock(relation, relation.getDefaultStorageBlockLayout());
+
+ {
+ MutableBlockReference block_expected = storage_manager_expected_->getBlockMutable(block, relation);
+
+ // Insert a tuple.
+ const int value_expected = -1;
+ {
+ vector<TypedValue> attrs(1, TypedValue(value_expected));
+ const Tuple tuple(move(attrs));
+
+ EXPECT_TRUE(block_expected->insertTuple(tuple));
+ }
+
+ const BlockReference block_checked = storage_manager_checked_->getBlock(block, relation);
+ EXPECT_FALSE(block_checked->isBlob());
+
+ const TupleStorageSubBlock &tuple_store_checked = block_checked->getTupleStorageSubBlock();
+
+ EXPECT_EQ(1, tuple_store_checked.numTuples());
+ EXPECT_EQ(value_expected, tuple_store_checked.getAttributeValueTyped(0 /* tuple_id */, attr_id).getLiteral<int>());
+ }
+
+ storage_manager_checked_->deleteBlockOrBlobFile(block);
+ storage_manager_expected_->deleteBlockOrBlobFile(block);
+}
+
+TEST_F(DataExchangeTest, BlobPull) {
+ const block_id blob = storage_manager_expected_->createBlob(kDefaultBlockSizeInSlots);
+ {
+ const BlobReference blob_expected = storage_manager_expected_->getBlob(blob);
+ const BlobReference blob_checked = storage_manager_checked_->getBlob(blob);
+ EXPECT_TRUE(blob_checked->isBlob());
+ EXPECT_EQ(blob, blob_checked->getID());
+ }
+
+ storage_manager_checked_->deleteBlockOrBlobFile(blob);
+ storage_manager_expected_->deleteBlockOrBlobFile(blob);
+}
+
+} // namespace quickstep
+
+int main(int argc, char **argv) {
+ google::InitGoogleLogging(argv[0]);
+ // Honor FLAGS_buffer_pool_slots in StorageManager.
+ gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+ ::testing::InitGoogleTest(&argc, argv);
+
+ return RUN_ALL_TESTS();
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae50520/third_party/iwyu/iwyu_helper.py
----------------------------------------------------------------------
diff --git a/third_party/iwyu/iwyu_helper.py b/third_party/iwyu/iwyu_helper.py
index a204c50..13697be 100755
--- a/third_party/iwyu/iwyu_helper.py
+++ b/third_party/iwyu/iwyu_helper.py
@@ -19,8 +19,9 @@ import sys
QUICKSTEP_INCLUDES = [ '.',
'./build',
'./build/third_party',
- './build/third_party/protobuf/include',
'./build/third_party/gflags/include',
+ './build/third_party/protobuf/include',
+ './build/third_party/tmb/include',
'./third_party/benchmark/include',
'./third_party/glog/src',
'./third_party/googletest/googletest/include',
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae50520/validate_cmakelists.py
----------------------------------------------------------------------
diff --git a/validate_cmakelists.py b/validate_cmakelists.py
index c7b5883..7dd6fc5 100755
--- a/validate_cmakelists.py
+++ b/validate_cmakelists.py
@@ -17,7 +17,7 @@ TODO List / Known Issues & Limitations:
"""
# Copyright 2011-2015 Quickstep Technologies LLC.
-# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2015-2016 Pivotal Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -40,7 +40,8 @@ EXCLUDED_TOP_LEVEL_DIRS = ["build", "third_party"]
# Explicitly ignored dependencies (special headers with no other quickstep
# dependencies).
IGNORED_DEPENDENCIES = frozenset(
- ["quickstep_threading_WinThreadsAPI",
+ ["quickstep_storage_DataExchange.grpc_proto",
+ "quickstep_threading_WinThreadsAPI",
"quickstep_utility_textbasedtest_TextBasedTest",
"quickstep_utility_textbasedtest_TextBasedTestDriver",
"quickstep_storage_bitweaving_BitWeavingHIndexSubBlock",
[43/48] incubator-quickstep git commit: Added Query ID to Relational
operators and WorkOrders.
Posted by hb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/SortMergeRunOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortMergeRunOperator.hpp b/relational_operators/SortMergeRunOperator.hpp
index f92affe..f54e925 100644
--- a/relational_operators/SortMergeRunOperator.hpp
+++ b/relational_operators/SortMergeRunOperator.hpp
@@ -88,6 +88,7 @@ class SortMergeRunOperator : public RelationalOperator {
* \c top_k is 0.
* @param input_relation_is_stored Boolean to indicate is input relation is
* stored or streamed.
+ * @param query_id The ID of the query to which this operator belongs.
**/
SortMergeRunOperator(const CatalogRelation &input_relation,
const CatalogRelation &output_relation,
@@ -97,8 +98,10 @@ class SortMergeRunOperator : public RelationalOperator {
const QueryContext::sort_config_id sort_config_index,
const std::size_t merge_factor,
const std::size_t top_k,
- const bool input_relation_is_stored)
- : input_relation_(input_relation),
+ const bool input_relation_is_stored,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ input_relation_(input_relation),
output_relation_(output_relation),
output_destination_index_(output_destination_index),
sort_config_index_(sort_config_index),
@@ -216,6 +219,7 @@ class SortMergeRunWorkOrder : public WorkOrder {
* @param input_runs Input runs to merge.
* @param top_k If non-zero will merge only \c top_k tuples.
* @param merge_level Merge level in the merge tree.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to create new blocks.
* @param storage_manager The StorageManager to use.
* @param operator_index Merge-run operator index to send feedback messages
@@ -229,12 +233,14 @@ class SortMergeRunWorkOrder : public WorkOrder {
std::vector<merge_run_operator::Run> &&input_runs,
const std::size_t top_k,
const std::size_t merge_level,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager,
const std::size_t operator_index,
const tmb::client_id scheduler_client_id,
MessageBus *bus)
- : sort_config_(sort_config),
+ : WorkOrder(query_id),
+ sort_config_(sort_config),
run_relation_(run_relation),
input_runs_(std::move(input_runs)),
top_k_(top_k),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/SortRunGenerationOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortRunGenerationOperator.cpp b/relational_operators/SortRunGenerationOperator.cpp
index 9bb3f51..e352f9e 100644
--- a/relational_operators/SortRunGenerationOperator.cpp
+++ b/relational_operators/SortRunGenerationOperator.cpp
@@ -54,6 +54,7 @@ bool SortRunGenerationOperator::getAllWorkOrders(
new SortRunGenerationWorkOrder(input_relation_,
input_block_id,
sort_config,
+ query_id_,
output_destination,
storage_manager),
op_index_);
@@ -69,6 +70,7 @@ bool SortRunGenerationOperator::getAllWorkOrders(
input_relation_,
input_relation_block_ids_[num_workorders_generated_],
sort_config,
+ query_id_,
output_destination,
storage_manager),
op_index_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/SortRunGenerationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortRunGenerationOperator.hpp b/relational_operators/SortRunGenerationOperator.hpp
index 04290a9..3da9813 100644
--- a/relational_operators/SortRunGenerationOperator.hpp
+++ b/relational_operators/SortRunGenerationOperator.hpp
@@ -83,13 +83,16 @@ class SortRunGenerationOperator : public RelationalOperator {
* @param input_relation_is_stored Does the input relation contain the blocks
* to sort. If \c false, the blocks are
* streamed.
+ * @param query_id The ID of the query to which this operator belongs.
**/
SortRunGenerationOperator(const CatalogRelation &input_relation,
const CatalogRelation &output_relation,
const QueryContext::insert_destination_id output_destination_index,
const QueryContext::sort_config_id sort_config_index,
- bool input_relation_is_stored)
- : input_relation_(input_relation),
+ bool input_relation_is_stored,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ input_relation_(input_relation),
output_relation_(output_relation),
output_destination_index_(output_destination_index),
sort_config_index_(sort_config_index),
@@ -152,6 +155,7 @@ class SortRunGenerationWorkOrder : public WorkOrder {
* @param input_block_id The block id.
* @param sort_config The Sort configuration specifying ORDER BY, ordering,
* and null ordering.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to store the sorted blocks
* of runs.
* @param storage_manager The StorageManager to use.
@@ -159,9 +163,11 @@ class SortRunGenerationWorkOrder : public WorkOrder {
SortRunGenerationWorkOrder(const CatalogRelationSchema &input_relation,
const block_id input_block_id,
const SortConfiguration &sort_config,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
- : input_relation_(input_relation),
+ : WorkOrder(query_id),
+ input_relation_(input_relation),
input_block_id_(input_block_id),
sort_config_(sort_config),
output_destination_(DCHECK_NOTNULL(output_destination)),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/TableGeneratorOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/TableGeneratorOperator.cpp b/relational_operators/TableGeneratorOperator.cpp
index 886d05f..fb1f743 100644
--- a/relational_operators/TableGeneratorOperator.cpp
+++ b/relational_operators/TableGeneratorOperator.cpp
@@ -42,8 +42,11 @@ bool TableGeneratorOperator::getAllWorkOrders(
// Currently the generator function is not abstracted to be parallelizable,
// so just produce one work order.
container->addNormalWorkOrder(
- new TableGeneratorWorkOrder(query_context->getGeneratorFunctionHandle(generator_function_index_),
- query_context->getInsertDestination(output_destination_index_)),
+ new TableGeneratorWorkOrder(
+ query_context->getGeneratorFunctionHandle(
+ generator_function_index_),
+ query_id_,
+ query_context->getInsertDestination(output_destination_index_)),
op_index_);
started_ = true;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/TableGeneratorOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/TableGeneratorOperator.hpp b/relational_operators/TableGeneratorOperator.hpp
index a26b227..bfc70c5 100644
--- a/relational_operators/TableGeneratorOperator.hpp
+++ b/relational_operators/TableGeneratorOperator.hpp
@@ -60,12 +60,14 @@ class TableGeneratorOperator : public RelationalOperator {
* QueryContext to insert the generated output.
* @param generator_function_index The index of the GeneratorFunctionHandle in
* the QueryContext.
- *
+ * @param query_id The ID of the query to which this operator belongs.
**/
TableGeneratorOperator(const CatalogRelation &output_relation,
const QueryContext::insert_destination_id output_destination_index,
- const QueryContext::generator_function_id generator_function_index)
- : output_relation_(output_relation),
+ const QueryContext::generator_function_id generator_function_index,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ output_relation_(output_relation),
output_destination_index_(output_destination_index),
generator_function_index_(generator_function_index),
started_(false) {
@@ -112,12 +114,15 @@ class TableGeneratorWorkOrder : public WorkOrder {
* @brief Constructor.
*
* @param generator_function The GeneratorFunctionHandle to use.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert the generated
* output.
**/
TableGeneratorWorkOrder(const GeneratorFunctionHandle &function_handle,
+ const std::size_t query_id,
InsertDestination *output_destination)
- : function_handle_(function_handle),
+ : WorkOrder(query_id),
+ function_handle_(function_handle),
output_destination_(DCHECK_NOTNULL(output_destination)) {}
~TableGeneratorWorkOrder() override {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/TextScanOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/TextScanOperator.cpp b/relational_operators/TextScanOperator.cpp
index 5ede6f7..8db5ef1 100644
--- a/relational_operators/TextScanOperator.cpp
+++ b/relational_operators/TextScanOperator.cpp
@@ -167,6 +167,7 @@ bool TextScanOperator::getAllWorkOrders(
container->addNormalWorkOrder(
new TextSplitWorkOrder(file,
process_escape_sequences_,
+ query_id_,
storage_manager,
op_index_,
scheduler_client_id,
@@ -185,6 +186,7 @@ bool TextScanOperator::getAllWorkOrders(
blob_work.size,
field_terminator_,
process_escape_sequences_,
+ query_id_,
output_destination,
storage_manager),
op_index_);
@@ -204,6 +206,7 @@ bool TextScanOperator::getAllWorkOrders(
new TextScanWorkOrder(file,
field_terminator_,
process_escape_sequences_,
+ query_id_,
output_destination,
storage_manager),
op_index_);
@@ -235,9 +238,11 @@ void TextScanOperator::receiveFeedbackMessage(const WorkOrder::FeedbackMessage &
TextScanWorkOrder::TextScanWorkOrder(const std::string &filename,
const char field_terminator,
const bool process_escape_sequences,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
- : is_file_(true),
+ : WorkOrder(query_id),
+ is_file_(true),
filename_(filename),
field_terminator_(field_terminator),
text_blob_(0),
@@ -253,9 +258,11 @@ TextScanWorkOrder::TextScanWorkOrder(const block_id text_blob,
const std::size_t text_size,
const char field_terminator,
const bool process_escape_sequences,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager)
- : is_file_(false),
+ : WorkOrder(query_id),
+ is_file_(false),
field_terminator_(field_terminator),
text_blob_(text_blob),
text_size_(text_size),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/TextScanOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/TextScanOperator.hpp b/relational_operators/TextScanOperator.hpp
index a2d4ced..1d0c04f 100644
--- a/relational_operators/TextScanOperator.hpp
+++ b/relational_operators/TextScanOperator.hpp
@@ -134,14 +134,17 @@ class TextScanOperator : public RelationalOperator {
* @param output_relation The output relation.
* @param output_destination_index The index of the InsertDestination in the
* QueryContext to insert tuples.
+ * @param query_id The ID of the query to which this operator belongs.
**/
TextScanOperator(const std::string &file_pattern,
const char field_terminator,
const bool process_escape_sequences,
const bool parallelize_load,
const CatalogRelation &output_relation,
- const QueryContext::insert_destination_id output_destination_index)
- : file_pattern_(file_pattern),
+ const QueryContext::insert_destination_id output_destination_index,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ file_pattern_(file_pattern),
field_terminator_(field_terminator),
process_escape_sequences_(process_escape_sequences),
parallelize_load_(parallelize_load),
@@ -202,6 +205,7 @@ class TextScanWorkOrder : public WorkOrder {
* the text file.
* @param process_escape_sequences Whether to decode escape sequences in the
* text file.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to insert tuples.
* @param storage_manager The StorageManager to use.
**/
@@ -209,6 +213,7 @@ class TextScanWorkOrder : public WorkOrder {
const std::string &filename,
const char field_terminator,
const bool process_escape_sequences,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager);
@@ -221,6 +226,7 @@ class TextScanWorkOrder : public WorkOrder {
* the text file.
* @param process_escape_sequences Whether to decode escape sequences in the
* text file.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param output_destination The InsertDestination to write the read tuples.
* @param storage_manager The StorageManager to use.
*/
@@ -229,6 +235,7 @@ class TextScanWorkOrder : public WorkOrder {
const std::size_t text_size,
const char field_terminator,
const bool process_escape_sequences,
+ const std::size_t query_id,
InsertDestination *output_destination,
StorageManager *storage_manager);
@@ -318,6 +325,7 @@ class TextSplitWorkOrder : public WorkOrder {
* @param filename File to split into row-aligned blobs.
* @param process_escape_sequences Whether to decode escape sequences in the
* text file.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param storage_manager The StorageManager to use.
* @param operator_index Operator index of the current operator. This is used
* to send new-work available message to Foreman.
@@ -326,11 +334,13 @@ class TextSplitWorkOrder : public WorkOrder {
*/
TextSplitWorkOrder(const std::string &filename,
const bool process_escape_sequences,
+ const std::size_t query_id,
StorageManager *storage_manager,
const std::size_t operator_index,
const tmb::client_id scheduler_client_id,
MessageBus *bus)
- : filename_(filename),
+ : WorkOrder(query_id),
+ filename_(filename),
process_escape_sequences_(process_escape_sequences),
storage_manager_(DCHECK_NOTNULL(storage_manager)),
operator_index_(operator_index),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/UpdateOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/UpdateOperator.cpp b/relational_operators/UpdateOperator.cpp
index 7585db1..b331a9c 100644
--- a/relational_operators/UpdateOperator.cpp
+++ b/relational_operators/UpdateOperator.cpp
@@ -57,6 +57,7 @@ bool UpdateOperator::getAllWorkOrders(
input_block_id,
query_context->getPredicate(predicate_index_),
query_context->getUpdateGroup(update_group_index_),
+ query_id_,
query_context->getInsertDestination(relocation_destination_index_),
storage_manager,
op_index_,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/UpdateOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/UpdateOperator.hpp b/relational_operators/UpdateOperator.hpp
index 78f8fe0..ba2d6cf 100644
--- a/relational_operators/UpdateOperator.hpp
+++ b/relational_operators/UpdateOperator.hpp
@@ -72,6 +72,7 @@ class UpdateOperator : public RelationalOperator {
* @param update_group_index The index of a update group (the map of
* attribute_ids to Scalars) which should be evaluated to get the new
* value for the corresponding attribute.
+ * @param query_id The ID of the query to which this operator belongs.
*
* @warning The constructed InsertDestination should belong to relation, but
* must NOT contain any pre-existing blocks.
@@ -79,8 +80,10 @@ class UpdateOperator : public RelationalOperator {
UpdateOperator(const CatalogRelation &relation,
const QueryContext::insert_destination_id relocation_destination_index,
const QueryContext::predicate_id predicate_index,
- const QueryContext::update_group_id update_group_index)
- : relation_(relation),
+ const QueryContext::update_group_id update_group_index,
+ const std::size_t query_id)
+ : RelationalOperator(query_id),
+ relation_(relation),
relocation_destination_index_(relocation_destination_index),
predicate_index_(predicate_index),
update_group_index_(update_group_index),
@@ -130,6 +133,7 @@ class UpdateWorkOrder : public WorkOrder {
* @param assignments The assignments (the map of attribute_ids to Scalars)
* which should be evaluated to get the new value for the corresponding
* attribute.
+ * @param query_id The ID of the query to which this WorkOrder belongs.
* @param input_block_id The block id.
* @param relocation_destination The InsertDestination to relocate tuples
* which can not be updated in-place.
@@ -143,12 +147,14 @@ class UpdateWorkOrder : public WorkOrder {
const block_id input_block_id,
const Predicate *predicate,
const std::unordered_map<attribute_id, std::unique_ptr<const Scalar>> &assignments,
+ const std::size_t query_id,
InsertDestination *relocation_destination,
StorageManager *storage_manager,
const std::size_t update_operator_index,
const tmb::client_id scheduler_client_id,
MessageBus *bus)
- : relation_(relation),
+ : WorkOrder(query_id),
+ relation_(relation),
input_block_id_(input_block_id),
predicate_(predicate),
assignments_(assignments),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/WorkOrder.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrder.hpp b/relational_operators/WorkOrder.hpp
index 42cec2a..059865d 100644
--- a/relational_operators/WorkOrder.hpp
+++ b/relational_operators/WorkOrder.hpp
@@ -286,8 +286,15 @@ class WorkOrder {
}
protected:
- WorkOrder() {}
+ /**
+ * @brief Constructor.
+ *
+ * @param query_id The ID of the query to which this WorkOrder belongs.
+ **/
+ explicit WorkOrder(const std::size_t query_id)
+ : query_id_(query_id) {}
+ const std::size_t query_id_;
// A vector of preferred NUMA node IDs where this workorder should be executed.
// These node IDs typically indicate the NUMA node IDs of the input(s) of the
// workorder. Derived classes should ensure that there are no duplicate entries
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/WorkOrder.proto
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrder.proto b/relational_operators/WorkOrder.proto
index 5d0619a..fd731f7 100644
--- a/relational_operators/WorkOrder.proto
+++ b/relational_operators/WorkOrder.proto
@@ -45,6 +45,7 @@ enum WorkOrderType {
message WorkOrder {
required WorkOrderType work_order_type = 1;
+ required uint64 query_id = 2;
// The convention for extension numbering is that extensions for a particular
// WorkOrderID should begin from (operator_type + 1) * 16.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/WorkOrderFactory.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrderFactory.cpp b/relational_operators/WorkOrderFactory.cpp
index 4157d0f..fdd694f 100644
--- a/relational_operators/WorkOrderFactory.cpp
+++ b/relational_operators/WorkOrderFactory.cpp
@@ -76,6 +76,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
LOG(INFO) << "Creating AggregationWorkOrder";
return new AggregationWorkOrder(
proto.GetExtension(serialization::AggregationWorkOrder::block_id),
+ proto.query_id(),
query_context->getAggregationState(
proto.GetExtension(serialization::AggregationWorkOrder::aggr_state_index)));
}
@@ -93,6 +94,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
move(join_key_attributes),
proto.GetExtension(serialization::BuildHashWorkOrder::any_join_key_attributes_nullable),
proto.GetExtension(serialization::BuildHashWorkOrder::block_id),
+ proto.query_id(),
query_context->getJoinHashTable(
proto.GetExtension(serialization::BuildHashWorkOrder::join_hash_table_index)),
storage_manager);
@@ -108,12 +110,15 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
storage_manager,
proto.GetExtension(serialization::DeleteWorkOrder::operator_index),
shiftboss_client_id,
+ proto.query_id(),
bus);
}
case serialization::DESTROY_HASH: {
LOG(INFO) << "Creating DestroyHashWorkOrder";
return new DestroyHashWorkOrder(
- proto.GetExtension(serialization::DestroyHashWorkOrder::join_hash_table_index),
+ proto.GetExtension(
+ serialization::DestroyHashWorkOrder::join_hash_table_index),
+ proto.query_id(),
query_context);
}
case serialization::DROP_TABLE: {
@@ -125,6 +130,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
}
return new DropTableWorkOrder(
+ proto.query_id(),
move(blocks),
storage_manager,
proto.HasExtension(serialization::DropTableWorkOrder::relation_id)
@@ -135,10 +141,12 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
case serialization::FINALIZE_AGGREGATION: {
LOG(INFO) << "Creating FinalizeAggregationWorkOrder";
return new FinalizeAggregationWorkOrder(
- query_context->releaseAggregationState(
- proto.GetExtension(serialization::FinalizeAggregationWorkOrder::aggr_state_index)),
+ proto.query_id(),
+ query_context->releaseAggregationState(proto.GetExtension(
+ serialization::FinalizeAggregationWorkOrder::aggr_state_index)),
query_context->getInsertDestination(
- proto.GetExtension(serialization::FinalizeAggregationWorkOrder::insert_destination_index)));
+ proto.GetExtension(serialization::FinalizeAggregationWorkOrder::
+ insert_destination_index)));
}
case serialization::HASH_JOIN: {
const auto hash_join_work_order_type =
@@ -193,6 +201,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
residual_predicate,
selection,
hash_table,
+ proto.query_id(),
output_destination,
storage_manager);
}
@@ -207,6 +216,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
residual_predicate,
selection,
hash_table,
+ proto.query_id(),
output_destination,
storage_manager);
}
@@ -229,6 +239,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
selection,
move(is_selection_on_build),
hash_table,
+ proto.query_id(),
output_destination,
storage_manager);
}
@@ -243,6 +254,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
residual_predicate,
selection,
hash_table,
+ proto.query_id(),
output_destination,
storage_manager);
}
@@ -253,6 +265,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
case serialization::INSERT: {
LOG(INFO) << "Creating InsertWorkOrder";
return new InsertWorkOrder(
+ proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::InsertWorkOrder::insert_destination_index)),
query_context->releaseTuple(
@@ -271,6 +284,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::NestedLoopsJoinWorkOrder::join_predicate_index)),
query_context->getScalarGroup(
proto.GetExtension(serialization::NestedLoopsJoinWorkOrder::selection_index)),
+ proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::NestedLoopsJoinWorkOrder::insert_destination_index)),
storage_manager);
@@ -283,6 +297,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::SampleWorkOrder::block_id),
proto.GetExtension(serialization::SampleWorkOrder::is_block_sample),
proto.GetExtension(serialization::SampleWorkOrder::percentage),
+ proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::SampleWorkOrder::insert_destination_index)),
storage_manager);
@@ -292,6 +307,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
return new SaveBlocksWorkOrder(
proto.GetExtension(serialization::SaveBlocksWorkOrder::block_id),
proto.GetExtension(serialization::SaveBlocksWorkOrder::force),
+ proto.query_id(),
storage_manager);
}
case serialization::SELECT: {
@@ -315,6 +331,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
simple_projection ? nullptr
: &query_context->getScalarGroup(
proto.GetExtension(serialization::SelectWorkOrder::selection_index)),
+ proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::SelectWorkOrder::insert_destination_index)),
storage_manager);
@@ -340,6 +357,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
move(runs),
proto.GetExtension(serialization::SortMergeRunWorkOrder::top_k),
proto.GetExtension(serialization::SortMergeRunWorkOrder::merge_level),
+ proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::SortMergeRunWorkOrder::insert_destination_index)),
storage_manager,
@@ -355,6 +373,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::SortRunGenerationWorkOrder::block_id),
query_context->getSortConfig(
proto.GetExtension(serialization::SortRunGenerationWorkOrder::sort_config_index)),
+ proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::SortRunGenerationWorkOrder::insert_destination_index)),
storage_manager);
@@ -364,6 +383,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
return new TableGeneratorWorkOrder(
query_context->getGeneratorFunctionHandle(
proto.GetExtension(serialization::TableGeneratorWorkOrder::generator_function_index)),
+ proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::TableGeneratorWorkOrder::insert_destination_index)));
}
@@ -374,6 +394,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::TextScanWorkOrder::filename),
proto.GetExtension(serialization::TextScanWorkOrder::field_terminator),
proto.GetExtension(serialization::TextScanWorkOrder::process_escape_sequences),
+ proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::TextScanWorkOrder::insert_destination_index)),
storage_manager);
@@ -386,6 +407,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
text_blob_proto.size(),
proto.GetExtension(serialization::TextScanWorkOrder::field_terminator),
proto.GetExtension(serialization::TextScanWorkOrder::process_escape_sequences),
+ proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::TextScanWorkOrder::insert_destination_index)),
storage_manager);
@@ -396,6 +418,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
return new TextSplitWorkOrder(
proto.GetExtension(serialization::TextSplitWorkOrder::filename),
proto.GetExtension(serialization::TextSplitWorkOrder::process_escape_sequences),
+ proto.query_id(),
storage_manager,
proto.GetExtension(serialization::TextSplitWorkOrder::operator_index),
shiftboss_client_id,
@@ -411,6 +434,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::UpdateWorkOrder::predicate_index)),
query_context->getUpdateGroup(
proto.GetExtension(serialization::UpdateWorkOrder::update_group_index)),
+ proto.query_id(),
query_context->getInsertDestination(
proto.GetExtension(serialization::UpdateWorkOrder::insert_destination_index)),
storage_manager,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/tests/AggregationOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/AggregationOperator_unittest.cpp b/relational_operators/tests/AggregationOperator_unittest.cpp
index f2207c2..ace7951 100644
--- a/relational_operators/tests/AggregationOperator_unittest.cpp
+++ b/relational_operators/tests/AggregationOperator_unittest.cpp
@@ -270,7 +270,7 @@ class AggregationOperatorTest : public ::testing::Test {
aggr_state_proto->set_estimated_num_entries(estimated_entries);
// Create Operators.
- op_.reset(new AggregationOperator(*table_, true, aggr_state_index));
+ op_.reset(new AggregationOperator(*table_, true, aggr_state_index, 0));
// Setup the InsertDestination proto in the query context proto.
const QueryContext::insert_destination_id insert_destination_index =
@@ -281,8 +281,10 @@ class AggregationOperatorTest : public ::testing::Test {
insert_destination_proto->set_relation_id(result_table_->getID());
insert_destination_proto->set_relational_op_index(kOpIndex);
- finalize_op_.reset(
- new FinalizeAggregationOperator(aggr_state_index, *result_table_, insert_destination_index));
+ finalize_op_.reset(new FinalizeAggregationOperator(aggr_state_index,
+ *result_table_,
+ insert_destination_index,
+ 0 /* dummy query ID */));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -352,7 +354,7 @@ class AggregationOperatorTest : public ::testing::Test {
serialization::HashTableImplType::LINEAR_OPEN_ADDRESSING);
// Create Operators.
- op_.reset(new AggregationOperator(*table_, true, aggr_state_index));
+ op_.reset(new AggregationOperator(*table_, true, aggr_state_index, 0));
// Setup the InsertDestination proto in the query context proto.
const QueryContext::insert_destination_id insert_destination_index =
@@ -363,8 +365,10 @@ class AggregationOperatorTest : public ::testing::Test {
insert_destination_proto->set_relation_id(result_table_->getID());
insert_destination_proto->set_relational_op_index(kOpIndex);
- finalize_op_.reset(
- new FinalizeAggregationOperator(aggr_state_index, *result_table_, insert_destination_index));
+ finalize_op_.reset(new FinalizeAggregationOperator(aggr_state_index,
+ *result_table_,
+ insert_destination_index,
+ 0 /* dummy query ID */));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/tests/HashJoinOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/HashJoinOperator_unittest.cpp b/relational_operators/tests/HashJoinOperator_unittest.cpp
index 333c3f0..4ef5a5c 100644
--- a/relational_operators/tests/HashJoinOperator_unittest.cpp
+++ b/relational_operators/tests/HashJoinOperator_unittest.cpp
@@ -336,7 +336,8 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) {
true /* is_stored */,
std::vector<attribute_id>(1, dim_col_long.getID()),
dim_col_long.getType().isNullable(),
- join_hash_table_index));
+ join_hash_table_index,
+ 0)); // dummy query ID
// Create the prober operator with one selection attribute.
const QueryContext::scalar_group_id selection_index = query_context_proto.scalar_groups_size();
@@ -368,7 +369,8 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) {
output_destination_index,
join_hash_table_index,
QueryContext::kInvalidPredicateId /* residual_predicate_index */,
- selection_index));
+ selection_index,
+ 0 /* dummy query ID */));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -421,7 +423,7 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) {
}
// Create cleaner operator.
- unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index));
+ unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index, 0 /* dummy query ID */));
cleaner->informAllBlockingDependenciesMet();
fetchAndExecuteWorkOrders(cleaner.get());
@@ -477,7 +479,8 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) {
true /* is_stored */,
std::vector<attribute_id>(1, dim_col_int.getID()),
dim_col_int.getType().isNullable(),
- join_hash_table_index));
+ join_hash_table_index,
+ 0)); // dummy query ID
// Create the prober operator with two selection attributes.
const QueryContext::scalar_group_id selection_index = query_context_proto.scalar_groups_size();
@@ -514,7 +517,8 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) {
output_destination_index,
join_hash_table_index,
QueryContext::kInvalidPredicateId /* residual_predicate_index */,
- selection_index));
+ selection_index,
+ 0 /* dummy query ID */));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -588,7 +592,7 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) {
}
// Create cleaner operator.
- unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index));
+ unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index, 0 /* dummy query ID */));
cleaner->informAllBlockingDependenciesMet();
fetchAndExecuteWorkOrders(cleaner.get());
@@ -636,7 +640,8 @@ TEST_P(HashJoinOperatorTest, CharKeyCartesianProductHashJoinTest) {
true /* is_stored */,
std::vector<attribute_id>(1, dim_col_char.getID()),
dim_col_char.getType().isNullable(),
- join_hash_table_index));
+ join_hash_table_index,
+ 0)); // dummy query ID.
// Create prober operator with one selection attribute.
const QueryContext::scalar_group_id selection_index = query_context_proto.scalar_groups_size();
@@ -668,7 +673,8 @@ TEST_P(HashJoinOperatorTest, CharKeyCartesianProductHashJoinTest) {
output_destination_index,
join_hash_table_index,
QueryContext::kInvalidPredicateId /* residual_predicate_index */,
- selection_index));
+ selection_index,
+ 0 /* dummy query ID */));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -721,7 +727,7 @@ TEST_P(HashJoinOperatorTest, CharKeyCartesianProductHashJoinTest) {
}
// Create cleaner operator.
- unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index));
+ unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index, 0 /* dummy query ID */));
cleaner->informAllBlockingDependenciesMet();
fetchAndExecuteWorkOrders(cleaner.get());
@@ -770,7 +776,8 @@ TEST_P(HashJoinOperatorTest, VarCharDuplicateKeyHashJoinTest) {
true /* is_stored */,
std::vector<attribute_id>(1, dim_col_varchar.getID()),
dim_col_varchar.getType().isNullable(),
- join_hash_table_index));
+ join_hash_table_index,
+ 0)); // dummy query ID.
// Create prober operator with two selection attributes.
const QueryContext::scalar_group_id selection_index = query_context_proto.scalar_groups_size();
@@ -807,7 +814,8 @@ TEST_P(HashJoinOperatorTest, VarCharDuplicateKeyHashJoinTest) {
output_destination_index,
join_hash_table_index,
QueryContext::kInvalidPredicateId /* residual_predicate_index */,
- selection_index));
+ selection_index,
+ 0 /* dummy query ID */));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -885,7 +893,7 @@ TEST_P(HashJoinOperatorTest, VarCharDuplicateKeyHashJoinTest) {
}
// Create the cleaner operator.
- unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index));
+ unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index, 0 /* dummy query ID */));
cleaner->informAllBlockingDependenciesMet();
fetchAndExecuteWorkOrders(cleaner.get());
@@ -939,7 +947,8 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) {
true /* is_stored */,
dim_key_attrs,
dim_col_long.getType().isNullable() || dim_col_varchar.getType().isNullable(),
- join_hash_table_index));
+ join_hash_table_index,
+ 0)); // dummy query ID.
// Create the prober operator with two selection attributes.
const QueryContext::scalar_group_id selection_index = query_context_proto.scalar_groups_size();
@@ -980,7 +989,8 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) {
output_destination_index,
join_hash_table_index,
QueryContext::kInvalidPredicateId /* residual_predicate_index */,
- selection_index));
+ selection_index,
+ 0 /* dummy query ID */));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -1058,7 +1068,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) {
}
// Create cleaner operator.
- unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index));
+ unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index, 0 /* dummy query ID */));
cleaner->informAllBlockingDependenciesMet();
fetchAndExecuteWorkOrders(cleaner.get());
@@ -1113,7 +1123,8 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinWithResidualPredicateTest) {
true /* is_stored */,
dim_key_attrs,
dim_col_long.getType().isNullable() || dim_col_varchar.getType().isNullable(),
- join_hash_table_index));
+ join_hash_table_index,
+ 0)); // dummy query ID.
// Create prober operator with two selection attributes.
const QueryContext::scalar_group_id selection_index = query_context_proto.scalar_groups_size();
@@ -1164,7 +1175,8 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinWithResidualPredicateTest) {
output_destination_index,
join_hash_table_index,
residual_pred_index,
- selection_index));
+ selection_index,
+ 0 /* dummy query ID */));
// Set up the QueryContext.
query_context_.reset(new QueryContext(query_context_proto,
@@ -1242,7 +1254,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinWithResidualPredicateTest) {
}
// Create cleaner operator.
- unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index));
+ unique_ptr<DestroyHashOperator> cleaner(new DestroyHashOperator(join_hash_table_index, 0 /* dummy query ID */));
cleaner->informAllBlockingDependenciesMet();
fetchAndExecuteWorkOrders(cleaner.get());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/tests/SortMergeRunOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/SortMergeRunOperator_unittest.cpp b/relational_operators/tests/SortMergeRunOperator_unittest.cpp
index 50c508d..244091f 100644
--- a/relational_operators/tests/SortMergeRunOperator_unittest.cpp
+++ b/relational_operators/tests/SortMergeRunOperator_unittest.cpp
@@ -1573,7 +1573,8 @@ class SortMergeRunOperatorTest : public ::testing::Test {
sort_config_index,
merge_factor,
top_k,
- true));
+ true,
+ 0 /* dummy query ID */));
merge_op_->setOperatorIndex(kOpIndex);
// Set up the QueryContext.
@@ -1616,7 +1617,8 @@ class SortMergeRunOperatorTest : public ::testing::Test {
sort_config_index,
merge_factor,
top_k,
- false));
+ false,
+ 0 /* dummy query ID */));
merge_op_->setOperatorIndex(kOpIndex);
// Set up the QueryContext.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/SortRunGenerationOperator_unittest.cpp b/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
index 7491778..6f24b92 100644
--- a/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
+++ b/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
@@ -358,7 +358,8 @@ class SortRunGenerationOperatorTest : public ::testing::Test {
*result_table_,
insert_destination_index,
sort_config_index,
- true /* is_stored */));
+ true /* is_stored */,
+ 0 /* dummy query ID */));
run_gen->setOperatorIndex(kOpIndex);
// Set up the QueryContext.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40542682/relational_operators/tests/TextScanOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/TextScanOperator_unittest.cpp b/relational_operators/tests/TextScanOperator_unittest.cpp
index 1dfad7b..7626686 100644
--- a/relational_operators/tests/TextScanOperator_unittest.cpp
+++ b/relational_operators/tests/TextScanOperator_unittest.cpp
@@ -193,7 +193,8 @@ TEST_F(TextScanOperatorTest, ScanTest) {
true,
false,
*relation_,
- output_destination_index));
+ output_destination_index,
+ 0 /* dummy query ID */));
// Setup query_context_.
query_context_.reset(new QueryContext(query_context_proto,
[17/48] incubator-quickstep git commit: Enable semi-join optimization
for left-deep trees through bloom filters (#195)
Posted by hb...@apache.org.
Enable semi-join optimization for left-deep trees through bloom filters (#195)
Link: https://github.com/pivotalsoftware/quickstep/pull/195
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/2c07d709
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/2c07d709
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/2c07d709
Branch: refs/heads/query-manager-used-in-foreman
Commit: 2c07d709f44bf8b7a3654412748d1e6df996543e
Parents: 44ca622
Author: Saket Saurabh <sa...@users.noreply.github.com>
Authored: Mon May 9 12:29:47 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:42 2016 -0700
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 2 +
query_execution/QueryContext.cpp | 16 +-
query_execution/QueryContext.hpp | 57 +++-
query_execution/QueryContext.proto | 18 +-
query_optimizer/CMakeLists.txt | 11 +
query_optimizer/ExecutionGenerator.cpp | 55 +++-
query_optimizer/ExecutionGenerator.hpp | 5 +-
query_optimizer/ExecutionHeuristics.cpp | 127 ++++++++
query_optimizer/ExecutionHeuristics.hpp | 155 ++++++++++
query_optimizer/tests/CMakeLists.txt | 16 +
.../tests/ExecutionHeuristics_unittest.cpp | 301 +++++++++++++++++++
storage/CMakeLists.txt | 4 +-
storage/HashTable.hpp | 104 +++++++
storage/HashTable.proto | 8 +
storage/HashTableFactory.hpp | 44 ++-
utility/BloomFilter.hpp | 198 +++++++++++-
utility/BloomFilter.proto | 30 ++
utility/CMakeLists.txt | 15 +
18 files changed, 1135 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 5887237..04a0348 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -90,9 +90,11 @@ target_link_libraries(quickstep_queryexecution_QueryContext
quickstep_storage_InsertDestination_proto
quickstep_types_TypedValue
quickstep_types_containers_Tuple
+ quickstep_utility_BloomFilter
quickstep_utility_Macros
quickstep_utility_SortConfiguration)
target_link_libraries(quickstep_queryexecution_QueryContext_proto
+ quickstep_utility_BloomFilter_proto
quickstep_expressions_Expressions_proto
quickstep_expressions_tablegenerator_GeneratorFunction_proto
quickstep_storage_AggregationOperationState_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/query_execution/QueryContext.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.cpp b/query_execution/QueryContext.cpp
index b0e9cae..3bfce17 100644
--- a/query_execution/QueryContext.cpp
+++ b/query_execution/QueryContext.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -37,6 +39,7 @@
#include "storage/InsertDestination.pb.h"
#include "types/TypedValue.hpp"
#include "types/containers/Tuple.hpp"
+#include "utility/BloomFilter.hpp"
#include "utility/SortConfiguration.hpp"
#include "glog/logging.h"
@@ -65,6 +68,10 @@ QueryContext::QueryContext(const serialization::QueryContext &proto,
storage_manager));
}
+ for (int i = 0; i < proto.bloom_filters_size(); ++i) {
+ bloom_filters_.emplace_back(new BloomFilter(proto.bloom_filters(i)));
+ }
+
for (int i = 0; i < proto.generator_functions_size(); ++i) {
const GeneratorFunctionHandle *func_handle =
GeneratorFunctionFactory::Instance().reconstructFromProto(proto.generator_functions(i));
@@ -76,7 +83,8 @@ QueryContext::QueryContext(const serialization::QueryContext &proto,
for (int i = 0; i < proto.join_hash_tables_size(); ++i) {
join_hash_tables_.emplace_back(
JoinHashTableFactory::CreateResizableFromProto(proto.join_hash_tables(i),
- storage_manager));
+ storage_manager,
+ bloom_filters_));
}
for (int i = 0; i < proto.insert_destinations_size(); ++i) {
@@ -142,6 +150,12 @@ bool QueryContext::ProtoIsValid(const serialization::QueryContext &proto,
}
}
+ for (int i = 0; i < proto.bloom_filters_size(); ++i) {
+ if (!BloomFilter::ProtoIsValid(proto.bloom_filters(i))) {
+ return false;
+ }
+ }
+
// Each GeneratorFunctionHandle object is serialized as a function name with
// a list of arguments. Here checks that the arguments are valid TypedValue's.
for (int i = 0; i < proto.generator_functions_size(); ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/query_execution/QueryContext.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp
index 0e9e21c..9440fae 100644
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -32,6 +34,7 @@
#include "storage/HashTable.hpp"
#include "storage/InsertDestination.hpp"
#include "types/containers/Tuple.hpp"
+#include "utility/BloomFilter.hpp"
#include "utility/Macros.hpp"
#include "utility/SortConfiguration.hpp"
@@ -63,6 +66,11 @@ class QueryContext {
typedef std::uint32_t aggregation_state_id;
/**
+ * @brief A unique identifier for a BloomFilter per query.
+ **/
+ typedef std::uint32_t bloom_filter_id;
+
+ /**
* @brief A unique identifier for a GeneratorFunctionHandle per query.
**/
typedef std::uint32_t generator_function_id;
@@ -181,6 +189,52 @@ class QueryContext {
}
/**
+ * @brief Whether the given BloomFilter id is valid.
+ *
+ * @param id The BloomFilter id.
+ *
+ * @return True if valid, otherwise false.
+ **/
+ bool isValidBloomFilterId(const bloom_filter_id id) const {
+ return id < bloom_filters_.size();
+ }
+
+ /**
+ * @brief Get a mutable reference to the BloomFilter.
+ *
+ * @param id The BloomFilter id.
+ *
+ * @return The BloomFilter, already created in the constructor.
+ **/
+ inline BloomFilter* getBloomFilterMutable(const bloom_filter_id id) {
+ DCHECK_LT(id, bloom_filters_.size());
+ return bloom_filters_[id].get();
+ }
+
+ /**
+ * @brief Get a constant pointer to the BloomFilter.
+ *
+ * @param id The BloomFilter id.
+ *
+ * @return The constant pointer to BloomFilter that is
+ * already created in the constructor.
+ **/
+ inline const BloomFilter* getBloomFilter(const bloom_filter_id id) const {
+ DCHECK_LT(id, bloom_filters_.size());
+ return bloom_filters_[id].get();
+ }
+
+ /**
+ * @brief Destory the given BloomFilter.
+ *
+ * @param id The id of the BloomFilter to destroy.
+ **/
+ inline void destroyBloomFilter(const bloom_filter_id id) {
+ DCHECK_LT(id, bloom_filters_.size());
+ bloom_filters_[id].reset();
+ }
+
+ /**
* @brief Whether the given GeneratorFunctionHandle id is valid.
*
* @param id The GeneratorFunctionHandle id.
@@ -257,7 +311,7 @@ class QueryContext {
*
* @param id The JoinHashTable id in the query.
*
- * @return The JoinHashTable, alreadly created in the constructor.
+ * @return The JoinHashTable, already created in the constructor.
**/
inline JoinHashTable* getJoinHashTable(const join_hash_table_id id) {
DCHECK_LT(id, join_hash_tables_.size());
@@ -408,6 +462,7 @@ class QueryContext {
private:
std::vector<std::unique_ptr<AggregationOperationState>> aggregation_states_;
+ std::vector<std::unique_ptr<BloomFilter>> bloom_filters_;
std::vector<std::unique_ptr<const GeneratorFunctionHandle>> generator_functions_;
std::vector<std::unique_ptr<InsertDestination>> insert_destinations_;
std::vector<std::unique_ptr<JoinHashTable>> join_hash_tables_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/query_execution/QueryContext.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.proto b/query_execution/QueryContext.proto
index a7c2a25..b37286c 100644
--- a/query_execution/QueryContext.proto
+++ b/query_execution/QueryContext.proto
@@ -23,6 +23,7 @@ import "storage/AggregationOperationState.proto";
import "storage/HashTable.proto";
import "storage/InsertDestination.proto";
import "types/containers/Tuple.proto";
+import "utility/BloomFilter.proto";
import "utility/SortConfiguration.proto";
message QueryContext {
@@ -42,14 +43,15 @@ message QueryContext {
}
repeated AggregationOperationState aggregation_states = 1;
- repeated HashTable join_hash_tables = 2;
- repeated InsertDestination insert_destinations = 3;
- repeated Predicate predicates = 4;
- repeated ScalarGroup scalar_groups = 5;
- repeated SortConfiguration sort_configs = 6;
- repeated Tuple tuples = 7;
- repeated GeneratorFunctionHandle generator_functions = 8;
+ repeated BloomFilter bloom_filters = 2;
+ repeated GeneratorFunctionHandle generator_functions = 3;
+ repeated HashTable join_hash_tables = 4;
+ repeated InsertDestination insert_destinations = 5;
+ repeated Predicate predicates = 6;
+ repeated ScalarGroup scalar_groups = 7;
+ repeated SortConfiguration sort_configs = 8;
+ repeated Tuple tuples = 9;
// NOTE(zuyu): For UpdateWorkOrder only.
- repeated UpdateGroup update_groups = 9;
+ repeated UpdateGroup update_groups = 10;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 2d09bee..feaecb3 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -35,6 +35,7 @@ add_subdirectory(tests)
# Declare micro-libs:
add_library(quickstep_queryoptimizer_ExecutionGenerator ExecutionGenerator.cpp ExecutionGenerator.hpp)
+add_library(quickstep_queryoptimizer_ExecutionHeuristics ExecutionHeuristics.cpp ExecutionHeuristics.hpp)
add_library(quickstep_queryoptimizer_LogicalGenerator LogicalGenerator.cpp LogicalGenerator.hpp)
add_library(quickstep_queryoptimizer_LogicalToPhysicalMapper
../empty_src.cpp
@@ -64,6 +65,7 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_expressions_scalar_ScalarAttribute
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryContext_proto
+ quickstep_queryoptimizer_ExecutionHeuristics
quickstep_queryoptimizer_OptimizerContext
quickstep_queryoptimizer_QueryHandle
quickstep_queryoptimizer_QueryPlan
@@ -139,6 +141,14 @@ if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_catalog_Catalog_proto)
endif()
+target_link_libraries(quickstep_queryoptimizer_ExecutionHeuristics
+ glog
+ quickstep_catalog_CatalogRelation
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_QueryContext_proto
+ quickstep_queryoptimizer_QueryPlan
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_LogicalGenerator
glog
quickstep_parser_ParseStatement
@@ -211,6 +221,7 @@ target_link_libraries(quickstep_queryoptimizer_Validator
add_library(quickstep_queryoptimizer ../empty_src.cpp QueryOptimizerModule.hpp)
target_link_libraries(quickstep_queryoptimizer
quickstep_queryoptimizer_ExecutionGenerator
+ quickstep_queryoptimizer_ExecutionHeuristics
quickstep_queryoptimizer_LogicalGenerator
quickstep_queryoptimizer_LogicalToPhysicalMapper
quickstep_queryoptimizer_Optimizer
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 077d35d..7f26e85 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -50,6 +50,7 @@
#include "expressions/scalar/ScalarAttribute.hpp"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/ExecutionHeuristics.hpp"
#include "query_optimizer/OptimizerContext.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryPlan.hpp"
@@ -144,6 +145,9 @@ static const volatile bool aggregate_hashtable_type_dummy
DEFINE_bool(parallelize_load, true, "Parallelize loading data files.");
+DEFINE_bool(optimize_joins, false,
+ "Enable post execution plan generation optimizations for joins.");
+
namespace E = ::quickstep::optimizer::expressions;
namespace P = ::quickstep::optimizer::physical;
namespace S = ::quickstep::serialization;
@@ -198,6 +202,11 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) {
temporary_relation_info.producer_operator_index);
}
+ // Optimize execution plan based on heuristics captured during execution plan generation, if enabled.
+ if (FLAGS_optimize_joins) {
+ execution_heuristics_->optimizeExecutionPlan(execution_plan_, query_context_proto_);
+ }
+
#ifdef QUICKSTEP_DISTRIBUTED
catalog_database_cache_proto_->set_name(optimizer_context_->catalog_database()->getName());
@@ -576,12 +585,32 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::vector<attribute_id> probe_attribute_ids;
std::vector<attribute_id> build_attribute_ids;
+ std::vector<attribute_id> probe_original_attribute_ids;
+ std::vector<attribute_id> build_original_attribute_ids;
+
+ const CatalogRelation *referenced_stored_probe_relation;
+ const CatalogRelation *referenced_stored_build_relation;
+
bool any_probe_attributes_nullable = false;
bool any_build_attributes_nullable = false;
+ bool skip_hash_join_optimization = false;
+
const std::vector<E::AttributeReferencePtr> &left_join_attributes =
physical_plan->left_join_attributes();
for (const E::AttributeReferencePtr &left_join_attribute : left_join_attributes) {
+ // Try to determine the original stored relation referenced in the Hash Join.
+ referenced_stored_probe_relation =
+ optimizer_context_->catalog_database()->getRelationByName(left_join_attribute->relation_name());
+ if (referenced_stored_probe_relation == nullptr) {
+ // Hash Join optimizations are not possible, if the referenced relation cannot be determined.
+ skip_hash_join_optimization = true;
+ } else {
+ const attribute_id probe_operator_attribute_id =
+ referenced_stored_probe_relation->getAttributeByName(left_join_attribute->attribute_name())->getID();
+ probe_original_attribute_ids.emplace_back(probe_operator_attribute_id);
+ }
+
const CatalogAttribute *probe_catalog_attribute
= attribute_substitution_map_[left_join_attribute->id()];
probe_attribute_ids.emplace_back(probe_catalog_attribute->getID());
@@ -594,6 +623,18 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
const std::vector<E::AttributeReferencePtr> &right_join_attributes =
physical_plan->right_join_attributes();
for (const E::AttributeReferencePtr &right_join_attribute : right_join_attributes) {
+ // Try to determine the original stored relation referenced in the Hash Join.
+ referenced_stored_build_relation =
+ optimizer_context_->catalog_database()->getRelationByName(right_join_attribute->relation_name());
+ if (referenced_stored_build_relation == nullptr) {
+ // Hash Join optimizations are not possible, if the referenced relation cannot be determined.
+ skip_hash_join_optimization = true;
+ } else {
+ const attribute_id build_operator_attribute_id =
+ referenced_stored_build_relation->getAttributeByName(right_join_attribute->attribute_name())->getID();
+ build_original_attribute_ids.emplace_back(build_operator_attribute_id);
+ }
+
const CatalogAttribute *build_catalog_attribute
= attribute_substitution_map_[right_join_attribute->id()];
build_attribute_ids.emplace_back(build_catalog_attribute->getID());
@@ -629,6 +670,8 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::swap(probe_cardinality, build_cardinality);
std::swap(probe_attribute_ids, build_attribute_ids);
std::swap(any_probe_attributes_nullable, any_build_attributes_nullable);
+ std::swap(probe_original_attribute_ids, build_original_attribute_ids);
+ std::swap(referenced_stored_probe_relation, referenced_stored_build_relation);
}
}
@@ -783,6 +826,17 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::forward_as_tuple(join_operator_index,
output_relation));
temporary_relation_info_vec_.emplace_back(join_operator_index, output_relation);
+
+ // Add heuristics for the Hash Join, if enabled.
+ if (FLAGS_optimize_joins && !skip_hash_join_optimization) {
+ execution_heuristics_->addHashJoinInfo(build_operator_index,
+ join_operator_index,
+ referenced_stored_build_relation,
+ referenced_stored_probe_relation,
+ std::move(build_original_attribute_ids),
+ std::move(probe_original_attribute_ids),
+ join_hash_table_index);
+ }
}
void ExecutionGenerator::convertNestedLoopsJoin(
@@ -895,7 +949,6 @@ void ExecutionGenerator::convertCopyFrom(
false /* is_pipeline_breaker */);
}
-
void ExecutionGenerator::convertCreateIndex(
const P::CreateIndexPtr &physical_plan) {
// CreateIndex is converted to a CreateIndex operator.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/query_optimizer/ExecutionGenerator.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.hpp b/query_optimizer/ExecutionGenerator.hpp
index 7c563d4..0630bca 100644
--- a/query_optimizer/ExecutionGenerator.hpp
+++ b/query_optimizer/ExecutionGenerator.hpp
@@ -33,6 +33,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/ExecutionHeuristics.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryPlan.hpp"
#include "query_optimizer/cost_model/CostModel.hpp"
@@ -102,7 +103,8 @@ class ExecutionGenerator {
: optimizer_context_(DCHECK_NOTNULL(optimizer_context)),
query_handle_(DCHECK_NOTNULL(query_handle)),
execution_plan_(DCHECK_NOTNULL(query_handle->getQueryPlanMutable())),
- query_context_proto_(DCHECK_NOTNULL(query_handle->getQueryContextProtoMutable())) {
+ query_context_proto_(DCHECK_NOTNULL(query_handle->getQueryContextProtoMutable())),
+ execution_heuristics_(new ExecutionHeuristics()) {
#ifdef QUICKSTEP_DISTRIBUTED
catalog_database_cache_proto_ = DCHECK_NOTNULL(query_handle->getCatalogDatabaseCacheProtoMutable());
#endif
@@ -376,6 +378,7 @@ class ExecutionGenerator {
QueryHandle *query_handle_;
QueryPlan *execution_plan_; // A part of QueryHandle.
serialization::QueryContext *query_context_proto_; // A part of QueryHandle.
+ std::unique_ptr<ExecutionHeuristics> execution_heuristics_;
#ifdef QUICKSTEP_DISTRIBUTED
serialization::CatalogDatabase *catalog_database_cache_proto_; // A part of QueryHandle.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/query_optimizer/ExecutionHeuristics.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionHeuristics.cpp b/query_optimizer/ExecutionHeuristics.cpp
new file mode 100644
index 0000000..fc31c53
--- /dev/null
+++ b/query_optimizer/ExecutionHeuristics.cpp
@@ -0,0 +1,127 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_optimizer/ExecutionHeuristics.hpp"
+
+#include <cstddef>
+#include <utility>
+#include <unordered_map>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/QueryPlan.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+
+void ExecutionHeuristics::optimizeExecutionPlan(QueryPlan *query_plan,
+ serialization::QueryContext *query_context_proto) {
+ // Currently this only optimizes left deep joins using bloom filters.
+ // It uses a simple algorithm to discover the left deep joins.
+ // It starts with the first hash join in the plan and keeps on iterating
+ // over the next hash joins, till a probe on a different relation id is found.
+ // The set of hash joins found in this way forms a chain and can be recognized
+ // as a left deep join. It becomes a candidate for optimization.
+
+ // The optimization is done by modifying each of the build operators in the chain
+ // to generate a bloom filter on the build key during their hash table creation.
+ // The leaf-level probe operator is then modified to query all the bloom
+ // filters generated from all the build operators in the chain. These
+ // bloom filters are queried to test the membership of the probe key
+ // just prior to probing the hash table.
+
+ QueryPlan::DAGNodeIndex origin_node = 0;
+ while (origin_node < hash_joins_.size() - 1) {
+ std::vector<std::size_t> chained_nodes;
+ chained_nodes.push_back(origin_node);
+ for (std::size_t i = origin_node + 1; i < hash_joins_.size(); ++i) {
+ const relation_id checked_relation_id = hash_joins_[origin_node].referenced_stored_probe_relation_->getID();
+ const relation_id expected_relation_id = hash_joins_[i].referenced_stored_probe_relation_->getID();
+ if (checked_relation_id == expected_relation_id) {
+ chained_nodes.push_back(i);
+ } else {
+ break;
+ }
+ }
+
+ // Only chains of length greater than one are suitable candidates for semi-join optimization.
+ if (chained_nodes.size() > 1) {
+ std::unordered_map<QueryContext::bloom_filter_id, std::vector<attribute_id>> probe_bloom_filter_info;
+ for (const std::size_t node : chained_nodes) {
+ // Provision for a new bloom filter to be used by the build operator.
+ const QueryContext::bloom_filter_id bloom_filter_id = query_context_proto->bloom_filters_size();
+ serialization::BloomFilter *bloom_filter_proto = query_context_proto->add_bloom_filters();
+
+ // Modify the bloom filter properties based on the statistics of the relation.
+ setBloomFilterProperties(bloom_filter_proto, hash_joins_[node].referenced_stored_build_relation_);
+
+ // Add build-side bloom filter information to the corresponding hash table proto.
+ query_context_proto->mutable_join_hash_tables(hash_joins_[node].join_hash_table_id_)
+ ->add_build_side_bloom_filter_id(bloom_filter_id);
+
+ probe_bloom_filter_info.insert(std::make_pair(bloom_filter_id, hash_joins_[node].probe_attributes_));
+ }
+
+ // Add probe-side bloom filter information to the corresponding hash table proto for each build-side bloom filter.
+ for (const std::pair<QueryContext::bloom_filter_id, std::vector<attribute_id>>
+ &bloom_filter_info : probe_bloom_filter_info) {
+ auto *probe_side_bloom_filter =
+ query_context_proto->mutable_join_hash_tables(hash_joins_[origin_node].join_hash_table_id_)
+ ->add_probe_side_bloom_filters();
+ probe_side_bloom_filter->set_probe_side_bloom_filter_id(bloom_filter_info.first);
+ for (const attribute_id &probe_attribute_id : bloom_filter_info.second) {
+ probe_side_bloom_filter->add_probe_side_attr_ids(probe_attribute_id);
+ }
+ }
+
+ // Add node dependencies from chained build nodes to origin node probe.
+ for (std::size_t i = 1; i < chained_nodes.size(); ++i) { // Note: It starts from index 1.
+ query_plan->addDirectDependency(hash_joins_[origin_node].join_operator_index_,
+ hash_joins_[origin_node + i].build_operator_index_,
+ true /* is_pipeline_breaker */);
+ }
+ }
+
+ // Update the origin node.
+ origin_node = chained_nodes.back() + 1;
+ }
+}
+
+void ExecutionHeuristics::setBloomFilterProperties(serialization::BloomFilter *bloom_filter_proto,
+ const CatalogRelation *relation) {
+ const std::size_t cardinality = relation->estimateTupleCardinality();
+ if (cardinality < kOneThousand) {
+ bloom_filter_proto->set_bloom_filter_size(kOneThousand / kCompressionFactor);
+ bloom_filter_proto->set_number_of_hashes(kVeryLowSparsityHash);
+ } else if (cardinality < kTenThousand) {
+ bloom_filter_proto->set_bloom_filter_size(kTenThousand / kCompressionFactor);
+ bloom_filter_proto->set_number_of_hashes(kLowSparsityHash);
+ } else if (cardinality < kHundredThousand) {
+ bloom_filter_proto->set_bloom_filter_size(kHundredThousand / kCompressionFactor);
+ bloom_filter_proto->set_number_of_hashes(kMediumSparsityHash);
+ } else {
+ bloom_filter_proto->set_bloom_filter_size(kMillion / kCompressionFactor);
+ bloom_filter_proto->set_number_of_hashes(kHighSparsityHash);
+ }
+}
+
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/query_optimizer/ExecutionHeuristics.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionHeuristics.hpp b/query_optimizer/ExecutionHeuristics.hpp
new file mode 100644
index 0000000..92a7fe8
--- /dev/null
+++ b/query_optimizer/ExecutionHeuristics.hpp
@@ -0,0 +1,155 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_HEURISTICS_HPP_
+#define QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_HEURISTICS_HPP_
+
+#include <vector>
+
+#include "catalog/CatalogRelation.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/QueryPlan.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+
+/** \addtogroup QueryOptimizer
+ * @{
+ */
+
+/**
+ * @brief The ExecutionHeuristics compiles certain heuristics for an execution plan
+ * as it is being converted to a physical plan. These heuristics can then be
+ * used to optimize the execution plan after it has been generated.
+ **/
+class ExecutionHeuristics {
+ public:
+ static const std::size_t kOneHundred = 100;
+ static const std::size_t kOneThousand = 1000;
+ static const std::size_t kTenThousand = 10000;
+ static const std::size_t kHundredThousand = 100000;
+ static const std::size_t kMillion = 1000000;
+
+ static const std::size_t kCompressionFactor = 10;
+
+ static const std::size_t kVeryLowSparsityHash = 1;
+ static const std::size_t kLowSparsityHash = 2;
+ static const std::size_t kMediumSparsityHash = 5;
+ static const std::size_t kHighSparsityHash = 10;
+
+ /**
+ * @brief A simple internal class that holds information about various
+ * hash joins within the execution plan for a query.
+ **/
+ struct HashJoinInfo {
+ HashJoinInfo(const QueryPlan::DAGNodeIndex build_operator_index,
+ const QueryPlan::DAGNodeIndex join_operator_index,
+ const CatalogRelation *referenced_stored_build_relation,
+ const CatalogRelation *referenced_stored_probe_relation,
+ std::vector<attribute_id> &&build_attributes,
+ std::vector<attribute_id> &&probe_attributes,
+ const QueryContext::join_hash_table_id join_hash_table_id)
+ : build_operator_index_(build_operator_index),
+ join_operator_index_(join_operator_index),
+ referenced_stored_build_relation_(referenced_stored_build_relation),
+ referenced_stored_probe_relation_(referenced_stored_probe_relation),
+ build_attributes_(std::move(build_attributes)),
+ probe_attributes_(std::move(probe_attributes)),
+ join_hash_table_id_(join_hash_table_id) {
+ }
+
+ const QueryPlan::DAGNodeIndex build_operator_index_;
+ const QueryPlan::DAGNodeIndex join_operator_index_;
+ const CatalogRelation *referenced_stored_build_relation_;
+ const CatalogRelation *referenced_stored_probe_relation_;
+ const std::vector<attribute_id> build_attributes_;
+ const std::vector<attribute_id> probe_attributes_;
+ const QueryContext::join_hash_table_id join_hash_table_id_;
+ };
+
+
+ /**
+ * @brief Constructor.
+ **/
+ ExecutionHeuristics() {}
+
+ /**
+ * @brief Saves information about a hash join used within the execution plan
+ * for a query.
+ *
+ * @param build_operator_index Index of the build operator of the hash join.
+ * @param join_operator_index Index of the join operator of the hash join.
+ * @param build_relation_id Id of the relation on which hash table is being built.
+ * @param probe_relation_id Id of the relation on which hash table is being probed.
+ * @param build_attributes List of attributes on which hash table is being built.
+ * @param probe_attributes List of attributes on which hash table is being probed.
+ * @param join_hash_table_id Id of the hash table which refers to the actual hash
+ * table within the query context.
+ **/
+ inline void addHashJoinInfo(const QueryPlan::DAGNodeIndex build_operator_index,
+ const QueryPlan::DAGNodeIndex join_operator_index,
+ const CatalogRelation *referenced_stored_build_relation,
+ const CatalogRelation *referenced_stored_probe_relation,
+ std::vector<attribute_id> &&build_attributes,
+ std::vector<attribute_id> &&probe_attributes,
+ const QueryContext::join_hash_table_id join_hash_table_id) {
+ hash_joins_.push_back(HashJoinInfo(build_operator_index,
+ join_operator_index,
+ referenced_stored_build_relation,
+ referenced_stored_probe_relation,
+ std::move(build_attributes),
+ std::move(probe_attributes),
+ join_hash_table_id));
+ }
+
+ /**
+ * @brief Optimize the execution plan based on heuristics generated
+ * during physical plan to execution plan conversion.
+ *
+ * @param query_plan A mutable reference to the query execution plan.
+ * @param query_context_proto A mutable reference to the protobuf representation
+ * of the query context.
+ **/
+ void optimizeExecutionPlan(QueryPlan *query_plan, serialization::QueryContext *query_context_proto);
+
+ /**
+ * @brief Set the properties of the bloom filter proto based on the statistics
+ * of the given relation.
+ *
+ * @param bloom_filter_proto A mutable reference to the bloom filter protobuf representation.
+ * @param relation The catalog relation on which bloom filter is being built.
+ **/
+ void setBloomFilterProperties(serialization::BloomFilter *bloom_filter_proto,
+ const CatalogRelation *relation);
+
+ private:
+ std::vector<HashJoinInfo> hash_joins_;
+
+ DISALLOW_COPY_AND_ASSIGN(ExecutionHeuristics);
+};
+
+/** @} */
+
+} // namespace optimizer
+} // namespace quickstep
+
+#endif /* QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_HEURISTICS_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/CMakeLists.txt b/query_optimizer/tests/CMakeLists.txt
index 1d2fa10..5647bfd 100644
--- a/query_optimizer/tests/CMakeLists.txt
+++ b/query_optimizer/tests/CMakeLists.txt
@@ -78,6 +78,22 @@ add_executable(quickstep_queryoptimizer_tests_ExecutionGeneratorTest
ExecutionGeneratorTestRunner.hpp
"${PROJECT_SOURCE_DIR}/utility/textbased_test/TextBasedTest.cpp"
"${PROJECT_SOURCE_DIR}/utility/textbased_test/TextBasedTest.hpp")
+add_executable(ExecutionHeuristics_unittest ExecutionHeuristics_unittest.cpp)
+target_link_libraries(ExecutionHeuristics_unittest
+ gtest
+ gtest_main
+ quickstep_catalog_Catalog
+ quickstep_catalog_CatalogDatabase
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_QueryContext_proto
+ quickstep_queryoptimizer_ExecutionHeuristics
+ quickstep_queryoptimizer_QueryPlan
+ quickstep_relationaloperators_BuildHashOperator
+ quickstep_relationaloperators_HashJoinOperator
+ quickstep_utility_Macros)
+add_test(ExecutionHeuristics_unittest ExecutionHeuristics_unittest)
+
add_executable(quickstep_queryoptimizer_tests_OptimizerTextTest
OptimizerTextTest.cpp
OptimizerTextTestRunner.cpp
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/ExecutionHeuristics_unittest.cpp b/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
new file mode 100644
index 0000000..12acaff
--- /dev/null
+++ b/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
@@ -0,0 +1,301 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "catalog/Catalog.hpp"
+#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/ExecutionHeuristics.hpp"
+#include "query_optimizer/QueryPlan.hpp"
+#include "relational_operators/BuildHashOperator.hpp"
+#include "relational_operators/HashJoinOperator.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+namespace quickstep {
+namespace optimizer {
+
+class ExecutionHeuristicsTest : public ::testing::Test {
+ protected:
+ virtual void SetUp() {
+ db_ = cat_.getDatabaseByIdMutable(cat_.addDatabase(new CatalogDatabase(nullptr, "db")));
+ execution_heuristics_.reset(new ExecutionHeuristics());
+ query_plan_.reset(new QueryPlan());
+ query_context_proto_.reset(new serialization::QueryContext());
+ }
+
+ CatalogRelation* createCatalogRelation(const std::string &name, bool temporary = false) {
+ return db_->getRelationByIdMutable(db_->addRelation(new CatalogRelation(nullptr, name, -1, temporary)));
+ }
+
+ void addDummyHashJoinInfo(ExecutionHeuristics *execution_heuristics,
+ const QueryPlan::DAGNodeIndex build_operator_index,
+ const QueryPlan::DAGNodeIndex join_operator_index,
+ const CatalogRelation *build_relation,
+ const CatalogRelation *probe_relation,
+ const attribute_id build_attribute_id,
+ const attribute_id probe_attribute_id,
+ const QueryContext::join_hash_table_id join_hash_table_id) {
+ std::vector<attribute_id> build_attribute_ids(1, build_attribute_id);
+ std::vector<attribute_id> probe_attribute_ids(1, probe_attribute_id);
+ execution_heuristics->addHashJoinInfo(build_operator_index,
+ join_operator_index,
+ build_relation,
+ probe_relation,
+ std::move(build_attribute_ids),
+ std::move(probe_attribute_ids),
+ join_hash_table_id);
+ }
+
+ QueryPlan::DAGNodeIndex createDummyBuildHashOperator(QueryPlan *query_plan,
+ const CatalogRelation *build_relation,
+ const attribute_id build_attribute_id,
+ const QueryContext::join_hash_table_id join_hash_table_index) {
+ std::vector<attribute_id> build_attribute_ids;
+ build_attribute_ids.push_back(build_attribute_id);
+ QueryPlan::DAGNodeIndex build_operator_index =
+ query_plan->addRelationalOperator(new BuildHashOperator(*build_relation,
+ true,
+ build_attribute_ids,
+ false,
+ join_hash_table_index));
+ return build_operator_index;
+ }
+
+ QueryPlan::DAGNodeIndex createDummyHashJoinOperator(QueryPlan *query_plan,
+ const CatalogRelation *build_relation,
+ const CatalogRelation *probe_relation,
+ const attribute_id probe_attribute_id,
+ const QueryContext::join_hash_table_id join_hash_table_index) {
+ std::vector<attribute_id> probe_attribute_ids;
+ probe_attribute_ids.push_back(probe_attribute_id);
+ QueryPlan::DAGNodeIndex join_operator_index =
+ query_plan->addRelationalOperator(new HashJoinOperator(*build_relation,
+ *probe_relation,
+ true,
+ probe_attribute_ids,
+ false,
+ *probe_relation,
+ 0,
+ join_hash_table_index,
+ 0,
+ 0));
+ return join_operator_index;
+ }
+
+ Catalog cat_;
+ CatalogDatabase *db_; // db_ is owned by cat_.
+ std::unique_ptr<QueryPlan> query_plan_;
+ std::unique_ptr<serialization::QueryContext> query_context_proto_;
+ std::unique_ptr<ExecutionHeuristics> execution_heuristics_;
+};
+
+TEST_F(ExecutionHeuristicsTest, HashJoinOptimizedTest) {
+ // This test case creates three hash joins, all of which are being probed on the same relation.
+ // Since the probe are being made on the same relation, ExecutionHeuristics should optimize
+ // these hash joins using bloom filters.
+
+ const CatalogRelation *build_relation_1 = createCatalogRelation("build_relation_1");
+ const CatalogRelation *build_relation_2 = createCatalogRelation("build_relation_2");
+ const CatalogRelation *build_relation_3 = createCatalogRelation("build_relation_3");
+ const CatalogRelation *probe_relation_1 = createCatalogRelation("probe_relation_1");
+
+ const attribute_id build_attribute_id_1 = 0;
+ const attribute_id build_attribute_id_2 = 0;
+ const attribute_id build_attribute_id_3 = 0;
+ const attribute_id probe_attribute_id_1 = 1;
+ const attribute_id probe_attribute_id_2 = 2;
+ const attribute_id probe_attribute_id_3 = 3;
+
+ const QueryContext::join_hash_table_id join_hash_table_index_1 = 0;
+ const QueryContext::join_hash_table_id join_hash_table_index_2 = 1;
+ const QueryContext::join_hash_table_id join_hash_table_index_3 = 2;
+ query_context_proto_->add_join_hash_tables();
+ query_context_proto_->add_join_hash_tables();
+ query_context_proto_->add_join_hash_tables();
+
+ const QueryPlan::DAGNodeIndex build_operator_index_1 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_1,
+ build_attribute_id_1,
+ join_hash_table_index_1);
+ const QueryPlan::DAGNodeIndex probe_operator_index_1 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_1,
+ probe_relation_1,
+ probe_attribute_id_1,
+ join_hash_table_index_1);
+ const QueryPlan::DAGNodeIndex build_operator_index_2 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_2,
+ build_attribute_id_2,
+ join_hash_table_index_2);
+ const QueryPlan::DAGNodeIndex probe_operator_index_2 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_2,
+ probe_relation_1,
+ probe_attribute_id_2,
+ join_hash_table_index_2);
+ const QueryPlan::DAGNodeIndex build_operator_index_3 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_3,
+ build_attribute_id_3,
+ join_hash_table_index_3);
+ const QueryPlan::DAGNodeIndex probe_operator_index_3 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_3,
+ probe_relation_1,
+ probe_attribute_id_3,
+ join_hash_table_index_3);
+
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_1,
+ probe_operator_index_1,
+ build_relation_1,
+ probe_relation_1,
+ build_attribute_id_1,
+ probe_attribute_id_1,
+ join_hash_table_index_1);
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_2,
+ probe_operator_index_2,
+ build_relation_2,
+ probe_relation_1,
+ build_attribute_id_2,
+ probe_attribute_id_2,
+ join_hash_table_index_2);
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_3,
+ probe_operator_index_3,
+ build_relation_3,
+ probe_relation_1,
+ build_attribute_id_3,
+ probe_attribute_id_3,
+ join_hash_table_index_3);
+
+ execution_heuristics_->optimizeExecutionPlan(query_plan_.get(), query_context_proto_.get());
+
+ // Test whether correct number of bloom filters were added.
+ EXPECT_EQ(1, query_context_proto_->join_hash_tables(0).build_side_bloom_filter_id_size());
+ EXPECT_EQ(1, query_context_proto_->join_hash_tables(1).build_side_bloom_filter_id_size());
+ EXPECT_EQ(1, query_context_proto_->join_hash_tables(2).build_side_bloom_filter_id_size());
+ EXPECT_EQ(3, query_context_proto_->join_hash_tables(0).probe_side_bloom_filters_size());
+
+ // Test that the DAG was modified correctly or not.
+ // Probe operator 1 should have now build operator 1 and build operator 2 added as dependencies.
+ auto const probe_node_dependencies = query_plan_->getQueryPlanDAG().getDependencies(probe_operator_index_1);
+ EXPECT_EQ(1u, probe_node_dependencies.count(build_operator_index_2));
+ EXPECT_EQ(1u, probe_node_dependencies.count(build_operator_index_3));
+}
+
+TEST_F(ExecutionHeuristicsTest, HashJoinNotOptimizedTest) {
+ // This test case creates three hash joins, all of which are being probed on different relations.
+ // Since the probe are being made on the different relations, ExecutionHeuristics should optimize
+ // these hash joins using bloom filters.
+
+ const CatalogRelation *build_relation_1 = createCatalogRelation("build_relation_1");
+ const CatalogRelation *build_relation_2 = createCatalogRelation("build_relation_2");
+ const CatalogRelation *build_relation_3 = createCatalogRelation("build_relation_3");
+ const CatalogRelation *probe_relation_1 = createCatalogRelation("probe_relation_1");
+ const CatalogRelation *probe_relation_2 = createCatalogRelation("probe_relation_2");
+ const CatalogRelation *probe_relation_3 = createCatalogRelation("probe_relation_3");
+
+ const attribute_id build_attribute_id_1 = 0;
+ const attribute_id build_attribute_id_2 = 0;
+ const attribute_id build_attribute_id_3 = 0;
+ const attribute_id probe_attribute_id_1 = 1;
+ const attribute_id probe_attribute_id_2 = 2;
+ const attribute_id probe_attribute_id_3 = 3;
+
+ const QueryContext::join_hash_table_id join_hash_table_index_1 = 0;
+ const QueryContext::join_hash_table_id join_hash_table_index_2 = 1;
+ const QueryContext::join_hash_table_id join_hash_table_index_3 = 2;
+ query_context_proto_->add_join_hash_tables();
+ query_context_proto_->add_join_hash_tables();
+ query_context_proto_->add_join_hash_tables();
+
+ const QueryPlan::DAGNodeIndex build_operator_index_1 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_1,
+ build_attribute_id_1,
+ join_hash_table_index_1);
+ const QueryPlan::DAGNodeIndex probe_operator_index_1 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_1,
+ probe_relation_1,
+ probe_attribute_id_1,
+ join_hash_table_index_1);
+ const QueryPlan::DAGNodeIndex build_operator_index_2 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_2,
+ build_attribute_id_2,
+ join_hash_table_index_2);
+ const QueryPlan::DAGNodeIndex probe_operator_index_2 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_2,
+ probe_relation_2,
+ probe_attribute_id_2,
+ join_hash_table_index_2);
+ const QueryPlan::DAGNodeIndex build_operator_index_3 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_3,
+ build_attribute_id_3,
+ join_hash_table_index_3);
+ const QueryPlan::DAGNodeIndex probe_operator_index_3 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_3,
+ probe_relation_3,
+ probe_attribute_id_3,
+ join_hash_table_index_3);
+
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_1,
+ probe_operator_index_1,
+ build_relation_1,
+ probe_relation_1,
+ build_attribute_id_1,
+ probe_attribute_id_1,
+ join_hash_table_index_1);
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_2,
+ probe_operator_index_2,
+ build_relation_2,
+ probe_relation_2,
+ build_attribute_id_2,
+ probe_attribute_id_2,
+ join_hash_table_index_2);
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_3,
+ probe_operator_index_3,
+ build_relation_3,
+ probe_relation_3,
+ build_attribute_id_3,
+ probe_attribute_id_3,
+ join_hash_table_index_3);
+
+ execution_heuristics_->optimizeExecutionPlan(query_plan_.get(), query_context_proto_.get());
+
+ // Test that no bloom filters were added.
+ EXPECT_EQ(0, query_context_proto_->join_hash_tables(0).build_side_bloom_filter_id_size());
+ EXPECT_EQ(0, query_context_proto_->join_hash_tables(1).build_side_bloom_filter_id_size());
+ EXPECT_EQ(0, query_context_proto_->join_hash_tables(2).build_side_bloom_filter_id_size());
+ EXPECT_EQ(0, query_context_proto_->join_hash_tables(0).probe_side_bloom_filters_size());
+
+ // Test that the DAG was not modified at all.
+ // Probe operator 1 should not have build operator 1 and build operator 2 added as dependencies.
+ auto probe_node_dependencies = query_plan_->getQueryPlanDAG().getDependencies(probe_operator_index_1);
+ EXPECT_EQ(0u, probe_node_dependencies.count(build_operator_index_2));
+ EXPECT_EQ(0u, probe_node_dependencies.count(build_operator_index_3));
+}
+
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index dacacfa..115248c 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -629,6 +629,7 @@ target_link_libraries(quickstep_storage_HashTable
quickstep_threading_SpinSharedMutex
quickstep_types_Type
quickstep_types_TypedValue
+ quickstep_utility_BloomFilter
quickstep_utility_HashPair
quickstep_utility_Macros)
target_link_libraries(quickstep_storage_HashTableBase
@@ -648,6 +649,7 @@ target_link_libraries(quickstep_storage_HashTableFactory
quickstep_types_Type
quickstep_types_TypeFactory
quickstep_types_TypedValue
+ quickstep_utility_BloomFilter
quickstep_utility_Macros)
target_link_libraries(quickstep_storage_HashTableKeyManager
glog
@@ -1196,7 +1198,7 @@ target_link_libraries(BloomFilterIndexSubBlock_unittest
add_test(BloomFilterIndexSubBlock_unittest BloomFilterIndexSubBlock_unittest)
if(QUICKSTEP_HAVE_BITWEAVING)
- add_executable(BitWeavingIndexSubBlock_unittest
+ add_executable(BitWeavingIndexSubBlock_unittest
"${CMAKE_CURRENT_SOURCE_DIR}/bitweaving/tests/BitWeavingIndexSubBlock_unittest.cpp")
target_link_libraries(BitWeavingIndexSubBlock_unittest
glog
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/storage/HashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTable.hpp b/storage/HashTable.hpp
index 667848e..be31fd9 100644
--- a/storage/HashTable.hpp
+++ b/storage/HashTable.hpp
@@ -38,6 +38,7 @@
#include "threading/SpinSharedMutex.hpp"
#include "types/Type.hpp"
#include "types/TypedValue.hpp"
+#include "utility/BloomFilter.hpp"
#include "utility/HashPair.hpp"
#include "utility/Macros.hpp"
@@ -990,6 +991,61 @@ class HashTable : public HashTableBase<resizable,
template <typename FunctorT>
std::size_t forEachCompositeKey(FunctorT *functor) const;
+ /**
+ * @brief A call to this function will cause a bloom filter to be built
+ * during the build phase of this hash table.
+ **/
+ inline void enableBuildSideBloomFilter() {
+ has_build_side_bloom_filter_ = true;
+ }
+
+ /**
+ * @brief A call to this function will cause a set of bloom filters to be
+ * probed during the probe phase of this hash table.
+ **/
+ inline void enableProbeSideBloomFilter() {
+ has_probe_side_bloom_filter_ = true;
+ }
+
+ /**
+ * @brief This function sets the pointer to the bloom filter to be
+ * used during the build phase of this hash table.
+ * @warning Should call enable_build_side_bloom_filter() first to enable
+ * bloom filter usage during build phase.
+ * @note The ownership of the bloom filter lies with the caller.
+ *
+ * @param bloom_filter The pointer to the bloom filter.
+ **/
+ inline void setBuildSideBloomFilter(BloomFilter *bloom_filter) {
+ build_bloom_filter_ = bloom_filter;
+ }
+
+ /**
+ * @brief This function adds a pointer to the list of bloom filters to be
+ * used during the probe phase of this hash table.
+ * @warning Should call enable_probe_side_bloom_filter() first to enable
+ * bloom filter usage during probe phase.
+ * @note The ownership of the bloom filter lies with the caller.
+ *
+ * @param bloom_filter The pointer to the bloom filter.
+ **/
+ inline void addProbeSideBloomFilter(const BloomFilter *bloom_filter) {
+ probe_bloom_filters_.emplace_back(bloom_filter);
+ }
+
+ /**
+ * @brief This function adds a vector of attribute ids corresponding to a
+ * bloom filter used during the probe phase of this hash table.
+ * @warning Should call enable_probe_side_bloom_filter() first to enable
+ * bloom filter usage during probe phase.
+ *
+ * @param probe_attribute_ids The vector of attribute ids to use for probing
+ * the bloom filter.
+ **/
+ inline void addProbeSideAttributeIds(std::vector<attribute_id> &&probe_attribute_ids) {
+ probe_attribute_ids_.push_back(probe_attribute_ids);
+ }
+
protected:
/**
* @brief Constructor for new resizable hash table.
@@ -1270,6 +1326,13 @@ class HashTable : public HashTableBase<resizable,
const attribute_id key_attr_id,
FunctorT *functor) const;
+ // Data structures used for bloom filter optimized semi-joins.
+ bool has_build_side_bloom_filter_ = false;
+ bool has_probe_side_bloom_filter_ = false;
+ BloomFilter *build_bloom_filter_;
+ std::vector<const BloomFilter*> probe_bloom_filters_;
+ std::vector<std::vector<attribute_id>> probe_attribute_ids_;
+
DISALLOW_COPY_AND_ASSIGN(HashTable);
};
@@ -1414,6 +1477,12 @@ HashTablePutResult HashTable<ValueT, resizable, serializable, force_key_copy, al
&prealloc_state);
}
}
+ std::unique_ptr<BloomFilter> thread_local_bloom_filter;
+ if (has_build_side_bloom_filter_) {
+ thread_local_bloom_filter.reset(new BloomFilter(build_bloom_filter_->getRandomSeed(),
+ build_bloom_filter_->getNumberOfHashes(),
+ build_bloom_filter_->getBitArraySize()));
+ }
if (resizable) {
while (result == HashTablePutResult::kOutOfSpace) {
{
@@ -1429,6 +1498,11 @@ HashTablePutResult HashTable<ValueT, resizable, serializable, force_key_copy, al
variable_size,
(*functor)(*accessor),
using_prealloc ? &prealloc_state : nullptr);
+ // Insert into bloom filter, if enabled.
+ if (has_build_side_bloom_filter_) {
+ thread_local_bloom_filter->insertUnSafe(static_cast<const std::uint8_t *>(key.getDataPtr()),
+ key.getDataSize());
+ }
if (result == HashTablePutResult::kDuplicateKey) {
DEBUG_ASSERT(!using_prealloc);
return result;
@@ -1454,11 +1528,20 @@ HashTablePutResult HashTable<ValueT, resizable, serializable, force_key_copy, al
variable_size,
(*functor)(*accessor),
using_prealloc ? &prealloc_state : nullptr);
+ // Insert into bloom filter, if enabled.
+ if (has_build_side_bloom_filter_) {
+ thread_local_bloom_filter->insertUnSafe(static_cast<const std::uint8_t *>(key.getDataPtr()),
+ key.getDataSize());
+ }
if (result != HashTablePutResult::kOK) {
return result;
}
}
}
+ // Update the build side bloom filter with thread local copy, if available.
+ if (has_build_side_bloom_filter_) {
+ build_bloom_filter_->bitwiseOr(thread_local_bloom_filter.get());
+ }
return HashTablePutResult::kOK;
});
@@ -2164,6 +2247,27 @@ void HashTable<ValueT, resizable, serializable, force_key_copy, allow_duplicate_
accessor,
[&](auto *accessor) -> void { // NOLINT(build/c++11)
while (accessor->next()) {
+ // Probe any bloom filters, if enabled.
+ if (has_probe_side_bloom_filter_) {
+ DCHECK_EQ(probe_bloom_filters_.size(), probe_attribute_ids_.size());
+ // Check if the key is contained in the BloomFilters or not.
+ bool bloom_miss = false;
+ for (std::size_t i = 0; i < probe_bloom_filters_.size() && !bloom_miss; ++i) {
+ const BloomFilter *bloom_filter = probe_bloom_filters_[i];
+ for (const attribute_id &attr_id : probe_attribute_ids_[i]) {
+ TypedValue bloom_key = accessor->getTypedValue(attr_id);
+ if (!bloom_filter->contains(static_cast<const std::uint8_t*>(bloom_key.getDataPtr()),
+ bloom_key.getDataSize())) {
+ bloom_miss = true;
+ break;
+ }
+ }
+ }
+ if (bloom_miss) {
+ continue; // On a bloom filter miss, probing the hash table can be skipped.
+ }
+ }
+
TypedValue key = accessor->getTypedValue(key_attr_id);
if (check_for_null_keys && key.isNull()) {
continue;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/storage/HashTable.proto
----------------------------------------------------------------------
diff --git a/storage/HashTable.proto b/storage/HashTable.proto
index 653c3a7..7f00f29 100644
--- a/storage/HashTable.proto
+++ b/storage/HashTable.proto
@@ -1,5 +1,7 @@
// Copyright 2011-2015 Quickstep Technologies LLC.
// Copyright 2015-2016 Pivotal Software, Inc.
+// Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+// University of Wisconsin\u2014Madison.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -32,4 +34,10 @@ message HashTable {
required HashTableImplType hash_table_impl_type = 1;
repeated Type key_types = 2;
required uint64 estimated_num_entries = 3;
+ repeated uint32 build_side_bloom_filter_id = 4;
+ message ProbeSideBloomFilter {
+ required uint32 probe_side_bloom_filter_id = 1;
+ repeated uint32 probe_side_attr_ids = 2;
+ }
+ repeated ProbeSideBloomFilter probe_side_bloom_filters = 6;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/storage/HashTableFactory.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTableFactory.hpp b/storage/HashTableFactory.hpp
index 94a0721..34baaeb 100644
--- a/storage/HashTableFactory.hpp
+++ b/storage/HashTableFactory.hpp
@@ -29,6 +29,7 @@
#include "storage/SimpleScalarSeparateChainingHashTable.hpp"
#include "storage/TupleReference.hpp"
#include "types/TypeFactory.hpp"
+#include "utility/BloomFilter.hpp"
#include "utility/Macros.hpp"
#include "glog/logging.h"
@@ -291,11 +292,14 @@ class HashTableFactory {
* @param proto A protobuf description of a resizable HashTable.
* @param storage_manager The StorageManager to use (a StorageBlob will be
* allocated to hold the HashTable's contents).
+ * @param bloom_filters A vector of pointers to bloom filters that may be used
+ * during hash table construction in build/probe phase.
* @return A new resizable HashTable with parameters specified by proto.
**/
static HashTable<ValueT, resizable, serializable, force_key_copy, allow_duplicate_keys>*
CreateResizableFromProto(const serialization::HashTable &proto,
- StorageManager *storage_manager) {
+ StorageManager *storage_manager,
+ const std::vector<std::unique_ptr<BloomFilter>> &bloom_filters) {
DCHECK(ProtoIsValid(proto))
<< "Attempted to create HashTable from invalid proto description:\n"
<< proto.DebugString();
@@ -305,10 +309,40 @@ class HashTableFactory {
key_types.emplace_back(&TypeFactory::ReconstructFromProto(proto.key_types(i)));
}
- return CreateResizable(HashTableImplTypeFromProto(proto.hash_table_impl_type()),
- key_types,
- proto.estimated_num_entries(),
- storage_manager);
+ auto hash_table = CreateResizable(HashTableImplTypeFromProto(proto.hash_table_impl_type()),
+ key_types,
+ proto.estimated_num_entries(),
+ storage_manager);
+
+ // TODO(ssaurabh): These lazy initializations can be moved from here and pushed to the
+ // individual implementations of the hash table constructors.
+
+ // Check if there are any build side bloom filter defined on the hash table.
+ if (proto.build_side_bloom_filter_id_size() > 0) {
+ hash_table->enableBuildSideBloomFilter();
+ hash_table->setBuildSideBloomFilter(bloom_filters[proto.build_side_bloom_filter_id(0)].get());
+ }
+
+ // Check if there are any probe side bloom filters defined on the hash table.
+ if (proto.probe_side_bloom_filters_size() > 0) {
+ hash_table->enableProbeSideBloomFilter();
+ // Add as many probe bloom filters as defined by the proto.
+ for (int j = 0; j < proto.probe_side_bloom_filters_size(); ++j) {
+ // Add the pointer to the probe bloom filter within the list of probe bloom filters to use.
+ const auto probe_side_bloom_filter = proto.probe_side_bloom_filters(j);
+ hash_table->addProbeSideBloomFilter(bloom_filters[probe_side_bloom_filter.probe_side_bloom_filter_id()].get());
+
+ // Add the attribute ids corresponding to this probe bloom filter.
+ std::vector<attribute_id> probe_attribute_ids;
+ for (int k = 0; k < probe_side_bloom_filter.probe_side_attr_ids_size(); ++k) {
+ const attribute_id probe_attribute_id = probe_side_bloom_filter.probe_side_attr_ids(k);
+ probe_attribute_ids.push_back(probe_attribute_id);
+ }
+ hash_table->addProbeSideAttributeIds(std::move(probe_attribute_ids));
+ }
+ }
+
+ return hash_table;
}
private:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/utility/BloomFilter.hpp
----------------------------------------------------------------------
diff --git a/utility/BloomFilter.hpp b/utility/BloomFilter.hpp
index 1d4fdc7..b93df84 100644
--- a/utility/BloomFilter.hpp
+++ b/utility/BloomFilter.hpp
@@ -26,8 +26,15 @@
#include <algorithm>
#include <cstddef>
#include <cstdint>
+#include <memory>
+#include <utility>
#include <vector>
+#include "storage/StorageConstants.hpp"
+#include "threading/Mutex.hpp"
+#include "threading/SharedMutex.hpp"
+#include "threading/SpinSharedMutex.hpp"
+#include "utility/BloomFilter.pb.h"
#include "utility/Macros.hpp"
#include "glog/logging.h"
@@ -47,7 +54,30 @@ class BloomFilter {
/**
* @brief Constructor.
- * @note The ownership of the bit array lies with the caller.
+ * @note When no bit_array is being passed to the constructor,
+ * then the bit_array is owned and managed by this class.
+ *
+ * @param random_seed A random_seed that generates unique hash functions.
+ * @param hash_fn_count The number of hash functions used by this bloom filter.
+ * @param bit_array_size_in_bytes Size of the bit array.
+ **/
+ BloomFilter(const std::uint64_t random_seed,
+ const std::size_t hash_fn_count,
+ const std::uint64_t bit_array_size_in_bytes)
+ : random_seed_(random_seed),
+ hash_fn_count_(hash_fn_count),
+ array_size_in_bytes_(bit_array_size_in_bytes),
+ array_size_(array_size_in_bytes_ * kNumBitsPerByte),
+ bit_array_(new std::uint8_t[array_size_in_bytes_]),
+ is_bit_array_owner_(true) {
+ reset();
+ generate_unique_hash_fn();
+ }
+
+ /**
+ * @brief Constructor.
+ * @note When a bit_array is passed as an argument to the constructor,
+ * then the ownership of the bit array lies with the caller.
*
* @param random_seed A random_seed that generates unique hash functions.
* @param hash_fn_count The number of hash functions used by this bloom filter.
@@ -61,11 +91,12 @@ class BloomFilter {
const std::uint64_t bit_array_size_in_bytes,
std::uint8_t *bit_array,
const bool is_initialized)
- : hash_fn_count_(hash_fn_count),
- random_seed_(random_seed) {
- array_size_ = bit_array_size_in_bytes * kNumBitsPerByte;
- array_size_in_bytes_ = bit_array_size_in_bytes;
- bit_array_ = bit_array; // Owned by the calling method.
+ : random_seed_(random_seed),
+ hash_fn_count_(hash_fn_count),
+ array_size_in_bytes_(bit_array_size_in_bytes),
+ array_size_(bit_array_size_in_bytes * kNumBitsPerByte),
+ bit_array_(bit_array), // Owned by the calling method.
+ is_bit_array_owner_(false) {
if (!is_initialized) {
reset();
}
@@ -73,27 +104,149 @@ class BloomFilter {
}
/**
+ * @brief Constructor.
+ * @note When a bloom filter proto is passed as an initializer,
+ * then the bit_array is owned and managed by this class.
+ *
+ * @param bloom_filter_proto The protobuf representation of a
+ * bloom filter configuration.
+ **/
+ explicit BloomFilter(const serialization::BloomFilter &bloom_filter_proto)
+ : random_seed_(bloom_filter_proto.bloom_filter_seed()),
+ hash_fn_count_(bloom_filter_proto.number_of_hashes()),
+ array_size_in_bytes_(bloom_filter_proto.bloom_filter_size()),
+ array_size_(array_size_in_bytes_ * kNumBitsPerByte),
+ bit_array_(new std::uint8_t[array_size_in_bytes_]),
+ is_bit_array_owner_(true) {
+ reset();
+ generate_unique_hash_fn();
+ }
+
+ /**
+ * @brief Destructor.
+ **/
+ ~BloomFilter() {
+ if (is_bit_array_owner_) {
+ bit_array_.reset();
+ } else {
+ bit_array_.release();
+ }
+ }
+
+ static bool ProtoIsValid(const serialization::BloomFilter &bloom_filter_proto) {
+ return bloom_filter_proto.IsInitialized();
+ }
+
+ /**
* @brief Zeros out the contents of the bit array.
**/
inline void reset() {
// Initialize the bit_array with all zeros.
- std::fill_n(bit_array_, array_size_in_bytes_, 0x00);
+ std::fill_n(bit_array_.get(), array_size_in_bytes_, 0x00);
inserted_element_count_ = 0;
}
/**
+ * @brief Get the random seed that was used to initialize this bloom filter.
+ *
+ * @return Returns the random seed.
+ **/
+ inline std::uint64_t getRandomSeed() const {
+ return random_seed_;
+ }
+
+ /**
+ * @brief Get the number of hash functions used in this bloom filter.
+ *
+ * @return Returns the number of hash functions.
+ **/
+ inline std::uint32_t getNumberOfHashes() const {
+ return hash_fn_count_;
+ }
+
+ /**
+ * @brief Get the size of the bit array in bytes for this bloom filter.
+ *
+ * @return Returns the bit array size (in bytes).
+ **/
+ inline std::uint64_t getBitArraySize() const {
+ return array_size_in_bytes_;
+ }
+
+ /**
+ * @brief Get the constant pointer to the bit array for this bloom filter
+ *
+ * @return Returns constant pointer to the bit array.
+ **/
+ inline const std::uint8_t* getBitArray() const {
+ return bit_array_.get();
+ }
+
+ /**
+ * @brief Inserts a given value into the bloom filter in a thread-safe manner.
+ *
+ * @param key_begin A pointer to the value being inserted.
+ * @param length Size of the value being inserted in bytes.
+ */
+ inline void insert(const std::uint8_t *key_begin, const std::size_t length) {
+ // Locks are needed during insertion, when multiple workers may be modifying the
+ // bloom filter concurrently. However, locks are not required during membership test.
+ std::size_t bit_index = 0;
+ std::size_t bit = 0;
+ std::vector<std::pair<std::size_t, std::size_t>> modified_bit_positions;
+ std::vector<bool> is_bit_position_correct;
+
+ // Determine all the bit positions that are required to be set.
+ for (std::size_t i = 0; i < hash_fn_count_; ++i) {
+ compute_indices(hash_ap(key_begin, length, hash_fn_[i]), &bit_index, &bit);
+ modified_bit_positions.push_back(std::make_pair(bit_index, bit));
+ }
+
+ // Acquire a reader lock and check which of the bit positions are already set.
+ {
+ SpinSharedMutexSharedLock<false> shared_reader_lock(bloom_filter_insert_mutex_);
+ for (std::size_t i = 0; i < hash_fn_count_; ++i) {
+ bit_index = modified_bit_positions[i].first;
+ bit = modified_bit_positions[i].second;
+ if (((bit_array_.get())[bit_index / kNumBitsPerByte] & (1 << bit)) != (1 << bit)) {
+ is_bit_position_correct.push_back(false);
+ } else {
+ is_bit_position_correct.push_back(true);
+ }
+ }
+ }
+
+ // Acquire a writer lock and set the bit positions are which are not set.
+ {
+ SpinSharedMutexExclusiveLock<false> exclusive_writer_lock(bloom_filter_insert_mutex_);
+ for (std::size_t i = 0; i < hash_fn_count_; ++i) {
+ if (!is_bit_position_correct[i]) {
+ bit_index = modified_bit_positions[i].first;
+ bit = modified_bit_positions[i].second;
+ (bit_array_.get())[bit_index / kNumBitsPerByte] |= (1 << bit);
+ }
+ }
+ }
+ ++inserted_element_count_;
+ }
+
+ /**
* @brief Inserts a given value into the bloom filter.
+ * @Warning This is a faster thread-unsafe version of the insert() function.
+ * The caller needs to ensure the thread safety.
*
* @param key_begin A pointer to the value being inserted.
* @param length Size of the value being inserted in bytes.
*/
- inline void insert(const std::uint8_t *key_begin, const std::size_t &length) {
+ inline void insertUnSafe(const std::uint8_t *key_begin, const std::size_t length) {
std::size_t bit_index = 0;
std::size_t bit = 0;
+
for (std::size_t i = 0; i < hash_fn_count_; ++i) {
compute_indices(hash_ap(key_begin, length, hash_fn_[i]), &bit_index, &bit);
- bit_array_[bit_index / kNumBitsPerByte] |= (1 << bit);
+ (bit_array_.get())[bit_index / kNumBitsPerByte] |= (1 << bit);
}
+
++inserted_element_count_;
}
@@ -102,6 +255,9 @@ class BloomFilter {
* If true is returned, then a value may or may not be present in the bloom filter.
* If false is returned, a value is certainly not present in the bloom filter.
*
+ * @note The membersip test does not require any locks, because the assumption is that
+ * the bloom filter will only be used after it has been built.
+ *
* @param key_begin A pointer to the value being tested for membership.
* @param length Size of the value being inserted in bytes.
*/
@@ -110,7 +266,7 @@ class BloomFilter {
std::size_t bit = 0;
for (std::size_t i = 0; i < hash_fn_count_; ++i) {
compute_indices(hash_ap(key_begin, length, hash_fn_[i]), &bit_index, &bit);
- if ((bit_array_[bit_index / kNumBitsPerByte] & (1 << bit)) != (1 << bit)) {
+ if (((bit_array_.get())[bit_index / kNumBitsPerByte] & (1 << bit)) != (1 << bit)) {
return false;
}
}
@@ -118,6 +274,19 @@ class BloomFilter {
}
/**
+ * @brief Perform a bitwise-OR of the given Bloom filter with this bloom filter.
+ * Essentially, it does a union of this bloom filter with the passed bloom filter.
+ *
+ * @param bloom_filter A const pointer to the bloom filter object to do bitwise-OR with.
+ */
+ inline void bitwiseOr(const BloomFilter *bloom_filter) {
+ SpinSharedMutexExclusiveLock<false> exclusive_writer_lock(bloom_filter_insert_mutex_);
+ for (std::size_t byte_index = 0; byte_index < bloom_filter->getBitArraySize(); ++byte_index) {
+ (bit_array_.get())[byte_index] |= bloom_filter->getBitArray()[byte_index];
+ }
+ }
+
+ /**
* @brief Return the number of elements currently inserted into bloom filter.
*
* @return The number of elements inserted into bloom filter.
@@ -219,13 +388,16 @@ class BloomFilter {
}
private:
+ const std::uint64_t random_seed_;
std::vector<std::uint32_t> hash_fn_;
- std::uint8_t *bit_array_;
const std::uint32_t hash_fn_count_;
- std::uint64_t array_size_;
std::uint64_t array_size_in_bytes_;
+ std::uint64_t array_size_;
+ std::unique_ptr<std::uint8_t> bit_array_;
std::uint32_t inserted_element_count_;
- const std::uint64_t random_seed_;
+ const bool is_bit_array_owner_;
+
+ alignas(kCacheLineBytes) mutable SpinSharedMutex<false> bloom_filter_insert_mutex_;
DISALLOW_COPY_AND_ASSIGN(BloomFilter);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/utility/BloomFilter.proto
----------------------------------------------------------------------
diff --git a/utility/BloomFilter.proto b/utility/BloomFilter.proto
new file mode 100644
index 0000000..8dd9163
--- /dev/null
+++ b/utility/BloomFilter.proto
@@ -0,0 +1,30 @@
+// Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+// University of Wisconsin\u2014Madison.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package quickstep.serialization;
+
+message BloomFilter {
+ // The default values were determined from empirical experiments.
+ // These values control the amount of false positivity that
+ // is expected from Bloom Filter.
+ // - Default seed for initializing family of hashes = 0xA5A5A5A55A5A5A5A.
+ // - Default bloom filter size = 10 KB.
+ // - Default number of hash functions used in bloom filter = 5.
+ optional fixed64 bloom_filter_seed = 1 [default = 0xA5A5A5A55A5A5A5A];
+ optional uint32 bloom_filter_size = 2 [default = 10000];
+ optional uint32 number_of_hashes = 3 [default = 5];
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c07d709/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index bb59f65..6d1eeab 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -146,6 +146,10 @@ configure_file (
"${CMAKE_CURRENT_BINARY_DIR}/UtilityConfig.h"
)
+QS_PROTOBUF_GENERATE_CPP(quickstep_utility_BloomFilter_proto_srcs
+ quickstep_utility_BloomFilter_proto_hdrs
+ BloomFilter.proto)
+
QS_PROTOBUF_GENERATE_CPP(quickstep_utility_SortConfiguration_proto_srcs
quickstep_utility_SortConfiguration_proto_hdrs
SortConfiguration.proto)
@@ -155,6 +159,9 @@ add_library(quickstep_utility_Alignment ../empty_src.cpp Alignment.hpp)
add_library(quickstep_utility_BitManipulation ../empty_src.cpp BitManipulation.hpp)
add_library(quickstep_utility_BitVector ../empty_src.cpp BitVector.hpp)
add_library(quickstep_utility_BloomFilter ../empty_src.cpp BloomFilter.hpp)
+add_library(quickstep_utility_BloomFilter_proto
+ ${quickstep_utility_BloomFilter_proto_srcs}
+ ${quickstep_utility_BloomFilter_proto_hdrs})
add_library(quickstep_utility_CalculateInstalledMemory CalculateInstalledMemory.cpp CalculateInstalledMemory.hpp)
add_library(quickstep_utility_Cast ../empty_src.cpp Cast.hpp)
add_library(quickstep_utility_CheckSnprintf ../empty_src.cpp CheckSnprintf.hpp)
@@ -202,7 +209,14 @@ target_link_libraries(quickstep_utility_BitVector
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_BloomFilter
glog
+ quickstep_storage_StorageConstants
+ quickstep_threading_Mutex
+ quickstep_threading_SharedMutex
+ quickstep_threading_SpinSharedMutex
+ quickstep_utility_BloomFilter_proto
quickstep_utility_Macros)
+target_link_libraries(quickstep_utility_BloomFilter_proto
+ ${PROTOBUF_LIBRARY})
target_link_libraries(quickstep_utility_CalculateInstalledMemory
glog)
target_link_libraries(quickstep_utility_CheckSnprintf
@@ -271,6 +285,7 @@ target_link_libraries(quickstep_utility
quickstep_utility_BitManipulation
quickstep_utility_BitVector
quickstep_utility_BloomFilter
+ quickstep_utility_BloomFilter_proto
quickstep_utility_CalculateInstalledMemory
quickstep_utility_Cast
quickstep_utility_CheckSnprintf
[05/48] incubator-quickstep git commit: Partition aware selection
using NUMA-awareness (#175)
Posted by hb...@apache.org.
Partition aware selection using NUMA-awareness (#175)
Link: https://github.com/pivotalsoftware/quickstep/pull/175
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4f7725fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4f7725fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4f7725fd
Branch: refs/heads/query-manager-used-in-foreman
Commit: 4f7725fd42de5e2f2fad77826ae603aa92f4866e
Parents: 2ab9cd9
Author: Adalbert Gerald Soosai Raj <ad...@gmail.com>
Authored: Sun May 1 15:38:38 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:40 2016 -0700
----------------------------------------------------------------------
catalog/CMakeLists.txt | 5 +-
catalog/CatalogRelation.hpp | 9 ++
relational_operators/CMakeLists.txt | 5 ++
relational_operators/SelectOperator.cpp | 129 +++++++++++++++++++++------
relational_operators/SelectOperator.hpp | 115 +++++++++++++++++++++---
5 files changed, 226 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4f7725fd/catalog/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/catalog/CMakeLists.txt b/catalog/CMakeLists.txt
index 8c89d7e..94da838 100644
--- a/catalog/CMakeLists.txt
+++ b/catalog/CMakeLists.txt
@@ -175,9 +175,12 @@ target_link_libraries(quickstep_catalog
quickstep_catalog_CatalogRelationSchema
quickstep_catalog_CatalogTypedefs
quickstep_catalog_IndexScheme
- quickstep_catalog_NUMAPlacementScheme
quickstep_catalog_PartitionScheme
quickstep_catalog_PartitionSchemeHeader)
+if(QUICKSTEP_HAVE_LIBNUMA)
+target_link_libraries(quickstep_catalog
+ quickstep_catalog_NUMAPlacementScheme)
+endif()
# Tests:
add_executable(Catalog_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/Catalog_unittest.cpp")
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4f7725fd/catalog/CatalogRelation.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.hpp b/catalog/CatalogRelation.hpp
index 4cc8d79..3701090 100644
--- a/catalog/CatalogRelation.hpp
+++ b/catalog/CatalogRelation.hpp
@@ -165,6 +165,15 @@ class CatalogRelation : public CatalogRelationSchema {
}
/**
+ * @brief Get the NUMA placement scheme of the relation.
+ *
+ * @return A pointer to a const NUMA placement scheme.
+ **/
+ const NUMAPlacementScheme* getNUMAPlacementSchemePtr() const {
+ return placement_scheme_.get();
+ }
+
+ /**
* @brief Set the NUMA placement scheme for the catalog relation.
*
* @param placement_scheme The NUMA placement scheme object for the relation,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4f7725fd/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index a4600e6..eec5300 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -266,6 +266,7 @@ target_link_libraries(quickstep_relationaloperators_SelectOperator
glog
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
+ quickstep_catalog_PartitionSchemeHeader
quickstep_queryexecution_QueryContext
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
@@ -276,6 +277,10 @@ target_link_libraries(quickstep_relationaloperators_SelectOperator
quickstep_storage_StorageManager
quickstep_utility_Macros
tmb)
+if(QUICKSTEP_HAVE_LIBNUMA)
+target_link_libraries(quickstep_relationaloperators_SelectOperator
+ quickstep_catalog_NUMAPlacementScheme)
+endif()
target_link_libraries(quickstep_relationaloperators_SortMergeRunOperator
glog
quickstep_catalog_CatalogRelation
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4f7725fd/relational_operators/SelectOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.cpp b/relational_operators/SelectOperator.cpp
index 3cac199..69bb434 100644
--- a/relational_operators/SelectOperator.cpp
+++ b/relational_operators/SelectOperator.cpp
@@ -35,6 +35,93 @@ namespace quickstep {
class Predicate;
+void SelectOperator::addWorkOrders(WorkOrdersContainer *container,
+ StorageManager *storage_manager,
+ const Predicate *predicate,
+ const std::vector<std::unique_ptr<const Scalar>> *selection,
+ InsertDestination *output_destination) {
+ if (input_relation_is_stored_) {
+ for (const block_id input_block_id : input_relation_block_ids_) {
+ container->addNormalWorkOrder(
+ new SelectWorkOrder(input_relation_,
+ input_block_id,
+ predicate,
+ simple_projection_,
+ simple_selection_,
+ selection,
+ output_destination,
+ storage_manager),
+ op_index_);
+ }
+ } else {
+ while (num_workorders_generated_ < input_relation_block_ids_.size()) {
+ container->addNormalWorkOrder(
+ new SelectWorkOrder(
+ input_relation_,
+ input_relation_block_ids_[num_workorders_generated_],
+ predicate,
+ simple_projection_,
+ simple_selection_,
+ selection,
+ output_destination,
+ storage_manager),
+ op_index_);
+ ++num_workorders_generated_;
+ }
+ }
+}
+
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+void SelectOperator::addPartitionAwareWorkOrders(WorkOrdersContainer *container,
+ StorageManager *storage_manager,
+ const Predicate *predicate,
+ const std::vector<std::unique_ptr<const Scalar>> *selection,
+ InsertDestination *output_destination) {
+ DCHECK(placement_scheme_ != nullptr);
+ const std::size_t num_partitions = input_relation_.getPartitionScheme().getPartitionSchemeHeader().getNumPartitions();
+ if (input_relation_is_stored_) {
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ for (const block_id input_block_id :
+ input_relation_block_ids_in_partition_[part_id]) {
+ container->addNormalWorkOrder(
+ new SelectWorkOrder(
+ input_relation_,
+ input_block_id,
+ predicate,
+ simple_projection_,
+ simple_selection_,
+ selection,
+ output_destination,
+ storage_manager,
+ placement_scheme_->getNUMANodeForBlock(input_block_id)),
+ op_index_);
+ }
+ }
+ } else {
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ while (num_workorders_generated_in_partition_[part_id] <
+ input_relation_block_ids_in_partition_[part_id].size()) {
+ block_id block_in_partition
+ = input_relation_block_ids_in_partition_[part_id][num_workorders_generated_in_partition_[part_id]];
+ container->addNormalWorkOrder(
+ new SelectWorkOrder(
+ input_relation_,
+ block_in_partition,
+ predicate,
+ simple_projection_,
+ simple_selection_,
+ selection,
+ output_destination,
+ storage_manager,
+ placement_scheme_->getNUMANodeForBlock(block_in_partition)),
+ op_index_);
+ ++num_workorders_generated_in_partition_[part_id];
+ }
+ }
+ }
+}
+#endif
+
bool SelectOperator::getAllWorkOrders(
WorkOrdersContainer *container,
QueryContext *query_context,
@@ -54,35 +141,27 @@ bool SelectOperator::getAllWorkOrders(
if (input_relation_is_stored_) {
if (!started_) {
- for (const block_id input_block_id : input_relation_block_ids_) {
- container->addNormalWorkOrder(
- new SelectWorkOrder(input_relation_,
- input_block_id,
- predicate,
- simple_projection_,
- simple_selection_,
- selection,
- output_destination,
- storage_manager),
- op_index_);
+ if (input_relation_.hasPartitionScheme()) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ if (input_relation_.hasNUMAPlacementScheme()) {
+ addPartitionAwareWorkOrders(container, storage_manager, predicate, selection, output_destination);
+ }
+#endif
+ } else {
+ addWorkOrders(container, storage_manager, predicate, selection, output_destination);
}
started_ = true;
}
return started_;
} else {
- while (num_workorders_generated_ < input_relation_block_ids_.size()) {
- container->addNormalWorkOrder(
- new SelectWorkOrder(
- input_relation_,
- input_relation_block_ids_[num_workorders_generated_],
- predicate,
- simple_projection_,
- simple_selection_,
- selection,
- output_destination,
- storage_manager),
- op_index_);
- ++num_workorders_generated_;
+ if (input_relation_.hasPartitionScheme()) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ if (input_relation_.hasNUMAPlacementScheme()) {
+ addPartitionAwareWorkOrders(container, storage_manager, predicate, selection, output_destination);
+ }
+#endif
+ } else {
+ addWorkOrders(container, storage_manager, predicate, selection, output_destination);
}
return done_feeding_input_relation_;
}
@@ -90,7 +169,7 @@ bool SelectOperator::getAllWorkOrders(
void SelectWorkOrder::execute() {
BlockReference block(
- storage_manager_->getBlock(input_block_id_, input_relation_));
+ storage_manager_->getBlock(input_block_id_, input_relation_, getPreferredNUMANodes()[0]));
if (simple_projection_) {
block->selectSimple(simple_selection_,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4f7725fd/relational_operators/SelectOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.hpp b/relational_operators/SelectOperator.hpp
index 3da496c..76f4cb6 100644
--- a/relational_operators/SelectOperator.hpp
+++ b/relational_operators/SelectOperator.hpp
@@ -24,6 +24,12 @@
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
+
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include "catalog/NUMAPlacementScheme.hpp"
+#endif
+
+#include "catalog/PartitionSchemeHeader.hpp"
#include "query_execution/QueryContext.hpp"
#include "relational_operators/RelationalOperator.hpp"
#include "relational_operators/WorkOrder.hpp"
@@ -87,7 +93,28 @@ class SelectOperator : public RelationalOperator {
num_workorders_generated_(0),
simple_projection_(false),
input_relation_is_stored_(input_relation_is_stored),
- started_(false) {}
+ started_(false) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ placement_scheme_ = input_relation.getNUMAPlacementSchemePtr();
+#endif
+ if (input_relation.hasPartitionScheme()) {
+ const PartitionScheme &part_scheme = input_relation.getPartitionScheme();
+ const PartitionSchemeHeader &part_scheme_header = part_scheme.getPartitionSchemeHeader();
+ const std::size_t num_partitions = part_scheme_header.getNumPartitions();
+ input_relation_block_ids_in_partition_.resize(num_partitions);
+ num_workorders_generated_in_partition_.resize(num_partitions);
+ num_workorders_generated_in_partition_.assign(num_partitions, 0);
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ if (input_relation_is_stored) {
+ input_relation_block_ids_in_partition_[part_id] =
+ part_scheme.getBlocksInPartition(part_id);
+ } else {
+ input_relation_block_ids_in_partition_[part_id] =
+ std::vector<block_id>();
+ }
+ }
+ }
+ }
/**
* @brief Constructor for selection with simple projection of attributes.
@@ -124,7 +151,28 @@ class SelectOperator : public RelationalOperator {
num_workorders_generated_(0),
simple_projection_(true),
input_relation_is_stored_(input_relation_is_stored),
- started_(false) {}
+ started_(false) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ placement_scheme_ = input_relation.getNUMAPlacementSchemePtr();
+#endif
+ if (input_relation.hasPartitionScheme()) {
+ const PartitionScheme &part_scheme = input_relation.getPartitionScheme();
+ const PartitionSchemeHeader &part_scheme_header = part_scheme.getPartitionSchemeHeader();
+ const std::size_t num_partitions = part_scheme_header.getNumPartitions();
+ input_relation_block_ids_in_partition_.resize(num_partitions);
+ num_workorders_generated_in_partition_.resize(num_partitions);
+ num_workorders_generated_in_partition_.assign(num_partitions, 0);
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ if (input_relation_is_stored) {
+ input_relation_block_ids_in_partition_[part_id] =
+ part_scheme.getBlocksInPartition(part_id);
+ } else {
+ input_relation_block_ids_in_partition_[part_id] =
+ std::vector<block_id>();
+ }
+ }
+ }
+ }
~SelectOperator() override {}
@@ -135,13 +183,33 @@ class SelectOperator : public RelationalOperator {
tmb::MessageBus *bus) override;
void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
- input_relation_block_ids_.push_back(input_block_id);
+ if (input_relation_.hasPartitionScheme()) {
+ const partition_id part_id =
+ input_relation_.getPartitionScheme().getPartitionForBlock(input_block_id);
+ input_relation_block_ids_in_partition_[part_id].push_back(input_block_id);
+ } else {
+ input_relation_block_ids_.push_back(input_block_id);
+ }
}
+ // TODO(gerald): Each call to getPartitionForBlock() involves grabbing shared
+ // locks on each partition's mutex, checking if the block belongs to the
+ // partition. Instead, we can provide a method getPartitionsForBlocks() which
+ // accepts a list of blocks and returns corresponding list of their partition IDs.
+ // Therefore, once we grab a lock for a partition, we search for all the blocks
+ // and then release the lock.
void feedInputBlocks(const relation_id rel_id, std::vector<block_id> *partially_filled_blocks) override {
- input_relation_block_ids_.insert(input_relation_block_ids_.end(),
- partially_filled_blocks->begin(),
- partially_filled_blocks->end());
+ if (input_relation_.hasPartitionScheme()) {
+ for (auto it = partially_filled_blocks->begin(); it != partially_filled_blocks->end(); ++it) {
+ const partition_id part_id = input_relation_.getPartitionScheme().getPartitionForBlock((*it));
+ input_relation_block_ids_in_partition_[part_id].insert(input_relation_block_ids_in_partition_[part_id].end(),
+ *it);
+ }
+ } else {
+ input_relation_block_ids_.insert(input_relation_block_ids_.end(),
+ partially_filled_blocks->begin(),
+ partially_filled_blocks->end());
+ }
}
QueryContext::insert_destination_id getInsertDestinationID() const override {
@@ -152,9 +220,20 @@ class SelectOperator : public RelationalOperator {
return output_relation_.getID();
}
+ void addWorkOrders(WorkOrdersContainer *container,
+ StorageManager *storage_manager,
+ const Predicate *predicate,
+ const std::vector<std::unique_ptr<const Scalar>> *selection,
+ InsertDestination *output_destination);
+
+ void addPartitionAwareWorkOrders(WorkOrdersContainer *container,
+ StorageManager *storage_manager,
+ const Predicate *predicate,
+ const std::vector<std::unique_ptr<const Scalar>> *selection,
+ InsertDestination *output_destination);
+
private:
const CatalogRelation &input_relation_;
-
const CatalogRelation &output_relation_;
const QueryContext::insert_destination_id output_destination_index_;
const QueryContext::predicate_id predicate_index_;
@@ -163,12 +242,20 @@ class SelectOperator : public RelationalOperator {
const std::vector<attribute_id> simple_selection_;
std::vector<block_id> input_relation_block_ids_;
+ // A vector of vectors V where V[i] indicates the list of block IDs of the
+ // input relation that belong to the partition i.
+ std::vector<std::vector<block_id>> input_relation_block_ids_in_partition_;
// A single workorder is generated for each block of input relation.
std::vector<block_id>::size_type num_workorders_generated_;
+ // A single workorder is generated for each block in each partition of input relation.
+ std::vector<std::size_t> num_workorders_generated_in_partition_;
const bool simple_projection_;
const bool input_relation_is_stored_;
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ const NUMAPlacementScheme *placement_scheme_;
+#endif
bool started_;
DISALLOW_COPY_AND_ASSIGN(SelectOperator);
@@ -205,7 +292,8 @@ class SelectWorkOrder : public WorkOrder {
const std::vector<attribute_id> &simple_selection,
const std::vector<std::unique_ptr<const Scalar>> *selection,
InsertDestination *output_destination,
- StorageManager *storage_manager)
+ StorageManager *storage_manager,
+ const numa_node_id numa_node = 0)
: input_relation_(input_relation),
input_block_id_(input_block_id),
predicate_(predicate),
@@ -213,7 +301,9 @@ class SelectWorkOrder : public WorkOrder {
simple_selection_(simple_selection),
selection_(selection),
output_destination_(DCHECK_NOTNULL(output_destination)),
- storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
+ storage_manager_(DCHECK_NOTNULL(storage_manager)) {
+ preferred_numa_nodes_.push_back(numa_node);
+ }
/**
* @brief Constructor for the distributed version.
@@ -241,7 +331,8 @@ class SelectWorkOrder : public WorkOrder {
std::vector<attribute_id> &&simple_selection,
const std::vector<std::unique_ptr<const Scalar>> *selection,
InsertDestination *output_destination,
- StorageManager *storage_manager)
+ StorageManager *storage_manager,
+ const numa_node_id numa_node = 0)
: input_relation_(input_relation),
input_block_id_(input_block_id),
predicate_(predicate),
@@ -249,7 +340,9 @@ class SelectWorkOrder : public WorkOrder {
simple_selection_(std::move(simple_selection)),
selection_(selection),
output_destination_(DCHECK_NOTNULL(output_destination)),
- storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
+ storage_manager_(DCHECK_NOTNULL(storage_manager)) {
+ preferred_numa_nodes_.push_back(numa_node);
+ }
~SelectWorkOrder() override {}
[24/48] incubator-quickstep git commit: Fix bug in the SMA code (#223)
Posted by hb...@apache.org.
Fix bug in the SMA code (#223)
* Fix bug in the SMA code so that the SMA predicate evaluation is only
applied if at least one of the operands in the predicate is a
static value.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/847941df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/847941df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/847941df
Branch: refs/heads/query-manager-used-in-foreman
Commit: 847941dfe48f8055be1ea2f6a420fc5613c80653
Parents: 43f1626
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Mon May 16 12:04:32 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:43 2016 -0700
----------------------------------------------------------------------
storage/SMAIndexSubBlock.cpp | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/847941df/storage/SMAIndexSubBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/SMAIndexSubBlock.cpp b/storage/SMAIndexSubBlock.cpp
index 2a0e4f9..aa9bc54 100644
--- a/storage/SMAIndexSubBlock.cpp
+++ b/storage/SMAIndexSubBlock.cpp
@@ -621,9 +621,14 @@ Selectivity SMAIndexSubBlock::getSelectivityForPredicate(const ComparisonPredica
predicate_cost_t SMAIndexSubBlock::estimatePredicateEvaluationCost(
const ComparisonPredicate &predicate) const {
DCHECK(initialized_);
- Selectivity selectivity = getSelectivityForPredicate(predicate);
- if (selectivity == Selectivity::kAll || selectivity == Selectivity::kNone) {
- return predicate_cost::kConstantTime;
+
+ // Check that at least one of the operands has a static value.
+ if (predicate.getLeftOperand().hasStaticValue() ||
+ predicate.getRightOperand().hasStaticValue()) {
+ Selectivity selectivity = getSelectivityForPredicate(predicate);
+ if (selectivity == Selectivity::kAll || selectivity == Selectivity::kNone) {
+ return predicate_cost::kConstantTime;
+ }
}
return predicate_cost::kInfinite;
}
[15/48] incubator-quickstep git commit: Transaction Part 4:
LockManager, CycleDetector and DeadLockDetector. (#187)
Posted by hb...@apache.org.
Transaction Part 4: LockManager, CycleDetector and DeadLockDetector. (#187)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/fc9c3bf1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/fc9c3bf1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/fc9c3bf1
Branch: refs/heads/query-manager-used-in-foreman
Commit: fc9c3bf1eba9044074782e41fa203d2d053eb5f3
Parents: 812773f
Author: Hakan Memisoglu <ha...@gmail.com>
Authored: Wed May 4 23:30:51 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:41 2016 -0700
----------------------------------------------------------------------
transaction/AccessMode.hpp | 76 +++++-
transaction/CMakeLists.txt | 59 ++++-
transaction/CycleDetector.cpp | 120 ++++++++++
transaction/CycleDetector.hpp | 83 +++++++
transaction/DeadLockDetector.cpp | 177 ++++++++++++++
transaction/DeadLockDetector.hpp | 156 ++++++++++++
transaction/DirectedGraph.hpp | 56 ++---
transaction/LockManager.cpp | 237 +++++++++++++++++++
transaction/LockManager.hpp | 128 ++++++++++
transaction/LockTable.cpp | 22 +-
transaction/LockTable.hpp | 36 +--
transaction/ResourceId.hpp | 9 +-
transaction/StronglyConnectedComponents.cpp | 1 -
transaction/Transaction.cpp | 48 ----
transaction/TransactionTable.cpp | 8 +-
transaction/TransactionTable.hpp | 11 +-
transaction/tests/AccessMode_unittest.cpp | 12 +-
transaction/tests/CycleDetector_unittest.cpp | 157 ++++++++++++
transaction/tests/DeadLockDetector_unittest.cpp | 96 ++++++++
transaction/tests/DirectedGraph_unittest.cpp | 113 +++++----
transaction/tests/LockRequest_unittest.cpp | 4 +-
transaction/tests/LockTable_unittest.cpp | 63 ++---
transaction/tests/Lock_unittest.cpp | 13 +-
.../StronglyConnectedComponents_unittest.cpp | 15 +-
transaction/tests/TransactionTable_unittest.cpp | 102 ++++----
25 files changed, 1512 insertions(+), 290 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/AccessMode.hpp
----------------------------------------------------------------------
diff --git a/transaction/AccessMode.hpp b/transaction/AccessMode.hpp
index 34ace36..bb06689 100644
--- a/transaction/AccessMode.hpp
+++ b/transaction/AccessMode.hpp
@@ -32,12 +32,12 @@ namespace transaction {
* @brief Represents mode type. Possible options are NL, IS, IX, S, SIX, X.
**/
enum class AccessModeType : std::uint8_t {
- kNoLock = 0,
- kIsLock,
- kIxLock,
- kSLock,
- kSixLock,
- kXLock,
+ kNoLockMode = 0,
+ kIsLockMode,
+ kIxLockMode,
+ kSLockMode,
+ kSixLockMode,
+ kXLockMode,
kNumAccessModeTypes,
};
@@ -55,6 +55,60 @@ class AccessMode {
: access_mode_(access_mode) {}
/**
+ * @brief Factory method for NoLockMode.
+ *
+ * @return NoLockMode instance.
+ **/
+ static AccessMode NoLockMode() {
+ return AccessMode(AccessModeType::kNoLockMode);
+ }
+
+ /**
+ * @brief Factory method for IsLockMode.
+ *
+ * @return IsLockMode instance.
+ **/
+ static AccessMode IsLockMode() {
+ return AccessMode(AccessModeType::kIsLockMode);
+ }
+
+ /**
+ * @brief Factory method for IxLockMode.
+ *
+ * @return IxLockMode instance.
+ **/
+ static AccessMode IxLockMode() {
+ return AccessMode(AccessModeType::kIxLockMode);
+ }
+
+ /**
+ * @brief Factory method for SixLockMode.
+ *
+ * @return SixLockMode instance.
+ **/
+ static AccessMode SixLockMode() {
+ return AccessMode(AccessModeType::kSixLockMode);
+ }
+
+ /**
+ * @brief Factory method for SLockMode.
+ *
+ * @return SLockMode instance.
+ **/
+ static AccessMode SLockMode() {
+ return AccessMode(AccessModeType::kSLockMode);
+ }
+
+ /**
+ * @brief Factory method for XLockMode.
+ *
+ * @return XLockMode instance.
+ **/
+ static AccessMode XLockMode() {
+ return AccessMode(AccessModeType::kXLockMode);
+ }
+
+ /**
* @brief Checks whether this access mode is compatible with the other.
*
* @param other Other access mode that will be checked against to this one.
@@ -74,7 +128,7 @@ class AccessMode {
* @return True if it is IS mode, false otherwise.
**/
inline bool isIntentionShareLock() const {
- return access_mode_ == AccessModeType::kIsLock;
+ return access_mode_ == AccessModeType::kIsLockMode;
}
/**
@@ -83,7 +137,7 @@ class AccessMode {
* @return True if it is IX mode, false otherwise.
**/
inline bool isIntentionExclusiveLock() const {
- return access_mode_ == AccessModeType::kIxLock;
+ return access_mode_ == AccessModeType::kIxLockMode;
}
/**
@@ -92,7 +146,7 @@ class AccessMode {
* @return True if it is SIX mode, false otherwise.
**/
inline bool isShareAndIntentionExclusiveLock() const {
- return access_mode_ == AccessModeType::kSixLock;
+ return access_mode_ == AccessModeType::kSixLockMode;
}
/**
@@ -101,7 +155,7 @@ class AccessMode {
* @return True if it is S mode, false otherwise.
**/
inline bool isShareLock() const {
- return access_mode_ == AccessModeType::kSLock;
+ return access_mode_ == AccessModeType::kSLockMode;
}
/**
@@ -110,7 +164,7 @@ class AccessMode {
* @return True if it is X mode, false otherwise.
**/
inline bool isExclusiveLock() const {
- return access_mode_ == AccessModeType::kXLock;
+ return access_mode_ == AccessModeType::kXLockMode;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/transaction/CMakeLists.txt b/transaction/CMakeLists.txt
index 05fc96a..c6c87b6 100644
--- a/transaction/CMakeLists.txt
+++ b/transaction/CMakeLists.txt
@@ -16,12 +16,21 @@
add_library(quickstep_transaction_AccessMode
AccessMode.cpp
AccessMode.hpp)
+add_library(quickstep_transaction_CycleDetector
+ CycleDetector.cpp
+ CycleDetector.hpp)
+add_library(quickstep_transaction_DeadLockDetector
+ DeadLockDetector.cpp
+ DeadLockDetector.cpp)
add_library(quickstep_transaction_DirectedGraph
../empty_src.cpp
DirectedGraph.hpp)
add_library(quickstep_transaction_Lock
../empty_src.cpp
Lock.hpp)
+add_library(quickstep_transaction_LockManager
+ LockManager.hpp
+ LockManager.cpp)
add_library(quickstep_transaction_LockRequest
../empty_src.cpp
LockRequest.hpp)
@@ -40,7 +49,17 @@ add_library(quickstep_transaction_Transaction
add_library(quickstep_transaction_TransactionTable
TransactionTable.cpp
TransactionTable.hpp)
-
+
+target_link_libraries(quickstep_transaction_CycleDetector
+ quickstep_transaction_DirectedGraph
+ quickstep_transaction_StronglyConnectedComponents
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_transaction_DeadLockDetector
+ glog
+ quickstep_transaction_CycleDetector
+ quickstep_transaction_DirectedGraph
+ quickstep_transaction_LockTable
+ quickstep_transaction_Transaction)
target_link_libraries(quickstep_transaction_DirectedGraph
glog
quickstep_transaction_Transaction
@@ -48,6 +67,18 @@ target_link_libraries(quickstep_transaction_DirectedGraph
target_link_libraries(quickstep_transaction_Lock
quickstep_transaction_AccessMode
quickstep_transaction_ResourceId)
+target_link_libraries(quickstep_transaction_LockManager
+ gflags_nothreads-static
+ glog
+ quickstep_utility_ThreadSafeQueue
+ quickstep_threading_Thread
+ quickstep_transaction_AccessMode
+ quickstep_transaction_DeadLockDetector
+ quickstep_transaction_LockRequest
+ quickstep_transaction_LockTable
+ quickstep_transaction_ResourceId
+ quickstep_transaction_Transaction
+ quickstep_transaction_TransactionTable)
target_link_libraries(quickstep_transaction_LockRequest
quickstep_transaction_AccessMode
quickstep_transaction_ResourceId
@@ -80,8 +111,11 @@ add_library(quickstep_transaction
TransactionModule.hpp)
target_link_libraries(quickstep_transaction
quickstep_transaction_AccessMode
+ quickstep_transaction_CycleDetector
+ quickstep_transaction_DeadLockDetector
quickstep_transaction_DirectedGraph
quickstep_transaction_Lock
+ quickstep_transaction_LockManager
quickstep_transaction_LockRequest
quickstep_transaction_LockTable
quickstep_transaction_ResourceId
@@ -97,6 +131,29 @@ target_link_libraries(AccessMode_unittest
quickstep_transaction_AccessMode)
add_test(AccessMode_unittest AccessMode_unittest)
+add_executable(CycleDetector_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/CycleDetector_unittest.cpp")
+target_link_libraries(CycleDetector_unittest
+ gtest
+ gtest_main
+ quickstep_transaction_CycleDetector
+ quickstep_transaction_DirectedGraph
+ quickstep_transaction_StronglyConnectedComponents)
+add_test(CycleDetector_unittest CycleDetector_unittest)
+
+add_executable(DeadLockDetector_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/DeadLockDetector_unittest.cpp")
+target_link_libraries(DeadLockDetector_unittest
+ gtest
+ gtest_main
+ quickstep_threading_Thread
+ quickstep_transaction_AccessMode
+ quickstep_transaction_DeadLockDetector
+ quickstep_transaction_LockTable
+ quickstep_transaction_ResourceId
+ quickstep_transaction_Transaction)
+add_test(DeadLockDetector_unittest DeadLockDetector_unittest)
+
add_executable(DirectedGraph_unittest
"${CMAKE_CURRENT_SOURCE_DIR}/tests/DirectedGraph_unittest.cpp")
target_link_libraries(DirectedGraph_unittest
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/CycleDetector.cpp
----------------------------------------------------------------------
diff --git a/transaction/CycleDetector.cpp b/transaction/CycleDetector.cpp
new file mode 100644
index 0000000..b12897f
--- /dev/null
+++ b/transaction/CycleDetector.cpp
@@ -0,0 +1,120 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/CycleDetector.hpp"
+
+#include <cstdint>
+#include <memory>
+#include <stack>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/StronglyConnectedComponents.hpp"
+
+namespace quickstep {
+namespace transaction {
+
+CycleDetector::CycleDetector(DirectedGraph *wait_for_graph)
+ : wait_for_graph_(wait_for_graph),
+ strongly_connected_components_(
+ std::make_unique<StronglyConnectedComponents>(*wait_for_graph)) {
+}
+
+std::vector<DirectedGraph::node_id>
+CycleDetector::chooseVictimsToBreakCycle() const {
+ std::vector<DirectedGraph::node_id> nodes_to_kill;
+ const std::unordered_map<std::uint64_t, std::vector<DirectedGraph::node_id>>
+ component_mapping = strongly_connected_components_->getComponentMapping();
+ for (const auto &entry : component_mapping) {
+ // One node means no cycle.
+ if (entry.second.size() == 1) {
+ continue;
+ }
+ const std::vector<DirectedGraph::node_id> nodes =
+ chooseVictimsInComponent(entry.second);
+ nodes_to_kill.insert(nodes_to_kill.end(), nodes.begin(), nodes.end());
+ }
+ return nodes_to_kill;
+}
+
+std::vector<DirectedGraph::node_id> CycleDetector::chooseVictimsInComponent(
+ const std::vector<DirectedGraph::node_id> &nodes) const {
+ std::vector<DirectedGraph::node_id> targets;
+ // Convert it to set to ensure defensively that the elements are unique.
+ std::unordered_set<DirectedGraph::node_id> nodes_set(nodes.begin(),
+ nodes.end());
+
+ while (true) {
+ if (!hasCycle(nodes_set)) {
+ break;
+ }
+ // Connected component still has a cycle, therefore choose a
+ // victim and keep trying to remove nodes until there is no cycle.
+ const DirectedGraph::node_id victim = chooseVictim(nodes_set);
+ // Remove the victim node from the connected component.
+ nodes_set.erase(victim);
+ // Removed node is a victim now.
+ targets.push_back(victim);
+ }
+ return targets;
+}
+
+bool CycleDetector::hasCycle(
+ const std::unordered_set<DirectedGraph::node_id> &nodes) const {
+ // Keeps track of the nodes the algorithms visited.
+ std::unordered_set<DirectedGraph::node_id> visited;
+ for (const DirectedGraph::node_id node_id : nodes) {
+ // If it is visited, then pass to the next one.
+ if (visited.count(node_id) == 1) {
+ continue;
+ }
+ // Save the backtracking information.
+ std::stack<DirectedGraph::node_id> to_visit;
+ // Mark this id as "to be visited".
+ to_visit.push(node_id);
+ // Start to visit nodes until it is done.
+ while (!to_visit.empty()) {
+ const DirectedGraph::node_id current_node = to_visit.top();
+ to_visit.pop();
+ // Mark the node coming from stack as "visited".
+ visited.insert(current_node);
+ // For all adjacent nodes of this "visited" node,
+ const std::vector<DirectedGraph::node_id> adjacents
+ = wait_for_graph_->getAdjacentNodes(current_node);
+ for (const DirectedGraph::node_id adjacent : adjacents) {
+ if (visited.count(adjacent) == 1) {
+ // If this adjacent node is a node we already visited, then
+ // there is a cycle.
+ return true;
+ } else if (nodes.count(adjacent) == 1 && visited.count(adjacent) == 0) {
+ // Otherwise, if it is a node that we did not visit before
+ // mark this nodes as "to be visited".
+ to_visit.push(adjacent);
+ }
+ }
+ }
+ }
+ // If we have already visited all nodes and could not find a cycle,
+ // then we should return "no cycle" result.
+ return false;
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/CycleDetector.hpp
----------------------------------------------------------------------
diff --git a/transaction/CycleDetector.hpp b/transaction/CycleDetector.hpp
new file mode 100644
index 0000000..6865e2d
--- /dev/null
+++ b/transaction/CycleDetector.hpp
@@ -0,0 +1,83 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_TRANSACTION_CYCLE_DETECTOR_HPP_
+#define QUICKSTEP_TRANSACTION_CYCLE_DETECTOR_HPP_
+
+#include <memory>
+#include <unordered_set>
+#include <vector>
+
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/StronglyConnectedComponents.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+namespace transaction {
+
+/** \addtogroup Transaction
+ * @{
+ */
+
+/**
+ * @brief Class for running cycle detection algorithm on directed graph.
+ */
+class CycleDetector {
+ public:
+ /**
+ * @brief Constructor for DirectedGraph.
+ *
+ * @param wait_for_graph Pointer to a directed wait-for graph.
+ */
+ explicit CycleDetector(DirectedGraph *wait_for_graph);
+
+ /**
+ * @brief Calculate which nodes should be killed to eliminate all cycles
+ * in the graph.
+ *
+ * @return Vector of node ids that should be killed to break all cycles.
+ */
+ std::vector<DirectedGraph::node_id> chooseVictimsToBreakCycle() const;
+
+ private:
+ std::vector<DirectedGraph::node_id> chooseVictimsInComponent(
+ const std::vector<DirectedGraph::node_id> &nodes) const;
+
+ inline DirectedGraph::node_id chooseVictim(
+ const std::unordered_set<DirectedGraph::node_id> &nodes_set) const {
+ // TODO(Hakan): This is very inefficient scheme, however in the
+ // future, we can use the transaction's priority
+ // as the victim selection parameter.
+ return *(nodes_set.begin());
+ }
+
+ // Checks whether the nodes in the set make a cycle.
+ bool hasCycle(const std::unordered_set<DirectedGraph::node_id> &within) const;
+
+ DirectedGraph *wait_for_graph_;
+
+ std::unique_ptr<StronglyConnectedComponents> strongly_connected_components_;
+
+ DISALLOW_COPY_AND_ASSIGN(CycleDetector);
+};
+
+/** @} */
+
+} // namespace transaction
+} // namespace quickstep
+
+#endif // QUICKSTEP_TRANSACTION_CYCLE_DETECTOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/DeadLockDetector.cpp
----------------------------------------------------------------------
diff --git a/transaction/DeadLockDetector.cpp b/transaction/DeadLockDetector.cpp
new file mode 100644
index 0000000..26ab115
--- /dev/null
+++ b/transaction/DeadLockDetector.cpp
@@ -0,0 +1,177 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/DeadLockDetector.hpp"
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <memory>
+#include <thread> // NOLINT(build/c++11)
+#include <utility>
+#include <vector>
+
+#include "transaction/CycleDetector.hpp"
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/LockTable.hpp"
+#include "transaction/Transaction.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace transaction {
+
+constexpr std::int64_t DeadLockDetector::kSleepDurationInSeconds;
+
+DeadLockDetector::DeadLockDetector(LockTable *lock_table,
+ std::atomic<DeadLockDetectorStatus> *status,
+ std::vector<DirectedGraph::node_id> *victims)
+ : tid_node_mapping_(std::make_unique<transaction_id_node_map>()),
+ lock_table_(lock_table),
+ status_(status),
+ victims_(victims) {
+}
+
+void DeadLockDetector::run() {
+ while (true) {
+ if (status_->load() == DeadLockDetectorStatus::kQuit) {
+ // DeadLockDetector should stop.
+ return;
+ }
+ while (status_->load() == DeadLockDetectorStatus::kDone) {
+ // LockTable has not process the previous batch yet.
+ }
+
+ // TODO(Hakan): Implement logging mechanism for deadlock detection
+ // start and end times.
+ std::vector<DirectedGraph::node_id> victim_new_batch = getAllVictims();
+
+ // Swap new batch with old batch to make LockTable to see new victims.
+ std::swap(victim_new_batch, *victims_);
+
+ // Signal LockTable that new batch is ready.
+ status_->store(DeadLockDetectorStatus::kDone);
+
+ // DeadLockDetector should run once in a predefined interval.
+ std::this_thread::sleep_for(
+ std::chrono::seconds(kSleepDurationInSeconds));
+ }
+}
+
+void DeadLockDetector::addPendingInfo(const transaction_id pending,
+ const transaction_id owner) {
+ const DirectedGraph::node_id pending_node_id = getNodeId(pending);
+ const DirectedGraph::node_id owner_node_id = getNodeId(owner);
+
+ // TODO(Hakan): Check first whether link is already created. Use checked
+ // version for adding an edge.
+ wait_for_graph_->addEdgeUnchecked(pending_node_id, owner_node_id);
+}
+
+void DeadLockDetector::deletePendingInfo(const transaction_id pending,
+ const transaction_id owner) {
+ LOG(FATAL) << "Not implemented";
+}
+
+bool DeadLockDetector::isDependent(const transaction_id pending,
+ const transaction_id owner) const {
+ LOG(FATAL) << "Not implemented";
+}
+
+std::vector<transaction_id>
+DeadLockDetector::getAllDependents(const transaction_id owner) const {
+ LOG(FATAL) << "Not implemented";
+}
+
+std::vector<transaction_id>
+DeadLockDetector::getAllDependees(transaction_id pending) {
+ const DirectedGraph::node_id pending_node_id = getNodeId(pending);
+ const std::vector<DirectedGraph::node_id> nodes
+ = wait_for_graph_->getAdjacentNodes(pending_node_id);
+ std::vector<transaction_id> transactions;
+ transactions.reserve(nodes.size());
+ for (const DirectedGraph::node_id node_id : nodes) {
+ const transaction_id tid = wait_for_graph_->getDataFromNode(node_id);
+ transactions.push_back(tid);
+ }
+ return transactions;
+}
+
+DirectedGraph::node_id DeadLockDetector::getNodeId(const transaction_id tid) {
+ DirectedGraph::node_id node_id;
+ if (tid_node_mapping_->count(tid) == 0) {
+ // If it is not created, create it.
+ node_id = addNode(tid);
+ } else {
+ // Otherwise find it in the map.
+ node_id = (*tid_node_mapping_)[tid];
+ }
+ return node_id;
+}
+
+
+DirectedGraph::node_id DeadLockDetector::addNode(const transaction_id tid) {
+ const DirectedGraph::node_id node_id =
+ wait_for_graph_->addNodeUnchecked(tid);
+ tid_node_mapping_->emplace(tid, node_id);
+ return node_id;
+}
+
+std::vector<transaction_id> DeadLockDetector::getAllVictims() {
+ std::vector<transaction_id> result_victims;
+ wait_for_graph_.reset(new DirectedGraph());
+
+ // Critical region on LockTable starts here.
+ lock_table_->latchShared();
+ for (const auto &lock_control_block : *lock_table_) {
+ const LockTable::lock_own_list &own_list = lock_control_block.second.first;
+ const LockTable::lock_pending_list &pending_list =
+ lock_control_block.second.second;
+
+ for (const auto &owned_lock_info : own_list) {
+ const transaction_id owned_transaction = owned_lock_info.first;
+ const DirectedGraph::node_id owned_node = getNodeId(owned_transaction);
+
+ for (const auto &pending_lock_info : pending_list) {
+ const transaction_id pending_transaction = pending_lock_info.first;
+ const DirectedGraph::node_id pending_node = getNodeId(pending_transaction);
+
+ wait_for_graph_->addEdgeUnchecked(pending_node, owned_node);
+ }
+ }
+ }
+
+ lock_table_->unlatchShared();
+ // Critical region on LockTable ends here.
+
+ const CycleDetector cycle_detector(wait_for_graph_.get());
+ const std::vector<DirectedGraph::node_id> victim_nodes =
+ cycle_detector.chooseVictimsToBreakCycle();
+ for (const DirectedGraph::node_id node_id : victim_nodes) {
+ const transaction_id victim_tid = wait_for_graph_->getDataFromNode(node_id);
+ result_victims.push_back(victim_tid);
+ }
+
+ // Destroy the wait graph. It will be reconstructed kSleepDurationSeconds
+ // seconds later.
+ wait_for_graph_.reset();
+
+ return result_victims;
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/DeadLockDetector.hpp
----------------------------------------------------------------------
diff --git a/transaction/DeadLockDetector.hpp b/transaction/DeadLockDetector.hpp
new file mode 100644
index 0000000..6897afb
--- /dev/null
+++ b/transaction/DeadLockDetector.hpp
@@ -0,0 +1,156 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_TRANSACTION_DEAD_LOCK_DETECTOR_HPP_
+#define QUICKSTEP_TRANSACTION_DEAD_LOCK_DETECTOR_HPP_
+
+#include <atomic>
+#include <cstdint>
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "threading/Thread.hpp"
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/Transaction.hpp"
+
+namespace quickstep {
+namespace transaction {
+
+class LockTable;
+
+/** \addtogroup Transaction
+ * @{
+ */
+
+/**
+ * @brief Notification mechanism between LockManager and DeadLockDetector.
+ **/
+enum class DeadLockDetectorStatus {
+ kNotReady = 0,
+ kDone,
+ kQuit,
+};
+
+/**
+ * @brief Class for deadlock detection on wait-for graph.
+ **/
+class DeadLockDetector : public Thread {
+ public:
+ typedef std::unordered_map<transaction_id, DirectedGraph::node_id>
+ transaction_id_node_map;
+
+ /**
+ * @brief Constructor for DeadLockDetector.
+ *
+ * @param lock_table Pointer to lock table, which this class gets the
+ * necessary information.
+ * @param status Pointer to status object which will act as a message
+ * passing algorithm between LockManager.
+ * @param victims Message passing buffer betwen DeadLockDetector and
+ * LockManager.
+ **/
+ DeadLockDetector(LockTable *lock_table,
+ std::atomic<DeadLockDetectorStatus> *status,
+ std::vector<DirectedGraph::node_id> *victims);
+
+ void run() override;
+
+ /**
+ * @brief Adds pending information based on a resource conflict.
+ *
+ * @param pending Id of the transaction that waits for the resource lock.
+ * @param owner Id of the transaction that owns the resource lock.
+ */
+ void addPendingInfo(const transaction_id pending,
+ const transaction_id owner);
+
+ /**
+ * @brief Deletes pending information on a resource.
+ *
+ * @param pending Id of the transaction that waits for the resource lock.
+ * @param owner Id of the transaction that owns the resource lock.
+ *
+ * @warning This method is not implemented yet.
+ */
+ void deletePendingInfo(const transaction_id pending,
+ const transaction_id owner);
+
+ /**
+ * @brief Check whether first transaction waits for the latter.
+ *
+ * @param pending Id of the transaction which will be checked whether
+ * it waits for the other.
+ * @param owner Id of the transaction which will be checked whether
+ * it is waited by the first.
+ *
+ * @warning This method is not implemented yet.
+ */
+ bool isDependent(const transaction_id pending,
+ const transaction_id owner) const;
+
+ /**
+ * @brief Gives the ids of transactions that wait for the owner transaction.
+ *
+ * @param owner Id of the transaction whose the penders will be returned.
+ * @return Vector of transaction ids that wait for owner.
+ */
+ std::vector<transaction_id> getAllDependents(const transaction_id owner) const;
+
+ /**
+ * @brief Gives the ids of transaction that the pending transaction waits for.
+ * @warning This method is not implemented yet.
+
+ * @param pending Id of the transaction that is pending.
+ * @return Vector of transaction ids which the pending transaction waits for.
+ */
+ std::vector<transaction_id> getAllDependees(transaction_id pending);
+
+ /**
+ * @brief Gives the list of victims whose the terminations will end the cycle.
+ *
+ * @return Vector of victim transaction ids.
+ */
+ std::vector<transaction_id> getAllVictims();
+
+ private:
+ static constexpr std::int64_t kSleepDurationInSeconds = 5;
+
+ DirectedGraph::node_id getNodeId(const transaction_id tid);
+
+ DirectedGraph::node_id addNode(const transaction_id tid);
+
+ // Owned pointer to wait-for graph.
+ std::unique_ptr<DirectedGraph> wait_for_graph_;
+
+ // Mapping from TransactioId to NodeId in graph.
+ std::unique_ptr<transaction_id_node_map> tid_node_mapping_;
+
+ // Pointer to lock table. Dependence edges will be created
+ // by the information got from lock table.
+ LockTable *lock_table_;
+
+ std::atomic<DeadLockDetectorStatus> *status_;
+ std::vector<DirectedGraph::node_id> *victims_;
+};
+
+/** @} */
+
+} // namespace transaction
+} // namespace quickstep
+
+#endif // QUICKSTEP_TRANSACTION_DEAD_LOCK_DETECTOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/DirectedGraph.hpp
----------------------------------------------------------------------
diff --git a/transaction/DirectedGraph.hpp b/transaction/DirectedGraph.hpp
index 89ce9c6..16b551a 100644
--- a/transaction/DirectedGraph.hpp
+++ b/transaction/DirectedGraph.hpp
@@ -21,8 +21,8 @@
#include <algorithm>
#include <cstddef>
#include <cstdint>
+#include <iterator>
#include <memory>
-#include <stack>
#include <unordered_set>
#include <vector>
@@ -39,7 +39,7 @@ namespace transaction {
*/
/**
- * @brief Class for representing a directed graph. Vertices are transaction
+ * @brief Class for representing a directed graph. Vertices are transaction
* ids, edges are wait-for relations.
**/
class DirectedGraph {
@@ -54,34 +54,31 @@ class DirectedGraph {
/**
* @brief Adds a new node to the graph with the given transaction id.
* It does not check whether the transaction id is valid or not.
- * @warning Pointer ownership will pass to the graph, therefore it
- * should not be deleted.
*
- * @param data Pointer to the transaction id that will be contained
+ * @param transaction_id_payload Transaction id that will be contained
* in the node.
* @return Id of the newly created node.
**/
- inline node_id addNodeUnchecked(transaction_id *data) {
- nodes_.emplace_back(data);
+ inline
+ node_id addNodeUnchecked(const transaction_id transaction_id_payload) {
+ nodes_.emplace_back(transaction_id_payload);
return nodes_.size() - 1;
}
/**
* @brief Adds a new node to the graph with the given transaction id.
* It checks whether the transaction id is valid or not.
- * @warning Pointer ownership will pass to the graph, therefore it
- * should not be deleted.
*
- * @param data Pointer to the transaction id that will be contained
+ * @param transaction_id_payload Transaction id that will be contained
* in the node.
* @return Id of the newly created node.
**/
- inline node_id addNodeCheckExists(transaction_id *data) {
- for (std::vector<DirectedGraphNode>::const_iterator
- it = nodes_.cbegin(); it != nodes_.cend(); ++it) {
- CHECK(*data != it->getData());
+ node_id addNodeCheckExists(const transaction_id transaction_id_payload) {
+ for (const auto &node : nodes_) {
+ CHECK(transaction_id_payload != node.getData());
}
- nodes_.emplace_back(data);
+
+ nodes_.emplace_back(transaction_id_payload);
return nodes_.size() - 1;
}
@@ -91,10 +88,10 @@ class DirectedGraph {
* @warning Does not check arguments are legit. It may cause
* out of range errors.
*
- * @param fromNode The node that edge is orginated.
+ * @param fromNode The node that edge is originated.
* @param toNode The node that edge is ended.
**/
- inline void addEdgeUnchecked(node_id from_node, node_id to_node) {
+ void addEdgeUnchecked(node_id from_node, node_id to_node) {
nodes_[from_node].addOutgoingEdge(to_node);
}
@@ -105,7 +102,7 @@ class DirectedGraph {
* @param fromNode The node that edge is orginated.
* @param toNode The node that edge is ended.
**/
- inline void addEdgeCheckExists(node_id from_node, node_id to_node) {
+ void addEdgeCheckExists(node_id from_node, node_id to_node) {
CHECK(from_node < getNumNodes() && to_node < getNumNodes());
nodes_[from_node].addOutgoingEdge(to_node);
}
@@ -119,7 +116,7 @@ class DirectedGraph {
* @param toNode Id of the node that edge is ended.
* @return True if there is an edge, false otherwise.
**/
- inline bool hasEdge(node_id from_node, node_id to_node) const {
+ bool hasEdge(node_id from_node, node_id to_node) const {
DCHECK(from_node < getNumNodes() && to_node < getNumNodes());
return nodes_[from_node].hasOutgoingEdge(to_node);
}
@@ -130,7 +127,7 @@ class DirectedGraph {
* @param node Id of the node that the data is got from.
* @return Id of the transaction that this node contains.
**/
- inline transaction_id getDataFromNode(node_id node) const {
+ transaction_id getDataFromNode(node_id node) const {
DCHECK(node < getNumNodes());
return nodes_[node].getData();
}
@@ -140,7 +137,7 @@ class DirectedGraph {
*
* @return The number of nodes the graph has.
**/
- inline std::size_t getNumNodes() const {
+ std::size_t getNumNodes() const {
return nodes_.size();
}
@@ -158,18 +155,18 @@ class DirectedGraph {
// Class for representing a graph node.
class DirectedGraphNode {
public:
- explicit DirectedGraphNode(transaction_id *data)
- : data_(data) {}
+ explicit DirectedGraphNode(const transaction_id payload)
+ : transaction_id_payload_(payload) {}
- inline void addOutgoingEdge(node_id to_node) {
+ void addOutgoingEdge(node_id to_node) {
outgoing_edges_.insert(to_node);
}
- inline bool hasOutgoingEdge(node_id to_node) const {
+ bool hasOutgoingEdge(node_id to_node) const {
return outgoing_edges_.count(to_node) == 1;
}
- inline std::vector<node_id> getOutgoingEdges() const {
+ std::vector<node_id> getOutgoingEdges() const {
// TODO(hakan): Benchmark this version and the alternative which the
// function returns const reference and the uniqueness
// is imposed in the outgoing_edges_ as a vector.
@@ -179,13 +176,12 @@ class DirectedGraph {
return result;
}
- inline transaction_id getData() const {
- return *(data_.get());
+ transaction_id getData() const {
+ return transaction_id_payload_;
}
private:
- // Owner pointer to transaction id.
- std::unique_ptr<transaction_id> data_;
+ const transaction_id transaction_id_payload_;
// Endpoint nodes of outgoing edges originated from this node.
std::unordered_set<node_id> outgoing_edges_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/LockManager.cpp
----------------------------------------------------------------------
diff --git a/transaction/LockManager.cpp b/transaction/LockManager.cpp
new file mode 100644
index 0000000..da6181a
--- /dev/null
+++ b/transaction/LockManager.cpp
@@ -0,0 +1,237 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/LockManager.hpp"
+
+#include <cstdint>
+#include <stack>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "transaction/AccessMode.hpp"
+#include "transaction/DeadLockDetector.hpp"
+#include "transaction/LockRequest.hpp"
+#include "transaction/LockTable.hpp"
+#include "transaction/ResourceId.hpp"
+#include "transaction/Transaction.hpp"
+#include "transaction/TransactionTable.hpp"
+#include "utility/ThreadSafeQueue.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace transaction {
+
+DEFINE_uint64(max_try_incoming, 10000,
+ "The maximum number of tries that lock manager checks incoming "
+ "request buffer until the buffer is empty.");
+
+DEFINE_uint64(max_try_inner, 6000,
+ "The maximum number of tries that lock manager checks inner "
+ "request buffer until the buffer is empty.");
+
+LockManager::LockManager(ThreadSafeQueue<LockRequest> *incoming_requests,
+ ThreadSafeQueue<LockRequest> *permitted_requests)
+ : lock_table_(std::make_unique<LockTable>()),
+ transaction_table_(std::make_unique<TransactionTable>()),
+ detector_status_(DeadLockDetectorStatus::kNotReady),
+ deadlock_detector_(std::make_unique<DeadLockDetector>(lock_table_.get(),
+ &detector_status_,
+ &victim_result_)),
+ incoming_requests_(*incoming_requests),
+ permitted_requests_(*permitted_requests),
+ inner_pending_requests_() {
+}
+
+LockManager::~LockManager() {
+ deadlock_detector_->join();
+}
+
+void LockManager::run() {
+ deadlock_detector_->start();
+
+ const std::uint64_t kMaxTryIncoming =
+ static_cast<std::uint64_t>(FLAGS_max_try_incoming);
+ const std::uint64_t kMaxTryInner =
+ static_cast<std::uint64_t>(FLAGS_max_try_incoming);
+
+ while (true) {
+ for (std::uint64_t tries = 0; tries < kMaxTryIncoming; ++tries) {
+ if (!incoming_requests_.empty()) {
+ const LockRequest request = incoming_requests_.popOne();
+ if (request.getRequestType() == RequestType::kReleaseLocks) {
+ CHECK(releaseAllLocks(request.getTransactionId()))
+ << "Unexpected condition occured.";
+
+ } else if (acquireLock(request.getTransactionId(),
+ request.getResourceId(),
+ request.getAccessMode())) {
+ LOG(INFO) << "Transaction "
+ << std::to_string(request.getTransactionId())
+ << " is waiting " + request.getResourceId().toString();
+
+ inner_pending_requests_.push(request);
+ } else {
+ LOG(INFO) << "Transaction "
+ << std::to_string(request.getTransactionId())
+ << " acquired " + request.getResourceId().toString();
+
+ permitted_requests_.push(request);
+ }
+ }
+ }
+
+ for (std::uint64_t tries = 0; tries < kMaxTryInner; ++tries) {
+ if (!inner_pending_requests_.empty()) {
+ const LockRequest request = inner_pending_requests_.front();
+
+ if (acquireLock(request.getTransactionId(), request.getResourceId(),
+ request.getAccessMode())) {
+ inner_pending_requests_.pop();
+ permitted_requests_.push(request);
+ }
+ }
+ }
+
+ // Resolve deadlocks.
+ killVictims();
+ }
+}
+
+bool LockManager::acquireLock(const transaction_id tid,
+ const ResourceId &rid,
+ const AccessMode &access_mode) {
+ std::stack<std::pair<ResourceId, AccessMode>> stack;
+ ResourceId current_rid = rid;
+ AccessMode current_access_mode = access_mode;
+ stack.push(std::make_pair(current_rid, current_access_mode));
+
+ while (current_rid.hasParent()) {
+ current_rid = current_rid.getParentResourceId();
+ current_access_mode = (current_access_mode.isShareLock() ||
+ current_access_mode.isIntentionShareLock())
+ ? AccessMode(AccessMode::IsLockMode())
+ : AccessMode(AccessMode::IxLockMode());
+
+ stack.push(std::make_pair(current_rid, current_access_mode));
+ }
+
+ lock_table_->latchExclusive();
+
+ while (!stack.empty()) {
+ const std::pair<ResourceId, AccessMode> pair_to_pick = stack.top();
+ const ResourceId rid_to_pick = pair_to_pick.first;
+ const AccessMode access_mode_to_pick = pair_to_pick.second;
+
+ if (!acquireLockInternal(tid, rid_to_pick, access_mode_to_pick)) {
+ lock_table_->unlatchExclusive();
+ return false;
+ }
+ stack.pop();
+ }
+ lock_table_->unlatchExclusive();
+ return true;
+}
+
+bool LockManager::releaseAllLocks(const transaction_id tid,
+ const bool latch_table) {
+ const std::vector<ResourceId> resource_ids
+ = transaction_table_->getResourceIdList(tid);
+ const TransactionTableResult transaction_deleted
+ = transaction_table_->deleteTransaction(tid);
+
+ CHECK(transaction_deleted != TransactionTableResult::kTransactionDeleteError)
+ << "In LockManager.releaseAllLocks: Transaction could not be deleted!";
+
+ if (latch_table) {
+ lock_table_->latchExclusive();
+ }
+ for (const auto &resource_id : resource_ids) {
+ const LockTableResult lock_deleted = lock_table_->deleteLock(tid, resource_id);
+
+ LOG(INFO) << "Transaction "
+ << std::to_string(tid)
+ << " released lock:"
+ << resource_id.toString();
+ CHECK(lock_deleted != LockTableResult::kDeleteError)
+ << "In LockManager.releaseAllLock lock could not be deleted from "
+ "LockTable";
+ }
+ if (latch_table) {
+ lock_table_->unlatchExclusive();
+ }
+ return true;
+}
+
+bool LockManager::acquireLockInternal(const transaction_id tid,
+ const ResourceId &rid,
+ const AccessMode &access_mode) {
+ const LockTableResult lock_result = lock_table_->putLock(tid, rid, access_mode);
+ CHECK(lock_result != LockTableResult::kPutError)
+ << "Unexpected result in LockManager.acquireLockInternal";
+
+ switch (lock_result) {
+ case LockTableResult::kAlreadyInOwned: {
+ return true;
+ }
+ case LockTableResult::kPlacedInOwned: {
+ const TransactionTableResult transaction_result
+ = transaction_table_->putOwnEntry(tid, rid, access_mode);
+ CHECK(transaction_result == TransactionTableResult::kPlacedInOwned)
+ << "Unexpected result in LockManager.acquireLockInternal: "
+ "Mismatch of table results: LockTable entry is owned, "
+ "whereas TransactionTable entry is not owned.";
+ return true;
+ }
+ case LockTableResult::kAlreadyInPending: {
+ return false;
+ }
+ case LockTableResult::kPlacedInPending: {
+ const TransactionTableResult transaction_result =
+ transaction_table_->putPendingEntry(tid, rid, access_mode);
+ CHECK(transaction_result == TransactionTableResult::kPlacedInPending)
+ << "Unexpected result in LockManager.acquireLockInternal: "
+ "Mismatch of table results: LockTable entry is pending, "
+ "whereas TransactionTable entry is not pending";
+ return false;
+ }
+ default: {
+ return false;
+ }
+ }
+}
+
+void LockManager::killVictims() {
+ if (detector_status_.load() == DeadLockDetectorStatus::kDone) {
+ lock_table_->latchExclusive();
+ for (const auto victim_transaction_id : victim_result_) {
+ releaseAllLocks(victim_transaction_id, false);
+ // TODO(Hakan): Find a way to kill transaction, so that requests with this
+ // tid should be ignored.
+ LOG(INFO) << "Killed transaction "
+ << std::to_string(victim_transaction_id);
+ }
+ lock_table_->unlatchExclusive();
+ }
+ victim_result_.clear();
+ detector_status_.store(DeadLockDetectorStatus::kNotReady);
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/LockManager.hpp
----------------------------------------------------------------------
diff --git a/transaction/LockManager.hpp b/transaction/LockManager.hpp
new file mode 100644
index 0000000..40ee6c8
--- /dev/null
+++ b/transaction/LockManager.hpp
@@ -0,0 +1,128 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_TRANSACTION_LOCK_MANAGER_HPP_
+#define QUICKSTEP_TRANSACTION_LOCK_MANAGER_HPP_
+
+#include <atomic>
+#include <memory>
+#include <queue>
+#include <vector>
+
+#include "threading/Thread.hpp"
+#include "transaction/DeadLockDetector.hpp"
+#include "transaction/LockRequest.hpp"
+#include "transaction/Transaction.hpp"
+
+namespace quickstep {
+
+template <typename T> class ThreadSafeQueue;
+
+namespace transaction {
+
+class AccessMode;
+class LockTable;
+class ResourceId;
+class TransactionTable;
+
+/** \addtogroup Transaction
+ * @{
+ **/
+
+/**
+ * @brief Class for centralized location of acquisition and releasing
+ * of resource locks.
+ **/
+class LockManager : public Thread {
+ public:
+ /**
+ * @brief Constructor
+ *
+ * @param incoming_requests Queue for the lock requests that are waiting
+ * for the permission.
+ * @param permitted_requests Queue for the lock requests that are granted
+ * permission.
+ **/
+ LockManager(ThreadSafeQueue<LockRequest> *incoming_requests,
+ ThreadSafeQueue<LockRequest> *permitted_requests);
+
+ /**
+ * @brief Destructor for LockManager. It handles the thread
+ * joins that it owns.
+ **/
+ ~LockManager();
+
+ /**
+ * @brief Method for defining the LockManager's thread main logic.
+ *
+ * @warning Users must not use this function directly. Instead use
+ * start() method inherited from the Thread class.
+ **/
+ void run() override;
+
+ /**
+ * @brief Acquires the lock on resource with specified access mode.
+ *
+ * @param tid Id of the transaction which the resource lock is acquired for.
+ * @param rid Id of the resource on which the resource lock is acquired.
+ * @param access_mode Permissible access mode on resource.
+ *
+ * @return True if it can acquire the lock from root to leaf lock hierarchy,
+ * false otherwise.
+ **/
+ bool acquireLock(const transaction_id tid,
+ const ResourceId &rid,
+ const AccessMode &access_mode);
+
+ /**
+ * @brief Releases all locks hold by the transaction.
+ *
+ * @param tid Id of the transaction whose locks will be released.
+ * @param latch_table If it is true, the method latch the whole
+ * lock table, which is default.
+ * @return True if tid releases all of its locks.
+ **/
+ bool releaseAllLocks(const transaction_id tid,
+ const bool latch_table = true);
+
+ /**
+ * @brief Release the locks acquired by the transactions contained
+ * in victim buffer to break the deadlock.
+ **/
+ void killVictims();
+
+ private:
+ bool acquireLockInternal(const transaction_id tid,
+ const ResourceId &rid,
+ const AccessMode &access_mode);
+
+ std::unique_ptr<LockTable> lock_table_;
+ std::unique_ptr<TransactionTable> transaction_table_;
+ std::atomic<DeadLockDetectorStatus> detector_status_;
+ std::vector<transaction_id> victim_result_;
+ std::unique_ptr<DeadLockDetector> deadlock_detector_;
+ ThreadSafeQueue<LockRequest> &incoming_requests_;
+ ThreadSafeQueue<LockRequest> &permitted_requests_;
+ std::queue<LockRequest> inner_pending_requests_;
+};
+
+/** @} */
+
+} // namespace transaction
+} // namespace quickstep
+
+#endif // QUICKSTEP_TRANSACTION_LOCK_MANAGER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/LockTable.cpp
----------------------------------------------------------------------
diff --git a/transaction/LockTable.cpp b/transaction/LockTable.cpp
index 77986f6..7b568fc 100644
--- a/transaction/LockTable.cpp
+++ b/transaction/LockTable.cpp
@@ -18,15 +18,11 @@
#include "transaction/LockTable.hpp"
#include <list>
-#include <unordered_map>
#include <utility>
-#include "threading/SharedMutex.hpp"
#include "transaction/AccessMode.hpp"
#include "transaction/Lock.hpp"
-#include "transaction/ResourceId.hpp"
#include "transaction/Transaction.hpp"
-#include "utility/Macros.hpp"
namespace quickstep {
namespace transaction {
@@ -34,7 +30,7 @@ namespace transaction {
LockTableResult
LockTable::putLock(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
// TODO(hakan): Lock upgrade is not supported.
lock_list_pair &lock_list_pair = internal_map_[rid];
@@ -47,7 +43,7 @@ LockTable::putLock(const transaction_id tid,
for (lock_own_list::const_iterator it = lock_own_list.cbegin();
it != lock_own_list.cend(); ++it) {
if (it->first == tid && it->second.getAccessMode() == access_mode) {
- return LockTableResult::kALREADY_IN_OWNED;
+ return LockTableResult::kAlreadyInOwned;
}
}
@@ -56,7 +52,7 @@ LockTable::putLock(const transaction_id tid,
for (lock_pending_list::const_iterator it = lock_pending_list.cbegin();
it != lock_pending_list.cend(); ++it) {
if (it->first == tid && it->second.getAccessMode() == access_mode) {
- return LockTableResult::kALREADY_IN_PENDING;
+ return LockTableResult::kAlreadyInPending;
}
}
@@ -68,18 +64,18 @@ LockTable::putLock(const transaction_id tid,
if (!access_mode.isCompatible(it->second.getAccessMode())) {
lock_pending_list.push_back(std::make_pair(tid,
Lock(rid, access_mode)));
- return LockTableResult::kPLACED_IN_PENDING;
+ return LockTableResult::kPlacedInPending;
}
}
lock_own_list.push_back(std::make_pair(tid, Lock(rid, access_mode)));
- return LockTableResult::kPLACED_IN_OWNED;
+ return LockTableResult::kPlacedInOwned;
} else {
// If the pending list is not empty, even if the lock request is compatible
// with other owned lock entries, we put the new request into the pending
// list to eliminate starvation.
lock_pending_list.push_back(std::make_pair(tid, Lock(rid, access_mode)));
- return LockTableResult::kPLACED_IN_PENDING;
+ return LockTableResult::kPlacedInPending;
}
}
@@ -105,7 +101,7 @@ LockTable::deleteLock(const transaction_id tid,
// compatible with the remaining owned entries.
movePendingToOwned(rid);
- return LockTableResult::kDEL_FROM_OWNED;
+ return LockTableResult::kDeleteFromOwned;
}
}
@@ -116,13 +112,13 @@ LockTable::deleteLock(const transaction_id tid,
if (it->first == tid) {
// If it exists, erase it from pending list.
lock_pending_list.erase(it);
- return LockTableResult::kDEL_FROM_PENDING;
+ return LockTableResult::kDeleteFromPending;
}
}
// Execution reaches here, if we cannot find the corresponding lock entry
// in the both list.
- return LockTableResult::kDEL_ERROR;
+ return LockTableResult::kDeleteError;
}
void LockTable::movePendingToOwned(const ResourceId &rid) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/LockTable.hpp
----------------------------------------------------------------------
diff --git a/transaction/LockTable.hpp b/transaction/LockTable.hpp
index 5a0612e..529db12 100644
--- a/transaction/LockTable.hpp
+++ b/transaction/LockTable.hpp
@@ -23,7 +23,6 @@
#include <utility>
#include "threading/SharedMutex.hpp"
-#include "transaction/AccessMode.hpp"
#include "transaction/Lock.hpp"
#include "transaction/ResourceId.hpp"
#include "transaction/Transaction.hpp"
@@ -32,6 +31,8 @@
namespace quickstep {
namespace transaction {
+class AccessMode;
+
/** \addtogroup Transaction
* @{
*/
@@ -40,14 +41,14 @@ namespace transaction {
* @brief Represents different results for LockTable's methods.
**/
enum class LockTableResult {
- kPLACED_IN_OWNED = 0,
- kPLACED_IN_PENDING,
- kALREADY_IN_OWNED,
- kALREADY_IN_PENDING,
- kDEL_FROM_OWNED,
- kDEL_FROM_PENDING,
- kDEL_ERROR,
- kPUT_ERROR,
+ kPlacedInOwned = 0,
+ kPlacedInPending,
+ kAlreadyInOwned,
+ kAlreadyInPending,
+ kDeleteFromOwned,
+ kDeleteFromPending,
+ kDeleteError,
+ kPutError,
};
/**
@@ -79,32 +80,31 @@ class LockTable {
* @param rid Id of the resource to be locked.
* @param access_mode Access mode of the lock.
*
- * @return LockTableResult::kPLACED_IN_OWNED if lock is granted,
- * LockTableResult::kPLACED_IN_PENDING if lock is not granted,
- * LockTableResult::kALREADY_IN_OWNED if lock has been
+ * @return LockTableResult::kPlacedInOwned if lock is granted,
+ * LockTableResult::kPlacedInPending if lock is not granted,
+ * LockTableResult::kAlreadyInOwned if lock has been
* already granted,
- * LockTableResult::kALREADY_IN_PENDING if lock has been
+ * LockTableResult::kAlreadyInPending if lock has been
* already pending.
**/
LockTableResult putLock(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Deletes the lock entry.
*
* @param tid Id of the transaction that owns or awaits.
* @param rid Id of resource that the lock covers.
*
- * @return LockTableResult::kDEL_FROM_OWNED if the lock is deleted from
+ * @return LockTableResult::kDelFromOwned if the lock is deleted from
* owned list,
- * LockTableResult::kDEL_FROM_PENDING if the lock is deleted from
+ * LockTableResult::kDelFromPending if the lock is deleted from
* pending list,
- * LockTableResult::kDEL_ERROR if the lock cannot be found
+ * LockTableResult::kDelError if the lock cannot be found
**/
LockTableResult deleteLock(const transaction_id tid,
const ResourceId &rid);
-
/**
* @brief Iterator for begin position.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/ResourceId.hpp
----------------------------------------------------------------------
diff --git a/transaction/ResourceId.hpp b/transaction/ResourceId.hpp
index b9d1cdf..3a770dd 100644
--- a/transaction/ResourceId.hpp
+++ b/transaction/ResourceId.hpp
@@ -19,7 +19,6 @@
#define QUICKSTEP_TRANSACTION_RESOURCE_ID_HPP_
#include <cstddef>
-#include <limits>
#include <string>
#include "catalog/CatalogTypedefs.hpp"
@@ -202,10 +201,10 @@ class ResourceId {
return tuple_id_ == kTupleIdPlaceholder;
}
- const database_id db_id_;
- const relation_id rel_id_;
- const block_id block_id_;
- const tuple_id tuple_id_;
+ database_id db_id_;
+ relation_id rel_id_;
+ block_id block_id_;
+ tuple_id tuple_id_;
};
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/StronglyConnectedComponents.cpp
----------------------------------------------------------------------
diff --git a/transaction/StronglyConnectedComponents.cpp b/transaction/StronglyConnectedComponents.cpp
index 89daf46..f50ed85 100644
--- a/transaction/StronglyConnectedComponents.cpp
+++ b/transaction/StronglyConnectedComponents.cpp
@@ -17,7 +17,6 @@
#include "transaction/StronglyConnectedComponents.hpp"
-#include <cstddef>
#include <cstdint>
#include <stack>
#include <unordered_map>
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/Transaction.cpp
----------------------------------------------------------------------
diff --git a/transaction/Transaction.cpp b/transaction/Transaction.cpp
deleted file mode 100644
index 3478d01..0000000
--- a/transaction/Transaction.cpp
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#include "transaction/Transaction.hpp"
-
-#include <functional>
-
-namespace quickstep {
-
-namespace transaction {
-
-TransactionId Transaction::getTransactionId() const {
- return tid_;
-}
-
-void Transaction::setStatus(TransactionStatus status) {
- status_ = status;
-}
-
-TransactionStatus Transaction::getStatus() const {
- return status_;
-}
-
-bool Transaction::operator==(const Transaction &other) const {
- return tid_ == other.tid_;
-}
-
-std::size_t Transaction::TransactionHasher::operator()(const Transaction &transaction) const {
- return std::hash<TransactionId>()(transaction.tid_);
-}
-
-} // namespace transaction
-
-} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/TransactionTable.cpp
----------------------------------------------------------------------
diff --git a/transaction/TransactionTable.cpp b/transaction/TransactionTable.cpp
index 993703a..3e37439 100644
--- a/transaction/TransactionTable.cpp
+++ b/transaction/TransactionTable.cpp
@@ -33,7 +33,7 @@ namespace transaction {
TransactionTableResult
TransactionTable::putOwnEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
transaction_list_pair &transaction_list_pair = internal_map_[tid];
transaction_own_list &transaction_own_list = transaction_list_pair.first;
@@ -45,7 +45,7 @@ TransactionTable::putOwnEntry(const transaction_id tid,
TransactionTableResult
TransactionTable::putPendingEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
transaction_list_pair &transaction_list_pair = internal_map_[tid];
transaction_pending_list &transaction_pending_list
= transaction_list_pair.second;
@@ -59,7 +59,7 @@ TransactionTable::putPendingEntry(const transaction_id tid,
TransactionTableResult
TransactionTable::deleteOwnEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
transaction_list_pair &transaction_list_pair = internal_map_[tid];
transaction_own_list &transaction_own_list = transaction_list_pair.first;
@@ -79,7 +79,7 @@ TransactionTable::deleteOwnEntry(const transaction_id tid,
TransactionTableResult
TransactionTable::deletePendingEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
transaction_list_pair &transaction_list_pair = internal_map_[tid];
transaction_pending_list &transaction_pending_list
= transaction_list_pair.second;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/TransactionTable.hpp
----------------------------------------------------------------------
diff --git a/transaction/TransactionTable.hpp b/transaction/TransactionTable.hpp
index a5e1da4..29df536 100644
--- a/transaction/TransactionTable.hpp
+++ b/transaction/TransactionTable.hpp
@@ -23,7 +23,6 @@
#include <utility>
#include <vector>
-#include "transaction/AccessMode.hpp"
#include "transaction/Lock.hpp"
#include "transaction/ResourceId.hpp"
#include "transaction/Transaction.hpp"
@@ -32,6 +31,8 @@
namespace quickstep {
namespace transaction {
+class AccessMode;
+
/** \addtogroup Transaction
* @{
*/
@@ -81,7 +82,7 @@ class TransactionTable {
**/
TransactionTableResult putOwnEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Puts a pending entry of the given resource id in the given
@@ -95,7 +96,7 @@ class TransactionTable {
**/
TransactionTableResult putPendingEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Deletes the owned entry corresponding to the resource id
@@ -110,7 +111,7 @@ class TransactionTable {
**/
TransactionTableResult deleteOwnEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Deletes the pending entry corresponding to the resource id
@@ -124,7 +125,7 @@ class TransactionTable {
**/
TransactionTableResult deletePendingEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Returns a vector of resource ids which the corresponding transaction
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/tests/AccessMode_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/AccessMode_unittest.cpp b/transaction/tests/AccessMode_unittest.cpp
index fa51525..3287fb0 100644
--- a/transaction/tests/AccessMode_unittest.cpp
+++ b/transaction/tests/AccessMode_unittest.cpp
@@ -25,12 +25,12 @@ namespace transaction {
class AccessModeTest : public ::testing::Test {
protected:
AccessModeTest()
- : nl_mode_(AccessModeType::kNoLock),
- is_mode_(AccessModeType::kIsLock),
- ix_mode_(AccessModeType::kIxLock),
- s_mode_(AccessModeType::kSLock),
- six_mode_(AccessModeType::kSixLock),
- x_mode_(AccessModeType::kXLock) {
+ : nl_mode_(AccessMode::NoLockMode()),
+ is_mode_(AccessMode::IsLockMode()),
+ ix_mode_(AccessMode::IxLockMode()),
+ s_mode_(AccessMode::SLockMode()),
+ six_mode_(AccessMode::SixLockMode()),
+ x_mode_(AccessMode::XLockMode()) {
}
const AccessMode nl_mode_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/tests/CycleDetector_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/CycleDetector_unittest.cpp b/transaction/tests/CycleDetector_unittest.cpp
new file mode 100644
index 0000000..6edaa63
--- /dev/null
+++ b/transaction/tests/CycleDetector_unittest.cpp
@@ -0,0 +1,157 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/CycleDetector.hpp"
+
+#include <cstdint>
+#include <memory>
+#include <stack>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/Transaction.hpp"
+
+#include "gtest/gtest.h"
+
+namespace quickstep {
+namespace transaction {
+
+class CycleDetectorTest : public testing::Test {
+ protected:
+ const std::uint64_t kNumberOfTransactions = 12;
+
+ CycleDetectorTest()
+ : wait_for_graph_(std::make_unique<DirectedGraph>()) {
+ }
+
+ virtual void SetUp() {
+ std::vector<transaction_id> transactions(kNumberOfTransactions);
+ for (std::uint64_t i = 0; i < kNumberOfTransactions; ++i) {
+ transactions.push_back(transaction_id(i));
+ }
+
+ std::vector<DirectedGraph::node_id> node_ids;
+ for (std::uint64_t i = 0; i < kNumberOfTransactions; ++i) {
+ node_ids.push_back(wait_for_graph_->addNodeUnchecked(transactions[i]));
+ }
+ }
+
+ void initializeCycleDetector() {
+ for (const auto &edge : edges_) {
+ wait_for_graph_->addEdgeUnchecked(edge.first, edge.second);
+ }
+
+ cycle_detector_.reset(new CycleDetector(wait_for_graph_.get()));
+ }
+
+ void checkVictims(
+ const std::unordered_set<DirectedGraph::node_id> &expected_victims) {
+ const std::vector<DirectedGraph::node_id> victims =
+ cycle_detector_->chooseVictimsToBreakCycle();
+
+ std::unordered_set<DirectedGraph::node_id> remaining_nodes;
+
+ for (DirectedGraph::node_id node = 0; node < wait_for_graph_->getNumNodes();
+ ++node) {
+ if (std::find(victims.begin(), victims.end(), node) == victims.end()) {
+ // Node is not in victims, then insert it to remaining set.
+ remaining_nodes.insert(node);
+ }
+ }
+
+ for (const auto node : remaining_nodes) {
+ ASSERT_FALSE(isSelfReachableNode(node, remaining_nodes));
+ }
+ }
+
+ bool isSelfReachableNode(
+ const DirectedGraph::node_id start_node,
+ const std::unordered_set<DirectedGraph::node_id> &node_set) {
+ std::unordered_set<DirectedGraph::node_id> marked_nodes;
+ std::stack<DirectedGraph::node_id> to_be_visied_nodes;
+
+ const std::vector<DirectedGraph::node_id> neighbors_of_start_node =
+ wait_for_graph_->getAdjacentNodes(start_node);
+ for (const auto node : neighbors_of_start_node) {
+ marked_nodes.insert(node);
+ to_be_visied_nodes.push(node);
+ }
+
+ while (!to_be_visied_nodes.empty()) {
+ const DirectedGraph::node_id current_node = to_be_visied_nodes.top();
+ to_be_visied_nodes.pop();
+ if (current_node == start_node) {
+ return true;
+ }
+ if (node_set.count(current_node) == 1 &&
+ marked_nodes.count(current_node) == 0) {
+ // Means, we did not visited this node yet, and it is in the node set,
+ // so we should process it (mark it and push all of its neighbors
+ // into stack).
+ marked_nodes.insert(current_node);
+ const auto neighbors = wait_for_graph_->getAdjacentNodes(current_node);
+ for (const auto neighbor : neighbors) {
+ to_be_visied_nodes.push(neighbor);
+ }
+ }
+ }
+ return false;
+ }
+
+ std::vector<std::pair<DirectedGraph::node_id, DirectedGraph::node_id>> edges_;
+ std::unique_ptr<DirectedGraph> wait_for_graph_;
+ std::unique_ptr<CycleDetector> cycle_detector_;
+};
+
+TEST_F(CycleDetectorTest, Interleaving) {
+ edges_ = {{0, 1},
+ {1, 0}};
+
+ initializeCycleDetector();
+
+ std::unordered_set<DirectedGraph::node_id> expected_victims = {1};
+
+ checkVictims(expected_victims);
+}
+
+TEST_F(CycleDetectorTest, MultipleCycle) {
+ // This edge contains lots of cycles of degree 1, 2 and 3.
+ edges_ = {{0, 1},
+ {1, 2}, {1, 3}, {1, 4},
+ {2, 5},
+ {3, 4}, {3, 6},
+ {4, 1}, {4, 5}, {4, 6},
+ {5, 2}, {5, 7},
+ {6, 7}, {6, 9},
+ {7, 6},
+ {8, 6},
+ {9, 8}, {9, 10},
+ {10, 11},
+ {11, 9}};
+
+ initializeCycleDetector();
+
+ std::unordered_set<DirectedGraph::node_id> expected_victims
+ = {4, 5, 7, 8, 9, 10, 11};
+
+ checkVictims(expected_victims);
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/tests/DeadLockDetector_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/DeadLockDetector_unittest.cpp b/transaction/tests/DeadLockDetector_unittest.cpp
new file mode 100644
index 0000000..bc65ef5
--- /dev/null
+++ b/transaction/tests/DeadLockDetector_unittest.cpp
@@ -0,0 +1,96 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/DeadLockDetector.hpp"
+
+#include <atomic>
+#include <memory>
+#include <vector>
+
+#include "transaction/AccessMode.hpp"
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/LockTable.hpp"
+#include "transaction/ResourceId.hpp"
+#include "transaction/Transaction.hpp"
+
+#include "gtest/gtest.h"
+
+namespace quickstep {
+namespace transaction {
+
+class DeadLockDetectorTest : public ::testing::Test {
+ protected:
+ DeadLockDetectorTest()
+ : lock_table_(std::make_unique<LockTable>()),
+ status_(DeadLockDetectorStatus::kDone) {
+ }
+
+ std::unique_ptr<LockTable> lock_table_;
+ std::atomic<DeadLockDetectorStatus> status_;
+ std::vector<DirectedGraph::node_id> victims_;
+};
+
+TEST_F(DeadLockDetectorTest, SimpleCycle) {
+ const transaction_id transaction_one(1), transaction_two(2);
+ const ResourceId resource_one(1, 2), resource_two(4, 5);
+
+ const AccessMode x_lock_mode(AccessMode::XLockMode());
+
+ // Produce a conflicting schedule.
+ // Transaction 1 will acquire X lock on resource 1.
+ lock_table_->putLock(transaction_one,
+ resource_one,
+ x_lock_mode);
+
+ // Transaction 2 will acquire X lock on resource 2.
+ lock_table_->putLock(transaction_two,
+ resource_two,
+ x_lock_mode);
+
+ // Transaction 1 will try to acquire X lock on resource 2,
+ // but it will fail since Transaction 2 has already acquired
+ // X lock on resource 2.
+ lock_table_->putLock(transaction_one,
+ resource_two,
+ x_lock_mode);
+
+ // Transaction 2 will try to acquire X lock on resource 1,
+ // but it will fail since Transaction 1 has already acquired
+ // X lock on resource 2.
+ lock_table_->putLock(transaction_two,
+ resource_one,
+ x_lock_mode);
+
+ // Run deadlock detector.
+ DeadLockDetector deadlock_detector(lock_table_.get(), &status_, &victims_);
+ status_.store(DeadLockDetectorStatus::kNotReady);
+
+ deadlock_detector.start();
+
+ // Signal deadlock detector.
+ while (status_.load() == DeadLockDetectorStatus::kNotReady) {
+ }
+
+ status_.store(DeadLockDetectorStatus::kQuit);
+ deadlock_detector.join();
+
+ // Victim size must be 1.
+ ASSERT_EQ(1u, victims_.size());
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/tests/DirectedGraph_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/DirectedGraph_unittest.cpp b/transaction/tests/DirectedGraph_unittest.cpp
index 43ad972..00fe276 100644
--- a/transaction/tests/DirectedGraph_unittest.cpp
+++ b/transaction/tests/DirectedGraph_unittest.cpp
@@ -26,101 +26,96 @@
namespace quickstep {
namespace transaction {
-TEST(DirectedGraphTest, AddNode) {
- // Prepare the data, but do not include in the graph.
- DirectedGraph wait_for_graph;
- transaction_id *tid3 = new transaction_id(3);
- transaction_id *tid4 = new transaction_id(4);
- transaction_id *tid5 = new transaction_id(5);
- transaction_id *tid6 = new transaction_id(6);
-
+class DirectedGraphTest : public ::testing::Test {
+ protected:
+ DirectedGraphTest()
+ : tid3_(3),
+ tid4_(4),
+ tid5_(5),
+ tid6_(6) {
+ }
+
+ DirectedGraph wait_for_graph_;
+ transaction_id tid3_;
+ transaction_id tid4_;
+ transaction_id tid5_;
+ transaction_id tid6_;
+};
+
+TEST_F(DirectedGraphTest, AddNode) {
// The nodes are not added yet, total no of nodesshould be zero.
- EXPECT_EQ(0u, wait_for_graph.getNumNodes());
+ EXPECT_EQ(0u, wait_for_graph_.getNumNodes());
- wait_for_graph.addNodeUnchecked(tid3);
+ wait_for_graph_.addNodeUnchecked(tid3_);
// One node is added.
- EXPECT_EQ(1u, wait_for_graph.getNumNodes());
+ EXPECT_EQ(1u, wait_for_graph_.getNumNodes());
- wait_for_graph.addNodeUnchecked(tid4);
+ wait_for_graph_.addNodeUnchecked(tid4_);
// Another node is added.
- EXPECT_EQ(2u, wait_for_graph.getNumNodes());
+ EXPECT_EQ(2u, wait_for_graph_.getNumNodes());
- wait_for_graph.addNodeUnchecked(tid5);
- wait_for_graph.addNodeUnchecked(tid6);
+ wait_for_graph_.addNodeUnchecked(tid5_);
+ wait_for_graph_.addNodeUnchecked(tid6_);
// Total no of nodes should be 4 right now.
- EXPECT_EQ(4u, wait_for_graph.getNumNodes());
+ EXPECT_EQ(4u, wait_for_graph_.getNumNodes());
}
-TEST(DirectedGraphTest, AddEdge) {
- // Prepare the graph.
- DirectedGraph wait_for_graph;
- transaction_id *tid3 = new transaction_id(3);
- transaction_id *tid4 = new transaction_id(4);
- transaction_id *tid5 = new transaction_id(5);
- transaction_id *tid6 = new transaction_id(6);
-
- DirectedGraph::node_id nid3 = wait_for_graph.addNodeUnchecked(tid3);
- DirectedGraph::node_id nid6 = wait_for_graph.addNodeUnchecked(tid6);
- DirectedGraph::node_id nid4 = wait_for_graph.addNodeUnchecked(tid4);
- DirectedGraph::node_id nid5 = wait_for_graph.addNodeUnchecked(tid5);
+TEST_F(DirectedGraphTest, AddEdge) {
+ DirectedGraph::node_id nid3 = wait_for_graph_.addNodeUnchecked(tid3_);
+ DirectedGraph::node_id nid6 = wait_for_graph_.addNodeUnchecked(tid6_);
+ DirectedGraph::node_id nid4 = wait_for_graph_.addNodeUnchecked(tid4_);
+ DirectedGraph::node_id nid5 = wait_for_graph_.addNodeUnchecked(tid5_);
// Add edges.
- wait_for_graph.addEdgeUnchecked(nid3, nid5);
- wait_for_graph.addEdgeUnchecked(nid6, nid4);
- wait_for_graph.addEdgeUnchecked(nid3, nid6);
- wait_for_graph.addEdgeUnchecked(nid4, nid6);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid5);
+ wait_for_graph_.addEdgeUnchecked(nid6, nid4);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid6);
+ wait_for_graph_.addEdgeUnchecked(nid4, nid6);
// Check whether the edges are already there.
- EXPECT_TRUE(wait_for_graph.hasEdge(nid3, nid5));
- EXPECT_TRUE(wait_for_graph.hasEdge(nid6, nid4));
- EXPECT_TRUE(wait_for_graph.hasEdge(nid4, nid6));
- EXPECT_TRUE(wait_for_graph.hasEdge(nid3, nid6));
+ EXPECT_TRUE(wait_for_graph_.hasEdge(nid3, nid5));
+ EXPECT_TRUE(wait_for_graph_.hasEdge(nid6, nid4));
+ EXPECT_TRUE(wait_for_graph_.hasEdge(nid4, nid6));
+ EXPECT_TRUE(wait_for_graph_.hasEdge(nid3, nid6));
// Check non-existent edges.
- EXPECT_FALSE(wait_for_graph.hasEdge(nid5, nid3));
- EXPECT_FALSE(wait_for_graph.hasEdge(nid6, nid3));
- EXPECT_FALSE(wait_for_graph.hasEdge(nid4, nid5));
+ EXPECT_FALSE(wait_for_graph_.hasEdge(nid5, nid3));
+ EXPECT_FALSE(wait_for_graph_.hasEdge(nid6, nid3));
+ EXPECT_FALSE(wait_for_graph_.hasEdge(nid4, nid5));
}
-TEST(DirectedGraphTest, GetAdjacentNodes) {
- // Prepare the graph.
- DirectedGraph wait_for_graph;
- transaction_id *tid3 = new transaction_id(3);
- transaction_id *tid4 = new transaction_id(4);
- transaction_id *tid5 = new transaction_id(5);
- transaction_id *tid6 = new transaction_id(6);
-
+TEST_F(DirectedGraphTest, GetAdjacentNodes) {
// Add 4 disconnected nodes to the graph.
- DirectedGraph::node_id nid3 = wait_for_graph.addNodeUnchecked(tid3);
- DirectedGraph::node_id nid6 = wait_for_graph.addNodeUnchecked(tid6);
- DirectedGraph::node_id nid4 = wait_for_graph.addNodeUnchecked(tid4);
- DirectedGraph::node_id nid5 = wait_for_graph.addNodeUnchecked(tid5);
+ DirectedGraph::node_id nid3 = wait_for_graph_.addNodeUnchecked(tid3_);
+ DirectedGraph::node_id nid6 = wait_for_graph_.addNodeUnchecked(tid6_);
+ DirectedGraph::node_id nid4 = wait_for_graph_.addNodeUnchecked(tid4_);
+ DirectedGraph::node_id nid5 = wait_for_graph_.addNodeUnchecked(tid5_);
- std::vector<DirectedGraph::node_id> result1 = wait_for_graph.getAdjacentNodes(nid3);
+ std::vector<DirectedGraph::node_id> result1 = wait_for_graph_.getAdjacentNodes(nid3);
// nid3 has no edge to other nodes.
EXPECT_EQ(0u, result1.size());
// Now nid3 has outgoing edge to nid4 and nid5.
- wait_for_graph.addEdgeUnchecked(nid3, nid4);
- wait_for_graph.addEdgeUnchecked(nid3, nid5);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid4);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid5);
- std::vector<DirectedGraph::node_id> result2 = wait_for_graph.getAdjacentNodes(nid3);
+ std::vector<DirectedGraph::node_id> result2 = wait_for_graph_.getAdjacentNodes(nid3);
// Therefore, number of outgoing edges from nid3 is 2.
EXPECT_EQ(2u, result2.size());
// Add an edge from nid3 to nid6.
- wait_for_graph.addEdgeUnchecked(nid3, nid6);
- std::vector<DirectedGraph::node_id> result3 = wait_for_graph.getAdjacentNodes(nid3);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid6);
+ std::vector<DirectedGraph::node_id> result3 = wait_for_graph_.getAdjacentNodes(nid3);
// Now there are 3 outgoing edges.
EXPECT_EQ(3u, result3.size());
// Again add edge from nid3 to nid6.
- wait_for_graph.addEdgeUnchecked(nid3, nid6);
- std::vector<DirectedGraph::node_id> result4 = wait_for_graph.getAdjacentNodes(nid3);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid6);
+ std::vector<DirectedGraph::node_id> result4 = wait_for_graph_.getAdjacentNodes(nid3);
// Since we have already add same edge before, number of edges are still 3.
EXPECT_EQ(3u, result4.size());
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/tests/LockRequest_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/LockRequest_unittest.cpp b/transaction/tests/LockRequest_unittest.cpp
index 0e4138a..77047d9 100644
--- a/transaction/tests/LockRequest_unittest.cpp
+++ b/transaction/tests/LockRequest_unittest.cpp
@@ -31,7 +31,7 @@ class LockRequestTest : public ::testing::Test {
LockRequestTest()
: lock_request_(transaction_id(3),
ResourceId(5),
- AccessMode(AccessModeType::kSLock),
+ AccessMode::SLockMode(),
RequestType::kAcquireLock) {
}
@@ -41,7 +41,7 @@ class LockRequestTest : public ::testing::Test {
TEST_F(LockRequestTest, CheckGetters) {
EXPECT_EQ(transaction_id(3), lock_request_.getTransactionId());
EXPECT_EQ(ResourceId(5), lock_request_.getResourceId());
- EXPECT_EQ(AccessMode(AccessModeType::kSLock), lock_request_.getAccessMode());
+ EXPECT_EQ(AccessMode::SLockMode(), lock_request_.getAccessMode());
EXPECT_EQ(RequestType::kAcquireLock, lock_request_.getRequestType());
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/tests/LockTable_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/LockTable_unittest.cpp b/transaction/tests/LockTable_unittest.cpp
index 577cb79..1aed0b8 100644
--- a/transaction/tests/LockTable_unittest.cpp
+++ b/transaction/tests/LockTable_unittest.cpp
@@ -41,64 +41,73 @@ class LockTableTest : public ::testing::Test {
};
TEST_F(LockTableTest, CompatibleRequestsFromDifferentTransactions) {
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ const AccessMode is_lock_mode = AccessMode::IsLockMode();
+ const AccessMode s_lock_mode = AccessMode::SLockMode();
+
+ EXPECT_EQ(LockTableResult::kPlacedInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kPLACED_IN_OWNED);
+ is_lock_mode));
// Acquire the same lock mode on same resource.
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ EXPECT_EQ(LockTableResult::kAlreadyInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kALREADY_IN_OWNED);
+ is_lock_mode));
// Another transaction acquires compatible lock on the same resource.
- EXPECT_EQ(lock_table_.putLock(tid_2_,
+ EXPECT_EQ(LockTableResult::kPlacedInOwned,
+ lock_table_.putLock(tid_2_,
ResourceId(2),
- AccessMode(AccessModeType::kSLock)),
- LockTableResult::kPLACED_IN_OWNED);
+ s_lock_mode));
}
TEST_F(LockTableTest, IncompatibleRequestsFromDifferentTransactions) {
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ const AccessMode is_lock_mode = AccessMode::IsLockMode();
+ const AccessMode x_lock_mode = AccessMode::XLockMode();
+
+ EXPECT_EQ(LockTableResult::kPlacedInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kPLACED_IN_OWNED);
+ is_lock_mode));
// Acquire the same lock mode on same resource.
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ EXPECT_EQ(LockTableResult::kAlreadyInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kALREADY_IN_OWNED);
+ is_lock_mode));
// Another transaction acquires incompatible lock on the same resource.
- EXPECT_EQ(lock_table_.putLock(tid_2_,
+ EXPECT_EQ(LockTableResult::kPlacedInPending,
+ lock_table_.putLock(tid_2_,
ResourceId(2),
- AccessMode(AccessModeType::kXLock)),
- LockTableResult::kPLACED_IN_PENDING);
+ x_lock_mode));
}
TEST_F(LockTableTest, StarvationProtection) {
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ const AccessMode is_lock_mode = AccessMode::IsLockMode();
+ const AccessMode x_lock_mode = AccessMode::XLockMode();
+
+ EXPECT_EQ(LockTableResult::kPlacedInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kPLACED_IN_OWNED);
+ is_lock_mode));
// Another transaction requests incompatible lock on the same resource.
// It should wait for the previous transaction.
- EXPECT_EQ(lock_table_.putLock(tid_2_,
+ EXPECT_EQ(LockTableResult::kPlacedInPending,
+ lock_table_.putLock(tid_2_,
ResourceId(2),
- AccessMode(AccessModeType::kXLock)),
- LockTableResult::kPLACED_IN_PENDING);
+ x_lock_mode));
// Another third transaction requests a compatible lock on the same resource.
// Normally, it should acquire the lock, however, there is a pending
// transaction waiting on the same resource. To prevent starvation, we should
// put in the pending list.
- EXPECT_EQ(lock_table_.putLock(tid_3_,
+ EXPECT_EQ(LockTableResult::kPlacedInPending,
+ lock_table_.putLock(tid_3_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kPLACED_IN_PENDING);
+ is_lock_mode));
}
} // namespace transaction
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc9c3bf1/transaction/tests/Lock_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/Lock_unittest.cpp b/transaction/tests/Lock_unittest.cpp
index 2ab8b3e..59a5e7a 100644
--- a/transaction/tests/Lock_unittest.cpp
+++ b/transaction/tests/Lock_unittest.cpp
@@ -18,6 +18,7 @@
#include "transaction/Lock.hpp"
#include <cstddef>
+#include <memory>
#include <vector>
#include "transaction/AccessMode.hpp"
@@ -31,12 +32,12 @@ namespace transaction {
class LockTest : public ::testing::Test {
protected:
LockTest()
- : modes_({AccessMode(AccessModeType::kNoLock),
- AccessMode(AccessModeType::kIsLock),
- AccessMode(AccessModeType::kIxLock),
- AccessMode(AccessModeType::kSLock),
- AccessMode(AccessModeType::kSixLock),
- AccessMode(AccessModeType::kXLock)}),
+ : modes_({AccessMode::NoLockMode(),
+ AccessMode::IsLockMode(),
+ AccessMode::IxLockMode(),
+ AccessMode::SLockMode(),
+ AccessMode::SixLockMode(),
+ AccessMode::XLockMode()}),
resource_a_(3, 10, 2, 5),
resource_b_(4, 5, 3, 2),
locks_on_resource_a_({Lock(resource_a_, modes_[0]),
[08/48] incubator-quickstep git commit: Added print_query gflag (#213)
Posted by hb...@apache.org.
Added print_query gflag (#213)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/afcc9f09
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/afcc9f09
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/afcc9f09
Branch: refs/heads/query-manager-used-in-foreman
Commit: afcc9f099096e10d7476afb0838f14fe0d15dbe7
Parents: cb763ba
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Sat May 7 09:44:56 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:41 2016 -0700
----------------------------------------------------------------------
cli/QuickstepCli.cpp | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/afcc9f09/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index ec195f7..4c0a166 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -144,6 +146,10 @@ DEFINE_string(worker_affinities, "",
"means that they will all be runable on any CPU according to "
"the kernel's own scheduling policy).");
DEFINE_bool(initialize_db, false, "If true, initialize a database.");
+DEFINE_bool(print_query, false,
+ "Print each input query statement. This is useful when running a "
+ "large number of queries in a batch.");
+
} // namespace quickstep
int main(int argc, char* argv[]) {
@@ -336,6 +342,10 @@ int main(int argc, char* argv[]) {
break;
}
+ if (quickstep::FLAGS_print_query) {
+ printf("\n%s\n", command_string->c_str());
+ }
+
parser_wrapper->feedNextBuffer(command_string);
bool quitting = false;
[18/48] incubator-quickstep git commit: Leave 10% more memory for the
OS. (#218)
Posted by hb...@apache.org.
Leave 10% more memory for the OS. (#218)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/44ca6224
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/44ca6224
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/44ca6224
Branch: refs/heads/query-manager-used-in-foreman
Commit: 44ca62240899e53bb69fcb4987fbe8c130cc49ed
Parents: 19aeb6c
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Sun May 8 22:41:17 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:42 2016 -0700
----------------------------------------------------------------------
storage/StorageConstants.hpp | 4 ++--
storage/StorageManager.cpp | 6 +++---
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/44ca6224/storage/StorageConstants.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageConstants.hpp b/storage/StorageConstants.hpp
index 154d2f7..0609cba 100644
--- a/storage/StorageConstants.hpp
+++ b/storage/StorageConstants.hpp
@@ -58,8 +58,8 @@ const std::uint64_t kLargeMemorySystemThresholdInGB = 32;
// we grab a bigger fraction of the installed memory than for a "small" system.
// The two constants below define the percentages to grab in each case.
// TODO(jmp): May need to generalize this to more than two levels in the future.
-const std::uint64_t kPercentageToGrabForSmallSystems = 80;
-const std::uint64_t kPercentageToGrabForLargeSystems = 90;
+const std::uint64_t kPercentageToGrabForSmallSystems = 70;
+const std::uint64_t kPercentageToGrabForLargeSystems = 80;
// The default size of the buffer pool (in terms of the number of slots).
const std::uint64_t kDefaultBufferPoolSizeInSlots = 1024;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/44ca6224/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index b990a22..dfc95b8 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -100,8 +100,8 @@ static const volatile bool block_domain_dummy
/**
* @brief Set or validate the buffer pool slots. When automatically picking a
* default value, check if the system is "small" or "large." Set the
- * buffer pool space to 80% of the installed main memory for small
- * and 90% otherwise.
+ * buffer pool space to 70% of the installed main memory for small
+ * and 80% otherwise.
* This method follows the signature that is set by the gflags module.
* @param flagname The name of the buffer pool flag.
* @param value The value of this flag from the command line, or default (0)
@@ -142,7 +142,7 @@ static bool SetOrValidateBufferPoolSlots(const char *flagname,
DEFINE_uint64(buffer_pool_slots, 0,
"By default the value is 0 and the system automatically sets the "
- "buffer pool size/slots at 80-90% of the total installed memory. "
+ "buffer pool size/slots at 70-80% of the total installed memory. "
"The user can also explicity define the number of slots. "
"The units for this variable is the number of 2-megabyte slots "
"that is allocated in the buffer pool. This is a \"soft\" limit: "
[31/48] incubator-quickstep git commit: Revert "Quickstep gen stats"
(#231)
Posted by hb...@apache.org.
Revert "Quickstep gen stats" (#231)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5e2bb515
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5e2bb515
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5e2bb515
Branch: refs/heads/query-manager-used-in-foreman
Commit: 5e2bb515bb6c66a1a359218c99270227f79d7864
Parents: 4b849cd
Author: Rogers Jeffrey Leo John <ro...@gmail.com>
Authored: Fri May 20 14:51:35 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:45 2016 -0700
----------------------------------------------------------------------
catalog/Catalog.proto | 2 +-
query_optimizer/CMakeLists.txt | 1 -
query_optimizer/ExecutionGenerator.cpp | 10 ---
relational_operators/CMakeLists.txt | 11 ---
.../GenerateNumRowsStatsOperator.cpp | 42 -----------
.../GenerateNumRowsStatsOperator.hpp | 79 --------------------
6 files changed, 1 insertion(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5e2bb515/catalog/Catalog.proto
----------------------------------------------------------------------
diff --git a/catalog/Catalog.proto b/catalog/Catalog.proto
index 8e44181..ce4bc2e 100644
--- a/catalog/Catalog.proto
+++ b/catalog/Catalog.proto
@@ -82,7 +82,7 @@ message IndexScheme {
message CatalogRelationStatistics {
optional fixed64 num_tuples = 1;
-
+
message NumDistinctValuesEntry {
required int32 attr_id = 1;
required fixed64 num_distinct_values = 2;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5e2bb515/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 1cc38d1..aa2873e 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -111,7 +111,6 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_relationaloperators_DestroyHashOperator
quickstep_relationaloperators_DropTableOperator
quickstep_relationaloperators_FinalizeAggregationOperator
- quickstep_relationaloperators_GenerateNumRowsStatsOperator
quickstep_relationaloperators_HashJoinOperator
quickstep_relationaloperators_InsertOperator
quickstep_relationaloperators_NestedLoopsJoinOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5e2bb515/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 612efd9..c590b6e 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -91,7 +91,6 @@
#include "relational_operators/DestroyHashOperator.hpp"
#include "relational_operators/DropTableOperator.hpp"
#include "relational_operators/FinalizeAggregationOperator.hpp"
-#include "relational_operators/GenerateNumRowsStatsOperator.hpp"
#include "relational_operators/HashJoinOperator.hpp"
#include "relational_operators/InsertOperator.hpp"
#include "relational_operators/NestedLoopsJoinOperator.hpp"
@@ -948,15 +947,6 @@ void ExecutionGenerator::convertCopyFrom(
execution_plan_->addDirectDependency(save_blocks_operator_index,
scan_operator_index,
false /* is_pipeline_breaker */);
-
- const QueryPlan::DAGNodeIndex num_rows_operator_index =
- execution_plan_->addRelationalOperator(new GenerateNumRowsStatsOperator(
- optimizer_context_->catalog_database()->getRelationByIdMutable(
- output_relation->getID())));
- insert_destination_proto->set_relational_op_index(num_rows_operator_index);
- execution_plan_->addDirectDependency(num_rows_operator_index,
- scan_operator_index,
- true /* is_pipeline_breaker */);
}
void ExecutionGenerator::convertCreateIndex(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5e2bb515/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index e211630..eec5300 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -34,9 +34,6 @@ add_library(quickstep_relationaloperators_DropTableOperator DropTableOperator.cp
add_library(quickstep_relationaloperators_FinalizeAggregationOperator
FinalizeAggregationOperator.cpp
FinalizeAggregationOperator.hpp)
-add_library(quickstep_relationaloperators_GenerateNumRowsStatsOperator
- GenerateNumRowsStatsOperator.cpp
- GenerateNumRowsStatsOperator.hpp)
add_library(quickstep_relationaloperators_HashJoinOperator HashJoinOperator.cpp HashJoinOperator.hpp)
add_library(quickstep_relationaloperators_InsertOperator InsertOperator.cpp InsertOperator.hpp)
add_library(quickstep_relationaloperators_NestedLoopsJoinOperator
@@ -162,13 +159,6 @@ target_link_libraries(quickstep_relationaloperators_FinalizeAggregationOperator
quickstep_storage_AggregationOperationState
quickstep_utility_Macros
tmb)
-target_link_libraries(quickstep_relationaloperators_GenerateNumRowsStatsOperator
- glog
- quickstep_catalog_CatalogRelation
- quickstep_cli_PrintToScreen
- quickstep_relationaloperators_RelationalOperator
- quickstep_utility_Macros
- tmb)
target_link_libraries(quickstep_relationaloperators_HashJoinOperator
gflags_nothreads-static
glog
@@ -456,7 +446,6 @@ target_link_libraries(quickstep_relationaloperators
quickstep_relationaloperators_DestroyHashOperator
quickstep_relationaloperators_DropTableOperator
quickstep_relationaloperators_FinalizeAggregationOperator
- quickstep_relationaloperators_GenerateNumRowsStatsOperator
quickstep_relationaloperators_HashJoinOperator
quickstep_relationaloperators_InsertOperator
quickstep_relationaloperators_NestedLoopsJoinOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5e2bb515/relational_operators/GenerateNumRowsStatsOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/GenerateNumRowsStatsOperator.cpp b/relational_operators/GenerateNumRowsStatsOperator.cpp
deleted file mode 100644
index 074e1ca..0000000
--- a/relational_operators/GenerateNumRowsStatsOperator.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#include "relational_operators/GenerateNumRowsStatsOperator.hpp"
-
-#include <memory>
-
-#include "catalog/CatalogRelation.hpp"
-#include "cli/PrintToScreen.hpp"
-
-#include "tmb/id_typedefs.h"
-
-namespace quickstep {
-
-bool GenerateNumRowsStatsOperator::getAllWorkOrders(
- WorkOrdersContainer *container,
- QueryContext *query_context,
- StorageManager *storage_manager,
- const tmb::client_id scheduler_client_id,
- tmb::MessageBus *bus) {
- std::size_t num_tuples =
- PrintToScreen::GetNumTuplesInRelation(*relation_, storage_manager);
- relation_->getStatisticsMutable()->setNumTuples(num_tuples);
- return true;
-}
-
-} // namespace quickstep
-
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5e2bb515/relational_operators/GenerateNumRowsStatsOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/GenerateNumRowsStatsOperator.hpp b/relational_operators/GenerateNumRowsStatsOperator.hpp
deleted file mode 100644
index 8622a63..0000000
--- a/relational_operators/GenerateNumRowsStatsOperator.hpp
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#ifndef QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
-#define QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
-
-#include <memory>
-
-#include "catalog/CatalogRelation.hpp"
-#include "relational_operators/RelationalOperator.hpp"
-#include "utility/Macros.hpp"
-
-#include "glog/logging.h"
-
-#include "tmb/id_typedefs.h"
-
-namespace tmb { class MessageBus; }
-
-namespace quickstep {
-
-class CatalogRelation;
-class QueryContext;
-class StorageManager;
-class WorkOrdersContainer;
-
-/** \addtogroup RelationalOperators
- * @{
- */
-
-/**
- * @brief An operator that gets the number of rows after loading a relation.
- **/
-class GenerateNumRowsStatsOperator : public RelationalOperator {
- public:
- /**
- * @brief Constructor.
- *
- * @param relation The relation to get the number of rows from.
- * This GenNumRowStatsOperator owns relation until
- * the WorkOrder it produces is successfully executed.
- **/
- explicit GenerateNumRowsStatsOperator(CatalogRelation *relation)
- : relation_(relation) {}
- ~GenerateNumRowsStatsOperator() override {}
-
- /**
- * @note no WorkOrder is generated for this operator.
- **/
- bool getAllWorkOrders(WorkOrdersContainer *container,
- QueryContext *query_context,
- StorageManager *storage_manager,
- const tmb::client_id scheduler_client_id,
- tmb::MessageBus *bus) override;
-
- private:
- CatalogRelation *relation_;
-
- DISALLOW_COPY_AND_ASSIGN(GenerateNumRowsStatsOperator);
-};
-
-/** @} */
-
-} // namespace quickstep
-
-#endif // QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
[03/48] incubator-quickstep git commit: Change default
aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining (#207)
Posted by hb...@apache.org.
Change default aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining (#207)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b23341ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b23341ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b23341ed
Branch: refs/heads/query-manager-used-in-foreman
Commit: b23341ed72580c064d0b032935e3b44a06036708
Parents: 3b33fa2
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Wed May 4 13:54:55 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:40 2016 -0700
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b23341ed/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index c34f084..3698701 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -135,7 +135,7 @@ static const volatile bool join_hashtable_type_dummy
= gflags::RegisterFlagValidator(&FLAGS_join_hashtable_type,
&ValidateHashTableImplTypeString);
-DEFINE_string(aggregate_hashtable_type, "LinearOpenAddressing",
+DEFINE_string(aggregate_hashtable_type, "SeparateChaining",
"HashTable implementation to use for aggregates with GROUP BY "
"(valid options are SeparateChaining or LinearOpenAddressing)");
static const volatile bool aggregate_hashtable_type_dummy
[27/48] incubator-quickstep git commit: In ScopedBuffer,
initialize the allocated memory bytes to 0 (#230)
Posted by hb...@apache.org.
In ScopedBuffer, initialize the allocated memory bytes to 0 (#230)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ebfe9521
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ebfe9521
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ebfe9521
Branch: refs/heads/query-manager-used-in-foreman
Commit: ebfe9521f869543f058bb4c984fcd29511b9bd60
Parents: cd9db29
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 19 18:20:53 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:44 2016 -0700
----------------------------------------------------------------------
utility/ScopedBuffer.hpp | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ebfe9521/utility/ScopedBuffer.hpp
----------------------------------------------------------------------
diff --git a/utility/ScopedBuffer.hpp b/utility/ScopedBuffer.hpp
index b6fddc5..03290ff 100644
--- a/utility/ScopedBuffer.hpp
+++ b/utility/ScopedBuffer.hpp
@@ -43,9 +43,13 @@ class ScopedBuffer {
* size.
*
* @param alloc_size The number of bytes of memory to allocate.
+ * @param initialize If true, initialize all the bytes of the allocated memory to 0.
**/
- explicit ScopedBuffer(const std::size_t alloc_size) {
+ explicit ScopedBuffer(const std::size_t alloc_size, bool initialize = true) {
internal_ptr_ = std::malloc(alloc_size);
+ if (initialize) {
+ memset(internal_ptr_, 0x0, alloc_size);
+ }
}
/**
@@ -54,10 +58,15 @@ class ScopedBuffer {
*
* @param alloc_size The number of bytes of memory to allocate.
* @param alloc_alignment The alignment of the memory to allocate.
+ * @param initialize If true, initialize all the bytes of the allocated memory to 0.
**/
ScopedBuffer(const std::size_t alloc_size,
- const std::size_t alloc_alignment) {
+ const std::size_t alloc_alignment,
+ const bool initialize = true) {
internal_ptr_ = malloc_with_alignment(alloc_size, alloc_alignment);
+ if (initialize) {
+ memset(internal_ptr_, 0x0, alloc_size);
+ }
}
/**
@@ -110,12 +119,16 @@ class ScopedBuffer {
* size.
*
* @param alloc_size The number of bytes of memory to allocate.
+ * @param initialize If true, initialize all the bytes of the allocated memory to 0.
**/
- void reset(const std::size_t alloc_size) {
+ void reset(const std::size_t alloc_size, const bool initialize = true) {
if (internal_ptr_ != nullptr) {
std::free(internal_ptr_);
}
internal_ptr_ = std::malloc(alloc_size);
+ if (initialize) {
+ memset(internal_ptr_, 0x0, alloc_size);
+ }
}
/**
[16/48] incubator-quickstep git commit: Print number of rows in the
output (#219)
Posted by hb...@apache.org.
Print number of rows in the output (#219)
- Provide an API to calculate the number of rows in a table
- Print output time in milliseconds, upto 3 decimal places.
- Simplify output messages.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/06f73ca9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/06f73ca9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/06f73ca9
Branch: refs/heads/query-manager-used-in-foreman
Commit: 06f73ca92681b176746c0fc8014181e88ff9bb9b
Parents: 2c07d70
Author: Harshad Deshmukh <d....@gmail.com>
Authored: Tue May 10 16:27:06 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Wed Jun 8 11:57:42 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 3 ++-
cli/CMakeLists.txt | 1 +
cli/PrintToScreen.cpp | 22 ++++++++++++++++++++++
cli/PrintToScreen.hpp | 23 +++++++++++++++++++++++
cli/QuickstepCli.cpp | 12 +++++++++---
storage/StorageBlock.cpp | 5 +++++
storage/StorageBlock.hpp | 11 ++++++++---
7 files changed, 70 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/06f73ca9/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c9578b0..87a8f7c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -712,7 +712,8 @@ target_link_libraries(quickstep_cli_shell
quickstep_threading_ThreadIDBasedMap
quickstep_utility_Macros
quickstep_utility_PtrVector
- quickstep_utility_SqlError)
+ quickstep_utility_SqlError
+ quickstep_utility_StringUtil)
if (ENABLE_HDFS)
target_link_libraries(quickstep_cli_shell
quickstep_storage_FileManagerHdfs)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/06f73ca9/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 9c4073b..a1989d5 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -102,6 +102,7 @@ target_link_libraries(quickstep_cli_PrintToScreen
quickstep_storage_StorageManager
quickstep_storage_TupleIdSequence
quickstep_storage_TupleStorageSubBlock
+ quickstep_types_IntType
quickstep_types_Type
quickstep_types_TypedValue
quickstep_utility_Macros)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/06f73ca9/cli/PrintToScreen.cpp
----------------------------------------------------------------------
diff --git a/cli/PrintToScreen.cpp b/cli/PrintToScreen.cpp
index c543b70..227ff39 100644
--- a/cli/PrintToScreen.cpp
+++ b/cli/PrintToScreen.cpp
@@ -29,6 +29,7 @@
#include "storage/StorageManager.hpp"
#include "storage/TupleIdSequence.hpp"
#include "storage/TupleStorageSubBlock.hpp"
+#include "types/IntType.hpp"
#include "types/Type.hpp"
#include "types/TypedValue.hpp"
#include "utility/Macros.hpp"
@@ -147,4 +148,25 @@ void PrintToScreen::printTuple(const TupleStorageSubBlock &tuple_store,
fputc('\n', out);
}
+std::size_t PrintToScreen::GetNumTuplesInRelation(
+ const CatalogRelation &relation, StorageManager *storage_manager) {
+ const std::vector<block_id> &blocks = relation.getBlocksSnapshot();
+ std::size_t total_num_tuples = 0;
+ for (block_id block : blocks) {
+ total_num_tuples +=
+ storage_manager->getBlock(block, relation)->getNumTuples();
+ }
+ return total_num_tuples;
+}
+
+void PrintToScreen::PrintOutputSize(const CatalogRelation &relation,
+ StorageManager *storage_manager,
+ FILE *out) {
+ const std::size_t num_rows = GetNumTuplesInRelation(relation, storage_manager);
+ fprintf(out,
+ "(%lu %s)\n",
+ num_rows,
+ (num_rows == 1) ? "row" : "rows");
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/06f73ca9/cli/PrintToScreen.hpp
----------------------------------------------------------------------
diff --git a/cli/PrintToScreen.hpp b/cli/PrintToScreen.hpp
index 3c9afae..0b57b4b 100644
--- a/cli/PrintToScreen.hpp
+++ b/cli/PrintToScreen.hpp
@@ -46,6 +46,29 @@ class PrintToScreen {
static void printHBar(const std::vector<int> &column_widths,
FILE *out);
+
+ /**
+ * @brief Get the total number of tuples in the given relation.
+ *
+ * @param relation The given relation.
+ * @param storage_manager The storage manager.
+ *
+ * @return The total number of tuples in the relation.
+ **/
+ static std::size_t GetNumTuplesInRelation(const CatalogRelation &relation,
+ StorageManager *storage_manager);
+
+ /**
+ * @brief Print the size of the output (i.e. number of rows in the relation).
+ *
+ * @param relation The given relation.
+ * @param storage_manager The storage manager.
+ * @param out The output stream.
+ **/
+ static void PrintOutputSize(const CatalogRelation &relation,
+ StorageManager *storage_manager,
+ FILE *out);
+
private:
// Undefined default constructor. Class is all-static and should not be
// instantiated.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/06f73ca9/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 4c0a166..5881b3e 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -72,6 +72,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "utility/Macros.hpp"
#include "utility/PtrVector.hpp"
#include "utility/SqlError.hpp"
+#include "utility/StringUtil.hpp"
#include "gflags/gflags.h"
@@ -399,6 +400,10 @@ int main(int argc, char* argv[]) {
PrintToScreen::PrintRelation(*query_result_relation,
query_processor->getStorageManager(),
stdout);
+ PrintToScreen::PrintOutputSize(
+ *query_result_relation,
+ query_processor->getStorageManager(),
+ stdout);
DropRelation::Drop(*query_result_relation,
query_processor->getDefaultDatabase(),
@@ -406,13 +411,14 @@ int main(int argc, char* argv[]) {
}
query_processor->saveCatalog();
- printf("Execution time: %g seconds\n",
- std::chrono::duration<double>(end - start).count());
+ std::chrono::duration<double, std::milli> time_ms = end - start;
+ printf("Time: %s ms\n",
+ quickstep::DoubleToStringWithSignificantDigits(
+ time_ms.count(), 3).c_str());
} catch (const std::exception &e) {
fprintf(stderr, "QUERY EXECUTION ERROR: %s\n", e.what());
break;
}
- printf("Query Complete\n");
} else {
if (result.condition == ParseResult::kError) {
fprintf(stderr, "%s", result.error_message.c_str());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/06f73ca9/storage/StorageBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.cpp b/storage/StorageBlock.cpp
index 476a130..fdd438d 100644
--- a/storage/StorageBlock.cpp
+++ b/storage/StorageBlock.cpp
@@ -1361,4 +1361,9 @@ void StorageBlock::invalidateAllIndexes() {
}
}
+const std::size_t StorageBlock::getNumTuples() const {
+ DCHECK(tuple_store_ != nullptr);
+ return tuple_store_->numTuples();
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/06f73ca9/storage/StorageBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.hpp b/storage/StorageBlock.hpp
index 97813e2..3ae3812 100644
--- a/storage/StorageBlock.hpp
+++ b/storage/StorageBlock.hpp
@@ -311,17 +311,17 @@ class StorageBlock : public StorageBlockBase {
ValueAccessor *accessor);
/**
- * @brief Perform a random sampling of data on the StorageBlock. The number
+ * @brief Perform a random sampling of data on the StorageBlock. The number
* of records sampled is determined by the sample percentage in case of
* tuple sample. For block sample all the records in a block are taken.
*
* @param is_block_sample Flag to indicate if the Sampling method is a tuple
* sample or block sample.
* @param percentage The percentage of tuples to be sampled.Used only when the
- * sampling method is tuple sample.
+ * sampling method is tuple sample.
* @param destination Where to insert the tuples resulting from the SAMPLE.
*
- * @return Randomly sampled tuples whose count is determined by the
+ * @return Randomly sampled tuples whose count is determined by the
* sampling percentage. In the case of block sample the entire
* block is returned.
**/
@@ -583,6 +583,11 @@ class StorageBlock : public StorageBlockBase {
return rebuildIndexes(false);
}
+ /**
+ * @brief Get the number of tuples in this storage block.
+ **/
+ const std::size_t getNumTuples() const;
+
private:
static TupleStorageSubBlock* CreateTupleStorageSubBlock(
const CatalogRelationSchema &relation,