You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by zu...@apache.org on 2017/01/11 01:01:42 UTC
[50/50] incubator-quickstep git commit: Added optimizer support for
hash partitions.
Added optimizer support for hash partitions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4890bbe8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4890bbe8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4890bbe8
Branch: refs/heads/quickstep_partition_parser_support
Commit: 4890bbe8354d32789d4a0b111ccaedce8ce055bb
Parents: 5fee821
Author: Zuyu Zhang <zu...@apache.org>
Authored: Tue Jan 10 17:00:08 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Tue Jan 10 17:00:08 2017 -0800
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 4 +
query_optimizer/logical/CMakeLists.txt | 1 +
query_optimizer/logical/CreateTable.hpp | 23 ++++--
query_optimizer/physical/CMakeLists.txt | 1 +
query_optimizer/physical/CreateTable.hpp | 22 ++++--
query_optimizer/resolver/CMakeLists.txt | 3 +
query_optimizer/resolver/Resolver.cpp | 101 ++++++++++++++++++++------
query_optimizer/resolver/Resolver.hpp | 12 +++
query_optimizer/strategy/OneToOne.cpp | 3 +-
9 files changed, 135 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 29e67f7..6ee219e 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1017,6 +1017,10 @@ void ExecutionGenerator::convertCreateTable(
catalog_relation->setDefaultStorageBlockLayout(layout.release());
}
+ if (physical_plan->partition_scheme_header_proto()) {
+ catalog_relation->setPartitionScheme(nullptr);
+ }
+
execution_plan_->addRelationalOperator(
new CreateTableOperator(query_handle_->query_id(),
catalog_relation.release(),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/logical/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/logical/CMakeLists.txt b/query_optimizer/logical/CMakeLists.txt
index c67f96f..8aca550 100644
--- a/query_optimizer/logical/CMakeLists.txt
+++ b/query_optimizer/logical/CMakeLists.txt
@@ -86,6 +86,7 @@ target_link_libraries(quickstep_queryoptimizer_logical_CreateIndex
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_logical_CreateTable
glog
+ quickstep_catalog_Catalog_proto
quickstep_queryoptimizer_OptimizerTree
quickstep_queryoptimizer_expressions_AttributeReference
quickstep_queryoptimizer_logical_Logical
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/logical/CreateTable.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/logical/CreateTable.hpp b/query_optimizer/logical/CreateTable.hpp
index cc7c6d7..c01b912 100644
--- a/query_optimizer/logical/CreateTable.hpp
+++ b/query_optimizer/logical/CreateTable.hpp
@@ -24,6 +24,7 @@
#include <string>
#include <vector>
+#include "catalog/Catalog.pb.h"
#include "query_optimizer/OptimizerTree.hpp"
#include "query_optimizer/expressions/AttributeReference.hpp"
#include "query_optimizer/logical/Logical.hpp"
@@ -72,10 +73,17 @@ class CreateTable : public Logical {
return block_properties_;
}
+ /**
+ * @return Shared pointer to the partition scheme.
+ */
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto() const {
+ return partition_scheme_header_proto_;
+ }
+
LogicalPtr copyWithNewChildren(
const std::vector<LogicalPtr> &new_children) const override {
DCHECK_EQ(getNumChildren(), new_children.size());
- return Create(relation_name_, attributes_, block_properties_);
+ return Create(relation_name_, attributes_, block_properties_, partition_scheme_header_proto_);
}
std::vector<expressions::AttributeReferencePtr> getOutputAttributes() const override {
@@ -100,8 +108,9 @@ class CreateTable : public Logical {
static CreateTablePtr Create(
const std::string &relation_name,
const std::vector<expressions::AttributeReferencePtr> &attributes,
- const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties) {
- return CreateTablePtr(new CreateTable(relation_name, attributes, block_properties));
+ const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties,
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> &partition_scheme_header_proto) {
+ return CreateTablePtr(new CreateTable(relation_name, attributes, block_properties, partition_scheme_header_proto));
}
protected:
@@ -117,18 +126,20 @@ class CreateTable : public Logical {
CreateTable(
const std::string &relation_name,
const std::vector<expressions::AttributeReferencePtr> &attributes,
- const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties)
+ const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties,
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> &partition_scheme_header_proto)
: relation_name_(relation_name),
attributes_(attributes),
block_properties_(block_properties),
block_properties_representation_(
- getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())) {}
+ getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())),
+ partition_scheme_header_proto_(partition_scheme_header_proto){}
std::string relation_name_;
std::vector<expressions::AttributeReferencePtr> attributes_;
std::shared_ptr<const StorageBlockLayoutDescription> block_properties_;
std::shared_ptr<const OptimizerProtoRepresentation<OptimizerTreeBaseNodePtr> > block_properties_representation_;
-
+ std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto_;
DISALLOW_COPY_AND_ASSIGN(CreateTable);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/physical/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/CMakeLists.txt b/query_optimizer/physical/CMakeLists.txt
index 5c2cd0b..7f26943 100644
--- a/query_optimizer/physical/CMakeLists.txt
+++ b/query_optimizer/physical/CMakeLists.txt
@@ -85,6 +85,7 @@ target_link_libraries(quickstep_queryoptimizer_physical_CreateIndex
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_physical_CreateTable
glog
+ quickstep_catalog_Catalog_proto
quickstep_queryoptimizer_OptimizerTree
quickstep_queryoptimizer_expressions_AttributeReference
quickstep_queryoptimizer_expressions_ExpressionUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/physical/CreateTable.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/CreateTable.hpp b/query_optimizer/physical/CreateTable.hpp
index 8e3bbd4..5fb1c61 100644
--- a/query_optimizer/physical/CreateTable.hpp
+++ b/query_optimizer/physical/CreateTable.hpp
@@ -24,6 +24,7 @@
#include <string>
#include <vector>
+#include "catalog/Catalog.pb.h"
#include "query_optimizer/OptimizerTree.hpp"
#include "query_optimizer/expressions/AttributeReference.hpp"
#include "query_optimizer/expressions/ExpressionUtil.hpp"
@@ -75,10 +76,17 @@ class CreateTable : public Physical {
return block_properties_;
}
+ /**
+ * @return Shared pointer to the partition scheme
+ */
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto() const {
+ return partition_scheme_header_proto_;
+ }
+
PhysicalPtr copyWithNewChildren(
const std::vector<PhysicalPtr> &new_children) const override {
DCHECK_EQ(getNumChildren(), new_children.size());
- return Create(relation_name_, attributes_, block_properties_);
+ return Create(relation_name_, attributes_, block_properties_, partition_scheme_header_proto_);
}
std::vector<expressions::AttributeReferencePtr> getOutputAttributes() const override {
@@ -107,8 +115,9 @@ class CreateTable : public Physical {
static CreateTablePtr Create(
const std::string &relation_name,
const std::vector<expressions::AttributeReferencePtr> &attributes,
- const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties) {
- return CreateTablePtr(new CreateTable(relation_name, attributes, block_properties));
+ const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties,
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> &partition_scheme_header_proto) {
+ return CreateTablePtr(new CreateTable(relation_name, attributes, block_properties, partition_scheme_header_proto));
}
protected:
@@ -124,17 +133,20 @@ class CreateTable : public Physical {
CreateTable(
const std::string &relation_name,
const std::vector<expressions::AttributeReferencePtr> &attributes,
- const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties)
+ const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties,
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> &partition_scheme_header_proto)
: relation_name_(relation_name),
attributes_(attributes),
block_properties_(block_properties),
block_properties_representation_(
- getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())) {}
+ getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())),
+ partition_scheme_header_proto_(partition_scheme_header_proto) {}
std::string relation_name_;
std::vector<expressions::AttributeReferencePtr> attributes_;
std::shared_ptr<const StorageBlockLayoutDescription> block_properties_;
std::shared_ptr<const OptimizerProtoRepresentation<OptimizerTreeBaseNodePtr> > block_properties_representation_;
+ std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto_;
DISALLOW_COPY_AND_ASSIGN(CreateTable);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/resolver/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/CMakeLists.txt b/query_optimizer/resolver/CMakeLists.txt
index 5251ccc..a34273e 100644
--- a/query_optimizer/resolver/CMakeLists.txt
+++ b/query_optimizer/resolver/CMakeLists.txt
@@ -34,6 +34,8 @@ target_link_libraries(quickstep_queryoptimizer_resolver_NameResolver
target_link_libraries(quickstep_queryoptimizer_resolver_Resolver
glog
quickstep_catalog_CatalogDatabase
+ quickstep_catalog_CatalogTypedefs
+ quickstep_catalog_Catalog_proto
quickstep_expressions_aggregation_AggregateFunction
quickstep_expressions_aggregation_AggregateFunctionFactory
quickstep_expressions_tablegenerator_GeneratorFunction
@@ -53,6 +55,7 @@ target_link_libraries(quickstep_queryoptimizer_resolver_Resolver
quickstep_parser_ParseLimit
quickstep_parser_ParseLiteralValue
quickstep_parser_ParseOrderBy
+ quickstep_parser_ParsePartitionClause
quickstep_parser_ParsePredicate
quickstep_parser_ParsePredicateExists
quickstep_parser_ParsePredicateInTableQuery
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index 2580342..d363a06 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -29,7 +29,9 @@
#include <vector>
#include <utility>
+#include "catalog/Catalog.pb.h"
#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogTypedefs.hpp"
#include "expressions/aggregation/AggregateFunction.hpp"
#include "expressions/aggregation/AggregateFunctionFactory.hpp"
#include "expressions/table_generator/GeneratorFunction.hpp"
@@ -49,6 +51,7 @@
#include "parser/ParseLimit.hpp"
#include "parser/ParseLiteralValue.hpp"
#include "parser/ParseOrderBy.hpp"
+#include "parser/ParsePartitionClause.hpp"
#include "parser/ParsePredicate.hpp"
#include "parser/ParsePredicateExists.hpp"
#include "parser/ParsePredicateInTableQuery.hpp"
@@ -129,12 +132,15 @@
#include "glog/logging.h"
+using std::make_unique;
+
namespace quickstep {
namespace optimizer {
namespace resolver {
namespace E = ::quickstep::optimizer::expressions;
namespace L = ::quickstep::optimizer::logical;
+namespace S = ::quickstep::serialization;
struct Resolver::ExpressionResolutionInfo {
/**
@@ -478,9 +484,32 @@ L::LogicalPtr Resolver::resolveCreateTable(
std::shared_ptr<const StorageBlockLayoutDescription>
block_properties(resolveBlockProperties(create_table_statement));
- return L::CreateTable::Create(relation_name, attributes, block_properties);
+ std::shared_ptr<const S::PartitionSchemeHeader>
+ partition_scheme_header_proto(resolvePartitionClause(create_table_statement));
+
+ return L::CreateTable::Create(relation_name, attributes, block_properties, partition_scheme_header_proto);
+}
+
+namespace {
+
+attribute_id GetAttributeIdFromName(const PtrList<ParseAttributeDefinition>&attribute_definition_list,
+ const std::string &attribute_name) {
+ const std::string lower_attribute_name = ToLower(attribute_name);
+
+ attribute_id attr_id = 0;
+ for (const ParseAttributeDefinition &attribute_definition : attribute_definition_list) {
+ if (lower_attribute_name == ToLower(attribute_definition.name()->value())) {
+ return attr_id;
+ }
+
+ ++attr_id;
+ }
+
+ return kInvalidAttributeID;
}
+} // namespace
+
StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
const ParseStatementCreateTable &create_table_statement) {
const ParseBlockProperties *block_properties
@@ -540,25 +569,6 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
THROW_SQL_ERROR_AT(type_parse_string) << "Unrecognized storage type.";
}
- // Helper lambda function which will be used in COMPRESS and SORT resolution.
- // Returns the column id from the name of the given attribute. Returns -1 if
- // the attribute is not found.
- auto columnIdFromAttributeName = [&create_table_statement](
- const std::string& attribute_name) -> int {
- const std::string search_name = ToLower(attribute_name);
- int i = 0;
- for (const ParseAttributeDefinition &attribute_definition :
- create_table_statement.attribute_definition_list()) {
- const std::string lower_attribute_name =
- ToLower(attribute_definition.name()->value());
- if (lower_attribute_name.compare(search_name) == 0) {
- return i;
- }
- i++;
- }
- return -1;
- };
-
// Resolve the SORT property.
const ParseString *sort_parse_string = block_properties->getSort();
if (block_requires_sort) {
@@ -566,9 +576,9 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
THROW_SQL_ERROR_AT(type_parse_string)
<< "The SORT property must be specified as an attribute name.";
} else {
- const std::string &sort_name = sort_parse_string->value();
// Lookup the name and map to a column id.
- int sort_id = columnIdFromAttributeName(sort_name);
+ attribute_id sort_id = GetAttributeIdFromName(create_table_statement.attribute_definition_list(),
+ sort_parse_string->value());
if (sort_id == -1) {
THROW_SQL_ERROR_AT(sort_parse_string)
<< "The SORT property did not match any attribute name.";
@@ -609,7 +619,8 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
<< "The COMPRESS property must be specified as ALL or a list of attributes.";
}
for (const ParseString &compressed_attribute_name : *compress_parse_strings) {
- int column_id = columnIdFromAttributeName(compressed_attribute_name.value());
+ attribute_id column_id = GetAttributeIdFromName(create_table_statement.attribute_definition_list(),
+ compressed_attribute_name.value());
if (column_id == -1) {
THROW_SQL_ERROR_AT(&compressed_attribute_name)
<< "The given attribute was not found.";
@@ -671,6 +682,50 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
return storage_block_description.release();
}
+const S::PartitionSchemeHeader* Resolver::resolvePartitionClause(
+ const ParseStatementCreateTable &create_table_statement) {
+ const ParsePartitionClause *partition_clause = create_table_statement.opt_partition_clause();
+ if (partition_clause == nullptr) {
+ return nullptr;
+ }
+
+ const ParseString *partition_type_string = partition_clause->partition_type();
+ if (partition_type_string == nullptr) {
+ THROW_SQL_ERROR_AT(partition_clause)
+ << "Partition type must be specified and be a string.";
+ }
+
+ const PtrList<ParseString> &attribute_name_list = partition_clause->attribute_name_list();
+ if (attribute_name_list.size() != 1) {
+ THROW_SQL_ERROR_AT(partition_clause)
+ << "Partition is supported on only one attribute.";
+ }
+
+ const ParseString &partition_attribute_name = *(attribute_name_list.begin());
+ const attribute_id attr_id = GetAttributeIdFromName(create_table_statement.attribute_definition_list(),
+ partition_attribute_name.value());
+ if (attr_id == kInvalidAttributeID) {
+ THROW_SQL_ERROR_AT(&partition_attribute_name)
+ << "The given attribute was not found.";
+ }
+
+ auto proto = make_unique<S::PartitionSchemeHeader>();
+ proto->set_num_partitions(partition_clause->num_partitions()->long_value());
+ proto->set_partition_attribute_id(attr_id);
+
+ const std::string partition_type = ToLower(partition_type_string->value());
+ if (partition_type == "hash") {
+ proto->set_partition_type(S::PartitionSchemeHeader::HASH);
+ } else if (partition_type == "range") {
+ THROW_SQL_ERROR_AT(partition_clause)
+ << "Range partition is not supported.";
+ } else {
+ THROW_SQL_ERROR_AT(partition_type_string) << "Unrecognized partition type.";
+ }
+
+ return proto.release();
+}
+
L::LogicalPtr Resolver::resolveCreateIndex(
const ParseStatementCreateIndex &create_index_statement) {
// Resolve relation reference.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/resolver/Resolver.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.hpp b/query_optimizer/resolver/Resolver.hpp
index 855e6ba..e4351e0 100644
--- a/query_optimizer/resolver/Resolver.hpp
+++ b/query_optimizer/resolver/Resolver.hpp
@@ -74,6 +74,8 @@ class PtrList;
class StorageBlockLayoutDescription;
class Type;
+namespace serialization { class PartitionSchemeHeader; }
+
} // namespace quickstep
namespace quickstep {
@@ -213,6 +215,16 @@ class Resolver {
const ParseStatementCreateTable &create_table_statement);
/**
+ * @brief Resolves the PARTITION clause of a CREATE TABLE statement to a
+ * the serialized PartitionSchemeHeader describing the user input.
+ *
+ * @param create_table_statement The create table statement.
+ * @return A pointer to a user-owned serialized PartitionSchemeHeader.
+ */
+ const serialization::PartitionSchemeHeader* resolvePartitionClause(
+ const ParseStatementCreateTable &create_table_statement);
+
+ /**
* @brief Resolves a DELETE query and returns a logical plan.
*
* @param delete_statement The DELETE parse tree.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/strategy/OneToOne.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/strategy/OneToOne.cpp b/query_optimizer/strategy/OneToOne.cpp
index 78003f4..7d0c4cb 100644
--- a/query_optimizer/strategy/OneToOne.cpp
+++ b/query_optimizer/strategy/OneToOne.cpp
@@ -121,7 +121,8 @@ bool OneToOne::generatePlan(const L::LogicalPtr &logical_input,
std::static_pointer_cast<const L::CreateTable>(logical_input);
*physical_output = P::CreateTable::Create(create_table->relation_name(),
create_table->attributes(),
- create_table->block_properties());
+ create_table->block_properties(),
+ create_table->partition_scheme_header_proto());
return true;
}
case L::LogicalType::kDeleteTuples: {