You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by ji...@apache.org on 2017/02/05 23:51:42 UTC
[1/5] incubator-quickstep git commit: Fixed CopyFrom bug in the
distributed version. [Forced Update!]
Repository: incubator-quickstep
Updated Branches:
refs/heads/reduce-group-by-attrs 022787a1f -> 3880a16e3 (forced update)
Fixed CopyFrom bug in the distributed version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/61391caa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/61391caa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/61391caa
Branch: refs/heads/reduce-group-by-attrs
Commit: 61391caac81201383fa7ea03b1321be50194b525
Parents: 929e5f1
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Feb 3 00:09:37 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Fri Feb 3 00:09:37 2017 -0800
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/61391caa/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index b73de39..6918313 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -988,6 +988,11 @@ void ExecutionGenerator::convertCopyFrom(
// CopyFrom is converted to a TextScan and a SaveBlocks.
const CatalogRelation *output_relation = physical_plan->catalog_relation();
+ const relation_id output_rel_id = output_relation->getID();
+
+#ifdef QUICKSTEP_DISTRIBUTED
+ referenced_relation_ids_.insert(output_rel_id);
+#endif
// Create InsertDestination proto.
const QueryContext::insert_destination_id insert_destination_index =
@@ -995,7 +1000,7 @@ void ExecutionGenerator::convertCopyFrom(
S::InsertDestination *insert_destination_proto = query_context_proto_->add_insert_destinations();
insert_destination_proto->set_insert_destination_type(S::InsertDestinationType::BLOCK_POOL);
- insert_destination_proto->set_relation_id(output_relation->getID());
+ insert_destination_proto->set_relation_id(output_rel_id);
insert_destination_proto->mutable_layout()->MergeFrom(
output_relation->getDefaultStorageBlockLayout().getDescription());
@@ -1016,7 +1021,7 @@ void ExecutionGenerator::convertCopyFrom(
insert_destination_proto->set_relational_op_index(scan_operator_index);
CatalogRelation *mutable_output_relation =
- catalog_database_->getRelationByIdMutable(output_relation->getID());
+ catalog_database_->getRelationByIdMutable(output_rel_id);
const QueryPlan::DAGNodeIndex save_blocks_operator_index =
execution_plan_->addRelationalOperator(
new SaveBlocksOperator(query_handle_->query_id(), mutable_output_relation));
[4/5] incubator-quickstep git commit: Saved catalog in the
distributed version.
Posted by ji...@apache.org.
Saved catalog in the distributed version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/27a80558
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/27a80558
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/27a80558
Branch: refs/heads/reduce-group-by-attrs
Commit: 27a8055872f82737c35f6f0914ce43bcbe272ce3
Parents: dda085c
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Feb 5 02:16:34 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Feb 5 02:16:34 2017 -0800
----------------------------------------------------------------------
cli/distributed/Conductor.cpp | 5 ++++-
query_execution/ForemanDistributed.cpp | 3 +++
query_execution/ForemanDistributed.hpp | 4 ++++
query_execution/PolicyEnforcerDistributed.cpp | 2 ++
query_execution/PolicyEnforcerDistributed.hpp | 7 +++++++
.../tests/DistributedExecutionGeneratorTestRunner.cpp | 10 +++++++++-
6 files changed, 29 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/27a80558/cli/distributed/Conductor.cpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Conductor.cpp b/cli/distributed/Conductor.cpp
index c4a2721..13d4d57 100644
--- a/cli/distributed/Conductor.cpp
+++ b/cli/distributed/Conductor.cpp
@@ -22,6 +22,7 @@
#include <cstddef>
#include <cstdlib>
#include <exception>
+#include <functional>
#include <memory>
#include <sstream>
#include <string>
@@ -95,7 +96,9 @@ void Conductor::init() {
block_locator_ = make_unique<BlockLocator>(&bus_);
block_locator_->start();
- foreman_ = make_unique<ForemanDistributed>(*block_locator_, &bus_, query_processor_->getDefaultDatabase());
+ foreman_ = make_unique<ForemanDistributed>(*block_locator_,
+ std::bind(&QueryProcessor::saveCatalog, query_processor_.get()), &bus_,
+ query_processor_->getDefaultDatabase());
foreman_->start();
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/27a80558/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index fe4c483..4d95f16 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -17,6 +17,7 @@
#include <cstddef>
#include <cstdio>
#include <cstdlib>
+#include <functional>
#include <memory>
#include <unordered_map>
#include <unordered_set>
@@ -67,6 +68,7 @@ class QueryHandle;
ForemanDistributed::ForemanDistributed(
const BlockLocator &block_locator,
+ std::function<void()> &&save_catalog_callback,
MessageBus *bus,
CatalogDatabaseLite *catalog_database,
const int cpu_id)
@@ -106,6 +108,7 @@ ForemanDistributed::ForemanDistributed(
policy_enforcer_ = std::make_unique<PolicyEnforcerDistributed>(
foreman_client_id_,
+ move(save_catalog_callback),
catalog_database_,
&shiftboss_directory_,
bus_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/27a80558/query_execution/ForemanDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.hpp b/query_execution/ForemanDistributed.hpp
index ed09fda..5f1a14b 100644
--- a/query_execution/ForemanDistributed.hpp
+++ b/query_execution/ForemanDistributed.hpp
@@ -17,6 +17,7 @@
#include <cstddef>
#include <cstdio>
+#include <functional>
#include <memory>
#include <unordered_map>
#include <unordered_set>
@@ -55,6 +56,8 @@ class ForemanDistributed final : public ForemanBase {
* @param block_locator The block locator that manages block location info.
* @param bus A pointer to the TMB.
* @param catalog_database The catalog database where this query is executed.
+ * @param save_catalog_callback The callback used to save catalog upon the query
+ * completion.
* @param cpu_id The ID of the CPU to which the Foreman thread can be pinned.
*
* @note If cpu_id is not specified, Foreman thread can be possibly moved
@@ -62,6 +65,7 @@ class ForemanDistributed final : public ForemanBase {
**/
ForemanDistributed(
const BlockLocator &block_locator,
+ std::function<void()> &&save_catalog_callback,
tmb::MessageBus *bus,
CatalogDatabaseLite *catalog_database,
const int cpu_id = -1);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/27a80558/query_execution/PolicyEnforcerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.cpp b/query_execution/PolicyEnforcerDistributed.cpp
index e9f57d3..38b8a34 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -227,6 +227,8 @@ void PolicyEnforcerDistributed::onQueryCompletion(QueryManagerBase *query_manage
}
if (query_result == nullptr) {
+ save_catalog_callback_();
+
// Clean up query execution states, i.e., QueryContext, in Shiftbosses.
serialization::QueryTeardownMessage proto;
proto.set_query_id(query_id);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/27a80558/query_execution/PolicyEnforcerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.hpp b/query_execution/PolicyEnforcerDistributed.hpp
index 248948a..f8476c8 100644
--- a/query_execution/PolicyEnforcerDistributed.hpp
+++ b/query_execution/PolicyEnforcerDistributed.hpp
@@ -16,6 +16,7 @@
#define QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_DISTRIBUTED_HPP_
#include <cstddef>
+#include <functional>
#include <memory>
#include <unordered_map>
#include <utility>
@@ -57,15 +58,19 @@ class PolicyEnforcerDistributed final : public PolicyEnforcerBase {
* @brief Constructor.
*
* @param foreman_client_id The TMB client ID of the Foreman.
+ * @param save_catalog_callback The callback used to save catalog upon the query
+ * completion.
* @param catalog_database The CatalogDatabase used.
* @param bus The TMB.
**/
PolicyEnforcerDistributed(const tmb::client_id foreman_client_id,
+ std::function<void()> &&save_catalog_callback,
CatalogDatabaseLite *catalog_database,
ShiftbossDirectory *shiftboss_directory,
tmb::MessageBus *bus)
: PolicyEnforcerBase(catalog_database),
foreman_client_id_(foreman_client_id),
+ save_catalog_callback_(std::move(save_catalog_callback)),
shiftboss_directory_(shiftboss_directory),
bus_(bus) {}
@@ -154,6 +159,8 @@ class PolicyEnforcerDistributed final : public PolicyEnforcerBase {
const tmb::client_id foreman_client_id_;
+ const std::function<void()> save_catalog_callback_;
+
ShiftbossDirectory *shiftboss_directory_;
tmb::MessageBus *bus_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/27a80558/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
index 45d4fdf..2e18467 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
@@ -21,6 +21,7 @@
#include <cstdio>
#include <cstdlib>
+#include <functional>
#include <memory>
#include <set>
#include <string>
@@ -64,6 +65,12 @@ class CatalogRelation;
namespace optimizer {
+namespace {
+
+void nop() {}
+
+} // namespace
+
const char *DistributedExecutionGeneratorTestRunner::kResetOption =
"reset_before_execution";
@@ -98,7 +105,8 @@ DistributedExecutionGeneratorTestRunner::DistributedExecutionGeneratorTestRunner
// NOTE(zuyu): Foreman should initialize before Shiftboss so that the former
// could receive a registration message from the latter.
- foreman_ = make_unique<ForemanDistributed>(*block_locator_, &bus_, test_database_loader_->catalog_database());
+ foreman_ = make_unique<ForemanDistributed>(*block_locator_, std::bind(&nop), &bus_,
+ test_database_loader_->catalog_database());
// We don't use the NUMA aware version of worker code.
const vector<numa_node_id> numa_nodes(1 /* Number of worker threads per instance */,
[2/5] incubator-quickstep git commit: Extract getFilesize as a method.
Posted by ji...@apache.org.
Extract getFilesize as a method.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/43a9b398
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/43a9b398
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/43a9b398
Branch: refs/heads/reduce-group-by-attrs
Commit: 43a9b3981146ffaa6da125faf7bc0fa2a918834d
Parents: 61391ca
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Feb 4 16:08:02 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Feb 4 16:08:02 2017 -0800
----------------------------------------------------------------------
relational_operators/TextScanOperator.cpp | 25 ++++++++++++++++---------
1 file changed, 16 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/43a9b398/relational_operators/TextScanOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/TextScanOperator.cpp b/relational_operators/TextScanOperator.cpp
index aa734d3..6650319 100644
--- a/relational_operators/TextScanOperator.cpp
+++ b/relational_operators/TextScanOperator.cpp
@@ -78,6 +78,20 @@ static bool ValidateTextScanTextSegmentSize(const char *flagname,
static const volatile bool text_scan_text_segment_size_dummy = gflags::RegisterFlagValidator(
&FLAGS_textscan_text_segment_size, &ValidateTextScanTextSegmentSize);
+namespace {
+
+size_t getFileSize(const string &file_name) {
+ // Use standard C libary to retrieve the file size.
+ FILE *fp = std::fopen(file_name.c_str(), "rb");
+ std::fseek(fp, 0, SEEK_END);
+ const std::size_t file_size = std::ftell(fp);
+ std::fclose(fp);
+
+ return file_size;
+}
+
+} // namespace
+
bool TextScanOperator::getAllWorkOrders(
WorkOrdersContainer *container,
QueryContext *query_context,
@@ -105,10 +119,7 @@ bool TextScanOperator::getAllWorkOrders(
<< "File " << file << " is not readable due to permission issues.";
#endif // QUICKSTEP_HAVE_UNISTD
- FILE *fp = std::fopen(file.c_str(), "rb");
- std::fseek(fp, 0, SEEK_END);
- const std::size_t file_size = std::ftell(fp);
- std::fclose(fp);
+ const std::size_t file_size = getFileSize(file);
std::size_t text_offset = 0;
for (size_t num_full_segments = file_size / FLAGS_textscan_text_segment_size;
@@ -148,11 +159,7 @@ bool TextScanOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container
const std::vector<std::string> files = utility::file::GlobExpand(file_pattern_);
if (blocking_dependencies_met_ && !work_generated_) {
for (const string &file : files) {
- // Use standard C libary to retrieve the file size.
- FILE *fp = std::fopen(file.c_str(), "rb");
- std::fseek(fp, 0, SEEK_END);
- const std::size_t file_size = std::ftell(fp);
- std::fclose(fp);
+ const std::size_t file_size = getFileSize(file);
size_t text_offset = 0;
for (size_t num_full_segments = file_size / FLAGS_textscan_text_segment_size;
[5/5] incubator-quickstep git commit: Reduce the number of group-by
attributes by pulling tables up aggregations
Posted by ji...@apache.org.
Reduce the number of group-by attributes by pulling tables up aggregations
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3880a16e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3880a16e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3880a16e
Branch: refs/heads/reduce-group-by-attrs
Commit: 3880a16e308183966603ad4d5868b317492244a7
Parents: 27a8055
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Sun Jan 29 18:36:14 2017 -0600
Committer: jianqiao <ji...@node-2.jianqiao.quickstep-pg0.wisc.cloudlab.us>
Committed: Sun Feb 5 17:51:04 2017 -0600
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 1 +
query_optimizer/Optimizer.cpp | 3 +-
query_optimizer/Optimizer.hpp | 2 -
query_optimizer/PhysicalGenerator.cpp | 3 +
query_optimizer/PhysicalGenerator.hpp | 11 +-
query_optimizer/rules/CMakeLists.txt | 23 ++
.../rules/ReduceGroupByAttributes.cpp | 217 +++++++++++++++++++
.../rules/ReduceGroupByAttributes.hpp | 143 ++++++++++++
query_optimizer/tests/OptimizerTest.cpp | 2 +-
.../tests/OptimizerTextTestRunner.cpp | 7 +-
.../tests/OptimizerTextTestRunner.hpp | 3 +-
third_party/src/iwyu/iwyu_helper.py | 14 +-
12 files changed, 412 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3880a16e/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 7f90e11..bc9a52f 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -212,6 +212,7 @@ target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
quickstep_queryoptimizer_rules_InjectJoinFilters
quickstep_queryoptimizer_rules_PruneColumns
quickstep_queryoptimizer_rules_PushDownLowCostDisjunctivePredicate
+ quickstep_queryoptimizer_rules_ReduceGroupByAttributes
quickstep_queryoptimizer_rules_ReorderColumns
quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
quickstep_queryoptimizer_rules_SwapProbeBuild
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3880a16e/query_optimizer/Optimizer.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/Optimizer.cpp b/query_optimizer/Optimizer.cpp
index b14c938..1b91574 100644
--- a/query_optimizer/Optimizer.cpp
+++ b/query_optimizer/Optimizer.cpp
@@ -30,10 +30,11 @@ void Optimizer::generateQueryHandle(const ParseStatement &parse_statement,
OptimizerContext *optimizer_context,
QueryHandle *query_handle) {
LogicalGenerator logical_generator(optimizer_context);
+ PhysicalGenerator physical_generator(optimizer_context);
ExecutionGenerator execution_generator(catalog_database, query_handle);
execution_generator.generatePlan(
- physical_generator_.generatePlan(
+ physical_generator.generatePlan(
logical_generator.generatePlan(*catalog_database, parse_statement)));
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3880a16e/query_optimizer/Optimizer.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/Optimizer.hpp b/query_optimizer/Optimizer.hpp
index 36f956a..227dd04 100644
--- a/query_optimizer/Optimizer.hpp
+++ b/query_optimizer/Optimizer.hpp
@@ -70,8 +70,6 @@ class Optimizer {
QueryHandle *query_handle);
private:
- PhysicalGenerator physical_generator_;
-
DISALLOW_COPY_AND_ASSIGN(Optimizer);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3880a16e/query_optimizer/PhysicalGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/PhysicalGenerator.cpp b/query_optimizer/PhysicalGenerator.cpp
index 5dc0ffb..1b68f49 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -30,6 +30,7 @@
#include "query_optimizer/rules/InjectJoinFilters.hpp"
#include "query_optimizer/rules/PruneColumns.hpp"
#include "query_optimizer/rules/PushDownLowCostDisjunctivePredicate.hpp"
+#include "query_optimizer/rules/ReduceGroupByAttributes.hpp"
#include "query_optimizer/rules/ReorderColumns.hpp"
#include "query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp"
#include "query_optimizer/rules/SwapProbeBuild.hpp"
@@ -127,6 +128,8 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
// general FusePhysical optimization) in the future.
rules.emplace_back(new PushDownLowCostDisjunctivePredicate());
+ rules.emplace_back(new ReduceGroupByAttributes(optimizer_context_));
+
if (FLAGS_reorder_hash_joins) {
rules.emplace_back(new StarSchemaHashJoinOrderOptimization());
rules.emplace_back(new PruneColumns());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3880a16e/query_optimizer/PhysicalGenerator.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/PhysicalGenerator.hpp b/query_optimizer/PhysicalGenerator.hpp
index 886a173..42fea86 100644
--- a/query_optimizer/PhysicalGenerator.hpp
+++ b/query_optimizer/PhysicalGenerator.hpp
@@ -33,6 +33,8 @@
namespace quickstep {
namespace optimizer {
+class OptimizerContext;
+
/** \addtogroup QueryOptimizer
* @{
*/
@@ -43,9 +45,12 @@ namespace optimizer {
class PhysicalGenerator : public LogicalToPhysicalMapper {
public:
/**
- * @brief Constructor
+ * @brief Constructor.
+ *
+ * @param optimizer_context The optimizer context.
*/
- PhysicalGenerator() {
+ explicit PhysicalGenerator(OptimizerContext *optimizer_context)
+ : optimizer_context_(optimizer_context) {
createStrategies();
}
@@ -125,6 +130,8 @@ class PhysicalGenerator : public LogicalToPhysicalMapper {
*/
std::unordered_map<logical::LogicalPtr, physical::PhysicalPtr> logical_to_physical_map_;
+ OptimizerContext *optimizer_context_;
+
/**
* @brief The complete physical plan.
*/
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3880a16e/query_optimizer/rules/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/CMakeLists.txt b/query_optimizer/rules/CMakeLists.txt
index 223c78c..029d816 100644
--- a/query_optimizer/rules/CMakeLists.txt
+++ b/query_optimizer/rules/CMakeLists.txt
@@ -29,6 +29,9 @@ add_library(quickstep_queryoptimizer_rules_PushDownLowCostDisjunctivePredicate
PushDownLowCostDisjunctivePredicate.cpp
PushDownLowCostDisjunctivePredicate.hpp)
add_library(quickstep_queryoptimizer_rules_PushDownSemiAntiJoin PushDownSemiAntiJoin.cpp PushDownSemiAntiJoin.hpp)
+add_library(quickstep_queryoptimizer_rules_ReduceGroupByAttributes
+ ReduceGroupByAttributes.cpp
+ ReduceGroupByAttributes.hpp)
add_library(quickstep_queryoptimizer_rules_ReorderColumns ReorderColumns.cpp ReorderColumns.hpp)
add_library(quickstep_queryoptimizer_rules_Rule ../../empty_src.cpp Rule.hpp)
add_library(quickstep_queryoptimizer_rules_RuleHelper RuleHelper.cpp RuleHelper.hpp)
@@ -143,6 +146,25 @@ target_link_libraries(quickstep_queryoptimizer_rules_PushDownSemiAntiJoin
quickstep_queryoptimizer_logical_PatternMatcher
quickstep_queryoptimizer_rules_TopDownRule
quickstep_utility_Macros)
+target_link_libraries(quickstep_queryoptimizer_rules_ReduceGroupByAttributes
+ ${GFLAGS_LIB_NAME}
+ quickstep_catalog_CatalogRelation
+ quickstep_queryoptimizer_OptimizerContext
+ quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_expressions_ExprId
+ quickstep_queryoptimizer_expressions_ExpressionUtil
+ quickstep_queryoptimizer_expressions_NamedExpression
+ quickstep_queryoptimizer_physical_Aggregate
+ quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_PatternMatcher
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_queryoptimizer_physical_TableReference
+ quickstep_queryoptimizer_physical_TopLevelPlan
+ quickstep_queryoptimizer_rules_PruneColumns
+ quickstep_queryoptimizer_rules_Rule
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_rules_ReorderColumns
quickstep_queryoptimizer_expressions_AttributeReference
quickstep_queryoptimizer_expressions_ExprId
@@ -272,6 +294,7 @@ target_link_libraries(quickstep_queryoptimizer_rules
quickstep_queryoptimizer_rules_PushDownFilter
quickstep_queryoptimizer_rules_PushDownLowCostDisjunctivePredicate
quickstep_queryoptimizer_rules_PushDownSemiAntiJoin
+ quickstep_queryoptimizer_rules_ReduceGroupByAttributes
quickstep_queryoptimizer_rules_ReorderColumns
quickstep_queryoptimizer_rules_Rule
quickstep_queryoptimizer_rules_RuleHelper
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3880a16e/query_optimizer/rules/ReduceGroupByAttributes.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/ReduceGroupByAttributes.cpp b/query_optimizer/rules/ReduceGroupByAttributes.cpp
new file mode 100644
index 0000000..dcdd27a
--- /dev/null
+++ b/query_optimizer/rules/ReduceGroupByAttributes.cpp
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "query_optimizer/rules/ReduceGroupByAttributes.hpp"
+
+#include <algorithm>
+#include <map>
+#include <vector>
+#include <unordered_set>
+#include <utility>
+
+#include "catalog/CatalogRelation.hpp"
+#include "query_optimizer/OptimizerContext.hpp"
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/expressions/ExprId.hpp"
+#include "query_optimizer/expressions/ExpressionUtil.hpp"
+#include "query_optimizer/expressions/NamedExpression.hpp"
+#include "query_optimizer/physical/Aggregate.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/PatternMatcher.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/PhysicalType.hpp"
+#include "query_optimizer/physical/TableReference.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+#include "query_optimizer/rules/PruneColumns.hpp"
+
+#include "gflags/gflags.h"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+
+DEFINE_uint64(reduce_group_by_attributes_threshold, 3u,
+ "The threshold for a stored relation's number of attributes in a "
+ "group-by clause for the ReduceGroupByAttributes optimization "
+ "rule to pull the stored relation up the aggregation");
+
+namespace E = ::quickstep::optimizer::expressions;
+namespace P = ::quickstep::optimizer::physical;
+
+P::PhysicalPtr ReduceGroupByAttributes::apply(const P::PhysicalPtr &input) {
+ DCHECK(input->getPhysicalType() == P::PhysicalType::kTopLevelPlan);
+ cost_model_.reset(new cost::StarSchemaSimpleCostModel(
+ std::static_pointer_cast<const P::TopLevelPlan>(input)->shared_subplans()));
+
+ P::PhysicalPtr output = applyInternal(input);
+ if (output != input) {
+ output = PruneColumns().apply(output);
+ }
+ return output;
+}
+
+P::PhysicalPtr ReduceGroupByAttributes::applyInternal(const P::PhysicalPtr &input) {
+ std::vector<P::PhysicalPtr> new_children;
+ for (const P::PhysicalPtr &child : input->children()) {
+ new_children.push_back(applyInternal(child));
+ }
+
+ if (new_children != input->children()) {
+ return applyToNode(input->copyWithNewChildren(new_children));
+ } else {
+ return applyToNode(input);
+ }
+}
+
+P::PhysicalPtr ReduceGroupByAttributes::applyToNode(const P::PhysicalPtr &input) {
+ P::TableReferencePtr table_reference;
+ if (P::SomeTableReference::MatchesWithConditionalCast(input, &table_reference)) {
+ // Collect the attributes-to-TableReference mapping info.
+ for (const auto &attr : table_reference->attribute_list()) {
+ source_.emplace(attr->id(), std::make_pair(table_reference, attr));
+ }
+ return input;
+ }
+
+ P::AggregatePtr aggregate;
+ if (!P::SomeAggregate::MatchesWithConditionalCast(input, &aggregate) ||
+ aggregate->grouping_expressions().size() <= 1u) {
+ return input;
+ }
+
+ // Divide the group-by attributes into groups based on their source table.
+ std::map<P::TableReferencePtr, std::vector<E::AttributeReferencePtr>> table_attributes;
+ for (const auto &expr : aggregate->grouping_expressions()) {
+ const auto source_it = source_.find(expr->id());
+ if (source_it != source_.end()) {
+ table_attributes[source_it->second.first].emplace_back(source_it->second.second);
+ }
+ }
+
+ std::unordered_set<E::ExprId> erased_grouping_attr_ids;
+ std::vector<std::pair<P::TableReferencePtr, E::AttributeReferencePtr>> hoisted_tables;
+
+ // For each group (i.e. each source table), if it is profitable then we pull
+ // the table up the aggregation.
+ for (const auto &pair : table_attributes) {
+ const P::TableReferencePtr table = pair.first;
+ const std::vector<E::AttributeReferencePtr> &attributes = pair.second;
+ // TODO(jianqiao): find a cost-based metic instead of hard-coding the threshold
+ // number of group-by attributes.
+ if (attributes.size() <= FLAGS_reduce_group_by_attributes_threshold) {
+ continue;
+ }
+
+ std::vector<AttributeInfo> attr_infos;
+ for (const auto &attr : attributes) {
+ attr_infos.emplace_back(attr,
+ cost_model_->impliesUniqueAttributes(table, {attr}),
+ !attr->getValueType().isVariableLength(),
+ attr->getValueType().maximumByteLength());
+ }
+
+ std::vector<const AttributeInfo *> attr_info_refs;
+ for (const auto &info : attr_infos) {
+ attr_info_refs.emplace_back(&info);
+ }
+ std::sort(attr_info_refs.begin(),
+ attr_info_refs.end(),
+ AttributeInfo::IsBetterThan);
+
+ const AttributeInfo &best_candidate = *attr_info_refs.front();
+ if (!best_candidate.is_unique) {
+ // Cannot find a key attribute. Give up pulling this table up.
+ continue;
+ }
+
+ const E::AttributeReferencePtr key_attribute = best_candidate.attribute;
+ hoisted_tables.emplace_back(table, key_attribute);
+
+ for (const auto &attr : attributes) {
+ if (attr->id() != key_attribute->id()) {
+ erased_grouping_attr_ids.emplace(attr->id());
+ }
+ }
+ }
+
+ if (erased_grouping_attr_ids.empty()) {
+ return input;
+ }
+
+ // Reconstuct the Aggregate node with reduced group-by attributes and then
+ // construct HashJoin nodes on top of the Aggregate.
+ std::vector<E::NamedExpressionPtr> reduced_grouping_expressions;
+ for (const auto &expr : aggregate->grouping_expressions()) {
+ if (erased_grouping_attr_ids.find(expr->id()) == erased_grouping_attr_ids.end()) {
+ reduced_grouping_expressions.emplace_back(expr);
+ }
+ }
+
+ const P::AggregatePtr new_aggregate =
+ P::Aggregate::Create(aggregate->input(),
+ reduced_grouping_expressions,
+ aggregate->aggregate_expressions(),
+ aggregate->filter_predicate());
+
+ P::PhysicalPtr output = new_aggregate;
+ std::vector<E::NamedExpressionPtr> project_expressions =
+ E::ToNamedExpressions(output->getOutputAttributes());
+ for (const auto &pair : hoisted_tables) {
+ const P::TableReferencePtr &source_table = pair.first;
+ const E::AttributeReferencePtr &probe_attribute = pair.second;
+
+ E::AttributeReferencePtr build_attribute;
+ std::vector<E::AttributeReferencePtr> new_attribute_list;
+ for (const auto &attr : source_table->attribute_list()) {
+ if (attr->id() == probe_attribute->id()) {
+ build_attribute =
+ E::AttributeReference::Create(optimizer_context_->nextExprId(),
+ attr->attribute_name(),
+ attr->attribute_alias(),
+ attr->relation_name(),
+ attr->getValueType(),
+ E::AttributeReferenceScope::kLocal);
+ new_attribute_list.emplace_back(build_attribute);
+ } else {
+ new_attribute_list.emplace_back(attr);
+ project_expressions.emplace_back(attr);
+ }
+ }
+
+ DCHECK(build_attribute != nullptr);
+ const P::TableReferencePtr build_side_table =
+ P::TableReference::Create(source_table->relation(),
+ source_table->relation()->getName(),
+ new_attribute_list);
+ output = P::HashJoin::Create(output,
+ build_side_table,
+ {probe_attribute},
+ {build_attribute},
+ nullptr,
+ project_expressions,
+ P::HashJoin::JoinType::kInnerJoin);
+ }
+
+ return output;
+}
+
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3880a16e/query_optimizer/rules/ReduceGroupByAttributes.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/ReduceGroupByAttributes.hpp b/query_optimizer/rules/ReduceGroupByAttributes.hpp
new file mode 100644
index 0000000..5a1f295
--- /dev/null
+++ b/query_optimizer/rules/ReduceGroupByAttributes.hpp
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_OPTIMIZER_RULES_REDUCE_GROUP_BY_ATTRIBUTES_HPP_
+#define QUICKSTEP_QUERY_OPTIMIZER_RULES_REDUCE_GROUP_BY_ATTRIBUTES_HPP_
+
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <utility>
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/expressions/ExprId.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/TableReference.hpp"
+#include "query_optimizer/rules/Rule.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+namespace optimizer {
+
+class OptimizerContext;
+
+/**
+ * @brief Rule that applies to a physical plan to reduce the number of group-by
+ * attributes for Aggregate nodes (to improve performance) by pulling
+ * joins up the aggregations.
+ *
+ * For example, let R be a relation with PRIMARY KEY x and attributes y, z. Let
+ * S be a relation with FOREIGN KEY u refering to R(x) and attribute v. Then the
+ * optimization rule will transform the physical plan:
+ * Aggregate(
+ * [input relation]: HashJoin(
+ * [probe relation]: S
+ * [build relation]: R
+ * [join expression]: S.u = R.x
+ * [project attributes]: v, x, y, z
+ * )
+ * [aggregate expression]: SUM(v) AS sum_v
+ * [group-by attributes]: x, y, z
+ * )
+ *
+ * into:
+ * HashJoin(
+ * [probe relation]: Aggregate(
+ * [input relation]: S
+ * [aggregate expression]: SUM(v) AS sum_v
+ * [group-by attribute]: u
+ * ) AS T
+ * [build relation]: R
+ * [join expression]: T.u = R.x
+ * [project attributes]: sum_v, x, y, z
+ * )
+ */
+class ReduceGroupByAttributes : public Rule<physical::Physical> {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param optimizer_context The optimizer context.
+ */
+ explicit ReduceGroupByAttributes(OptimizerContext *optimizer_context)
+ : optimizer_context_(optimizer_context) {}
+
+ ~ReduceGroupByAttributes() override {}
+
+ std::string getName() const override {
+ return "ReduceGroupByAttributes";
+ }
+
+ physical::PhysicalPtr apply(const physical::PhysicalPtr &input) override;
+
+ private:
+ struct AttributeInfo {
+ AttributeInfo(const expressions::AttributeReferencePtr &attribute_in,
+ const bool is_unique_in,
+ const bool is_fixed_length_in,
+ const std::size_t maximum_size_in)
+ : attribute(attribute_in),
+ is_unique(is_unique_in),
+ is_fixed_length(is_fixed_length_in),
+ maximum_size(maximum_size_in) {}
+
+ // In the situation that there are multiple attributes that can serve as the
+ // group-by key, we define an ordering based on aggregation performance (e.g.
+ // it is faster to do aggregation with a fix-length attribute as the group-by
+ // key than with a variable-length attribute).
+ inline static bool IsBetterThan(const AttributeInfo *lhs,
+ const AttributeInfo *rhs) {
+ if (lhs->is_unique != rhs->is_unique) {
+ return lhs->is_unique;
+ }
+ if (lhs->is_fixed_length != rhs->is_fixed_length) {
+ return lhs->is_fixed_length;
+ }
+ if (lhs->maximum_size != rhs->maximum_size) {
+ return lhs->maximum_size < rhs->maximum_size;
+ }
+ return lhs->attribute->id() < rhs->attribute->id();
+ }
+
+ const expressions::AttributeReferencePtr attribute;
+ const bool is_unique;
+ const bool is_fixed_length;
+ const std::size_t maximum_size;
+ };
+
+ physical::PhysicalPtr applyInternal(const physical::PhysicalPtr &input);
+ physical::PhysicalPtr applyToNode(const physical::PhysicalPtr &input);
+
+ OptimizerContext *optimizer_context_;
+ std::unique_ptr<cost::StarSchemaSimpleCostModel> cost_model_;
+
+ // Maps an attribute's id to the TableReference that generates the attribute.
+ std::unordered_map<expressions::ExprId,
+ std::pair<physical::TableReferencePtr,
+ expressions::AttributeReferencePtr>> source_;
+
+ DISALLOW_COPY_AND_ASSIGN(ReduceGroupByAttributes);
+};
+
+} // namespace optimizer
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_OPTIMIZER_RULES_REDUCE_GROUP_BY_ATTRIBUTES_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3880a16e/query_optimizer/tests/OptimizerTest.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/OptimizerTest.cpp b/query_optimizer/tests/OptimizerTest.cpp
index 3838638..7eb7a11 100644
--- a/query_optimizer/tests/OptimizerTest.cpp
+++ b/query_optimizer/tests/OptimizerTest.cpp
@@ -62,7 +62,7 @@ OptimizerTest::OptimizerTest()
catalog_database_(
new CatalogDatabase(catalog_.get(), "TestDatabase" /* name */, 0)),
optimizer_context_(new OptimizerContext),
- physical_generator_(new PhysicalGenerator()) {}
+ physical_generator_(new PhysicalGenerator(optimizer_context_.get())) {}
E::AliasPtr OptimizerTest::createAlias(const E::ExpressionPtr &expression,
const std::string &attribute_name,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3880a16e/query_optimizer/tests/OptimizerTextTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/OptimizerTextTestRunner.cpp b/query_optimizer/tests/OptimizerTextTestRunner.cpp
index b9238c9..cb8f153 100644
--- a/query_optimizer/tests/OptimizerTextTestRunner.cpp
+++ b/query_optimizer/tests/OptimizerTextTestRunner.cpp
@@ -80,7 +80,7 @@ void OptimizerTextTestRunner::runTestCase(const std::string &input,
}
if (output_physical_plan) {
physical_plan =
- generatePhysicalPlan(optimized_logical_plan);
+ generatePhysicalPlan(optimized_logical_plan, &optimizer_context);
++num_options;
}
@@ -126,8 +126,9 @@ logical::LogicalPtr OptimizerTextTestRunner::generateLogicalPlan(
}
physical::PhysicalPtr OptimizerTextTestRunner::generatePhysicalPlan(
- const logical::LogicalPtr &logical_plan) {
- PhysicalGenerator physical_generator;
+ const logical::LogicalPtr &logical_plan,
+ OptimizerContext *optimizer_context) {
+ PhysicalGenerator physical_generator(optimizer_context);
return physical_generator.generatePlan(logical_plan);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3880a16e/query_optimizer/tests/OptimizerTextTestRunner.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/OptimizerTextTestRunner.hpp b/query_optimizer/tests/OptimizerTextTestRunner.hpp
index 27fa14f..d8f604b 100644
--- a/query_optimizer/tests/OptimizerTextTestRunner.hpp
+++ b/query_optimizer/tests/OptimizerTextTestRunner.hpp
@@ -73,7 +73,8 @@ class OptimizerTextTestRunner : public TextBasedTestRunner {
OptimizerContext *optimizer_context);
physical::PhysicalPtr generatePhysicalPlan(
- const logical::LogicalPtr &logical_plan);
+ const logical::LogicalPtr &logical_plan,
+ OptimizerContext *optimizer_context);
SqlParserWrapper sql_parser_;
TestDatabaseLoader test_database_loader_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3880a16e/third_party/src/iwyu/iwyu_helper.py
----------------------------------------------------------------------
diff --git a/third_party/src/iwyu/iwyu_helper.py b/third_party/src/iwyu/iwyu_helper.py
index dff4d55..93ddbae 100755
--- a/third_party/src/iwyu/iwyu_helper.py
+++ b/third_party/src/iwyu/iwyu_helper.py
@@ -22,15 +22,15 @@ QUICKSTEP_INCLUDES = [ '.',
'./build/third_party/gflags/include',
'./build/third_party/protobuf/include',
'./build/third_party/tmb/include',
- './third_party/benchmark/include',
- './third_party/glog/src',
- './third_party/googletest/googletest/include',
- './third_party/protobuf/src',
- './third_party/re2',
- './third_party/tmb/include']
+ './third_party/src/benchmark/include',
+ './third_party/src/glog/src',
+ './third_party/src/googletest/googletest/include',
+ './third_party/src/protobuf/src',
+ './third_party/src/re2',
+ './third_party/src/tmb/include']
QUICKSTEP_DEFINES = [ '-DQUICKSTEP_DEBUG',
'-DQUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION', ]
-CLANG_FLAGS = [ '-std=c++14', '-x', 'c++', ]
+CLANG_FLAGS = [ '-std=c++14', '-x', 'c++', '-ferror-limit=-1' ]
# Custom configuration filenames.
CUSTOM_IWYU_MAPPINGS = '.iwyu.imp'
[3/5] incubator-quickstep git commit: Moved hdfs-related flags into a
file.
Posted by ji...@apache.org.
Moved hdfs-related flags into a file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/dda085c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/dda085c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/dda085c7
Branch: refs/heads/reduce-group-by-attrs
Commit: dda085c7aeaa4f7f353c3392b547ca788d1507b8
Parents: 43a9b39
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Feb 4 14:46:44 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Feb 4 16:25:03 2017 -0800
----------------------------------------------------------------------
CMakeLists.txt | 1 +
cli/QuickstepCli.cpp | 1 +
storage/CMakeLists.txt | 13 ++++++---
storage/FileManagerHdfs.cpp | 35 +----------------------
storage/FileManagerHdfs.hpp | 6 ----
storage/Flags.cpp | 62 ++++++++++++++++++++++++++++++++++++++++
storage/Flags.hpp | 51 +++++++++++++++++++++++++++++++++
storage/StorageManager.cpp | 5 +---
storage/StorageManager.hpp | 4 ---
9 files changed, 126 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 85210ef..e74a274 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -789,6 +789,7 @@ target_link_libraries(quickstep_cli_shell
quickstep_queryexecution_WorkerDirectory
quickstep_queryoptimizer_QueryHandle
quickstep_queryoptimizer_QueryProcessor
+ quickstep_storage_Flags
quickstep_storage_PreloaderThread
quickstep_storage_StorageConstants
quickstep_storage_StorageManager
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 9db7577..02ec4ec 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -58,6 +58,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "query_execution/WorkerDirectory.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryProcessor.hpp"
+#include "storage/Flags.hpp"
#include "storage/StorageConfig.h" // For QUICKSTEP_HAVE_FILE_MANAGER_HDFS.
#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index fddea1f..a44c3a7 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -207,6 +207,7 @@ if (QUICKSTEP_HAVE_FILE_MANAGER_POSIX)
elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS)
add_library(quickstep_storage_FileManagerWindows FileManagerWindows.cpp FileManagerWindows.hpp)
endif()
+add_library(quickstep_storage_Flags Flags.cpp Flags.hpp)
add_library(quickstep_storage_HashTable ../empty_src.cpp HashTable.hpp)
add_library(quickstep_storage_HashTable_proto ${storage_HashTable_proto_srcs})
add_library(quickstep_storage_HashTableBase ../empty_src.cpp HashTableBase.hpp)
@@ -679,8 +680,8 @@ target_link_libraries(quickstep_storage_FileManager
if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
target_link_libraries(quickstep_storage_FileManagerHdfs
glog
- ${GFLAGS_LIB_NAME}
quickstep_storage_FileManager
+ quickstep_storage_Flags
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageConstants
quickstep_storage_StorageErrors
@@ -711,6 +712,8 @@ elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS)
quickstep_utility_Macros
quickstep_utility_StringUtil)
endif()
+target_link_libraries(quickstep_storage_Flags
+ ${GFLAGS_LIB_NAME})
target_link_libraries(quickstep_storage_HashTable
quickstep_catalog_CatalogTypedefs
quickstep_storage_HashTableBase
@@ -1000,6 +1003,7 @@ target_link_libraries(quickstep_storage_StorageManager
quickstep_storage_EvictionPolicy
quickstep_storage_FileManager
quickstep_storage_FileManagerLocal
+ quickstep_storage_Flags
quickstep_storage_StorageBlob
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockBase
@@ -1121,17 +1125,18 @@ target_link_libraries(quickstep_storage
quickstep_storage_CompressedTupleStorageSubBlock
quickstep_storage_CountedReference
quickstep_storage_EvictionPolicy
- quickstep_storage_FileManager
- quickstep_storage_FileManagerLocal
quickstep_storage_FastHashTable
quickstep_storage_FastHashTableFactory
quickstep_storage_FastSeparateChainingHashTable
+ quickstep_storage_FileManager
+ quickstep_storage_FileManagerLocal
+ quickstep_storage_Flags
quickstep_storage_HashTable
- quickstep_storage_HashTable_proto
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
quickstep_storage_HashTableKeyManager
quickstep_storage_HashTablePool
+ quickstep_storage_HashTable_proto
quickstep_storage_IndexSubBlock
quickstep_storage_IndexSubBlockDescriptionFactory
quickstep_storage_InsertDestination
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/FileManagerHdfs.cpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerHdfs.cpp b/storage/FileManagerHdfs.cpp
index e28ed3b..937750a 100644
--- a/storage/FileManagerHdfs.cpp
+++ b/storage/FileManagerHdfs.cpp
@@ -31,12 +31,12 @@
#include <cstring>
#include <string>
+#include "storage/Flags.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageConstants.hpp"
#include "storage/StorageErrors.hpp"
#include "utility/StringUtil.hpp"
-#include "gflags/gflags.h"
#include "glog/logging.h"
using std::size_t;
@@ -46,41 +46,8 @@ using std::string;
namespace quickstep {
-DEFINE_string(hdfs_namenode_host, "localhost", "Host name or IP address of HDFS namenode.");
-
-static bool ValidateHdfsNameNodePort(const char *flagname,
- std::int32_t value) {
- if (value > 0 && value < 65536) {
- return true;
- } else {
- std::fprintf(stderr, "--%s must be between 1 and 65535 (inclusive)\n", flagname);
- return false;
- }
-}
-DEFINE_int32(hdfs_namenode_port, 9000, "Port of HDFS namenode.");
-static const bool hdfs_namenode_port_dummy
- = gflags::RegisterFlagValidator(&FLAGS_hdfs_namenode_port, &ValidateHdfsNameNodePort);
-
-static bool ValidateHdfsNumReplications(const char *flagname,
- std::int32_t value) {
- // The default dfs.namenode.replication.min is 1, while the default
- // dfs.replication.max is 512.
- if (value > 0) {
- return true;
- } else {
- std::fprintf(stderr, "--%s must be at least 1\n", flagname);
- return false;
- }
-}
-DEFINE_int32(hdfs_num_replications, 1, "Number of HDFS file replications.");
-static const bool hdfs_num_replications_dummy
- = gflags::RegisterFlagValidator(&FLAGS_hdfs_num_replications, &ValidateHdfsNumReplications);
-
FileManagerHdfs::FileManagerHdfs(const string &storage_path)
: FileManager(storage_path) {
- DCHECK(hdfs_namenode_port_dummy);
- DCHECK(hdfs_num_replications_dummy);
-
struct hdfsBuilder *builder = hdfsNewBuilder();
hdfsBuilderSetNameNode(builder, FLAGS_hdfs_namenode_host.c_str());
hdfsBuilderSetNameNodePort(builder, FLAGS_hdfs_namenode_port);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/FileManagerHdfs.hpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerHdfs.hpp b/storage/FileManagerHdfs.hpp
index 3c751b5..f47e4a8 100644
--- a/storage/FileManagerHdfs.hpp
+++ b/storage/FileManagerHdfs.hpp
@@ -29,14 +29,8 @@
#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
-#include "gflags/gflags.h"
-
namespace quickstep {
-DECLARE_string(hdfs_namenode_host);
-DECLARE_int32(hdfs_namenode_port);
-DECLARE_int32(hdfs_num_replications);
-
/** \addtogroup Storage
* @{
*/
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/Flags.cpp
----------------------------------------------------------------------
diff --git a/storage/Flags.cpp b/storage/Flags.cpp
new file mode 100644
index 0000000..4672f81
--- /dev/null
+++ b/storage/Flags.cpp
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "storage/Flags.hpp"
+
+#include "storage/StorageConfig.h"
+
+#include "gflags/gflags.h"
+
+namespace quickstep {
+
+#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
+DEFINE_bool(use_hdfs, false, "Use HDFS as the persistent storage, instead of the local disk.");
+
+DEFINE_string(hdfs_namenode_host, "localhost", "Host name or IP address of HDFS namenode.");
+
+static bool ValidateHdfsNameNodePort(const char *flagname,
+ std::int32_t value) {
+ if (value > 0 && value < 65536) {
+ return true;
+ } else {
+ std::fprintf(stderr, "--%s must be between 1 and 65535 (inclusive)\n", flagname);
+ return false;
+ }
+}
+DEFINE_int32(hdfs_namenode_port, 9000, "Port of HDFS namenode.");
+static const bool hdfs_namenode_port_dummy
+ = gflags::RegisterFlagValidator(&FLAGS_hdfs_namenode_port, &ValidateHdfsNameNodePort);
+
+static bool ValidateHdfsNumReplications(const char *flagname,
+ std::int32_t value) {
+ // The default dfs.namenode.replication.min is 1, while the default
+ // dfs.replication.max is 512.
+ if (value > 0) {
+ return true;
+ } else {
+ std::fprintf(stderr, "--%s must be at least 1\n", flagname);
+ return false;
+ }
+}
+DEFINE_int32(hdfs_num_replications, 1, "Number of HDFS file replications.");
+static const bool hdfs_num_replications_dummy
+ = gflags::RegisterFlagValidator(&FLAGS_hdfs_num_replications, &ValidateHdfsNumReplications);
+#endif
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/Flags.hpp
----------------------------------------------------------------------
diff --git a/storage/Flags.hpp b/storage/Flags.hpp
new file mode 100644
index 0000000..1d5527c
--- /dev/null
+++ b/storage/Flags.hpp
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_FLAGS_HPP_
+#define QUICKSTEP_STORAGE_FLAGS_HPP_
+
+#include "storage/StorageConfig.h" // For QUICKSTEP_HAVE_FILE_MANAGER_HDFS.
+
+#include "gflags/gflags_declare.h"
+
+namespace quickstep {
+
+/** \addtogroup STORAGE
+ * @{
+ */
+
+/**
+ * @brief A collection of common flags shared by Quickstep STORAGEs in both the
+ * single-node and the distributed version.
+ **/
+
+#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
+DECLARE_bool(use_hdfs);
+
+DECLARE_string(hdfs_namenode_host);
+DECLARE_int32(hdfs_namenode_port);
+DECLARE_int32(hdfs_num_replications);
+
+#endif
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_STORAGE_FLAGS_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 6299cda..6f7d38b 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -76,6 +76,7 @@
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManagerLocal.hpp"
+#include "storage/Flags.hpp"
#include "storage/StorageBlob.hpp"
#include "storage/StorageBlock.hpp"
#include "storage/StorageBlockBase.hpp"
@@ -189,10 +190,6 @@ DEFINE_uint64(buffer_pool_slots, 0,
static const volatile bool buffer_pool_slots_dummy
= gflags::RegisterFlagValidator(&FLAGS_buffer_pool_slots, &SetOrValidateBufferPoolSlots);
-#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
-DEFINE_bool(use_hdfs, false, "Use HDFS as the persistent storage, instead of the local disk.");
-#endif
-
StorageManager::StorageManager(
const std::string &storage_path,
const block_id_domain block_domain,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index b61f10a..42176ee 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -63,10 +63,6 @@ namespace quickstep {
DECLARE_int32(block_domain);
DECLARE_uint64(buffer_pool_slots);
-#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
-DECLARE_bool(use_hdfs);
-#endif
-
class CatalogRelationSchema;
#ifdef QUICKSTEP_DISTRIBUTED