You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by ji...@apache.org on 2016/12/15 03:50:41 UTC
[01/39] incubator-quickstep git commit: Allows filters to be
evaluated. [Forced Update!]
Repository: incubator-quickstep
Updated Branches:
refs/heads/min-max-stats e915cbdbb -> f8feba19b (forced update)
Allows filters to be evaluated.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3093e744
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3093e744
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3093e744
Branch: refs/heads/min-max-stats
Commit: 3093e74484b620f828e435f434d674390f81a1c9
Parents: c45d68e
Author: cramja <ma...@gmail.com>
Authored: Tue Nov 8 17:08:22 2016 -0600
Committer: cramja <ma...@gmail.com>
Committed: Wed Nov 9 11:26:20 2016 -0600
----------------------------------------------------------------------
storage/SMAIndexSubBlock.cpp | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3093e744/storage/SMAIndexSubBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/SMAIndexSubBlock.cpp b/storage/SMAIndexSubBlock.cpp
index 621bafd..3b3b879 100644
--- a/storage/SMAIndexSubBlock.cpp
+++ b/storage/SMAIndexSubBlock.cpp
@@ -672,25 +672,25 @@ predicate_cost_t SMAIndexSubBlock::estimatePredicateEvaluationCost(
TupleIdSequence* SMAIndexSubBlock::getMatchesForPredicate(
const ComparisonPredicate &predicate,
const TupleIdSequence *filter) const {
- if (filter != nullptr) {
- LOG(FATAL) << "SMAIndex cannot evaluate filters.";
- }
-
Selectivity selectivity = getSelectivityForPredicate(predicate);
if (selectivity == Selectivity::kAll) {
- TupleIdSequence* tidseq = new TupleIdSequence(tuple_store_.numTuples());
-
- // Set all existing tuples to true, selected.
- if (tuple_store_.isPacked()) {
- tidseq->setRange(0, tuple_store_.numTuples(), true);
+ if (filter != nullptr) {
+ return new TupleIdSequence(filter->length(), filter->getInternalBitVector());
} else {
- for (tuple_id tid = 0; tid <= tuple_store_.getMaxTupleID(); ++tid) {
- if (tuple_store_.hasTupleWithID(tid)) {
- tidseq->set(tid, true);
+ TupleIdSequence* tidseq = new TupleIdSequence(tuple_store_.numTuples());
+
+ // Set all existing tuples to true, selected.
+ if (tuple_store_.isPacked()) {
+ tidseq->setRange(0, tuple_store_.numTuples(), true);
+ } else {
+ for (tuple_id tid = 0; tid <= tuple_store_.getMaxTupleID(); ++tid) {
+ if (tuple_store_.hasTupleWithID(tid)) {
+ tidseq->set(tid, true);
+ }
}
}
+ return tidseq;
}
- return tidseq;
} else if (selectivity == Selectivity::kNone) {
// A new tuple ID sequence is initialized to false for all values.
return new TupleIdSequence(tuple_store_.numTuples());
[08/39] incubator-quickstep git commit: Unified
WorkOrderCompletionMessage.
Posted by ji...@apache.org.
Unified WorkOrderCompletionMessage.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/178ed4b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/178ed4b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/178ed4b3
Branch: refs/heads/min-max-stats
Commit: 178ed4b3241d5c16736459f9e9073922a03d99b2
Parents: 7095987
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Nov 12 16:44:39 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Fri Nov 18 13:21:38 2016 -0800
----------------------------------------------------------------------
query_execution/PolicyEnforcerBase.cpp | 6 ++---
query_execution/PolicyEnforcerBase.hpp | 4 +--
query_execution/QueryExecutionMessages.proto | 32 ++++++++---------------
query_execution/Worker.cpp | 28 +++++++++-----------
query_execution/Worker.hpp | 13 +++++----
5 files changed, 35 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/178ed4b3/query_execution/PolicyEnforcerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.cpp b/query_execution/PolicyEnforcerBase.cpp
index 745ded6..4e8c782 100644
--- a/query_execution/PolicyEnforcerBase.cpp
+++ b/query_execution/PolicyEnforcerBase.cpp
@@ -45,7 +45,7 @@ void PolicyEnforcerBase::processMessage(const TaggedMessage &tagged_message) {
switch (tagged_message.message_type()) {
case kWorkOrderCompleteMessage: {
- serialization::NormalWorkOrderCompletionMessage proto;
+ serialization::WorkOrderCompletionMessage proto;
// Note: This proto message contains the time it took to execute the
// WorkOrder. It can be accessed in this scope.
CHECK(proto.ParseFromArray(tagged_message.message(),
@@ -64,7 +64,7 @@ void PolicyEnforcerBase::processMessage(const TaggedMessage &tagged_message) {
break;
}
case kRebuildWorkOrderCompleteMessage: {
- serialization::RebuildWorkOrderCompletionMessage proto;
+ serialization::WorkOrderCompletionMessage proto;
// Note: This proto message contains the time it took to execute the
// rebuild WorkOrder. It can be accessed in this scope.
CHECK(proto.ParseFromArray(tagged_message.message(),
@@ -157,7 +157,7 @@ bool PolicyEnforcerBase::admitQueries(
}
void PolicyEnforcerBase::recordTimeForWorkOrder(
- const serialization::NormalWorkOrderCompletionMessage &proto) {
+ const serialization::WorkOrderCompletionMessage &proto) {
const std::size_t query_id = proto.query_id();
std::vector<WorkOrderTimeEntry> &workorder_time_entries
= workorder_time_recorder_[query_id];
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/178ed4b3/query_execution/PolicyEnforcerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.hpp b/query_execution/PolicyEnforcerBase.hpp
index ea2c06f..7009a0a 100644
--- a/query_execution/PolicyEnforcerBase.hpp
+++ b/query_execution/PolicyEnforcerBase.hpp
@@ -38,7 +38,7 @@ namespace quickstep {
class CatalogDatabaseLite;
class QueryHandle;
-namespace serialization { class NormalWorkOrderCompletionMessage; }
+namespace serialization { class WorkOrderCompletionMessage; }
/** \addtogroup QueryExecution
* @{
@@ -165,7 +165,7 @@ class PolicyEnforcerBase {
* execution.
**/
void recordTimeForWorkOrder(
- const serialization::NormalWorkOrderCompletionMessage &proto);
+ const serialization::WorkOrderCompletionMessage &proto);
CatalogDatabaseLite *catalog_database_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/178ed4b3/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 1a2cb78..165a194 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -31,31 +31,21 @@ import "relational_operators/WorkOrder.proto";
// order completion message, we may be interested in adding the compression
// ratio or dictionary size of the rebuilt block.
-// TODO(harshad) : If there are different fields in the two message types below,
-// create a base message class called WorkOrderCompletionMessage and make the
-// two classes below extend the base class. All the common fields in both the
-// classes can be moved to the base class.
+message WorkOrderCompletionMessage {
+ enum WorkOrderType {
+ NORMAL = 0;
+ REBUILD = 1;
+ }
-// A message sent upon completion of a normal (not rebuild) WorkOrder execution.
-message NormalWorkOrderCompletionMessage {
- required uint64 operator_index = 1;
- required uint64 worker_thread_index = 2;
- required uint64 query_id = 3;
-
- // Epoch time in microseconds.
- optional uint64 execution_start_time = 4;
- optional uint64 execution_end_time = 5;
-}
+ required WorkOrderType work_order_type = 1;
-// A message sent upon completion of a rebuild WorkOrder execution.
-message RebuildWorkOrderCompletionMessage {
- required uint64 operator_index = 1;
- required uint64 worker_thread_index = 2;
- required uint64 query_id = 3;
+ required uint64 operator_index = 2;
+ required uint64 worker_thread_index = 3;
+ required uint64 query_id = 4;
// Epoch time in microseconds.
- optional uint64 execution_start_time = 4;
- optional uint64 execution_end_time = 5;
+ optional uint64 execution_start_time = 5;
+ optional uint64 execution_end_time = 6;
}
message CatalogRelationNewBlockMessage {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/178ed4b3/query_execution/Worker.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Worker.cpp b/query_execution/Worker.cpp
index 0b1efba..0db17b4 100644
--- a/query_execution/Worker.cpp
+++ b/query_execution/Worker.cpp
@@ -47,6 +47,8 @@ using tmb::TaggedMessage;
namespace quickstep {
+using serialization::WorkOrderCompletionMessage;
+
void Worker::run() {
if (cpu_id_ >= 0) {
ThreadUtil::BindToCPU(cpu_id_);
@@ -61,21 +63,16 @@ void Worker::run() {
const TaggedMessage &tagged_message = annotated_msg.tagged_message;
switch (tagged_message.message_type()) {
case kWorkOrderMessage: {
- serialization::NormalWorkOrderCompletionMessage proto;
- executeWorkOrderHelper<serialization::NormalWorkOrderCompletionMessage>(
- tagged_message, &proto);
- sendWorkOrderCompleteMessage<
- serialization::NormalWorkOrderCompletionMessage>(
+ WorkOrderCompletionMessage proto;
+ executeWorkOrderHelper(tagged_message, &proto);
+ sendWorkOrderCompleteMessage(
annotated_msg.sender, proto, kWorkOrderCompleteMessage);
break;
}
case kRebuildWorkOrderMessage: {
- serialization::RebuildWorkOrderCompletionMessage proto;
- executeWorkOrderHelper<
- serialization::RebuildWorkOrderCompletionMessage>(tagged_message,
- &proto);
- sendWorkOrderCompleteMessage<
- serialization::RebuildWorkOrderCompletionMessage>(
+ WorkOrderCompletionMessage proto;
+ executeWorkOrderHelper(tagged_message, &proto, true /* is_rebuild */);
+ sendWorkOrderCompleteMessage(
annotated_msg.sender, proto, kRebuildWorkOrderCompleteMessage);
break;
}
@@ -88,9 +85,8 @@ void Worker::run() {
}
}
-template <typename CompletionMessageProtoT>
void Worker::sendWorkOrderCompleteMessage(const tmb::client_id receiver,
- const CompletionMessageProtoT &proto,
+ const WorkOrderCompletionMessage &proto,
const message_type_id message_type) {
// NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
const size_t proto_length = proto.ByteSize();
@@ -109,9 +105,9 @@ void Worker::sendWorkOrderCompleteMessage(const tmb::client_id receiver,
CHECK(send_status == tmb::MessageBus::SendStatus::kOK);
}
-template <typename CompletionMessageProtoT>
void Worker::executeWorkOrderHelper(const TaggedMessage &tagged_message,
- CompletionMessageProtoT *proto) {
+ WorkOrderCompletionMessage *proto,
+ const bool is_rebuild_work_order) {
std::chrono::time_point<std::chrono::steady_clock> start, end;
WorkerMessage worker_message(
*static_cast<const WorkerMessage *>(tagged_message.message()));
@@ -133,6 +129,8 @@ void Worker::executeWorkOrderHelper(const TaggedMessage &tagged_message,
end.time_since_epoch()).count();
// Construct the proto message.
+ proto->set_work_order_type(is_rebuild_work_order ? WorkOrderCompletionMessage::REBUILD
+ : WorkOrderCompletionMessage::NORMAL);
proto->set_operator_index(worker_message.getRelationalOpIndex());
proto->set_query_id(query_id_for_workorder);
proto->set_worker_thread_index(worker_thread_index_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/178ed4b3/query_execution/Worker.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Worker.hpp b/query_execution/Worker.hpp
index aa39bb3..fec2242 100644
--- a/query_execution/Worker.hpp
+++ b/query_execution/Worker.hpp
@@ -34,6 +34,8 @@ namespace tmb { class TaggedMessge; }
namespace quickstep {
+namespace serialization { class WorkOrderCompletionMessage; }
+
/** \addtogroup QueryExecution
* @{
*/
@@ -100,7 +102,6 @@ class Worker : public Thread {
* @brief A helper method to execute the WorkOrder and construct a
* completion message.
*
- * @note CompletionMessageProtoT is the type of the completion message.
* @note Right now a single helper method works for all message types.
* If different message types need to collect different statistics for
* the WorkOrder execution, we need to create different helper methods,
@@ -108,23 +109,21 @@ class Worker : public Thread {
*
* @param tagged_message The TaggedMessage which consists of the WorkOrder.
* @param proto The proto message to be sent.
+ * @param is_rebuild_work_order Whether it is used for a RebuildWorkOrder.
**/
- template <typename CompletionMessageProtoT>
void executeWorkOrderHelper(const TaggedMessage &tagged_message,
- CompletionMessageProtoT *proto);
+ serialization::WorkOrderCompletionMessage *proto,
+ const bool is_rebuild_work_order = false);
/**
* @brief A helper method to send the WorkOrder completion message.
*
- * @note CompletionMessageProtoT is the type of the completion message.
- *
* @param receiver The TMB client ID of the receiver.
* @param proto The proto message to be sent.
* @param message_type The ID of the type of the message being sent.
**/
- template <typename CompletionMessageProtoT>
void sendWorkOrderCompleteMessage(const tmb::client_id receiver,
- const CompletionMessageProtoT &proto,
+ const serialization::WorkOrderCompletionMessage &proto,
const message_type_id message_type);
const std::size_t worker_thread_index_;
[32/39] incubator-quickstep git commit: Transfered the ownership of
QueryHandle to QueryManager.
Posted by ji...@apache.org.
Transfered the ownership of QueryHandle to QueryManager.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c608e99e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c608e99e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c608e99e
Branch: refs/heads/min-max-stats
Commit: c608e99eff883afacafa2f0fefce99a0513f9963
Parents: 57730e4
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 20 12:04:24 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 27 18:00:34 2016 -0800
----------------------------------------------------------------------
cli/CommandExecutor.cpp | 30 +++++++------
cli/QuickstepCli.cpp | 46 ++++++++++----------
cli/tests/CommandExecutorTestRunner.cpp | 31 +++++++------
query_execution/QueryManagerBase.hpp | 7 ++-
.../tests/QueryManagerSingleNode_unittest.cpp | 8 ++--
.../DistributedExecutionGeneratorTestRunner.cpp | 34 ++++++++-------
.../tests/ExecutionGeneratorTestRunner.cpp | 35 ++++++++-------
7 files changed, 99 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 4ab32de..3c510e7 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -213,25 +213,27 @@ inline std::vector<TypedValue> executeQueryForSingleRow(
DCHECK(result.condition == ParseResult::kSuccess);
const ParseStatement &statement = *result.parsed_statement;
+ const CatalogRelation *query_result_relation = nullptr;
- // Generate the query plan.
- std::unique_ptr<QueryHandle> query_handle(
- std::make_unique<QueryHandle>(query_processor->query_id(),
- main_thread_client_id,
- statement.getPriority()));
- query_processor->generateQueryHandle(statement, query_handle.get());
- DCHECK(query_handle->getQueryPlanMutable() != nullptr);
-
- // Use foreman to execute the query plan.
- QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
- main_thread_client_id, foreman_client_id, query_handle.get(), bus);
+ {
+ // Generate the query plan.
+ auto query_handle =
+ std::make_unique<QueryHandle>(query_processor->query_id(),
+ main_thread_client_id,
+ statement.getPriority());
+ query_processor->generateQueryHandle(statement, query_handle.get());
+ DCHECK(query_handle->getQueryPlanMutable() != nullptr);
+ query_result_relation = query_handle->getQueryResultRelation();
+ DCHECK(query_result_relation != nullptr);
+
+ // Use foreman to execute the query plan.
+ QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
+ main_thread_client_id, foreman_client_id, query_handle.release(), bus);
+ }
QueryExecutionUtil::ReceiveQueryCompletionMessage(main_thread_client_id, bus);
// Retrieve the scalar result from the result relation.
- const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
- DCHECK(query_result_relation != nullptr);
-
std::vector<TypedValue> values;
{
std::vector<block_id> blocks = query_result_relation->getBlocksSnapshot();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index f4816a8..9db7577 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -341,38 +341,41 @@ int main(int argc, char* argv[]) {
continue;
}
- std::unique_ptr<QueryHandle> query_handle(
- std::make_unique<QueryHandle>(query_processor->query_id(),
- main_thread_client_id,
- statement.getPriority()));
+ const std::size_t query_id = query_processor->query_id();
+ const CatalogRelation *query_result_relation = nullptr;
+ std::unique_ptr<quickstep::ExecutionDAGVisualizer> dag_visualizer;
+
try {
+ auto query_handle = std::make_unique<QueryHandle>(query_id,
+ main_thread_client_id,
+ statement.getPriority());
query_processor->generateQueryHandle(statement, query_handle.get());
+ DCHECK(query_handle->getQueryPlanMutable() != nullptr);
+
+ if (quickstep::FLAGS_visualize_execution_dag) {
+ dag_visualizer =
+ std::make_unique<quickstep::ExecutionDAGVisualizer>(*query_handle->getQueryPlanMutable());
+ }
+
+ query_result_relation = query_handle->getQueryResultRelation();
+
+ start = std::chrono::steady_clock::now();
+ QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
+ main_thread_client_id,
+ foreman.getBusClientID(),
+ query_handle.release(),
+ &bus);
} catch (const quickstep::SqlError &sql_error) {
fprintf(stderr, "%s", sql_error.formatMessage(*command_string).c_str());
reset_parser = true;
break;
}
- DCHECK(query_handle->getQueryPlanMutable() != nullptr);
- std::unique_ptr<quickstep::ExecutionDAGVisualizer> dag_visualizer;
- if (quickstep::FLAGS_visualize_execution_dag) {
- dag_visualizer.reset(
- new quickstep::ExecutionDAGVisualizer(*query_handle->getQueryPlanMutable()));
- }
-
- start = std::chrono::steady_clock::now();
- QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
- main_thread_client_id,
- foreman.getBusClientID(),
- query_handle.get(),
- &bus);
-
try {
QueryExecutionUtil::ReceiveQueryCompletionMessage(
main_thread_client_id, &bus);
end = std::chrono::steady_clock::now();
- const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
if (query_result_relation) {
PrintToScreen::PrintRelation(*query_result_relation,
&storage_manager,
@@ -394,12 +397,11 @@ int main(int argc, char* argv[]) {
time_ms.count(), 3).c_str());
if (quickstep::FLAGS_profile_and_report_workorder_perf) {
// TODO(harshad) - Allow user specified file instead of stdout.
- foreman.printWorkOrderProfilingResults(query_handle->query_id(),
- stdout);
+ foreman.printWorkOrderProfilingResults(query_id, stdout);
}
if (quickstep::FLAGS_visualize_execution_dag) {
const auto &profiling_stats =
- foreman.getWorkOrderProfilingResults(query_handle->query_id());
+ foreman.getWorkOrderProfilingResults(query_id);
dag_visualizer->bindProfilingStats(profiling_stats);
std::cerr << "\n" << dag_visualizer->toDOT() << "\n";
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/cli/tests/CommandExecutorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/cli/tests/CommandExecutorTestRunner.cpp b/cli/tests/CommandExecutorTestRunner.cpp
index 41cc9da..e352b24 100644
--- a/cli/tests/CommandExecutorTestRunner.cpp
+++ b/cli/tests/CommandExecutorTestRunner.cpp
@@ -20,6 +20,7 @@
#include "cli/tests/CommandExecutorTestRunner.hpp"
#include <cstdio>
+#include <memory>
#include <set>
#include <string>
#include <utility>
@@ -88,28 +89,26 @@ void CommandExecutorTestRunner::runTestCase(
nullptr,
output_stream.file());
} else {
- QueryHandle query_handle(0 /* query_id */, main_thread_client_id_);
- O::OptimizerContext optimizer_context;
-
- optimizer_.generateQueryHandle(parse_statement,
- test_database_loader_.catalog_database(),
- &optimizer_context,
- &query_handle);
-
- AdmitRequestMessage request_message(&query_handle);
- TaggedMessage admit_tagged_message(
- &request_message, sizeof(request_message), kAdmitRequestMessage);
- QueryExecutionUtil::SendTMBMessage(&bus_,
- main_thread_client_id_,
- foreman_->getBusClientID(),
- std::move(admit_tagged_message));
+ const CatalogRelation *query_result_relation = nullptr;
+ {
+ auto query_handle = std::make_unique<QueryHandle>(0 /* query_id */, main_thread_client_id_);
+ O::OptimizerContext optimizer_context;
+
+ optimizer_.generateQueryHandle(parse_statement,
+ test_database_loader_.catalog_database(),
+ &optimizer_context,
+ query_handle.get());
+ query_result_relation = query_handle->getQueryResultRelation();
+
+ QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
+ main_thread_client_id_, foreman_->getBusClientID(), query_handle.release(), &bus_);
+ }
// Receive workload completion message from Foreman.
const AnnotatedMessage annotated_msg =
bus_.Receive(main_thread_client_id_, 0, true);
const TaggedMessage &tagged_message = annotated_msg.tagged_message;
DCHECK_EQ(kWorkloadCompletionMessage, tagged_message.message_type());
- const CatalogRelation *query_result_relation = query_handle.getQueryResultRelation();
if (query_result_relation) {
PrintToScreen::PrintRelation(*query_result_relation,
test_database_loader_.storage_manager(),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/query_execution/QueryManagerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.hpp b/query_execution/QueryManagerBase.hpp
index a274742..ddb76d5 100644
--- a/query_execution/QueryManagerBase.hpp
+++ b/query_execution/QueryManagerBase.hpp
@@ -26,6 +26,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/QueryExecutionState.hpp"
+#include "query_optimizer/QueryHandle.hpp"
#include "relational_operators/RelationalOperator.hpp"
#include "relational_operators/WorkOrder.hpp"
#include "storage/StorageBlockInfo.hpp"
@@ -34,8 +35,6 @@
namespace quickstep {
-class QueryHandle;
-
/** \addtogroup QueryExecution
* @{
*/
@@ -77,7 +76,7 @@ class QueryManagerBase {
* @brief Get the query handle.
**/
const QueryHandle* query_handle() const {
- return query_handle_;
+ return query_handle_.get();
}
/**
@@ -259,7 +258,7 @@ class QueryManagerBase {
return query_exec_state_->hasRebuildInitiated(index);
}
- const QueryHandle *query_handle_;
+ std::unique_ptr<QueryHandle> query_handle_;
const std::size_t query_id_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/query_execution/tests/QueryManagerSingleNode_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/QueryManagerSingleNode_unittest.cpp b/query_execution/tests/QueryManagerSingleNode_unittest.cpp
index f65ec53..6ec6521 100644
--- a/query_execution/tests/QueryManagerSingleNode_unittest.cpp
+++ b/query_execution/tests/QueryManagerSingleNode_unittest.cpp
@@ -234,14 +234,14 @@ class QueryManagerTest : public ::testing::Test {
db_.reset(new CatalogDatabase(nullptr /* catalog */, "database"));
storage_manager_.reset(new StorageManager("./"));
bus_.Initialize();
- query_handle_.reset(new QueryHandle(0 /* dummy query ID */, tmb::kClientIdNone /* cli_id */));
+ query_handle_ = new QueryHandle(0 /* dummy query ID */, tmb::kClientIdNone /* cli_id */);
query_plan_ = query_handle_->getQueryPlanMutable();
query_handle_->getQueryContextProtoMutable()->set_query_id(query_handle_->query_id());
}
inline void constructQueryManager() {
query_manager_.reset(new QueryManagerSingleNode(
- 0, 1, query_handle_.get(), db_.get(), storage_manager_.get(), &bus_));
+ 0, 1, query_handle_, db_.get(), storage_manager_.get(), &bus_));
}
inline const int getNumWorkOrdersInExecution(const QueryPlan::DAGNodeIndex index) const {
@@ -291,8 +291,8 @@ class QueryManagerTest : public ::testing::Test {
unique_ptr<CatalogDatabase> db_;
unique_ptr<StorageManager> storage_manager_;
- QueryPlan *query_plan_;
- unique_ptr<QueryHandle> query_handle_;
+ QueryPlan *query_plan_; // Owned by 'query_handle_'.
+ QueryHandle* query_handle_; // Owned by 'query_manager_'.
unique_ptr<QueryManagerSingleNode> query_manager_;
MessageBusImpl bus_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
index 2351dcd..5100651 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
@@ -160,37 +160,39 @@ void DistributedExecutionGeneratorTestRunner::runTestCase(
const ParseStatement &parse_statement = *result.parsed_statement;
std::printf("%s\n", parse_statement.toString().c_str());
+
+ const CatalogRelation *query_result_relation = nullptr;
try {
OptimizerContext optimizer_context;
- QueryHandle query_handle(query_id_++, cli_id_);
+ auto query_handle = std::make_unique<QueryHandle>(query_id_++, cli_id_);
optimizer_.generateQueryHandle(parse_statement,
test_database_loader_->catalog_database(),
&optimizer_context,
- &query_handle);
+ query_handle.get());
+ query_result_relation = query_handle->getQueryResultRelation();
QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
cli_id_,
foreman_->getBusClientID(),
- &query_handle,
+ query_handle.release(),
&bus_);
-
- const tmb::AnnotatedMessage annotated_message = bus_.Receive(cli_id_, 0, true);
- DCHECK_EQ(kQueryExecutionSuccessMessage, annotated_message.tagged_message.message_type());
-
- const CatalogRelation *query_result_relation = query_handle.getQueryResultRelation();
- if (query_result_relation) {
- PrintToScreen::PrintRelation(*query_result_relation,
- test_database_loader_->storage_manager(),
- output_stream.file());
- DropRelation::Drop(*query_result_relation,
- test_database_loader_->catalog_database(),
- test_database_loader_->storage_manager());
- }
} catch (const SqlError &error) {
*output = error.formatMessage(input);
break;
}
+
+ const tmb::AnnotatedMessage annotated_message = bus_.Receive(cli_id_, 0, true);
+ DCHECK_EQ(kQueryExecutionSuccessMessage, annotated_message.tagged_message.message_type());
+
+ if (query_result_relation) {
+ PrintToScreen::PrintRelation(*query_result_relation,
+ test_database_loader_->storage_manager(),
+ output_stream.file());
+ DropRelation::Drop(*query_result_relation,
+ test_database_loader_->catalog_database(),
+ test_database_loader_->storage_manager());
+ }
}
if (output->empty()) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
index 06397d4..ee9bee7 100644
--- a/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
@@ -20,6 +20,7 @@
#include "query_optimizer/tests/ExecutionGeneratorTestRunner.hpp"
#include <cstdio>
+#include <memory>
#include <set>
#include <string>
@@ -71,37 +72,39 @@ void ExecutionGeneratorTestRunner::runTestCase(
} else {
const ParseStatement &parse_statement = *result.parsed_statement;
std::printf("%s\n", parse_statement.toString().c_str());
+
+ const CatalogRelation *query_result_relation = nullptr;
try {
- QueryHandle query_handle(0 /* query_id */, main_thread_client_id_);
OptimizerContext optimizer_context;
+ auto query_handle = std::make_unique<QueryHandle>(0 /* query_id */, main_thread_client_id_);
optimizer_.generateQueryHandle(parse_statement,
test_database_loader_.catalog_database(),
&optimizer_context,
- &query_handle);
+ query_handle.get());
+ query_result_relation = query_handle->getQueryResultRelation();
QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
main_thread_client_id_,
foreman_->getBusClientID(),
- &query_handle,
+ query_handle.release(),
&bus_);
-
- QueryExecutionUtil::ReceiveQueryCompletionMessage(
- main_thread_client_id_, &bus_);
-
- const CatalogRelation *query_result_relation = query_handle.getQueryResultRelation();
- if (query_result_relation) {
- PrintToScreen::PrintRelation(*query_result_relation,
- test_database_loader_.storage_manager(),
- output_stream.file());
- DropRelation::Drop(*query_result_relation,
- test_database_loader_.catalog_database(),
- test_database_loader_.storage_manager());
- }
} catch (const SqlError &error) {
*output = error.formatMessage(input);
break;
}
+
+ QueryExecutionUtil::ReceiveQueryCompletionMessage(
+ main_thread_client_id_, &bus_);
+
+ if (query_result_relation) {
+ PrintToScreen::PrintRelation(*query_result_relation,
+ test_database_loader_.storage_manager(),
+ output_stream.file());
+ DropRelation::Drop(*query_result_relation,
+ test_database_loader_.catalog_database(),
+ test_database_loader_.storage_manager());
+ }
}
}
[37/39] incubator-quickstep git commit: Added DistributedCli
executable.
Posted by ji...@apache.org.
Added DistributedCli executable.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b949c504
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b949c504
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b949c504
Branch: refs/heads/min-max-stats
Commit: b949c50450625f16c40a581db3ed811be2b9ccf2
Parents: bc4086b
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 27 22:32:24 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Mon Dec 5 00:15:16 2016 -0800
----------------------------------------------------------------------
CMakeLists.txt | 17 ++
cli/CMakeLists.txt | 3 +
cli/distributed/CMakeLists.txt | 93 +++++++++
cli/distributed/Cli.cpp | 239 ++++++++++++++++++++++
cli/distributed/Cli.hpp | 71 +++++++
cli/distributed/CliDistributedModule.hpp | 23 +++
cli/distributed/Conductor.cpp | 180 ++++++++++++++++
cli/distributed/Conductor.hpp | 80 ++++++++
cli/distributed/Executor.cpp | 87 ++++++++
cli/distributed/Executor.hpp | 83 ++++++++
cli/distributed/QuickstepDistributedCli.cpp | 81 ++++++++
cli/distributed/Role.cpp | 51 +++++
cli/distributed/Role.hpp | 69 +++++++
query_execution/QueryExecutionMessages.proto | 8 +
query_execution/QueryExecutionTypedefs.hpp | 6 +-
validate_cmakelists.py | 4 +-
16 files changed, 1093 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 126b47b..391cb26 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -782,3 +782,20 @@ endif()
# Link against other required system and third-party libraries.
target_link_libraries(quickstep_cli_shell ${LIBS})
+
+if (ENABLE_DISTRIBUTED)
+ # Build the quickstep_distributed_cli_shell executable.
+ add_executable (quickstep_distributed_cli_shell cli/distributed/QuickstepDistributedCli.cpp)
+ # Link against direct deps (will transitively pull in everything needed).
+ # NOTE(zuyu): Link quickstep_cli_LineReader on behalf of quickstep_cli_distributed_Cli,
+ # as a workaround for bypassing conditionally built target checks in validate_cmakelists.py.
+ target_link_libraries(quickstep_distributed_cli_shell
+ glog
+ quickstep_cli_LineReader
+ quickstep_cli_distributed_Cli
+ quickstep_cli_distributed_Conductor
+ quickstep_cli_distributed_Executor
+ quickstep_utility_StringUtil
+ ${GFLAGS_LIB_NAME}
+ ${GRPCPLUSPLUS_LIBRARIES})
+endif(ENABLE_DISTRIBUTED)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 9b62af9..be13c82 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -16,6 +16,9 @@
# under the License.
include_directories(${CMAKE_CURRENT_BINARY_DIR})
+if (ENABLE_DISTRIBUTED)
+ add_subdirectory(distributed)
+endif(ENABLE_DISTRIBUTED)
add_subdirectory(tests)
if (WIN32)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/distributed/CMakeLists.txt b/cli/distributed/CMakeLists.txt
new file mode 100644
index 0000000..e16d8af
--- /dev/null
+++ b/cli/distributed/CMakeLists.txt
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if (BUILD_SHARED_LIBS)
+ set(GFLAGS_LIB_NAME gflags_nothreads-shared)
+else()
+ set(GFLAGS_LIB_NAME gflags_nothreads-static)
+endif()
+
+# Declare micro-libs and link dependencies:
+add_library(quickstep_cli_distributed_Cli Cli.cpp Cli.hpp)
+add_library(quickstep_cli_distributed_Conductor Conductor.cpp Conductor.hpp)
+add_library(quickstep_cli_distributed_Executor Executor.cpp Executor.hpp)
+add_library(quickstep_cli_distributed_Role Role.cpp Role.hpp)
+
+# Link dependencies:
+target_link_libraries(quickstep_cli_distributed_Cli
+ glog
+ quickstep_catalog_CatalogRelation
+ quickstep_cli_Flags
+ quickstep_cli_PrintToScreen
+ quickstep_cli_distributed_Role
+ quickstep_parser_ParseStatement
+ quickstep_parser_SqlParserWrapper
+ quickstep_queryexecution_BlockLocatorUtil
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_DataExchangerAsync
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageManager
+ quickstep_utility_Macros
+ quickstep_utility_StringUtil
+ tmb)
+target_link_libraries(quickstep_cli_distributed_Conductor
+ glog
+ quickstep_cli_DefaultsConfigurator
+ quickstep_cli_Flags
+ quickstep_cli_distributed_Role
+ quickstep_parser_ParseStatement
+ quickstep_parser_SqlParserWrapper
+ quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_ForemanDistributed
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_queryoptimizer_QueryHandle
+ quickstep_queryoptimizer_QueryProcessor
+ quickstep_storage_StorageConstants
+ quickstep_utility_Macros
+ quickstep_utility_SqlError
+ tmb)
+target_link_libraries(quickstep_cli_distributed_Executor
+ glog
+ quickstep_catalog_CatalogTypedefs
+ quickstep_cli_Flags
+ quickstep_cli_distributed_Role
+ quickstep_queryexecution_BlockLocatorUtil
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_Shiftboss
+ quickstep_queryexecution_Worker
+ quickstep_queryexecution_WorkerDirectory
+ quickstep_storage_DataExchangerAsync
+ quickstep_storage_StorageManager
+ quickstep_utility_Macros
+ tmb)
+target_link_libraries(quickstep_cli_distributed_Role
+ quickstep_utility_Macros
+ tmb
+ ${GFLAGS_LIB_NAME})
+
+# Module all-in-one library:
+add_library(quickstep_cli_distributed ../../empty_src.cpp CliDistributedModule.hpp)
+
+target_link_libraries(quickstep_cli_distributed
+ quickstep_cli_distributed_Cli
+ quickstep_cli_distributed_Conductor
+ quickstep_cli_distributed_Executor
+ quickstep_cli_distributed_Role)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Cli.cpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Cli.cpp b/cli/distributed/Cli.cpp
new file mode 100644
index 0000000..01f824d
--- /dev/null
+++ b/cli/distributed/Cli.cpp
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "cli/distributed/Cli.hpp"
+
+#include <chrono>
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogRelation.hpp"
+#include "cli/CliConfig.h" // For QUICKSTEP_USE_LINENOISE.
+#include "cli/Flags.hpp"
+
+#ifdef QUICKSTEP_USE_LINENOISE
+#include "cli/LineReaderLineNoise.hpp"
+typedef quickstep::LineReaderLineNoise LineReaderImpl;
+#else
+#include "cli/LineReaderDumb.hpp"
+typedef quickstep::LineReaderDumb LineReaderImpl;
+#endif
+
+#include "cli/PrintToScreen.hpp"
+#include "parser/ParseStatement.hpp"
+#include "parser/SqlParserWrapper.hpp"
+#include "query_execution/BlockLocatorUtil.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "utility/StringUtil.hpp"
+#include "storage/StorageBlockInfo.hpp"
+
+#include "tmb/address.h"
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/message_style.h"
+#include "tmb/native_net_client_message_bus.h"
+#include "tmb/tagged_message.h"
+
+#include "glog/logging.h"
+
+using std::fprintf;
+using std::free;
+using std::make_unique;
+using std::malloc;
+using std::move;
+using std::printf;
+using std::size_t;
+using std::string;
+using std::vector;
+
+using tmb::AnnotatedMessage;
+using tmb::TaggedMessage;
+using tmb::client_id;
+
+namespace quickstep {
+
+namespace S = ::quickstep::serialization;
+
+void Cli::init() {
+ cli_id_ = bus_.Connect();
+ DLOG(INFO) << "DistributedCli TMB Client ID: " << cli_id_;
+
+ bus_.RegisterClientAsSender(cli_id_, kDistributedCliRegistrationMessage);
+ bus_.RegisterClientAsReceiver(cli_id_, kDistributedCliRegistrationResponseMessage);
+
+ DLOG(INFO) << "DistributedCli sent DistributedCliRegistrationMessage (typed '"
+ << kDistributedCliRegistrationMessage
+ << "') to all";
+
+ tmb::Address all_addresses;
+ all_addresses.All(true);
+ // NOTE(zuyu): The singleton Conductor would need one copy of the message.
+ tmb::MessageStyle style;
+
+ TaggedMessage cli_reg_message(kDistributedCliRegistrationMessage);
+ DCHECK(tmb::MessageBus::SendStatus::kOK ==
+ bus_.Send(cli_id_, all_addresses, style, move(cli_reg_message)));
+
+ // Wait for Conductor to response.
+ const AnnotatedMessage cli_reg_response_message(bus_.Receive(cli_id_, 0, true));
+ DCHECK_EQ(kDistributedCliRegistrationResponseMessage,
+ cli_reg_response_message.tagged_message.message_type());
+ conductor_client_id_ = cli_reg_response_message.sender;
+
+ DLOG(INFO) << "DistributedCli received typed '" << kDistributedCliRegistrationResponseMessage
+ << "' message from Conductor (id'" << conductor_client_id_ << "').";
+
+ // Setup StorageManager.
+ bus_.RegisterClientAsSender(cli_id_, kBlockDomainRegistrationMessage);
+ bus_.RegisterClientAsReceiver(cli_id_, kBlockDomainRegistrationResponseMessage);
+
+ client_id locator_client_id;
+ storage_manager_ = make_unique<StorageManager>(
+ FLAGS_storage_path,
+ block_locator::getBlockDomain(data_exchanger_.network_address(), cli_id_, &locator_client_id, &bus_),
+ locator_client_id, &bus_);
+
+ data_exchanger_.set_storage_manager(storage_manager_.get());
+ data_exchanger_.start();
+
+ // Prepare for submitting a query.
+ bus_.RegisterClientAsSender(cli_id_, kSqlQueryMessage);
+ bus_.RegisterClientAsReceiver(cli_id_, kQueryExecutionSuccessMessage);
+ bus_.RegisterClientAsReceiver(cli_id_, kQueryExecutionErrorMessage);
+}
+
+void Cli::run() {
+ LineReaderImpl line_reader("distributed_quickstep> ",
+ " ...> ");
+ auto parser_wrapper = make_unique<SqlParserWrapper>();
+ std::chrono::time_point<std::chrono::steady_clock> start, end;
+
+ for (;;) {
+ string *command_string = new string();
+ *command_string = line_reader.getNextCommand();
+ if (command_string->size() == 0) {
+ delete command_string;
+ break;
+ }
+
+ parser_wrapper->feedNextBuffer(command_string);
+
+ bool quitting = false;
+ // A parse error should reset the parser. This is because the thrown quickstep
+ // SqlError does not do the proper reset work of the YYABORT macro.
+ bool reset_parser = false;
+ for (;;) {
+ ParseResult result = parser_wrapper->getNextStatement();
+ const ParseStatement &statement = *result.parsed_statement;
+ if (result.condition == ParseResult::kSuccess) {
+ if (statement.getStatementType() == ParseStatement::kQuit) {
+ quitting = true;
+ break;
+ }
+
+ CHECK_NE(statement.getStatementType(), ParseStatement::kCommand)
+ << "TODO(quickstep-team)";
+
+ DLOG(INFO) << "DistributedCli sent SqlQueryMessage (typed '" << kSqlQueryMessage
+ << "') to Conductor";
+ S::SqlQueryMessage proto;
+ proto.set_sql_query(*command_string);
+
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage sql_query_message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kSqlQueryMessage);
+ free(proto_bytes);
+
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ cli_id_,
+ conductor_client_id_,
+ move(sql_query_message));
+
+ start = std::chrono::steady_clock::now();
+
+ const AnnotatedMessage annotated_message(bus_.Receive(cli_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ DLOG(INFO) << "DistributedCli received typed '" << tagged_message.message_type()
+ << "' message from client " << annotated_message.sender;
+ switch (tagged_message.message_type()) {
+ case kQueryExecutionSuccessMessage: {
+ end = std::chrono::steady_clock::now();
+
+ S::QueryExecutionSuccessMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ if (proto.has_result_relation()) {
+ CatalogRelation result_relation(proto.result_relation());
+
+ PrintToScreen::PrintRelation(result_relation, storage_manager_.get(), stdout);
+
+ const vector<block_id> blocks(result_relation.getBlocksSnapshot());
+ for (const block_id block : blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
+ }
+
+ std::chrono::duration<double, std::milli> time_in_ms = end - start;
+ printf("Time: %s ms\n", DoubleToStringWithSignificantDigits(time_in_ms.count(), 3).c_str());
+ break;
+ }
+ case kQueryExecutionErrorMessage: {
+ S::QueryExecutionErrorMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ fprintf(stderr, "%s", proto.error_message().c_str());
+ break;
+ }
+ default: {
+ LOG(ERROR) << "Unknown TMB message type";
+ }
+ }
+ } else {
+ if (result.condition == ParseResult::kError) {
+ fprintf(stderr, "%s", result.error_message.c_str());
+ }
+ reset_parser = true;
+ break;
+ }
+ }
+
+ if (quitting) {
+ break;
+ } else if (reset_parser) {
+ parser_wrapper = make_unique<SqlParserWrapper>();
+ reset_parser = false;
+ }
+ }
+
+ bus_.Disconnect(cli_id_);
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Cli.hpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Cli.hpp b/cli/distributed/Cli.hpp
new file mode 100644
index 0000000..32c178f
--- /dev/null
+++ b/cli/distributed/Cli.hpp
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_CLI_DISTRIBUTED_CLI_HPP_
+#define QUICKSTEP_CLI_DISTRIBUTED_CLI_HPP_
+
+#include <memory>
+
+#include "cli/distributed/Role.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageManager.hpp"
+#include "utility/Macros.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace quickstep {
+
+/** \addtogroup CliDistributed
+ * @{
+ */
+
+/**
+ * @brief A class for the Cli component in the distributed version.
+ **/
+class Cli final : public Role {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ Cli() = default;
+
+ ~Cli() override {
+ data_exchanger_.shutdown();
+ storage_manager_.reset();
+ data_exchanger_.join();
+ }
+
+ void init() override;
+
+ void run() override;
+
+ private:
+ tmb::client_id cli_id_, conductor_client_id_;
+
+ DataExchangerAsync data_exchanger_;
+ std::unique_ptr<StorageManager> storage_manager_;
+
+ DISALLOW_COPY_AND_ASSIGN(Cli);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_CLI_DISTRIBUTED_CLI_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/CliDistributedModule.hpp
----------------------------------------------------------------------
diff --git a/cli/distributed/CliDistributedModule.hpp b/cli/distributed/CliDistributedModule.hpp
new file mode 100644
index 0000000..cfa1e1b
--- /dev/null
+++ b/cli/distributed/CliDistributedModule.hpp
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+/** @defgroup CliDistributed
+ *
+ * The distributed QuickStep command-line interface.
+ **/
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Conductor.cpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Conductor.cpp b/cli/distributed/Conductor.cpp
new file mode 100644
index 0000000..c4a2721
--- /dev/null
+++ b/cli/distributed/Conductor.cpp
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "cli/distributed/Conductor.hpp"
+
+#include <cstddef>
+#include <cstdlib>
+#include <exception>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <utility>
+
+#include "cli/DefaultsConfigurator.hpp"
+#include "cli/Flags.hpp"
+#include "parser/ParseStatement.hpp"
+#include "parser/SqlParserWrapper.hpp"
+#include "query_execution/BlockLocator.hpp"
+#include "query_execution/ForemanDistributed.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "query_optimizer/QueryHandle.hpp"
+#include "query_optimizer/QueryProcessor.hpp"
+#include "storage/StorageConstants.hpp"
+#include "utility/SqlError.hpp"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/native_net_client_message_bus.h"
+#include "tmb/tagged_message.h"
+
+#include "glog/logging.h"
+
+using std::free;
+using std::make_unique;
+using std::malloc;
+using std::move;
+using std::size_t;
+using std::string;
+
+using tmb::AnnotatedMessage;
+using tmb::MessageBus;
+using tmb::TaggedMessage;
+using tmb::client_id;
+
+namespace quickstep {
+
+namespace S = ::quickstep::serialization;
+
+void Conductor::init() {
+ try {
+ string catalog_path = FLAGS_storage_path + kCatalogFilename;
+
+ if (quickstep::FLAGS_initialize_db) { // Initialize the database
+ DefaultsConfigurator::InitializeDefaultDatabase(FLAGS_storage_path, catalog_path);
+ }
+
+ query_processor_ = make_unique<QueryProcessor>(move(catalog_path));
+ } catch (const std::exception &e) {
+ LOG(FATAL) << "FATAL ERROR DURING STARTUP: " << e.what()
+ << "\nIf you intended to create a new database, "
+ << "please use the \"-initialize_db=true\" command line option.";
+ } catch (...) {
+ LOG(FATAL) << "NON-STANDARD EXCEPTION DURING STARTUP";
+ }
+
+ bus_.ResetBus();
+
+ conductor_client_id_ = bus_.Connect();
+ DLOG(INFO) << "Conductor TMB Client ID: " << conductor_client_id_;
+
+ bus_.RegisterClientAsReceiver(conductor_client_id_, kDistributedCliRegistrationMessage);
+ bus_.RegisterClientAsSender(conductor_client_id_, kDistributedCliRegistrationResponseMessage);
+
+ bus_.RegisterClientAsReceiver(conductor_client_id_, kSqlQueryMessage);
+ bus_.RegisterClientAsSender(conductor_client_id_, kQueryExecutionErrorMessage);
+ bus_.RegisterClientAsSender(conductor_client_id_, kAdmitRequestMessage);
+
+ block_locator_ = make_unique<BlockLocator>(&bus_);
+ block_locator_->start();
+
+ foreman_ = make_unique<ForemanDistributed>(*block_locator_, &bus_, query_processor_->getDefaultDatabase());
+ foreman_->start();
+}
+
+void Conductor::run() {
+ for (;;) {
+ AnnotatedMessage annotated_message(bus_.Receive(conductor_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ const client_id sender = annotated_message.sender;
+
+ DLOG(INFO) << "Conductor received typed '" << tagged_message.message_type()
+ << "' message from client " << sender;
+ switch (tagged_message.message_type()) {
+ case kDistributedCliRegistrationMessage: {
+ TaggedMessage message(kDistributedCliRegistrationResponseMessage);
+
+ DLOG(INFO) << "Conductor sent DistributedCliRegistrationResponseMessage (typed '"
+ << kDistributedCliRegistrationResponseMessage
+ << "') to Distributed CLI " << sender;
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_, conductor_client_id_, sender, move(message)));
+ break;
+ }
+ case kSqlQueryMessage: {
+ S::SqlQueryMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+ DLOG(INFO) << "Conductor received the following SQL query: " << proto.sql_query();
+
+ processSqlQueryMessage(sender, new string(move(proto.sql_query())));
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown TMB message type";
+ }
+ }
+}
+
+void Conductor::processSqlQueryMessage(const tmb::client_id sender, string *command_string) {
+ parser_wrapper_.feedNextBuffer(command_string);
+ ParseResult parse_result = parser_wrapper_.getNextStatement();
+
+ CHECK(parse_result.condition == ParseResult::kSuccess)
+ << "Any SQL syntax error should be addressed in the DistributedCli.";
+
+ const ParseStatement &statement = *parse_result.parsed_statement;
+ CHECK(statement.getStatementType() != ParseStatement::kCommand)
+ << "TODO(quickstep-team)";
+
+ try {
+ auto query_handle = make_unique<QueryHandle>(query_processor_->query_id(),
+ sender,
+ statement.getPriority());
+ query_processor_->generateQueryHandle(statement, query_handle.get());
+ DCHECK(query_handle->getQueryPlanMutable() != nullptr);
+
+ QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
+ conductor_client_id_,
+ foreman_->getBusClientID(),
+ query_handle.release(),
+ &bus_);
+ } catch (const SqlError &sql_error) {
+ // Set the query execution status along with the error message.
+ S::QueryExecutionErrorMessage proto;
+ proto.set_error_message(sql_error.formatMessage(*command_string));
+
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kQueryExecutionErrorMessage);
+ free(proto_bytes);
+
+ DLOG(INFO) << "Conductor (on behalf of Optimizer) sent QueryExecutionErrorMessage (typed '"
+ << kQueryExecutionErrorMessage
+ << "') to Distributed CLI " << sender;
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_, conductor_client_id_, sender, move(message)));
+ }
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Conductor.hpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Conductor.hpp b/cli/distributed/Conductor.hpp
new file mode 100644
index 0000000..e8c9582
--- /dev/null
+++ b/cli/distributed/Conductor.hpp
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_CLI_DISTRIBUTED_CONDUCTOR_HPP_
+#define QUICKSTEP_CLI_DISTRIBUTED_CONDUCTOR_HPP_
+
+#include <memory>
+#include <string>
+
+#include "cli/distributed/Role.hpp"
+#include "parser/SqlParserWrapper.hpp"
+#include "query_execution/BlockLocator.hpp"
+#include "query_execution/ForemanDistributed.hpp"
+#include "query_optimizer/QueryProcessor.hpp"
+#include "utility/Macros.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace quickstep {
+
+/** \addtogroup CliDistributed
+ * @{
+ */
+
+/**
+ * @brief A class for the Conductor component in the distributed version.
+ **/
+class Conductor final : public Role {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ Conductor() = default;
+
+ ~Conductor() override {
+ foreman_->join();
+ block_locator_->join();
+ }
+
+ void init() override;
+
+ void run() override;
+
+ private:
+ void processSqlQueryMessage(const tmb::client_id sender, std::string *command_string);
+
+ SqlParserWrapper parser_wrapper_;
+
+ std::unique_ptr<QueryProcessor> query_processor_;
+
+ tmb::client_id conductor_client_id_;
+
+ std::unique_ptr<BlockLocator> block_locator_;
+
+ std::unique_ptr<ForemanDistributed> foreman_;
+
+ DISALLOW_COPY_AND_ASSIGN(Conductor);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_CLI_DISTRIBUTED_CONDUCTOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Executor.cpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Executor.cpp b/cli/distributed/Executor.cpp
new file mode 100644
index 0000000..1d03579
--- /dev/null
+++ b/cli/distributed/Executor.cpp
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "cli/distributed/Executor.hpp"
+
+#include <cstddef>
+#include <memory>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "cli/Flags.hpp"
+#include "query_execution/BlockLocatorUtil.hpp"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/Shiftboss.hpp"
+#include "query_execution/Worker.hpp"
+#include "query_execution/WorkerDirectory.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageManager.hpp"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/native_net_client_message_bus.h"
+
+#include "glog/logging.h"
+
+using std::make_unique;
+using std::size_t;
+using std::vector;
+
+using tmb::client_id;
+
+namespace quickstep {
+
+void Executor::init() {
+ executor_client_id_ = bus_.Connect();
+ DLOG(INFO) << "Executor TMB Client ID: " << executor_client_id_;
+
+ bus_.RegisterClientAsSender(executor_client_id_, kBlockDomainRegistrationMessage);
+ bus_.RegisterClientAsReceiver(executor_client_id_, kBlockDomainRegistrationResponseMessage);
+
+ vector<client_id> worker_client_ids;
+ vector<numa_node_id> worker_numa_nodes(FLAGS_num_workers, kAnyNUMANodeID);
+
+ for (std::size_t worker_thread_index = 0;
+ worker_thread_index < FLAGS_num_workers;
+ ++worker_thread_index) {
+ workers_.push_back(make_unique<Worker>(worker_thread_index, &bus_));
+ worker_client_ids.push_back(workers_.back()->getBusClientID());
+ }
+
+ worker_directory_ =
+ make_unique<WorkerDirectory>(worker_client_ids.size(), worker_client_ids, worker_numa_nodes);
+
+ client_id locator_client_id;
+ storage_manager_ = make_unique<StorageManager>(
+ FLAGS_storage_path,
+ block_locator::getBlockDomain(data_exchanger_.network_address(), executor_client_id_, &locator_client_id, &bus_),
+ locator_client_id, &bus_);
+
+ data_exchanger_.set_storage_manager(storage_manager_.get());
+ data_exchanger_.start();
+
+ shiftboss_ =
+ make_unique<Shiftboss>(&bus_, storage_manager_.get(), worker_directory_.get());
+ shiftboss_->start();
+
+ for (const auto &worker : workers_) {
+ worker->start();
+ }
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Executor.hpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Executor.hpp b/cli/distributed/Executor.hpp
new file mode 100644
index 0000000..6ffa756
--- /dev/null
+++ b/cli/distributed/Executor.hpp
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_CLI_DISTRIBUTED_EXECUTOR_HPP_
+#define QUICKSTEP_CLI_DISTRIBUTED_EXECUTOR_HPP_
+
+#include <memory>
+#include <vector>
+
+#include "cli/distributed/Role.hpp"
+#include "query_execution/Shiftboss.hpp"
+#include "query_execution/Worker.hpp"
+#include "query_execution/WorkerDirectory.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageManager.hpp"
+#include "utility/Macros.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace quickstep {
+
+/** \addtogroup CliDistributed
+ * @{
+ */
+
+/**
+ * @brief A class for the Executor component in the distributed version.
+ **/
+class Executor final : public Role {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ Executor() = default;
+
+ ~Executor() override {
+ for (const auto &worker : workers_) {
+ worker->join();
+ }
+ shiftboss_->join();
+
+ data_exchanger_.shutdown();
+ storage_manager_.reset();
+ data_exchanger_.join();
+ }
+
+ void init() override;
+
+ void run() override {}
+
+ private:
+ tmb::client_id executor_client_id_;
+
+ std::vector<std::unique_ptr<Worker>> workers_;
+ std::unique_ptr<WorkerDirectory> worker_directory_;
+ DataExchangerAsync data_exchanger_;
+ std::unique_ptr<StorageManager> storage_manager_;
+ std::unique_ptr<Shiftboss> shiftboss_;
+
+ DISALLOW_COPY_AND_ASSIGN(Executor);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_CLI_DISTRIBUTED_EXECUTOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/QuickstepDistributedCli.cpp
----------------------------------------------------------------------
diff --git a/cli/distributed/QuickstepDistributedCli.cpp b/cli/distributed/QuickstepDistributedCli.cpp
new file mode 100644
index 0000000..f01cd13
--- /dev/null
+++ b/cli/distributed/QuickstepDistributedCli.cpp
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+/* A standalone command-line interface to Distributed QuickStep */
+
+#include <memory>
+#include <string>
+
+#include "cli/distributed/Cli.hpp"
+#include "cli/distributed/Conductor.hpp"
+#include "cli/distributed/Executor.hpp"
+#include "cli/distributed/Role.hpp"
+#include "utility/StringUtil.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+#include "grpc/grpc.h"
+
+using std::make_unique;
+
+namespace quickstep {
+
+constexpr char kCliRole[] = "cli";
+constexpr char kConductorRole[] = "conductor";
+constexpr char kExecutorRole[] = "executor";
+
+DEFINE_string(role, "",
+ "The role in the distributed Quickstep: Conductor, Executor, or Cli.");
+static bool ValidateRole(const char *flagname,
+ const std::string &value) {
+ if (value.empty()) {
+ return false;
+ }
+
+ FLAGS_role = ToLower(value);
+ return FLAGS_role == kCliRole ||
+ FLAGS_role == kConductorRole ||
+ FLAGS_role == kExecutorRole;
+}
+static const volatile bool role_dummy
+ = gflags::RegisterFlagValidator(&FLAGS_role, &ValidateRole);
+
+} // namespace quickstep
+
+using quickstep::FLAGS_role;
+
+int main(int argc, char *argv[]) {
+ google::InitGoogleLogging(argv[0]);
+ gflags::ParseCommandLineFlags(&argc, &argv, true);
+ grpc_init();
+
+ std::unique_ptr<quickstep::Role> role;
+ if (FLAGS_role == quickstep::kCliRole) {
+ role = make_unique<quickstep::Cli>();
+ } else if (FLAGS_role == quickstep::kConductorRole) {
+ role = make_unique<quickstep::Conductor>();
+ } else if (FLAGS_role == quickstep::kExecutorRole) {
+ role = make_unique<quickstep::Executor>();
+ }
+
+ role->init();
+ role->run();
+
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Role.cpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Role.cpp b/cli/distributed/Role.cpp
new file mode 100644
index 0000000..d56ef09
--- /dev/null
+++ b/cli/distributed/Role.cpp
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "cli/distributed/Role.hpp"
+
+#include <cstdio>
+#include <cstdint>
+
+#include "gflags/gflags.h"
+
+#include "tmb/native_net_client_message_bus.h"
+
+namespace quickstep {
+
+DEFINE_string(tmb_server_ip, "127.0.0.1", "IP Address of the TMB Server.");
+
+static bool ValidateTmbServerPort(const char *flagname,
+ std::int32_t value) {
+ if (value > 0 && value < 65536) {
+ return true;
+ } else {
+ std::fprintf(stderr, "--%s must be between 1 and 65535 (inclusive)\n", flagname);
+ return false;
+ }
+}
+DEFINE_int32(tmb_server_port, 4575, "Port of the TMB Server.");
+static const bool tmb_server_port_dummy
+ = gflags::RegisterFlagValidator(&FLAGS_tmb_server_port, &ValidateTmbServerPort);
+
+Role::Role() {
+ bus_.AddServer(FLAGS_tmb_server_ip, FLAGS_tmb_server_port);
+ bus_.Initialize();
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Role.hpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Role.hpp b/cli/distributed/Role.hpp
new file mode 100644
index 0000000..b802543
--- /dev/null
+++ b/cli/distributed/Role.hpp
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_CLI_DISTRIBUTED_ROLE_HPP_
+#define QUICKSTEP_CLI_DISTRIBUTED_ROLE_HPP_
+
+#include "utility/Macros.hpp"
+
+#include "tmb/native_net_client_message_bus.h"
+
+namespace quickstep {
+
+/** \addtogroup CliDistributed
+ * @{
+ */
+
+/**
+ * @brief A base class for all components in the distributed version.
+ **/
+class Role {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ Role();
+
+ /**
+ * @brief Virtual destructor.
+ **/
+ virtual ~Role() {}
+
+ /**
+ * @brief Initialize the component.
+ **/
+ virtual void init() = 0;
+
+ /**
+ * @brief Start the component.
+ **/
+ virtual void run() = 0;
+
+ protected:
+ tmb::NativeNetClientMessageBus bus_;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(Role);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_CLI_DISTRIBUTED_ROLE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 93e458c..28b5ebd 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -78,6 +78,10 @@ message ShiftbossRegistrationResponseMessage {
required uint64 shiftboss_index = 1;
}
+message SqlQueryMessage {
+ required string sql_query = 1;
+}
+
message QueryInitiateMessage {
required uint64 query_id = 1;
required CatalogDatabase catalog_database_cache = 2;
@@ -131,6 +135,10 @@ message QueryExecutionSuccessMessage {
optional CatalogRelationSchema result_relation = 1;
}
+message QueryExecutionErrorMessage {
+ required string error_message = 1;
+}
+
// BlockLocator related messages.
message BlockDomainRegistrationMessage {
// Format IP:Port, i.e., "0.0.0.0:0".
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionTypedefs.hpp b/query_execution/QueryExecutionTypedefs.hpp
index 919e45b..faf2132 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -81,6 +81,9 @@ enum QueryExecutionMessageType : message_type_id {
kShiftbossRegistrationMessage, // From Shiftboss to Foreman.
kShiftbossRegistrationResponseMessage, // From Foreman to Shiftboss, or from
// Shiftboss to Worker.
+ kDistributedCliRegistrationMessage, // From CLI to Conductor.
+ kDistributedCliRegistrationResponseMessage, // From Conductor to CLI.
+ kSqlQueryMessage, // From CLI to Conductor.
kQueryInitiateMessage, // From Foreman to Shiftboss.
kQueryInitiateResponseMessage, // From Shiftboss to Foreman.
@@ -92,8 +95,9 @@ enum QueryExecutionMessageType : message_type_id {
kSaveQueryResultMessage, // From Foreman to Shiftboss.
kSaveQueryResultResponseMessage, // From Shiftboss to Foreman.
- // From Foreman to CLI.
+ // From Foreman / Conductor to CLI.
kQueryExecutionSuccessMessage,
+ kQueryExecutionErrorMessage,
// BlockLocator related messages, sorted in a life cycle of StorageManager
// with a unique block domain.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/validate_cmakelists.py
----------------------------------------------------------------------
diff --git a/validate_cmakelists.py b/validate_cmakelists.py
index f691d1f..9d1f530 100755
--- a/validate_cmakelists.py
+++ b/validate_cmakelists.py
@@ -46,7 +46,9 @@ EXCLUDED_TOP_LEVEL_DIRS = ["build", "third_party"]
# Explicitly ignored dependencies (special headers with no other quickstep
# dependencies).
IGNORED_DEPENDENCIES = frozenset(
- ["quickstep_storage_DataExchange.grpc_proto",
+ ["quickstep_cli_LineReaderDumb",
+ "quickstep_cli_LineReaderLineNoise",
+ "quickstep_storage_DataExchange.grpc_proto",
"quickstep_threading_WinThreadsAPI",
"quickstep_utility_textbasedtest_TextBasedTest",
"quickstep_utility_textbasedtest_TextBasedTestDriver",
[02/39] incubator-quickstep git commit: Use BlockLocator and
DataExchangerAsync in the distributed tests.
Posted by ji...@apache.org.
Use BlockLocator and DataExchangerAsync in the distributed tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/787a3251
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/787a3251
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/787a3251
Branch: refs/heads/min-max-stats
Commit: 787a3251019162610ebe13efbd341b3f9ac7a268
Parents: 3093e74
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Nov 4 23:12:09 2016 -0700
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Mon Nov 14 20:38:04 2016 -0800
----------------------------------------------------------------------
query_optimizer/tests/CMakeLists.txt | 8 ++
.../DistributedExecutionGeneratorTestRunner.cpp | 104 ++++++++++++++++---
.../DistributedExecutionGeneratorTestRunner.hpp | 32 +++++-
query_optimizer/tests/TestDatabaseLoader.hpp | 59 +++++++++--
storage/DataExchangerAsync.cpp | 4 +-
5 files changed, 178 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/787a3251/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/CMakeLists.txt b/query_optimizer/tests/CMakeLists.txt
index ac4548a..9c764e4 100644
--- a/query_optimizer/tests/CMakeLists.txt
+++ b/query_optimizer/tests/CMakeLists.txt
@@ -79,6 +79,10 @@ target_link_libraries(quickstep_queryoptimizer_tests_TestDatabaseLoader
quickstep_types_containers_Tuple
quickstep_utility_Macros
tmb)
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryoptimizer_tests_TestDatabaseLoader
+ quickstep_storage_StorageBlockInfo)
+endif(ENABLE_DISTRIBUTED)
if (ENABLE_DISTRIBUTED)
add_executable(quickstep_queryoptimizer_tests_DistributedExecutionGeneratorTest
@@ -110,7 +114,9 @@ if (ENABLE_DISTRIBUTED)
quickstep_cli_PrintToScreen
quickstep_parser_ParseStatement
quickstep_parser_SqlParserWrapper
+ quickstep_queryexecution_BlockLocator
quickstep_queryexecution_ForemanDistributed
+ quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
quickstep_queryexecution_Shiftboss
@@ -120,6 +126,8 @@ if (ENABLE_DISTRIBUTED)
quickstep_queryoptimizer_OptimizerContext
quickstep_queryoptimizer_QueryHandle
quickstep_queryoptimizer_tests_TestDatabaseLoader
+ quickstep_storage_DataExchangerAsync
+ quickstep_storage_StorageBlockInfo
quickstep_utility_Macros
quickstep_utility_MemStream
quickstep_utility_SqlError
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/787a3251/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
index 5cccc21..0403e77 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
@@ -20,18 +20,26 @@
#include "query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp"
#include <cstdio>
+#include <cstdlib>
+#include <memory>
#include <set>
#include <string>
+#include <utility>
#include <vector>
#include "catalog/CatalogTypedefs.hpp"
#include "cli/DropRelation.hpp"
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
+#include "query_execution/BlockLocator.hpp"
#include "query_execution/ForemanDistributed.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
#include "query_optimizer/OptimizerContext.hpp"
#include "query_optimizer/QueryHandle.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageBlockInfo.hpp"
#include "utility/MemStream.hpp"
#include "utility/SqlError.hpp"
@@ -41,10 +49,15 @@
#include "tmb/message_bus.h"
#include "tmb/tagged_message.h"
-using std::string;
+using std::free;
using std::make_unique;
+using std::malloc;
+using std::move;
+using std::string;
using std::vector;
+using tmb::TaggedMessage;
+
namespace quickstep {
class CatalogRelation;
@@ -56,10 +69,7 @@ const char *DistributedExecutionGeneratorTestRunner::kResetOption =
DistributedExecutionGeneratorTestRunner::DistributedExecutionGeneratorTestRunner(const string &storage_path)
: query_id_(0),
- test_database_loader_(storage_path) {
- test_database_loader_.createTestRelation(false /* allow_vchar */);
- test_database_loader_.loadTestRelation();
-
+ data_exchangers_(kNumInstances) {
bus_.Initialize();
cli_id_ = bus_.Connect();
@@ -67,9 +77,27 @@ DistributedExecutionGeneratorTestRunner::DistributedExecutionGeneratorTestRunner
bus_.RegisterClientAsSender(cli_id_, kPoisonMessage);
bus_.RegisterClientAsReceiver(cli_id_, kQueryExecutionSuccessMessage);
+ bus_.RegisterClientAsSender(cli_id_, kBlockDomainRegistrationMessage);
+ bus_.RegisterClientAsReceiver(cli_id_, kBlockDomainRegistrationResponseMessage);
+
+ block_locator_ = make_unique<BlockLocator>(&bus_);
+ locator_client_id_ = block_locator_->getBusClientID();
+ block_locator_->start();
+
+ test_database_loader_ = make_unique<TestDatabaseLoader>(
+ storage_path,
+ getBlockDomain(test_database_loader_data_exchanger_.network_address()),
+ locator_client_id_,
+ &bus_);
+ test_database_loader_data_exchanger_.set_storage_manager(test_database_loader_->storage_manager());
+ test_database_loader_data_exchanger_.start();
+
+ test_database_loader_->createTestRelation(false /* allow_vchar */);
+ test_database_loader_->loadTestRelation();
+
// NOTE(zuyu): Foreman should initialize before Shiftboss so that the former
// could receive a registration message from the latter.
- foreman_ = make_unique<ForemanDistributed>(&bus_, test_database_loader_.catalog_database());
+ foreman_ = make_unique<ForemanDistributed>(&bus_, test_database_loader_->catalog_database());
// We don't use the NUMA aware version of worker code.
const vector<numa_node_id> numa_nodes(1 /* Number of worker threads per instance */,
@@ -78,17 +106,24 @@ DistributedExecutionGeneratorTestRunner::DistributedExecutionGeneratorTestRunner
for (int i = 0; i < kNumInstances; ++i) {
workers_.push_back(make_unique<Worker>(0 /* worker_thread_index */, &bus_));
- const vector<tmb::client_id> worker_client_ids(1, workers_[i]->getBusClientID());
+ const vector<tmb::client_id> worker_client_ids(1, workers_.back()->getBusClientID());
worker_directories_.push_back(
make_unique<WorkerDirectory>(worker_client_ids.size(), worker_client_ids, numa_nodes));
+ auto storage_manager = make_unique<StorageManager>(
+ storage_path, getBlockDomain(data_exchangers_[i].network_address()), locator_client_id_, &bus_);
+
+ data_exchangers_[i].set_storage_manager(storage_manager.get());
shiftbosses_.push_back(
- make_unique<Shiftboss>(&bus_, test_database_loader_.storage_manager(), worker_directories_[i].get()));
+ make_unique<Shiftboss>(&bus_, storage_manager.get(), worker_directories_.back().get()));
+
+ storage_managers_.push_back(move(storage_manager));
}
foreman_->start();
for (int i = 0; i < kNumInstances; ++i) {
+ data_exchangers_[i].start();
shiftbosses_[i]->start();
workers_[i]->start();
}
@@ -101,9 +136,9 @@ void DistributedExecutionGeneratorTestRunner::runTestCase(
VLOG(4) << "Test SQL(s): " << input;
if (options.find(kResetOption) != options.end()) {
- test_database_loader_.clear();
- test_database_loader_.createTestRelation(false /* allow_vchar */);
- test_database_loader_.loadTestRelation();
+ test_database_loader_->clear();
+ test_database_loader_->createTestRelation(false /* allow_vchar */);
+ test_database_loader_->loadTestRelation();
}
MemStream output_stream;
@@ -125,7 +160,7 @@ void DistributedExecutionGeneratorTestRunner::runTestCase(
QueryHandle query_handle(query_id_++, cli_id_);
optimizer_.generateQueryHandle(parse_statement,
- test_database_loader_.catalog_database(),
+ test_database_loader_->catalog_database(),
&optimizer_context,
&query_handle);
@@ -141,11 +176,11 @@ void DistributedExecutionGeneratorTestRunner::runTestCase(
const CatalogRelation *query_result_relation = query_handle.getQueryResultRelation();
if (query_result_relation) {
PrintToScreen::PrintRelation(*query_result_relation,
- test_database_loader_.storage_manager(),
+ test_database_loader_->storage_manager(),
output_stream.file());
DropRelation::Drop(*query_result_relation,
- test_database_loader_.catalog_database(),
- test_database_loader_.storage_manager());
+ test_database_loader_->catalog_database(),
+ test_database_loader_->storage_manager());
}
} catch (const SqlError &error) {
*output = error.formatMessage(input);
@@ -158,5 +193,44 @@ void DistributedExecutionGeneratorTestRunner::runTestCase(
}
}
+block_id_domain DistributedExecutionGeneratorTestRunner::getBlockDomain(
+ const string &network_address) {
+ serialization::BlockDomainRegistrationMessage proto;
+ proto.set_domain_network_address(network_address);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainRegistrationMessage);
+ free(proto_bytes);
+
+ DLOG(INFO) << "Client (id '" << cli_id_
+ << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ cli_id_,
+ locator_client_id_,
+ move(message)));
+
+ const tmb::AnnotatedMessage annotated_message(bus_.Receive(cli_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ CHECK_EQ(locator_client_id_, annotated_message.sender);
+ CHECK_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
+ DLOG(INFO) << "Client (id '" << cli_id_
+ << "') received BlockDomainRegistrationResponseMessage (typed '"
+ << kBlockDomainRegistrationResponseMessage
+ << "') from BlockLocator";
+
+ serialization::BlockDomainMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ return static_cast<block_id_domain>(response_proto.block_domain());
+}
+
} // namespace optimizer
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/787a3251/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
index ab10841..d2b13e4 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
@@ -28,6 +28,7 @@
#include <vector>
#include "parser/SqlParserWrapper.hpp"
+#include "query_execution/BlockLocator.hpp"
#include "query_execution/ForemanDistributed.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
@@ -36,6 +37,8 @@
#include "query_execution/WorkerDirectory.hpp"
#include "query_optimizer/Optimizer.hpp"
#include "query_optimizer/tests/TestDatabaseLoader.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
#include "utility/textbased_test/TextBasedTestRunner.hpp"
@@ -86,6 +89,25 @@ class DistributedExecutionGeneratorTestRunner : public TextBasedTestRunner {
}
foreman_->join();
+
+ test_database_loader_data_exchanger_.shutdown();
+ test_database_loader_.reset();
+ for (int i = 0; i < kNumInstances; ++i) {
+ data_exchangers_[i].shutdown();
+ storage_managers_[i].reset();
+ }
+
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ cli_id_,
+ locator_client_id_,
+ tmb::TaggedMessage(quickstep::kPoisonMessage)));
+
+ test_database_loader_data_exchanger_.join();
+ for (int i = 0; i < kNumInstances; ++i) {
+ data_exchangers_[i].join();
+ }
+ block_locator_->join();
}
void runTestCase(const std::string &input,
@@ -93,20 +115,26 @@ class DistributedExecutionGeneratorTestRunner : public TextBasedTestRunner {
std::string *output) override;
private:
+ block_id_domain getBlockDomain(const std::string &network_address);
+
std::size_t query_id_;
SqlParserWrapper sql_parser_;
- TestDatabaseLoader test_database_loader_;
+ std::unique_ptr<TestDatabaseLoader> test_database_loader_;
+ DataExchangerAsync test_database_loader_data_exchanger_;
Optimizer optimizer_;
MessageBusImpl bus_;
+ tmb::client_id cli_id_, locator_client_id_;
- tmb::client_id cli_id_;
+ std::unique_ptr<BlockLocator> block_locator_;
std::unique_ptr<ForemanDistributed> foreman_;
std::vector<std::unique_ptr<Worker>> workers_;
std::vector<std::unique_ptr<WorkerDirectory>> worker_directories_;
+ std::vector<DataExchangerAsync> data_exchangers_;
+ std::vector<std::unique_ptr<StorageManager>> storage_managers_;
std::vector<std::unique_ptr<Shiftboss>> shiftbosses_;
DISALLOW_COPY_AND_ASSIGN(DistributedExecutionGeneratorTestRunner);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/787a3251/query_optimizer/tests/TestDatabaseLoader.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/TestDatabaseLoader.hpp b/query_optimizer/tests/TestDatabaseLoader.hpp
index d49719d..87c19c6 100644
--- a/query_optimizer/tests/TestDatabaseLoader.hpp
+++ b/query_optimizer/tests/TestDatabaseLoader.hpp
@@ -24,12 +24,21 @@
#include "catalog/CatalogDatabase.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
+
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "storage/StorageBlockInfo.hpp"
+#endif // QUICKSTEP_DISTRIBUTED
+
#include "storage/StorageManager.hpp"
#include "threading/ThreadIDBasedMap.hpp"
#include "utility/Macros.hpp"
#include "tmb/id_typedefs.h"
+#ifdef QUICKSTEP_DISTRIBUTED
+namespace tmb { class MessageBus; }
+#endif // QUICKSTEP_DISTRIBUTED
+
namespace quickstep {
class CatalogRelation;
@@ -60,18 +69,34 @@ class TestDatabaseLoader {
0 /* id */),
storage_manager_(storage_path),
test_relation_(nullptr) {
- bus_.Initialize();
-
- const tmb::client_id worker_thread_client_id = bus_.Connect();
- bus_.RegisterClientAsSender(worker_thread_client_id, kCatalogRelationNewBlockMessage);
-
- // Refer to InsertDestination::sendBlockFilledMessage for the rationale
- // behind using ClientIDMap.
- thread_id_map_->addValue(worker_thread_client_id);
+ init();
+ }
- scheduler_client_id_ = bus_.Connect();
- bus_.RegisterClientAsReceiver(scheduler_client_id_, kCatalogRelationNewBlockMessage);
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Constructor for the distributed version.
+ *
+ * @param storage_path A filesystem directory where the blocks may be
+ * evicted to during the execution of a test query.
+ * Can be empty if the test query is not executed
+ * in the query engine.
+ * @param block_domain The block_domain for StorageManager.
+ * @param locator_client_id The client id of BlockLocator for StorageManager.
+ * @param bus_global The Bus for StorageManager.
+ */
+ TestDatabaseLoader(const std::string &storage_path,
+ const block_id_domain block_domain,
+ const tmb::client_id locator_client_id,
+ tmb::MessageBus *bus_global)
+ : thread_id_map_(ClientIDMap::Instance()),
+ catalog_database_(nullptr /* parent */,
+ "TestDatabase" /* name */,
+ 0 /* id */),
+ storage_manager_(storage_path, block_domain, locator_client_id, bus_global),
+ test_relation_(nullptr) {
+ init();
}
+#endif // QUICKSTEP_DISTRIBUTED
~TestDatabaseLoader() {
clear();
@@ -139,6 +164,20 @@ class TestDatabaseLoader {
void clear();
private:
+ void init() {
+ bus_.Initialize();
+
+ const tmb::client_id worker_thread_client_id = bus_.Connect();
+ bus_.RegisterClientAsSender(worker_thread_client_id, kCatalogRelationNewBlockMessage);
+
+ // Refer to InsertDestination::sendBlockFilledMessage for the rationale
+ // behind using ClientIDMap.
+ thread_id_map_->addValue(worker_thread_client_id);
+
+ scheduler_client_id_ = bus_.Connect();
+ bus_.RegisterClientAsReceiver(scheduler_client_id_, kCatalogRelationNewBlockMessage);
+ }
+
/**
* @brief Simulate Foreman to add all new blocks to the relation.
*/
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/787a3251/storage/DataExchangerAsync.cpp
----------------------------------------------------------------------
diff --git a/storage/DataExchangerAsync.cpp b/storage/DataExchangerAsync.cpp
index 59f5ebf..1d2f7db 100644
--- a/storage/DataExchangerAsync.cpp
+++ b/storage/DataExchangerAsync.cpp
@@ -155,11 +155,11 @@ void DataExchangerAsync::run() {
if (ok) {
call_context->Proceed();
} else {
- LOG(WARNING) << "Not ok\n";
+ LOG(WARNING) << "DataExchangerAsync " << server_address_ << " is not ok";
delete call_context;
}
} else {
- LOG(INFO) << "Shutdown\n";
+ LOG(INFO) << "DataExchangerAsync " << server_address_ << " shuts down";
return;
}
}
[39/39] incubator-quickstep git commit: Analyze min/max stat for each
column.
Posted by ji...@apache.org.
Analyze min/max stat for each column.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f8feba19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f8feba19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f8feba19
Branch: refs/heads/min-max-stats
Commit: f8feba19b85786dfa8e98fe4abf6f9d01b6d4870
Parents: 6ae0cdd
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu Oct 27 14:16:32 2016 -0500
Committer: Jianqiao Zhu <ji...@cs.wisc.edu>
Committed: Wed Dec 14 21:50:19 2016 -0600
----------------------------------------------------------------------
catalog/Catalog.proto | 10 +-
catalog/CatalogRelationStatistics.cpp | 49 ++++--
catalog/CatalogRelationStatistics.hpp | 151 ++++++++++++++++---
cli/CommandExecutor.cpp | 34 ++++-
cli/QuickstepCli.cpp | 6 +
.../cost_model/StarSchemaSimpleCostModel.cpp | 16 +-
6 files changed, 220 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f8feba19/catalog/Catalog.proto
----------------------------------------------------------------------
diff --git a/catalog/Catalog.proto b/catalog/Catalog.proto
index 90ce37e..58de363 100644
--- a/catalog/Catalog.proto
+++ b/catalog/Catalog.proto
@@ -81,13 +81,15 @@ message IndexScheme {
}
message CatalogRelationStatistics {
- optional fixed64 num_tuples = 1;
+ optional TypedValue num_tuples = 1;
- message NumDistinctValuesEntry {
+ message ColumnStats {
required int32 attr_id = 1;
- required fixed64 num_distinct_values = 2;
+ optional TypedValue num_distinct_values = 2;
+ optional TypedValue min_value = 3;
+ optional TypedValue max_value = 4;
}
- repeated NumDistinctValuesEntry num_distinct_values_map = 2;
+ repeated ColumnStats column_stats = 2;
}
message CatalogRelationSchema {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f8feba19/catalog/CatalogRelationStatistics.cpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelationStatistics.cpp b/catalog/CatalogRelationStatistics.cpp
index 6a51570..0f67234 100644
--- a/catalog/CatalogRelationStatistics.cpp
+++ b/catalog/CatalogRelationStatistics.cpp
@@ -23,27 +23,56 @@
namespace quickstep {
+const TypedValue CatalogRelationStatistics::kNullValue =
+ NullType::InstanceNullable().makeNullValue();
+
CatalogRelationStatistics::CatalogRelationStatistics(
const serialization::CatalogRelationStatistics &proto) {
if (proto.has_num_tuples()) {
- num_tuples_ = proto.num_tuples();
+ num_tuples_ = TypedValue::ReconstructFromProto(proto.num_tuples());
+ } else {
+ num_tuples_ = kNullValue;
}
- for (int i = 0; i < proto.num_distinct_values_map_size(); ++i) {
- const auto &entry = proto.num_distinct_values_map(i);
- num_distinct_values_map_.emplace(entry.attr_id(),
- entry.num_distinct_values());
+
+ for (int i = 0; i < proto.column_stats_size(); ++i) {
+ const auto &stat_proto = proto.column_stats(i);
+ auto &stat = column_stats_[stat_proto.attr_id()];
+ if (stat_proto.has_num_distinct_values()) {
+ stat.num_distinct_values =
+ TypedValue::ReconstructFromProto(stat_proto.num_distinct_values());
+ }
+ if (stat_proto.has_min_value()) {
+ stat.min_value =
+ TypedValue::ReconstructFromProto(stat_proto.min_value());
+ }
+ if (stat_proto.has_max_value()) {
+ stat.max_value =
+ TypedValue::ReconstructFromProto(stat_proto.max_value());
+ }
}
}
serialization::CatalogRelationStatistics CatalogRelationStatistics::getProto() const {
serialization::CatalogRelationStatistics proto;
- if (num_tuples_ != 0) {
- proto.set_num_tuples(num_tuples_);
+ if (!num_tuples_.isNull()) {
+ proto.mutable_num_tuples()->CopyFrom(num_tuples_.getProto());
}
- for (const auto &pair : num_distinct_values_map_) {
- auto entry = proto.add_num_distinct_values_map();
+ for (const auto &pair : column_stats_) {
+ auto entry = proto.add_column_stats();
entry->set_attr_id(pair.first);
- entry->set_num_distinct_values(pair.second);
+ const auto &stat = pair.second;
+ if (!stat.num_distinct_values.isNull()) {
+ entry->mutable_num_distinct_values()->CopyFrom(
+ stat.num_distinct_values.getProto());
+ }
+ if (!stat.min_value.isNull()) {
+ entry->mutable_min_value()->CopyFrom(
+ stat.min_value.getProto());
+ }
+ if (!stat.max_value.isNull()) {
+ entry->mutable_max_value()->CopyFrom(
+ stat.max_value.getProto());
+ }
}
return proto;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f8feba19/catalog/CatalogRelationStatistics.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelationStatistics.hpp b/catalog/CatalogRelationStatistics.hpp
index f2056f3..ecb3110 100644
--- a/catalog/CatalogRelationStatistics.hpp
+++ b/catalog/CatalogRelationStatistics.hpp
@@ -26,6 +26,9 @@
#include "catalog/Catalog.pb.h"
#include "catalog/CatalogTypedefs.hpp"
+#include "types/LongType.hpp"
+#include "types/NullType.hpp"
+#include "types/TypedValue.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
@@ -44,7 +47,7 @@ class CatalogRelationStatistics {
* @brief Constructor.
**/
CatalogRelationStatistics()
- : num_tuples_(0) {}
+ : num_tuples_(kNullValue) {}
/**
* @brief Reconstruct a CatalogRelationStatistics object from its serialized
@@ -64,12 +67,12 @@ class CatalogRelationStatistics {
serialization::CatalogRelationStatistics getProto() const;
/**
- * @brief Set the number of tuples statistic.
+ * @brief Check whether the number of tuples statistic is available.
*
- * @param num_tuples The number of tuples statistic.
+ * @return True if the number of tuples statistic is available, false otherwise.
*/
- void setNumTuples(std::size_t num_tuples) {
- num_tuples_ = num_tuples;
+ bool hasNumTuples() const {
+ return !num_tuples_.isNull();
}
/**
@@ -78,41 +81,145 @@ class CatalogRelationStatistics {
* @return The number of tuples. Returns 0 if the statistic is not set.
*/
std::size_t getNumTuples() const {
- return num_tuples_;
+ DCHECK(hasNumTuples());
+ return num_tuples_.getLiteral<std::int64_t>();
+ }
+
+ /**
+ * @brief Set the number of tuples statistic.
+ *
+ * @param num_tuples The number of tuples statistic.
+ */
+ void setNumTuples(const std::size_t num_tuples) {
+ num_tuples_ = LongType::InstanceNonNullable().makeValue(&num_tuples);
+ }
+
+ /**
+ * @brief Check whether the number of distinct values statistic is available
+ * for a column.
+ *
+ * @param The id of the column.
+ * @return True if the number of distinct values statistic is available,
+ * false otherwise.
+ */
+ bool hasNumDistinctValues(const attribute_id attr_id) const {
+ const ColumnStats *stats = getColumnStats(attr_id);
+ return (stats != nullptr && !stats->num_distinct_values.isNull());
+ }
+
+ /**
+ * @brief Get the number of distinct values statistic for a column.
+ *
+ * @param The id of the column.
+ * @return The number of distinct values statistic for the column.
+ */
+ std::size_t getNumDistinctValues(const attribute_id attr_id) const {
+ DCHECK(hasNumDistinctValues(attr_id));
+ return column_stats_.at(attr_id).num_distinct_values.getLiteral<std::int64_t>();
}
/**
- * @brief Set the number of distinct values statistic for a column (catalog attribute).
+ * @brief Set the number of distinct values statistic for a column.
*
* @param attr_id The id of the column.
* @param num_distinct_values The number of distinct values statistic.
*/
- void setNumDistinctValues(attribute_id attr_id, std::size_t num_distinct_values) {
- num_distinct_values_map_[attr_id] = num_distinct_values;
+ void setNumDistinctValues(const attribute_id attr_id,
+ const std::size_t num_distinct_values) {
+ column_stats_[attr_id].num_distinct_values =
+ LongType::InstanceNonNullable().makeValue(&num_distinct_values);
}
/**
- * @brief Get the number of distinct values statistic for a column (catalog attribute).
+ * @brief Check whether the minimum value statistic is available for a column.
*
* @param The id of the column.
- * @return The number of distinct values statistic for the column. Returns 0
- * if the statistic is not set.
+ * @return True if the minimum value statistic is available, false otherwise.
*/
- std::size_t getNumDistinctValues(attribute_id attr_id) const {
- const auto it = num_distinct_values_map_.find(attr_id);
- if (it == num_distinct_values_map_.end()) {
- return static_cast<std::size_t>(0);
- } else {
- return it->second;
- }
+ bool hasMinValue(const attribute_id attr_id) const {
+ const ColumnStats *stats = getColumnStats(attr_id);
+ return (stats != nullptr && !stats->min_value.isNull());
+ }
+
+ /**
+ * @brief Get the minimum value statistic for a column.
+ *
+ * @param The id of the column.
+ * @return The minimum value statistic for the column.
+ */
+ const TypedValue& getMinValue(const attribute_id attr_id) const {
+ DCHECK(hasMinValue(attr_id));
+ return column_stats_.at(attr_id).min_value;
+ }
+
+ /**
+ * @brief Set the minimum value statistic for a column.
+ *
+ * @param The id of the column.
+ * @return The minimum value statistic for the column.
+ */
+ void setMinValue(const attribute_id attr_id,
+ const TypedValue &min_value) {
+ column_stats_[attr_id].min_value = min_value;
+ }
+
+ /**
+ * @brief Check whether the maximum value statistic is available for a column.
+ *
+ * @param The id of the column.
+ * @return True if the maximum value statistic is available, false otherwise.
+ */
+ bool hasMaxValue(const attribute_id attr_id) const {
+ const ColumnStats *stats = getColumnStats(attr_id);
+ return (stats != nullptr && !stats->max_value.isNull());
+ }
+
+ /**
+ * @brief Get the maximum value statistic for a column.
+ *
+ * @param The id of the column.
+ * @return The maximum value statistic for the column.
+ */
+ const TypedValue& getMaxValue(const attribute_id attr_id) const {
+ DCHECK(hasMaxValue(attr_id));
+ return column_stats_.at(attr_id).max_value;
+ }
+
+ /**
+ * @brief Set the maximum value statistic for a column.
+ *
+ * @param The id of the column.
+ * @return The maximum value statistic for the column.
+ */
+ void setMaxValue(const attribute_id attr_id,
+ const TypedValue &max_value) {
+ column_stats_[attr_id].max_value = max_value;
}
private:
+ struct ColumnStats {
+ ColumnStats()
+ : num_distinct_values(kNullValue),
+ min_value(kNullValue),
+ max_value(kNullValue) {
+ }
+ TypedValue num_distinct_values;
+ TypedValue min_value;
+ TypedValue max_value;
+ };
+
+ inline const ColumnStats* getColumnStats(const attribute_id attr_id) const {
+ const auto it = column_stats_.find(attr_id);
+ return (it == column_stats_.end() ? nullptr : &it->second);
+ }
+
+ static const TypedValue kNullValue;
+
// Total number of tuples in the relation.
- std::size_t num_tuples_;
+ TypedValue num_tuples_;
- // Number of distinct values for each column.
- std::unordered_map<attribute_id, std::size_t> num_distinct_values_map_;
+ // Statistics for each column.
+ std::unordered_map<attribute_id, ColumnStats> column_stats_;
DISALLOW_COPY_AND_ASSIGN(CatalogRelationStatistics);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f8feba19/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 3c510e7..3e1312a 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -33,6 +33,8 @@
#include "catalog/CatalogRelationSchema.hpp"
#include "cli/DropRelation.hpp"
#include "cli/PrintToScreen.hpp"
+#include "expressions/aggregation/AggregateFunctionMax.hpp"
+#include "expressions/aggregation/AggregateFunctionMin.hpp"
#include "parser/ParseStatement.hpp"
#include "parser/ParseString.hpp"
#include "parser/SqlParserWrapper.hpp"
@@ -324,9 +326,26 @@ void executeAnalyze(const PtrVector<ParseString> *arguments,
// Get the number of distinct values for each column.
for (const CatalogAttribute &attribute : relation) {
+ const Type &attr_type = attribute.getType();
+ bool is_min_applicable =
+ AggregateFunctionMin::Instance().canApplyToTypes({&attr_type});
+ bool is_max_applicable =
+ AggregateFunctionMax::Instance().canApplyToTypes({&attr_type});
+
std::string query_string = "SELECT COUNT(DISTINCT ";
query_string.append(attribute.getName());
- query_string.append(") FROM ");
+ query_string.append(")");
+ if (is_min_applicable) {
+ query_string.append(", MIN(");
+ query_string.append(attribute.getName());
+ query_string.append(")");
+ }
+ if (is_max_applicable) {
+ query_string.append(", MAX(");
+ query_string.append(attribute.getName());
+ query_string.append(")");
+ }
+ query_string.append(" FROM ");
query_string.append(relation.getName());
query_string.append(";");
@@ -342,9 +361,18 @@ void executeAnalyze(const PtrVector<ParseString> *arguments,
auto *stat = mutable_relation->getStatisticsMutable();
const attribute_id attr_id = attribute.getID();
- DCHECK(results[0].getTypeID() == TypeID::kLong);
+ auto results_it = results.begin();
+ DCHECK(results_it->getTypeID() == TypeID::kLong);
stat->setNumDistinctValues(attr_id,
- results[0].getLiteral<std::int64_t>());
+ results_it->getLiteral<std::int64_t>());
+ if (is_min_applicable) {
+ ++results_it;
+ stat->setMinValue(attr_id, *results_it);
+ }
+ if (is_max_applicable) {
+ ++results_it;
+ stat->setMaxValue(attr_id, *results_it);
+ }
}
// Get the number of tuples for the relation.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f8feba19/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 9db7577..379683d 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -77,6 +77,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "gflags/gflags.h"
#include "glog/logging.h"
+#include "google/protobuf/text_format.h"
#include "tmb/id_typedefs.h"
#include "tmb/message_bus.h"
@@ -212,6 +213,11 @@ int main(int argc, char* argv[]) {
LOG(FATAL) << "NON-STANDARD EXCEPTION DURING STARTUP";
}
+ std::string str;
+ google::protobuf::TextFormat::PrintToString(
+ query_processor->getDefaultDatabase()->getProto(), &str);
+ std::cerr << str << "\n";
+
// Parse the CPU affinities for workers and the preloader thread, if enabled
// to warm up the buffer pool.
const vector<int> worker_cpu_affinities =
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f8feba19/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
index 1075739..d8682f8 100644
--- a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -25,6 +25,7 @@
#include <vector>
#include "catalog/CatalogRelation.hpp"
+#include "catalog/CatalogRelationStatistics.hpp"
#include "query_optimizer/expressions/AttributeReference.hpp"
#include "query_optimizer/expressions/ComparisonExpression.hpp"
#include "query_optimizer/expressions/ExprId.hpp"
@@ -104,10 +105,11 @@ std::size_t StarSchemaSimpleCostModel::estimateCardinalityForTopLevelPlan(
std::size_t StarSchemaSimpleCostModel::estimateCardinalityForTableReference(
const P::TableReferencePtr &physical_plan) {
- std::size_t num_tuples = physical_plan->relation()->getStatistics().getNumTuples();
- if (num_tuples == 0) {
- num_tuples = physical_plan->relation()->estimateTupleCardinality();
- }
+ const CatalogRelation *relation = physical_plan->relation();
+ const CatalogRelationStatistics &stat = relation->getStatistics();
+ const std::size_t num_tuples =
+ stat.hasNumTuples() ? stat.getNumTuples()
+ : relation->estimateTupleCardinality();
return num_tuples;
}
@@ -384,9 +386,9 @@ std::size_t StarSchemaSimpleCostModel::getNumDistinctValues(
const std::vector<E::AttributeReferencePtr> &attributes = table_reference->attribute_list();
for (std::size_t i = 0; i < attributes.size(); ++i) {
if (attributes[i]->id() == attribute_id) {
- std::size_t num_distinct_values = relation.getStatistics().getNumDistinctValues(i);
- if (num_distinct_values > 0) {
- return num_distinct_values;
+ const CatalogRelationStatistics &stat = relation.getStatistics();
+ if (stat.hasNumDistinctValues(i)) {
+ return stat.getNumDistinctValues(i);
}
break;
}
[16/39] incubator-quickstep git commit: Added an util to get block
domain from BlockLocator.
Posted by ji...@apache.org.
Added an util to get block domain from BlockLocator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/56555117
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/56555117
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/56555117
Branch: refs/heads/min-max-stats
Commit: 565551173e797c65651eef0408c74b55e27a5796
Parents: f7d1543
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 20 15:17:46 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:58:14 2016 -0800
----------------------------------------------------------------------
query_execution/BlockLocatorUtil.cpp | 93 ++++++++++++++++++++
query_execution/BlockLocatorUtil.hpp | 59 +++++++++++++
query_execution/CMakeLists.txt | 9 ++
query_execution/tests/BlockLocator_unittest.cpp | 42 +--------
query_optimizer/tests/CMakeLists.txt | 4 +-
.../DistributedExecutionGeneratorTestRunner.cpp | 54 +++---------
.../DistributedExecutionGeneratorTestRunner.hpp | 4 +-
storage/CMakeLists.txt | 4 +-
storage/tests/DataExchange_unittest.cpp | 50 ++---------
9 files changed, 187 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_execution/BlockLocatorUtil.cpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocatorUtil.cpp b/query_execution/BlockLocatorUtil.cpp
new file mode 100644
index 0000000..d2d1e96
--- /dev/null
+++ b/query_execution/BlockLocatorUtil.cpp
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "query_execution/BlockLocatorUtil.hpp"
+
+#include <cstdlib>
+#include <string>
+#include <utility>
+
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "storage/StorageBlockInfo.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/address.h"
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/message_style.h"
+#include "tmb/tagged_message.h"
+
+using tmb::TaggedMessage;
+using tmb::MessageBus;
+using tmb::client_id;
+
+namespace quickstep {
+namespace block_locator {
+
+namespace S = ::quickstep::serialization;
+
+block_id_domain getBlockDomain(const std::string &network_address,
+ const client_id cli_id,
+ client_id *locator_client_id,
+ MessageBus *bus) {
+ tmb::Address address;
+ address.All(true);
+ // NOTE(zuyu): The singleton BlockLocator would need only one copy of the message.
+ tmb::MessageStyle style;
+
+ S::BlockDomainRegistrationMessage proto;
+ proto.set_domain_network_address(network_address);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainRegistrationMessage);
+ std::free(proto_bytes);
+
+ DLOG(INFO) << "Client (id '" << cli_id
+ << "') broadcasts BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
+ << "') to BlockLocator.";
+
+ CHECK(MessageBus::SendStatus::kOK ==
+ bus->Send(cli_id, address, style, std::move(message)));
+
+ const tmb::AnnotatedMessage annotated_message(bus->Receive(cli_id, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ CHECK_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
+
+ *locator_client_id = annotated_message.sender;
+
+ DLOG(INFO) << "Client (id '" << cli_id
+ << "') received BlockDomainRegistrationResponseMessage (typed '"
+ << kBlockDomainRegistrationResponseMessage
+ << "') from BlockLocator (id '" << *locator_client_id << "').";
+
+ S::BlockDomainMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ return static_cast<block_id_domain>(response_proto.block_domain());
+}
+
+} // namespace block_locator
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_execution/BlockLocatorUtil.hpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocatorUtil.hpp b/query_execution/BlockLocatorUtil.hpp
new file mode 100644
index 0000000..74f65e4
--- /dev/null
+++ b/query_execution/BlockLocatorUtil.hpp
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_UTIL_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_UTIL_HPP_
+
+#include <string>
+
+#include "storage/StorageBlockInfo.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace tmb { class MessageBus; }
+
+namespace quickstep {
+namespace block_locator {
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief Broadcast to find BlockLocator to get a block domain for
+ * StorageManager with the given network address.
+ *
+ * @param network_address The network address of the StorageManager.
+ * @param cli_id The client ID of the block domain requester.
+ * @param locator_client_id The client ID of BlockLocator to set.
+ * @param bus A pointer to the TMB.
+ *
+ * @return The requested block domain.
+ **/
+block_id_domain getBlockDomain(const std::string &network_address,
+ const tmb::client_id cli_id,
+ tmb::client_id *locator_client_id,
+ tmb::MessageBus *bus);
+
+/** @} */
+
+} // namespace block_locator
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_UTIL_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index eec0029..719d9f3 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -31,6 +31,7 @@ endif()
add_library(quickstep_queryexecution_AdmitRequestMessage ../empty_src.cpp AdmitRequestMessage.hpp)
if (ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp BlockLocator.hpp)
+ add_library(quickstep_queryexecution_BlockLocatorUtil BlockLocatorUtil.cpp BlockLocatorUtil.hpp)
endif(ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_ForemanBase ../empty_src.cpp ForemanBase.hpp)
if (ENABLE_DISTRIBUTED)
@@ -83,6 +84,12 @@ if (ENABLE_DISTRIBUTED)
quickstep_threading_ThreadUtil
quickstep_utility_Macros
tmb)
+ target_link_libraries(quickstep_queryexecution_BlockLocatorUtil
+ glog
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_storage_StorageBlockInfo
+ tmb)
endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution_ForemanBase
glog
@@ -345,6 +352,7 @@ target_link_libraries(quickstep_queryexecution
if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_BlockLocatorUtil
quickstep_queryexecution_ForemanDistributed
quickstep_queryexecution_PolicyEnforcerDistributed
quickstep_queryexecution_QueryManagerDistributed
@@ -363,6 +371,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_catalog_CatalogAttribute
quickstep_catalog_CatalogRelation
quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_BlockLocatorUtil
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_execution/tests/BlockLocator_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/BlockLocator_unittest.cpp b/query_execution/tests/BlockLocator_unittest.cpp
index 465f2a3..32437c3 100644
--- a/query_execution/tests/BlockLocator_unittest.cpp
+++ b/query_execution/tests/BlockLocator_unittest.cpp
@@ -26,6 +26,7 @@
#include "catalog/CatalogAttribute.hpp"
#include "catalog/CatalogRelation.hpp"
#include "query_execution/BlockLocator.hpp"
+#include "query_execution/BlockLocatorUtil.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
@@ -71,7 +72,6 @@ class BlockLocatorTest : public ::testing::Test {
bus_.Initialize();
locator_.reset(new BlockLocator(&bus_));
- locator_client_id_ = locator_->getBusClientID();
locator_->start();
worker_client_id_ = bus_.Connect();
@@ -84,7 +84,9 @@ class BlockLocatorTest : public ::testing::Test {
bus_.RegisterClientAsSender(worker_client_id_, kPoisonMessage);
- block_domain_ = getBlockDomain(kDomainNetworkAddress);
+ block_domain_ =
+ block_locator::getBlockDomain(kDomainNetworkAddress, worker_client_id_, &locator_client_id_, &bus_);
+ DCHECK_EQ(locator_->getBusClientID(), locator_client_id_);
storage_manager_.reset(
new StorageManager(kStoragePath, block_domain_, locator_client_id_, &bus_));
@@ -168,42 +170,6 @@ class BlockLocatorTest : public ::testing::Test {
unique_ptr<StorageManager> storage_manager_;
private:
- block_id_domain getBlockDomain(const string &network_address) {
- serialization::BlockDomainRegistrationMessage proto;
- proto.set_domain_network_address(network_address);
-
- const int proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kBlockDomainRegistrationMessage);
- free(proto_bytes);
-
- LOG(INFO) << "Worker (id '" << worker_client_id_
- << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
- << "') to BlockLocator (id '" << locator_client_id_ << "')";
-
- CHECK(MessageBus::SendStatus::kOK ==
- QueryExecutionUtil::SendTMBMessage(&bus_,
- worker_client_id_,
- locator_client_id_,
- move(message)));
-
- const AnnotatedMessage annotated_message(bus_.Receive(worker_client_id_, 0, true));
- const TaggedMessage &tagged_message = annotated_message.tagged_message;
- EXPECT_EQ(locator_client_id_, annotated_message.sender);
- EXPECT_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
- LOG(INFO) << "Worker (id '" << worker_client_id_
- << "') received BlockDomainRegistrationResponseMessage from BlockLocator";
-
- serialization::BlockDomainMessage response_proto;
- CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
-
- return static_cast<block_id_domain>(response_proto.block_domain());
- }
-
MessageBusImpl bus_;
unique_ptr<BlockLocator> locator_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/CMakeLists.txt b/query_optimizer/tests/CMakeLists.txt
index 9c764e4..b987322 100644
--- a/query_optimizer/tests/CMakeLists.txt
+++ b/query_optimizer/tests/CMakeLists.txt
@@ -115,8 +115,8 @@ if (ENABLE_DISTRIBUTED)
quickstep_parser_ParseStatement
quickstep_parser_SqlParserWrapper
quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_BlockLocatorUtil
quickstep_queryexecution_ForemanDistributed
- quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
quickstep_queryexecution_Shiftboss
@@ -127,7 +127,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_queryoptimizer_QueryHandle
quickstep_queryoptimizer_tests_TestDatabaseLoader
quickstep_storage_DataExchangerAsync
- quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageManager
quickstep_utility_Macros
quickstep_utility_MemStream
quickstep_utility_SqlError
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
index 0403e77..2351dcd 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
@@ -32,14 +32,14 @@
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
#include "query_execution/BlockLocator.hpp"
+#include "query_execution/BlockLocatorUtil.hpp"
#include "query_execution/ForemanDistributed.hpp"
-#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_optimizer/OptimizerContext.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "storage/DataExchangerAsync.hpp"
-#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageManager.hpp"
#include "utility/MemStream.hpp"
#include "utility/SqlError.hpp"
@@ -81,14 +81,15 @@ DistributedExecutionGeneratorTestRunner::DistributedExecutionGeneratorTestRunner
bus_.RegisterClientAsReceiver(cli_id_, kBlockDomainRegistrationResponseMessage);
block_locator_ = make_unique<BlockLocator>(&bus_);
- locator_client_id_ = block_locator_->getBusClientID();
block_locator_->start();
test_database_loader_ = make_unique<TestDatabaseLoader>(
storage_path,
- getBlockDomain(test_database_loader_data_exchanger_.network_address()),
+ block_locator::getBlockDomain(
+ test_database_loader_data_exchanger_.network_address(), cli_id_, &locator_client_id_, &bus_),
locator_client_id_,
&bus_);
+ DCHECK_EQ(block_locator_->getBusClientID(), locator_client_id_);
test_database_loader_data_exchanger_.set_storage_manager(test_database_loader_->storage_manager());
test_database_loader_data_exchanger_.start();
@@ -111,7 +112,11 @@ DistributedExecutionGeneratorTestRunner::DistributedExecutionGeneratorTestRunner
make_unique<WorkerDirectory>(worker_client_ids.size(), worker_client_ids, numa_nodes));
auto storage_manager = make_unique<StorageManager>(
- storage_path, getBlockDomain(data_exchangers_[i].network_address()), locator_client_id_, &bus_);
+ storage_path,
+ block_locator::getBlockDomain(
+ data_exchangers_[i].network_address(), cli_id_, &locator_client_id_, &bus_),
+ locator_client_id_, &bus_);
+ DCHECK_EQ(block_locator_->getBusClientID(), locator_client_id_);
data_exchangers_[i].set_storage_manager(storage_manager.get());
shiftbosses_.push_back(
@@ -193,44 +198,5 @@ void DistributedExecutionGeneratorTestRunner::runTestCase(
}
}
-block_id_domain DistributedExecutionGeneratorTestRunner::getBlockDomain(
- const string &network_address) {
- serialization::BlockDomainRegistrationMessage proto;
- proto.set_domain_network_address(network_address);
-
- const int proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kBlockDomainRegistrationMessage);
- free(proto_bytes);
-
- DLOG(INFO) << "Client (id '" << cli_id_
- << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
- << "') to BlockLocator (id '" << locator_client_id_ << "')";
-
- CHECK(MessageBus::SendStatus::kOK ==
- QueryExecutionUtil::SendTMBMessage(&bus_,
- cli_id_,
- locator_client_id_,
- move(message)));
-
- const tmb::AnnotatedMessage annotated_message(bus_.Receive(cli_id_, 0, true));
- const TaggedMessage &tagged_message = annotated_message.tagged_message;
- CHECK_EQ(locator_client_id_, annotated_message.sender);
- CHECK_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
- DLOG(INFO) << "Client (id '" << cli_id_
- << "') received BlockDomainRegistrationResponseMessage (typed '"
- << kBlockDomainRegistrationResponseMessage
- << "') from BlockLocator";
-
- serialization::BlockDomainMessage response_proto;
- CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
-
- return static_cast<block_id_domain>(response_proto.block_domain());
-}
-
} // namespace optimizer
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
index d2b13e4..63e320d 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
@@ -38,7 +38,7 @@
#include "query_optimizer/Optimizer.hpp"
#include "query_optimizer/tests/TestDatabaseLoader.hpp"
#include "storage/DataExchangerAsync.hpp"
-#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageManager.hpp"
#include "utility/Macros.hpp"
#include "utility/textbased_test/TextBasedTestRunner.hpp"
@@ -115,8 +115,6 @@ class DistributedExecutionGeneratorTestRunner : public TextBasedTestRunner {
std::string *output) override;
private:
- block_id_domain getBlockDomain(const std::string &network_address);
-
std::size_t query_id_;
SqlParserWrapper sql_parser_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index be60662..559d86d 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -862,7 +862,7 @@ target_link_libraries(quickstep_storage_PartitionedHashTablePool
quickstep_storage_FastHashTableFactory
quickstep_storage_HashTableBase
quickstep_utility_Macros
- quickstep_utility_StringUtil)
+ quickstep_utility_StringUtil)
target_link_libraries(quickstep_storage_PreloaderThread
glog
quickstep_catalog_CatalogDatabase
@@ -1502,7 +1502,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_BlockLocator
- quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_BlockLocatorUtil
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
quickstep_storage_CountedReference
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/storage/tests/DataExchange_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/DataExchange_unittest.cpp b/storage/tests/DataExchange_unittest.cpp
index 9c75150..ac39728 100644
--- a/storage/tests/DataExchange_unittest.cpp
+++ b/storage/tests/DataExchange_unittest.cpp
@@ -27,7 +27,7 @@
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/BlockLocator.hpp"
-#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/BlockLocatorUtil.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "storage/CountedReference.hpp"
@@ -77,7 +77,6 @@ class DataExchangeTest : public ::testing::Test {
bus_.Initialize();
locator_.reset(new BlockLocator(&bus_));
- locator_client_id_ = locator_->getBusClientID();
locator_->start();
worker_client_id_ = bus_.Connect();
@@ -88,18 +87,22 @@ class DataExchangeTest : public ::testing::Test {
storage_manager_expected_.reset(new StorageManager(
kStoragePath,
- getBlockDomain(data_exchanger_expected_.network_address()),
+ block_locator::getBlockDomain(
+ data_exchanger_expected_.network_address(), worker_client_id_, &locator_client_id_, &bus_),
locator_client_id_,
&bus_));
+ DCHECK_EQ(locator_->getBusClientID(), locator_client_id_);
data_exchanger_expected_.set_storage_manager(storage_manager_expected_.get());
data_exchanger_expected_.start();
storage_manager_checked_.reset(new StorageManager(
kStoragePath,
- getBlockDomain(kCheckedDomainNetworkAddress),
+ block_locator::getBlockDomain(
+ kCheckedDomainNetworkAddress, worker_client_id_, &locator_client_id_, &bus_),
locator_client_id_,
&bus_));
+ DCHECK_EQ(locator_->getBusClientID(), locator_client_id_);
}
virtual void TearDown() {
@@ -123,45 +126,6 @@ class DataExchangeTest : public ::testing::Test {
unique_ptr<StorageManager> storage_manager_expected_, storage_manager_checked_;
private:
- block_id_domain getBlockDomain(const string &network_address) {
- serialization::BlockDomainRegistrationMessage proto;
- proto.set_domain_network_address(network_address);
-
- const int proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kBlockDomainRegistrationMessage);
- free(proto_bytes);
-
- LOG(INFO) << "Worker (id '" << worker_client_id_
- << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
- << "') to BlockLocator (id '" << locator_client_id_ << "')";
-
- CHECK(MessageBus::SendStatus::kOK ==
- QueryExecutionUtil::SendTMBMessage(&bus_,
- worker_client_id_,
- locator_client_id_,
- move(message)));
-
- const tmb::AnnotatedMessage annotated_message(bus_.Receive(worker_client_id_, 0, true));
- const TaggedMessage &tagged_message = annotated_message.tagged_message;
- EXPECT_EQ(locator_client_id_, annotated_message.sender);
- EXPECT_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
-
- LOG(INFO) << "Worker (id '" << worker_client_id_
- << "') received BlockDomainRegistrationResponseMessage (typed '"
- << kBlockDomainRegistrationResponseMessage
- << "') from BlockLocator";
-
- serialization::BlockDomainMessage response_proto;
- CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
-
- return static_cast<block_id_domain>(response_proto.block_domain());
- }
-
MessageBusImpl bus_;
unique_ptr<BlockLocator> locator_;
[29/39] incubator-quickstep git commit: Used multiple Shiftbosses in
the distributed unit tests.
Posted by ji...@apache.org.
Used multiple Shiftbosses in the distributed unit tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c4f4b285
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c4f4b285
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c4f4b285
Branch: refs/heads/min-max-stats
Commit: c4f4b285b63e8a3d6bb8fd312f6d00cc8f1cbd45
Parents: 4286d75
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Nov 12 22:31:42 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 27 11:46:23 2016 -0800
----------------------------------------------------------------------
query_execution/ForemanDistributed.cpp | 14 +++---
query_execution/PolicyEnforcerDistributed.cpp | 6 ++-
query_execution/QueryExecutionState.hpp | 53 ++++++++++++++++++++++
query_execution/QueryManagerDistributed.cpp | 35 ++++++++------
query_execution/QueryManagerDistributed.hpp | 4 +-
5 files changed, 88 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f4b285/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index d619657..7dccce4 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -238,17 +238,19 @@ void ForemanDistributed::run() {
bool ForemanDistributed::canCollectNewMessages(const tmb::message_type_id message_type) {
return !QUICKSTEP_EQUALS_ANY_CONSTANT(message_type,
kCatalogRelationNewBlockMessage,
- kWorkOrderFeedbackMessage) &&
- // TODO(zuyu): Multiple Shiftbosses support.
- !shiftboss_directory_.hasReachedCapacity(0);
+ kWorkOrderFeedbackMessage);
}
void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::WorkOrderMessage>> &messages) {
+ const size_t num_shiftbosses = shiftboss_directory_.size();
+ size_t shiftboss_index = 0u;
for (const auto &message : messages) {
DCHECK(message != nullptr);
- // TODO(zuyu): Multiple Shiftbosses support.
- sendWorkOrderMessage(0, *message);
- shiftboss_directory_.incrementNumQueuedWorkOrders(0);
+ sendWorkOrderMessage(shiftboss_index, *message);
+ shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index);
+
+ // TO(zuyu): Take data-locality into account for scheduling.
+ shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f4b285/query_execution/PolicyEnforcerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.cpp b/query_execution/PolicyEnforcerDistributed.cpp
index c06fd86..6e09ea8 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -140,8 +140,10 @@ void PolicyEnforcerDistributed::processInitiateRebuildResponseMessage(const tmb:
QueryManagerDistributed *query_manager = static_cast<QueryManagerDistributed*>(admitted_queries_[query_id].get());
const std::size_t num_rebuild_work_orders = proto.num_rebuild_work_orders();
- query_manager->processInitiateRebuildResponseMessage(proto.operator_index(), num_rebuild_work_orders);
- shiftboss_directory_->addNumQueuedWorkOrders(proto.shiftboss_index(), num_rebuild_work_orders);
+ const size_t shiftboss_index = proto.shiftboss_index();
+ query_manager->processInitiateRebuildResponseMessage(
+ proto.operator_index(), num_rebuild_work_orders, shiftboss_index);
+ shiftboss_directory_->addNumQueuedWorkOrders(shiftboss_index, num_rebuild_work_orders);
if (query_manager->getQueryExecutionState().hasQueryExecutionFinished()) {
onQueryCompletion(query_manager);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f4b285/query_execution/QueryExecutionState.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionState.hpp b/query_execution/QueryExecutionState.hpp
index f5281d5..a4273dc 100644
--- a/query_execution/QueryExecutionState.hpp
+++ b/query_execution/QueryExecutionState.hpp
@@ -22,6 +22,12 @@
#include <cstddef>
#include <unordered_map>
+
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED.
+#ifdef QUICKSTEP_DISTRIBUTED
+#include <unordered_set>
+#endif // QUICKSTEP_DISTRIBUTED
+
#include <utility>
#include <vector>
@@ -103,6 +109,49 @@ class QueryExecutionState {
}
}
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Update the rebuild status of the given operator the number of
+ * pending rebuild work orders, after the rebuild has been initiated.
+ *
+ * @param operator_index The index of the given operator.
+ * @param num_rebuild_workorders The number of rebuild workorders of the given
+ * operator.
+ * @param shiftboss_index The index of the Shiftboss that rebuilt.
+ **/
+ void updateRebuildStatus(const std::size_t operator_index,
+ const std::size_t num_rebuild_workorders,
+ const std::size_t shiftboss_index) {
+ DCHECK_LT(operator_index, num_operators_);
+ auto search_res = rebuild_status_.find(operator_index);
+ DCHECK(search_res != rebuild_status_.end() && search_res->second.has_initiated);
+ search_res->second.num_pending_workorders += num_rebuild_workorders;
+ search_res->second.rebuilt_shiftboss_indexes.insert(shiftboss_index);
+ }
+
+ /**
+ * @brief Check if the rebuild has been finished for the given operator.
+ *
+ * @param operator_index The index of the given operator.
+ * @param num_shiftbosses The number of the Shiftbosses for rebuilt.
+ *
+ * @return True if the rebuild has been finished, false otherwise.
+ **/
+ inline bool hasRebuildFinished(const std::size_t operator_index,
+ const std::size_t num_shiftbosses) const {
+ DCHECK_LT(operator_index, num_operators_);
+ const auto search_res = rebuild_status_.find(operator_index);
+ DCHECK(search_res != rebuild_status_.end());
+
+ const auto &rebuild_status = search_res->second;
+ DCHECK(rebuild_status.has_initiated);
+
+ return rebuild_status.rebuilt_shiftboss_indexes.size() == num_shiftbosses &&
+ rebuild_status.num_pending_workorders == 0u;
+ }
+
+#endif // QUICKSTEP_DISTRIBUTED
+
/**
* @brief Check if the rebuild has been initiated for the given operator.
*
@@ -314,6 +363,10 @@ class QueryExecutionState {
// The number of pending rebuild workorders for the operator.
// Valid if and only if 'has_initiated' is true.
std::size_t num_pending_workorders;
+
+#ifdef QUICKSTEP_DISTRIBUTED
+ std::unordered_set<std::size_t> rebuilt_shiftboss_indexes;
+#endif // QUICKSTEP_DISTRIBUTED
};
// Key is dag_node_index for which rebuild is required.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f4b285/query_execution/QueryManagerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.cpp b/query_execution/QueryManagerDistributed.cpp
index 7d45933..20650d0 100644
--- a/query_execution/QueryManagerDistributed.cpp
+++ b/query_execution/QueryManagerDistributed.cpp
@@ -36,7 +36,9 @@
#include "glog/logging.h"
+#include "tmb/address.h"
#include "tmb/id_typedefs.h"
+#include "tmb/tagged_message.h"
using std::free;
using std::malloc;
@@ -125,16 +127,16 @@ bool QueryManagerDistributed::fetchNormalWorkOrders(const dag_node_index index)
}
void QueryManagerDistributed::processInitiateRebuildResponseMessage(const dag_node_index op_index,
- const std::size_t num_rebuild_work_orders) {
- // TODO(zuyu): Multiple Shiftbosses support.
- query_exec_state_->setRebuildStatus(op_index, num_rebuild_work_orders, true);
+ const std::size_t num_rebuild_work_orders,
+ const std::size_t shiftboss_index) {
+ query_exec_state_->updateRebuildStatus(op_index, num_rebuild_work_orders, shiftboss_index);
- if (num_rebuild_work_orders != 0u) {
+ if (!query_exec_state_->hasRebuildFinished(op_index, shiftboss_directory_->size())) {
// Wait for the rebuild work orders to finish.
return;
}
- // No needs for rebuilds.
+ // No needs for rebuilds, or the rebuild has finished.
markOperatorFinished(op_index);
for (const std::pair<dag_node_index, bool> &dependent_link :
@@ -168,17 +170,20 @@ bool QueryManagerDistributed::initiateRebuild(const dag_node_index index) {
kInitiateRebuildMessage);
free(proto_bytes);
+ // TODO(quickstep-team): Dynamically scale-up/down Shiftbosses.
+ tmb::Address shiftboss_addresses;
+ for (std::size_t i = 0; i < shiftboss_directory_->size(); ++i) {
+ shiftboss_addresses.AddRecipient(shiftboss_directory_->getClientId(i));
+ }
+
LOG(INFO) << "ForemanDistributed sent InitiateRebuildMessage (typed '" << kInitiateRebuildMessage
- << "') to Shiftboss";
- // TODO(zuyu): Multiple workers support.
- QueryExecutionUtil::SendTMBMessage(bus_,
- foreman_client_id_,
- shiftboss_directory_->getClientId(0),
- move(tagged_msg));
-
- // The negative value indicates that the number of rebuild work orders is to be
- // determined.
- query_exec_state_->setRebuildStatus(index, -1, true);
+ << "') to all Shiftbosses";
+ QueryExecutionUtil::BroadcastMessage(foreman_client_id_,
+ shiftboss_addresses,
+ move(tagged_msg),
+ bus_);
+
+ query_exec_state_->setRebuildStatus(index, 0, true);
// Wait for Shiftbosses to report the number of rebuild work orders.
return false;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f4b285/query_execution/QueryManagerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.hpp b/query_execution/QueryManagerDistributed.hpp
index e609ab8..f8ac53c 100644
--- a/query_execution/QueryManagerDistributed.hpp
+++ b/query_execution/QueryManagerDistributed.hpp
@@ -73,9 +73,11 @@ class QueryManagerDistributed final : public QueryManagerBase {
* for initiating the rebuild work order.
* @param num_rebuild_work_orders The number of the rebuild work orders
* generated for the operator indexed by 'op_index'.
+ * @param shiftboss_index The index of the Shiftboss that sends the message.
**/
void processInitiateRebuildResponseMessage(const dag_node_index op_index,
- const std::size_t num_rebuild_work_orders);
+ const std::size_t num_rebuild_work_orders,
+ const std::size_t shiftboss_index);
/**
* @brief Get the next normal workorder to be excuted, wrapped in a
[27/39] incubator-quickstep git commit: Added back the debug build
with llvm.
Posted by ji...@apache.org.
Added back the debug build with llvm.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a31dde4e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a31dde4e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a31dde4e
Branch: refs/heads/min-max-stats
Commit: a31dde4eaab6122768e21812b958a05d3048e50f
Parents: 3047d89
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Nov 25 21:52:43 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Nov 26 23:49:57 2016 -0800
----------------------------------------------------------------------
.travis.yml | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a31dde4e/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 9b43b16..8915eeb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,4 @@
-# NOTE(quickstep-team): In Travis-CI, jobs timeout if they take more than 120
+# NOTE(quickstep-team): In Travis-CI, jobs timeout if they take more than 50
# mins or if there is no log output for more than 10 mins. Hence, we use -O0 to
# speed up compilation in release build. Also, jobs can only use upto 20GB of
# disk space. Hence, we minimize the amount of debug symbol using -g0 (DEBUG
@@ -15,9 +15,18 @@ compiler:
- clang
env:
+ - BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=joinwithbinaryexpressions
+ - BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=selection
- BUILD_TYPE=Release VECTOR_COPY_ELISION_LEVEL=joinwithbinaryexpressions
- BUILD_TYPE=Release VECTOR_COPY_ELISION_LEVEL=selection
+matrix:
+ exclude: # Due to time-out.
+ - compiler: gcc
+ env: BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=joinwithbinaryexpressions
+ - compiler: gcc
+ env: BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=selection
+
install:
- export TEST_JOBS=2;
- if [ "$CC" = "gcc" ]; then
[17/39] incubator-quickstep git commit: Added an util method to get
the ip address.
Posted by ji...@apache.org.
Added an util method to get the ip address.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8f1b0e56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8f1b0e56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8f1b0e56
Branch: refs/heads/min-max-stats
Commit: 8f1b0e56947b49f544bb6e0f5a4b425e8735ccf9
Parents: 5655511
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 13 15:13:09 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 20:04:21 2016 -0800
----------------------------------------------------------------------
utility/CMakeLists.txt | 20 ++++++++
utility/NetworkUtil.cpp | 78 +++++++++++++++++++++++++++++
utility/NetworkUtil.hpp | 42 ++++++++++++++++
utility/tests/NetworkUtil_unittest.cpp | 44 ++++++++++++++++
4 files changed, 184 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f1b0e56/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index e9be2ec..6a656fb 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -15,6 +15,12 @@
# specific language governing permissions and limitations
# under the License.
+if (BUILD_SHARED_LIBS)
+ set(GFLAGS_LIB_NAME gflags_nothreads-shared)
+else()
+ set(GFLAGS_LIB_NAME gflags_nothreads-static)
+endif()
+
include(CheckCXXSourceCompiles)
# Look for GCC-style builtins (also in clang, ICC, and possibly others) that
@@ -180,6 +186,7 @@ add_library(quickstep_utility_Glob Glob.cpp Glob.hpp)
add_library(quickstep_utility_HashPair ../empty_src.cpp HashPair.hpp)
add_library(quickstep_utility_Macros ../empty_src.cpp Macros.hpp)
add_library(quickstep_utility_MemStream ../empty_src.cpp MemStream.hpp)
+add_library(quickstep_utility_NetworkUtil NetworkUtil.cpp NetworkUtil.hpp)
add_library(quickstep_utility_PlanVisualizer PlanVisualizer.cpp PlanVisualizer.hpp)
add_library(quickstep_utility_PrimeNumber PrimeNumber.cpp PrimeNumber.hpp)
add_library(quickstep_utility_PtrList ../empty_src.cpp PtrList.hpp)
@@ -257,6 +264,9 @@ target_link_libraries(quickstep_utility_Glob
target_link_libraries(quickstep_utility_MemStream
glog
quickstep_utility_Macros)
+target_link_libraries(quickstep_utility_NetworkUtil
+ glog
+ ${GFLAGS_LIB_NAME})
target_link_libraries(quickstep_utility_PrimeNumber
glog)
target_link_libraries(quickstep_utility_PlanVisualizer
@@ -339,6 +349,7 @@ target_link_libraries(quickstep_utility
quickstep_utility_HashPair
quickstep_utility_Macros
quickstep_utility_MemStream
+ quickstep_utility_NetworkUtil
quickstep_utility_PlanVisualizer
quickstep_utility_PrimeNumber
quickstep_utility_PtrList
@@ -404,6 +415,15 @@ target_link_libraries(EqualsAnyConstant_unittest
${LIBS})
add_test(EqualsAnyConstant_unittest EqualsAnyConstant_unittest)
+add_executable(NetworkUtil_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/NetworkUtil_unittest.cpp")
+target_link_libraries(NetworkUtil_unittest
+ gtest
+ gtest_main
+ quickstep_utility_NetworkUtil
+ ${GFLAGS_LIB_NAME}
+ ${LIBS})
+add_test(NetworkUtil_unittest NetworkUtil_unittest)
+
add_executable(PrimeNumber_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/PrimeNumber_unittest.cpp")
target_link_libraries(PrimeNumber_unittest
gtest
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f1b0e56/utility/NetworkUtil.cpp
----------------------------------------------------------------------
diff --git a/utility/NetworkUtil.cpp b/utility/NetworkUtil.cpp
new file mode 100644
index 0000000..f665b62
--- /dev/null
+++ b/utility/NetworkUtil.cpp
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "utility/NetworkUtil.hpp"
+
+#include "utility/UtilityConfig.h" // For QUICKSTEP_HAVE_WINDOWS.
+
+#ifndef QUICKSTEP_HAVE_WINDOWS
+#include <arpa/inet.h>
+#include <net/if.h> // Need to include before <ifaddrs.h> for NetBSD.
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <ifaddrs.h>
+#include <netdb.h>
+
+#include <cstring>
+#endif // QUICKSTEP_HAVE_WINDOWS
+
+#include <string>
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+
+namespace quickstep {
+
+DEFINE_bool(use_ethernet_ip, false,
+ "Use the Ethernet ip address, instead of the loopback.");
+
+std::string GetIpv4Address() {
+#ifndef QUICKSTEP_HAVE_WINDOWS
+ struct ifaddrs *ifaddrs;
+ CHECK_EQ(0, getifaddrs(&ifaddrs));
+
+ char ip_address[NI_MAXHOST] = { '\0' };
+
+ for (struct ifaddrs *ifa = ifaddrs; ifa; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr == nullptr ||
+ ifa->ifa_addr->sa_family != AF_INET ||
+ (ifa->ifa_flags & IFF_UP) == 0) {
+ continue;
+ }
+
+ if (FLAGS_use_ethernet_ip && !std::strncmp(ifa->ifa_name, "lo", 2)) {
+ // NOTE: On Linux it starts with 'eth', while on Mac OS X, 'en'.
+ continue;
+ }
+
+ struct sockaddr_in *s4 = (struct sockaddr_in *)(ifa->ifa_addr);
+ CHECK(inet_ntop(AF_INET, reinterpret_cast<void *>(&(s4->sin_addr)), ip_address, sizeof(ip_address)) != nullptr);
+ break;
+ }
+
+ freeifaddrs(ifaddrs);
+
+ return ip_address;
+#else
+ CHECK(!FLAGS_use_ethernet_ip);
+ return kLocalIpv4Address;
+#endif // QUICKSTEP_HAVE_WINDOWS
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f1b0e56/utility/NetworkUtil.hpp
----------------------------------------------------------------------
diff --git a/utility/NetworkUtil.hpp b/utility/NetworkUtil.hpp
new file mode 100644
index 0000000..b825fd8
--- /dev/null
+++ b/utility/NetworkUtil.hpp
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_UTILITY_NETWORK_UTIL_HPP_
+#define QUICKSTEP_UTILITY_NETWORK_UTIL_HPP_
+
+#include <string>
+
+namespace quickstep {
+
+/** \addtogroup Utility
+ * @{
+ */
+
+constexpr char kLocalIpv4Address[] = "127.0.0.1";
+
+/**
+ * @brief Get the IPv4 network address in the text format, i.e., "127.0.0.1".
+ */
+extern std::string GetIpv4Address();
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_UTILITY_NETWORK_UTIL_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f1b0e56/utility/tests/NetworkUtil_unittest.cpp
----------------------------------------------------------------------
diff --git a/utility/tests/NetworkUtil_unittest.cpp b/utility/tests/NetworkUtil_unittest.cpp
new file mode 100644
index 0000000..0013d7c
--- /dev/null
+++ b/utility/tests/NetworkUtil_unittest.cpp
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include <string>
+
+#include "utility/NetworkUtil.hpp"
+
+#include "gflags/gflags_declare.h"
+#include "gtest/gtest.h"
+
+using std::string;
+
+namespace quickstep {
+
+DECLARE_bool(use_ethernet_ip);
+
+TEST(NetworkUtilTest, LoopbackTest) {
+ string lo_ip_address = GetIpv4Address();
+ EXPECT_STREQ(kLocalIpv4Address, lo_ip_address.c_str());
+}
+
+TEST(NetworkUtilTest, EthernetTest) {
+ FLAGS_use_ethernet_ip = true;
+ string ethernet_ip_address = GetIpv4Address();
+ EXPECT_STRNE(kLocalIpv4Address, ethernet_ip_address.c_str());
+}
+
+} // namespace quickstep
[03/39] incubator-quickstep git commit: Moved some storage-related
constants.
Posted by ji...@apache.org.
Moved some storage-related constants.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/6fa7be6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/6fa7be6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/6fa7be6b
Branch: refs/heads/min-max-stats
Commit: 6fa7be6bab6f64fd534b1a7cd56f3a2250f9d330
Parents: 787a325
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Nov 14 22:24:55 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Tue Nov 15 17:55:44 2016 -0800
----------------------------------------------------------------------
CMakeLists.txt | 1 +
cli/QuickstepCli.cpp | 12 +++---------
storage/StorageConstants.hpp | 10 ++++++++++
3 files changed, 14 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6fa7be6b/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 487aaf9..6191de0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -769,6 +769,7 @@ target_link_libraries(quickstep_cli_shell
quickstep_queryoptimizer_QueryPlan
quickstep_queryoptimizer_QueryProcessor
quickstep_storage_PreloaderThread
+ quickstep_storage_StorageConstants
quickstep_threading_ThreadIDBasedMap
quickstep_utility_ExecutionDAGVisualizer
quickstep_utility_Macros
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6fa7be6b/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 8269197..31d9834 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -29,6 +29,8 @@
#include <vector>
#include <fstream>
+#include "cli/CliConfig.h" // For QUICKSTEP_USE_LINENOISE, QUICKSTEP_ENABLE_GOOGLE_PROFILER, and QUICKSTEP_OS_WINDOWS.
+
// TODO(jmp): If filesystem shows up in C++-17, we can switch to just using that.
#ifdef QUICKSTEP_OS_WINDOWS
#include <filesystem>
@@ -36,7 +38,6 @@
#include <stdlib.h>
#endif
-#include "cli/CliConfig.h" // For QUICKSTEP_USE_LINENOISE, QUICKSTEP_ENABLE_GOOGLE_PROFILER.
#include "cli/CommandExecutor.hpp"
#include "cli/DropRelation.hpp"
@@ -74,6 +75,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#endif
#include "storage/PreloaderThread.hpp"
+#include "storage/StorageConstants.hpp"
#include "threading/ThreadIDBasedMap.hpp"
#include "utility/ExecutionDAGVisualizer.hpp"
#include "utility/Macros.hpp"
@@ -130,14 +132,6 @@ using tmb::client_id;
namespace quickstep {
-#ifdef QUICKSTEP_OS_WINDOWS
-static constexpr char kPathSeparator = '\\';
-static constexpr char kDefaultStoragePath[] = "qsstor\\";
-#else
-static constexpr char kPathSeparator = '/';
-static constexpr char kDefaultStoragePath[] = "qsstor/";
-#endif
-
DEFINE_bool(profile_and_report_workorder_perf, false,
"If true, Quickstep will record the exceution time of all the individual "
"normal work orders and report it at the end of query execution.");
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6fa7be6b/storage/StorageConstants.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageConstants.hpp b/storage/StorageConstants.hpp
index 90028db..34a714d 100644
--- a/storage/StorageConstants.hpp
+++ b/storage/StorageConstants.hpp
@@ -23,12 +23,22 @@
#include <cstddef>
#include <cstdint>
+#include "cli/CliConfig.h" // For QUICKSTEP_OS_WINDOWS.
+
namespace quickstep {
/** \addtogroup Storage
* @{
*/
+#ifdef QUICKSTEP_OS_WINDOWS
+constexpr char kPathSeparator = '\\';
+constexpr char kDefaultStoragePath[] = "qsstor\\";
+#else
+constexpr char kPathSeparator = '/';
+constexpr char kDefaultStoragePath[] = "qsstor/";
+#endif
+
// Size of a memory slot managed by the StorageManager. This is the smallest
// quantum of allocation for StorageBlocks and StorageBlobs. 2 MB is the large
// page size on x86.
[23/39] incubator-quickstep git commit: IWYU for QuickstepCli.
Posted by ji...@apache.org.
IWYU for QuickstepCli.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/172b51b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/172b51b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/172b51b5
Branch: refs/heads/min-max-stats
Commit: 172b51b5eaa9a2354792dc0a8a51fbad13ba10f2
Parents: f673f94
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Nov 21 00:06:05 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Mon Nov 21 00:06:05 2016 -0800
----------------------------------------------------------------------
CMakeLists.txt | 3 ---
cli/DefaultsConfigurator.cpp | 1 +
cli/QuickstepCli.cpp | 21 +--------------------
3 files changed, 2 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/172b51b5/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index cd53967..126b47b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -758,16 +758,13 @@ target_link_libraries(quickstep_cli_shell
quickstep_cli_PrintToScreen
quickstep_parser_ParseStatement
quickstep_parser_SqlParserWrapper
- quickstep_queryexecution_AdmitRequestMessage
quickstep_queryexecution_ForemanSingleNode
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
quickstep_queryexecution_Worker
quickstep_queryexecution_WorkerDirectory
- quickstep_queryexecution_WorkerMessage
quickstep_queryoptimizer_QueryHandle
- quickstep_queryoptimizer_QueryPlan
quickstep_queryoptimizer_QueryProcessor
quickstep_storage_PreloaderThread
quickstep_storage_StorageConstants
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/172b51b5/cli/DefaultsConfigurator.cpp
----------------------------------------------------------------------
diff --git a/cli/DefaultsConfigurator.cpp b/cli/DefaultsConfigurator.cpp
index 355031c..94280a7 100644
--- a/cli/DefaultsConfigurator.cpp
+++ b/cli/DefaultsConfigurator.cpp
@@ -22,6 +22,7 @@
#include "cli/CliConfig.h" // For QUICKSTEP_OS_WINDOWS.
#ifdef QUICKSTEP_OS_WINDOWS
+// TODO(jmp): If filesystem shows up in C++-17, we can switch to just using that.
#include <experimental/filesystem>
#else
#include <cstdlib>
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/172b51b5/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 465793c..656786a 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -27,16 +27,8 @@
#include <string>
#include <utility>
#include <vector>
-#include <fstream>
-#include "cli/CliConfig.h" // For QUICKSTEP_USE_LINENOISE, QUICKSTEP_ENABLE_GOOGLE_PROFILER, and QUICKSTEP_OS_WINDOWS.
-
-// TODO(jmp): If filesystem shows up in C++-17, we can switch to just using that.
-#ifdef QUICKSTEP_OS_WINDOWS
-#include <filesystem>
-#else
-#include <stdlib.h>
-#endif
+#include "cli/CliConfig.h" // For QUICKSTEP_USE_LINENOISE, QUICKSTEP_ENABLE_GOOGLE_PROFILER.
#include "cli/CommandExecutor.hpp"
#include "cli/DropRelation.hpp"
@@ -59,15 +51,12 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
#include "parser/SqlParserWrapper.hpp"
-#include "query_execution/AdmitRequestMessage.hpp"
#include "query_execution/ForemanSingleNode.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_execution/Worker.hpp"
#include "query_execution/WorkerDirectory.hpp"
-#include "query_execution/WorkerMessage.hpp"
#include "query_optimizer/QueryHandle.hpp"
-#include "query_optimizer/QueryPlan.hpp"
#include "query_optimizer/QueryProcessor.hpp"
#include "storage/StorageConfig.h" // For QUICKSTEP_HAVE_FILE_MANAGER_HDFS.
@@ -89,10 +78,8 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "glog/logging.h"
-#include "tmb/address.h"
#include "tmb/id_typedefs.h"
#include "tmb/message_bus.h"
-#include "tmb/message_style.h"
namespace quickstep {
class CatalogRelation;
@@ -104,8 +91,6 @@ using std::printf;
using std::string;
using std::vector;
-using quickstep::Address;
-using quickstep::AdmitRequestMessage;
using quickstep::CatalogRelation;
using quickstep::DefaultsConfigurator;
using quickstep::DropRelation;
@@ -114,20 +99,16 @@ using quickstep::FLAGS_storage_path;
using quickstep::ForemanSingleNode;
using quickstep::InputParserUtil;
using quickstep::MessageBusImpl;
-using quickstep::MessageStyle;
-using quickstep::ParseCommand;
using quickstep::ParseResult;
using quickstep::ParseStatement;
using quickstep::PrintToScreen;
using quickstep::PtrVector;
using quickstep::QueryExecutionUtil;
using quickstep::QueryHandle;
-using quickstep::QueryPlan;
using quickstep::QueryProcessor;
using quickstep::SqlParserWrapper;
using quickstep::Worker;
using quickstep::WorkerDirectory;
-using quickstep::WorkerMessage;
using quickstep::kAdmitRequestMessage;
using quickstep::kCatalogFilename;
using quickstep::kPoisonMessage;
[06/39] incubator-quickstep git commit: Refactor Shiftboss for better
debug info.
Posted by ji...@apache.org.
Refactor Shiftboss for better debug info.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/365fff6a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/365fff6a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/365fff6a
Branch: refs/heads/min-max-stats
Commit: 365fff6a3371f6516488a808cc79e929ff789b4a
Parents: 9a005f3
Author: Zuyu Zhang <zu...@apache.org>
Authored: Thu Nov 17 15:02:17 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Thu Nov 17 15:02:17 2016 -0800
----------------------------------------------------------------------
query_execution/Shiftboss.cpp | 56 ++++++++++++++++++++------------------
query_execution/Shiftboss.hpp | 2 ++
2 files changed, 32 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/365fff6a/query_execution/Shiftboss.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index a434527..09d7846 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -56,6 +56,7 @@ using std::string;
using std::unique_ptr;
using std::vector;
+using tmb::AnnotatedMessage;
using tmb::MessageBus;
using tmb::TaggedMessage;
@@ -69,25 +70,16 @@ void Shiftboss::run() {
ThreadUtil::BindToCPU(cpu_id_);
}
+ processShiftbossRegistrationResponseMessage();
+
for (;;) {
// Receive() is a blocking call, causing this thread to sleep until next
// message is received.
AnnotatedMessage annotated_message(bus_->Receive(shiftboss_client_id_, 0, true));
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') received the typed '" << annotated_message.tagged_message.message_type()
<< "' message from client " << annotated_message.sender;
switch (annotated_message.tagged_message.message_type()) {
- case kShiftbossRegistrationResponseMessage: {
- foreman_client_id_ = annotated_message.sender;
-
- const TaggedMessage &tagged_message = annotated_message.tagged_message;
-
- serialization::ShiftbossRegistrationResponseMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
-
- shiftboss_index_ = proto.shiftboss_index();
- break;
- }
case kQueryInitiateMessage: {
const TaggedMessage &tagged_message = annotated_message.tagged_message;
@@ -121,7 +113,7 @@ void Shiftboss::run() {
kWorkOrderMessage);
const size_t worker_index = getSchedulableWorker();
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') forwarded WorkOrderMessage (typed '" << kWorkOrderMessage
<< "') from Foreman to worker " << worker_index;
@@ -152,7 +144,7 @@ void Shiftboss::run() {
case kWorkOrderFeedbackMessage:
case kWorkOrderCompleteMessage:
case kRebuildWorkOrderCompleteMessage: {
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') forwarded typed '" << annotated_message.tagged_message.message_type()
<< "' message from Worker with TMB client ID '" << annotated_message.sender
<< "' to Foreman with TMB client ID " << foreman_client_id_;
@@ -203,7 +195,7 @@ void Shiftboss::run() {
kSaveQueryResultResponseMessage);
free(proto_response_bytes);
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') sent SaveQueryResultResponseMessage (typed '" << kSaveQueryResultResponseMessage
<< "') to Foreman with TMB client ID " << foreman_client_id_;
const MessageBus::SendStatus send_status =
@@ -215,7 +207,7 @@ void Shiftboss::run() {
break;
}
case kPoisonMessage: {
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') forwarded PoisonMessage (typed '" << kPoisonMessage
<< "') from Foreman to all workers";
@@ -271,7 +263,7 @@ void Shiftboss::registerWithForeman() {
kShiftbossRegistrationMessage);
free(proto_bytes);
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') sent ShiftbossRegistrationMessage (typed '" << kShiftbossRegistrationMessage
<< "') to all";
tmb::MessageBus::SendStatus send_status =
@@ -279,18 +271,30 @@ void Shiftboss::registerWithForeman() {
DCHECK(send_status == tmb::MessageBus::SendStatus::kOK);
}
+void Shiftboss::processShiftbossRegistrationResponseMessage() {
+ const AnnotatedMessage annotated_message(bus_->Receive(shiftboss_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ DCHECK_EQ(kShiftbossRegistrationResponseMessage, tagged_message.message_type());
+
+ foreman_client_id_ = annotated_message.sender;
+ DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ << "') received the typed '" << kShiftbossRegistrationResponseMessage
+ << "' message from ForemanDistributed with client " << foreman_client_id_;
+
+ serialization::ShiftbossRegistrationResponseMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ shiftboss_index_ = proto.shiftboss_index();
+}
+
void Shiftboss::processQueryInitiateMessage(
const std::size_t query_id,
const serialization::CatalogDatabase &catalog_database_cache_proto,
const serialization::QueryContext &query_context_proto) {
database_cache_.update(catalog_database_cache_proto);
- unique_ptr<QueryContext> query_context(
- new QueryContext(query_context_proto,
- database_cache_,
- storage_manager_,
- shiftboss_client_id_,
- bus_));
+ auto query_context = std::make_unique<QueryContext>(
+ query_context_proto, database_cache_, storage_manager_, shiftboss_client_id_, bus_);
query_contexts_.emplace(query_id, move(query_context));
serialization::QueryInitiateResponseMessage proto;
@@ -305,7 +309,7 @@ void Shiftboss::processQueryInitiateMessage(
kQueryInitiateResponseMessage);
free(proto_bytes);
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') sent QueryInitiateResponseMessage (typed '" << kQueryInitiateResponseMessage
<< "') to Foreman with TMB client ID " << foreman_client_id_;
const MessageBus::SendStatus send_status =
@@ -344,7 +348,7 @@ void Shiftboss::processInitiateRebuildMessage(const std::size_t query_id,
kInitiateRebuildResponseMessage);
free(proto_bytes);
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') sent InitiateRebuildResponseMessage (typed '" << kInitiateRebuildResponseMessage
<< "') to Foreman with TMB client ID " << foreman_client_id_;
const MessageBus::SendStatus send_status =
@@ -373,7 +377,7 @@ void Shiftboss::processInitiateRebuildMessage(const std::size_t query_id,
kRebuildWorkOrderMessage);
const size_t worker_index = getSchedulableWorker();
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') sent RebuildWorkOrderMessage (typed '" << kRebuildWorkOrderMessage
<< "') to worker " << worker_index;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/365fff6a/query_execution/Shiftboss.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.hpp b/query_execution/Shiftboss.hpp
index 94b10a2..442e61e 100644
--- a/query_execution/Shiftboss.hpp
+++ b/query_execution/Shiftboss.hpp
@@ -191,6 +191,8 @@ class Shiftboss : public Thread {
private:
void registerWithForeman();
+ void processShiftbossRegistrationResponseMessage();
+
/**
* @brief Process the Shiftboss initiate message and ack back.
*
[36/39] incubator-quickstep git commit: Scheduling based on data
locality info.
Posted by ji...@apache.org.
Scheduling based on data locality info.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/bc4086be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/bc4086be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/bc4086be
Branch: refs/heads/min-max-stats
Commit: bc4086be44bb400f29ae9cada17b951241040bee
Parents: 0859a17
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Dec 5 00:13:59 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Mon Dec 5 00:13:59 2016 -0800
----------------------------------------------------------------------
query_execution/BlockLocator.cpp | 53 +++++++++++++++++---
query_execution/BlockLocator.hpp | 53 ++++++++++++++++++++
query_execution/CMakeLists.txt | 3 ++
query_execution/ForemanDistributed.cpp | 53 +++++++++++++++++++-
query_execution/ForemanDistributed.hpp | 13 +++--
query_execution/QueryExecutionMessages.proto | 6 +++
query_execution/QueryExecutionTypedefs.hpp | 1 +
query_execution/Shiftboss.cpp | 1 +
.../DistributedExecutionGeneratorTestRunner.cpp | 2 +-
storage/StorageManager.cpp | 29 +++++++++++
storage/StorageManager.hpp | 9 ++++
11 files changed, 210 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/BlockLocator.cpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocator.cpp b/query_execution/BlockLocator.cpp
index 5de6a54..fa6db51 100644
--- a/query_execution/BlockLocator.cpp
+++ b/query_execution/BlockLocator.cpp
@@ -27,6 +27,7 @@
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "storage/StorageBlockInfo.hpp"
+#include "threading/SpinSharedMutex.hpp"
#include "threading/ThreadUtil.hpp"
#include "glog/logging.h"
@@ -65,6 +66,18 @@ void BlockLocator::run() {
processBlockDomainRegistrationMessage(sender, proto.domain_network_address());
break;
}
+ case kBlockDomainToShiftbossIndexMessage: {
+ serialization::BlockDomainToShiftbossIndexMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ {
+ // Lock 'block_domain_to_shiftboss_index_shared_mutex_' as briefly as
+ // possible to insert an entry for the new Shiftboss index.
+ SpinSharedMutexExclusiveLock<false> write_lock(block_domain_to_shiftboss_index_shared_mutex_);
+ block_domain_to_shiftboss_index_.emplace(proto.block_domain(), proto.shiftboss_index());
+ }
+ break;
+ }
case kAddBlockLocationMessage: {
serialization::BlockLocationMessage proto;
CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
@@ -72,9 +85,15 @@ void BlockLocator::run() {
const block_id block = proto.block_id();
const block_id_domain domain = proto.block_domain();
- const auto result_block_locations = block_locations_[block].insert(domain);
const auto result_domain_blocks = domain_blocks_[domain].insert(block);
- DCHECK_EQ(result_block_locations.second, result_domain_blocks.second);
+
+ {
+ // Lock 'block_locations_shared_mutex_' as briefly as possible to
+ // insert an entry for the new block location.
+ SpinSharedMutexExclusiveLock<false> write_lock(block_locations_shared_mutex_);
+ const auto result_block_locations = block_locations_[block].insert(domain);
+ DCHECK_EQ(result_block_locations.second, result_domain_blocks.second);
+ }
if (result_domain_blocks.second) {
DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " loaded in Domain " << domain;
@@ -90,11 +109,20 @@ void BlockLocator::run() {
const block_id block = proto.block_id();
const block_id_domain domain = proto.block_domain();
- const auto cit = block_locations_[block].find(domain);
- if (cit != block_locations_[block].end()) {
- block_locations_[block].erase(domain);
- domain_blocks_[domain].erase(block);
+ bool block_found = false;
+ {
+ // Lock 'block_locations_shared_mutex_' as briefly as possible to
+ // delete an entry for the block location.
+ SpinSharedMutexExclusiveLock<false> write_lock(block_locations_shared_mutex_);
+ const auto cit = block_locations_[block].find(domain);
+ if (cit != block_locations_[block].end()) {
+ block_locations_[block].erase(domain);
+ block_found = true;
+ }
+ }
+ if (block_found) {
+ domain_blocks_[domain].erase(block);
DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " evicted in Domain " << domain;
} else {
DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " not found in Domain " << domain;
@@ -123,8 +151,13 @@ void BlockLocator::run() {
domain_network_addresses_.erase(domain);
- for (const block_id block : domain_blocks_[domain]) {
- block_locations_[block].erase(domain);
+ {
+ // Lock 'block_locations_shared_mutex_' as briefly as possible to
+ // delete all entry for the block domain.
+ SpinSharedMutexExclusiveLock<false> write_lock(block_locations_shared_mutex_);
+ for (const block_id block : domain_blocks_[domain]) {
+ block_locations_[block].erase(domain);
+ }
}
domain_blocks_.erase(domain);
@@ -172,6 +205,8 @@ void BlockLocator::processLocateBlockMessage(const client_id receiver,
const block_id block) {
serialization::LocateBlockResponseMessage proto;
+ // NOTE(zuyu): We don't need to protect here, as all the writers are in the
+ // single thread.
for (const block_id_domain domain : block_locations_[block]) {
proto.add_block_domains(domain);
}
@@ -199,6 +234,8 @@ void BlockLocator::processGetPeerDomainNetworkAddressesMessage(const client_id r
const block_id block) {
serialization::GetPeerDomainNetworkAddressesResponseMessage proto;
+ // NOTE(zuyu): We don't need to protect here, as all the writers are in the
+ // single thread.
for (const block_id_domain domain : block_locations_[block]) {
proto.add_domain_network_addresses(domain_network_addresses_[domain]);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/BlockLocator.hpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocator.hpp b/query_execution/BlockLocator.hpp
index a83a394..4690369 100644
--- a/query_execution/BlockLocator.hpp
+++ b/query_execution/BlockLocator.hpp
@@ -21,6 +21,7 @@
#define QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_HPP_
#include <atomic>
+#include <cstddef>
#include <string>
#include <unordered_map>
#include <unordered_set>
@@ -28,6 +29,7 @@
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageConstants.hpp"
+#include "threading/SpinSharedMutex.hpp"
#include "threading/Thread.hpp"
#include "utility/Macros.hpp"
@@ -67,6 +69,8 @@ class BlockLocator : public Thread {
bus_->RegisterClientAsReceiver(locator_client_id_, kBlockDomainRegistrationMessage);
bus_->RegisterClientAsSender(locator_client_id_, kBlockDomainRegistrationResponseMessage);
+ bus_->RegisterClientAsReceiver(locator_client_id_, kBlockDomainToShiftbossIndexMessage);
+
bus_->RegisterClientAsReceiver(locator_client_id_, kAddBlockLocationMessage);
bus_->RegisterClientAsReceiver(locator_client_id_, kDeleteBlockLocationMessage);
@@ -91,6 +95,46 @@ class BlockLocator : public Thread {
return locator_client_id_;
}
+ /**
+ * @brief Get the block locality info for scheduling in ForemanDistributed.
+ *
+ * @param block The given block.
+ * @param shiftboss_index_for_block The index of Shiftboss that has loaded the
+ * block in the buffer pool.
+ *
+ * @return Whether the block locality info has found.
+ **/
+ bool getBlockLocalityInfo(const block_id block, std::size_t *shiftboss_index_for_block) const {
+ std::unordered_set<block_id_domain> block_domains;
+ {
+ // Lock 'block_locations_shared_mutex_' as briefly as possible as a
+ // reader.
+ SpinSharedMutexSharedLock<false> read_lock(block_locations_shared_mutex_);
+ const auto cit = block_locations_.find(block);
+ if (cit != block_locations_.end()) {
+ block_domains = cit->second;
+ } else {
+ return false;
+ }
+ }
+
+ {
+ // NOTE(zuyu): This lock is held for the rest duration of this call, as the
+ // exclusive case is rare.
+ SpinSharedMutexSharedLock<false> read_lock(block_domain_to_shiftboss_index_shared_mutex_);
+ for (const block_id_domain block_domain : block_domains) {
+ // TODO(quickstep-team): choose the best node, instead of the first.
+ const auto cit = block_domain_to_shiftboss_index_.find(block_domain);
+ if (cit != block_domain_to_shiftboss_index_.end()) {
+ *shiftboss_index_for_block = cit->second;
+ return true;
+ }
+ }
+ }
+
+ return false;
+ }
+
protected:
void run() override;
@@ -110,8 +154,17 @@ class BlockLocator : public Thread {
// "0.0.0.0:0".
std::unordered_map<block_id_domain, const std::string> domain_network_addresses_;
+ // From a block domain to its Shiftboss index, used by ForemanDistributed
+ // to schedule based on the data-locality info.
+ // Note that not every 'block_id_domain' has a Shiftboss index. For example,
+ // DistributedCli has StorageManager with a 'block_id_domain', which is not
+ // a part of Shiftboss.
+ std::unordered_map<block_id_domain, std::size_t> block_domain_to_shiftboss_index_;
+ alignas(kCacheLineBytes) mutable SpinSharedMutex<false> block_domain_to_shiftboss_index_shared_mutex_;
+
// From a block to its domains.
std::unordered_map<block_id, std::unordered_set<block_id_domain>> block_locations_;
+ alignas(kCacheLineBytes) mutable SpinSharedMutex<false> block_locations_shared_mutex_;
// From a block domain to all blocks loaded in its buffer pool.
std::unordered_map<block_id_domain, std::unordered_set<block_id>> domain_blocks_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 1f7add8..0f74384 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -80,6 +80,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_queryexecution_QueryExecutionUtil
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageConstants
+ quickstep_threading_SpinSharedMutex
quickstep_threading_Thread
quickstep_threading_ThreadUtil
quickstep_utility_Macros
@@ -105,6 +106,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_catalog_CatalogTypedefs
quickstep_catalog_Catalog_proto
quickstep_queryexecution_AdmitRequestMessage
+ quickstep_queryexecution_BlockLocator
quickstep_queryexecution_ForemanBase
quickstep_queryexecution_PolicyEnforcerBase
quickstep_queryexecution_PolicyEnforcerDistributed
@@ -114,6 +116,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_queryexecution_QueryExecutionUtil
quickstep_queryexecution_ShiftbossDirectory
quickstep_relationaloperators_WorkOrder_proto
+ quickstep_storage_StorageBlockInfo
quickstep_threading_ThreadUtil
quickstep_utility_EqualsAnyConstant
quickstep_utility_Macros
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index 0dad8b0..0fa701d 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -28,6 +28,7 @@
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/AdmitRequestMessage.hpp"
+#include "query_execution/BlockLocator.hpp"
#include "query_execution/PolicyEnforcerBase.hpp"
#include "query_execution/PolicyEnforcerDistributed.hpp"
#include "query_execution/QueryContext.hpp"
@@ -36,6 +37,7 @@
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_execution/ShiftbossDirectory.hpp"
#include "relational_operators/WorkOrder.pb.h"
+#include "storage/StorageBlockInfo.hpp"
#include "threading/ThreadUtil.hpp"
#include "utility/EqualsAnyConstant.hpp"
@@ -64,10 +66,12 @@ namespace S = serialization;
class QueryHandle;
ForemanDistributed::ForemanDistributed(
+ const BlockLocator &block_locator,
MessageBus *bus,
CatalogDatabaseLite *catalog_database,
const int cpu_id)
: ForemanBase(bus, cpu_id),
+ block_locator_(block_locator),
catalog_database_(DCHECK_NOTNULL(catalog_database)) {
const std::vector<QueryExecutionMessageType> sender_message_types{
kShiftbossRegistrationResponseMessage,
@@ -296,7 +300,50 @@ bool ForemanDistributed::isHashJoinRelatedWorkOrder(const S::WorkOrderMessage &p
}
namespace {
+
constexpr size_t kDefaultShiftbossIndex = 0u;
+
+bool isNestedLoopsJoinWorkOrder(const serialization::WorkOrder &work_order_proto,
+ const BlockLocator &block_locator,
+ std::size_t *shiftboss_index_for_join) {
+ if (work_order_proto.work_order_type() != S::NESTED_LOOP_JOIN) {
+ return false;
+ }
+
+ const block_id left_block = work_order_proto.GetExtension(S::NestedLoopsJoinWorkOrder::left_block_id);
+ if (block_locator.getBlockLocalityInfo(left_block, shiftboss_index_for_join)) {
+ return true;
+ }
+
+ const block_id right_block = work_order_proto.GetExtension(S::NestedLoopsJoinWorkOrder::right_block_id);
+ return block_locator.getBlockLocalityInfo(right_block, shiftboss_index_for_join);
+}
+
+bool hasBlockLocalityInfo(const serialization::WorkOrder &work_order_proto,
+ const BlockLocator &block_locator,
+ std::size_t *shiftboss_index_for_block) {
+ block_id block = kInvalidBlockId;
+ switch (work_order_proto.work_order_type()) {
+ case S::SAVE_BLOCKS: {
+ block = work_order_proto.GetExtension(S::SaveBlocksWorkOrder::block_id);
+ break;
+ }
+ case S::SELECT: {
+ block = work_order_proto.GetExtension(S::SelectWorkOrder::block_id);
+ break;
+ }
+ case S::SORT_RUN_GENERATION: {
+ block = work_order_proto.GetExtension(S::SortRunGenerationWorkOrder::block_id);
+ break;
+ }
+ default:
+ return false;
+ }
+
+ DCHECK_NE(block, kInvalidBlockId);
+ return block_locator.getBlockLocalityInfo(block, shiftboss_index_for_block);
+}
+
} // namespace
void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::WorkOrderMessage>> &messages) {
@@ -306,14 +353,18 @@ void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::Wo
for (const auto &message : messages) {
DCHECK(message != nullptr);
const S::WorkOrderMessage &proto = *message;
+ const S::WorkOrder &work_order_proto = proto.work_order();
size_t shiftboss_index_for_particular_work_order_type;
if (policy_enforcer_dist->isSingleNodeQuery(proto.query_id())) {
// Always schedule the single-node query to the same Shiftboss.
shiftboss_index_for_particular_work_order_type = kDefaultShiftbossIndex;
} else if (isAggregationRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
} else if (isHashJoinRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
+ } else if (hasBlockLocalityInfo(work_order_proto, block_locator_,
+ &shiftboss_index_for_particular_work_order_type)) {
+ } else if (isNestedLoopsJoinWorkOrder(work_order_proto, block_locator_,
+ &shiftboss_index_for_particular_work_order_type)) {
} else {
- // TODO(zuyu): Take data-locality into account for scheduling.
shiftboss_index_for_particular_work_order_type = shiftboss_index;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/ForemanDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.hpp b/query_execution/ForemanDistributed.hpp
index 34bac07..ed09fda 100644
--- a/query_execution/ForemanDistributed.hpp
+++ b/query_execution/ForemanDistributed.hpp
@@ -33,6 +33,7 @@ namespace tmb { class MessageBus; }
namespace quickstep {
+class BlockLocator;
class CatalogDatabaseLite;
namespace serialization { class WorkOrderMessage; }
@@ -51,6 +52,7 @@ class ForemanDistributed final : public ForemanBase {
/**
* @brief Constructor.
*
+ * @param block_locator The block locator that manages block location info.
* @param bus A pointer to the TMB.
* @param catalog_database The catalog database where this query is executed.
* @param cpu_id The ID of the CPU to which the Foreman thread can be pinned.
@@ -58,9 +60,11 @@ class ForemanDistributed final : public ForemanBase {
* @note If cpu_id is not specified, Foreman thread can be possibly moved
* around on different CPUs by the OS.
**/
- ForemanDistributed(tmb::MessageBus *bus,
- CatalogDatabaseLite *catalog_database,
- const int cpu_id = -1);
+ ForemanDistributed(
+ const BlockLocator &block_locator,
+ tmb::MessageBus *bus,
+ CatalogDatabaseLite *catalog_database,
+ const int cpu_id = -1);
~ForemanDistributed() override {}
@@ -111,6 +115,9 @@ class ForemanDistributed final : public ForemanBase {
**/
bool canCollectNewMessages(const tmb::message_type_id message_type);
+ // To get block locality info for scheduling.
+ const BlockLocator &block_locator_;
+
ShiftbossDirectory shiftboss_directory_;
CatalogDatabaseLite *catalog_database_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index e6d741a..93e458c 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -142,6 +142,12 @@ message BlockDomainMessage {
required uint32 block_domain = 1;
}
+// Used for the block locality based scheduling in ForemanDistributed.
+message BlockDomainToShiftbossIndexMessage {
+ required uint32 block_domain = 1;
+ required uint64 shiftboss_index = 2;
+}
+
// Used when StorageManager loads or evicts a block or a blob from its buffer
// pool.
message BlockLocationMessage {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionTypedefs.hpp b/query_execution/QueryExecutionTypedefs.hpp
index fb9a9d6..919e45b 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -99,6 +99,7 @@ enum QueryExecutionMessageType : message_type_id {
// with a unique block domain.
kBlockDomainRegistrationMessage, // From Worker to BlockLocator.
kBlockDomainRegistrationResponseMessage, // From BlockLocator to Worker.
+ kBlockDomainToShiftbossIndexMessage, // From StorageManager to BlockLocator.
kAddBlockLocationMessage, // From StorageManager to BlockLocator.
kDeleteBlockLocationMessage, // From StorageManager to BlockLocator.
kLocateBlockMessage, // From StorageManager to BlockLocator.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/Shiftboss.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index ed4bade..2ed42d0 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -286,6 +286,7 @@ void Shiftboss::processShiftbossRegistrationResponseMessage() {
CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
shiftboss_index_ = proto.shiftboss_index();
+ storage_manager_->sendBlockDomainToShiftbossIndexMessage(shiftboss_index_);
// Forward this message to Workers regarding <shiftboss_index_>.
QueryExecutionUtil::BroadcastMessage(shiftboss_client_id_,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
index 5100651..45d4fdf 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
@@ -98,7 +98,7 @@ DistributedExecutionGeneratorTestRunner::DistributedExecutionGeneratorTestRunner
// NOTE(zuyu): Foreman should initialize before Shiftboss so that the former
// could receive a registration message from the latter.
- foreman_ = make_unique<ForemanDistributed>(&bus_, test_database_loader_->catalog_database());
+ foreman_ = make_unique<ForemanDistributed>(*block_locator_, &bus_, test_database_loader_->catalog_database());
// We don't use the NUMA aware version of worker code.
const vector<numa_node_id> numa_nodes(1 /* Number of worker threads per instance */,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index a202f71..6299cda 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -230,6 +230,8 @@ StorageManager::StorageManager(
bus_->RegisterClientAsSender(storage_manager_client_id_, kGetPeerDomainNetworkAddressesMessage);
bus_->RegisterClientAsReceiver(storage_manager_client_id_, kGetPeerDomainNetworkAddressesResponseMessage);
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kBlockDomainToShiftbossIndexMessage);
+
bus_->RegisterClientAsSender(storage_manager_client_id_, kAddBlockLocationMessage);
bus_->RegisterClientAsSender(storage_manager_client_id_, kDeleteBlockLocationMessage);
bus_->RegisterClientAsSender(storage_manager_client_id_, kBlockDomainUnregistrationMessage);
@@ -470,6 +472,33 @@ block_id StorageManager::allocateNewBlockOrBlob(const std::size_t num_slots,
}
#ifdef QUICKSTEP_DISTRIBUTED
+void StorageManager::sendBlockDomainToShiftbossIndexMessage(const std::size_t shiftboss_index) {
+ serialization::BlockDomainToShiftbossIndexMessage proto;
+ proto.set_block_domain(block_domain_);
+ proto.set_shiftboss_index(shiftboss_index);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainToShiftbossIndexMessage);
+ free(proto_bytes);
+
+ DLOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent BlockDomainToShiftbossIndexMessage (typed '" << kBlockDomainToShiftbossIndexMessage
+ << "') to BlockLocator";
+
+ DCHECK_NE(block_locator_client_id_, tmb::kClientIdNone);
+ DCHECK(bus_ != nullptr);
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ storage_manager_client_id_,
+ block_locator_client_id_,
+ move(message)));
+}
+
void StorageManager::pullBlockOrBlob(const block_id block,
PullResponse *response) const {
SpinSharedMutexSharedLock<false> read_lock(blocks_shared_mutex_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index 066953b..b61f10a 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -382,6 +382,15 @@ class StorageManager {
#ifdef QUICKSTEP_DISTRIBUTED
/**
+ * @brief Send BlockDomainToShiftbossIndexMessage to BlockLocator so that
+ * ForemanDistributed could take advantages of block locality info
+ * for a better scheduling policy.
+ *
+ * @param shiftboss_index The Shiftboss index.
+ **/
+ void sendBlockDomainToShiftbossIndexMessage(const std::size_t shiftboss_index);
+
+ /**
* @brief Pull a block or a blob. Used by DataExchangerAsync.
*
* @param block The id of the block or blob.
[30/39] incubator-quickstep git commit: Scheduled HashJoinWorkOrder
on the same Shiftboss of BuildHashWorkOrder.
Posted by ji...@apache.org.
Scheduled HashJoinWorkOrder on the same Shiftboss of BuildHashWorkOrder.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e75c265e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e75c265e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e75c265e
Branch: refs/heads/min-max-stats
Commit: e75c265eb279256d89f4fa634c542d566ce3cd7a
Parents: c4f4b28
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 13 20:20:56 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 27 16:45:22 2016 -0800
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 3 ++
query_execution/ForemanDistributed.cpp | 47 ++++++++++++++++++++--
query_execution/ForemanDistributed.hpp | 4 ++
query_execution/PolicyEnforcerDistributed.cpp | 12 ++++++
query_execution/PolicyEnforcerDistributed.hpp | 19 +++++++++
query_execution/QueryManagerDistributed.hpp | 25 ++++++++++++
6 files changed, 106 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e75c265e/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 719d9f3..1f7add8 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -108,10 +108,12 @@ if (ENABLE_DISTRIBUTED)
quickstep_queryexecution_ForemanBase
quickstep_queryexecution_PolicyEnforcerBase
quickstep_queryexecution_PolicyEnforcerDistributed
+ quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
quickstep_queryexecution_ShiftbossDirectory
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_threading_ThreadUtil
quickstep_utility_EqualsAnyConstant
quickstep_utility_Macros
@@ -153,6 +155,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_catalog_CatalogRelation
quickstep_catalog_Catalog_proto
quickstep_queryexecution_PolicyEnforcerBase
+ quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryContext_proto
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionState
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e75c265e/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index 7dccce4..fc9cd3c 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -30,10 +30,12 @@
#include "query_execution/AdmitRequestMessage.hpp"
#include "query_execution/PolicyEnforcerBase.hpp"
#include "query_execution/PolicyEnforcerDistributed.hpp"
+#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_execution/ShiftbossDirectory.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "threading/ThreadUtil.hpp"
#include "utility/EqualsAnyConstant.hpp"
@@ -241,16 +243,53 @@ bool ForemanDistributed::canCollectNewMessages(const tmb::message_type_id messag
kWorkOrderFeedbackMessage);
}
+bool ForemanDistributed::isHashJoinRelatedWorkOrder(const S::WorkOrderMessage &proto,
+ const size_t next_shiftboss_index_to_schedule,
+ size_t *shiftboss_index_for_hash_join) {
+ const S::WorkOrder &work_order_proto = proto.work_order();
+ QueryContext::join_hash_table_id join_hash_table_index;
+
+ switch (work_order_proto.work_order_type()) {
+ case S::BUILD_HASH:
+ join_hash_table_index = work_order_proto.GetExtension(serialization::BuildHashWorkOrder::join_hash_table_index);
+ break;
+ case S::DESTROY_HASH:
+ join_hash_table_index = work_order_proto.GetExtension(serialization::DestroyHashWorkOrder::join_hash_table_index);
+ break;
+ case S::HASH_JOIN:
+ join_hash_table_index = work_order_proto.GetExtension(serialization::HashJoinWorkOrder::join_hash_table_index);
+ break;
+ default:
+ return false;
+ }
+
+ static_cast<PolicyEnforcerDistributed*>(policy_enforcer_.get())->getShiftbossIndexForHashJoin(
+ proto.query_id(), join_hash_table_index, next_shiftboss_index_to_schedule, shiftboss_index_for_hash_join);
+
+ return true;
+}
+
void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::WorkOrderMessage>> &messages) {
const size_t num_shiftbosses = shiftboss_directory_.size();
size_t shiftboss_index = 0u;
for (const auto &message : messages) {
DCHECK(message != nullptr);
- sendWorkOrderMessage(shiftboss_index, *message);
- shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index);
+ const S::WorkOrderMessage &proto = *message;
+ size_t shiftboss_index_for_hash_join;
+ if (isHashJoinRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_hash_join)) {
+ sendWorkOrderMessage(shiftboss_index_for_hash_join, proto);
+ shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index_for_hash_join);
+
+ if (shiftboss_index == shiftboss_index_for_hash_join) {
+ shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
+ }
+ } else {
+ sendWorkOrderMessage(shiftboss_index, proto);
+ shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index);
- // TO(zuyu): Take data-locality into account for scheduling.
- shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
+ // TODO(zuyu): Take data-locality into account for scheduling.
+ shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e75c265e/query_execution/ForemanDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.hpp b/query_execution/ForemanDistributed.hpp
index ccdd0ae..0616f30 100644
--- a/query_execution/ForemanDistributed.hpp
+++ b/query_execution/ForemanDistributed.hpp
@@ -71,6 +71,10 @@ class ForemanDistributed final : public ForemanBase {
void run() override;
private:
+ bool isHashJoinRelatedWorkOrder(const serialization::WorkOrderMessage &proto,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index_for_hash_join);
+
/**
* @brief Dispatch schedulable WorkOrders, wrapped in WorkOrderMessages to the
* worker threads.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e75c265e/query_execution/PolicyEnforcerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.cpp b/query_execution/PolicyEnforcerDistributed.cpp
index 6e09ea8..86b36c8 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -158,6 +158,18 @@ void PolicyEnforcerDistributed::processInitiateRebuildResponseMessage(const tmb:
}
}
+void PolicyEnforcerDistributed::getShiftbossIndexForHashJoin(
+ const std::size_t query_id,
+ const QueryContext::join_hash_table_id join_hash_table_index,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index) {
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+ QueryManagerDistributed *query_manager = static_cast<QueryManagerDistributed*>(admitted_queries_[query_id].get());
+ query_manager->getShiftbossIndexForHashJoin(join_hash_table_index,
+ next_shiftboss_index_to_schedule,
+ shiftboss_index);
+}
+
void PolicyEnforcerDistributed::initiateQueryInShiftboss(QueryHandle *query_handle) {
S::QueryInitiateMessage proto;
proto.set_query_id(query_handle->query_id());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e75c265e/query_execution/PolicyEnforcerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.hpp b/query_execution/PolicyEnforcerDistributed.hpp
index 146e9af..37326bd 100644
--- a/query_execution/PolicyEnforcerDistributed.hpp
+++ b/query_execution/PolicyEnforcerDistributed.hpp
@@ -20,6 +20,7 @@
#include <vector>
#include "query_execution/PolicyEnforcerBase.hpp"
+#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/ShiftbossDirectory.hpp"
#include "utility/Macros.hpp"
@@ -88,6 +89,24 @@ class PolicyEnforcerDistributed final : public PolicyEnforcerBase {
**/
void processInitiateRebuildResponseMessage(const tmb::TaggedMessage &tagged_message);
+ /**
+ * @brief Get or set the index of Shiftboss for a HashJoin related WorkOrder.
+ * If it is the first BuildHash on <join_hash_table_index>, <shiftboss_index>
+ * will be set to <next_shiftboss_index_to_schedule>. Otherwise,
+ * <shiftboss_index> will be set to the index of the Shiftboss that has
+ * executed the first BuildHash.
+ *
+ * @param query_id The query id.
+ * @param join_hash_table_index The Hash Table for the Join.
+ * @param next_shiftboss_index The index of Shiftboss to schedule a next WorkOrder.
+ * @param shiftboss_index The index of Shiftboss to schedule the WorkOrder.
+ **/
+ void getShiftbossIndexForHashJoin(
+ const std::size_t query_id,
+ const QueryContext::join_hash_table_id join_hash_table_index,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index);
+
private:
void decrementNumQueuedWorkOrders(const serialization::WorkOrderCompletionMessage &proto) override {
shiftboss_directory_->decrementNumQueuedWorkOrders(proto.shiftboss_index());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e75c265e/query_execution/QueryManagerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.hpp b/query_execution/QueryManagerDistributed.hpp
index f8ac53c..2b21303 100644
--- a/query_execution/QueryManagerDistributed.hpp
+++ b/query_execution/QueryManagerDistributed.hpp
@@ -22,7 +22,9 @@
#include <cstddef>
#include <memory>
+#include <unordered_map>
+#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryExecutionState.hpp"
#include "query_execution/QueryManagerBase.hpp"
#include "query_execution/WorkOrderProtosContainer.hpp"
@@ -92,6 +94,26 @@ class QueryManagerDistributed final : public QueryManagerBase {
serialization::WorkOrderMessage* getNextWorkOrderMessage(
const dag_node_index start_operator_index);
+ /**
+ * @brief Get the index of Shiftboss for a HashJoin related WorkOrder. If the
+ * Shiftboss index is not found, set using <next_shiftboss_index_to_schedule>.
+ *
+ * @param join_hash_table_index The Hash Table for the Join.
+ * @param next_shiftboss_index The index of Shiftboss to schedule a next WorkOrder.
+ * @param shiftboss_index The index of Shiftboss to schedule the WorkOrder.
+ **/
+ void getShiftbossIndexForHashJoin(const QueryContext::join_hash_table_id join_hash_table_index,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index) {
+ const auto cit = shiftboss_indexes_for_hash_joins_.find(join_hash_table_index);
+ if (cit != shiftboss_indexes_for_hash_joins_.end()) {
+ *shiftboss_index = cit->second;
+ } else {
+ shiftboss_indexes_for_hash_joins_.emplace(join_hash_table_index, next_shiftboss_index_to_schedule);
+ *shiftboss_index = next_shiftboss_index_to_schedule;
+ }
+ }
+
private:
bool checkNormalExecutionOver(const dag_node_index index) const override {
return (checkAllDependenciesMet(index) &&
@@ -114,6 +136,9 @@ class QueryManagerDistributed final : public QueryManagerBase {
std::unique_ptr<WorkOrderProtosContainer> normal_workorder_protos_container_;
+ // A map from a join hash table to its scheduled Shiftboss index.
+ std::unordered_map<QueryContext::join_hash_table_id, std::size_t> shiftboss_indexes_for_hash_joins_;
+
DISALLOW_COPY_AND_ASSIGN(QueryManagerDistributed);
};
[19/39] incubator-quickstep git commit: Fixed the lint issue.
Posted by ji...@apache.org.
Fixed the lint issue.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/212e6cda
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/212e6cda
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/212e6cda
Branch: refs/heads/min-max-stats
Commit: 212e6cdaf89c89bd0cd1ac4b5e27f7cda0785c4a
Parents: c886f7a
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 20 20:38:35 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 20:38:35 2016 -0800
----------------------------------------------------------------------
query_optimizer/QueryProcessor.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/212e6cda/query_optimizer/QueryProcessor.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryProcessor.hpp b/query_optimizer/QueryProcessor.hpp
index 3e37c9d..2b39b84 100644
--- a/query_optimizer/QueryProcessor.hpp
+++ b/query_optimizer/QueryProcessor.hpp
@@ -131,7 +131,7 @@ class QueryProcessor {
*
* @param catalog_filename The file to read the serialized catalog from.
**/
- explicit QueryProcessor(std::string &&catalog_filename)
+ explicit QueryProcessor(std::string &&catalog_filename) // NOLINT(whitespace/operators)
: catalog_filename_(std::move(catalog_filename)),
catalog_altered_(false),
query_id_(0) {
[24/39] incubator-quickstep git commit: Adds support for
PartialBulkInserts in StorageBlocks
Posted by ji...@apache.org.
Adds support for PartialBulkInserts in StorageBlocks
- Enables use of PartialBulkInserts in StorageBlocks
- Value accessor changes to allow use of 2 insert destinations
- Enables PartialInserts for SplitRow
- Changes HashJoin operator so that it can take advantage of the
PartialInserts code.
- This also cleans up code from Previous commit.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/2d11ec58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/2d11ec58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/2d11ec58
Branch: refs/heads/min-max-stats
Commit: 2d11ec588e7e8d7a7a1a8adfc28fb30e5fe0852e
Parents: 172b51b
Author: navsan <na...@gmail.com>
Authored: Mon Nov 7 14:35:01 2016 -0600
Committer: cramja <ma...@gmail.com>
Committed: Mon Nov 21 14:28:02 2016 -0600
----------------------------------------------------------------------
relational_operators/HashJoinOperator.cpp | 150 ++++++++++++++++---
storage/InsertDestination.cpp | 84 +++++++++++
storage/InsertDestination.hpp | 16 ++
storage/InsertDestinationInterface.hpp | 22 +++
storage/SplitRowStoreTupleStorageSubBlock.hpp | 4 +-
storage/StorageBlock.cpp | 24 +++
storage/StorageBlock.hpp | 44 ++++++
storage/TupleStorageSubBlock.hpp | 50 +++++++
types/containers/ColumnVectorsValueAccessor.hpp | 4 +
9 files changed, 373 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index 4a91f86..2028046 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -65,10 +65,11 @@ namespace {
// Functor passed to HashTable::getAllFromValueAccessor() to collect matching
// tuples from the inner relation. It stores matching tuple ID pairs
-// in an unordered_map keyed by inner block ID.
-class MapBasedJoinedTupleCollector {
+// in an unordered_map keyed by inner block ID and a vector of
+// pairs of (build-tupleID, probe-tuple-ID).
+class VectorsOfPairsJoinedTuplesCollector {
public:
- MapBasedJoinedTupleCollector() {
+ VectorsOfPairsJoinedTuplesCollector() {
}
template <typename ValueAccessorT>
@@ -95,6 +96,34 @@ class MapBasedJoinedTupleCollector {
std::unordered_map<block_id, std::vector<std::pair<tuple_id, tuple_id>>> joined_tuples_;
};
+// Another collector using an unordered_map keyed on inner block just like above,
+// except that it uses of a pair of (build-tupleIDs-vector, probe-tuple-IDs-vector).
+class PairsOfVectorsJoinedTuplesCollector {
+ public:
+ PairsOfVectorsJoinedTuplesCollector() {
+ }
+
+ template <typename ValueAccessorT>
+ inline void operator()(const ValueAccessorT &accessor,
+ const TupleReference &tref) {
+ joined_tuples_[tref.block].first.push_back(tref.tuple);
+ joined_tuples_[tref.block].second.push_back(accessor.getCurrentPosition());
+ }
+
+ // Get a mutable pointer to the collected map of joined tuple ID pairs. The
+ // key is inner block_id, value is a pair consisting of
+ // inner block tuple IDs (first) and outer block tuple IDs (second).
+ inline std::unordered_map< block_id, std::pair<std::vector<tuple_id>, std::vector<tuple_id>>>*
+ getJoinedTuples() {
+ return &joined_tuples_;
+ }
+
+ private:
+ std::unordered_map<
+ block_id,
+ std::pair<std::vector<tuple_id>, std::vector<tuple_id>>> joined_tuples_;
+};
+
class SemiAntiJoinTupleCollector {
public:
explicit SemiAntiJoinTupleCollector(TupleIdSequence *filter)
@@ -432,7 +461,7 @@ void HashInnerJoinWorkOrder::execute() {
base_accessor->createSharedTupleIdSequenceAdapterVirtual(*existence_map));
}
- MapBasedJoinedTupleCollector collector;
+ PairsOfVectorsJoinedTuplesCollector collector;
if (join_key_attributes_.size() == 1) {
hash_table_.getAllFromValueAccessor(
probe_accessor.get(),
@@ -450,12 +479,14 @@ void HashInnerJoinWorkOrder::execute() {
const relation_id build_relation_id = build_relation_.getID();
const relation_id probe_relation_id = probe_relation_.getID();
- for (std::pair<const block_id, std::vector<std::pair<tuple_id, tuple_id>>>
+ for (std::pair<const block_id, std::pair<std::vector<tuple_id>, std::vector<tuple_id>>>
&build_block_entry : *collector.getJoinedTuples()) {
BlockReference build_block =
storage_manager_->getBlock(build_block_entry.first, build_relation_);
const TupleStorageSubBlock &build_store = build_block->getTupleStorageSubBlock();
std::unique_ptr<ValueAccessor> build_accessor(build_store.createValueAccessor());
+ const std::vector<tuple_id> &build_tids = build_block_entry.second.first;
+ const std::vector<tuple_id> &probe_tids = build_block_entry.second.second;
// Evaluate '*residual_predicate_', if any.
//
@@ -468,17 +499,16 @@ void HashInnerJoinWorkOrder::execute() {
// hash join is below a reasonable threshold so that we don't blow up
// temporary memory requirements to an unreasonable degree.
if (residual_predicate_ != nullptr) {
- std::vector<std::pair<tuple_id, tuple_id>> filtered_matches;
-
- for (const std::pair<tuple_id, tuple_id> &hash_match
- : build_block_entry.second) {
+ std::pair<std::vector<tuple_id>, std::vector<tuple_id>> filtered_matches;
+ for (std::size_t i = 0; i < build_tids.size(); ++i) {
if (residual_predicate_->matchesForJoinedTuples(*build_accessor,
build_relation_id,
- hash_match.first,
+ build_tids[i],
*probe_accessor,
probe_relation_id,
- hash_match.second)) {
- filtered_matches.emplace_back(hash_match);
+ probe_tids[i])) {
+ filtered_matches.first.push_back(build_tids[i]);
+ filtered_matches.second.push_back(probe_tids[i]);
}
}
@@ -501,22 +531,96 @@ void HashInnerJoinWorkOrder::execute() {
// benefit (probably only a real performance win when there are very few
// matching tuples in each individual inner block but very many inner
// blocks with at least one match).
+
+ // We now create ordered value accessors for both build and probe side,
+ // using the joined tuple TIDs. Note that we have to use this Lambda-based
+ // invocation method here because the accessors don't have a virtual
+ // function that creates such an OrderedTupleIdSequenceAdapterValueAccessor.
+ std::unique_ptr<ValueAccessor> ordered_build_accessor, ordered_probe_accessor;
+ InvokeOnValueAccessorNotAdapter(
+ build_accessor.get(),
+ [&](auto *accessor) -> void { // NOLINT(build/c++11)
+ ordered_build_accessor.reset(
+ accessor->createSharedOrderedTupleIdSequenceAdapter(build_tids));
+ });
+
+ if (probe_accessor->isTupleIdSequenceAdapter()) {
+ InvokeOnTupleIdSequenceAdapterValueAccessor(
+ probe_accessor.get(),
+ [&](auto *accessor) -> void { // NOLINT(build/c++11)
+ ordered_probe_accessor.reset(
+ accessor->createSharedOrderedTupleIdSequenceAdapter(probe_tids));
+ });
+ } else {
+ InvokeOnValueAccessorNotAdapter(
+ probe_accessor.get(),
+ [&](auto *accessor) -> void { // NOLINT(build/c++11)
+ ordered_probe_accessor.reset(
+ accessor->createSharedOrderedTupleIdSequenceAdapter(probe_tids));
+ });
+ }
+
+
+ // We also need a temp value accessor to store results of any scalar expressions.
ColumnVectorsValueAccessor temp_result;
- for (vector<unique_ptr<const Scalar>>::const_iterator selection_cit = selection_.begin();
- selection_cit != selection_.end();
- ++selection_cit) {
- temp_result.addColumn((*selection_cit)->getAllValuesForJoin(build_relation_id,
- build_accessor.get(),
- probe_relation_id,
- probe_accessor.get(),
- build_block_entry.second));
+
+ // Create a map of ValueAccessors and what attributes we want to pick from them
+ std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>> accessor_attribute_map;
+ const std::vector<ValueAccessor *> accessors{
+ ordered_build_accessor.get(), ordered_probe_accessor.get(), &temp_result};
+ const unsigned int build_index = 0, probe_index = 1, temp_index = 2;
+ for (auto &accessor : accessors) {
+ accessor_attribute_map.push_back(std::make_pair(
+ accessor,
+ std::vector<attribute_id>(selection_.size(), kInvalidCatalogId)));
+ }
+
+ attribute_id dest_attr = 0;
+ std::vector<std::pair<tuple_id, tuple_id>> zipped_joined_tuple_ids;
+
+ for (auto &selection_cit : selection_) {
+ // If the Scalar (column) is not an attribute in build/probe blocks, then
+ // insert it into a ColumnVectorsValueAccessor.
+ if (selection_cit->getDataSource() != Scalar::ScalarDataSource::kAttribute) {
+ // Current destination attribute maps to the column we'll create now.
+ accessor_attribute_map[temp_index].second[dest_attr] = temp_result.getNumColumns();
+
+ if (temp_result.getNumColumns() == 0) {
+ // The getAllValuesForJoin function below needs joined tuple IDs as
+ // a vector of pair of (build-tuple-ID, probe-tuple-ID), and we have
+ // a pair of (build-tuple-IDs-vector, probe-tuple-IDs-vector). So
+ // we'll have to zip our two vectors together. We do this inside
+ // the loop because most queries don't exercise this code since
+ // they don't have scalar expressions with attributes from both
+ // build and probe relations (other expressions would have been
+ // pushed down to before the join).
+ zipped_joined_tuple_ids.reserve(build_tids.size());
+ for (std::size_t i = 0; i < build_tids.size(); ++i) {
+ zipped_joined_tuple_ids.push_back(std::make_pair(build_tids[i], probe_tids[i]));
+ }
+ }
+ temp_result.addColumn(
+ selection_cit
+ ->getAllValuesForJoin(build_relation_id, build_accessor.get(),
+ probe_relation_id, probe_accessor.get(),
+ zipped_joined_tuple_ids));
+ } else {
+ auto scalar_attr = static_cast<const ScalarAttribute *>(selection_cit.get());
+ const attribute_id attr_id = scalar_attr->getAttribute().getID();
+ if (scalar_attr->getAttribute().getParent().getID() == build_relation_id) {
+ accessor_attribute_map[build_index].second[dest_attr] = attr_id;
+ } else {
+ accessor_attribute_map[probe_index].second[dest_attr] = attr_id;
+ }
+ }
+ ++dest_attr;
}
// NOTE(chasseur): calling the bulk-insert method of InsertDestination once
// for each pair of joined blocks incurs some extra overhead that could be
// avoided by keeping checked-out MutableBlockReferences across iterations
// of this loop, but that would get messy when combined with partitioning.
- output_destination_->bulkInsertTuples(&temp_result);
+ output_destination_->bulkInsertTuplesFromValueAccessors(accessor_attribute_map);
}
}
@@ -550,7 +654,7 @@ void HashSemiJoinWorkOrder::executeWithResidualPredicate() {
// We collect all the matching probe relation tuples, as there's a residual
// preidcate that needs to be applied after collecting these matches.
- MapBasedJoinedTupleCollector collector;
+ VectorsOfPairsJoinedTuplesCollector collector;
if (join_key_attributes_.size() == 1) {
hash_table_.getAllFromValueAccessor(
probe_accessor.get(),
@@ -759,7 +863,7 @@ void HashAntiJoinWorkOrder::executeWithResidualPredicate() {
base_accessor->createSharedTupleIdSequenceAdapterVirtual(*existence_map));
}
- MapBasedJoinedTupleCollector collector;
+ VectorsOfPairsJoinedTuplesCollector collector;
// We probe the hash table and get all the matches. Unlike
// executeWithoutResidualPredicate(), we have to collect all the matching
// tuples, because after this step we still have to evalute the residual
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/InsertDestination.cpp
----------------------------------------------------------------------
diff --git a/storage/InsertDestination.cpp b/storage/InsertDestination.cpp
index 5e83453..067edf6 100644
--- a/storage/InsertDestination.cpp
+++ b/storage/InsertDestination.cpp
@@ -247,6 +247,90 @@ void InsertDestination::bulkInsertTuplesWithRemappedAttributes(
});
}
+// A common case that we can optimize away is when the attribute_map
+// for an accessor only contains gaps. e.g. This happens for a join when
+// there are no attributes selected from one side.
+void removeGapOnlyAccessors(
+ const std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>>* accessor_attribute_map,
+ std::vector<std::pair<ValueAccessor *, const std::vector<attribute_id>>>* reduced_accessor_attribute_map) {
+ for (std::size_t i = 0; i < accessor_attribute_map->size(); ++i) {
+ bool all_gaps = true;
+ for (const auto &attr : (*accessor_attribute_map)[i].second)
+ if (attr != kInvalidCatalogId) {
+ all_gaps = false;
+ break;
+ }
+ if (all_gaps)
+ continue;
+ reduced_accessor_attribute_map->push_back((*accessor_attribute_map)[i]);
+ (*accessor_attribute_map)[i].first->beginIterationVirtual();
+ }
+}
+
+void InsertDestination::bulkInsertTuplesFromValueAccessors(
+ const std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>> &accessor_attribute_map,
+ bool always_mark_full) {
+ // Handle pathological corner case where there are no accessors
+ if (accessor_attribute_map.size() == 0)
+ return;
+
+ std::vector<std::pair<ValueAccessor *, const std::vector<attribute_id>>> reduced_accessor_attribute_map;
+ removeGapOnlyAccessors(&accessor_attribute_map, &reduced_accessor_attribute_map);
+
+ // We assume that all input accessors have the same number of tuples, so
+ // the iterations finish together. Therefore, we can just check the first one.
+ auto first_accessor = reduced_accessor_attribute_map[0].first;
+ while (!first_accessor->iterationFinishedVirtual()) {
+ tuple_id num_tuples_to_insert = kCatalogMaxID;
+ tuple_id num_tuples_inserted = 0;
+ MutableBlockReference output_block = this->getBlockForInsertion();
+
+ // Now iterate through all the accessors and do one round of bulk-insertion
+ // of partial tuples into the selected output_block.
+ // While inserting from the first ValueAccessor, space is reserved for
+ // all the columns including those coming from other ValueAccessors.
+ // Thereafter, in a given round, we only insert the remaining columns of the
+ // same tuples from the other ValueAccessors.
+ for (auto &p : reduced_accessor_attribute_map) {
+ ValueAccessor *accessor = p.first;
+ std::vector<attribute_id> attribute_map = p.second;
+
+
+ InvokeOnAnyValueAccessor(
+ accessor,
+ [&](auto *accessor) -> void { // NOLINT(build/c++11)
+ num_tuples_inserted = output_block->bulkInsertPartialTuples(
+ attribute_map, accessor, num_tuples_to_insert);
+ });
+
+ if (accessor == first_accessor) {
+ // Now we know how many full tuples can be inserted into this
+ // output_block (viz. number of tuples inserted from first ValueAccessor).
+ // We should only insert that many tuples from the remaining
+ // ValueAccessors as well.
+ num_tuples_to_insert = num_tuples_inserted;
+ } else {
+ // Since the bulk insertion of the first ValueAccessor should already
+ // have reserved the space for all the other ValueAccessors' columns,
+ // we must have been able to insert all the tuples we asked to insert.
+ DCHECK(num_tuples_inserted == num_tuples_to_insert);
+ }
+ }
+
+ // After one round of insertions, we have successfully inserted as many
+ // tuples as possible into the output_block. Strictly speaking, it's
+ // possible that there is more space for insertions because the size
+ // estimation of variable length columns is conservative. But we will ignore
+ // that case and proceed assuming that this output_block is full.
+
+ // Update the header for output_block and then return it.
+ output_block->bulkInsertPartialTuplesFinalize(num_tuples_inserted);
+ const bool mark_full = always_mark_full
+ || !first_accessor->iterationFinishedVirtual();
+ this->returnBlock(std::move(output_block), mark_full);
+ }
+}
+
void InsertDestination::insertTuplesFromVector(std::vector<Tuple>::const_iterator begin,
std::vector<Tuple>::const_iterator end) {
if (begin == end) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/InsertDestination.hpp
----------------------------------------------------------------------
diff --git a/storage/InsertDestination.hpp b/storage/InsertDestination.hpp
index 408e76b..3487638 100644
--- a/storage/InsertDestination.hpp
+++ b/storage/InsertDestination.hpp
@@ -152,6 +152,10 @@ class InsertDestination : public InsertDestinationInterface {
ValueAccessor *accessor,
bool always_mark_full = false) override;
+ void bulkInsertTuplesFromValueAccessors(
+ const std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>> &accessor_attribute_map,
+ bool always_mark_full = false) override;
+
void insertTuplesFromVector(std::vector<Tuple>::const_iterator begin,
std::vector<Tuple>::const_iterator end) override;
@@ -313,6 +317,12 @@ class AlwaysCreateBlockInsertDestination : public InsertDestination {
~AlwaysCreateBlockInsertDestination() override {
}
+ void bulkInsertTuplesFromValueAccessors(
+ const std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>> &accessor_attribute_map,
+ bool always_mark_full = false) override {
+ LOG(FATAL) << "bulkInsertTuplesFromValueAccessors is not implemented for AlwaysCreateBlockInsertDestination";
+ }
+
protected:
MutableBlockReference getBlockForInsertion() override;
@@ -517,6 +527,12 @@ class PartitionAwareInsertDestination : public InsertDestination {
ValueAccessor *accessor,
bool always_mark_full = false) override;
+ void bulkInsertTuplesFromValueAccessors(
+ const std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>> &accessor_attribute_map,
+ bool always_mark_full = false) override {
+ LOG(FATAL) << "bulkInsertTuplesFromValueAccessors is not implemented for PartitionAwareInsertDestination";
+ }
+
void insertTuplesFromVector(std::vector<Tuple>::const_iterator begin,
std::vector<Tuple>::const_iterator end) override;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/InsertDestinationInterface.hpp
----------------------------------------------------------------------
diff --git a/storage/InsertDestinationInterface.hpp b/storage/InsertDestinationInterface.hpp
index 423dff1..b62d3e5 100644
--- a/storage/InsertDestinationInterface.hpp
+++ b/storage/InsertDestinationInterface.hpp
@@ -20,6 +20,7 @@
#ifndef QUICKSTEP_STORAGE_INSERT_DESTINATION_INTERFACE_HPP_
#define QUICKSTEP_STORAGE_INSERT_DESTINATION_INTERFACE_HPP_
+#include <utility>
#include <vector>
#include "catalog/CatalogTypedefs.hpp"
@@ -122,6 +123,27 @@ class InsertDestinationInterface {
bool always_mark_full = false) = 0;
/**
+ * @brief Bulk-insert tuples from one or more ValueAccessors
+ * into blocks managed by this InsertDestination.
+ *
+ * @warning It is implicitly assumed that all the input ValueAccessors have
+ * the same number of tuples in them.
+ *
+ * @param accessor_attribute_map A vector of pairs of ValueAccessor and
+ * corresponding attribute map
+ * The i-th attribute ID in the attr map for a value accessor is "n"
+ * if the attribute_id "i" in the output relation
+ * is the attribute_id "n" in corresponding input value accessor.
+ * Set the i-th element to kInvalidCatalogId if it doesn't come from
+ * the corresponding value accessor.
+ * @param always_mark_full If \c true, always mark the blocks full after
+ * insertion from ValueAccessor even when partially full.
+ **/
+ virtual void bulkInsertTuplesFromValueAccessors(
+ const std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>> &accessor_attribute_map,
+ bool always_mark_full = false) = 0;
+
+ /**
* @brief Insert tuples from a range of Tuples in a vector.
* @warning Unlike bulkInsertTuples(), this is not well-optimized and not
* intended for general use. It should only be used by
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/SplitRowStoreTupleStorageSubBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/SplitRowStoreTupleStorageSubBlock.hpp b/storage/SplitRowStoreTupleStorageSubBlock.hpp
index 681001e..89c756d 100644
--- a/storage/SplitRowStoreTupleStorageSubBlock.hpp
+++ b/storage/SplitRowStoreTupleStorageSubBlock.hpp
@@ -304,9 +304,9 @@ class SplitRowStoreTupleStorageSubBlock: public TupleStorageSubBlock {
tuple_id bulkInsertPartialTuples(
const std::vector<attribute_id> &attribute_map,
ValueAccessor *accessor,
- const tuple_id max_num_tuples_to_insert);
+ const tuple_id max_num_tuples_to_insert) override;
- void bulkInsertPartialTuplesFinalize(const tuple_id num_tuples_inserted);
+ void bulkInsertPartialTuplesFinalize(const tuple_id num_tuples_inserted) override;
const void* getAttributeValue(const tuple_id tuple,
const attribute_id attr) const override;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/StorageBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.cpp b/storage/StorageBlock.cpp
index ea74ee6..6267d6b 100644
--- a/storage/StorageBlock.cpp
+++ b/storage/StorageBlock.cpp
@@ -284,6 +284,30 @@ tuple_id StorageBlock::bulkInsertTuplesWithRemappedAttributes(
return num_inserted;
}
+tuple_id StorageBlock::bulkInsertPartialTuples(
+ const std::vector<attribute_id> &attribute_map,
+ ValueAccessor *accessor,
+ const tuple_id max_num_tuples_to_insert) {
+ const tuple_id num_inserted
+ = tuple_store_->bulkInsertPartialTuples(attribute_map,
+ accessor,
+ max_num_tuples_to_insert);
+ if (num_inserted != 0) {
+ invalidateAllIndexes();
+ dirty_ = true;
+ } else if (tuple_store_->isEmpty()) {
+ if (!accessor->iterationFinishedVirtual()) {
+ throw TupleTooLargeForBlock(0);
+ }
+ }
+ return num_inserted;
+}
+
+void StorageBlock::bulkInsertPartialTuplesFinalize(
+ const tuple_id num_tuples_inserted) {
+ tuple_store_->bulkInsertPartialTuplesFinalize(num_tuples_inserted);
+}
+
void StorageBlock::sample(const bool is_block_sample,
const int percentage,
InsertDestinationInterface *destination) const {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/StorageBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.hpp b/storage/StorageBlock.hpp
index 56b3bdc..ed252c5 100644
--- a/storage/StorageBlock.hpp
+++ b/storage/StorageBlock.hpp
@@ -307,6 +307,7 @@ class StorageBlock : public StorageBlockBase {
* iteration will be advanced to the first non-inserted tuple or, if
* all accessible tuples were inserted in this block, to the end
* position.
+ * @param max_tuples_to_insert Insert at most these many tuples
* @return The number of tuples inserted from accessor.
**/
tuple_id bulkInsertTuplesWithRemappedAttributes(
@@ -314,6 +315,49 @@ class StorageBlock : public StorageBlockBase {
ValueAccessor *accessor);
/**
+ * @brief Insert up to max_num_tuples_to_insert tuples from a ValueAccessor
+ * as a single batch, using the attribute_map to project and reorder
+ * columns from the input ValueAccessor. Does not update header.
+ *
+ * @note Typical usage is where you want to bulk-insert columns from two
+ * or more value accessors. Instead of writing out the columns into
+ * one or more column vector value accessors, you can simply use this
+ * function with the appropriate attribute_map for each value
+ * accessor (InsertDestination::bulkInsertTuplesFromValueAccessors
+ * handles all the details) to insert tuples without an extra temp copy.
+ *
+ * @warning Must call bulkInsertPartialTuplesFinalize() to update the header,
+ * until which point, the insertion is not visible to others.
+ * @warning The inserted tuples may be placed in sub-optimal locations in this
+ * TupleStorageSubBlock.
+ *
+ * @param attribute_map A vector which maps the attributes of this
+ * TupleStorageSubBlock's relation (gaps indicated with kInvalidCatalogId)
+ * to the corresponding attributes which should be read from accessor.
+ * @param accessor A ValueAccessor to insert tuples from. The accessor's
+ * iteration will be advanced to the first non-inserted tuple or, if
+ * all accessible tuples were inserted in this sub-block, to the end
+ * position.
+ * @return The number of tuples inserted from accessor.
+ **/
+ tuple_id bulkInsertPartialTuples(
+ const std::vector<attribute_id> &attribute_map,
+ ValueAccessor *accessor,
+ const tuple_id max_num_tuples_to_insert);
+
+ /**
+ * @brief Update header after a bulkInsertPartialTuples.
+ *
+ * @warning Only call this after a bulkInsertPartialTuples, passing in the
+ * number of tuples that were inserted (return value of that function).
+ *
+ * @param num_tuples_inserted Number of tuples inserted (i.e., how much to
+ * advance the header.num_tuples by). Should be equal to the return
+ * value of bulkInsertPartialTuples.
+ **/
+ void bulkInsertPartialTuplesFinalize(tuple_id num_tuples_inserted);
+
+ /**
* @brief Get the IDs of tuples in this StorageBlock which match a given Predicate.
*
* @param predicate The predicate to match.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/TupleStorageSubBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/TupleStorageSubBlock.hpp b/storage/TupleStorageSubBlock.hpp
index aed6eea..26e8027 100644
--- a/storage/TupleStorageSubBlock.hpp
+++ b/storage/TupleStorageSubBlock.hpp
@@ -272,6 +272,56 @@ class TupleStorageSubBlock {
ValueAccessor *accessor) = 0;
/**
+ * @brief Insert up to max_num_tuples_to_insert tuples from a ValueAccessor
+ * as a single batch, using the attribute_map to project and reorder
+ * columns from the input ValueAccessor. Does not update header.
+ *
+ * @note Typical usage is where you want to bulk-insert columns from two
+ * or more value accessors. Instead of writing out the columns into
+ * one or more column vector value accessors, you can simply use this
+ * function with the appropriate attribute_map for each value
+ * accessor (InsertDestination::bulkInsertTuplesFromValueAccessors
+ * handles all the details) to insert tuples without an extra temp copy.
+ *
+ * @warning Must call bulkInsertPartialTuplesFinalize() to update the header,
+ * until which point, the insertion is not visible to others.
+ * @warning The inserted tuples may be placed in a suboptimal position in the
+ * block.
+ *
+ * @param attribute_map A vector which maps the attributes of this
+ * TupleStorageSubBlock's relation (gaps indicated with kInvalidCatalogId)
+ * to the corresponding attributes which should be read from accessor.
+ * @param accessor A ValueAccessor to insert tuples from. The accessor's
+ * iteration will be advanced to the first non-inserted tuple or, if
+ * all accessible tuples were inserted in this sub-block, to the end
+ * position.
+ * @return The number of tuples inserted from accessor.
+ **/
+ virtual tuple_id bulkInsertPartialTuples(
+ const std::vector<attribute_id> &attribute_map,
+ ValueAccessor *accessor,
+ const tuple_id max_num_tuples_to_insert) {
+ LOG(FATAL) << "Partial bulk insert is not supported for this TupleStorageBlock type ("
+ << getTupleStorageSubBlockType() << ").";
+ }
+
+ /**
+ * @brief Update header after a bulkInsertPartialTuples.
+ *
+ * @warning Only call this after a bulkInsertPartialTuples, passing in the
+ * number of tuples that were inserted (return value of that function).
+ *
+ * @param num_tuples_inserted Number of tuples inserted (i.e., how much to
+ * advance the header.num_tuples by). Should be equal to the return
+ * value of bulkInsertPartialTuples.
+ **/
+ virtual void bulkInsertPartialTuplesFinalize(
+ const tuple_id num_tuples_inserted) {
+ LOG(FATAL) << "Partial bulk insert is not supported for this TupleStorageBlock type ("
+ << getTupleStorageSubBlockType() << ").";
+ }
+
+ /**
* @brief Get the (untyped) value of an attribute in a tuple in this buffer.
* @warning This method may not be supported for all implementations of
* TupleStorageSubBlock. supportsUntypedGetAttributeValue() MUST be
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/types/containers/ColumnVectorsValueAccessor.hpp
----------------------------------------------------------------------
diff --git a/types/containers/ColumnVectorsValueAccessor.hpp b/types/containers/ColumnVectorsValueAccessor.hpp
index fe413a0..fbbdc1b 100644
--- a/types/containers/ColumnVectorsValueAccessor.hpp
+++ b/types/containers/ColumnVectorsValueAccessor.hpp
@@ -139,6 +139,10 @@ class ColumnVectorsValueAccessor : public ValueAccessor {
return nullptr;
}
+ inline std::size_t getNumColumns() const {
+ return columns_.size();
+ }
+
template <bool check_null = true>
inline const void* getUntypedValue(const attribute_id attr_id) const {
return getUntypedValueAtAbsolutePosition<check_null>(attr_id, current_position_);
[09/39] incubator-quickstep git commit: Build only the release jobs.
Posted by ji...@apache.org.
Build only the release jobs.
- Use only one thread to avoid OOM.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/dec87234
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/dec87234
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/dec87234
Branch: refs/heads/min-max-stats
Commit: dec87234c53db8bd27bbe6a22b28b7e461cae9c6
Parents: 178ed4b
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Nov 14 21:57:19 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:22:51 2016 -0800
----------------------------------------------------------------------
.travis.yml | 9 +--------
1 file changed, 1 insertion(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dec87234/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 784a46f..9b43b16 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,17 +15,10 @@ compiler:
- clang
env:
- - BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=joinwithbinaryexpressions
- BUILD_TYPE=Release VECTOR_COPY_ELISION_LEVEL=joinwithbinaryexpressions
- - BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=selection
- BUILD_TYPE=Release VECTOR_COPY_ELISION_LEVEL=selection
install:
- - if [ "$CC" = "gcc" ] || [[ "$BUILD_TYPE" = "Release" ]]; then
- export MAKE_JOBS=1;
- else
- export MAKE_JOBS=2;
- fi
- export TEST_JOBS=2;
- if [ "$CC" = "gcc" ]; then
export CC="gcc-5";
@@ -61,7 +54,7 @@ script:
- ./third_party/cpplint/lint_everything.py
- ./validate_cmakelists.py
- ./cyclic_dependency.py
- - (cd build && make -j$MAKE_JOBS)
+ - (cd build && make)
- (cd build && ctest --output-on-failure -j$TEST_JOBS)
after_failure:
[22/39] incubator-quickstep git commit: Refactored the database init
code.
Posted by ji...@apache.org.
Refactored the database init code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f673f945
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f673f945
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f673f945
Branch: refs/heads/min-max-stats
Commit: f673f94561e104c62d0441f1ded04737e23b0ba0
Parents: 54e0654
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 20 23:55:22 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 23:55:22 2016 -0800
----------------------------------------------------------------------
cli/CMakeLists.txt | 8 +++-
cli/DefaultsConfigurator.cpp | 77 +++++++++++++++++++++++++++++++++++++++
cli/DefaultsConfigurator.hpp | 10 +++++
cli/QuickstepCli.cpp | 36 +-----------------
4 files changed, 94 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f673f945/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 9847787..9b62af9 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -75,7 +75,7 @@ else()
quickstep_utility_Macros)
endif()
-add_library(quickstep_cli_DefaultsConfigurator ../empty_src.cpp DefaultsConfigurator.hpp)
+add_library(quickstep_cli_DefaultsConfigurator DefaultsConfigurator.cpp DefaultsConfigurator.hpp)
add_library(quickstep_cli_InputParserUtil InputParserUtil.cpp InputParserUtil.hpp)
add_library(quickstep_cli_PrintToScreen PrintToScreen.cpp PrintToScreen.hpp)
@@ -106,10 +106,14 @@ target_link_libraries(quickstep_cli_CommandExecutor
quickstep_utility_SqlError)
target_link_libraries(quickstep_cli_DefaultsConfigurator
+ glog
+ quickstep_catalog_Catalog
+ quickstep_catalog_Catalog_proto
+ quickstep_catalog_CatalogDatabase
quickstep_utility_Macros)
if(QUICKSTEP_HAVE_LIBNUMA)
target_link_libraries(quickstep_cli_DefaultsConfigurator
- ${LIBNUMA_LIBRARY})
+ ${LIBNUMA_LIBRARY})
endif()
target_link_libraries(quickstep_cli_Flags
quickstep_cli_DefaultsConfigurator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f673f945/cli/DefaultsConfigurator.cpp
----------------------------------------------------------------------
diff --git a/cli/DefaultsConfigurator.cpp b/cli/DefaultsConfigurator.cpp
new file mode 100644
index 0000000..355031c
--- /dev/null
+++ b/cli/DefaultsConfigurator.cpp
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "cli/DefaultsConfigurator.hpp"
+
+#include "cli/CliConfig.h" // For QUICKSTEP_OS_WINDOWS.
+
+#ifdef QUICKSTEP_OS_WINDOWS
+#include <experimental/filesystem>
+#else
+#include <cstdlib>
+#endif // QUICKSTEP_OS_WINDOWS
+
+#include <fstream>
+#include <string>
+
+#include "catalog/Catalog.hpp"
+#include "catalog/Catalog.pb.h"
+#include "catalog/CatalogDatabase.hpp"
+
+#include "glog/logging.h"
+
+using std::string;
+
+namespace quickstep {
+
+void DefaultsConfigurator::InitializeDefaultDatabase(const string &storage_path, const string &catalog_path) {
+ // TODO(jmp): Refactor the code in this file!
+ LOG(INFO) << "Initializing the database, creating a new catalog file and storage directory";
+
+ // Create the directory
+ // TODO(jmp): At some point, likely in C++-17, we will just have the
+ // filesystem path, and we can clean this up
+#ifdef QUICKSTEP_OS_WINDOWS
+ CHECK(std::experimental::filesystem::create_directories(storage_path))
+ << "Failed when attempting to create the directory: " << storage_path
+ << "\nCheck if the directory already exists. If so, delete it or move it before initializing";
+#else
+ {
+ const string path_name = "mkdir " + storage_path;
+ CHECK(std::system(path_name.c_str()))
+ << "Failed when attempting to create the directory: " << storage_path;
+ }
+#endif // QUICKSTEP_OS_WINDOWS
+
+ // Create the default catalog file.
+ std::ofstream catalog_file(catalog_path.c_str());
+ CHECK(catalog_file.good())
+ << "ERROR: Unable to open " << catalog_path << " for writing.";
+
+ Catalog catalog;
+ catalog.addDatabase(new CatalogDatabase(nullptr, "default"));
+
+ CHECK(catalog.getProto().SerializeToOstream(&catalog_file))
+ << "ERROR: Unable to serialize catalog proto to file " << catalog_path;
+
+ // Close the catalog file - it will be reopened below by the QueryProcessor.
+ catalog_file.close();
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f673f945/cli/DefaultsConfigurator.hpp
----------------------------------------------------------------------
diff --git a/cli/DefaultsConfigurator.hpp b/cli/DefaultsConfigurator.hpp
index 4fd62ac..4b534d6 100644
--- a/cli/DefaultsConfigurator.hpp
+++ b/cli/DefaultsConfigurator.hpp
@@ -27,6 +27,7 @@
#endif // QUICKSTEP_HAVE_LIBNUMA
#include <cstddef>
+#include <string>
#include <thread> // NOLINT(build/c++11)
#ifdef QUICKSTEP_HAVE_LIBNUMA
@@ -106,6 +107,15 @@ class DefaultsConfigurator {
return 1;
}
+ /**
+ * @brief Initialize the default database with no relations.
+ *
+ * @param storage_path The filesystem directory to store catalog.
+ * @param catalog_path The full path of the catalog file.
+ **/
+ static void InitializeDefaultDatabase(const std::string &storage_path,
+ const std::string &catalog_path);
+
private:
/**
* @brief Private constructor to disable instantiation of the class.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f673f945/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 6a7c500..465793c 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -216,41 +216,7 @@ int main(int argc, char* argv[]) {
string catalog_path(FLAGS_storage_path);
catalog_path.append(kCatalogFilename);
if (quickstep::FLAGS_initialize_db) { // Initialize the database
- // TODO(jmp): Refactor the code in this file!
- LOG(INFO) << "Initializing the database, creating a new catalog file and storage directory\n";
-
- // Create the directory
- // TODO(jmp): At some point, likely in C++-17, we will just have the
- // filesystem path, and we can clean this up
-#ifdef QUICKSTEP_OS_WINDOWS
- std::filesystem::create_directories(FLAGS_storage_path);
- LOG(FATAL) << "Failed when attempting to create the directory: " << FLAGS_storage_path << "\n";
- LOG(FATAL) << "Check if the directory already exists. If so, delete it or move it before initializing \n";
-#else
- {
- string path_name = "mkdir " + FLAGS_storage_path;
- if (std::system(path_name.c_str())) {
- LOG(FATAL) << "Failed when attempting to create the directory: " << FLAGS_storage_path << "\n";
- }
- }
-#endif
-
- // Create the default catalog file.
- std::ofstream catalog_file(catalog_path);
- if (!catalog_file.good()) {
- LOG(FATAL) << "ERROR: Unable to open " << kCatalogFilename << " for writing.\n";
- }
-
- quickstep::Catalog catalog;
- catalog.addDatabase(new quickstep::CatalogDatabase(nullptr, "default"));
-
- if (!catalog.getProto().SerializeToOstream(&catalog_file)) {
- LOG(FATAL) << "ERROR: Unable to serialize catalog proto to file " << kCatalogFilename;
- return 1;
- }
-
- // Close the catalog file - it will be reopened below by the QueryProcessor.
- catalog_file.close();
+ DefaultsConfigurator::InitializeDefaultDatabase(FLAGS_storage_path, catalog_path);
}
// Setup QueryProcessor, including CatalogDatabase.
[33/39] incubator-quickstep git commit: Logged only in the debug mode.
Posted by ji...@apache.org.
Logged only in the debug mode.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5ff89dd5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5ff89dd5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5ff89dd5
Branch: refs/heads/min-max-stats
Commit: 5ff89dd5427bf904e1406111bc2877836f586196
Parents: c608e99
Author: Zuyu Zhang <zu...@apache.org>
Authored: Wed Nov 30 23:07:25 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Wed Nov 30 23:07:25 2016 -0800
----------------------------------------------------------------------
storage/StorageManager.cpp | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ff89dd5/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 56ca323..a202f71 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -557,9 +557,9 @@ vector<string> StorageManager::getPeerDomainNetworkAddresses(const block_id bloc
kGetPeerDomainNetworkAddressesMessage);
free(proto_bytes);
- LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
- << "') sent GetPeerDomainNetworkAddressesMessage (typed '" << kGetPeerDomainNetworkAddressesMessage
- << "') to BlockLocator";
+ DLOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent GetPeerDomainNetworkAddressesMessage (typed '" << kGetPeerDomainNetworkAddressesMessage
+ << "') to BlockLocator";
DCHECK_NE(block_locator_client_id_, tmb::kClientIdNone);
DCHECK(bus_ != nullptr);
@@ -573,8 +573,8 @@ vector<string> StorageManager::getPeerDomainNetworkAddresses(const block_id bloc
const TaggedMessage &tagged_message = annotated_message.tagged_message;
CHECK_EQ(block_locator_client_id_, annotated_message.sender);
CHECK_EQ(kGetPeerDomainNetworkAddressesResponseMessage, tagged_message.message_type());
- LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
- << "') received GetPeerDomainNetworkAddressesResponseMessage from BlockLocator";
+ DLOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') received GetPeerDomainNetworkAddressesResponseMessage from BlockLocator";
serialization::GetPeerDomainNetworkAddressesResponseMessage response_proto;
CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
@@ -591,10 +591,10 @@ void StorageManager::sendBlockLocationMessage(const block_id block,
const tmb::message_type_id message_type) {
switch (message_type) {
case kAddBlockLocationMessage:
- LOG(INFO) << "Loaded Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
+ DLOG(INFO) << "Loaded Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
break;
case kDeleteBlockLocationMessage:
- LOG(INFO) << "Evicted Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
+ DLOG(INFO) << "Evicted Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
break;
default:
LOG(FATAL) << "Unknown message type " << message_type;
@@ -613,9 +613,9 @@ void StorageManager::sendBlockLocationMessage(const block_id block,
message_type);
free(proto_bytes);
- LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
- << "') sent BlockLocationMessage (typed '" << message_type
- << "') to BlockLocator";
+ DLOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent BlockLocationMessage (typed '" << message_type
+ << "') to BlockLocator";
CHECK(MessageBus::SendStatus::kOK ==
QueryExecutionUtil::SendTMBMessage(bus_,
storage_manager_client_id_,
@@ -635,7 +635,7 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
// TODO(quickstep-team): Use a cost model to determine whether to load from
// a remote peer or the disk.
if (BlockIdUtil::Domain(block) != block_domain_) {
- LOG(INFO) << "Pulling Block " << BlockIdUtil::ToString(block) << " from a remote peer";
+ DLOG(INFO) << "Pulling Block " << BlockIdUtil::ToString(block) << " from a remote peer";
const vector<string> peer_domain_network_addresses = getPeerDomainNetworkAddresses(block);
for (const string &peer_domain_network_address : peer_domain_network_addresses) {
DataExchangerClientAsync client(
@@ -648,8 +648,8 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
}
}
- LOG(INFO) << "Failed to pull Block " << BlockIdUtil::ToString(block)
- << " from remote peers, so try to load from disk.";
+ DLOG(INFO) << "Failed to pull Block " << BlockIdUtil::ToString(block)
+ << " from remote peers, so try to load from disk.";
}
#endif
[28/39] incubator-quickstep git commit: A better debug log for Worker.
Posted by ji...@apache.org.
A better debug log for Worker.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4286d75e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4286d75e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4286d75e
Branch: refs/heads/min-max-stats
Commit: 4286d75ec622ac6e60813c6892b131660ce3a931
Parents: a31dde4
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 27 11:45:28 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 27 11:45:28 2016 -0800
----------------------------------------------------------------------
query_execution/Worker.cpp | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4286d75e/query_execution/Worker.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Worker.cpp b/query_execution/Worker.cpp
index 31eb964..9a548fd 100644
--- a/query_execution/Worker.cpp
+++ b/query_execution/Worker.cpp
@@ -107,7 +107,14 @@ void Worker::sendWorkOrderCompleteMessage(const tmb::client_id receiver,
static_cast<const void *>(proto_bytes), proto_length, message_type);
std::free(proto_bytes);
- DLOG(INFO) << "Worker sent WorkOrderCompleteMessage (typed '" << message_type
+ DLOG(INFO) << "Worker " << worker_thread_index_
+#ifdef QUICKSTEP_DISTRIBUTED
+ << " in Shiftboss " << shiftboss_index_
+#endif // QUICKSTEP_DISTRIBUTED
+ << " sent "
+ << (message_type == kWorkOrderCompleteMessage ? "WorkOrderCompleteMessage"
+ : "RebuildWorkOrderCompleteMessage")
+ << " (typed '" << message_type
<< "') to Scheduler with TMB client ID " << receiver;
const tmb::MessageBus::SendStatus send_status =
QueryExecutionUtil::SendTMBMessage(
[11/39] incubator-quickstep git commit: Maintained the catalog
filename as a constant.
Posted by ji...@apache.org.
Maintained the catalog filename as a constant.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/37a78cb1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/37a78cb1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/37a78cb1
Branch: refs/heads/min-max-stats
Commit: 37a78cb112da2df57973fcef054dba77bd9d9ee1
Parents: 24367d7
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Nov 19 11:59:13 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:27:54 2016 -0800
----------------------------------------------------------------------
BUILDING.md | 8 ++------
cli/QuickstepCli.cpp | 7 ++++---
storage/StorageConstants.hpp | 2 ++
3 files changed, 8 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/37a78cb1/BUILDING.md
----------------------------------------------------------------------
diff --git a/BUILDING.md b/BUILDING.md
index 581fbdd..97552c6 100644
--- a/BUILDING.md
+++ b/BUILDING.md
@@ -190,16 +190,12 @@ number of jobs).
Running Quickstep
-----------------
-To use quickstep, just run `quickstep_cli_shell` in the build directory.
+To use quickstep, just run `quickstep_cli_shell` in the build directory. For the
+first time user, run once with `-initialize_db=true` to set up an empty catalog.
Quickstep has number of command-line flags that control its behavior. Run
`quickstep_cli_shell --help` to see a listing of the options and how to use
them.
-The provided build directory has a skeleton qsstor directory with a fresh
-catalog.pb.bin file which is needed to run `quickstep_cli_shell`. If you want
-to wipe out your database and start fresh, delete all the files in qsstor and
-copy `catalog.pb.bin.orig` to `qsstor/catalog.pb.bin`.
-
Running Tests
-------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/37a78cb1/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index f17e18b..0e8468e 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -125,6 +125,7 @@ using quickstep::Worker;
using quickstep::WorkerDirectory;
using quickstep::WorkerMessage;
using quickstep::kAdmitRequestMessage;
+using quickstep::kCatalogFilename;
using quickstep::kPoisonMessage;
using quickstep::kWorkloadCompletionMessage;
@@ -243,7 +244,7 @@ int main(int argc, char* argv[]) {
quickstep::StorageManager storage_manager(fixed_storage_path);
string catalog_path(fixed_storage_path);
- catalog_path.append("catalog.pb.bin");
+ catalog_path.append(kCatalogFilename);
if (quickstep::FLAGS_initialize_db) { // Initialize the database
// TODO(jmp): Refactor the code in this file!
LOG(INFO) << "Initializing the database, creating a new catalog file and storage directory\n";
@@ -267,14 +268,14 @@ int main(int argc, char* argv[]) {
// Create the default catalog file.
std::ofstream catalog_file(catalog_path);
if (!catalog_file.good()) {
- LOG(FATAL) << "ERROR: Unable to open catalog.pb.bin for writing.\n";
+ LOG(FATAL) << "ERROR: Unable to open " << kCatalogFilename << " for writing.\n";
}
quickstep::Catalog catalog;
catalog.addDatabase(new quickstep::CatalogDatabase(nullptr, "default"));
if (!catalog.getProto().SerializeToOstream(&catalog_file)) {
- LOG(FATAL) << "ERROR: Unable to serialize catalog proto to file catalog.pb.bin\n";
+ LOG(FATAL) << "ERROR: Unable to serialize catalog proto to file " << kCatalogFilename;
return 1;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/37a78cb1/storage/StorageConstants.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageConstants.hpp b/storage/StorageConstants.hpp
index 34a714d..037a8a9 100644
--- a/storage/StorageConstants.hpp
+++ b/storage/StorageConstants.hpp
@@ -39,6 +39,8 @@ constexpr char kPathSeparator = '/';
constexpr char kDefaultStoragePath[] = "qsstor/";
#endif
+constexpr char kCatalogFilename[] = "catalog.pb.bin";
+
// Size of a memory slot managed by the StorageManager. This is the smallest
// quantum of allocation for StorageBlocks and StorageBlobs. 2 MB is the large
// page size on x86.
[21/39] incubator-quickstep git commit: IWYU for DefaultsConfigurator.
Posted by ji...@apache.org.
IWYU for DefaultsConfigurator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/54e0654b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/54e0654b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/54e0654b
Branch: refs/heads/min-max-stats
Commit: 54e0654bfb9a42553129428fa94f78535e2781f4
Parents: e666a81
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 20 23:30:42 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 23:30:42 2016 -0800
----------------------------------------------------------------------
cli/DefaultsConfigurator.hpp | 28 +++++++++++++++++-----------
1 file changed, 17 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/54e0654b/cli/DefaultsConfigurator.hpp
----------------------------------------------------------------------
diff --git a/cli/DefaultsConfigurator.hpp b/cli/DefaultsConfigurator.hpp
index 7450d88..4fd62ac 100644
--- a/cli/DefaultsConfigurator.hpp
+++ b/cli/DefaultsConfigurator.hpp
@@ -20,17 +20,23 @@
#ifndef QUICKSTEP_CLI_DEFAULTS_CONFIGURATOR_HPP_
#define QUICKSTEP_CLI_DEFAULTS_CONFIGURATOR_HPP_
+#include "storage/StorageConfig.h" // For QUICKSTEP_HAVE_LIBNUMA.
+
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include <numa.h>
+#endif // QUICKSTEP_HAVE_LIBNUMA
+
+#include <cstddef>
#include <thread> // NOLINT(build/c++11)
+
+#ifdef QUICKSTEP_HAVE_LIBNUMA
#include <unordered_map>
+#endif // QUICKSTEP_HAVE_LIBNUMA
+
#include <vector>
-#include "storage/StorageConfig.h"
#include "utility/Macros.hpp"
-#ifdef QUICKSTEP_HAVE_LIBNUMA
-#include <numa.h>
-#endif
-
namespace quickstep {
/** \addtogroup CLI
@@ -60,12 +66,12 @@ class DefaultsConfigurator {
* have libnuma.
**/
static std::size_t GetNumNUMANodes() {
- #ifdef QUICKSTEP_HAVE_LIBNUMA
+#ifdef QUICKSTEP_HAVE_LIBNUMA
// Id of the maximum node.
return numa_max_node() + 1;
- #else
+#else
return 1;
- #endif
+#endif // QUICKSTEP_HAVE_LIBNUMA
}
/**
@@ -80,7 +86,7 @@ class DefaultsConfigurator {
**/
static std::size_t GetNumNUMANodesCoveredByWorkers(const std::vector<int> &worker_cpu_affinities) {
if (!worker_cpu_affinities.empty()) {
- #ifdef QUICKSTEP_HAVE_LIBNUMA
+#ifdef QUICKSTEP_HAVE_LIBNUMA
// Key = NUMA node, value = whether there is at least one worker whose
// affinity is set to a core on the given NUMA node.
std::unordered_map<int, bool> any_worker_on_numa_node;
@@ -93,7 +99,7 @@ class DefaultsConfigurator {
}
}
return any_worker_on_numa_node.size();
- #endif
+#endif // QUICKSTEP_HAVE_LIBNUMA
}
// When libnuma is not available, or worker affinities are not specified,
// the default return value is 1.
@@ -108,9 +114,9 @@ class DefaultsConfigurator {
DISALLOW_COPY_AND_ASSIGN(DefaultsConfigurator);
};
+
/** @} */
} // namespace quickstep
#endif // QUICKSTEP_CLI_DEFAULTS_CONFIGURATOR_HPP_
-
[35/39] incubator-quickstep git commit: Marked SingleNodeQuery for
Insertions.
Posted by ji...@apache.org.
Marked SingleNodeQuery for Insertions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0859a17a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0859a17a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0859a17a
Branch: refs/heads/min-max-stats
Commit: 0859a17aa4e71ef8d3d261f15e52518b39f617f6
Parents: e50a2b7
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Dec 4 14:11:58 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Dec 4 15:44:24 2016 -0800
----------------------------------------------------------------------
query_execution/ForemanDistributed.cpp | 16 +++++++---
query_execution/PolicyEnforcerDistributed.hpp | 23 +++++++++++---
query_optimizer/ExecutionGenerator.cpp | 3 ++
query_optimizer/QueryHandle.hpp | 37 ++++++++++++++++++++++
4 files changed, 71 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0859a17a/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index 61f0603..0dad8b0 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -295,14 +295,22 @@ bool ForemanDistributed::isHashJoinRelatedWorkOrder(const S::WorkOrderMessage &p
return true;
}
+namespace {
+constexpr size_t kDefaultShiftbossIndex = 0u;
+} // namespace
+
void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::WorkOrderMessage>> &messages) {
- const size_t num_shiftbosses = shiftboss_directory_.size();
- size_t shiftboss_index = 0u;
+ static size_t shiftboss_index = kDefaultShiftbossIndex;
+
+ PolicyEnforcerDistributed* policy_enforcer_dist = static_cast<PolicyEnforcerDistributed*>(policy_enforcer_.get());
for (const auto &message : messages) {
DCHECK(message != nullptr);
const S::WorkOrderMessage &proto = *message;
size_t shiftboss_index_for_particular_work_order_type;
- if (isAggregationRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
+ if (policy_enforcer_dist->isSingleNodeQuery(proto.query_id())) {
+ // Always schedule the single-node query to the same Shiftboss.
+ shiftboss_index_for_particular_work_order_type = kDefaultShiftbossIndex;
+ } else if (isAggregationRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
} else if (isHashJoinRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
} else {
// TODO(zuyu): Take data-locality into account for scheduling.
@@ -313,7 +321,7 @@ void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::Wo
shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index_for_particular_work_order_type);
if (shiftboss_index == shiftboss_index_for_particular_work_order_type) {
- shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
+ shiftboss_index = (shiftboss_index + 1) % shiftboss_directory_.size();
} else {
// NOTE(zuyu): This is not the exact round-robin scheduling, as in this case,
// <shiftboss_index_for_particular_work_order_type> might be scheduled one
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0859a17a/query_execution/PolicyEnforcerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.hpp b/query_execution/PolicyEnforcerDistributed.hpp
index e8bc394..2c00a6b 100644
--- a/query_execution/PolicyEnforcerDistributed.hpp
+++ b/query_execution/PolicyEnforcerDistributed.hpp
@@ -17,14 +17,20 @@
#include <cstddef>
#include <memory>
+#include <unordered_map>
+#include <utility>
#include <vector>
#include "query_execution/PolicyEnforcerBase.hpp"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryManagerBase.hpp"
#include "query_execution/ShiftbossDirectory.hpp"
+#include "query_optimizer/QueryHandle.hpp"
#include "utility/Macros.hpp"
+#include "glog/logging.h"
+
#include "tmb/id_typedefs.h"
namespace tmb {
@@ -35,10 +41,6 @@ class TaggedMessage;
namespace quickstep {
class CatalogDatabaseLite;
-class QueryHandle;
-class QueryManagerBase;
-
-namespace serialization { class WorkOrderMessage; }
/** \addtogroup QueryExecution
* @{
@@ -90,6 +92,19 @@ class PolicyEnforcerDistributed final : public PolicyEnforcerBase {
void processInitiateRebuildResponseMessage(const tmb::TaggedMessage &tagged_message);
/**
+ * @brief Whether the query should be executed on one Shiftboss.
+ *
+ * @param query_id The query id.
+ *
+ * @return Whether the query should be executed on one Shiftboss.
+ **/
+ bool isSingleNodeQuery(const std::size_t query_id) const {
+ const auto cit = admitted_queries_.find(query_id);
+ DCHECK(cit != admitted_queries_.end());
+ return cit->second->query_handle()->is_single_node_query();
+ }
+
+ /**
* @brief Get or set the index of Shiftboss for an Aggregation related
* WorkOrder. If it is the first Aggregation on <aggr_state_index>,
* <shiftboss_index> will be set to <next_shiftboss_index_to_schedule>.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0859a17a/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 2e0d8f3..5a2c450 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1096,6 +1096,9 @@ void ExecutionGenerator::convertDropTable(
void ExecutionGenerator::convertInsertTuple(
const P::InsertTuplePtr &physical_plan) {
// InsertTuple is converted to an Insert and a SaveBlocks.
+#ifdef QUICKSTEP_DISTRIBUTED
+ query_handle_->set_is_single_node_query();
+#endif // QUICKSTEP_DISTRIBUTED
const CatalogRelationInfo *input_relation_info =
findRelationInfoOutputByPhysical(physical_plan->input());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0859a17a/query_optimizer/QueryHandle.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryHandle.hpp b/query_optimizer/QueryHandle.hpp
index 1ca6021..cbd1cd9 100644
--- a/query_optimizer/QueryHandle.hpp
+++ b/query_optimizer/QueryHandle.hpp
@@ -26,6 +26,7 @@
#include "catalog/Catalog.pb.h"
#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED.
#include "query_optimizer/QueryPlan.hpp"
#include "utility/Macros.hpp"
@@ -134,6 +135,22 @@ class QueryHandle {
query_result_relation_ = relation;
}
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Whether the query will be executed in the single node.
+ */
+ bool is_single_node_query() const {
+ return is_single_node_query_;
+ }
+
+ /**
+ * @brief Set the query to be executed in the single node.
+ */
+ void set_is_single_node_query() {
+ is_single_node_query_ = true;
+ }
+#endif // QUICKSTEP_DISTRIBUTED
+
private:
const std::size_t query_id_;
@@ -153,6 +170,26 @@ class QueryHandle {
// and deleted by the Cli shell.
const CatalogRelation *query_result_relation_;
+#ifdef QUICKSTEP_DISTRIBUTED
+ // Indicate whether the query should be executed on the default Shiftboss for
+ // correctness purpose.
+ // An example would be the insert query that might otherwise need block
+ // invalidation among multiple StorageManagers. In this case, an insert query
+ // has scheduled on node 0, and the block is in the buffer pool of node 0.
+ // Another insert query on the same relation might be scheduled on another
+ // node, say node 1, which will pull the block from node 0, and do the
+ // insertion. Thus, two blocks with the same block id in two nodes
+ // have different contents, which is incorrect.
+ // One approach is to evict blocks cached in all other nodes for every
+ // change. It, however, does not scale, and even worse, it will also affect
+ // the performance of each select query.
+ // Alternatively, we choose to mark the query as a single-node query to
+ // modify blocks on the default node only. But if the changed block has also
+ // cached in another node, this approach would still produce inconsistent
+ // query result.
+ bool is_single_node_query_ = false;
+#endif // QUICKSTEP_DISTRIBUTED
+
DISALLOW_COPY_AND_ASSIGN(QueryHandle);
};
[13/39] incubator-quickstep git commit: Refactored
printWorkOrderProfilingResults in Foreman.
Posted by ji...@apache.org.
Refactored printWorkOrderProfilingResults in Foreman.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/7fb96f3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/7fb96f3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/7fb96f3d
Branch: refs/heads/min-max-stats
Commit: 7fb96f3da140f4631060132b58ded2fd38191167
Parents: 76e37ca
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Nov 18 22:07:27 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:41:16 2016 -0800
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 3 +++
query_execution/ForemanBase.hpp | 33 +++++++++++++++++++++++++++++
query_execution/ForemanDistributed.cpp | 11 ++++++----
query_execution/ForemanDistributed.hpp | 14 +-----------
query_execution/ForemanSingleNode.cpp | 17 ++++++++-------
query_execution/ForemanSingleNode.hpp | 29 +------------------------
query_execution/PolicyEnforcerBase.hpp | 18 ++++++++--------
7 files changed, 63 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 0f6c282..cf9d5b0 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -86,6 +86,7 @@ if (ENABLE_DISTRIBUTED)
endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution_ForemanBase
glog
+ quickstep_queryexecution_PolicyEnforcerBase
quickstep_threading_Thread
quickstep_utility_Macros
tmb)
@@ -98,6 +99,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_catalog_Catalog_proto
quickstep_queryexecution_AdmitRequestMessage
quickstep_queryexecution_ForemanBase
+ quickstep_queryexecution_PolicyEnforcerBase
quickstep_queryexecution_PolicyEnforcerDistributed
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
@@ -113,6 +115,7 @@ target_link_libraries(quickstep_queryexecution_ForemanSingleNode
glog
quickstep_queryexecution_AdmitRequestMessage
quickstep_queryexecution_ForemanBase
+ quickstep_queryexecution_PolicyEnforcerBase
quickstep_queryexecution_PolicyEnforcerSingleNode
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/ForemanBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanBase.hpp b/query_execution/ForemanBase.hpp
index 951f34d..ee6c7ce 100644
--- a/query_execution/ForemanBase.hpp
+++ b/query_execution/ForemanBase.hpp
@@ -20,6 +20,11 @@
#ifndef QUICKSTEP_QUERY_EXECUTION_FOREMAN_BASE_HPP_
#define QUICKSTEP_QUERY_EXECUTION_FOREMAN_BASE_HPP_
+#include <cstdio>
+#include <memory>
+#include <vector>
+
+#include "query_execution/PolicyEnforcerBase.hpp"
#include "threading/Thread.hpp"
#include "utility/Macros.hpp"
@@ -30,6 +35,8 @@
namespace quickstep {
+struct WorkOrderTimeEntry;
+
/** \addtogroup QueryExecution
* @{
*/
@@ -59,6 +66,30 @@ class ForemanBase : public Thread {
~ForemanBase() override {}
/**
+ * @brief Print the results of profiling individual work orders for a given
+ * query.
+ *
+ * TODO(harshad) - Add the name of the operator to the output.
+ *
+ * @param query_id The ID of the query for which the results are to be printed.
+ * @param out The file stream.
+ **/
+ virtual void printWorkOrderProfilingResults(const std::size_t query_id,
+ std::FILE *out) const = 0;
+
+ /**
+ * @brief Get the results of profiling individual work orders for a given
+ * query.
+ *
+ * @param query_id The ID of the query for which the results are to be printed.
+ * @return A vector of records, each being a single profiling entry.
+ **/
+ const std::vector<WorkOrderTimeEntry>& getWorkOrderProfilingResults(
+ const std::size_t query_id) const {
+ return policy_enforcer_->getProfilingResults(query_id);
+ }
+
+ /**
* @brief Get the TMB client ID of Foreman thread.
*
* @return TMB client ID of foreman thread.
@@ -77,6 +108,8 @@ class ForemanBase : public Thread {
// The ID of the CPU that the Foreman thread can optionally be pinned to.
const int cpu_id_;
+ std::unique_ptr<PolicyEnforcerBase> policy_enforcer_;
+
private:
DISALLOW_COPY_AND_ASSIGN(ForemanBase);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index 56b319b..aa4db17 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -28,6 +28,7 @@
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/AdmitRequestMessage.hpp"
+#include "query_execution/PolicyEnforcerBase.hpp"
#include "query_execution/PolicyEnforcerDistributed.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionTypedefs.hpp"
@@ -98,12 +99,12 @@ ForemanDistributed::ForemanDistributed(
bus_->RegisterClientAsReceiver(foreman_client_id_, message_type);
}
- policy_enforcer_.reset(new PolicyEnforcerDistributed(
+ policy_enforcer_ = std::make_unique<PolicyEnforcerDistributed>(
foreman_client_id_,
catalog_database_,
&shiftboss_directory_,
bus_,
- profile_individual_workorders));
+ profile_individual_workorders);
}
void ForemanDistributed::run() {
@@ -180,7 +181,8 @@ void ForemanDistributed::run() {
}
case kInitiateRebuildResponseMessage: {
// A unique case in the distributed version.
- policy_enforcer_->processInitiateRebuildResponseMessage(tagged_message);
+ static_cast<PolicyEnforcerDistributed*>(policy_enforcer_.get())->
+ processInitiateRebuildResponseMessage(tagged_message);
break;
}
case kSaveQueryResultResponseMessage: {
@@ -228,7 +230,8 @@ void ForemanDistributed::run() {
if (canCollectNewMessages(message_type)) {
vector<unique_ptr<S::WorkOrderMessage>> new_messages;
- policy_enforcer_->getWorkOrderProtoMessages(&new_messages);
+ static_cast<PolicyEnforcerDistributed*>(policy_enforcer_.get())->
+ getWorkOrderProtoMessages(&new_messages);
dispatchWorkOrderMessages(new_messages);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/ForemanDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.hpp b/query_execution/ForemanDistributed.hpp
index b42795c..7f2d2f9 100644
--- a/query_execution/ForemanDistributed.hpp
+++ b/query_execution/ForemanDistributed.hpp
@@ -24,7 +24,6 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/ForemanBase.hpp"
-#include "query_execution/PolicyEnforcerDistributed.hpp"
#include "query_execution/ShiftbossDirectory.hpp"
#include "utility/Macros.hpp"
@@ -68,17 +67,8 @@ class ForemanDistributed final : public ForemanBase {
~ForemanDistributed() override {}
- /**
- * @brief Print the results of profiling individual work orders for a given
- * query.
- *
- * TODO(harshad) - Add the name of the operator to the output.
- *
- * @param query_id The ID of the query for which the results are to be printed.
- * @param out The file stream.
- **/
void printWorkOrderProfilingResults(const std::size_t query_id,
- std::FILE *out) const;
+ std::FILE *out) const override;
protected:
void run() override;
@@ -120,8 +110,6 @@ class ForemanDistributed final : public ForemanBase {
CatalogDatabaseLite *catalog_database_;
- std::unique_ptr<PolicyEnforcerDistributed> policy_enforcer_;
-
// From a query id to a set of Shiftbosses that save query result.
std::unordered_map<std::size_t, std::unordered_set<std::size_t>> query_result_saved_shiftbosses_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/ForemanSingleNode.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanSingleNode.cpp b/query_execution/ForemanSingleNode.cpp
index 4661c37..dfdfb71 100644
--- a/query_execution/ForemanSingleNode.cpp
+++ b/query_execution/ForemanSingleNode.cpp
@@ -26,6 +26,7 @@
#include <vector>
#include "query_execution/AdmitRequestMessage.hpp"
+#include "query_execution/PolicyEnforcerBase.hpp"
#include "query_execution/PolicyEnforcerSingleNode.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
@@ -92,14 +93,14 @@ ForemanSingleNode::ForemanSingleNode(
bus_->RegisterClientAsReceiver(foreman_client_id_, message_type);
}
- policy_enforcer_.reset(new PolicyEnforcerSingleNode(
+ policy_enforcer_ = std::make_unique<PolicyEnforcerSingleNode>(
foreman_client_id_,
num_numa_nodes,
catalog_database_,
storage_manager_,
worker_directory_,
bus_,
- profile_individual_workorders));
+ profile_individual_workorders);
}
void ForemanSingleNode::run() {
@@ -157,7 +158,8 @@ void ForemanSingleNode::run() {
if (canCollectNewMessages(message_type)) {
vector<unique_ptr<WorkerMessage>> new_messages;
- policy_enforcer_->getWorkerMessages(&new_messages);
+ static_cast<PolicyEnforcerSingleNode*>(policy_enforcer_.get())->
+ getWorkerMessages(&new_messages);
dispatchWorkerMessages(new_messages);
}
@@ -233,13 +235,12 @@ void ForemanSingleNode::sendWorkerMessage(const size_t worker_thread_index,
CHECK(send_status == tmb::MessageBus::SendStatus::kOK);
}
-const std::vector<WorkOrderTimeEntry>& ForemanSingleNode
- ::getWorkOrderProfilingResults(const std::size_t query_id) const {
- return policy_enforcer_->getProfilingResults(query_id);
-}
-
void ForemanSingleNode::printWorkOrderProfilingResults(const std::size_t query_id,
std::FILE *out) const {
+ // TODO(harshad) - Add the CPU core ID of the operator to the output. This
+ // will require modifying the WorkerDirectory to remember worker affinities.
+ // Until then, the users can refer to the worker_affinities provided to the
+ // cli to infer the CPU core ID where a given worker is pinned.
const std::vector<WorkOrderTimeEntry> &recorded_times =
policy_enforcer_->getProfilingResults(query_id);
fputs("Query ID,Worker ID,NUMA Socket,Operator ID,Time (microseconds)\n", out);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/ForemanSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanSingleNode.hpp b/query_execution/ForemanSingleNode.hpp
index 5a368aa..4cc7a63 100644
--- a/query_execution/ForemanSingleNode.hpp
+++ b/query_execution/ForemanSingleNode.hpp
@@ -26,7 +26,6 @@
#include <vector>
#include "query_execution/ForemanBase.hpp"
-#include "query_execution/PolicyEnforcerSingleNode.hpp"
#include "utility/Macros.hpp"
#include "tmb/id_typedefs.h"
@@ -78,32 +77,8 @@ class ForemanSingleNode final : public ForemanBase {
~ForemanSingleNode() override {}
-
- /**
- * @brief Get the results of profiling individual work orders for a given
- * query.
- *
- * @param query_id The ID of the query for which the results are to be printed.
- * @return A vector of records, each being a single profiling entry.
- **/
- const std::vector<WorkOrderTimeEntry>& getWorkOrderProfilingResults(
- const std::size_t query_id) const;
-
- /**
- * @brief Print the results of profiling individual work orders for a given
- * query.
- *
- * TODO(harshad) - Add the name of the operator to the output.
- * TODO(harshad) - Add the CPU core ID of the operator to the output. This
- * will require modifying the WorkerDirectory to remember worker affinities.
- * Until then, the users can refer to the worker_affinities provided to the
- * cli to infer the CPU core ID where a given worker is pinned.
- *
- * @param query_id The ID of the query for which the results are to be printed.
- * @param out The file stream.
- **/
void printWorkOrderProfilingResults(const std::size_t query_id,
- std::FILE *out) const;
+ std::FILE *out) const override;
protected:
void run() override;
@@ -142,8 +117,6 @@ class ForemanSingleNode final : public ForemanBase {
CatalogDatabaseLite *catalog_database_;
StorageManager *storage_manager_;
- std::unique_ptr<PolicyEnforcerSingleNode> policy_enforcer_;
-
DISALLOW_COPY_AND_ASSIGN(ForemanSingleNode);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/PolicyEnforcerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.hpp b/query_execution/PolicyEnforcerBase.hpp
index 414367e..aa070b8 100644
--- a/query_execution/PolicyEnforcerBase.hpp
+++ b/query_execution/PolicyEnforcerBase.hpp
@@ -146,6 +146,15 @@ class PolicyEnforcerBase {
return workorder_time_recorder_.at(query_id);
}
+ /**
+ * @brief Admit a query to the system.
+ *
+ * @param query_handle The QueryHandle for the new query.
+ *
+ * @return Whether the query was admitted to the system.
+ **/
+ virtual bool admitQuery(QueryHandle *query_handle) = 0;
+
protected:
static constexpr std::size_t kMaxConcurrentQueries = 1;
@@ -185,15 +194,6 @@ class PolicyEnforcerBase {
private:
/**
- * @brief Admit a query to the system.
- *
- * @param query_handle The QueryHandle for the new query.
- *
- * @return Whether the query was admitted to the system.
- **/
- virtual bool admitQuery(QueryHandle *query_handle) = 0;
-
- /**
* @brief Decrement the number of queued workorders for the given worker by 1.
*
* @param proto The completion message proto received after the WorkOrder
[05/39] incubator-quickstep git commit: Removed StorageManager from
QueryProcessor.
Posted by ji...@apache.org.
Removed StorageManager from QueryProcessor.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9a005f36
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9a005f36
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9a005f36
Branch: refs/heads/min-max-stats
Commit: 9a005f36944de0eebd1bf764124fc7876e62d934
Parents: ea5f28b
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Nov 14 21:09:28 2016 -0800
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Thu Nov 17 14:02:51 2016 -0600
----------------------------------------------------------------------
cli/CommandExecutor.cpp | 5 +++--
cli/QuickstepCli.cpp | 17 +++++++++--------
query_optimizer/CMakeLists.txt | 1 -
query_optimizer/QueryProcessor.hpp | 15 +--------------
4 files changed, 13 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9a005f36/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index fea4b74..4ab32de 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -260,7 +260,7 @@ inline std::vector<TypedValue> executeQueryForSingleRow(
// Drop the result relation.
DropRelation::Drop(*query_result_relation,
query_processor->getDefaultDatabase(),
- query_processor->getStorageManager());
+ storage_manager);
return values;
}
@@ -292,10 +292,10 @@ void executeAnalyze(const PtrVector<ParseString> *arguments,
const tmb::client_id main_thread_client_id,
const tmb::client_id foreman_client_id,
MessageBus *bus,
+ StorageManager *storage_manager,
QueryProcessor *query_processor,
FILE *out) {
const CatalogDatabase &database = *query_processor->getDefaultDatabase();
- StorageManager *storage_manager = query_processor->getStorageManager();
std::unique_ptr<SqlParserWrapper> parser_wrapper(new SqlParserWrapper());
std::vector<std::reference_wrapper<const CatalogRelation>> relations;
@@ -396,6 +396,7 @@ void executeCommand(const ParseStatement &statement,
main_thread_client_id,
foreman_client_id,
bus,
+ storage_manager,
query_processor, out);
} else {
THROW_SQL_ERROR_AT(command.command()) << "Invalid Command";
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9a005f36/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 31d9834..31215f6 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -240,6 +240,7 @@ int main(int argc, char* argv[]) {
&& (fixed_storage_path.back() != quickstep::kPathSeparator)) {
fixed_storage_path.push_back(quickstep::kPathSeparator);
}
+ quickstep::StorageManager storage_manager(fixed_storage_path);
string catalog_path(fixed_storage_path);
catalog_path.append("catalog.pb.bin");
@@ -281,10 +282,10 @@ int main(int argc, char* argv[]) {
catalog_file.close();
}
- // Setup QueryProcessor, including CatalogDatabase and StorageManager.
+ // Setup QueryProcessor, including CatalogDatabase.
std::unique_ptr<QueryProcessor> query_processor;
try {
- query_processor.reset(new QueryProcessor(catalog_path, fixed_storage_path));
+ query_processor = std::make_unique<QueryProcessor>(catalog_path);
} catch (const std::exception &e) {
LOG(FATAL) << "FATAL ERROR DURING STARTUP: "
<< e.what()
@@ -308,7 +309,7 @@ int main(int argc, char* argv[]) {
printf("Preloading the buffer pool ... ");
fflush(stdout);
quickstep::PreloaderThread preloader(*query_processor->getDefaultDatabase(),
- query_processor->getStorageManager(),
+ &storage_manager,
worker_cpu_affinities.front());
preloader.start();
@@ -357,7 +358,7 @@ int main(int argc, char* argv[]) {
&worker_directory,
&bus,
query_processor->getDefaultDatabase(),
- query_processor->getStorageManager(),
+ &storage_manager,
-1, // Don't pin the Foreman thread.
num_numa_nodes_system,
quickstep::FLAGS_profile_and_report_workorder_perf || quickstep::FLAGS_visualize_execution_dag);
@@ -412,7 +413,7 @@ int main(int argc, char* argv[]) {
main_thread_client_id,
foreman.getBusClientID(),
&bus,
- query_processor->getStorageManager(),
+ &storage_manager,
query_processor.get(),
stdout);
} catch (const quickstep::SqlError &sql_error) {
@@ -458,16 +459,16 @@ int main(int argc, char* argv[]) {
const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
if (query_result_relation) {
PrintToScreen::PrintRelation(*query_result_relation,
- query_processor->getStorageManager(),
+ &storage_manager,
stdout);
PrintToScreen::PrintOutputSize(
*query_result_relation,
- query_processor->getStorageManager(),
+ &storage_manager,
stdout);
DropRelation::Drop(*query_result_relation,
query_processor->getDefaultDatabase(),
- query_processor->getStorageManager());
+ &storage_manager);
}
query_processor->saveCatalog();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9a005f36/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 00d5163..10c52a1 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -234,7 +234,6 @@ target_link_libraries(quickstep_queryoptimizer_QueryProcessor
quickstep_catalog_Catalog_proto
quickstep_queryoptimizer_Optimizer
quickstep_queryoptimizer_OptimizerContext
- quickstep_storage_StorageManager
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_Validator
glog
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9a005f36/query_optimizer/QueryProcessor.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryProcessor.hpp b/query_optimizer/QueryProcessor.hpp
index 5d3818e..5e5c115 100644
--- a/query_optimizer/QueryProcessor.hpp
+++ b/query_optimizer/QueryProcessor.hpp
@@ -27,7 +27,6 @@
#include "catalog/Catalog.hpp"
#include "query_optimizer/Optimizer.hpp"
-#include "storage/StorageManager.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
@@ -130,13 +129,9 @@ class QueryProcessor {
* @brief Constructor.
*
* @param catalog_filename The file to read the serialized catalog from.
- * @param storage_path The filesystem directory where blocks are stored on
- * disk.
**/
- QueryProcessor(const std::string &catalog_filename,
- const std::string &storage_path)
+ explicit QueryProcessor(const std::string &catalog_filename)
: catalog_filename_(catalog_filename),
- storage_manager_(std::make_unique<StorageManager>(storage_path)),
catalog_altered_(false),
query_id_(0) {
loadCatalog();
@@ -185,13 +180,6 @@ class QueryProcessor {
return catalog_->getDatabaseByNameMutable("default");
}
- /**
- * @brief Get the StorageManager held by this QueryProcessor.
- **/
- StorageManager* getStorageManager() const {
- return storage_manager_.get();
- }
-
private:
void loadCatalog(); // If it exists, free catalog_ before calling this
@@ -200,7 +188,6 @@ class QueryProcessor {
std::string catalog_filename_;
std::unique_ptr<Catalog> catalog_;
- std::unique_ptr<StorageManager> storage_manager_;
bool catalog_altered_;
[14/39] incubator-quickstep git commit: Refactor regarding the
profile flag.
Posted by ji...@apache.org.
Refactor regarding the profile flag.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e7c664ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e7c664ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e7c664ba
Branch: refs/heads/min-max-stats
Commit: e7c664baa216fad0c7d04f08cad8398d6d9db2bd
Parents: 7fb96f3
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Nov 18 17:38:17 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:51:25 2016 -0800
----------------------------------------------------------------------
cli/QuickstepCli.cpp | 13 ++++---------
query_execution/CMakeLists.txt | 3 ++-
query_execution/ForemanDistributed.cpp | 6 ++----
query_execution/ForemanDistributed.hpp | 5 +----
query_execution/ForemanSingleNode.cpp | 6 ++----
query_execution/ForemanSingleNode.hpp | 5 +----
query_execution/PolicyEnforcerBase.cpp | 15 +++++++++++++++
query_execution/PolicyEnforcerBase.hpp | 6 +-----
query_execution/PolicyEnforcerDistributed.hpp | 5 ++---
query_execution/PolicyEnforcerSingleNode.hpp | 5 ++---
10 files changed, 32 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 0e8468e..a4e55d7 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -133,9 +133,6 @@ using tmb::client_id;
namespace quickstep {
-DEFINE_bool(profile_and_report_workorder_perf, false,
- "If true, Quickstep will record the exceution time of all the individual "
- "normal work orders and report it at the end of query execution.");
DEFINE_int32(num_workers, 0, "Number of worker threads. If this value is "
"specified and is greater than 0, then this "
"user-supplied value is used. Else (i.e. the"
@@ -181,10 +178,9 @@ DEFINE_string(profile_file_name, "",
// To put things in perspective, the first run is, in my experiments, about 5-10
// times more expensive than the average run. That means the query needs to be
// run at least a hundred times to make the impact of the first run small (< 5 %).
-DEFINE_bool(visualize_execution_dag, false,
- "If true, visualize the execution plan DAG into a graph in DOT "
- "format (DOT is a plain text graph description language) which is "
- "then printed via stderr.");
+
+DECLARE_bool(profile_and_report_workorder_perf);
+DECLARE_bool(visualize_execution_dag);
} // namespace quickstep
@@ -361,8 +357,7 @@ int main(int argc, char* argv[]) {
query_processor->getDefaultDatabase(),
&storage_manager,
-1, // Don't pin the Foreman thread.
- num_numa_nodes_system,
- quickstep::FLAGS_profile_and_report_workorder_perf || quickstep::FLAGS_visualize_execution_dag);
+ num_numa_nodes_system);
// Start the worker threads.
for (Worker &worker : workers) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index cf9d5b0..eec0029 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -138,7 +138,8 @@ target_link_libraries(quickstep_queryexecution_PolicyEnforcerBase
quickstep_relationaloperators_WorkOrder
quickstep_storage_StorageBlockInfo
quickstep_utility_Macros
- tmb)
+ tmb
+ ${GFLAGS_LIB_NAME})
if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution_PolicyEnforcerDistributed
glog
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index aa4db17..d619657 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -64,8 +64,7 @@ class QueryHandle;
ForemanDistributed::ForemanDistributed(
MessageBus *bus,
CatalogDatabaseLite *catalog_database,
- const int cpu_id,
- const bool profile_individual_workorders)
+ const int cpu_id)
: ForemanBase(bus, cpu_id),
catalog_database_(DCHECK_NOTNULL(catalog_database)) {
const std::vector<QueryExecutionMessageType> sender_message_types{
@@ -103,8 +102,7 @@ ForemanDistributed::ForemanDistributed(
foreman_client_id_,
catalog_database_,
&shiftboss_directory_,
- bus_,
- profile_individual_workorders);
+ bus_);
}
void ForemanDistributed::run() {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/ForemanDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.hpp b/query_execution/ForemanDistributed.hpp
index 7f2d2f9..ccdd0ae 100644
--- a/query_execution/ForemanDistributed.hpp
+++ b/query_execution/ForemanDistributed.hpp
@@ -54,16 +54,13 @@ class ForemanDistributed final : public ForemanBase {
* @param bus A pointer to the TMB.
* @param catalog_database The catalog database where this query is executed.
* @param cpu_id The ID of the CPU to which the Foreman thread can be pinned.
- * @param profile_individual_workorders Whether every workorder's execution
- * be profiled or not.
*
* @note If cpu_id is not specified, Foreman thread can be possibly moved
* around on different CPUs by the OS.
**/
ForemanDistributed(tmb::MessageBus *bus,
CatalogDatabaseLite *catalog_database,
- const int cpu_id = -1,
- const bool profile_individual_workorders = false);
+ const int cpu_id = -1);
~ForemanDistributed() override {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/ForemanSingleNode.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanSingleNode.cpp b/query_execution/ForemanSingleNode.cpp
index dfdfb71..02799c7 100644
--- a/query_execution/ForemanSingleNode.cpp
+++ b/query_execution/ForemanSingleNode.cpp
@@ -63,8 +63,7 @@ ForemanSingleNode::ForemanSingleNode(
CatalogDatabaseLite *catalog_database,
StorageManager *storage_manager,
const int cpu_id,
- const size_t num_numa_nodes,
- const bool profile_individual_workorders)
+ const size_t num_numa_nodes)
: ForemanBase(bus, cpu_id),
main_thread_client_id_(main_thread_client_id),
worker_directory_(DCHECK_NOTNULL(worker_directory)),
@@ -99,8 +98,7 @@ ForemanSingleNode::ForemanSingleNode(
catalog_database_,
storage_manager_,
worker_directory_,
- bus_,
- profile_individual_workorders);
+ bus_);
}
void ForemanSingleNode::run() {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/ForemanSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanSingleNode.hpp b/query_execution/ForemanSingleNode.hpp
index 4cc7a63..d2db51b 100644
--- a/query_execution/ForemanSingleNode.hpp
+++ b/query_execution/ForemanSingleNode.hpp
@@ -60,8 +60,6 @@ class ForemanSingleNode final : public ForemanBase {
* @param storage_manager The StorageManager to use.
* @param cpu_id The ID of the CPU to which the Foreman thread can be pinned.
* @param num_numa_nodes The number of NUMA nodes in the system.
- * @param profile_individual_workorders Whether every workorder's execution
- * be profiled or not.
*
* @note If cpu_id is not specified, Foreman thread can be possibly moved
* around on different CPUs by the OS.
@@ -72,8 +70,7 @@ class ForemanSingleNode final : public ForemanBase {
CatalogDatabaseLite *catalog_database,
StorageManager *storage_manager,
const int cpu_id = -1,
- const std::size_t num_numa_nodes = 1,
- const bool profile_individual_workorders = false);
+ const std::size_t num_numa_nodes = 1);
~ForemanSingleNode() override {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/PolicyEnforcerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.cpp b/query_execution/PolicyEnforcerBase.cpp
index 6e922a8..b799d5f 100644
--- a/query_execution/PolicyEnforcerBase.cpp
+++ b/query_execution/PolicyEnforcerBase.cpp
@@ -35,10 +35,25 @@
#include "relational_operators/WorkOrder.hpp"
#include "storage/StorageBlockInfo.hpp"
+#include "gflags/gflags.h"
#include "glog/logging.h"
namespace quickstep {
+DEFINE_bool(profile_and_report_workorder_perf, false,
+ "If true, Quickstep will record the exceution time of all the individual "
+ "normal work orders and report it at the end of query execution.");
+
+DEFINE_bool(visualize_execution_dag, false,
+ "If true, visualize the execution plan DAG into a graph in DOT "
+ "format (DOT is a plain text graph description language) which is "
+ "then printed via stderr.");
+
+PolicyEnforcerBase::PolicyEnforcerBase(CatalogDatabaseLite *catalog_database)
+ : catalog_database_(catalog_database),
+ profile_individual_workorders_(FLAGS_profile_and_report_workorder_perf || FLAGS_visualize_execution_dag) {
+}
+
void PolicyEnforcerBase::processMessage(const TaggedMessage &tagged_message) {
std::size_t query_id;
QueryManagerBase::dag_node_index op_index;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/PolicyEnforcerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.hpp b/query_execution/PolicyEnforcerBase.hpp
index aa070b8..baf9c68 100644
--- a/query_execution/PolicyEnforcerBase.hpp
+++ b/query_execution/PolicyEnforcerBase.hpp
@@ -54,12 +54,8 @@ class PolicyEnforcerBase {
* @brief Constructor.
*
* @param catalog_database The CatalogDatabase used.
- * @param profile_individual_workorders If true, profile each normal work order.
**/
- PolicyEnforcerBase(CatalogDatabaseLite *catalog_database,
- const bool profile_individual_workorders)
- : catalog_database_(catalog_database),
- profile_individual_workorders_(profile_individual_workorders) {}
+ explicit PolicyEnforcerBase(CatalogDatabaseLite *catalog_database);
/**
* @brief Virtual Destructor.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/PolicyEnforcerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.hpp b/query_execution/PolicyEnforcerDistributed.hpp
index 0bf249c..146e9af 100644
--- a/query_execution/PolicyEnforcerDistributed.hpp
+++ b/query_execution/PolicyEnforcerDistributed.hpp
@@ -59,9 +59,8 @@ class PolicyEnforcerDistributed final : public PolicyEnforcerBase {
PolicyEnforcerDistributed(const tmb::client_id foreman_client_id,
CatalogDatabaseLite *catalog_database,
ShiftbossDirectory *shiftboss_directory,
- tmb::MessageBus *bus,
- const bool profile_individual_workorders = false)
- : PolicyEnforcerBase(catalog_database, profile_individual_workorders),
+ tmb::MessageBus *bus)
+ : PolicyEnforcerBase(catalog_database),
foreman_client_id_(foreman_client_id),
shiftboss_directory_(shiftboss_directory),
bus_(bus) {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/PolicyEnforcerSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerSingleNode.hpp b/query_execution/PolicyEnforcerSingleNode.hpp
index 16c7a0d..f87d670 100644
--- a/query_execution/PolicyEnforcerSingleNode.hpp
+++ b/query_execution/PolicyEnforcerSingleNode.hpp
@@ -64,9 +64,8 @@ class PolicyEnforcerSingleNode final : public PolicyEnforcerBase {
CatalogDatabaseLite *catalog_database,
StorageManager *storage_manager,
WorkerDirectory *worker_directory,
- tmb::MessageBus *bus,
- const bool profile_individual_workorders = false)
- : PolicyEnforcerBase(catalog_database, profile_individual_workorders),
+ tmb::MessageBus *bus)
+ : PolicyEnforcerBase(catalog_database),
foreman_client_id_(foreman_client_id),
num_numa_nodes_(num_numa_nodes),
storage_manager_(storage_manager),
[12/39] incubator-quickstep git commit: Refactored
PolicyEnforcerBase::decrementNumQueuedWorkOrders.
Posted by ji...@apache.org.
Refactored PolicyEnforcerBase::decrementNumQueuedWorkOrders.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/76e37cab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/76e37cab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/76e37cab
Branch: refs/heads/min-max-stats
Commit: 76e37cabcf24ff4fec7e50d5bd2b590fb9fe93d2
Parents: 37a78cb
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Nov 18 12:20:12 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:29:11 2016 -0800
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 1 +
query_execution/PolicyEnforcerBase.cpp | 4 ++--
query_execution/PolicyEnforcerBase.hpp | 6 ++++--
query_execution/PolicyEnforcerDistributed.hpp | 5 +++--
query_execution/PolicyEnforcerSingleNode.cpp | 4 ----
query_execution/PolicyEnforcerSingleNode.hpp | 7 +++++--
query_execution/QueryExecutionMessages.proto | 3 +++
query_execution/QueryExecutionTypedefs.hpp | 3 ++-
query_execution/Shiftboss.cpp | 8 +++++++-
query_execution/Shiftboss.hpp | 1 +
query_execution/Worker.cpp | 14 ++++++++++++++
query_execution/Worker.hpp | 9 +++++++++
12 files changed, 51 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index b5e07df..0f6c282 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -160,6 +160,7 @@ target_link_libraries(quickstep_queryexecution_PolicyEnforcerSingleNode
glog
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_PolicyEnforcerBase
+ quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionState
quickstep_queryexecution_QueryManagerBase
quickstep_queryexecution_QueryManagerSingleNode
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/PolicyEnforcerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.cpp b/query_execution/PolicyEnforcerBase.cpp
index 4e8c782..6e922a8 100644
--- a/query_execution/PolicyEnforcerBase.cpp
+++ b/query_execution/PolicyEnforcerBase.cpp
@@ -50,7 +50,7 @@ void PolicyEnforcerBase::processMessage(const TaggedMessage &tagged_message) {
// WorkOrder. It can be accessed in this scope.
CHECK(proto.ParseFromArray(tagged_message.message(),
tagged_message.message_bytes()));
- decrementNumQueuedWorkOrders(proto.worker_thread_index());
+ decrementNumQueuedWorkOrders(proto);
if (profile_individual_workorders_) {
recordTimeForWorkOrder(proto);
@@ -69,7 +69,7 @@ void PolicyEnforcerBase::processMessage(const TaggedMessage &tagged_message) {
// rebuild WorkOrder. It can be accessed in this scope.
CHECK(proto.ParseFromArray(tagged_message.message(),
tagged_message.message_bytes()));
- decrementNumQueuedWorkOrders(proto.worker_thread_index());
+ decrementNumQueuedWorkOrders(proto);
query_id = proto.query_id();
DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/PolicyEnforcerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.hpp b/query_execution/PolicyEnforcerBase.hpp
index 7009a0a..414367e 100644
--- a/query_execution/PolicyEnforcerBase.hpp
+++ b/query_execution/PolicyEnforcerBase.hpp
@@ -196,9 +196,11 @@ class PolicyEnforcerBase {
/**
* @brief Decrement the number of queued workorders for the given worker by 1.
*
- * @param worker_index The logical ID of the given worker.
+ * @param proto The completion message proto received after the WorkOrder
+ * execution.
**/
- virtual void decrementNumQueuedWorkOrders(const std::size_t worker_index) = 0;
+ virtual void decrementNumQueuedWorkOrders(
+ const serialization::WorkOrderCompletionMessage &proto) = 0;
DISALLOW_COPY_AND_ASSIGN(PolicyEnforcerBase);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/PolicyEnforcerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.hpp b/query_execution/PolicyEnforcerDistributed.hpp
index bce3e0c..0bf249c 100644
--- a/query_execution/PolicyEnforcerDistributed.hpp
+++ b/query_execution/PolicyEnforcerDistributed.hpp
@@ -20,6 +20,7 @@
#include <vector>
#include "query_execution/PolicyEnforcerBase.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/ShiftbossDirectory.hpp"
#include "utility/Macros.hpp"
@@ -89,8 +90,8 @@ class PolicyEnforcerDistributed final : public PolicyEnforcerBase {
void processInitiateRebuildResponseMessage(const tmb::TaggedMessage &tagged_message);
private:
- void decrementNumQueuedWorkOrders(const std::size_t shiftboss_index) override {
- shiftboss_directory_->decrementNumQueuedWorkOrders(shiftboss_index);
+ void decrementNumQueuedWorkOrders(const serialization::WorkOrderCompletionMessage &proto) override {
+ shiftboss_directory_->decrementNumQueuedWorkOrders(proto.shiftboss_index());
}
void onQueryCompletion(QueryManagerBase *query_manager) override;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/PolicyEnforcerSingleNode.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerSingleNode.cpp b/query_execution/PolicyEnforcerSingleNode.cpp
index 0bdb9b1..0aa2ca8 100644
--- a/query_execution/PolicyEnforcerSingleNode.cpp
+++ b/query_execution/PolicyEnforcerSingleNode.cpp
@@ -108,8 +108,4 @@ bool PolicyEnforcerSingleNode::admitQuery(QueryHandle *query_handle) {
}
}
-void PolicyEnforcerSingleNode::decrementNumQueuedWorkOrders(const std::size_t worker_index) {
- worker_directory_->decrementNumQueuedWorkOrders(worker_index);
-}
-
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/PolicyEnforcerSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerSingleNode.hpp b/query_execution/PolicyEnforcerSingleNode.hpp
index 870df95..16c7a0d 100644
--- a/query_execution/PolicyEnforcerSingleNode.hpp
+++ b/query_execution/PolicyEnforcerSingleNode.hpp
@@ -25,6 +25,8 @@
#include <vector>
#include "query_execution/PolicyEnforcerBase.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/WorkerDirectory.hpp"
#include "utility/Macros.hpp"
#include "tmb/id_typedefs.h"
@@ -36,7 +38,6 @@ namespace quickstep {
class CatalogDatabaseLite;
class QueryHandle;
class StorageManager;
-class WorkerDirectory;
class WorkerMessage;
/** \addtogroup QueryExecution
@@ -89,7 +90,9 @@ class PolicyEnforcerSingleNode final : public PolicyEnforcerBase {
std::vector<std::unique_ptr<WorkerMessage>> *worker_messages);
private:
- void decrementNumQueuedWorkOrders(const std::size_t worker_index) override;
+ void decrementNumQueuedWorkOrders(const serialization::WorkOrderCompletionMessage &proto) override {
+ worker_directory_->decrementNumQueuedWorkOrders(proto.worker_thread_index());
+ }
const tmb::client_id foreman_client_id_;
const std::size_t num_numa_nodes_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 165a194..e6d741a 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -46,6 +46,9 @@ message WorkOrderCompletionMessage {
// Epoch time in microseconds.
optional uint64 execution_start_time = 5;
optional uint64 execution_end_time = 6;
+
+ // Required in the distributed version.
+ optional uint64 shiftboss_index = 7;
}
message CatalogRelationNewBlockMessage {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionTypedefs.hpp b/query_execution/QueryExecutionTypedefs.hpp
index 22c0ae1..fb9a9d6 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -79,7 +79,8 @@ enum QueryExecutionMessageType : message_type_id {
#ifdef QUICKSTEP_DISTRIBUTED
kShiftbossRegistrationMessage, // From Shiftboss to Foreman.
- kShiftbossRegistrationResponseMessage, // From Foreman to Shiftboss.
+ kShiftbossRegistrationResponseMessage, // From Foreman to Shiftboss, or from
+ // Shiftboss to Worker.
kQueryInitiateMessage, // From Foreman to Shiftboss.
kQueryInitiateResponseMessage, // From Shiftboss to Foreman.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/Shiftboss.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index 30b2ae2..ed4bade 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -273,7 +273,7 @@ void Shiftboss::registerWithForeman() {
}
void Shiftboss::processShiftbossRegistrationResponseMessage() {
- const AnnotatedMessage annotated_message(bus_->Receive(shiftboss_client_id_, 0, true));
+ AnnotatedMessage annotated_message(bus_->Receive(shiftboss_client_id_, 0, true));
const TaggedMessage &tagged_message = annotated_message.tagged_message;
DCHECK_EQ(kShiftbossRegistrationResponseMessage, tagged_message.message_type());
@@ -286,6 +286,12 @@ void Shiftboss::processShiftbossRegistrationResponseMessage() {
CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
shiftboss_index_ = proto.shiftboss_index();
+
+ // Forward this message to Workers regarding <shiftboss_index_>.
+ QueryExecutionUtil::BroadcastMessage(shiftboss_client_id_,
+ worker_addresses_,
+ move(annotated_message.tagged_message),
+ bus_);
}
void Shiftboss::processQueryInitiateMessage(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/Shiftboss.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.hpp b/query_execution/Shiftboss.hpp
index 442e61e..6538d48 100644
--- a/query_execution/Shiftboss.hpp
+++ b/query_execution/Shiftboss.hpp
@@ -102,6 +102,7 @@ class Shiftboss : public Thread {
bus_->RegisterClientAsSender(shiftboss_client_id_, kSaveQueryResultResponseMessage);
// Message sent to Worker.
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kShiftbossRegistrationResponseMessage);
bus_->RegisterClientAsSender(shiftboss_client_id_, kRebuildWorkOrderMessage);
// Forward the following message types from Foreman to Workers.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/Worker.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Worker.cpp b/query_execution/Worker.cpp
index 0db17b4..31eb964 100644
--- a/query_execution/Worker.cpp
+++ b/query_execution/Worker.cpp
@@ -29,6 +29,7 @@
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_execution/WorkerMessage.hpp"
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED.
#include "relational_operators/WorkOrder.hpp"
#include "threading/ThreadIDBasedMap.hpp"
#include "threading/ThreadUtil.hpp"
@@ -62,6 +63,15 @@ void Worker::run() {
bus_->Receive(worker_client_id_, 0, true);
const TaggedMessage &tagged_message = annotated_msg.tagged_message;
switch (tagged_message.message_type()) {
+#ifdef QUICKSTEP_DISTRIBUTED
+ case kShiftbossRegistrationResponseMessage: {
+ serialization::ShiftbossRegistrationResponseMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ shiftboss_index_ = proto.shiftboss_index();
+ break;
+ }
+#endif // QUICKSTEP_DISTRIBUTED
case kWorkOrderMessage: {
WorkOrderCompletionMessage proto;
executeWorkOrderHelper(tagged_message, &proto);
@@ -136,6 +146,10 @@ void Worker::executeWorkOrderHelper(const TaggedMessage &tagged_message,
proto->set_worker_thread_index(worker_thread_index_);
proto->set_execution_start_time(execution_start_time);
proto->set_execution_end_time(execution_end_time);
+
+#ifdef QUICKSTEP_DISTRIBUTED
+ proto->set_shiftboss_index(shiftboss_index_);
+#endif // QUICKSTEP_DISTRIBUTED
}
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/Worker.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Worker.hpp b/query_execution/Worker.hpp
index fec2242..feb7c7d 100644
--- a/query_execution/Worker.hpp
+++ b/query_execution/Worker.hpp
@@ -24,6 +24,7 @@
#include <cstdint>
#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED.
#include "threading/Thread.hpp"
#include "utility/Macros.hpp"
@@ -75,6 +76,10 @@ class Worker : public Thread {
bus_->RegisterClientAsReceiver(worker_client_id_, kWorkOrderMessage);
bus_->RegisterClientAsReceiver(worker_client_id_, kRebuildWorkOrderMessage);
bus_->RegisterClientAsReceiver(worker_client_id_, kPoisonMessage);
+
+#ifdef QUICKSTEP_DISTRIBUTED
+ bus_->RegisterClientAsReceiver(worker_client_id_, kShiftbossRegistrationResponseMessage);
+#endif // QUICKSTEP_DISTRIBUTED
}
~Worker() override {}
@@ -132,6 +137,10 @@ class Worker : public Thread {
const int cpu_id_;
client_id worker_client_id_;
+#ifdef QUICKSTEP_DISTRIBUTED
+ std::size_t shiftboss_index_;
+#endif // QUICKSTEP_DISTRIBUTED
+
DISALLOW_COPY_AND_ASSIGN(Worker);
};
[20/39] incubator-quickstep git commit: Moved FLAGS_num_workers into
a file for shared flags.
Posted by ji...@apache.org.
Moved FLAGS_num_workers into a file for shared flags.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e666a81f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e666a81f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e666a81f
Branch: refs/heads/min-max-stats
Commit: e666a81f472e4c8ed81e98995c1914ce655e247a
Parents: 212e6cd
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 20 22:42:49 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 22:42:49 2016 -0800
----------------------------------------------------------------------
cli/CMakeLists.txt | 1 +
cli/Flags.cpp | 29 +++++++++++++++++++++++++++++
cli/Flags.hpp | 2 ++
cli/QuickstepCli.cpp | 31 +++++--------------------------
4 files changed, 37 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 9928f3f..9847787 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -112,6 +112,7 @@ if(QUICKSTEP_HAVE_LIBNUMA)
${LIBNUMA_LIBRARY})
endif()
target_link_libraries(quickstep_cli_Flags
+ quickstep_cli_DefaultsConfigurator
quickstep_storage_StorageConstants
${GFLAGS_LIB_NAME})
target_link_libraries(quickstep_cli_InputParserUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/Flags.cpp
----------------------------------------------------------------------
diff --git a/cli/Flags.cpp b/cli/Flags.cpp
index 384ef71..87f9f73 100644
--- a/cli/Flags.cpp
+++ b/cli/Flags.cpp
@@ -19,14 +19,43 @@
#include "cli/Flags.hpp"
+#include <cstddef>
+#include <cstdio>
#include <string>
+#include "cli/DefaultsConfigurator.hpp"
#include "storage/StorageConstants.hpp"
#include "gflags/gflags.h"
+using std::fprintf;
+
namespace quickstep {
+static bool ValidateNumWorkers(const char *flagname, int value) {
+ if (value > 0) {
+ return true;
+ }
+
+ // Detect the hardware concurrency level.
+ const std::size_t num_hw_threads =
+ DefaultsConfigurator::GetNumHardwareThreads();
+
+ // Use the command-line value if that was supplied, else use the value
+ // that we computed above, provided it did return a valid value.
+ // TODO(jmp): May need to change this at some point to keep one thread
+ // available for the OS if the hardware concurrency level is high.
+ FLAGS_num_workers = num_hw_threads != 0 ? num_hw_threads : 1;
+
+ return FLAGS_num_workers > 0;
+}
+DEFINE_int32(num_workers, 0, "Number of worker threads. If this value is "
+ "specified and is greater than 0, then this user-supplied value is "
+ "used. Else (i.e. the default case), we examine the reported "
+ "hardware concurrency level, and use that.");
+static const volatile bool num_workers_dummy
+ = gflags::RegisterFlagValidator(&FLAGS_num_workers, &ValidateNumWorkers);
+
static bool ValidateStoragePath(const char *flagname,
const std::string &value) {
if (!value.empty() && value.back() != kPathSeparator) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/Flags.hpp
----------------------------------------------------------------------
diff --git a/cli/Flags.hpp b/cli/Flags.hpp
index a623448..b020a3e 100644
--- a/cli/Flags.hpp
+++ b/cli/Flags.hpp
@@ -32,6 +32,8 @@ namespace quickstep {
* @brief A collection of common flags shared by Quickstep CLIs in both the
* single-node and the distributed version.
**/
+DECLARE_int32(num_workers);
+
DECLARE_string(storage_path);
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 33a9ff4..6a7c500 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -109,6 +109,7 @@ using quickstep::AdmitRequestMessage;
using quickstep::CatalogRelation;
using quickstep::DefaultsConfigurator;
using quickstep::DropRelation;
+using quickstep::FLAGS_num_workers;
using quickstep::FLAGS_storage_path;
using quickstep::ForemanSingleNode;
using quickstep::InputParserUtil;
@@ -136,11 +137,6 @@ using tmb::client_id;
namespace quickstep {
-DEFINE_int32(num_workers, 0, "Number of worker threads. If this value is "
- "specified and is greater than 0, then this "
- "user-supplied value is used. Else (i.e. the"
- "default case), we examine the reported "
- "hardware concurrency level, and use that.");
DEFINE_bool(preload_buffer_pool, false,
"If true, pre-load all known blocks into buffer pool before "
"accepting queries (should also set --buffer_pool_slots to be "
@@ -189,25 +185,8 @@ int main(int argc, char* argv[]) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
- // Detect the hardware concurrency level.
- const std::size_t num_hw_threads =
- DefaultsConfigurator::GetNumHardwareThreads();
-
- // Use the command-line value if that was supplied, else use the value
- // that we computed above, provided it did return a valid value.
- // TODO(jmp): May need to change this at some point to keep one thread
- // available for the OS if the hardware concurrency level is high.
- if (quickstep::FLAGS_num_workers <= 0) {
- LOG(INFO) << "Quickstep expects at least one worker thread, switching to "
- "the default number of worker threads";
- }
- const int real_num_workers = quickstep::FLAGS_num_workers > 0
- ? quickstep::FLAGS_num_workers
- : (num_hw_threads != 0 ? num_hw_threads : 1);
-
- DCHECK_GT(real_num_workers, 0);
- printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool\n",
- real_num_workers,
+ printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool.\n",
+ FLAGS_num_workers,
(static_cast<double>(quickstep::FLAGS_buffer_pool_slots) * quickstep::kSlotSizeBytes)/quickstep::kAGigaByte);
#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
@@ -290,7 +269,7 @@ int main(int argc, char* argv[]) {
// Parse the CPU affinities for workers and the preloader thread, if enabled
// to warm up the buffer pool.
const vector<int> worker_cpu_affinities =
- InputParserUtil::ParseWorkerAffinities(real_num_workers,
+ InputParserUtil::ParseWorkerAffinities(FLAGS_num_workers,
quickstep::FLAGS_worker_affinities);
const std::size_t num_numa_nodes_system = DefaultsConfigurator::GetNumNUMANodes();
@@ -323,7 +302,7 @@ int main(int argc, char* argv[]) {
vector<client_id> worker_client_ids;
// Initialize the worker threads.
- DCHECK_EQ(static_cast<std::size_t>(real_num_workers),
+ DCHECK_EQ(static_cast<std::size_t>(FLAGS_num_workers),
worker_cpu_affinities.size());
for (std::size_t worker_thread_index = 0;
worker_thread_index < worker_cpu_affinities.size();
[34/39] incubator-quickstep git commit: LOG only in the debug mode
for BlockLocator.
Posted by ji...@apache.org.
LOG only in the debug mode for BlockLocator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e50a2b7a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e50a2b7a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e50a2b7a
Branch: refs/heads/min-max-stats
Commit: e50a2b7aaed97f3282c338be0b36071a5cffd523
Parents: 5ff89dd
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Dec 3 21:24:25 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Dec 3 21:24:25 2016 -0800
----------------------------------------------------------------------
query_execution/BlockLocator.cpp | 48 ++++++++++++------------
query_execution/QueryManagerDistributed.cpp | 4 +-
2 files changed, 26 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e50a2b7a/query_execution/BlockLocator.cpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocator.cpp b/query_execution/BlockLocator.cpp
index 81684ba..5de6a54 100644
--- a/query_execution/BlockLocator.cpp
+++ b/query_execution/BlockLocator.cpp
@@ -55,8 +55,8 @@ void BlockLocator::run() {
const tmb::AnnotatedMessage annotated_message = bus_->Receive(locator_client_id_, 0, true);
const TaggedMessage &tagged_message = annotated_message.tagged_message;
const client_id sender = annotated_message.sender;
- LOG(INFO) << "BlockLocator received the typed '" << tagged_message.message_type()
- << "' message from TMB Client " << sender;
+ DLOG(INFO) << "BlockLocator received the typed '" << tagged_message.message_type()
+ << "' message from TMB Client " << sender;
switch (tagged_message.message_type()) {
case kBlockDomainRegistrationMessage: {
serialization::BlockDomainRegistrationMessage proto;
@@ -77,9 +77,9 @@ void BlockLocator::run() {
DCHECK_EQ(result_block_locations.second, result_domain_blocks.second);
if (result_domain_blocks.second) {
- LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " loaded in Domain " << domain;
+ DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " loaded in Domain " << domain;
} else {
- LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " existed in Domain " << domain;
+ DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " existed in Domain " << domain;
}
break;
}
@@ -95,9 +95,9 @@ void BlockLocator::run() {
block_locations_[block].erase(domain);
domain_blocks_[domain].erase(block);
- LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " evicted in Domain " << domain;
+ DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " evicted in Domain " << domain;
} else {
- LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " not found in Domain " << domain;
+ DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " not found in Domain " << domain;
}
break;
}
@@ -128,7 +128,7 @@ void BlockLocator::run() {
}
domain_blocks_.erase(domain);
- LOG(INFO) << "Unregistered Domain " << domain;
+ DLOG(INFO) << "Unregistered Domain " << domain;
break;
}
case kPoisonMessage: {
@@ -153,14 +153,14 @@ void BlockLocator::processBlockDomainRegistrationMessage(const client_id receive
CHECK(proto.SerializeToArray(proto_bytes, proto_length));
TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kBlockDomainRegistrationResponseMessage);
+ proto_length,
+ kBlockDomainRegistrationResponseMessage);
free(proto_bytes);
- LOG(INFO) << "BlockLocator (id '" << locator_client_id_
- << "') sent BlockDomainRegistrationResponseMessage (typed '"
- << kBlockDomainRegistrationResponseMessage
- << "') to Worker (id '" << receiver << "')";
+ DLOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent BlockDomainRegistrationResponseMessage (typed '"
+ << kBlockDomainRegistrationResponseMessage
+ << "') to TMB Client (id '" << receiver << "')";
CHECK(tmb::MessageBus::SendStatus::kOK ==
QueryExecutionUtil::SendTMBMessage(bus_,
locator_client_id_,
@@ -181,13 +181,13 @@ void BlockLocator::processLocateBlockMessage(const client_id receiver,
CHECK(proto.SerializeToArray(proto_bytes, proto_length));
TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kLocateBlockResponseMessage);
+ proto_length,
+ kLocateBlockResponseMessage);
free(proto_bytes);
- LOG(INFO) << "BlockLocator (id '" << locator_client_id_
- << "') sent LocateBlockResponseMessage (typed '" << kLocateBlockResponseMessage
- << "') to StorageManager (id '" << receiver << "')";
+ DLOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent LocateBlockResponseMessage (typed '" << kLocateBlockResponseMessage
+ << "') to StorageManager (id '" << receiver << "')";
CHECK(tmb::MessageBus::SendStatus::kOK ==
QueryExecutionUtil::SendTMBMessage(bus_,
locator_client_id_,
@@ -208,14 +208,14 @@ void BlockLocator::processGetPeerDomainNetworkAddressesMessage(const client_id r
CHECK(proto.SerializeToArray(proto_bytes, proto_length));
TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kGetPeerDomainNetworkAddressesResponseMessage);
+ proto_length,
+ kGetPeerDomainNetworkAddressesResponseMessage);
free(proto_bytes);
- LOG(INFO) << "BlockLocator (id '" << locator_client_id_
- << "') sent GetPeerDomainNetworkAddressesResponseMessage (typed '"
- << kGetPeerDomainNetworkAddressesResponseMessage
- << "') to StorageManager (id '" << receiver << "')";
+ DLOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent GetPeerDomainNetworkAddressesResponseMessage (typed '"
+ << kGetPeerDomainNetworkAddressesResponseMessage
+ << "') to StorageManager (id '" << receiver << "')";
CHECK(tmb::MessageBus::SendStatus::kOK ==
QueryExecutionUtil::SendTMBMessage(bus_,
locator_client_id_,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e50a2b7a/query_execution/QueryManagerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.cpp b/query_execution/QueryManagerDistributed.cpp
index 20650d0..5c7e0d8 100644
--- a/query_execution/QueryManagerDistributed.cpp
+++ b/query_execution/QueryManagerDistributed.cpp
@@ -176,8 +176,8 @@ bool QueryManagerDistributed::initiateRebuild(const dag_node_index index) {
shiftboss_addresses.AddRecipient(shiftboss_directory_->getClientId(i));
}
- LOG(INFO) << "ForemanDistributed sent InitiateRebuildMessage (typed '" << kInitiateRebuildMessage
- << "') to all Shiftbosses";
+ DLOG(INFO) << "ForemanDistributed sent InitiateRebuildMessage (typed '" << kInitiateRebuildMessage
+ << "') to all Shiftbosses";
QueryExecutionUtil::BroadcastMessage(foreman_client_id_,
shiftboss_addresses,
move(tagged_msg),
[15/39] incubator-quickstep git commit: Moved a CLI flag into a new
file.
Posted by ji...@apache.org.
Moved a CLI flag into a new file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f7d15436
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f7d15436
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f7d15436
Branch: refs/heads/min-max-stats
Commit: f7d1543671a3d3b3e52d2fe0c8170b64796aa9d6
Parents: e7c664b
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Nov 19 00:18:55 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:56:10 2016 -0800
----------------------------------------------------------------------
CMakeLists.txt | 2 ++
cli/CMakeLists.txt | 5 +++++
cli/Flags.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++
cli/Flags.hpp | 41 +++++++++++++++++++++++++++++++++++++++++
cli/QuickstepCli.cpp | 23 +++++++++--------------
5 files changed, 100 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f7d15436/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6191de0..cd53967 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -752,6 +752,7 @@ target_link_libraries(quickstep_cli_shell
quickstep_cli_CommandExecutor
quickstep_cli_DefaultsConfigurator
quickstep_cli_DropRelation
+ quickstep_cli_Flags
quickstep_cli_InputParserUtil
quickstep_cli_LineReader
quickstep_cli_PrintToScreen
@@ -770,6 +771,7 @@ target_link_libraries(quickstep_cli_shell
quickstep_queryoptimizer_QueryProcessor
quickstep_storage_PreloaderThread
quickstep_storage_StorageConstants
+ quickstep_storage_StorageManager
quickstep_threading_ThreadIDBasedMap
quickstep_utility_ExecutionDAGVisualizer
quickstep_utility_Macros
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f7d15436/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index b86821a..9928f3f 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -54,6 +54,7 @@ target_link_libraries(quickstep_cli_DropRelation
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageManager
quickstep_utility_Macros)
+add_library(quickstep_cli_Flags Flags.cpp Flags.hpp)
if(USE_LINENOISE)
add_library(quickstep_cli_LineReader
@@ -110,6 +111,9 @@ if(QUICKSTEP_HAVE_LIBNUMA)
target_link_libraries(quickstep_cli_DefaultsConfigurator
${LIBNUMA_LIBRARY})
endif()
+target_link_libraries(quickstep_cli_Flags
+ quickstep_storage_StorageConstants
+ ${GFLAGS_LIB_NAME})
target_link_libraries(quickstep_cli_InputParserUtil
glog
quickstep_utility_Macros
@@ -139,6 +143,7 @@ target_link_libraries(quickstep_cli
quickstep_cli_CommandExecutor
quickstep_cli_DefaultsConfigurator
quickstep_cli_DropRelation
+ quickstep_cli_Flags
quickstep_cli_InputParserUtil
quickstep_cli_LineReader
quickstep_cli_PrintToScreen)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f7d15436/cli/Flags.cpp
----------------------------------------------------------------------
diff --git a/cli/Flags.cpp b/cli/Flags.cpp
new file mode 100644
index 0000000..384ef71
--- /dev/null
+++ b/cli/Flags.cpp
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "cli/Flags.hpp"
+
+#include <string>
+
+#include "storage/StorageConstants.hpp"
+
+#include "gflags/gflags.h"
+
+namespace quickstep {
+
+static bool ValidateStoragePath(const char *flagname,
+ const std::string &value) {
+ if (!value.empty() && value.back() != kPathSeparator) {
+ FLAGS_storage_path.push_back(kPathSeparator);
+ }
+
+ return true;
+}
+DEFINE_string(storage_path, kDefaultStoragePath,
+ "Filesystem path to store the Quickstep database.");
+static const volatile bool storage_path_dummy
+ = gflags::RegisterFlagValidator(&FLAGS_storage_path, &ValidateStoragePath);
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f7d15436/cli/Flags.hpp
----------------------------------------------------------------------
diff --git a/cli/Flags.hpp b/cli/Flags.hpp
new file mode 100644
index 0000000..a623448
--- /dev/null
+++ b/cli/Flags.hpp
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_CLI_FLAGS_HPP_
+#define QUICKSTEP_CLI_FLAGS_HPP_
+
+#include "gflags/gflags_declare.h"
+
+namespace quickstep {
+
+/** \addtogroup CLI
+ * @{
+ */
+
+/**
+ * @brief A collection of common flags shared by Quickstep CLIs in both the
+ * single-node and the distributed version.
+ **/
+DECLARE_string(storage_path);
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_CLI_FLAGS_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f7d15436/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index a4e55d7..33a9ff4 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -54,6 +54,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#endif
#include "cli/DefaultsConfigurator.hpp"
+#include "cli/Flags.hpp"
#include "cli/InputParserUtil.hpp"
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
@@ -76,6 +77,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "storage/PreloaderThread.hpp"
#include "storage/StorageConstants.hpp"
+#include "storage/StorageManager.hpp"
#include "threading/ThreadIDBasedMap.hpp"
#include "utility/ExecutionDAGVisualizer.hpp"
#include "utility/Macros.hpp"
@@ -107,6 +109,7 @@ using quickstep::AdmitRequestMessage;
using quickstep::CatalogRelation;
using quickstep::DefaultsConfigurator;
using quickstep::DropRelation;
+using quickstep::FLAGS_storage_path;
using quickstep::ForemanSingleNode;
using quickstep::InputParserUtil;
using quickstep::MessageBusImpl;
@@ -142,8 +145,6 @@ DEFINE_bool(preload_buffer_pool, false,
"If true, pre-load all known blocks into buffer pool before "
"accepting queries (should also set --buffer_pool_slots to be "
"large enough to accomodate the entire database).");
-DEFINE_string(storage_path, kDefaultStoragePath,
- "Filesystem path to store the Quickstep database.");
DEFINE_string(worker_affinities, "",
"A comma-separated list of CPU IDs to pin worker threads to "
"(leaving this empty will cause all worker threads to inherit "
@@ -231,15 +232,9 @@ int main(int argc, char* argv[]) {
bus.RegisterClientAsSender(main_thread_client_id, kPoisonMessage);
bus.RegisterClientAsReceiver(main_thread_client_id, kWorkloadCompletionMessage);
- // Setup the paths used by StorageManager.
- string fixed_storage_path(quickstep::FLAGS_storage_path);
- if (!fixed_storage_path.empty()
- && (fixed_storage_path.back() != quickstep::kPathSeparator)) {
- fixed_storage_path.push_back(quickstep::kPathSeparator);
- }
- quickstep::StorageManager storage_manager(fixed_storage_path);
+ quickstep::StorageManager storage_manager(FLAGS_storage_path);
- string catalog_path(fixed_storage_path);
+ string catalog_path(FLAGS_storage_path);
catalog_path.append(kCatalogFilename);
if (quickstep::FLAGS_initialize_db) { // Initialize the database
// TODO(jmp): Refactor the code in this file!
@@ -249,14 +244,14 @@ int main(int argc, char* argv[]) {
// TODO(jmp): At some point, likely in C++-17, we will just have the
// filesystem path, and we can clean this up
#ifdef QUICKSTEP_OS_WINDOWS
- std::filesystem::create_directories(fixed_storage_path);
- LOG(FATAL) << "Failed when attempting to create the directory: " << fixed_storage_path << "\n";
+ std::filesystem::create_directories(FLAGS_storage_path);
+ LOG(FATAL) << "Failed when attempting to create the directory: " << FLAGS_storage_path << "\n";
LOG(FATAL) << "Check if the directory already exists. If so, delete it or move it before initializing \n";
#else
{
- string path_name = "mkdir " + fixed_storage_path;
+ string path_name = "mkdir " + FLAGS_storage_path;
if (std::system(path_name.c_str())) {
- LOG(FATAL) << "Failed when attempting to create the directory: " << fixed_storage_path << "\n";
+ LOG(FATAL) << "Failed when attempting to create the directory: " << FLAGS_storage_path << "\n";
}
}
#endif
[04/39] incubator-quickstep git commit: Removed using-declaration in
headers.
Posted by ji...@apache.org.
Removed using-declaration in headers.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ea5f28bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ea5f28bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ea5f28bf
Branch: refs/heads/min-max-stats
Commit: ea5f28bf6825071d04fb3b30bc0be0f9c4bf9937
Parents: 6fa7be6
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Nov 14 17:49:39 2016 -0800
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Wed Nov 16 22:15:12 2016 -0600
----------------------------------------------------------------------
cli/CommandExecutor.hpp | 5 -----
cli/LineReader.hpp | 3 ---
2 files changed, 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ea5f28bf/cli/CommandExecutor.hpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.hpp b/cli/CommandExecutor.hpp
index 25b47cc..a1d9af9 100644
--- a/cli/CommandExecutor.hpp
+++ b/cli/CommandExecutor.hpp
@@ -21,14 +21,9 @@
#define QUICKSTEP_CLI_COMMAND_COMMAND_EXECUTOR_HPP_
#include <cstdio>
-#include <string>
#include "tmb/id_typedefs.h"
-using std::fprintf;
-using std::fputc;
-using std::string;
-
namespace tmb { class MessageBus; }
namespace quickstep {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ea5f28bf/cli/LineReader.hpp
----------------------------------------------------------------------
diff --git a/cli/LineReader.hpp b/cli/LineReader.hpp
index 867d676..c7c073f 100644
--- a/cli/LineReader.hpp
+++ b/cli/LineReader.hpp
@@ -24,9 +24,6 @@
#include "utility/Macros.hpp"
-using std::size_t;
-using std::string;
-
namespace quickstep {
/** \addtogroup CLI
[26/39] incubator-quickstep git commit: Moved the init-db flag and
fixed a bug regarding db-init.
Posted by ji...@apache.org.
Moved the init-db flag and fixed a bug regarding db-init.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3047d89a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3047d89a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3047d89a
Branch: refs/heads/min-max-stats
Commit: 3047d89ab0e02bf0a30e914f55e6b49435c07e6a
Parents: 1b27888
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Nov 21 23:44:13 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Mon Nov 21 23:44:13 2016 -0800
----------------------------------------------------------------------
cli/DefaultsConfigurator.cpp | 2 +-
cli/Flags.cpp | 2 ++
cli/Flags.hpp | 3 +++
cli/QuickstepCli.cpp | 1 -
4 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3047d89a/cli/DefaultsConfigurator.cpp
----------------------------------------------------------------------
diff --git a/cli/DefaultsConfigurator.cpp b/cli/DefaultsConfigurator.cpp
index 94280a7..21b0af6 100644
--- a/cli/DefaultsConfigurator.cpp
+++ b/cli/DefaultsConfigurator.cpp
@@ -55,7 +55,7 @@ void DefaultsConfigurator::InitializeDefaultDatabase(const string &storage_path,
#else
{
const string path_name = "mkdir " + storage_path;
- CHECK(std::system(path_name.c_str()))
+ CHECK(!std::system(path_name.c_str()))
<< "Failed when attempting to create the directory: " << storage_path;
}
#endif // QUICKSTEP_OS_WINDOWS
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3047d89a/cli/Flags.cpp
----------------------------------------------------------------------
diff --git a/cli/Flags.cpp b/cli/Flags.cpp
index 87f9f73..1e3ce1d 100644
--- a/cli/Flags.cpp
+++ b/cli/Flags.cpp
@@ -32,6 +32,8 @@ using std::fprintf;
namespace quickstep {
+DEFINE_bool(initialize_db, false, "If true, initialize a database.");
+
static bool ValidateNumWorkers(const char *flagname, int value) {
if (value > 0) {
return true;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3047d89a/cli/Flags.hpp
----------------------------------------------------------------------
diff --git a/cli/Flags.hpp b/cli/Flags.hpp
index b020a3e..70aee98 100644
--- a/cli/Flags.hpp
+++ b/cli/Flags.hpp
@@ -32,6 +32,9 @@ namespace quickstep {
* @brief A collection of common flags shared by Quickstep CLIs in both the
* single-node and the distributed version.
**/
+
+DECLARE_bool(initialize_db);
+
DECLARE_int32(num_workers);
DECLARE_string(storage_path);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3047d89a/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 656786a..f4816a8 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -128,7 +128,6 @@ DEFINE_string(worker_affinities, "",
"the affinity mask of the Quickstep process, which typically "
"means that they will all be runable on any CPU according to "
"the kernel's own scheduling policy).");
-DEFINE_bool(initialize_db, false, "If true, initialize a database.");
DEFINE_bool(print_query, false,
"Print each input query statement. This is useful when running a "
"large number of queries in a batch.");
[25/39] incubator-quickstep git commit: Set Default block type to
only be the splitrow store.
Posted by ji...@apache.org.
Set Default block type to only be the splitrow store.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1b27888c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1b27888c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1b27888c
Branch: refs/heads/min-max-stats
Commit: 1b27888c3ced7ff6b104893cc6fe1a02d5cb33b2
Parents: 2d11ec5
Author: cramja <ma...@gmail.com>
Authored: Thu Nov 17 16:12:55 2016 -0600
Committer: cramja <ma...@gmail.com>
Committed: Mon Nov 21 14:28:02 2016 -0600
----------------------------------------------------------------------
storage/StorageBlockLayout.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1b27888c/storage/StorageBlockLayout.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlockLayout.cpp b/storage/StorageBlockLayout.cpp
index bdeb9cb..3b1bcc0 100644
--- a/storage/StorageBlockLayout.cpp
+++ b/storage/StorageBlockLayout.cpp
@@ -148,17 +148,17 @@ void StorageBlockLayout::copyHeaderTo(void *dest) const {
StorageBlockLayout* StorageBlockLayout::GenerateDefaultLayout(const CatalogRelationSchema &relation,
const bool relation_variable_length) {
+ // TODO(marc): In the future we may use relation_variable_length (columnstores as intermediate
+ // blocks), but for now, all we do is add (void) so that the compiler will not complain
+ // about an unused variable.
+ (void) relation_variable_length;
StorageBlockLayout *layout = new StorageBlockLayout(relation);
StorageBlockLayoutDescription *description = layout->getDescriptionMutable();
description->set_num_slots(1);
TupleStorageSubBlockDescription *tuple_store_description = description->mutable_tuple_store_description();
- if (relation_variable_length) {
- tuple_store_description->set_sub_block_type(TupleStorageSubBlockDescription::SPLIT_ROW_STORE);
- } else {
- tuple_store_description->set_sub_block_type(TupleStorageSubBlockDescription::PACKED_ROW_STORE);
- }
+ tuple_store_description->set_sub_block_type(TupleStorageSubBlockDescription::SPLIT_ROW_STORE);
layout->finalize();
return layout;
[38/39] incubator-quickstep git commit: QUICKSTEP-65 Fix Quickstep
build failure on Mac OSX 10.12 by turning off deprecation errors
Posted by ji...@apache.org.
QUICKSTEP-65 Fix Quickstep build failure on Mac OSX 10.12 by turning off deprecation errors
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/6ae0cdd3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/6ae0cdd3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/6ae0cdd3
Branch: refs/heads/min-max-stats
Commit: 6ae0cdd3d5db40febc98be00ce94baeed31ba680
Parents: b949c50
Author: Saket Saurabh <ss...@cs.wisc.edu>
Authored: Tue Dec 6 09:53:57 2016 -0600
Committer: Saket Saurabh <ss...@cs.wisc.edu>
Committed: Tue Dec 6 10:05:42 2016 -0600
----------------------------------------------------------------------
CMakeLists.txt | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6ae0cdd3/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 391cb26..4dcc56a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -291,6 +291,25 @@ else()
endif()
endif()
+ # OSX 10.12 has deprecated certain system-level APIs which causes protobuf & glog
+ # builds to fail. As a short-term workaround for now, we turn off deprecated
+ # warnings so that they do not cause build failures anymore.
+ # TODO: Remove this workaround by fixing the protobuf_cmake and glog_cmake.
+ if (${CMAKE_SYSTEM} MATCHES "Darwin-16.1.0")
+ if (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+ CHECK_CXX_COMPILER_FLAG("-Wno-error=deprecated-declarations" COMPILER_HAS_WNO_DEPRECATED)
+ if (COMPILER_HAS_WNO_DEPRECATED)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-declarations")
+ endif()
+ endif()
+ if(CMAKE_COMPILER_IS_GNUCXX)
+ CHECK_CXX_COMPILER_FLAG("-Wno-deprecated-declarations" COMPILER_HAS_WNO_DEPRECATED)
+ if (COMPILER_HAS_WNO_DEPRECATED)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations")
+ endif()
+ endif()
+ endif()
+
# One of the protobuf headers includes a nested anonymous union within
# another anonymous type. Ordinarily we work around this by compiling the
# protobuf libraries themselves with "-Wno-nested-anon-types" and including
[07/39] incubator-quickstep git commit: Refactor WorkOrderFactory for
a better debug info.
Posted by ji...@apache.org.
Refactor WorkOrderFactory for a better debug info.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/70959876
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/70959876
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/70959876
Branch: refs/heads/min-max-stats
Commit: 70959876590fb65b8f195a77d7af62e33b166dd4
Parents: 365fff6
Author: Zuyu Zhang <zu...@apache.org>
Authored: Thu Nov 17 15:09:17 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Fri Nov 18 11:30:44 2016 -0800
----------------------------------------------------------------------
query_execution/Shiftboss.cpp | 1 +
relational_operators/WorkOrderFactory.cpp | 48 ++++++++++++++------------
relational_operators/WorkOrderFactory.hpp | 4 +++
3 files changed, 30 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/70959876/query_execution/Shiftboss.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index 09d7846..30b2ae2 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -99,6 +99,7 @@ void Shiftboss::run() {
DCHECK_EQ(1u, query_contexts_.count(query_id));
WorkOrder *work_order = WorkOrderFactory::ReconstructFromProto(proto.work_order(),
+ shiftboss_index_,
&database_cache_,
query_contexts_[query_id].get(),
storage_manager_,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/70959876/relational_operators/WorkOrderFactory.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrderFactory.cpp b/relational_operators/WorkOrderFactory.cpp
index 6943a52..871f74d 100644
--- a/relational_operators/WorkOrderFactory.cpp
+++ b/relational_operators/WorkOrderFactory.cpp
@@ -19,6 +19,7 @@
#include "relational_operators/WorkOrderFactory.hpp"
+#include <cstddef>
#include <memory>
#include <utility>
#include <vector>
@@ -67,6 +68,7 @@ class Predicate;
class Scalar;
WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder &proto,
+ const std::size_t shiftboss_index,
CatalogDatabaseLite *catalog_database,
QueryContext *query_context,
StorageManager *storage_manager,
@@ -79,7 +81,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
switch (proto.work_order_type()) {
case serialization::AGGREGATION: {
- LOG(INFO) << "Creating AggregationWorkOrder";
+ LOG(INFO) << "Creating AggregationWorkOrder in Shiftboss " << shiftboss_index;
return new AggregationWorkOrder(
proto.query_id(),
proto.GetExtension(serialization::AggregationWorkOrder::block_id),
@@ -89,7 +91,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::AggregationWorkOrder::lip_deployment_index), query_context));
}
case serialization::BUILD_HASH: {
- LOG(INFO) << "Creating BuildHashWorkOrder";
+ LOG(INFO) << "Creating BuildHashWorkOrder in Shiftboss " << shiftboss_index;
vector<attribute_id> join_key_attributes;
for (int i = 0; i < proto.ExtensionSize(serialization::BuildHashWorkOrder::join_key_attributes); ++i) {
join_key_attributes.push_back(
@@ -110,7 +112,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::BuildHashWorkOrder::lip_deployment_index), query_context));
}
case serialization::DELETE: {
- LOG(INFO) << "Creating DeleteWorkOrder";
+ LOG(INFO) << "Creating DeleteWorkOrder in Shiftboss " << shiftboss_index;
return new DeleteWorkOrder(
proto.query_id(),
catalog_database->getRelationSchemaById(
@@ -124,7 +126,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
bus);
}
case serialization::DESTROY_AGGREGATION_STATE: {
- LOG(INFO) << "Creating DestroyAggregationStateWorkOrder";
+ LOG(INFO) << "Creating DestroyAggregationStateWorkOrder in Shiftboss " << shiftboss_index;
return new DestroyAggregationStateWorkOrder(
proto.query_id(),
proto.GetExtension(
@@ -132,7 +134,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
query_context);
}
case serialization::DESTROY_HASH: {
- LOG(INFO) << "Creating DestroyHashWorkOrder";
+ LOG(INFO) << "Creating DestroyHashWorkOrder in Shiftboss " << shiftboss_index;
return new DestroyHashWorkOrder(
proto.query_id(),
proto.GetExtension(
@@ -140,7 +142,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
query_context);
}
case serialization::DROP_TABLE: {
- LOG(INFO) << "Creating DropTableWorkOrder";
+ LOG(INFO) << "Creating DropTableWorkOrder in Shiftboss " << shiftboss_index;
vector<block_id> blocks;
for (int i = 0; i < proto.ExtensionSize(serialization::DropTableWorkOrder::block_ids); ++i) {
blocks.push_back(
@@ -157,7 +159,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
catalog_database);
}
case serialization::FINALIZE_AGGREGATION: {
- LOG(INFO) << "Creating FinalizeAggregationWorkOrder";
+ LOG(INFO) << "Creating FinalizeAggregationWorkOrder in Shiftboss " << shiftboss_index;
return new FinalizeAggregationWorkOrder(
proto.query_id(),
query_context->getAggregationState(proto.GetExtension(
@@ -212,7 +214,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
switch (hash_join_work_order_type) {
case serialization::HashJoinWorkOrder::HASH_ANTI_JOIN: {
- LOG(INFO) << "Creating HashAntiJoinWorkOrder";
+ LOG(INFO) << "Creating HashAntiJoinWorkOrder in Shiftboss " << shiftboss_index;
return new HashAntiJoinWorkOrder(
proto.query_id(),
build_relation,
@@ -228,7 +230,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
lip_filter_adaptive_prober);
}
case serialization::HashJoinWorkOrder::HASH_INNER_JOIN: {
- LOG(INFO) << "Creating HashInnerJoinWorkOrder";
+ LOG(INFO) << "Creating HashInnerJoinWorkOrder in Shiftboss " << shiftboss_index;
return new HashInnerJoinWorkOrder(
proto.query_id(),
build_relation,
@@ -252,7 +254,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::HashJoinWorkOrder::is_selection_on_build, i));
}
- LOG(INFO) << "Creating HashOuterJoinWorkOrder";
+ LOG(INFO) << "Creating HashOuterJoinWorkOrder in Shiftboss " << shiftboss_index;
return new HashOuterJoinWorkOrder(
proto.query_id(),
build_relation,
@@ -268,7 +270,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
lip_filter_adaptive_prober);
}
case serialization::HashJoinWorkOrder::HASH_SEMI_JOIN: {
- LOG(INFO) << "Creating HashSemiJoinWorkOrder";
+ LOG(INFO) << "Creating HashSemiJoinWorkOrder in Shiftboss " << shiftboss_index;
return new HashSemiJoinWorkOrder(
proto.query_id(),
build_relation,
@@ -288,7 +290,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
}
}
case serialization::INSERT: {
- LOG(INFO) << "Creating InsertWorkOrder";
+ LOG(INFO) << "Creating InsertWorkOrder in Shiftboss " << shiftboss_index;
return new InsertWorkOrder(
proto.query_id(),
query_context->getInsertDestination(
@@ -297,7 +299,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::InsertWorkOrder::tuple_index)));
}
case serialization::NESTED_LOOP_JOIN: {
- LOG(INFO) << "Creating NestedLoopsJoinWorkOrder";
+ LOG(INFO) << "Creating NestedLoopsJoinWorkOrder in Shiftboss " << shiftboss_index;
return new NestedLoopsJoinWorkOrder(
proto.query_id(),
catalog_database->getRelationSchemaById(
@@ -315,7 +317,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
storage_manager);
}
case serialization::SAMPLE: {
- LOG(INFO) << "Creating SampleWorkOrder";
+ LOG(INFO) << "Creating SampleWorkOrder in Shiftboss " << shiftboss_index;
return new SampleWorkOrder(
proto.query_id(),
catalog_database->getRelationSchemaById(
@@ -328,7 +330,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
storage_manager);
}
case serialization::SAVE_BLOCKS: {
- LOG(INFO) << "Creating SaveBlocksWorkOrder";
+ LOG(INFO) << "Creating SaveBlocksWorkOrder in Shiftboss " << shiftboss_index;
return new SaveBlocksWorkOrder(
proto.query_id(),
proto.GetExtension(serialization::SaveBlocksWorkOrder::block_id),
@@ -336,7 +338,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
storage_manager);
}
case serialization::SELECT: {
- LOG(INFO) << "Creating SelectWorkOrder";
+ LOG(INFO) << "Creating SelectWorkOrder in Shiftboss " << shiftboss_index;
const bool simple_projection =
proto.GetExtension(serialization::SelectWorkOrder::simple_projection);
vector<attribute_id> simple_selection;
@@ -364,7 +366,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::SelectWorkOrder::lip_deployment_index), query_context));
}
case serialization::SORT_MERGE_RUN: {
- LOG(INFO) << "Creating SortMergeRunWorkOrder";
+ LOG(INFO) << "Creating SortMergeRunWorkOrder in Shiftboss " << shiftboss_index;
vector<merge_run_operator::Run> runs;
for (int i = 0; i < proto.ExtensionSize(serialization::SortMergeRunWorkOrder::runs); ++i) {
merge_run_operator::Run run;
@@ -393,7 +395,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
bus);
}
case serialization::SORT_RUN_GENERATION: {
- LOG(INFO) << "Creating SortRunGenerationWorkOrder";
+ LOG(INFO) << "Creating SortRunGenerationWorkOrder in Shiftboss " << shiftboss_index;
return new SortRunGenerationWorkOrder(
proto.query_id(),
catalog_database->getRelationSchemaById(
@@ -406,7 +408,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
storage_manager);
}
case serialization::TABLE_GENERATOR: {
- LOG(INFO) << "Creating SortRunGenerationWorkOrder";
+ LOG(INFO) << "Creating SortRunGenerationWorkOrder in Shiftboss " << shiftboss_index;
return new TableGeneratorWorkOrder(
proto.query_id(),
query_context->getGeneratorFunctionHandle(
@@ -415,7 +417,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::TableGeneratorWorkOrder::insert_destination_index)));
}
case serialization::TEXT_SCAN: {
- LOG(INFO) << "Creating TextScanWorkOrder";
+ LOG(INFO) << "Creating TextScanWorkOrder in Shiftboss " << shiftboss_index;
return new TextScanWorkOrder(
proto.query_id(),
proto.GetExtension(serialization::TextScanWorkOrder::filename),
@@ -427,7 +429,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::TextScanWorkOrder::insert_destination_index)));
}
case serialization::UPDATE: {
- LOG(INFO) << "Creating UpdateWorkOrder";
+ LOG(INFO) << "Creating UpdateWorkOrder in Shiftboss " << shiftboss_index;
return new UpdateWorkOrder(
proto.query_id(),
catalog_database->getRelationSchemaById(
@@ -445,7 +447,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
bus);
}
case serialization::WINDOW_AGGREGATION: {
- LOG(INFO) << "Creating WindowAggregationWorkOrder";
+ LOG(INFO) << "Creating WindowAggregationWorkOrder in Shiftboss " << shiftboss_index;
vector<block_id> blocks;
for (int i = 0; i < proto.ExtensionSize(serialization::WindowAggregationWorkOrder::block_ids); ++i) {
blocks.push_back(
@@ -461,7 +463,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::WindowAggregationWorkOrder::insert_destination_index)));
}
default:
- LOG(FATAL) << "Unknown WorkOrder Type in WorkOrderFactory::ReconstructFromProto";
+ LOG(FATAL) << "Unknown WorkOrder Type in WorkOrderFactory::ReconstructFromProto in Shiftboss" << shiftboss_index;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/70959876/relational_operators/WorkOrderFactory.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrderFactory.hpp b/relational_operators/WorkOrderFactory.hpp
index 45e2cbc..acf3855 100644
--- a/relational_operators/WorkOrderFactory.hpp
+++ b/relational_operators/WorkOrderFactory.hpp
@@ -20,6 +20,8 @@
#ifndef QUICKSTEP_RELATIONAL_OPERATORS_WORK_ORDER_FACTORY_HPP_
#define QUICKSTEP_RELATIONAL_OPERATORS_WORK_ORDER_FACTORY_HPP_
+#include <cstddef>
+
#include "utility/Macros.hpp"
#include "tmb/id_typedefs.h"
@@ -50,6 +52,7 @@ class WorkOrderFactory {
*
* @param proto The Protocol Buffer representation of a WorkOrder object,
* originally generated by RelationalOperator::getAllWorkOrders.
+ * @param shiftboss_index The index of Shiftboss that is using.
* @param catalog_database The database to resolve relation and attribute
* references in.
* @param query_context A pointer to QueryContext.
@@ -60,6 +63,7 @@ class WorkOrderFactory {
* @return A new WorkOrder reconstructed from the supplied Protocol Buffer.
**/
static WorkOrder* ReconstructFromProto(const serialization::WorkOrder &proto,
+ const std::size_t shiftboss_index,
CatalogDatabaseLite *catalog_database,
QueryContext *query_context,
StorageManager *storage_manager,
[31/39] incubator-quickstep git commit: Scheduled WorkOrders w/ the
same aggr_state_index on the same Shiftboss.
Posted by ji...@apache.org.
Scheduled WorkOrders w/ the same aggr_state_index on the same Shiftboss.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/57730e40
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/57730e40
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/57730e40
Branch: refs/heads/min-max-stats
Commit: 57730e406533fa02280568380c853ce1c5dc19ee
Parents: e75c265
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 27 17:04:30 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 27 17:28:30 2016 -0800
----------------------------------------------------------------------
query_execution/ForemanDistributed.cpp | 61 ++++++++++++++++------
query_execution/ForemanDistributed.hpp | 4 ++
query_execution/PolicyEnforcerDistributed.cpp | 12 +++++
query_execution/PolicyEnforcerDistributed.hpp | 18 +++++++
query_execution/QueryManagerDistributed.hpp | 23 ++++++++
5 files changed, 102 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/57730e40/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index fc9cd3c..61f0603 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -243,6 +243,32 @@ bool ForemanDistributed::canCollectNewMessages(const tmb::message_type_id messag
kWorkOrderFeedbackMessage);
}
+bool ForemanDistributed::isAggregationRelatedWorkOrder(const S::WorkOrderMessage &proto,
+ const size_t next_shiftboss_index_to_schedule,
+ size_t *shiftboss_index_for_aggregation) {
+ const S::WorkOrder &work_order_proto = proto.work_order();
+ QueryContext::aggregation_state_id aggr_state_index;
+
+ switch (work_order_proto.work_order_type()) {
+ case S::AGGREGATION:
+ aggr_state_index = work_order_proto.GetExtension(S::AggregationWorkOrder::aggr_state_index);
+ break;
+ case S::FINALIZE_AGGREGATION:
+ aggr_state_index = work_order_proto.GetExtension(S::FinalizeAggregationWorkOrder::aggr_state_index);
+ break;
+ case S::DESTROY_AGGREGATION_STATE:
+ aggr_state_index = work_order_proto.GetExtension(S::DestroyAggregationStateWorkOrder::aggr_state_index);
+ break;
+ default:
+ return false;
+ }
+
+ static_cast<PolicyEnforcerDistributed*>(policy_enforcer_.get())->getShiftbossIndexForAggregation(
+ proto.query_id(), aggr_state_index, next_shiftboss_index_to_schedule, shiftboss_index_for_aggregation);
+
+ return true;
+}
+
bool ForemanDistributed::isHashJoinRelatedWorkOrder(const S::WorkOrderMessage &proto,
const size_t next_shiftboss_index_to_schedule,
size_t *shiftboss_index_for_hash_join) {
@@ -251,13 +277,13 @@ bool ForemanDistributed::isHashJoinRelatedWorkOrder(const S::WorkOrderMessage &p
switch (work_order_proto.work_order_type()) {
case S::BUILD_HASH:
- join_hash_table_index = work_order_proto.GetExtension(serialization::BuildHashWorkOrder::join_hash_table_index);
- break;
- case S::DESTROY_HASH:
- join_hash_table_index = work_order_proto.GetExtension(serialization::DestroyHashWorkOrder::join_hash_table_index);
+ join_hash_table_index = work_order_proto.GetExtension(S::BuildHashWorkOrder::join_hash_table_index);
break;
case S::HASH_JOIN:
- join_hash_table_index = work_order_proto.GetExtension(serialization::HashJoinWorkOrder::join_hash_table_index);
+ join_hash_table_index = work_order_proto.GetExtension(S::HashJoinWorkOrder::join_hash_table_index);
+ break;
+ case S::DESTROY_HASH:
+ join_hash_table_index = work_order_proto.GetExtension(S::DestroyHashWorkOrder::join_hash_table_index);
break;
default:
return false;
@@ -275,20 +301,23 @@ void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::Wo
for (const auto &message : messages) {
DCHECK(message != nullptr);
const S::WorkOrderMessage &proto = *message;
- size_t shiftboss_index_for_hash_join;
- if (isHashJoinRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_hash_join)) {
- sendWorkOrderMessage(shiftboss_index_for_hash_join, proto);
- shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index_for_hash_join);
-
- if (shiftboss_index == shiftboss_index_for_hash_join) {
- shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
- }
+ size_t shiftboss_index_for_particular_work_order_type;
+ if (isAggregationRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
+ } else if (isHashJoinRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
} else {
- sendWorkOrderMessage(shiftboss_index, proto);
- shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index);
-
// TODO(zuyu): Take data-locality into account for scheduling.
+ shiftboss_index_for_particular_work_order_type = shiftboss_index;
+ }
+
+ sendWorkOrderMessage(shiftboss_index_for_particular_work_order_type, proto);
+ shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index_for_particular_work_order_type);
+
+ if (shiftboss_index == shiftboss_index_for_particular_work_order_type) {
shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
+ } else {
+ // NOTE(zuyu): This is not the exact round-robin scheduling, as in this case,
+ // <shiftboss_index_for_particular_work_order_type> might be scheduled one
+ // more WorkOrder for an Aggregation or a HashJoin.
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/57730e40/query_execution/ForemanDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.hpp b/query_execution/ForemanDistributed.hpp
index 0616f30..34bac07 100644
--- a/query_execution/ForemanDistributed.hpp
+++ b/query_execution/ForemanDistributed.hpp
@@ -71,6 +71,10 @@ class ForemanDistributed final : public ForemanBase {
void run() override;
private:
+ bool isAggregationRelatedWorkOrder(const serialization::WorkOrderMessage &proto,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index_for_aggregation);
+
bool isHashJoinRelatedWorkOrder(const serialization::WorkOrderMessage &proto,
const std::size_t next_shiftboss_index_to_schedule,
std::size_t *shiftboss_index_for_hash_join);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/57730e40/query_execution/PolicyEnforcerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.cpp b/query_execution/PolicyEnforcerDistributed.cpp
index 86b36c8..c5642bc 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -158,6 +158,18 @@ void PolicyEnforcerDistributed::processInitiateRebuildResponseMessage(const tmb:
}
}
+void PolicyEnforcerDistributed::getShiftbossIndexForAggregation(
+ const std::size_t query_id,
+ const QueryContext::aggregation_state_id aggr_state_index,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index) {
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+ QueryManagerDistributed *query_manager = static_cast<QueryManagerDistributed*>(admitted_queries_[query_id].get());
+ query_manager->getShiftbossIndexForAggregation(aggr_state_index,
+ next_shiftboss_index_to_schedule,
+ shiftboss_index);
+}
+
void PolicyEnforcerDistributed::getShiftbossIndexForHashJoin(
const std::size_t query_id,
const QueryContext::join_hash_table_id join_hash_table_index,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/57730e40/query_execution/PolicyEnforcerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.hpp b/query_execution/PolicyEnforcerDistributed.hpp
index 37326bd..e8bc394 100644
--- a/query_execution/PolicyEnforcerDistributed.hpp
+++ b/query_execution/PolicyEnforcerDistributed.hpp
@@ -90,6 +90,24 @@ class PolicyEnforcerDistributed final : public PolicyEnforcerBase {
void processInitiateRebuildResponseMessage(const tmb::TaggedMessage &tagged_message);
/**
+ * @brief Get or set the index of Shiftboss for an Aggregation related
+ * WorkOrder. If it is the first Aggregation on <aggr_state_index>,
+ * <shiftboss_index> will be set to <next_shiftboss_index_to_schedule>.
+ * Otherwise, <shiftboss_index> will be set to the index of the Shiftboss that
+ * has executed the first Aggregation.
+ *
+ * @param query_id The query id.
+ * @param aggr_state_index The Hash Table for the Aggregation.
+ * @param next_shiftboss_index The index of Shiftboss to schedule a next WorkOrder.
+ * @param shiftboss_index The index of Shiftboss to schedule the WorkOrder.
+ **/
+ void getShiftbossIndexForAggregation(
+ const std::size_t query_id,
+ const QueryContext::aggregation_state_id aggr_state_index,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index);
+
+ /**
* @brief Get or set the index of Shiftboss for a HashJoin related WorkOrder.
* If it is the first BuildHash on <join_hash_table_index>, <shiftboss_index>
* will be set to <next_shiftboss_index_to_schedule>. Otherwise,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/57730e40/query_execution/QueryManagerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.hpp b/query_execution/QueryManagerDistributed.hpp
index 2b21303..7a07fcb 100644
--- a/query_execution/QueryManagerDistributed.hpp
+++ b/query_execution/QueryManagerDistributed.hpp
@@ -95,6 +95,26 @@ class QueryManagerDistributed final : public QueryManagerBase {
const dag_node_index start_operator_index);
/**
+ * @brief Get the index of Shiftboss for an Aggregation related WorkOrder. If
+ * the Shiftboss index is not found, set using <next_shiftboss_index_to_schedule>.
+ *
+ * @param aggr_state_index The Hash Table for the Aggregation.
+ * @param next_shiftboss_index The index of Shiftboss to schedule a next WorkOrder.
+ * @param shiftboss_index The index of Shiftboss to schedule the WorkOrder.
+ **/
+ void getShiftbossIndexForAggregation(const QueryContext::aggregation_state_id aggr_state_index,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index) {
+ const auto cit = shiftboss_indexes_for_aggrs_.find(aggr_state_index);
+ if (cit != shiftboss_indexes_for_aggrs_.end()) {
+ *shiftboss_index = cit->second;
+ } else {
+ shiftboss_indexes_for_aggrs_.emplace(aggr_state_index, next_shiftboss_index_to_schedule);
+ *shiftboss_index = next_shiftboss_index_to_schedule;
+ }
+ }
+
+ /**
* @brief Get the index of Shiftboss for a HashJoin related WorkOrder. If the
* Shiftboss index is not found, set using <next_shiftboss_index_to_schedule>.
*
@@ -136,6 +156,9 @@ class QueryManagerDistributed final : public QueryManagerBase {
std::unique_ptr<WorkOrderProtosContainer> normal_workorder_protos_container_;
+ // A map from an aggregation id to its scheduled Shiftboss index.
+ std::unordered_map<QueryContext::aggregation_state_id, std::size_t> shiftboss_indexes_for_aggrs_;
+
// A map from a join hash table to its scheduled Shiftboss index.
std::unordered_map<QueryContext::join_hash_table_id, std::size_t> shiftboss_indexes_for_hash_joins_;
[18/39] incubator-quickstep git commit: Used the network util in
DataExchanger.
Posted by ji...@apache.org.
Used the network util in DataExchanger.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c886f7af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c886f7af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c886f7af
Branch: refs/heads/min-max-stats
Commit: c886f7af5835a8577f516b2c7a39692d18d5b9b2
Parents: 8f1b0e5
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 13 15:55:31 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 20:08:27 2016 -0800
----------------------------------------------------------------------
storage/CMakeLists.txt | 1 +
storage/DataExchangerAsync.cpp | 9 +++++----
storage/DataExchangerAsync.hpp | 2 --
3 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c886f7af/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 559d86d..61a8a99 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -625,6 +625,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_storage_StorageManager
quickstep_threading_Thread
quickstep_utility_Macros
+ quickstep_utility_NetworkUtil
${GRPCPLUSPLUS_LIBRARIES})
endif()
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c886f7af/storage/DataExchangerAsync.cpp
----------------------------------------------------------------------
diff --git a/storage/DataExchangerAsync.cpp b/storage/DataExchangerAsync.cpp
index 1d2f7db..d349430 100644
--- a/storage/DataExchangerAsync.cpp
+++ b/storage/DataExchangerAsync.cpp
@@ -28,6 +28,7 @@
#include "storage/DataExchange.grpc.pb.h"
#include "storage/DataExchange.pb.h"
#include "storage/StorageManager.hpp"
+#include "utility/NetworkUtil.hpp"
#include "glog/logging.h"
@@ -127,18 +128,18 @@ void CallContext::Proceed() {
} // namespace
-const char *DataExchangerAsync::kLocalNetworkAddress = "0.0.0.0:";
-
DataExchangerAsync::DataExchangerAsync() {
+ const std::string ipv4_address(GetIpv4Address() + ':');
+
grpc::ServerBuilder builder;
- builder.AddListeningPort(kLocalNetworkAddress, grpc::InsecureServerCredentials(), &port_);
+ builder.AddListeningPort(ipv4_address, grpc::InsecureServerCredentials(), &port_);
builder.RegisterService(&service_);
queue_ = builder.AddCompletionQueue();
server_ = builder.BuildAndStart();
DCHECK_GT(port_, 0);
- server_address_ = kLocalNetworkAddress + std::to_string(port_);
+ server_address_ = ipv4_address + std::to_string(port_);
LOG(INFO) << "DataExchangerAsync Service listening on " << server_address_;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c886f7af/storage/DataExchangerAsync.hpp
----------------------------------------------------------------------
diff --git a/storage/DataExchangerAsync.hpp b/storage/DataExchangerAsync.hpp
index b777108..4504c08 100644
--- a/storage/DataExchangerAsync.hpp
+++ b/storage/DataExchangerAsync.hpp
@@ -79,8 +79,6 @@ class DataExchangerAsync final : public Thread {
void run() override;
private:
- static const char *kLocalNetworkAddress;
-
DataExchange::AsyncService service_;
int port_ = -1;
[10/39] incubator-quickstep git commit: Used the move semantics in
constructing QueryProcessor.
Posted by ji...@apache.org.
Used the move semantics in constructing QueryProcessor.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/24367d7d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/24367d7d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/24367d7d
Branch: refs/heads/min-max-stats
Commit: 24367d7d878e066ce131a6e13032b2ba0ecef8f7
Parents: dec8723
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Nov 18 23:28:34 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:25:55 2016 -0800
----------------------------------------------------------------------
cli/QuickstepCli.cpp | 2 +-
query_optimizer/QueryProcessor.hpp | 7 ++++---
2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/24367d7d/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 31215f6..f17e18b 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -285,7 +285,7 @@ int main(int argc, char* argv[]) {
// Setup QueryProcessor, including CatalogDatabase.
std::unique_ptr<QueryProcessor> query_processor;
try {
- query_processor = std::make_unique<QueryProcessor>(catalog_path);
+ query_processor = std::make_unique<QueryProcessor>(std::move(catalog_path));
} catch (const std::exception &e) {
LOG(FATAL) << "FATAL ERROR DURING STARTUP: "
<< e.what()
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/24367d7d/query_optimizer/QueryProcessor.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryProcessor.hpp b/query_optimizer/QueryProcessor.hpp
index 5e5c115..3e37c9d 100644
--- a/query_optimizer/QueryProcessor.hpp
+++ b/query_optimizer/QueryProcessor.hpp
@@ -24,6 +24,7 @@
#include <exception>
#include <memory>
#include <string>
+#include <utility>
#include "catalog/Catalog.hpp"
#include "query_optimizer/Optimizer.hpp"
@@ -130,8 +131,8 @@ class QueryProcessor {
*
* @param catalog_filename The file to read the serialized catalog from.
**/
- explicit QueryProcessor(const std::string &catalog_filename)
- : catalog_filename_(catalog_filename),
+ explicit QueryProcessor(std::string &&catalog_filename)
+ : catalog_filename_(std::move(catalog_filename)),
catalog_altered_(false),
query_id_(0) {
loadCatalog();
@@ -185,7 +186,7 @@ class QueryProcessor {
optimizer::Optimizer optimizer_;
- std::string catalog_filename_;
+ const std::string catalog_filename_;
std::unique_ptr<Catalog> catalog_;