You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by zu...@apache.org on 2017/02/11 19:41:33 UTC
[1/2] incubator-quickstep git commit: Visualized Execution DAG in the
distributed version.
Repository: incubator-quickstep
Updated Branches:
refs/heads/dist-patch 2539ce33b -> 1ccbb60b4
Visualized Execution DAG in the distributed version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/911f3dcd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/911f3dcd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/911f3dcd
Branch: refs/heads/dist-patch
Commit: 911f3dcd2227a96d1474b667cb51c8bc081bd747
Parents: 2539ce3
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Feb 10 22:01:48 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Feb 11 11:41:24 2017 -0800
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 5 ++++-
query_execution/PolicyEnforcerBase.cpp | 7 ++----
query_execution/PolicyEnforcerDistributed.cpp | 26 +++++++++++++++++++++-
query_execution/QueryManagerBase.cpp | 11 +++++++++
query_execution/QueryManagerBase.hpp | 12 ++++++++++
query_optimizer/QueryHandle.hpp | 7 ++++++
6 files changed, 61 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/911f3dcd/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 3a69f77..12d6be0 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -166,6 +166,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_queryexecution_ShiftbossDirectory
quickstep_queryoptimizer_QueryHandle
quickstep_storage_StorageBlockInfo
+ quickstep_utility_ExecutionDAGVisualizer
quickstep_utility_Macros
tmb
${GFLAGS_LIB_NAME})
@@ -246,7 +247,9 @@ target_link_libraries(quickstep_queryexecution_QueryManagerBase
quickstep_relationaloperators_WorkOrder
quickstep_storage_StorageBlockInfo
quickstep_utility_DAG
- quickstep_utility_Macros)
+ quickstep_utility_ExecutionDAGVisualizer
+ quickstep_utility_Macros
+ ${GFLAGS_LIB_NAME})
if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution_QueryManagerDistributed
quickstep_catalog_CatalogTypedefs
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/911f3dcd/query_execution/PolicyEnforcerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.cpp b/query_execution/PolicyEnforcerBase.cpp
index 082f6e9..1ffde4d 100644
--- a/query_execution/PolicyEnforcerBase.cpp
+++ b/query_execution/PolicyEnforcerBase.cpp
@@ -40,15 +40,12 @@
namespace quickstep {
+DECLARE_bool(visualize_execution_dag);
+
DEFINE_bool(profile_and_report_workorder_perf, false,
"If true, Quickstep will record the exceution time of all the individual "
"normal work orders and report it at the end of query execution.");
-DEFINE_bool(visualize_execution_dag, false,
- "If true, visualize the execution plan DAG into a graph in DOT "
- "format (DOT is a plain text graph description language) which is "
- "then printed via stderr.");
-
PolicyEnforcerBase::PolicyEnforcerBase(CatalogDatabaseLite *catalog_database)
: catalog_database_(catalog_database),
profile_individual_workorders_(FLAGS_profile_and_report_workorder_perf || FLAGS_visualize_execution_dag) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/911f3dcd/query_execution/PolicyEnforcerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.cpp b/query_execution/PolicyEnforcerDistributed.cpp
index 8f0332d..d5881b9 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -15,11 +15,14 @@
#include "query_execution/PolicyEnforcerDistributed.hpp"
#include <cstddef>
+#include <cstdio>
#include <cstdlib>
#include <memory>
#include <queue>
-#include <utility>
+#include <sstream>
+#include <string>
#include <unordered_map>
+#include <utility>
#include <vector>
#include "catalog/Catalog.pb.h"
@@ -33,6 +36,7 @@
#include "query_execution/QueryManagerDistributed.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "storage/StorageBlockInfo.hpp"
+#include "utility/ExecutionDAGVisualizer.hpp"
#include "gflags/gflags.h"
#include "glog/logging.h"
@@ -53,6 +57,8 @@ using tmb::TaggedMessage;
namespace quickstep {
+DECLARE_bool(visualize_execution_dag);
+
namespace S = serialization;
DEFINE_uint64(max_msgs_per_dispatch_round, 20, "Maximum number of messages that"
@@ -228,6 +234,24 @@ void PolicyEnforcerDistributed::onQueryCompletion(QueryManagerBase *query_manage
const tmb::client_id cli_id = query_handle->getClientId();
const std::size_t query_id = query_handle->query_id();
+ if (FLAGS_visualize_execution_dag) {
+ ExecutionDAGVisualizer* dag_visualizer = query_manager->dag_visualizer();
+ dag_visualizer->bindProfilingStats(getProfilingResults(query_id));
+
+ std::ostringstream dot_filename;
+ dot_filename << query_id << ".dot";
+
+ FILE *fp = std::fopen(dot_filename.str().c_str(), "w");
+ CHECK_NOTNULL(fp);
+ const std::string dot_file_content(dag_visualizer->toDOT());
+ const std::size_t dot_file_length = dot_file_content.length();
+
+ CHECK_EQ(dot_file_length,
+ std::fwrite(dot_file_content.c_str(), sizeof(char), dot_file_length, fp));
+
+ std::fclose(fp);
+ }
+
// TODO(quickstep-team): Dynamically scale-up/down Shiftbosses.
tmb::Address shiftboss_addresses;
for (std::size_t i = 0; i < shiftboss_directory_->size(); ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/911f3dcd/query_execution/QueryManagerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.cpp b/query_execution/QueryManagerBase.cpp
index 5f8c6a3..f84ad4e 100644
--- a/query_execution/QueryManagerBase.cpp
+++ b/query_execution/QueryManagerBase.cpp
@@ -30,12 +30,18 @@
#include "relational_operators/WorkOrder.hpp"
#include "storage/StorageBlockInfo.hpp"
+#include "gflags/gflags.h"
#include "glog/logging.h"
using std::pair;
namespace quickstep {
+DEFINE_bool(visualize_execution_dag, false,
+ "If true, visualize the execution plan DAG into a graph in DOT "
+ "format (DOT is a plain text graph description language) which is "
+ "then printed via stderr.");
+
QueryManagerBase::QueryManagerBase(QueryHandle *query_handle)
: query_handle_(DCHECK_NOTNULL(query_handle)),
query_id_(query_handle->query_id()),
@@ -45,6 +51,11 @@ QueryManagerBase::QueryManagerBase(QueryHandle *query_handle)
output_consumers_(num_operators_in_dag_),
blocking_dependencies_(num_operators_in_dag_),
query_exec_state_(new QueryExecutionState(num_operators_in_dag_)) {
+ if (FLAGS_visualize_execution_dag) {
+ dag_visualizer_ =
+ std::make_unique<quickstep::ExecutionDAGVisualizer>(query_handle_->getQueryPlan());
+ }
+
for (dag_node_index node_index = 0;
node_index < num_operators_in_dag_;
++node_index) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/911f3dcd/query_execution/QueryManagerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.hpp b/query_execution/QueryManagerBase.hpp
index d0bb0ea..27fa6dc 100644
--- a/query_execution/QueryManagerBase.hpp
+++ b/query_execution/QueryManagerBase.hpp
@@ -31,6 +31,7 @@
#include "relational_operators/WorkOrder.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "utility/DAG.hpp"
+#include "utility/ExecutionDAGVisualizer.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
@@ -149,6 +150,15 @@ class QueryManagerBase {
**/
QueryStatusCode queryStatus(const dag_node_index op_index);
+ /**
+ * @brief Get the execution DAG visualizer.
+ *
+ * @return the execution DAG visualizer.
+ **/
+ ExecutionDAGVisualizer* dag_visualizer() {
+ return dag_visualizer_.get();
+ }
+
protected:
/**
* @brief Process a current relational operator: Get its workorders and store
@@ -276,6 +286,8 @@ class QueryManagerBase {
std::unique_ptr<QueryExecutionState> query_exec_state_;
+ std::unique_ptr<ExecutionDAGVisualizer> dag_visualizer_;
+
private:
/**
* @brief Check if the given operator's normal execution is over.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/911f3dcd/query_optimizer/QueryHandle.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryHandle.hpp b/query_optimizer/QueryHandle.hpp
index cbd1cd9..7cb4f68 100644
--- a/query_optimizer/QueryHandle.hpp
+++ b/query_optimizer/QueryHandle.hpp
@@ -87,6 +87,13 @@ class QueryHandle {
}
/**
+ * @return The const query plan.
+ */
+ const QueryPlan& getQueryPlan() const {
+ return *query_plan_;
+ }
+
+ /**
* @return The mutable query plan.
*/
QueryPlan* getQueryPlanMutable() {
[2/2] incubator-quickstep git commit: Enabled dag viz by default.
Posted by zu...@apache.org.
Enabled dag viz by default.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1ccbb60b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1ccbb60b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1ccbb60b
Branch: refs/heads/dist-patch
Commit: 1ccbb60b47e38c2408bf8752320efa4a2f45be4c
Parents: 911f3dc
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Feb 11 11:40:33 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Feb 11 11:41:25 2017 -0800
----------------------------------------------------------------------
query_execution/QueryManagerBase.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1ccbb60b/query_execution/QueryManagerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.cpp b/query_execution/QueryManagerBase.cpp
index f84ad4e..2dd5467 100644
--- a/query_execution/QueryManagerBase.cpp
+++ b/query_execution/QueryManagerBase.cpp
@@ -37,7 +37,7 @@ using std::pair;
namespace quickstep {
-DEFINE_bool(visualize_execution_dag, false,
+DEFINE_bool(visualize_execution_dag, true,
"If true, visualize the execution plan DAG into a graph in DOT "
"format (DOT is a plain text graph description language) which is "
"then printed via stderr.");