You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by zu...@apache.org on 2017/02/11 20:55:00 UTC
incubator-quickstep git commit: Visualized Execution DAG in the
distributed version. [Forced Update!]
Repository: incubator-quickstep
Updated Branches:
refs/heads/dag_viz 7b7ba09d6 -> 303db0ee1 (forced update)
Visualized Execution DAG in the distributed version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/303db0ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/303db0ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/303db0ee
Branch: refs/heads/dag_viz
Commit: 303db0ee179ebb37aabe211a97f8e4fadfa87eba
Parents: ab46d78
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Feb 10 22:01:48 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Feb 11 12:54:50 2017 -0800
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 5 ++++-
query_execution/PolicyEnforcerBase.cpp | 7 ++----
query_execution/PolicyEnforcerBase.hpp | 16 +++++++++++--
query_execution/PolicyEnforcerDistributed.cpp | 26 +++++++++++++++++++++-
query_execution/QueryManagerBase.cpp | 11 +++++++++
query_execution/QueryManagerBase.hpp | 12 ++++++++++
query_optimizer/QueryHandle.hpp | 7 ++++++
7 files changed, 75 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/303db0ee/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 5ad6999..50bf694 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -166,6 +166,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_queryexecution_ShiftbossDirectory
quickstep_queryoptimizer_QueryHandle
quickstep_storage_StorageBlockInfo
+ quickstep_utility_ExecutionDAGVisualizer
quickstep_utility_Macros
tmb
${GFLAGS_LIB_NAME})
@@ -246,7 +247,9 @@ target_link_libraries(quickstep_queryexecution_QueryManagerBase
quickstep_relationaloperators_WorkOrder
quickstep_storage_StorageBlockInfo
quickstep_utility_DAG
- quickstep_utility_Macros)
+ quickstep_utility_ExecutionDAGVisualizer
+ quickstep_utility_Macros
+ ${GFLAGS_LIB_NAME})
if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution_QueryManagerDistributed
quickstep_catalog_CatalogTypedefs
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/303db0ee/query_execution/PolicyEnforcerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.cpp b/query_execution/PolicyEnforcerBase.cpp
index 082f6e9..1ffde4d 100644
--- a/query_execution/PolicyEnforcerBase.cpp
+++ b/query_execution/PolicyEnforcerBase.cpp
@@ -40,15 +40,12 @@
namespace quickstep {
+DECLARE_bool(visualize_execution_dag);
+
DEFINE_bool(profile_and_report_workorder_perf, false,
"If true, Quickstep will record the exceution time of all the individual "
"normal work orders and report it at the end of query execution.");
-DEFINE_bool(visualize_execution_dag, false,
- "If true, visualize the execution plan DAG into a graph in DOT "
- "format (DOT is a plain text graph description language) which is "
- "then printed via stderr.");
-
PolicyEnforcerBase::PolicyEnforcerBase(CatalogDatabaseLite *catalog_database)
: catalog_database_(catalog_database),
profile_individual_workorders_(FLAGS_profile_and_report_workorder_perf || FLAGS_visualize_execution_dag) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/303db0ee/query_execution/PolicyEnforcerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.hpp b/query_execution/PolicyEnforcerBase.hpp
index 4107817..f66134b 100644
--- a/query_execution/PolicyEnforcerBase.hpp
+++ b/query_execution/PolicyEnforcerBase.hpp
@@ -113,6 +113,19 @@ class PolicyEnforcerBase {
}
/**
+ * @brief Check if the given query has profiling results.
+ *
+ * @note Even enabled profiling, not every query has profiling results.
+ * For example, CreateTable and CreateIndex do not produce work orders,
+ * so they do not have profiling results.
+ *
+ * @return True if it has profiling results, otherwise false.
+ **/
+ bool hasProfilingResults(const std::size_t query_id) const {
+ return workorder_time_recorder_.find(query_id) != workorder_time_recorder_.end();
+ }
+
+ /**
* @brief Get the profiling results for individual work order execution for a
* given query.
*
@@ -127,8 +140,7 @@ class PolicyEnforcerBase {
inline const std::vector<WorkOrderTimeEntry>& getProfilingResults(
const std::size_t query_id) const {
DCHECK(profile_individual_workorders_);
- DCHECK(workorder_time_recorder_.find(query_id) !=
- workorder_time_recorder_.end());
+ DCHECK(hasProfilingResults(query_id));
return workorder_time_recorder_.at(query_id);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/303db0ee/query_execution/PolicyEnforcerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.cpp b/query_execution/PolicyEnforcerDistributed.cpp
index 8f0332d..6ee58a8 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -15,11 +15,14 @@
#include "query_execution/PolicyEnforcerDistributed.hpp"
#include <cstddef>
+#include <cstdio>
#include <cstdlib>
#include <memory>
#include <queue>
-#include <utility>
+#include <sstream>
+#include <string>
#include <unordered_map>
+#include <utility>
#include <vector>
#include "catalog/Catalog.pb.h"
@@ -33,6 +36,7 @@
#include "query_execution/QueryManagerDistributed.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "storage/StorageBlockInfo.hpp"
+#include "utility/ExecutionDAGVisualizer.hpp"
#include "gflags/gflags.h"
#include "glog/logging.h"
@@ -53,6 +57,8 @@ using tmb::TaggedMessage;
namespace quickstep {
+DECLARE_bool(visualize_execution_dag);
+
namespace S = serialization;
DEFINE_uint64(max_msgs_per_dispatch_round, 20, "Maximum number of messages that"
@@ -228,6 +234,24 @@ void PolicyEnforcerDistributed::onQueryCompletion(QueryManagerBase *query_manage
const tmb::client_id cli_id = query_handle->getClientId();
const std::size_t query_id = query_handle->query_id();
+ if (FLAGS_visualize_execution_dag && hasProfilingResults(query_id)) {
+ ExecutionDAGVisualizer* dag_visualizer = query_manager->dag_visualizer();
+ dag_visualizer->bindProfilingStats(getProfilingResults(query_id));
+
+ std::ostringstream dot_filename;
+ dot_filename << query_id << ".dot";
+
+ FILE *fp = std::fopen(dot_filename.str().c_str(), "w");
+ CHECK_NOTNULL(fp);
+ const std::string dot_file_content(dag_visualizer->toDOT());
+ const std::size_t dot_file_length = dot_file_content.length();
+
+ CHECK_EQ(dot_file_length,
+ std::fwrite(dot_file_content.c_str(), sizeof(char), dot_file_length, fp));
+
+ std::fclose(fp);
+ }
+
// TODO(quickstep-team): Dynamically scale-up/down Shiftbosses.
tmb::Address shiftboss_addresses;
for (std::size_t i = 0; i < shiftboss_directory_->size(); ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/303db0ee/query_execution/QueryManagerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.cpp b/query_execution/QueryManagerBase.cpp
index 5f8c6a3..f84ad4e 100644
--- a/query_execution/QueryManagerBase.cpp
+++ b/query_execution/QueryManagerBase.cpp
@@ -30,12 +30,18 @@
#include "relational_operators/WorkOrder.hpp"
#include "storage/StorageBlockInfo.hpp"
+#include "gflags/gflags.h"
#include "glog/logging.h"
using std::pair;
namespace quickstep {
+DEFINE_bool(visualize_execution_dag, false,
+ "If true, visualize the execution plan DAG into a graph in DOT "
+ "format (DOT is a plain text graph description language) which is "
+ "then printed via stderr.");
+
QueryManagerBase::QueryManagerBase(QueryHandle *query_handle)
: query_handle_(DCHECK_NOTNULL(query_handle)),
query_id_(query_handle->query_id()),
@@ -45,6 +51,11 @@ QueryManagerBase::QueryManagerBase(QueryHandle *query_handle)
output_consumers_(num_operators_in_dag_),
blocking_dependencies_(num_operators_in_dag_),
query_exec_state_(new QueryExecutionState(num_operators_in_dag_)) {
+ if (FLAGS_visualize_execution_dag) {
+ dag_visualizer_ =
+ std::make_unique<quickstep::ExecutionDAGVisualizer>(query_handle_->getQueryPlan());
+ }
+
for (dag_node_index node_index = 0;
node_index < num_operators_in_dag_;
++node_index) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/303db0ee/query_execution/QueryManagerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.hpp b/query_execution/QueryManagerBase.hpp
index d0bb0ea..27fa6dc 100644
--- a/query_execution/QueryManagerBase.hpp
+++ b/query_execution/QueryManagerBase.hpp
@@ -31,6 +31,7 @@
#include "relational_operators/WorkOrder.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "utility/DAG.hpp"
+#include "utility/ExecutionDAGVisualizer.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
@@ -149,6 +150,15 @@ class QueryManagerBase {
**/
QueryStatusCode queryStatus(const dag_node_index op_index);
+ /**
+ * @brief Get the execution DAG visualizer.
+ *
+ * @return the execution DAG visualizer.
+ **/
+ ExecutionDAGVisualizer* dag_visualizer() {
+ return dag_visualizer_.get();
+ }
+
protected:
/**
* @brief Process a current relational operator: Get its workorders and store
@@ -276,6 +286,8 @@ class QueryManagerBase {
std::unique_ptr<QueryExecutionState> query_exec_state_;
+ std::unique_ptr<ExecutionDAGVisualizer> dag_visualizer_;
+
private:
/**
* @brief Check if the given operator's normal execution is over.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/303db0ee/query_optimizer/QueryHandle.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryHandle.hpp b/query_optimizer/QueryHandle.hpp
index cbd1cd9..7cb4f68 100644
--- a/query_optimizer/QueryHandle.hpp
+++ b/query_optimizer/QueryHandle.hpp
@@ -87,6 +87,13 @@ class QueryHandle {
}
/**
+ * @return The const query plan.
+ */
+ const QueryPlan& getQueryPlan() const {
+ return *query_plan_;
+ }
+
+ /**
* @return The mutable query plan.
*/
QueryPlan* getQueryPlanMutable() {