You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by hb...@apache.org on 2016/06/16 20:19:00 UTC
incubator-quickstep git commit: Basic support to report individual
work order profiling results [Forced Update!]
Repository: incubator-quickstep
Updated Branches:
refs/heads/workorder-time-reporting 3d1a4be27 -> acad02c99 (forced update)
Basic support to report individual work order profiling results
- A flag to enable work order profiling report generation.
- At the end of each query, a report is generated which includes worker
ID, its NUMA socket, the query ID, the operator that produced the WorkOrder
and the execution time in microseconds.
- The output is printed on stdout in CSV format as of now.
- As this is a rudimentary support for the functionality, there is a lot of
future work in this regards, which includes printing of CPU core information,
printing operator name, allowing user to specify a file where the output can
be written etc.
Added query ID to the output.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/acad02c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/acad02c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/acad02c9
Branch: refs/heads/workorder-time-reporting
Commit: acad02c99e2d7e9fa52f1fa807bf8fc133f5405b
Parents: 8e825f1
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Thu Jun 16 14:03:34 2016 -0500
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Thu Jun 16 15:15:47 2016 -0500
----------------------------------------------------------------------
CMakeLists.txt | 5 ++++
cli/QuickstepCli.cpp | 5 ++++
query_execution/CMakeLists.txt | 13 ++++++++
query_execution/Foreman.cpp | 18 +++++++++++
query_execution/Foreman.hpp | 20 +++++++++++++
query_execution/PolicyEnforcer.cpp | 13 ++++++++
query_execution/PolicyEnforcer.hpp | 41 ++++++++++++++++++++++++++
query_execution/QueryExecutionConfig.h.in | 18 +++++++++++
8 files changed, 133 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/acad02c9/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 20e1fb9..3aa461a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -185,6 +185,11 @@ if (ENABLE_GOOGLE_PROFILER)
set(LIBS ${LIBS} profiler)
endif()
+# This option enables profiling individual work order performances. If enabled,
+# at the end of each query execution, we print on stdout in CSV format the
+# statistics about every work order that was executed.
+option(ENABLE_WORKORDER_PROFILING "Enable profiling individual WorkOrder execution" OFF)
+
# Link against the system's threading library.
find_package(Threads REQUIRED)
set(LIBS ${LIBS} ${CMAKE_THREAD_LIBS_INIT})
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/acad02c9/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 35bd16e..27c4235 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -59,6 +59,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "parser/SqlParserWrapper.hpp"
#include "query_execution/AdmitRequestMessage.hpp"
#include "query_execution/Foreman.hpp"
+#include "query_execution/QueryExecutionConfig.h" // For QUICKSTEP_ENABLE_WORKORDER_PROFILING
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_execution/Worker.hpp"
@@ -461,6 +462,10 @@ int main(int argc, char* argv[]) {
printf("Time: %s ms\n",
quickstep::DoubleToStringWithSignificantDigits(
time_ms.count(), 3).c_str());
+#ifdef QUICKSTEP_ENABLE_WORKORDER_PROFILING
+ // TODO(harshad) - Allow user specified file instead of stdout.
+ foreman.printWorkOrderProfilingResults(stdout);
+#endif
} catch (const std::exception &e) {
fprintf(stderr, "QUERY EXECUTION ERROR: %s\n", e.what());
break;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/acad02c9/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 501166e..a645152 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -1,5 +1,7 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015-2016 Pivotal Software, Inc.
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -19,12 +21,23 @@ QS_PROTOBUF_GENERATE_CPP(queryexecution_QueryExecutionMessages_proto_srcs
queryexecution_QueryExecutionMessages_proto_hdrs
QueryExecutionMessages.proto)
+if (ENABLE_WORKORDER_PROFILING)
+ set(QUICKSTEP_ENABLE_WORKORDER_PROFILING TRUE)
+else()
+ set(QUICKSTEP_ENABLE_WORKORDER_PROFILING FALSE)
+endif()
+
if (BUILD_SHARED_LIBS)
set(GFLAGS_LIB_NAME gflags_nothreads-shared)
else()
set(GFLAGS_LIB_NAME gflags_nothreads-static)
endif()
+configure_file (
+ "${CMAKE_CURRENT_SOURCE_DIR}/QueryExecutionConfig.h.in"
+ "${CMAKE_CURRENT_BINARY_DIR}/QueryExecutionConfig.h"
+)
+
# Declare micro-libs:
if (ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp BlockLocator.hpp)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/acad02c9/query_execution/Foreman.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Foreman.cpp b/query_execution/Foreman.cpp
index 828834d..6dbe392 100644
--- a/query_execution/Foreman.cpp
+++ b/query_execution/Foreman.cpp
@@ -229,4 +229,22 @@ void Foreman::sendWorkerMessage(const size_t worker_thread_index,
<< worker_directory_->getClientID(worker_thread_index);
}
+#ifdef QUICKSTEP_ENABLE_WORKORDER_PROFILING
+void Foreman::printWorkOrderProfilingResults(std::FILE *out) const {
+ const std::vector<
+ std::tuple<std::size_t, std::size_t, std::size_t, std::size_t>>
+ &recorded_times = policy_enforcer_->getProfilingResults();
+ fputs("Worker ID, NUMA Socket, Query ID, Operator ID, Time (microseconds)\n", out);
+ for (auto workorder_entry : recorded_times) {
+ // Note: Index of the "worker thread index" in the tuple is 0.
+ const std::size_t worker_id = std::get<0>(workorder_entry);
+ fprintf(out, "%lu,", worker_id);
+ fprintf(out, "%d,", worker_directory_->getNUMANode(worker_id));
+ fprintf(out, "%lu,", std::get<1>(workorder_entry));
+ fprintf(out, "%lu,", std::get<2>(workorder_entry));
+ fprintf(out, "%lu\n", std::get<3>(workorder_entry));
+ }
+}
+#endif
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/acad02c9/query_execution/Foreman.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Foreman.hpp b/query_execution/Foreman.hpp
index 94cb9fc..db49ee2 100644
--- a/query_execution/Foreman.hpp
+++ b/query_execution/Foreman.hpp
@@ -18,6 +18,13 @@
#ifndef QUICKSTEP_QUERY_EXECUTION_FOREMAN_HPP_
#define QUICKSTEP_QUERY_EXECUTION_FOREMAN_HPP_
+#include "query_execution/QueryExecutionConfig.h"
+
+#ifdef QUICKSTEP_ENABLE_WORKORDER_PROFILING
+#include <cstdio>
+#include <tuple>
+#endif
+
#include <cstddef>
#include <memory>
#include <vector>
@@ -71,6 +78,19 @@ class Foreman final : public ForemanLite {
~Foreman() override {}
+#ifdef QUICKSTEP_ENABLE_WORKORDER_PROFILING
+ /**
+ * @brief Print the results of profiling individual work orders.
+ *
+ * TODO(harshad) - Add the name of the operator to the output.
+ * TODO(harshad) - Add the CPU core ID of the operator to the output. This
+ * will require modifying the WorkerDirectory to remember worker affinities.
+ *
+ * @param out The file stream.
+ **/
+ void printWorkOrderProfilingResults(std::FILE *out) const;
+#endif
+
protected:
void run() override;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/acad02c9/query_execution/PolicyEnforcer.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcer.cpp b/query_execution/PolicyEnforcer.cpp
index 9f0502d..78343e3 100644
--- a/query_execution/PolicyEnforcer.cpp
+++ b/query_execution/PolicyEnforcer.cpp
@@ -76,6 +76,9 @@ void PolicyEnforcer::processMessage(const TaggedMessage &tagged_message) {
query_id = proto.query_id();
worker_directory_->decrementNumQueuedWorkOrders(
proto.worker_thread_index());
+#ifdef QUICKSTEP_ENABLE_WORKORDER_PROFILING
+ recordTimeForWorkOrder(proto);
+#endif
break;
}
case kRebuildWorkOrderCompleteMessage: {
@@ -197,4 +200,14 @@ bool PolicyEnforcer::admitQueries(
return true;
}
+#ifdef QUICKSTEP_ENABLE_WORKORDER_PROFILING
+void PolicyEnforcer::recordTimeForWorkOrder(
+ const serialization::NormalWorkOrderCompletionMessage &proto) {
+ workorder_time_recorder_.emplace_back(proto.worker_thread_index(),
+ proto.query_id(),
+ proto.operator_index(),
+ proto.execution_time_in_microseconds());
+}
+#endif
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/acad02c9/query_execution/PolicyEnforcer.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcer.hpp b/query_execution/PolicyEnforcer.hpp
index 9f87056..ca8012d 100644
--- a/query_execution/PolicyEnforcer.hpp
+++ b/query_execution/PolicyEnforcer.hpp
@@ -24,6 +24,12 @@
#include <unordered_map>
#include <vector>
+#include "query_execution/QueryExecutionConfig.h"
+
+#ifdef QUICKSTEP_ENABLE_WORKORDER_PROFILING
+#include <tuple>
+#endif
+
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryManager.hpp"
#include "query_execution/WorkerMessage.hpp"
@@ -143,9 +149,35 @@ class PolicyEnforcer {
return !(admitted_queries_.empty() && waiting_queries_.empty());
}
+#ifdef QUICKSTEP_ENABLE_WORKORDER_PROFILING
+ /**
+ * @brief Get the profiling results for individual work order execution.
+ **/
+ inline const std::vector<
+ std::tuple<std::size_t, std::size_t, std::size_t, std::size_t>>&
+ getProfilingResults() const {
+ return workorder_time_recorder_;
+ }
+#endif
+
private:
static constexpr std::size_t kMaxConcurrentQueries = 1;
+#ifdef QUICKSTEP_ENABLE_WORKORDER_PROFILING
+ /**
+ * @brief Record the execution time for a finished WorkOrder.
+ *
+ * TODO(harshad) - Extend the functionality to rebuild work orders.
+ *
+ * @param proto The completion message proto sent after the WorkOrder
+ * execution.
+ **/
+ void recordTimeForWorkOrder(
+ const serialization::NormalWorkOrderCompletionMessage &proto);
+
+ void resetTimeRecord();
+#endif
+
const tmb::client_id foreman_client_id_;
const std::size_t num_numa_nodes_;
@@ -161,6 +193,15 @@ class PolicyEnforcer {
// The queries which haven't been admitted yet.
std::queue<QueryHandle*> waiting_queries_;
+#ifdef QUICKSTEP_ENABLE_WORKORDER_PROFILING
+ // Each entry indicates a record of executing a work order.
+ // 1st element: Logical worker ID.
+ // 2nd element: Query ID.
+ // 2nd element: Operator ID.
+ // 3rd element: Time in microseconds to execute the work order.
+ std::vector<std::tuple<std::size_t, std::size_t, std::size_t, std::size_t>> workorder_time_recorder_;
+#endif
+
DISALLOW_COPY_AND_ASSIGN(PolicyEnforcer);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/acad02c9/query_execution/QueryExecutionConfig.h.in
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionConfig.h.in b/query_execution/QueryExecutionConfig.h.in
new file mode 100644
index 0000000..a7f3efc
--- /dev/null
+++ b/query_execution/QueryExecutionConfig.h.in
@@ -0,0 +1,18 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#cmakedefine QUICKSTEP_ENABLE_WORKORDER_PROFILING