You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by ji...@apache.org on 2017/02/21 05:05:18 UTC
incubator-quickstep git commit: Initial commit [Forced Update!]
Repository: incubator-quickstep
Updated Branches:
refs/heads/LIP-time-decomposition 0ba3eb695 -> 1e56713ee (forced update)
Initial commit
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1e56713e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1e56713e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1e56713e
Branch: refs/heads/LIP-time-decomposition
Commit: 1e56713ee9b7c24f3ae786a3f4f48e96951c2d71
Parents: f6480fb
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Mon Feb 20 21:38:55 2017 -0600
Committer: Jianqiao Zhu <ji...@cs.wisc.edu>
Committed: Mon Feb 20 23:05:38 2017 -0600
----------------------------------------------------------------------
cli/QuickstepCli.cpp | 15 ++
relational_operators/BuildHashOperator.cpp | 9 +
relational_operators/CMakeLists.txt | 2 +
relational_operators/HashJoinOperator.cpp | 7 +
utility/CMakeLists.txt | 4 +
utility/EventProfiler.cpp | 30 ++++
utility/EventProfiler.hpp | 190 ++++++++++++++++++++
utility/lip_filter/CMakeLists.txt | 1 +
utility/lip_filter/LIPFilterAdaptiveProber.hpp | 10 +-
9 files changed, 266 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1e56713e/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index eddee8c..c10b655 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -23,6 +23,7 @@
#include <cstddef>
#include <cstdio>
#include <exception>
+#include <fstream>
#include <memory>
#include <string>
#include <utility>
@@ -69,6 +70,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "storage/StorageConstants.hpp"
#include "storage/StorageManager.hpp"
#include "threading/ThreadIDBasedMap.hpp"
+#include "utility/EventProfiler.hpp"
#include "utility/ExecutionDAGVisualizer.hpp"
#include "utility/Macros.hpp"
#include "utility/PtrVector.hpp"
@@ -156,6 +158,8 @@ DEFINE_string(profile_file_name, "",
// To put things in perspective, the first run is, in my experiments, about 5-10
// times more expensive than the average run. That means the query needs to be
// run at least a hundred times to make the impact of the first run small (< 5 %).
+DEFINE_string(profile_output, "",
+ "Output file name for dumping the profiled events.");
DECLARE_bool(profile_and_report_workorder_perf);
DECLARE_bool(visualize_execution_dag);
@@ -344,6 +348,8 @@ int main(int argc, char* argv[]) {
const std::size_t query_id = query_processor->query_id();
const CatalogRelation *query_result_relation = nullptr;
std::unique_ptr<quickstep::ExecutionDAGVisualizer> dag_visualizer;
+ quickstep::simple_profiler.clear();
+ auto *event_container = quickstep::simple_profiler.getContainer();
try {
auto query_handle = std::make_unique<QueryHandle>(query_id,
@@ -360,6 +366,7 @@ int main(int argc, char* argv[]) {
query_result_relation = query_handle->getQueryResultRelation();
start = std::chrono::steady_clock::now();
+ event_container->startEvent("overall");
QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
main_thread_client_id,
foreman.getBusClientID(),
@@ -374,6 +381,7 @@ int main(int argc, char* argv[]) {
try {
QueryExecutionUtil::ReceiveQueryCompletionMessage(
main_thread_client_id, &bus);
+ event_container->endEvent("overall");
end = std::chrono::steady_clock::now();
if (query_result_relation) {
@@ -405,6 +413,13 @@ int main(int argc, char* argv[]) {
dag_visualizer->bindProfilingStats(profiling_stats);
std::cerr << "\n" << dag_visualizer->toDOT() << "\n";
}
+ if (!quickstep::FLAGS_profile_output.empty()) {
+ std::ofstream ofs(
+ quickstep::FLAGS_profile_output + std::to_string(query_processor->query_id()),
+ std::ios::out);
+ quickstep::simple_profiler.writeToStream(ofs);
+ ofs.close();
+ }
} catch (const std::exception &e) {
fprintf(stderr, "QUERY EXECUTION ERROR: %s\n", e.what());
break;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1e56713e/relational_operators/BuildHashOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/BuildHashOperator.cpp b/relational_operators/BuildHashOperator.cpp
index 8f40fbb..0ec2915 100644
--- a/relational_operators/BuildHashOperator.cpp
+++ b/relational_operators/BuildHashOperator.cpp
@@ -34,6 +34,7 @@
#include "storage/TupleReference.hpp"
#include "storage/TupleStorageSubBlock.hpp"
#include "storage/ValueAccessor.hpp"
+#include "utility/EventProfiler.hpp"
#include "utility/lip_filter/LIPFilterBuilder.hpp"
#include "utility/lip_filter/LIPFilterUtil.hpp"
@@ -157,12 +158,19 @@ void BuildHashWorkOrder::execute() {
TupleReferenceGenerator generator(build_block_id_);
std::unique_ptr<ValueAccessor> accessor(block->getTupleStorageSubBlock().createValueAccessor());
+ auto *container = simple_profiler.getContainer();
+
// Build LIPFilters if enabled.
if (lip_filter_builder_ != nullptr) {
+ auto *event_lip = container->getEventLine("BuildLIP");
+ event_lip->emplace_back();
lip_filter_builder_->insertValueAccessor(accessor.get());
accessor->beginIterationVirtual();
+ event_lip->back().endEvent();
}
+ auto *event_hash = container->getEventLine("BuildHash");
+ event_hash->emplace_back();
HashTablePutResult result;
if (join_key_attributes_.size() == 1) {
result = hash_table_->putValueAccessor(accessor.get(),
@@ -175,6 +183,7 @@ void BuildHashWorkOrder::execute() {
any_join_key_attributes_nullable_,
&generator);
}
+ event_hash->back().endEvent();
CHECK(result == HashTablePutResult::kOK)
<< "Failed to add entries to join hash table.";
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1e56713e/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index 457d58a..fd44cfc 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -141,6 +141,7 @@ target_link_libraries(quickstep_relationaloperators_BuildHashOperator
quickstep_storage_TupleReference
quickstep_storage_TupleStorageSubBlock
quickstep_storage_ValueAccessor
+ quickstep_utility_EventProfiler
quickstep_utility_Macros
quickstep_utility_lipfilter_LIPFilterBuilder
quickstep_utility_lipfilter_LIPFilterUtil
@@ -281,6 +282,7 @@ target_link_libraries(quickstep_relationaloperators_HashJoinOperator
quickstep_types_TypedValue
quickstep_types_containers_ColumnVector
quickstep_types_containers_ColumnVectorsValueAccessor
+ quickstep_utility_EventProfiler
quickstep_utility_Macros
quickstep_utility_lipfilter_LIPFilterAdaptiveProber
quickstep_utility_lipfilter_LIPFilterUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1e56713e/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index 0e75411..3f6eaab 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -50,6 +50,7 @@
#include "types/TypedValue.hpp"
#include "types/containers/ColumnVector.hpp"
#include "types/containers/ColumnVectorsValueAccessor.hpp"
+#include "utility/EventProfiler.hpp"
#include "utility/lip_filter/LIPFilterAdaptiveProber.hpp"
#include "utility/lip_filter/LIPFilterUtil.hpp"
@@ -470,11 +471,17 @@ void HashInnerJoinWorkOrder::execute() {
base_accessor->createSharedTupleIdSequenceAdapterVirtual(*existence_map));
}
+ auto *container = simple_profiler.getContainer();
+ auto *event_hash = container->getEventLine("ProbeHash");
+ event_hash->emplace_back();
+
if (probe_accessor->getImplementationType() == ValueAccessor::Implementation::kSplitRowStore) {
executeWithCopyElision(probe_accessor.get());
} else {
executeWithoutCopyElision(probe_accessor.get());
}
+
+ event_hash->back().endEvent();
}
void HashInnerJoinWorkOrder::executeWithoutCopyElision(ValueAccessor *probe_accessor) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1e56713e/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index ca04462..c4f8160 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -178,6 +178,7 @@ add_library(quickstep_utility_BarrieredReadWriteConcurrentBitVector
add_library(quickstep_utility_DAG ../empty_src.cpp DAG.hpp)
add_library(quickstep_utility_DisjointTreeForest ../empty_src.cpp DisjointTreeForest.hpp)
add_library(quickstep_utility_EqualsAnyConstant ../empty_src.cpp EqualsAnyConstant.hpp)
+add_library(quickstep_utility_EventProfiler EventProfiler.cpp EventProfiler.hpp)
add_library(quickstep_utility_ExecutionDAGVisualizer
ExecutionDAGVisualizer.cpp
ExecutionDAGVisualizer.hpp)
@@ -249,6 +250,8 @@ target_link_libraries(quickstep_utility_DAG
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_DisjointTreeForest
glog)
+target_link_libraries(quickstep_utility_EventProfiler
+ quickstep_threading_Mutex)
target_link_libraries(quickstep_utility_ExecutionDAGVisualizer
quickstep_catalog_CatalogRelationSchema
quickstep_queryexecution_QueryExecutionTypedefs
@@ -348,6 +351,7 @@ target_link_libraries(quickstep_utility
quickstep_utility_DAG
quickstep_utility_DisjointTreeForest
quickstep_utility_EqualsAnyConstant
+ quickstep_utility_EventProfiler
quickstep_utility_ExecutionDAGVisualizer
quickstep_utility_Glob
quickstep_utility_HashPair
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1e56713e/utility/EventProfiler.cpp
----------------------------------------------------------------------
diff --git a/utility/EventProfiler.cpp b/utility/EventProfiler.cpp
new file mode 100644
index 0000000..cf89cb9
--- /dev/null
+++ b/utility/EventProfiler.cpp
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "utility/EventProfiler.hpp"
+
+#include <cstddef>
+#include <string>
+#include <vector>
+
+namespace quickstep {
+
+EventProfiler<std::string> simple_profiler;
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1e56713e/utility/EventProfiler.hpp
----------------------------------------------------------------------
diff --git a/utility/EventProfiler.hpp b/utility/EventProfiler.hpp
new file mode 100644
index 0000000..c28f49b
--- /dev/null
+++ b/utility/EventProfiler.hpp
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_UTILITY_EVENT_PROFILER_HPP_
+#define QUICKSTEP_UTILITY_EVENT_PROFILER_HPP_
+
+#include <chrono>
+#include <cstddef>
+#include <cstring>
+#include <ctime>
+#include <iomanip>
+#include <map>
+#include <ostream>
+#include <thread>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "threading/Mutex.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+/** \addtogroup Utility
+ * @{
+ */
+
+using clock = std::chrono::steady_clock;
+
+template <typename TagT, typename ...PayloadT>
+class EventProfiler {
+
+ public:
+ EventProfiler()
+ : zero_time_(clock::now()) {
+ }
+
+ struct EventInfo {
+ clock::time_point start_time;
+ clock::time_point end_time;
+ bool is_finished;
+ std::tuple<PayloadT...> payload;
+
+ explicit EventInfo(const clock::time_point &start_time_in)
+ : start_time(start_time_in),
+ is_finished(false) {
+ }
+
+ EventInfo()
+ : start_time(clock::now()),
+ is_finished(false) {
+ }
+
+ inline void setPayload(PayloadT &&...in_payload) {
+ payload = std::make_tuple(in_payload...);
+ }
+
+ inline void endEvent() {
+ end_time = clock::now();
+ is_finished = true;
+ }
+ };
+
+ struct EventContainer {
+ EventContainer()
+ : context(0) {}
+
+ inline void startEvent(const TagT &tag) {
+ events[tag].emplace_back(clock::now());
+ }
+
+ inline void endEvent(const TagT &tag) {
+ auto &event_info = events.at(tag).back();
+ event_info.is_finished = true;
+ event_info.end_time = clock::now();
+ }
+
+ inline std::vector<EventInfo> *getEventLine(const TagT &tag) {
+ return &events[tag];
+ }
+
+ inline void setContext(int context_in) {
+ context = context_in;
+ }
+
+ inline int getContext() const {
+ return context;
+ }
+
+ std::map<TagT, std::vector<EventInfo>> events;
+ int context;
+ };
+
+ EventContainer *getContainer() {
+ MutexLock lock(mutex_);
+ return &thread_map_[std::this_thread::get_id()];
+ }
+
+ void writeToStream(std::ostream &os) const {
+ time_t rawtime;
+ time(&rawtime);
+ char event_id[32];
+ strftime(event_id, sizeof event_id, "%Y-%m-%d %H:%M:%S", localtime(&rawtime));
+
+ int thread_id = 0;
+ for (const auto &thread_ctx : thread_map_) {
+ for (const auto &event_group : thread_ctx.second.events) {
+ for (const auto &event_info : event_group.second) {
+ CHECK(event_info.is_finished)
+ << "Unfinished profiling event at thread " << thread_id
+ << ": " << event_group.first;
+
+ os << std::setprecision(12)
+ << event_id << ","
+ << thread_id << "," << event_group.first << ",";
+
+ PrintTuple(os, event_info.payload, ",");
+
+ os << std::chrono::duration<double>(event_info.start_time - zero_time_).count()
+ << ","
+ << std::chrono::duration<double>(event_info.end_time - zero_time_).count()
+ << "\n";
+ }
+ }
+ ++thread_id;
+ }
+ }
+
+ void clear() {
+ zero_time_ = clock::now();
+ thread_map_.clear();
+ }
+
+ const std::map<std::thread::id, EventContainer> &containers() {
+ return thread_map_;
+ }
+
+ const clock::time_point &zero_time() {
+ return zero_time_;
+ }
+
+ private:
+ template<class Tuple, std::size_t N>
+ struct TuplePrinter {
+ static void Print(std::ostream &os, const Tuple &t, const std::string &sep) {
+ TuplePrinter<Tuple, N-1>::Print(os, t, sep);
+ os << std::get<N-1>(t) << sep;
+ }
+ };
+
+ template<class Tuple>
+ struct TuplePrinter<Tuple, 0> {
+ static void Print(std::ostream &os, const Tuple &t, const std::string &sep) {
+ }
+ };
+
+ template<class... Args>
+ static void PrintTuple(std::ostream &os, const std::tuple<Args...>& t, const std::string &sep) {
+ TuplePrinter<decltype(t), sizeof...(Args)>::Print(os, t, sep);
+ }
+
+ clock::time_point zero_time_;
+ std::map<std::thread::id, EventContainer> thread_map_;
+ Mutex mutex_;
+};
+
+extern EventProfiler<std::string> simple_profiler;
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_UTILITY_EVENT_PROFILER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1e56713e/utility/lip_filter/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/lip_filter/CMakeLists.txt b/utility/lip_filter/CMakeLists.txt
index 519d3e9..4f7dd9a 100644
--- a/utility/lip_filter/CMakeLists.txt
+++ b/utility/lip_filter/CMakeLists.txt
@@ -51,6 +51,7 @@ target_link_libraries(quickstep_utility_lipfilter_LIPFilterAdaptiveProber
quickstep_storage_TupleIdSequence
quickstep_storage_ValueAccessor
quickstep_types_Type
+ quickstep_utility_EventProfiler
quickstep_utility_Macros
quickstep_utility_lipfilter_LIPFilter)
target_link_libraries(quickstep_utility_lipfilter_LIPFilterBuilder
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1e56713e/utility/lip_filter/LIPFilterAdaptiveProber.hpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilterAdaptiveProber.hpp b/utility/lip_filter/LIPFilterAdaptiveProber.hpp
index e1a75d6..b2f498f 100644
--- a/utility/lip_filter/LIPFilterAdaptiveProber.hpp
+++ b/utility/lip_filter/LIPFilterAdaptiveProber.hpp
@@ -31,6 +31,7 @@
#include "storage/TupleIdSequence.hpp"
#include "storage/ValueAccessor.hpp"
#include "types/Type.hpp"
+#include "utility/EventProfiler.hpp"
#include "utility/Macros.hpp"
#include "utility/lip_filter/LIPFilter.hpp"
@@ -87,12 +88,17 @@ class LIPFilterAdaptiveProber {
* @return A TupleIdSequence for the hit tuples in the ValueAccessor.
*/
TupleIdSequence* filterValueAccessor(ValueAccessor *accessor) {
+ TupleIdSequence *matches;
const TupleIdSequence *existence_map = accessor->getTupleIdSequenceVirtual();
+ auto *event_lip = simple_profiler.getContainer()->getEventLine("ProbeLIP");
+ event_lip->emplace_back();
if (existence_map == nullptr) {
- return filterValueAccessorNoExistenceMap(accessor);
+ matches = filterValueAccessorNoExistenceMap(accessor);
} else {
- return filterValueAccessorWithExistenceMap(accessor, existence_map);
+ matches = filterValueAccessorWithExistenceMap(accessor, existence_map);
}
+ event_lip->back().endEvent();
+ return matches;
}
private: