You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by hb...@apache.org on 2016/07/15 21:52:36 UTC
[1/2] incubator-quickstep git commit: Learner keeps a STL set of
priority levels.
Repository: incubator-quickstep
Updated Branches:
refs/heads/scheduler++ 884a50e18 -> eea560336
Learner keeps a STL set of priority levels.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8caae5a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8caae5a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8caae5a0
Branch: refs/heads/scheduler++
Commit: 8caae5a0f87b8e3dfc19c08af6a1595d5dd24048
Parents: 884a50e
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Fri Jul 15 16:30:46 2016 -0500
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Fri Jul 15 16:30:46 2016 -0500
----------------------------------------------------------------------
query_execution/Learner.cpp | 21 ++-------------------
query_execution/Learner.hpp | 12 +++++++-----
2 files changed, 9 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8caae5a0/query_execution/Learner.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Learner.cpp b/query_execution/Learner.cpp
index 264e84e..272d582 100644
--- a/query_execution/Learner.cpp
+++ b/query_execution/Learner.cpp
@@ -40,8 +40,7 @@ DEFINE_uint64(max_past_entries_learner,
"The maximum number of past WorkOrder execution statistics"
" entries for a query");
-Learner::Learner()
- : highest_priority_level_(kInvalidPriorityLevel) {
+Learner::Learner() {
probabilities_of_priority_levels_.reset(new ProbabilityStore());
// Format: Query ID, Operator ID, Worker ID, Time in micros, WO execution end timestamp.
LOG(INFO) << "Query ID|Operator ID|Worker ID|Time in microseconds|Workorder end timestamp";
@@ -232,23 +231,7 @@ void Learner::checkAndRemovePriorityLevel(const std::size_t priority_level) {
default_probabilities_.erase(priority_level);
// NOTE(harshad) : Not using this cache as it gets confusing.
// has_feedback_from_all_queries_.erase(priority_level);
- if (hasActiveQueries()) {
- if (static_cast<int>(priority_level) == highest_priority_level_) {
- // The priority level to be removed is the highest priority level.
- std::size_t new_highest_priority_level = 0;
- // Find the new highest priority level.
- for (auto priority_level_it = execution_stats_.cbegin();
- priority_level_it != execution_stats_.cend();
- ++priority_level_it) {
- if (priority_level_it->first > new_highest_priority_level) {
- new_highest_priority_level = priority_level_it->first;
- }
- }
- highest_priority_level_ = static_cast<int>(new_highest_priority_level);
- }
- } else {
- highest_priority_level_ = kInvalidPriorityLevel;
- }
+ priority_levels_set_.erase(priority_level);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8caae5a0/query_execution/Learner.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Learner.hpp b/query_execution/Learner.hpp
index f61e4f9..36f1509 100644
--- a/query_execution/Learner.hpp
+++ b/query_execution/Learner.hpp
@@ -162,7 +162,11 @@ class Learner {
* kInvalidPriorityLevel.
**/
inline const int getHighestPriorityLevel() const {
- return highest_priority_level_;
+ if (!priority_levels_set_.empty()) {
+ return static_cast<int>(*priority_levels_set_.rbegin());
+ } else {
+ return kInvalidPriorityLevel;
+ }
}
/**
@@ -325,9 +329,7 @@ class Learner {
if (!isPriorityLevelPresent(priority_level)) {
current_probabilities_[priority_level].reset(new ProbabilityStore());
execution_stats_.emplace(priority_level, std::vector<std::pair<std::size_t, std::unique_ptr<ExecutionStats>>>());
- if (static_cast<int>(priority_level) > highest_priority_level_) {
- highest_priority_level_ = priority_level;
- }
+ priority_levels_set_.insert(priority_level);
}
}
@@ -555,7 +557,7 @@ class Learner {
// feedback from all the queries in the given priority level.
// std::unordered_map<std::size_t, bool> has_feedback_from_all_queries_;
- int highest_priority_level_;
+ std::set<std::size_t> priority_levels_set_;
DISALLOW_COPY_AND_ASSIGN(Learner);
};
[2/2] incubator-quickstep git commit: HPF can schedule work from
lower priority levels.
Posted by hb...@apache.org.
HPF can schedule work from lower priority levels.
- If highest priority level doesn't have work and we have some free
cycles, we will schedule work from lower priority levels.
- We will probe priority levels in decreasing order to see if there is
any work to be scheduled.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/eea56033
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/eea56033
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/eea56033
Branch: refs/heads/scheduler++
Commit: eea560336e29d15a4611f7ae4910c878a2e6bd0d
Parents: 8caae5a
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Fri Jul 15 16:38:46 2016 -0500
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Fri Jul 15 16:38:46 2016 -0500
----------------------------------------------------------------------
query_execution/Learner.hpp | 4 ++++
query_execution/PriorityPolicyEnforcer.cpp | 15 +++++++++------
2 files changed, 13 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/eea56033/query_execution/Learner.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Learner.hpp b/query_execution/Learner.hpp
index 36f1509..8f63c19 100644
--- a/query_execution/Learner.hpp
+++ b/query_execution/Learner.hpp
@@ -169,6 +169,10 @@ class Learner {
}
}
+ inline const std::set<std::size_t>& getSetOfPriorityLevels() const {
+ return priority_levels_set_;
+ }
+
/**
* @brief Randomly pick a priority level.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/eea56033/query_execution/PriorityPolicyEnforcer.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PriorityPolicyEnforcer.cpp b/query_execution/PriorityPolicyEnforcer.cpp
index 9bfe979..e0290f1 100644
--- a/query_execution/PriorityPolicyEnforcer.cpp
+++ b/query_execution/PriorityPolicyEnforcer.cpp
@@ -229,18 +229,21 @@ void PriorityPolicyEnforcer::getWorkerMessagesHPF(std::vector<std::unique_ptr<Wo
// been collected.
DCHECK(worker_messages->empty());
if (learner_->hasActiveQueries()) {
+ const std::set<std::size_t> &priority_set = learner_->getSetOfPriorityLevels();
std::unordered_map<std::size_t, bool> finished_queries_ids;
- const int chosen_priority_level = learner_->getHighestPriorityLevel();
- if (chosen_priority_level == kInvalidPriorityLevel) {
- DLOG(INFO) << "No valid priority level chosen";
+ auto priority_set_reverse_it = priority_set.rbegin();
+ if (priority_set.empty()) {
+ DLOG(INFO) << "No priority level available";
return;
}
- while (worker_messages->size() < FLAGS_max_msgs_per_dispatch_round) {
- WorkerMessage *next_worker_message = getNextWorkerMessageFromPriorityLevel(chosen_priority_level, &finished_queries_ids);
+ while (worker_messages->size() < FLAGS_max_msgs_per_dispatch_round && priority_set_reverse_it != priority_set.rend()) {
+ WorkerMessage *next_worker_message = getNextWorkerMessageFromPriorityLevel(*priority_set_reverse_it, &finished_queries_ids);
if (next_worker_message != nullptr) {
worker_messages->push_back(std::unique_ptr<WorkerMessage>(next_worker_message));
} else {
- break;
+ // No work orders available in the current priority level, check the
+ // next largest priority level.
+ ++priority_set_reverse_it;
}
}
for (auto finished_qid_pair : finished_queries_ids) {