You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by hb...@apache.org on 2016/06/25 14:40:44 UTC
[11/11] incubator-quickstep git commit: Added GFLAG to learner.
Added GFLAG to learner.
- To control the number of number of work order execution statistics
that are maintained in the Learner, for a given query.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9ad3281f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9ad3281f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9ad3281f
Branch: refs/heads/scheduler++
Commit: 9ad3281f2c358a4232413792565e573cfa358c9f
Parents: 35bf47e
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Fri Jun 24 11:39:33 2016 -0500
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Sat Jun 25 09:40:02 2016 -0500
----------------------------------------------------------------------
query_execution/Learner.cpp | 25 +++++++-
query_execution/Learner.hpp | 76 ++++++++++++-------------
query_execution/tests/Learner_unittest.cpp | 17 +++---
3 files changed, 69 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9ad3281f/query_execution/Learner.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Learner.cpp b/query_execution/Learner.cpp
index 38a773b..0f17e7a 100644
--- a/query_execution/Learner.cpp
+++ b/query_execution/Learner.cpp
@@ -34,6 +34,11 @@
namespace quickstep {
+DEFINE_uint64(max_past_entries_learner,
+ 10,
+ "The maximum number of past WorkOrder execution statistics"
+ " entries for a query");
+
void Learner::addCompletionFeedback(
const serialization::NormalWorkOrderCompletionMessage
&workorder_completion_proto) {
@@ -90,8 +95,7 @@ void Learner::updateProbabilitiesForQueriesInPriorityLevel(
}
}
-void Learner::updateProbabilitiesOfAllPriorityLevels(
- const std::size_t priority_level) {
+void Learner::updateProbabilitiesOfAllPriorityLevels() {
if (!hasFeedbackFromAllPriorityLevels() ||
has_feedback_from_all_queries_.empty()) {
// Either we don't have enough feedback messages from all the priority
@@ -114,7 +118,7 @@ void Learner::updateProbabilitiesOfAllPriorityLevels(
total_time_curr_level += mean_workorder_entry.second;
}
const std::size_t num_queries_in_priority_level =
- execution_stats_[priority_level].size();
+ execution_stats_[curr_priority_level].size();
DCHECK_GT(num_queries_in_priority_level, 0u);
predicted_time_for_level[curr_priority_level] =
total_time_curr_level / num_queries_in_priority_level;
@@ -195,4 +199,19 @@ void Learner::initializeDefaultProbabilitiesForPriorityLevels() {
}
}
+void Learner::initializeQuery(const QueryHandle &query_handle) {
+ const std::size_t priority_level = query_handle.query_priority();
+ const std::size_t query_id = query_handle.query_id();
+ DCHECK(isPriorityLevelPresent(priority_level));
+ query_id_to_priority_lookup_[query_id] = priority_level;
+ // TODO(harshad) - Create a gflag for max_past_entries_learner.
+ execution_stats_[priority_level].emplace_back(
+ query_id,
+ std::unique_ptr<ExecutionStats>(
+ new ExecutionStats(FLAGS_max_past_entries_learner)));
+ // As we are initializing the query, we obviously haven't gotten any
+ // feedback message for this query. Hence mark the following field as false.
+ has_feedback_from_all_queries_[priority_level] = false;
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9ad3281f/query_execution/Learner.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Learner.hpp b/query_execution/Learner.hpp
index fb0e4cb..073b693 100644
--- a/query_execution/Learner.hpp
+++ b/query_execution/Learner.hpp
@@ -30,15 +30,10 @@
#include "query_optimizer/QueryHandle.hpp"
#include "utility/Macros.hpp"
-#include "gflags/gflags.h"
#include "glog/logging.h"
namespace quickstep {
-/*DECLARE_int32(max_past_entries_learner,
- 10,
- "The maximum number of past WorkOrder execution statistics"
- " entries for a query");*/
/** \addtogroup QueryExecution
* @{
*/
@@ -94,14 +89,6 @@ class Learner {
}
}
- void updateProbabilitiesForQueriesInPriorityLevel(
- const std::size_t priority_level, const std::size_t query_id);
-
- // TODO(harshad) - Cache internal results from previous invocation of this
- // function and reuse them. There's a lot of redundancy in computations
- // at this point.
- void updateProbabilitiesOfAllPriorityLevels(const std::size_t priority_level);
-
inline const bool hasActiveQueries() const {
return !query_id_to_priority_lookup_.empty();
}
@@ -122,6 +109,29 @@ class Learner {
private:
/**
+ * @brief Update the probabilities for queries in the given priority level.
+ *
+ * @note This function is called after the learner receives a completion
+ * feedback message from a given query.
+ *
+ * @param priority_level The priority level.
+ * @param query_id The ID of the query for which a completion feedback message
+ * has been received.
+ *
+ **/
+ void updateProbabilitiesForQueriesInPriorityLevel(
+ const std::size_t priority_level, const std::size_t query_id);
+
+ /**
+ * @brief Update the probabilities of all the priority levels.
+ *
+ * TODO(harshad) - Cache internal results from previous invocation of this
+ * function and reuse them. There's a lot of redundancy in computations
+ * at this point.
+ **/
+ void updateProbabilitiesOfAllPriorityLevels();
+
+ /**
* @brief Initialize the default probabilities for the queries.
**/
void initializeDefaultProbabilitiesForAllQueries();
@@ -135,18 +145,14 @@ class Learner {
* @brief Initialize the data structures for a given priority level, if none
* exist. If there are already data structures for the given priority
* level, do nothing.
+ *
+ * @note This function should be followed by a relearn() call, to insert this
+ * priority levels in probabilities_of_priority_levels_.
**/
inline void initializePriorityLevelIfNotPresent(
const std::size_t priority_level) {
if (!isPriorityLevelPresent(priority_level)) {
current_probabilities_[priority_level].reset(new ProbabilityStore());
- // Calculate the default probability for the priority level here and use
- // it instead of 0.5 here.
- // TODO(harshad) - Correct this.
- /*const float new_denominator =
- probabilities_of_priority_levels_[priority_level]->getDenominator();
- probabilities_of_priority_levels_->addOrUpdateObjectNewDenominator(
- priority_level, priority_level, new_denominator);*/
execution_stats_[priority_level];
}
}
@@ -169,6 +175,10 @@ class Learner {
/**
* @brief Check if the Learner has presence of the given priority level.
+ *
+ * @param priority_level The priority level.
+ *
+ * @return True if present, false otherwise.
**/
inline bool isPriorityLevelPresent(const std::size_t priority_level) const {
DCHECK_EQ((current_probabilities_.find(priority_level) ==
@@ -178,7 +188,7 @@ class Learner {
}
/**
- * @brief Check if the query is present.
+ * @brief Check if the query is present in local data structures.
**/
inline bool isQueryPresent(const std::size_t query_id) const {
return query_id_to_priority_lookup_.find(query_id) !=
@@ -190,20 +200,7 @@ class Learner {
*
* @param query_handle The query handle for the new query.
**/
- void initializeQuery(const QueryHandle &query_handle) {
- const std::size_t priority_level = query_handle.query_priority();
- const std::size_t query_id = query_handle.query_id();
- DCHECK(isPriorityLevelPresent(priority_level));
- query_id_to_priority_lookup_[query_id] = priority_level;
- execution_stats_[priority_level].emplace_back(
- query_id,
- std::unique_ptr<ExecutionStats>(
- // new ExecutionStats(FLAGS_max_past_entries_learner)));
- new ExecutionStats(10)));
- // As we are initializing the query, we obviously haven't gotten any
- // feedback message for this query. Hence mark the following field as false.
- has_feedback_from_all_queries_[priority_level] = false;
- }
+ void initializeQuery(const QueryHandle &query_handle);
/**
* @brief Get the execution stats object for the given query.
@@ -222,9 +219,10 @@ class Learner {
}
/**
- * @brief This function works well when the query and priority level exists
- * in the data structures.
+ * @breif Get a mutable iterator to the execution stats for a given query.
*
+ * @note This function works well when the query and priority level exists
+ * in the data structures.
**/
inline std::vector<
std::pair<std::size_t, std::unique_ptr<ExecutionStats>>>::const_iterator
@@ -244,6 +242,9 @@ class Learner {
return stats_iter;
}
+ /**
+ * @brief Get a query's priority level given its ID.
+ **/
inline const std::size_t getQueryPriority(const std::size_t query_id) const {
const auto it = query_id_to_priority_lookup_.find(query_id);
DCHECK(it != query_id_to_priority_lookup_.end());
@@ -366,7 +367,6 @@ class Learner {
// Key = priority level. Value = A boolean that indicates if we have received
// feedback from all the queries in the given priority level.
- // TODO(harshad) - Invalidate the cache whenever needed.
std::unordered_map<std::size_t, bool> has_feedback_from_all_queries_;
DISALLOW_COPY_AND_ASSIGN(Learner);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9ad3281f/query_execution/tests/Learner_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/Learner_unittest.cpp b/query_execution/tests/Learner_unittest.cpp
index 864bb22..a1a144d 100644
--- a/query_execution/tests/Learner_unittest.cpp
+++ b/query_execution/tests/Learner_unittest.cpp
@@ -27,28 +27,29 @@ namespace quickstep {
TEST(LearnerTest, AddAndRemoveQueryTest) {
Learner learner;
std::unique_ptr<QueryHandle> handle;
- const std::size_t kPriorityLevel1 = 1;
- handle.reset(new QueryHandle(1, kPriorityLevel1));
+ const std::size_t kPriorityLevel = 1;
+ handle.reset(new QueryHandle(1, kPriorityLevel));
EXPECT_FALSE(learner.hasActiveQueries());
learner.addQuery(*handle);
EXPECT_EQ(1u, learner.getTotalNumActiveQueries());
- EXPECT_EQ(1u, learner.getNumActiveQueriesInPriorityLevel(kPriorityLevel1));
+ EXPECT_EQ(1u, learner.getNumActiveQueriesInPriorityLevel(kPriorityLevel));
EXPECT_TRUE(learner.hasActiveQueries());
+
learner.removeQuery(handle->query_id());
EXPECT_EQ(0u, learner.getTotalNumActiveQueries());
- EXPECT_EQ(0u, learner.getNumActiveQueriesInPriorityLevel(kPriorityLevel1));
+ EXPECT_EQ(0u, learner.getNumActiveQueriesInPriorityLevel(kPriorityLevel));
EXPECT_FALSE(learner.hasActiveQueries());
- const std::size_t kPriorityLevel2 = 1;
- handle.reset(new QueryHandle(1, kPriorityLevel2));
+ handle.reset(new QueryHandle(2, kPriorityLevel));
learner.addQuery(*handle);
EXPECT_EQ(1u, learner.getTotalNumActiveQueries());
- EXPECT_EQ(1u, learner.getNumActiveQueriesInPriorityLevel(kPriorityLevel2));
+ EXPECT_EQ(1u, learner.getNumActiveQueriesInPriorityLevel(kPriorityLevel));
EXPECT_TRUE(learner.hasActiveQueries());
+
learner.removeQuery(handle->query_id());
EXPECT_EQ(0u, learner.getTotalNumActiveQueries());
- EXPECT_EQ(0u, learner.getNumActiveQueriesInPriorityLevel(kPriorityLevel2));
+ EXPECT_EQ(0u, learner.getNumActiveQueriesInPriorityLevel(kPriorityLevel));
EXPECT_FALSE(learner.hasActiveQueries());
}