You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by hb...@apache.org on 2016/07/17 17:24:44 UTC
incubator-quickstep git commit: Memory footprint for a running query.
Repository: incubator-quickstep
Updated Branches:
refs/heads/memory-estimate 21be79baf -> 3e5783b45
Memory footprint for a running query.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3e5783b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3e5783b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3e5783b4
Branch: refs/heads/memory-estimate
Commit: 3e5783b45edf114669a2448ca5c6e26fe2e7d0c7
Parents: 21be79b
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Sun Jul 17 12:24:12 2016 -0500
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Sun Jul 17 12:24:12 2016 -0500
----------------------------------------------------------------------
catalog/CatalogRelation.hpp | 5 ++++
query_execution/CMakeLists.txt | 1 +
query_execution/PriorityPolicyEnforcer.cpp | 9 ++++++
query_execution/PriorityPolicyEnforcer.hpp | 2 ++
query_execution/QueryContext.cpp | 10 +++++++
query_execution/QueryContext.hpp | 10 +++++++
query_execution/QueryManager.hpp | 4 +++
query_optimizer/CMakeLists.txt | 4 ++-
query_optimizer/QueryHandle.cpp | 39 +++++++++++++++++++++++++
query_optimizer/QueryHandle.hpp | 23 +++++++++++++++
query_optimizer/QueryProcessor.cpp | 2 +-
storage/HashTable.hpp | 7 +++++
12 files changed, 114 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e5783b4/catalog/CatalogRelation.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.hpp b/catalog/CatalogRelation.hpp
index e0d5350..911495a 100644
--- a/catalog/CatalogRelation.hpp
+++ b/catalog/CatalogRelation.hpp
@@ -338,6 +338,11 @@ class CatalogRelation : public CatalogRelationSchema {
return blocks_.size();
}
+ inline std::size_t getRelationSizeBytes() const {
+ SpinSharedMutexSharedLock<false> lock(blocks_mutex_);
+ return blocks_.size() * getDefaultStorageBlockLayout().getDescription().num_slots() * kSlotSizeBytes;
+ }
+
/**
* @brief Get the current set of blocks belonging to this relation.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e5783b4/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 11e0e1d..73ed293 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -154,6 +154,7 @@ target_link_libraries(quickstep_queryexecution_QueryContext
quickstep_storage_HashTableFactory
quickstep_storage_InsertDestination
quickstep_storage_InsertDestination_proto
+ quickstep_threading_SpinSharedMutex
quickstep_types_TypedValue
quickstep_types_containers_Tuple
quickstep_utility_BloomFilter
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e5783b4/query_execution/PriorityPolicyEnforcer.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PriorityPolicyEnforcer.cpp b/query_execution/PriorityPolicyEnforcer.cpp
index 2496b74..37db4d0 100644
--- a/query_execution/PriorityPolicyEnforcer.cpp
+++ b/query_execution/PriorityPolicyEnforcer.cpp
@@ -392,4 +392,13 @@ bool PriorityPolicyEnforcer::admissionMemoryCheck(const QueryHandle *query_handl
return false;
}
+const std::size_t PriorityPolicyEnforcer::getMemoryForQueryInBytes(const std::size_t query_id) {
+ DCHECK(query_id_to_handle_.find(query_id) != query_id_to_handle_.end());
+ QueryHandle *query_handle = query_id_to_handle_[query_id];
+ std::size_t memory = query_handle->getMemoryTempRelationsBytes();
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+ memory += admitted_queries_[query_id]->getMemoryBytes();
+ return memory;
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e5783b4/query_execution/PriorityPolicyEnforcer.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PriorityPolicyEnforcer.hpp b/query_execution/PriorityPolicyEnforcer.hpp
index 9d727ed..33d0e1f 100644
--- a/query_execution/PriorityPolicyEnforcer.hpp
+++ b/query_execution/PriorityPolicyEnforcer.hpp
@@ -193,6 +193,8 @@ class PriorityPolicyEnforcer {
bool admissionMemoryCheck(const QueryHandle *query_handle);
+ const std::size_t getMemoryForQueryInBytes(const std::size_t query_id);
+
const tmb::client_id foreman_client_id_;
const std::size_t num_numa_nodes_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e5783b4/query_execution/QueryContext.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.cpp b/query_execution/QueryContext.cpp
index 54dd557..fc35475 100644
--- a/query_execution/QueryContext.cpp
+++ b/query_execution/QueryContext.cpp
@@ -234,4 +234,14 @@ bool QueryContext::ProtoIsValid(const serialization::QueryContext &proto,
return proto.IsInitialized();
}
+
+const std::size_t QueryContext::getMemoryBytes() {
+ SpinSharedMutexSharedLock<false> lock(hash_tables_mutex_);
+ std::size_t memory = 0;
+ for (std::size_t i = 0; i < join_hash_tables_.size(); ++i) {
+ memory += join_hash_tables_[i]->getHashTableMemorySizeBytes();
+ }
+ return memory;
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e5783b4/query_execution/QueryContext.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp
index 7d5628d..eeb1512 100644
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@ -33,6 +33,7 @@
#include "storage/AggregationOperationState.hpp"
#include "storage/HashTable.hpp"
#include "storage/InsertDestination.hpp"
+#include "threading/SpinSharedMutex.hpp"
#include "types/containers/Tuple.hpp"
#include "utility/BloomFilter.hpp"
#include "utility/Macros.hpp"
@@ -303,6 +304,7 @@ class QueryContext {
* @return True if valid, otherwise false.
**/
bool isValidJoinHashTableId(const join_hash_table_id id) const {
+ SpinSharedMutexSharedLock<false> lock(hash_tables_mutex_);
return id < join_hash_tables_.size();
}
@@ -314,6 +316,7 @@ class QueryContext {
* @return The JoinHashTable, already created in the constructor.
**/
inline JoinHashTable* getJoinHashTable(const join_hash_table_id id) {
+ SpinSharedMutexSharedLock<false> lock(hash_tables_mutex_);
DCHECK_LT(id, join_hash_tables_.size());
return join_hash_tables_[id].get();
}
@@ -324,6 +327,7 @@ class QueryContext {
* @param id The id of the JoinHashTable to destroy.
**/
inline void destroyJoinHashTable(const join_hash_table_id id) {
+ SpinSharedMutexExclusiveLock<false> lock(hash_tables_mutex_);
DCHECK_LT(id, join_hash_tables_.size());
join_hash_tables_[id].reset();
}
@@ -460,6 +464,11 @@ class QueryContext {
return update_groups_[id];
}
+ /**
+ * @brief Get the total memory footprint of the temp structures in bytes.
+ **/
+ const std::size_t getMemoryBytes();
+
private:
std::vector<std::unique_ptr<AggregationOperationState>> aggregation_states_;
std::vector<std::unique_ptr<BloomFilter>> bloom_filters_;
@@ -472,6 +481,7 @@ class QueryContext {
std::vector<std::unique_ptr<Tuple>> tuples_;
std::vector<std::unordered_map<attribute_id, std::unique_ptr<const Scalar>>> update_groups_;
+ mutable SpinSharedMutex<false> hash_tables_mutex_;
DISALLOW_COPY_AND_ASSIGN(QueryContext);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e5783b4/query_execution/QueryManager.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManager.hpp b/query_execution/QueryManager.hpp
index b52460f..2306f37 100644
--- a/query_execution/QueryManager.hpp
+++ b/query_execution/QueryManager.hpp
@@ -123,6 +123,10 @@ class QueryManager {
return query_context_.get();
}
+ const std::size_t getMemoryBytes() {
+ return query_context_->getMemoryBytes();
+ }
+
private:
/**
* @brief Process the received WorkOrder complete message.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e5783b4/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index fdc607a..86a0834 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -50,7 +50,7 @@ add_library(quickstep_queryoptimizer_Optimizer Optimizer.cpp Optimizer.hpp)
add_library(quickstep_queryoptimizer_OptimizerContext OptimizerContext.cpp OptimizerContext.hpp)
add_library(quickstep_queryoptimizer_OptimizerTree ../empty_src.cpp OptimizerTree.hpp)
add_library(quickstep_queryoptimizer_PhysicalGenerator PhysicalGenerator.cpp PhysicalGenerator.hpp)
-add_library(quickstep_queryoptimizer_QueryHandle ../empty_src.cpp QueryHandle.hpp)
+add_library(quickstep_queryoptimizer_QueryHandle QueryHandle.cpp QueryHandle.hpp)
add_library(quickstep_queryoptimizer_QueryPlan ../empty_src.cpp QueryPlan.hpp)
add_library(quickstep_queryoptimizer_QueryProcessor QueryProcessor.cpp QueryProcessor.hpp)
add_library(quickstep_queryoptimizer_Validator ../empty_src.cpp Validator.hpp)
@@ -205,6 +205,8 @@ target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
quickstep_utility_Macros
quickstep_utility_PlanVisualizer)
target_link_libraries(quickstep_queryoptimizer_QueryHandle
+ quickstep_catalog_CatalogDatabase
+ quickstep_catalog_CatalogRelation
quickstep_catalog_Catalog_proto
quickstep_queryexecution_QueryContext_proto
quickstep_queryoptimizer_QueryPlan
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e5783b4/query_optimizer/QueryHandle.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryHandle.cpp b/query_optimizer/QueryHandle.cpp
new file mode 100644
index 0000000..9a90922
--- /dev/null
+++ b/query_optimizer/QueryHandle.cpp
@@ -0,0 +1,39 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_optimizer/QueryHandle.hpp"
+
+#include <set>
+
+#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogRelation.hpp"
+
+namespace quickstep {
+
+const std::size_t QueryHandle::getMemoryTempRelationsBytes() {
+ std::size_t memory = 0;
+ for (relation_id rid : temp_relation_ids_) {
+ if (catalog_database_->hasRelationWithId(rid)) {
+ memory += catalog_database_->getRelationById(rid)->getRelationSizeBytes();
+ } else {
+ removeTempRelationID(rid);
+ }
+ }
+ return memory;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e5783b4/query_optimizer/QueryHandle.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryHandle.hpp b/query_optimizer/QueryHandle.hpp
index a07313f..63a1c18 100644
--- a/query_optimizer/QueryHandle.hpp
+++ b/query_optimizer/QueryHandle.hpp
@@ -21,15 +21,18 @@
#include <cstddef>
#include <cstdint>
#include <memory>
+#include <set>
#include <utility>
#include "catalog/Catalog.pb.h"
+#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/QueryContext.pb.h"
#include "query_optimizer/QueryPlan.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
+class CatalogDatabase;
class CatalogRelation;
/** \addtogroup QueryOptimizer
@@ -47,11 +50,13 @@ class QueryHandle {
* @param query_id The given query id.
*/
explicit QueryHandle(const std::size_t query_id,
+ CatalogDatabase *catalog_database,
const std::uint64_t query_priority = 1)
: query_id_(query_id),
query_priority_(query_priority),
estimated_max_memory_in_bytes_(0),
query_plan_(new QueryPlan()),
+ catalog_database_(catalog_database),
query_result_relation_(nullptr) {}
~QueryHandle() {}
@@ -162,7 +167,21 @@ class QueryHandle {
estimated_max_memory_in_bytes_ += memory_bytes;
}
+ /**
+ * @brief Find the memory used by the temp relations in this query at the time
+ * when this function is called.
+ **/
+ const std::size_t getMemoryTempRelationsBytes();
+
+ void addTempRelationID(const relation_id rid) {
+ temp_relation_ids_.insert(rid);
+ }
+
private:
+ void removeTempRelationID(const relation_id rid) {
+ temp_relation_ids_.erase(rid);
+ }
+
const std::size_t query_id_;
const std::uint64_t query_priority_;
std::size_t estimated_max_memory_in_bytes_;
@@ -174,6 +193,8 @@ class QueryHandle {
// TODO(quickstep-team): Use Catalog to support multiple databases.
serialization::CatalogDatabase catalog_database_cache_proto_;
+ CatalogDatabase *catalog_database_;
+
// NOTE(zuyu): The relation gets created by the optimizer,
// and deleted by the Cli shell.
const CatalogRelation *query_result_relation_;
@@ -185,6 +206,8 @@ class QueryHandle {
// Time when query finished its execution.
std::chrono::steady_clock::time_point completion_time_;
+ std::set<relation_id> temp_relation_ids_;
+
DISALLOW_COPY_AND_ASSIGN(QueryHandle);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e5783b4/query_optimizer/QueryProcessor.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryProcessor.cpp b/query_optimizer/QueryProcessor.cpp
index 6381d3a..8a4355e 100644
--- a/query_optimizer/QueryProcessor.cpp
+++ b/query_optimizer/QueryProcessor.cpp
@@ -39,7 +39,7 @@ namespace quickstep {
QueryHandle* QueryProcessor::generateQueryHandle(const ParseStatement &statement) {
std::unique_ptr<QueryHandle> query_handle(
- new QueryHandle(query_id_, statement.getPriority()));
+ new QueryHandle(query_id_, getDefaultDatabase(), statement.getPriority()));
optimizer::Optimizer optimizer(query_id_, getDefaultDatabase(), storage_manager_.get());
optimizer.generateQueryHandle(statement, query_handle.get());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e5783b4/storage/HashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTable.hpp b/storage/HashTable.hpp
index be31fd9..5a36f5c 100644
--- a/storage/HashTable.hpp
+++ b/storage/HashTable.hpp
@@ -551,6 +551,13 @@ class HashTable : public HashTableBase<resizable,
FunctorT *functor);
/**
+ * @brief Get the size of the hash table at the time this function is called.
+ **/
+ inline const std::size_t getHashTableMemorySizeBytes() const {
+ return hash_table_memory_size_;
+ }
+
+ /**
* @brief Determine the number of entries (key-value pairs) contained in this
* HashTable.
* @note For some HashTable implementations, this is O(1), but for others it