You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by ji...@apache.org on 2016/10/20 22:03:13 UTC
[3/3] incubator-quickstep git commit: updates
updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b9db77d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b9db77d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b9db77d9
Branch: refs/heads/exact-filter
Commit: b9db77d96e19821c65f97eb1dc29df743dde0bf0
Parents: f298303
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu Oct 20 16:35:32 2016 -0500
Committer: Jianqiao Zhu <ji...@cs.wisc.edu>
Committed: Thu Oct 20 16:35:32 2016 -0500
----------------------------------------------------------------------
query_execution/QueryContext.cpp | 3 +-
utility/lip_filter/LIPFilter.proto | 12 +++++-
utility/lip_filter/LIPFilterFactory.cpp | 40 +++++++++++++++++++-
utility/lip_filter/LIPFilterFactory.hpp | 5 ++-
utility/lip_filter/SimpleHashSetExactFilter.hpp | 17 ++++++---
5 files changed, 65 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b9db77d9/query_execution/QueryContext.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.cpp b/query_execution/QueryContext.cpp
index 0e6636d..bbb552a 100644
--- a/query_execution/QueryContext.cpp
+++ b/query_execution/QueryContext.cpp
@@ -98,7 +98,8 @@ QueryContext::QueryContext(const serialization::QueryContext &proto,
for (int i = 0; i < proto.lip_filters_size(); ++i) {
lip_filters_.emplace_back(
std::unique_ptr<LIPFilter>(
- LIPFilterFactory::ReconstructFromProto(proto.lip_filters(i))));
+ LIPFilterFactory::ReconstructFromProto(proto.lip_filters(i),
+ storage_manager)));
}
for (int i = 0; i < proto.lip_filter_deployments_size(); ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b9db77d9/utility/lip_filter/LIPFilter.proto
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilter.proto b/utility/lip_filter/LIPFilter.proto
index 0ec2978..2f80094 100644
--- a/utility/lip_filter/LIPFilter.proto
+++ b/utility/lip_filter/LIPFilter.proto
@@ -33,14 +33,22 @@ message LIPFilter {
extensions 16 to max;
}
-message SingleIdentityHashFilter {
+message SimpleHashSetExactFilter {
extend LIPFilter {
// All required
- optional uint64 filter_cardinality = 16;
+ optional uint64 initial_num_entries = 16;
optional uint64 attribute_size = 17;
}
}
+message SingleIdentityHashFilter {
+ extend LIPFilter {
+ // All required
+ optional uint64 filter_cardinality = 24;
+ optional uint64 attribute_size = 25;
+ }
+}
+
enum LIPFilterActionType {
BUILD = 1;
PROBE = 2;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b9db77d9/utility/lip_filter/LIPFilterFactory.cpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilterFactory.cpp b/utility/lip_filter/LIPFilterFactory.cpp
index e69e8f3..9f990e1 100644
--- a/utility/lip_filter/LIPFilterFactory.cpp
+++ b/utility/lip_filter/LIPFilterFactory.cpp
@@ -30,8 +30,37 @@
namespace quickstep {
-LIPFilter* LIPFilterFactory::ReconstructFromProto(const serialization::LIPFilter &proto) {
+LIPFilter* LIPFilterFactory::ReconstructFromProto(const serialization::LIPFilter &proto,
+ StorageManager *storage_manager) {
switch (proto.lip_filter_type()) {
+ case serialization::LIPFilterType::SIMPLE_HASH_SET_EXACT_FILTER: {
+ const std::size_t attr_size =
+ proto.GetExtension(serialization::SimpleHashSetExactFilter::attribute_size);
+ const std::size_t initalial_num_entries =
+ proto.GetExtension(serialization::SimpleHashSetExactFilter::initial_num_entries);
+
+ switch (attr_size) {
+ case 1: {
+ return new SimpleHashSetExactFilter<std::uint8_t>(initalial_num_entries,
+ storage_manager);
+ }
+ case 2: {
+ return new SimpleHashSetExactFilter<std::uint16_t>(initalial_num_entries,
+ storage_manager);
+ }
+ case 4: {
+ return new SimpleHashSetExactFilter<std::uint32_t>(initalial_num_entries,
+ storage_manager);
+ }
+ case 8: {
+ return new SimpleHashSetExactFilter<std::uint64_t>(initalial_num_entries,
+ storage_manager);
+ }
+ default:
+ LOG(FATAL) << "Unsupported attribute size for SimpleHashSetExactFilter: "
+ << attr_size;
+ }
+ }
case serialization::LIPFilterType::SINGLE_IDENTITY_HASH_FILTER: {
const std::size_t attr_size =
proto.GetExtension(serialization::SingleIdentityHashFilter::attribute_size);
@@ -48,7 +77,7 @@ LIPFilter* LIPFilterFactory::ReconstructFromProto(const serialization::LIPFilter
return new SingleIdentityHashFilter<std::uint8_t>(filter_cardinality);
}
}
- // TODO(jianqiao): handle the BLOOM_FILTER and EXACT_FILTER implementations.
+ // TODO(jianqiao): handle the BLOOM_FILTER implementations.
default:
LOG(FATAL) << "Unsupported LIP filter type: "
<< serialization::LIPFilterType_Name(proto.lip_filter_type());
@@ -58,6 +87,13 @@ LIPFilter* LIPFilterFactory::ReconstructFromProto(const serialization::LIPFilter
bool LIPFilterFactory::ProtoIsValid(const serialization::LIPFilter &proto) {
switch (proto.lip_filter_type()) {
+ case serialization::LIPFilterType::SIMPLE_HASH_SET_EXACT_FILTER: {
+ const std::size_t attr_size =
+ proto.GetExtension(serialization::SimpleHashSetExactFilter::attribute_size);
+ const std::size_t initial_num_entries =
+ proto.GetExtension(serialization::SimpleHashSetExactFilter::initial_num_entries);
+ return (attr_size != 0 && initial_num_entries != 0);
+ }
case serialization::LIPFilterType::SINGLE_IDENTITY_HASH_FILTER: {
const std::size_t attr_size =
proto.GetExtension(serialization::SingleIdentityHashFilter::attribute_size);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b9db77d9/utility/lip_filter/LIPFilterFactory.hpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilterFactory.hpp b/utility/lip_filter/LIPFilterFactory.hpp
index b8301b8..7527c55 100644
--- a/utility/lip_filter/LIPFilterFactory.hpp
+++ b/utility/lip_filter/LIPFilterFactory.hpp
@@ -27,6 +27,7 @@ namespace quickstep {
namespace serialization { class LIPFilter; }
class LIPFilter;
+class StorageManager;
/** \addtogroup Utility
* @{
@@ -42,10 +43,12 @@ class LIPFilterFactory {
* @brief Reconstruct a LIPFilter from its serialized Protocol Buffer form.
*
* @param proto The Protocol Buffer representation of a LIPFilter object,
+ * @param storage_manager The StorageManager to use.
* @return A new LIPFilter reconstructed from the supplied Protocol Buffer.
* Caller should take ownership of the returned object.
*/
- static LIPFilter* ReconstructFromProto(const serialization::LIPFilter &proto);
+ static LIPFilter* ReconstructFromProto(const serialization::LIPFilter &proto,
+ StorageManager *storage_manager);
/**
* @brief Check whether a serialization::LIPFilter is fully-formed and
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b9db77d9/utility/lip_filter/SimpleHashSetExactFilter.hpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/SimpleHashSetExactFilter.hpp b/utility/lip_filter/SimpleHashSetExactFilter.hpp
index 8c7afd6..c2eb980 100644
--- a/utility/lip_filter/SimpleHashSetExactFilter.hpp
+++ b/utility/lip_filter/SimpleHashSetExactFilter.hpp
@@ -44,19 +44,21 @@ template <typename KeyCppType>
class SimpleHashSetExactFilter : public LIPFilter {
public:
SimpleHashSetExactFilter(const std::size_t num_entries,
- StorageManager *storage_manager) {
+ StorageManager *storage_manager)
+ : LIPFilter(LIPFilterType::kSimpleHashSetExactFilter),
+ storage_manager_(storage_manager) {
// Pick out a prime number of slots and calculate storage requirements.
std::size_t num_slots_tmp = get_next_prime_number(num_entries * kHashTableLoadFactor);
std::size_t required_memory = sizeof(Header)
+ num_slots_tmp * sizeof(std::atomic<std::size_t>)
+ (num_slots_tmp / kHashTableLoadFactor) * sizeof(Bucket);
- std::size_t num_storage_slots = this->storage_manager_->SlotsNeededForBytes(required_memory);
+ std::size_t num_storage_slots = storage_manager_->SlotsNeededForBytes(required_memory);
// Get a StorageBlob to hold the hash table.
- const block_id blob_id = this->storage_manager_->createBlob(num_storage_slots);
- this->blob_ = this->storage_manager_->getBlobMutable(blob_id);
+ const block_id blob_id = storage_manager_->createBlob(num_storage_slots);
+ blob_ = storage_manager_->getBlobMutable(blob_id);
- void *aligned_memory_start = this->blob_->getMemoryMutable();
+ void *aligned_memory_start = blob_->getMemoryMutable();
std::size_t available_memory = num_storage_slots * kSlotSizeBytes;
// Locate the header.
@@ -168,7 +170,7 @@ class SimpleHashSetExactFilter : public LIPFilter {
continue;
}
// Chase the next pointer.
- const Bucket *bucket = buckets_ + (existing_chain_ptr - 1);
+ Bucket *bucket = buckets_ + (existing_chain_ptr - 1);
if (bucket->hash == hash_code) {
// The key is already in the hash table, done.
return;
@@ -243,6 +245,9 @@ class SimpleHashSetExactFilter : public LIPFilter {
std::atomic<std::size_t> *slots_;
Bucket *buckets_;
+ StorageManager *storage_manager_;
+ MutableBlobReference blob_;
+
DISALLOW_COPY_AND_ASSIGN(SimpleHashSetExactFilter);
};