You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by ji...@apache.org on 2016/10/20 22:03:13 UTC

[3/3] incubator-quickstep git commit: updates

updates


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b9db77d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b9db77d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b9db77d9

Branch: refs/heads/exact-filter
Commit: b9db77d96e19821c65f97eb1dc29df743dde0bf0
Parents: f298303
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu Oct 20 16:35:32 2016 -0500
Committer: Jianqiao Zhu <ji...@cs.wisc.edu>
Committed: Thu Oct 20 16:35:32 2016 -0500

----------------------------------------------------------------------
 query_execution/QueryContext.cpp                |  3 +-
 utility/lip_filter/LIPFilter.proto              | 12 +++++-
 utility/lip_filter/LIPFilterFactory.cpp         | 40 +++++++++++++++++++-
 utility/lip_filter/LIPFilterFactory.hpp         |  5 ++-
 utility/lip_filter/SimpleHashSetExactFilter.hpp | 17 ++++++---
 5 files changed, 65 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b9db77d9/query_execution/QueryContext.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.cpp b/query_execution/QueryContext.cpp
index 0e6636d..bbb552a 100644
--- a/query_execution/QueryContext.cpp
+++ b/query_execution/QueryContext.cpp
@@ -98,7 +98,8 @@ QueryContext::QueryContext(const serialization::QueryContext &proto,
   for (int i = 0; i < proto.lip_filters_size(); ++i) {
     lip_filters_.emplace_back(
         std::unique_ptr<LIPFilter>(
-            LIPFilterFactory::ReconstructFromProto(proto.lip_filters(i))));
+            LIPFilterFactory::ReconstructFromProto(proto.lip_filters(i),
+                                                   storage_manager)));
   }
 
   for (int i = 0; i < proto.lip_filter_deployments_size(); ++i) {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b9db77d9/utility/lip_filter/LIPFilter.proto
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilter.proto b/utility/lip_filter/LIPFilter.proto
index 0ec2978..2f80094 100644
--- a/utility/lip_filter/LIPFilter.proto
+++ b/utility/lip_filter/LIPFilter.proto
@@ -33,14 +33,22 @@ message LIPFilter {
   extensions 16 to max;
 }
 
-message SingleIdentityHashFilter {
+message SimpleHashSetExactFilter {
   extend LIPFilter {
     // All required
-    optional uint64 filter_cardinality = 16;
+    optional uint64 initial_num_entries = 16;
     optional uint64 attribute_size = 17;
   }
 }
 
+message SingleIdentityHashFilter {
+  extend LIPFilter {
+    // All required
+    optional uint64 filter_cardinality = 24;
+    optional uint64 attribute_size = 25;
+  }
+}
+
 enum LIPFilterActionType {
   BUILD = 1;
   PROBE = 2;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b9db77d9/utility/lip_filter/LIPFilterFactory.cpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilterFactory.cpp b/utility/lip_filter/LIPFilterFactory.cpp
index e69e8f3..9f990e1 100644
--- a/utility/lip_filter/LIPFilterFactory.cpp
+++ b/utility/lip_filter/LIPFilterFactory.cpp
@@ -30,8 +30,37 @@
 
 namespace quickstep {
 
-LIPFilter* LIPFilterFactory::ReconstructFromProto(const serialization::LIPFilter &proto) {
+LIPFilter* LIPFilterFactory::ReconstructFromProto(const serialization::LIPFilter &proto,
+                                                  StorageManager *storage_manager) {
   switch (proto.lip_filter_type()) {
+    case serialization::LIPFilterType::SIMPLE_HASH_SET_EXACT_FILTER: {
+      const std::size_t attr_size =
+          proto.GetExtension(serialization::SimpleHashSetExactFilter::attribute_size);
+      const std::size_t initalial_num_entries =
+          proto.GetExtension(serialization::SimpleHashSetExactFilter::initial_num_entries);
+
+      switch (attr_size) {
+        case 1: {
+          return new SimpleHashSetExactFilter<std::uint8_t>(initalial_num_entries,
+                                                            storage_manager);
+        }
+        case 2: {
+          return new SimpleHashSetExactFilter<std::uint16_t>(initalial_num_entries,
+                                                             storage_manager);
+        }
+        case 4: {
+          return new SimpleHashSetExactFilter<std::uint32_t>(initalial_num_entries,
+                                                             storage_manager);
+        }
+        case 8: {
+          return new SimpleHashSetExactFilter<std::uint64_t>(initalial_num_entries,
+                                                             storage_manager);
+        }
+        default:
+          LOG(FATAL) << "Unsupported attribute size for SimpleHashSetExactFilter: "
+                     << attr_size;
+      }
+    }
     case serialization::LIPFilterType::SINGLE_IDENTITY_HASH_FILTER: {
       const std::size_t attr_size =
           proto.GetExtension(serialization::SingleIdentityHashFilter::attribute_size);
@@ -48,7 +77,7 @@ LIPFilter* LIPFilterFactory::ReconstructFromProto(const serialization::LIPFilter
         return new SingleIdentityHashFilter<std::uint8_t>(filter_cardinality);
       }
     }
-    // TODO(jianqiao): handle the BLOOM_FILTER and EXACT_FILTER implementations.
+    // TODO(jianqiao): handle the BLOOM_FILTER implementations.
     default:
       LOG(FATAL) << "Unsupported LIP filter type: "
                  << serialization::LIPFilterType_Name(proto.lip_filter_type());
@@ -58,6 +87,13 @@ LIPFilter* LIPFilterFactory::ReconstructFromProto(const serialization::LIPFilter
 
 bool LIPFilterFactory::ProtoIsValid(const serialization::LIPFilter &proto) {
   switch (proto.lip_filter_type()) {
+    case serialization::LIPFilterType::SIMPLE_HASH_SET_EXACT_FILTER: {
+      const std::size_t attr_size =
+          proto.GetExtension(serialization::SimpleHashSetExactFilter::attribute_size);
+      const std::size_t initial_num_entries =
+          proto.GetExtension(serialization::SimpleHashSetExactFilter::initial_num_entries);
+      return (attr_size != 0 && initial_num_entries != 0);
+    }
     case serialization::LIPFilterType::SINGLE_IDENTITY_HASH_FILTER: {
       const std::size_t attr_size =
           proto.GetExtension(serialization::SingleIdentityHashFilter::attribute_size);

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b9db77d9/utility/lip_filter/LIPFilterFactory.hpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/LIPFilterFactory.hpp b/utility/lip_filter/LIPFilterFactory.hpp
index b8301b8..7527c55 100644
--- a/utility/lip_filter/LIPFilterFactory.hpp
+++ b/utility/lip_filter/LIPFilterFactory.hpp
@@ -27,6 +27,7 @@ namespace quickstep {
 namespace serialization { class LIPFilter; }
 
 class LIPFilter;
+class StorageManager;
 
 /** \addtogroup Utility
  *  @{
@@ -42,10 +43,12 @@ class LIPFilterFactory {
    * @brief Reconstruct a LIPFilter from its serialized Protocol Buffer form.
    *
    * @param proto The Protocol Buffer representation of a LIPFilter object,
+   * @param storage_manager The StorageManager to use.
    * @return A new LIPFilter reconstructed from the supplied Protocol Buffer.
    *         Caller should take ownership of the returned object.
    */
-  static LIPFilter* ReconstructFromProto(const serialization::LIPFilter &proto);
+  static LIPFilter* ReconstructFromProto(const serialization::LIPFilter &proto,
+                                         StorageManager *storage_manager);
 
   /**
    * @brief Check whether a serialization::LIPFilter is fully-formed and

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b9db77d9/utility/lip_filter/SimpleHashSetExactFilter.hpp
----------------------------------------------------------------------
diff --git a/utility/lip_filter/SimpleHashSetExactFilter.hpp b/utility/lip_filter/SimpleHashSetExactFilter.hpp
index 8c7afd6..c2eb980 100644
--- a/utility/lip_filter/SimpleHashSetExactFilter.hpp
+++ b/utility/lip_filter/SimpleHashSetExactFilter.hpp
@@ -44,19 +44,21 @@ template <typename KeyCppType>
 class SimpleHashSetExactFilter : public LIPFilter {
  public:
   SimpleHashSetExactFilter(const std::size_t num_entries,
-                           StorageManager *storage_manager) {
+                           StorageManager *storage_manager)
+      : LIPFilter(LIPFilterType::kSimpleHashSetExactFilter),
+        storage_manager_(storage_manager) {
     // Pick out a prime number of slots and calculate storage requirements.
     std::size_t num_slots_tmp = get_next_prime_number(num_entries * kHashTableLoadFactor);
     std::size_t required_memory = sizeof(Header)
                                   + num_slots_tmp * sizeof(std::atomic<std::size_t>)
                                   + (num_slots_tmp / kHashTableLoadFactor) * sizeof(Bucket);
-    std::size_t num_storage_slots = this->storage_manager_->SlotsNeededForBytes(required_memory);
+    std::size_t num_storage_slots = storage_manager_->SlotsNeededForBytes(required_memory);
 
     // Get a StorageBlob to hold the hash table.
-    const block_id blob_id = this->storage_manager_->createBlob(num_storage_slots);
-    this->blob_ = this->storage_manager_->getBlobMutable(blob_id);
+    const block_id blob_id = storage_manager_->createBlob(num_storage_slots);
+    blob_ = storage_manager_->getBlobMutable(blob_id);
 
-    void *aligned_memory_start = this->blob_->getMemoryMutable();
+    void *aligned_memory_start = blob_->getMemoryMutable();
     std::size_t available_memory = num_storage_slots * kSlotSizeBytes;
 
     // Locate the header.
@@ -168,7 +170,7 @@ class SimpleHashSetExactFilter : public LIPFilter {
         continue;
       }
       // Chase the next pointer.
-      const Bucket *bucket = buckets_ + (existing_chain_ptr - 1);
+      Bucket *bucket = buckets_ + (existing_chain_ptr - 1);
       if (bucket->hash == hash_code) {
         // The key is already in the hash table, done.
         return;
@@ -243,6 +245,9 @@ class SimpleHashSetExactFilter : public LIPFilter {
   std::atomic<std::size_t> *slots_;
   Bucket *buckets_;
 
+  StorageManager *storage_manager_;
+  MutableBlobReference blob_;
+
   DISALLOW_COPY_AND_ASSIGN(SimpleHashSetExactFilter);
 };