You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by pe...@apache.org on 2021/07/07 03:32:18 UTC

[pulsar] 05/07: [C++] Fix Setting KeySharedMeta in SubscribeCommand (#11088)

This is an automated email from the ASF dual-hosted git repository.

penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 72479b8a9a51bc763efae5ed873532e832861fa2
Author: Yuto Furuta <mz...@gmail.com>
AuthorDate: Sat Jun 26 03:51:04 2021 +0900

    [C++] Fix Setting KeySharedMeta in SubscribeCommand (#11088)
    
    ### Motivation
    https://github.com/apache/pulsar/blob/67adf0aedf4b435a43fc570afa92154915f5b212/pulsar-client-cpp/lib/Commands.cc#L295-L306
    Regarding the above, it seems that `KeySharedMeta ksm` is not set as subscribe command.
    As a test, when `STICKY` is set, the consumer can connect to the broker without ranges.
    It seems that `AUTO_SPLIT` as default value is used.
    
    ### Modification
    Fix `that KeySharedMeta meta` is able to be set.
    Also, implement sticky consumer in C++ client.
    
    
    (cherry picked from commit eaa4ab03ab8e9905b3dd24a10b86d6c416ed4507)
---
 pulsar-client-cpp/include/pulsar/KeySharedPolicy.h |  16 ++
 pulsar-client-cpp/lib/Commands.cc                  |   9 +-
 pulsar-client-cpp/lib/KeySharedPolicy.cc           |  28 +++
 pulsar-client-cpp/lib/KeySharedPolicyImpl.h        |   1 +
 pulsar-client-cpp/tests/KeySharedPolicyTest.cc     | 203 +++++++++++++++++++++
 5 files changed, 255 insertions(+), 2 deletions(-)

diff --git a/pulsar-client-cpp/include/pulsar/KeySharedPolicy.h b/pulsar-client-cpp/include/pulsar/KeySharedPolicy.h
index 09f0d1f..53efc4c 100644
--- a/pulsar-client-cpp/include/pulsar/KeySharedPolicy.h
+++ b/pulsar-client-cpp/include/pulsar/KeySharedPolicy.h
@@ -22,6 +22,9 @@
 
 #include <memory>
 
+#include <utility>
+#include <vector>
+
 namespace pulsar {
 
 /**
@@ -44,6 +47,9 @@ enum KeySharedMode
 
 struct KeySharedPolicyImpl;
 
+typedef std::pair<int, int> StickyRange;
+typedef std::vector<StickyRange> StickyRanges;
+
 class PULSAR_PUBLIC KeySharedPolicy {
    public:
     KeySharedPolicy();
@@ -88,6 +94,16 @@ class PULSAR_PUBLIC KeySharedPolicy {
      */
     bool isAllowOutOfOrderDelivery() const;
 
+    /**
+     * @param ranges used with sticky mode
+     */
+    KeySharedPolicy& setStickyRanges(std::initializer_list<StickyRange> ranges);
+
+    /**
+     * @return ranges used with sticky mode
+     */
+    StickyRanges getStickyRanges() const;
+
    private:
     std::shared_ptr<KeySharedPolicyImpl> impl_;
 };
diff --git a/pulsar-client-cpp/lib/Commands.cc b/pulsar-client-cpp/lib/Commands.cc
index 5fbfdde..5b17d24 100644
--- a/pulsar-client-cpp/lib/Commands.cc
+++ b/pulsar-client-cpp/lib/Commands.cc
@@ -293,15 +293,20 @@ SharedBuffer Commands::newSubscribe(const std::string& topic, const std::string&
     }
 
     if (subType == CommandSubscribe_SubType_Key_Shared) {
-        KeySharedMeta ksm;
+        KeySharedMeta& ksm = *subscribe->mutable_keysharedmeta();
         switch (keySharedPolicy.getKeySharedMode()) {
             case pulsar::AUTO_SPLIT:
                 ksm.set_keysharedmode(proto::KeySharedMode::AUTO_SPLIT);
                 break;
             case pulsar::STICKY:
                 ksm.set_keysharedmode(proto::KeySharedMode::STICKY);
+                for (StickyRange range : keySharedPolicy.getStickyRanges()) {
+                    IntRange* intRange = IntRange().New();
+                    intRange->set_start(range.first);
+                    intRange->set_end(range.second);
+                    ksm.mutable_hashranges()->AddAllocated(intRange);
+                }
         }
-
         ksm.set_allowoutoforderdelivery(keySharedPolicy.isAllowOutOfOrderDelivery());
     }
 
diff --git a/pulsar-client-cpp/lib/KeySharedPolicy.cc b/pulsar-client-cpp/lib/KeySharedPolicy.cc
index 0fdf576..3e3c5eb 100644
--- a/pulsar-client-cpp/lib/KeySharedPolicy.cc
+++ b/pulsar-client-cpp/lib/KeySharedPolicy.cc
@@ -18,8 +18,12 @@
  */
 #include <lib/KeySharedPolicyImpl.h>
 
+#include <stdexcept>
+
 namespace pulsar {
 
+static const int DefaultHashRangeSize = 2 << 15;
+
 KeySharedPolicy::KeySharedPolicy() : impl_(std::make_shared<KeySharedPolicyImpl>()) {}
 
 KeySharedPolicy::~KeySharedPolicy() {}
@@ -45,6 +49,30 @@ KeySharedPolicy &KeySharedPolicy::setAllowOutOfOrderDelivery(bool allowOutOfOrde
 
 bool KeySharedPolicy::isAllowOutOfOrderDelivery() const { return impl_->allowOutOfOrderDelivery; }
 
+KeySharedPolicy &KeySharedPolicy::setStickyRanges(std::initializer_list<StickyRange> ranges) {
+    if (ranges.size() == 0) {
+        throw std::invalid_argument("Ranges for KeyShared policy must not be empty.");
+    }
+    for (StickyRange range : ranges) {
+        if (range.first < 0 || range.second >= DefaultHashRangeSize) {
+            throw std::invalid_argument("KeySharedPolicy Exception: Ranges must be [0, 65535].");
+        }
+        for (StickyRange range2 : ranges) {
+            int start = std::max(range.first, range2.first);
+            int end = std::min(range.second, range2.second);
+            if (range != range2 && end >= start) {
+                throw std::invalid_argument("Ranges for KeyShared policy with overlap.");
+            }
+        }
+        for (StickyRange range : ranges) {
+            impl_->ranges.push_back(range);
+        }
+    }
+    return *this;
+}
+
+StickyRanges KeySharedPolicy::getStickyRanges() const { return impl_->ranges; }
+
 KeySharedPolicy KeySharedPolicy::clone() const {
     KeySharedPolicy newConf;
     newConf.impl_.reset(new KeySharedPolicyImpl(*this->impl_));
diff --git a/pulsar-client-cpp/lib/KeySharedPolicyImpl.h b/pulsar-client-cpp/lib/KeySharedPolicyImpl.h
index a12f48a..51ad491 100644
--- a/pulsar-client-cpp/lib/KeySharedPolicyImpl.h
+++ b/pulsar-client-cpp/lib/KeySharedPolicyImpl.h
@@ -25,6 +25,7 @@ namespace pulsar {
 struct KeySharedPolicyImpl {
     bool allowOutOfOrderDelivery;
     KeySharedMode keySharedMode;
+    StickyRanges ranges;
 
     KeySharedPolicyImpl() : allowOutOfOrderDelivery(false), keySharedMode(AUTO_SPLIT) {}
 };
diff --git a/pulsar-client-cpp/tests/KeySharedPolicyTest.cc b/pulsar-client-cpp/tests/KeySharedPolicyTest.cc
new file mode 100644
index 0000000..49fef3f
--- /dev/null
+++ b/pulsar-client-cpp/tests/KeySharedPolicyTest.cc
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <cmath>
+#include <ctime>
+#include <vector>
+#include <map>
+
+#include <gtest/gtest.h>
+#include <pulsar/Client.h>
+#include <pulsar/KeySharedPolicy.h>
+#include "lib/LogUtils.h"
+
+#include "HttpHelper.h"
+#include "LogHelper.h"
+
+DECLARE_LOG_OBJECT()
+
+using namespace pulsar;
+
+static std::string lookupUrl = "pulsar://localhost:6650";
+
+class KeySharedPolicyTest : public ::testing::Test {
+   protected:
+    KeySharedPolicyTest() : client(lookupUrl, ClientConfiguration().setPartititionsUpdateInterval(1)) {}
+
+    void TearDown() override { client.close(); }
+
+    void addProducer(const std::string& topicName) {
+        producers.emplace_back();
+        auto conf = ProducerConfiguration().setBatchingEnabled(false).setPartitionsRoutingMode(
+            ProducerConfiguration::RoundRobinDistribution);
+        ASSERT_EQ(ResultOk, client.createProducer(topicName, conf, producers.back()));
+    }
+
+    ConsumerConfiguration getConsumerConfiguration() {
+        ConsumerConfiguration conf;
+        conf.setConsumerType(ConsumerKeyShared);
+        return conf;
+    }
+
+    static constexpr int NUMBER_OF_KEYS = 300;
+
+    static Message newIntMessage(int i, const std::string& key, const char* orderingKey = nullptr) {
+        MessageBuilder builder;
+        if (orderingKey) {
+            builder.setOrderingKey(orderingKey);
+        }
+        return builder.setPartitionKey(key).setContent(std::to_string(i)).build();
+    }
+
+    static void sendCallback(Result result, const MessageId&) { ASSERT_EQ(result, ResultOk); }
+
+    void receiveAndCheckDistribution(int expectedNumTotalMessages) {
+        keyToConsumer.clear();
+        messagesPerConsumer.clear();
+        int totalMessages = 0;
+
+        for (size_t i = 0; i < consumers.size(); i++) {
+            auto& consumer = consumers[i];
+            while (true) {
+                Message msg;
+                Result result = consumer.receive(msg, 3000);
+                if (result == ResultTimeout) {
+                    break;
+                }
+
+                ASSERT_EQ(result, ResultOk);
+                totalMessages++;
+                messagesPerConsumer[i]++;
+                ASSERT_EQ(ResultOk, consumer.acknowledge(msg));
+
+                if (msg.hasPartitionKey() || msg.hasOrderingKey()) {
+                    std::string key = msg.hasOrderingKey() ? msg.getOrderingKey() : msg.getPartitionKey();
+                    auto iter = keyToConsumer.find(key);
+                    if (iter == keyToConsumer.end()) {
+                        keyToConsumer[key] = i;
+                    } else {
+                        // check messages with the same key will be consumed by the same consumer
+                        ASSERT_EQ(iter->second, i);
+                    }
+                }
+            }
+        }
+
+        LOG_INFO("messagesPerConsumer: " << messagesPerConsumer);
+        int numTotalMessages = 0;
+        for (const auto& kv : messagesPerConsumer) {
+            numTotalMessages += kv.second;
+        }
+        ASSERT_EQ(numTotalMessages, expectedNumTotalMessages);
+
+        const double expectedMessagesPerConsumer = static_cast<double>(totalMessages) / consumers.size();
+        constexpr double PERCENT_ERROR = 0.50;
+        for (const auto& kv : messagesPerConsumer) {
+            int count = kv.second;
+            ASSERT_LT(fabs(count - expectedMessagesPerConsumer), expectedMessagesPerConsumer * PERCENT_ERROR);
+        }
+    }
+
+    Client client;
+    std::vector<Producer> producers;
+    std::vector<Consumer> consumers;
+    const std::string subName = "SubscriptionName";
+
+    // key is message's ordering key or partitioned key, value is consumer index
+    std::map<std::string, size_t> keyToConsumer;
+    // key is consumer index, value is the number of message received by
+    std::map<size_t, int> messagesPerConsumer;
+};
+
+TEST_F(KeySharedPolicyTest, testStickyConsumer) {
+    const std::string topicName = "KeySharedPolicyTest-sticky-consumer" + std::to_string(time(nullptr));
+
+    consumers.emplace_back();
+    KeySharedPolicy ksp1;
+    ksp1.setKeySharedMode(STICKY);
+    ksp1.setStickyRanges({StickyRange(0, 20000)});
+    ConsumerConfiguration consumerConfig1 = getConsumerConfiguration().setKeySharedPolicy(ksp1);
+    Result result = client.subscribe(topicName, subName, consumerConfig1, consumers.back());
+    ASSERT_EQ(ResultOk, result);
+
+    consumers.emplace_back();
+    KeySharedPolicy ksp2;
+    ksp2.setKeySharedMode(STICKY);
+    ksp2.setStickyRanges({StickyRange(20001, 40000)});
+    ConsumerConfiguration consumerConfig2 = getConsumerConfiguration().setKeySharedPolicy(ksp2);
+    result = client.subscribe(topicName, subName, consumerConfig2, consumers.back());
+    ASSERT_EQ(ResultOk, result);
+
+    consumers.emplace_back();
+    KeySharedPolicy ksp3;
+    ksp3.setKeySharedMode(STICKY);
+    ksp3.setStickyRanges({StickyRange(40001, 65535)});
+    ConsumerConfiguration consumerConfig3 = getConsumerConfiguration().setKeySharedPolicy(ksp3);
+    result = client.subscribe(topicName, subName, consumerConfig3, consumers.back());
+    ASSERT_EQ(ResultOk, result);
+
+    addProducer(topicName);
+
+    srand(time(nullptr));
+    constexpr int numMessagesPerProducer = 1000;
+    for (int i = 0; i < numMessagesPerProducer; i++) {
+        std::string key = std::to_string(rand() % NUMBER_OF_KEYS);
+        producers[0].sendAsync(newIntMessage(i, key), sendCallback);
+    }
+    ASSERT_EQ(ResultOk, producers[0].flush());
+
+    receiveAndCheckDistribution(numMessagesPerProducer);
+}
+
+TEST_F(KeySharedPolicyTest, ResultConsumerAssignError) {
+    const std::string topicName =
+        "KeySharedPolicyTest-result-consumer-assign-error" + std::to_string(time(nullptr));
+
+    // empty range
+    KeySharedPolicy ksp;
+    ksp.setKeySharedMode(STICKY);
+    ConsumerConfiguration consumerConfig = getConsumerConfiguration().setKeySharedPolicy(ksp);
+    Consumer consumer;
+    ASSERT_EQ(ResultConsumerAssignError, client.subscribe(topicName, subName, consumerConfig, consumer));
+
+    // intersect range
+    KeySharedPolicy ksp1;
+    ksp1.setKeySharedMode(STICKY);
+    ksp1.setStickyRanges({StickyRange(0, 65535)});
+    ConsumerConfiguration consumerConfig1 = getConsumerConfiguration().setKeySharedPolicy(ksp1);
+    Consumer consumer1;
+    Result result = client.subscribe(topicName, subName, consumerConfig1, consumer1);
+    ASSERT_EQ(ResultOk, result);
+
+    KeySharedPolicy ksp2;
+    ksp2.setKeySharedMode(STICKY);
+    ksp2.setStickyRanges({StickyRange(0, 65535)});
+    ConsumerConfiguration consumerConfig2 = getConsumerConfiguration().setKeySharedPolicy(ksp2);
+    Consumer consumer2;
+    ASSERT_EQ(ResultConsumerAssignError, client.subscribe(topicName, subName, consumerConfig2, consumer2));
+
+    ASSERT_EQ(ResultOk, consumer1.close());
+}
+
+TEST_F(KeySharedPolicyTest, InvalidStickyRanges) {
+    KeySharedPolicy ksp;
+    ASSERT_THROW(ksp.setStickyRanges({}), std::invalid_argument);
+    ASSERT_THROW(ksp.setStickyRanges({StickyRange(-1, 10)}), std::invalid_argument);
+    ASSERT_THROW(ksp.setStickyRanges({StickyRange(0, 65536)}), std::invalid_argument);
+    ASSERT_THROW(ksp.setStickyRanges({StickyRange(0, 10), StickyRange(9, 20)}), std::invalid_argument);
+}