You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by pe...@apache.org on 2021/07/07 03:32:18 UTC
[pulsar] 05/07: [C++] Fix Setting KeySharedMeta in SubscribeCommand
(#11088)
This is an automated email from the ASF dual-hosted git repository.
penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 72479b8a9a51bc763efae5ed873532e832861fa2
Author: Yuto Furuta <mz...@gmail.com>
AuthorDate: Sat Jun 26 03:51:04 2021 +0900
[C++] Fix Setting KeySharedMeta in SubscribeCommand (#11088)
### Motivation
https://github.com/apache/pulsar/blob/67adf0aedf4b435a43fc570afa92154915f5b212/pulsar-client-cpp/lib/Commands.cc#L295-L306
Regarding the above, it seems that `KeySharedMeta ksm` is not set as subscribe command.
As a test, when `STICKY` is set, the consumer can connect to the broker without ranges.
It seems that `AUTO_SPLIT` as default value is used.
### Modification
Fix `that KeySharedMeta meta` is able to be set.
Also, implement sticky consumer in C++ client.
(cherry picked from commit eaa4ab03ab8e9905b3dd24a10b86d6c416ed4507)
---
pulsar-client-cpp/include/pulsar/KeySharedPolicy.h | 16 ++
pulsar-client-cpp/lib/Commands.cc | 9 +-
pulsar-client-cpp/lib/KeySharedPolicy.cc | 28 +++
pulsar-client-cpp/lib/KeySharedPolicyImpl.h | 1 +
pulsar-client-cpp/tests/KeySharedPolicyTest.cc | 203 +++++++++++++++++++++
5 files changed, 255 insertions(+), 2 deletions(-)
diff --git a/pulsar-client-cpp/include/pulsar/KeySharedPolicy.h b/pulsar-client-cpp/include/pulsar/KeySharedPolicy.h
index 09f0d1f..53efc4c 100644
--- a/pulsar-client-cpp/include/pulsar/KeySharedPolicy.h
+++ b/pulsar-client-cpp/include/pulsar/KeySharedPolicy.h
@@ -22,6 +22,9 @@
#include <memory>
+#include <utility>
+#include <vector>
+
namespace pulsar {
/**
@@ -44,6 +47,9 @@ enum KeySharedMode
struct KeySharedPolicyImpl;
+typedef std::pair<int, int> StickyRange;
+typedef std::vector<StickyRange> StickyRanges;
+
class PULSAR_PUBLIC KeySharedPolicy {
public:
KeySharedPolicy();
@@ -88,6 +94,16 @@ class PULSAR_PUBLIC KeySharedPolicy {
*/
bool isAllowOutOfOrderDelivery() const;
+ /**
+ * @param ranges used with sticky mode
+ */
+ KeySharedPolicy& setStickyRanges(std::initializer_list<StickyRange> ranges);
+
+ /**
+ * @return ranges used with sticky mode
+ */
+ StickyRanges getStickyRanges() const;
+
private:
std::shared_ptr<KeySharedPolicyImpl> impl_;
};
diff --git a/pulsar-client-cpp/lib/Commands.cc b/pulsar-client-cpp/lib/Commands.cc
index 5fbfdde..5b17d24 100644
--- a/pulsar-client-cpp/lib/Commands.cc
+++ b/pulsar-client-cpp/lib/Commands.cc
@@ -293,15 +293,20 @@ SharedBuffer Commands::newSubscribe(const std::string& topic, const std::string&
}
if (subType == CommandSubscribe_SubType_Key_Shared) {
- KeySharedMeta ksm;
+ KeySharedMeta& ksm = *subscribe->mutable_keysharedmeta();
switch (keySharedPolicy.getKeySharedMode()) {
case pulsar::AUTO_SPLIT:
ksm.set_keysharedmode(proto::KeySharedMode::AUTO_SPLIT);
break;
case pulsar::STICKY:
ksm.set_keysharedmode(proto::KeySharedMode::STICKY);
+ for (StickyRange range : keySharedPolicy.getStickyRanges()) {
+ IntRange* intRange = IntRange().New();
+ intRange->set_start(range.first);
+ intRange->set_end(range.second);
+ ksm.mutable_hashranges()->AddAllocated(intRange);
+ }
}
-
ksm.set_allowoutoforderdelivery(keySharedPolicy.isAllowOutOfOrderDelivery());
}
diff --git a/pulsar-client-cpp/lib/KeySharedPolicy.cc b/pulsar-client-cpp/lib/KeySharedPolicy.cc
index 0fdf576..3e3c5eb 100644
--- a/pulsar-client-cpp/lib/KeySharedPolicy.cc
+++ b/pulsar-client-cpp/lib/KeySharedPolicy.cc
@@ -18,8 +18,12 @@
*/
#include <lib/KeySharedPolicyImpl.h>
+#include <stdexcept>
+
namespace pulsar {
+static const int DefaultHashRangeSize = 2 << 15;
+
KeySharedPolicy::KeySharedPolicy() : impl_(std::make_shared<KeySharedPolicyImpl>()) {}
KeySharedPolicy::~KeySharedPolicy() {}
@@ -45,6 +49,30 @@ KeySharedPolicy &KeySharedPolicy::setAllowOutOfOrderDelivery(bool allowOutOfOrde
bool KeySharedPolicy::isAllowOutOfOrderDelivery() const { return impl_->allowOutOfOrderDelivery; }
+KeySharedPolicy &KeySharedPolicy::setStickyRanges(std::initializer_list<StickyRange> ranges) {
+ if (ranges.size() == 0) {
+ throw std::invalid_argument("Ranges for KeyShared policy must not be empty.");
+ }
+ for (StickyRange range : ranges) {
+ if (range.first < 0 || range.second >= DefaultHashRangeSize) {
+ throw std::invalid_argument("KeySharedPolicy Exception: Ranges must be [0, 65535].");
+ }
+ for (StickyRange range2 : ranges) {
+ int start = std::max(range.first, range2.first);
+ int end = std::min(range.second, range2.second);
+ if (range != range2 && end >= start) {
+ throw std::invalid_argument("Ranges for KeyShared policy with overlap.");
+ }
+ }
+ for (StickyRange range : ranges) {
+ impl_->ranges.push_back(range);
+ }
+ }
+ return *this;
+}
+
+StickyRanges KeySharedPolicy::getStickyRanges() const { return impl_->ranges; }
+
KeySharedPolicy KeySharedPolicy::clone() const {
KeySharedPolicy newConf;
newConf.impl_.reset(new KeySharedPolicyImpl(*this->impl_));
diff --git a/pulsar-client-cpp/lib/KeySharedPolicyImpl.h b/pulsar-client-cpp/lib/KeySharedPolicyImpl.h
index a12f48a..51ad491 100644
--- a/pulsar-client-cpp/lib/KeySharedPolicyImpl.h
+++ b/pulsar-client-cpp/lib/KeySharedPolicyImpl.h
@@ -25,6 +25,7 @@ namespace pulsar {
struct KeySharedPolicyImpl {
bool allowOutOfOrderDelivery;
KeySharedMode keySharedMode;
+ StickyRanges ranges;
KeySharedPolicyImpl() : allowOutOfOrderDelivery(false), keySharedMode(AUTO_SPLIT) {}
};
diff --git a/pulsar-client-cpp/tests/KeySharedPolicyTest.cc b/pulsar-client-cpp/tests/KeySharedPolicyTest.cc
new file mode 100644
index 0000000..49fef3f
--- /dev/null
+++ b/pulsar-client-cpp/tests/KeySharedPolicyTest.cc
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <cmath>
+#include <ctime>
+#include <vector>
+#include <map>
+
+#include <gtest/gtest.h>
+#include <pulsar/Client.h>
+#include <pulsar/KeySharedPolicy.h>
+#include "lib/LogUtils.h"
+
+#include "HttpHelper.h"
+#include "LogHelper.h"
+
+DECLARE_LOG_OBJECT()
+
+using namespace pulsar;
+
+static std::string lookupUrl = "pulsar://localhost:6650";
+
+class KeySharedPolicyTest : public ::testing::Test {
+ protected:
+ KeySharedPolicyTest() : client(lookupUrl, ClientConfiguration().setPartititionsUpdateInterval(1)) {}
+
+ void TearDown() override { client.close(); }
+
+ void addProducer(const std::string& topicName) {
+ producers.emplace_back();
+ auto conf = ProducerConfiguration().setBatchingEnabled(false).setPartitionsRoutingMode(
+ ProducerConfiguration::RoundRobinDistribution);
+ ASSERT_EQ(ResultOk, client.createProducer(topicName, conf, producers.back()));
+ }
+
+ ConsumerConfiguration getConsumerConfiguration() {
+ ConsumerConfiguration conf;
+ conf.setConsumerType(ConsumerKeyShared);
+ return conf;
+ }
+
+ static constexpr int NUMBER_OF_KEYS = 300;
+
+ static Message newIntMessage(int i, const std::string& key, const char* orderingKey = nullptr) {
+ MessageBuilder builder;
+ if (orderingKey) {
+ builder.setOrderingKey(orderingKey);
+ }
+ return builder.setPartitionKey(key).setContent(std::to_string(i)).build();
+ }
+
+ static void sendCallback(Result result, const MessageId&) { ASSERT_EQ(result, ResultOk); }
+
+ void receiveAndCheckDistribution(int expectedNumTotalMessages) {
+ keyToConsumer.clear();
+ messagesPerConsumer.clear();
+ int totalMessages = 0;
+
+ for (size_t i = 0; i < consumers.size(); i++) {
+ auto& consumer = consumers[i];
+ while (true) {
+ Message msg;
+ Result result = consumer.receive(msg, 3000);
+ if (result == ResultTimeout) {
+ break;
+ }
+
+ ASSERT_EQ(result, ResultOk);
+ totalMessages++;
+ messagesPerConsumer[i]++;
+ ASSERT_EQ(ResultOk, consumer.acknowledge(msg));
+
+ if (msg.hasPartitionKey() || msg.hasOrderingKey()) {
+ std::string key = msg.hasOrderingKey() ? msg.getOrderingKey() : msg.getPartitionKey();
+ auto iter = keyToConsumer.find(key);
+ if (iter == keyToConsumer.end()) {
+ keyToConsumer[key] = i;
+ } else {
+ // check messages with the same key will be consumed by the same consumer
+ ASSERT_EQ(iter->second, i);
+ }
+ }
+ }
+ }
+
+ LOG_INFO("messagesPerConsumer: " << messagesPerConsumer);
+ int numTotalMessages = 0;
+ for (const auto& kv : messagesPerConsumer) {
+ numTotalMessages += kv.second;
+ }
+ ASSERT_EQ(numTotalMessages, expectedNumTotalMessages);
+
+ const double expectedMessagesPerConsumer = static_cast<double>(totalMessages) / consumers.size();
+ constexpr double PERCENT_ERROR = 0.50;
+ for (const auto& kv : messagesPerConsumer) {
+ int count = kv.second;
+ ASSERT_LT(fabs(count - expectedMessagesPerConsumer), expectedMessagesPerConsumer * PERCENT_ERROR);
+ }
+ }
+
+ Client client;
+ std::vector<Producer> producers;
+ std::vector<Consumer> consumers;
+ const std::string subName = "SubscriptionName";
+
+ // key is message's ordering key or partitioned key, value is consumer index
+ std::map<std::string, size_t> keyToConsumer;
+ // key is consumer index, value is the number of message received by
+ std::map<size_t, int> messagesPerConsumer;
+};
+
+TEST_F(KeySharedPolicyTest, testStickyConsumer) {
+ const std::string topicName = "KeySharedPolicyTest-sticky-consumer" + std::to_string(time(nullptr));
+
+ consumers.emplace_back();
+ KeySharedPolicy ksp1;
+ ksp1.setKeySharedMode(STICKY);
+ ksp1.setStickyRanges({StickyRange(0, 20000)});
+ ConsumerConfiguration consumerConfig1 = getConsumerConfiguration().setKeySharedPolicy(ksp1);
+ Result result = client.subscribe(topicName, subName, consumerConfig1, consumers.back());
+ ASSERT_EQ(ResultOk, result);
+
+ consumers.emplace_back();
+ KeySharedPolicy ksp2;
+ ksp2.setKeySharedMode(STICKY);
+ ksp2.setStickyRanges({StickyRange(20001, 40000)});
+ ConsumerConfiguration consumerConfig2 = getConsumerConfiguration().setKeySharedPolicy(ksp2);
+ result = client.subscribe(topicName, subName, consumerConfig2, consumers.back());
+ ASSERT_EQ(ResultOk, result);
+
+ consumers.emplace_back();
+ KeySharedPolicy ksp3;
+ ksp3.setKeySharedMode(STICKY);
+ ksp3.setStickyRanges({StickyRange(40001, 65535)});
+ ConsumerConfiguration consumerConfig3 = getConsumerConfiguration().setKeySharedPolicy(ksp3);
+ result = client.subscribe(topicName, subName, consumerConfig3, consumers.back());
+ ASSERT_EQ(ResultOk, result);
+
+ addProducer(topicName);
+
+ srand(time(nullptr));
+ constexpr int numMessagesPerProducer = 1000;
+ for (int i = 0; i < numMessagesPerProducer; i++) {
+ std::string key = std::to_string(rand() % NUMBER_OF_KEYS);
+ producers[0].sendAsync(newIntMessage(i, key), sendCallback);
+ }
+ ASSERT_EQ(ResultOk, producers[0].flush());
+
+ receiveAndCheckDistribution(numMessagesPerProducer);
+}
+
+TEST_F(KeySharedPolicyTest, ResultConsumerAssignError) {
+ const std::string topicName =
+ "KeySharedPolicyTest-result-consumer-assign-error" + std::to_string(time(nullptr));
+
+ // empty range
+ KeySharedPolicy ksp;
+ ksp.setKeySharedMode(STICKY);
+ ConsumerConfiguration consumerConfig = getConsumerConfiguration().setKeySharedPolicy(ksp);
+ Consumer consumer;
+ ASSERT_EQ(ResultConsumerAssignError, client.subscribe(topicName, subName, consumerConfig, consumer));
+
+ // intersect range
+ KeySharedPolicy ksp1;
+ ksp1.setKeySharedMode(STICKY);
+ ksp1.setStickyRanges({StickyRange(0, 65535)});
+ ConsumerConfiguration consumerConfig1 = getConsumerConfiguration().setKeySharedPolicy(ksp1);
+ Consumer consumer1;
+ Result result = client.subscribe(topicName, subName, consumerConfig1, consumer1);
+ ASSERT_EQ(ResultOk, result);
+
+ KeySharedPolicy ksp2;
+ ksp2.setKeySharedMode(STICKY);
+ ksp2.setStickyRanges({StickyRange(0, 65535)});
+ ConsumerConfiguration consumerConfig2 = getConsumerConfiguration().setKeySharedPolicy(ksp2);
+ Consumer consumer2;
+ ASSERT_EQ(ResultConsumerAssignError, client.subscribe(topicName, subName, consumerConfig2, consumer2));
+
+ ASSERT_EQ(ResultOk, consumer1.close());
+}
+
+TEST_F(KeySharedPolicyTest, InvalidStickyRanges) {
+ KeySharedPolicy ksp;
+ ASSERT_THROW(ksp.setStickyRanges({}), std::invalid_argument);
+ ASSERT_THROW(ksp.setStickyRanges({StickyRange(-1, 10)}), std::invalid_argument);
+ ASSERT_THROW(ksp.setStickyRanges({StickyRange(0, 65536)}), std::invalid_argument);
+ ASSERT_THROW(ksp.setStickyRanges({StickyRange(0, 10), StickyRange(9, 20)}), std::invalid_argument);
+}