You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2019/02/12 20:44:06 UTC

[pulsar] branch master updated: [cpp-client] add SubscriptionInitialPosition option in ConsumerConfiguration #3494 (#3567)

This is an automated email from the ASF dual-hosted git repository.

mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new d0a3d5f  [cpp-client] add SubscriptionInitialPosition option in ConsumerConfiguration  #3494 (#3567)
d0a3d5f is described below

commit d0a3d5f328cee22501b90389977a39d3a71f99cb
Author: 冉小龙 <rx...@qq.com>
AuthorDate: Wed Feb 13 04:43:57 2019 +0800

    [cpp-client] add SubscriptionInitialPosition option in ConsumerConfiguration  #3494 (#3567)
    
    * [cpp-client] add multi topic for consumer #3494
    
    Signed-off-by: xiaolong.ran <ra...@gmail.com>
    
    * use initial_position instead to multi-topics
    
    Signed-off-by: xiaolong.ran <ra...@gmail.com>
---
 .../include/pulsar/ConsumerConfiguration.h         |  4 ++
 pulsar-client-cpp/include/pulsar/InitialPosition.h | 31 +++++++++++++
 .../include/pulsar/c/consumer_configuration.h      | 19 ++++++++
 pulsar-client-cpp/lib/Commands.cc                  |  4 +-
 pulsar-client-cpp/lib/Commands.h                   |  3 +-
 pulsar-client-cpp/lib/ConsumerConfiguration.cc     |  8 ++++
 pulsar-client-cpp/lib/ConsumerConfigurationImpl.h  |  2 +
 pulsar-client-cpp/lib/ConsumerImpl.cc              | 17 ++++++--
 pulsar-client-cpp/lib/ConsumerImpl.h               |  1 +
 pulsar-client-cpp/lib/c/c_ConsumerConfiguration.cc | 11 +++++
 .../tests/ConsumerConfigurationTest.cc             | 51 +++++++++++++++++++++-
 11 files changed, 145 insertions(+), 6 deletions(-)

diff --git a/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h b/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h
index d63514e..e30ab80 100644
--- a/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h
+++ b/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h
@@ -27,6 +27,7 @@
 #include <pulsar/Schema.h>
 #include <pulsar/ConsumerCryptoFailureAction.h>
 #include <pulsar/CryptoKeyReader.h>
+#include <pulsar/InitialPosition.h>
 
 #pragma GCC visibility push(default)
 namespace pulsar {
@@ -179,6 +180,9 @@ class ConsumerConfiguration {
     void setPatternAutoDiscoveryPeriod(int periodInSeconds);
     int getPatternAutoDiscoveryPeriod() const;
 
+    void setSubscriptionInitialPosition(InitialPosition subscriptionInitialPosition);
+    InitialPosition getSubscriptionInitialPosition() const;
+
     /**
      * Check whether the message has a specific property attached.
      *
diff --git a/pulsar-client-cpp/include/pulsar/InitialPosition.h b/pulsar-client-cpp/include/pulsar/InitialPosition.h
new file mode 100644
index 0000000..adb97e9
--- /dev/null
+++ b/pulsar-client-cpp/include/pulsar/InitialPosition.h
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef PULSAR_CPP_INITIAL_POSITION_H
+#define PULSAR_CPP_INITIAL_POSITION_H
+
+#pragma GCC visibility push(default)
+namespace pulsar {
+enum InitialPosition
+{
+    InitialPositionLatest,
+    InitialPositionEarliest
+};
+}
+#pragma GCC visibility pop
+#endif  // PULSAR_CPP_INITIAL_POSITION_H
diff --git a/pulsar-client-cpp/include/pulsar/c/consumer_configuration.h b/pulsar-client-cpp/include/pulsar/c/consumer_configuration.h
index fca47eb..b90963a 100644
--- a/pulsar-client-cpp/include/pulsar/c/consumer_configuration.h
+++ b/pulsar-client-cpp/include/pulsar/c/consumer_configuration.h
@@ -18,6 +18,8 @@
  */
 #pragma once
 
+#include "consumer.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -44,6 +46,17 @@ typedef enum {
     pulsar_ConsumerFailover
 } pulsar_consumer_type;
 
+typedef enum {
+    /**
+     * the latest position which means the start consuming position will be the last message
+     */
+    initial_position_latest,
+    /**
+     * the earliest position which means the start consuming position will be the first message
+     */
+    initial_position_earliest
+} initial_position;
+
 /// Callback definition for MessageListener
 typedef void (*pulsar_message_listener)(pulsar_consumer_t *consumer, pulsar_message_t *msg, void *ctx);
 
@@ -152,6 +165,12 @@ int pulsar_consumer_is_read_compacted(pulsar_consumer_configuration_t *consumer_
 void pulsar_consumer_set_read_compacted(pulsar_consumer_configuration_t *consumer_configuration,
                                         int compacted);
 
+int pulsar_consumer_get_subscription_initial_position(
+    pulsar_consumer_configuration_t *consumer_configuration);
+
+void pulsar_consumer_set_subscription_initial_position(
+    pulsar_consumer_configuration_t *consumer_configuration, initial_position subscriptionInitialPosition);
+
 void pulsar_consumer_configuration_set_property(pulsar_consumer_configuration_t *conf, const char *name,
                                                 const char *value);
 
diff --git a/pulsar-client-cpp/lib/Commands.cc b/pulsar-client-cpp/lib/Commands.cc
index 384e54c..726939f 100644
--- a/pulsar-client-cpp/lib/Commands.cc
+++ b/pulsar-client-cpp/lib/Commands.cc
@@ -233,7 +233,8 @@ SharedBuffer Commands::newSubscribe(const std::string& topic, const std::string&
                                     const std::string& consumerName, SubscriptionMode subscriptionMode,
                                     Optional<MessageId> startMessageId, bool readCompacted,
                                     const std::map<std::string, std::string>& metadata,
-                                    const SchemaInfo& schemaInfo) {
+                                    const SchemaInfo& schemaInfo,
+                                    CommandSubscribe_InitialPosition subscriptionInitialPosition) {
     BaseCommand cmd;
     cmd.set_type(BaseCommand::SUBSCRIBE);
     CommandSubscribe* subscribe = cmd.mutable_subscribe();
@@ -245,6 +246,7 @@ SharedBuffer Commands::newSubscribe(const std::string& topic, const std::string&
     subscribe->set_consumer_name(consumerName);
     subscribe->set_durable(subscriptionMode == SubscriptionModeDurable);
     subscribe->set_read_compacted(readCompacted);
+    subscribe->set_initialposition(subscriptionInitialPosition);
 
     if (isBuiltInSchema(schemaInfo.getSchemaType())) {
         subscribe->set_allocated_schema(getSchema(schemaInfo));
diff --git a/pulsar-client-cpp/lib/Commands.h b/pulsar-client-cpp/lib/Commands.h
index fd018c3..967270c 100644
--- a/pulsar-client-cpp/lib/Commands.h
+++ b/pulsar-client-cpp/lib/Commands.h
@@ -80,7 +80,8 @@ class Commands {
                                      proto::CommandSubscribe_SubType subType, const std::string& consumerName,
                                      SubscriptionMode subscriptionMode, Optional<MessageId> startMessageId,
                                      bool readCompacted, const std::map<std::string, std::string>& metadata,
-                                     const SchemaInfo& schemaInfo);
+                                     const SchemaInfo& schemaInfo,
+                                     proto::CommandSubscribe_InitialPosition subscriptionInitialPosition);
 
     static SharedBuffer newUnsubscribe(uint64_t consumerId, uint64_t requestId);
 
diff --git a/pulsar-client-cpp/lib/ConsumerConfiguration.cc b/pulsar-client-cpp/lib/ConsumerConfiguration.cc
index 1a33da6..4530ebd 100644
--- a/pulsar-client-cpp/lib/ConsumerConfiguration.cc
+++ b/pulsar-client-cpp/lib/ConsumerConfiguration.cc
@@ -114,6 +114,14 @@ bool ConsumerConfiguration::isReadCompacted() const { return impl_->readCompacte
 
 void ConsumerConfiguration::setReadCompacted(bool compacted) { impl_->readCompacted = compacted; }
 
+void ConsumerConfiguration::setSubscriptionInitialPosition(InitialPosition subscriptionInitialPosition) {
+    impl_->subscriptionInitialPosition = subscriptionInitialPosition;
+}
+
+InitialPosition ConsumerConfiguration::getSubscriptionInitialPosition() const {
+    return impl_->subscriptionInitialPosition;
+}
+
 void ConsumerConfiguration::setPatternAutoDiscoveryPeriod(int periodInSeconds) {
     impl_->patternAutoDiscoveryPeriod = periodInSeconds;
 }
diff --git a/pulsar-client-cpp/lib/ConsumerConfigurationImpl.h b/pulsar-client-cpp/lib/ConsumerConfigurationImpl.h
index 556e9a3..70f725b 100644
--- a/pulsar-client-cpp/lib/ConsumerConfigurationImpl.h
+++ b/pulsar-client-cpp/lib/ConsumerConfigurationImpl.h
@@ -35,6 +35,7 @@ struct ConsumerConfigurationImpl {
     CryptoKeyReaderPtr cryptoKeyReader;
     ConsumerCryptoFailureAction cryptoFailureAction;
     bool readCompacted;
+    InitialPosition subscriptionInitialPosition;
     int patternAutoDiscoveryPeriod;
     std::map<std::string, std::string> properties;
     ConsumerConfigurationImpl()
@@ -49,6 +50,7 @@ struct ConsumerConfigurationImpl {
           cryptoKeyReader(),
           cryptoFailureAction(ConsumerCryptoFailureAction::FAIL),
           readCompacted(false),
+          subscriptionInitialPosition(InitialPosition::InitialPositionLatest),
           patternAutoDiscoveryPeriod(60),
           properties() {}
 };
diff --git a/pulsar-client-cpp/lib/ConsumerImpl.cc b/pulsar-client-cpp/lib/ConsumerImpl.cc
index e254480..1c46323 100644
--- a/pulsar-client-cpp/lib/ConsumerImpl.cc
+++ b/pulsar-client-cpp/lib/ConsumerImpl.cc
@@ -137,9 +137,9 @@ void ConsumerImpl::connectionOpened(const ClientConnectionPtr& cnx) {
 
     ClientImplPtr client = client_.lock();
     uint64_t requestId = client->newRequestId();
-    SharedBuffer cmd = Commands::newSubscribe(topic_, subscription_, consumerId_, requestId, getSubType(),
-                                              consumerName_, subscriptionMode_, startMessageId_,
-                                              readCompacted_, config_.getProperties(), config_.getSchema());
+    SharedBuffer cmd = Commands::newSubscribe(
+        topic_, subscription_, consumerId_, requestId, getSubType(), consumerName_, subscriptionMode_,
+        startMessageId_, readCompacted_, config_.getProperties(), config_.getSchema(), getInitialPosition());
     cnx->sendRequestWithId(cmd, requestId)
         .addListener(
             std::bind(&ConsumerImpl::handleCreateConsumer, shared_from_this(), cnx, std::placeholders::_1));
@@ -720,6 +720,17 @@ inline proto::CommandSubscribe_SubType ConsumerImpl::getSubType() {
     }
 }
 
+inline proto::CommandSubscribe_InitialPosition ConsumerImpl::getInitialPosition() {
+    InitialPosition initialPosition = config_.getSubscriptionInitialPosition();
+    switch (initialPosition) {
+        case InitialPositionLatest:
+            return proto::CommandSubscribe_InitialPosition ::CommandSubscribe_InitialPosition_Latest;
+
+        case InitialPositionEarliest:
+            return proto::CommandSubscribe_InitialPosition ::CommandSubscribe_InitialPosition_Earliest;
+    }
+}
+
 void ConsumerImpl::statsCallback(Result res, ResultCallback callback, proto::CommandAck_AckType ackType) {
     consumerStatsBasePtr_->messageAcknowledged(res, ackType);
     if (callback) {
diff --git a/pulsar-client-cpp/lib/ConsumerImpl.h b/pulsar-client-cpp/lib/ConsumerImpl.h
index 01aa55f..747c276 100644
--- a/pulsar-client-cpp/lib/ConsumerImpl.h
+++ b/pulsar-client-cpp/lib/ConsumerImpl.h
@@ -78,6 +78,7 @@ class ConsumerImpl : public ConsumerImplBase,
     int incrementAndGetPermits(uint64_t cnxSequenceId);
     void messageProcessed(Message& msg);
     inline proto::CommandSubscribe_SubType getSubType();
+    inline proto::CommandSubscribe_InitialPosition getInitialPosition();
     void unsubscribeAsync(ResultCallback callback);
     void handleUnsubscribe(Result result, ResultCallback callback);
     void doAcknowledge(const MessageId& messageId, proto::CommandAck_AckType ackType,
diff --git a/pulsar-client-cpp/lib/c/c_ConsumerConfiguration.cc b/pulsar-client-cpp/lib/c/c_ConsumerConfiguration.cc
index 924060d..40a6a8f 100644
--- a/pulsar-client-cpp/lib/c/c_ConsumerConfiguration.cc
+++ b/pulsar-client-cpp/lib/c/c_ConsumerConfiguration.cc
@@ -118,3 +118,14 @@ void pulsar_consumer_configuration_set_property(pulsar_consumer_configuration_t
                                                 const char *value) {
     conf->consumerConfiguration.setProperty(name, value);
 }
+
+void pulsar_consumer_set_subscription_initial_position(
+    pulsar_consumer_configuration_t *consumer_configuration, initial_position subscriptionInitialPosition) {
+    consumer_configuration->consumerConfiguration.setSubscriptionInitialPosition(
+        (pulsar::InitialPosition)subscriptionInitialPosition);
+}
+
+int pulsar_consumer_get_subscription_initial_position(
+    pulsar_consumer_configuration_t *consumer_configuration) {
+    return consumer_configuration->consumerConfiguration.getSubscriptionInitialPosition();
+}
diff --git a/pulsar-client-cpp/tests/ConsumerConfigurationTest.cc b/pulsar-client-cpp/tests/ConsumerConfigurationTest.cc
index e4944ec..0a7411d 100644
--- a/pulsar-client-cpp/tests/ConsumerConfigurationTest.cc
+++ b/pulsar-client-cpp/tests/ConsumerConfigurationTest.cc
@@ -18,6 +18,11 @@
  */
 #include <pulsar/Client.h>
 #include <gtest/gtest.h>
+#include <boost/date_time/posix_time/ptime.hpp>
+#include <boost/bind.hpp>
+#include <lib/LogUtils.h>
+
+DECLARE_LOG_OBJECT()
 
 #include "../lib/Future.h"
 #include "../lib/Utils.h"
@@ -102,4 +107,48 @@ TEST(ConsumerConfigurationTest, testReadCompactNonPersistentExclusive) {
     result = client.subscribe(topicName, subName, config, consumer);
     ASSERT_EQ(ResultInvalidConfiguration, result);
     consumer.close();
-}
\ No newline at end of file
+}
+
+TEST(ConsumerConfigurationTest, testSubscriptionInitialPosition) {
+    std::string lookupUrl = "pulsar://localhost:6650";
+    std::string topicName = "persist-topic-test-position";
+    std::string subName = "test-subscription-initial-earliest-position";
+
+    ClientConfiguration clientConfig;
+    Client client(lookupUrl, clientConfig);
+
+    LOG_INFO("create 1 producer...");
+    Producer producer;
+    Result result;
+    ProducerConfiguration conf;
+    result = client.createProducer(topicName, conf, producer);
+    ASSERT_EQ(ResultOk, result);
+
+    // Send synchronously
+    std::string content1 = "msg-1-content-1";
+    Message msg = MessageBuilder().setContent(content1).build();
+    result = producer.send(msg);
+    ASSERT_EQ(ResultOk, result);
+
+    std::string content2 = "msg-2-content-2";
+    msg = MessageBuilder().setContent(content2).build();
+    result = producer.send(msg);
+    ASSERT_EQ(ResultOk, result);
+
+    Consumer consumer;
+    ConsumerConfiguration config;
+    config.setSubscriptionInitialPosition(InitialPosition::InitialPositionEarliest);
+    result = client.subscribe(topicName, subName, config, consumer);
+    ASSERT_EQ(ResultOk, result);
+
+    Message receivedMsg;
+
+    result = consumer.receive(receivedMsg, 2000);
+    ASSERT_EQ(ResultOk, result);
+    ASSERT_EQ(content1, receivedMsg.getDataAsString());
+
+    ASSERT_EQ(ResultOk, consumer.unsubscribe());
+    ASSERT_EQ(ResultAlreadyClosed, consumer.close());
+    ASSERT_EQ(ResultOk, producer.close());
+    ASSERT_EQ(ResultOk, client.close());
+}