You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by pe...@apache.org on 2021/06/25 04:01:03 UTC

[pulsar] branch branch-2.8 updated (b61832d -> 0734191)

This is an automated email from the ASF dual-hosted git repository.

penghui pushed a change to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git.


    from b61832d  [Broker] Fix the backlog issue with --precise-backlog=true (#10966)
     new 3f000b8  Load credentials from secrets for Kinesis connectors (#10822)
     new b20da1c  When topic does not exist, optimize the prompt message (#10845)
     new 489d24d  [Fixed]  throwable exception not thrown. (#10863)
     new c2d3707  Fix documentation for transactions to disable send timeout by default (#10901)
     new ec86a9d  When the Replicator is enabled, no managedLedger is created when updating the number of partitions (#10910)
     new f13d6ad  Fix compaction not working for system topic (#10941)
     new 0734191  [Transactions] Prevent NPE in case of closeAsync() without a successful execution of startAsync() (#10948)

The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../pulsar/broker/admin/impl/NamespacesBase.java   |  4 +-
 .../broker/admin/impl/PersistentTopicsBase.java    |  3 +-
 .../apache/pulsar/broker/service/ServerCnx.java    |  3 +-
 .../broker/service/persistent/PersistentTopic.java |  3 +-
 .../pulsar/broker/admin/TopicPoliciesTest.java     | 31 ++++++++-
 .../pulsar/broker/service/ReplicatorTest.java      | 37 +++++++++++
 .../client/api/SimpleProducerConsumerTest.java     | 21 +++++++
 .../TransactionCoordinatorClientImpl.java          | 12 ++--
 pulsar-io/kinesis/pom.xml                          |  5 ++
 .../pulsar/io/kinesis/BaseKinesisConfig.java       |  1 +
 .../org/apache/pulsar/io/kinesis/KinesisSink.java  | 11 ++--
 .../pulsar/io/kinesis/KinesisSinkConfig.java       | 21 +++----
 .../apache/pulsar/io/kinesis/KinesisSource.java    | 19 +++---
 .../pulsar/io/kinesis/KinesisSourceConfig.java     |  5 --
 .../pulsar/io/kinesis/KinesisSinkConfigTests.java  | 36 +++++++++--
 .../io/kinesis/KinesisSourceConfigTests.java       | 73 +++++++++++++++++-----
 site2/docs/transaction-api.md                      |  1 +
 .../version-2.7.0/transaction-api.md               |  1 +
 .../version-2.7.1/transaction-api.md               |  1 +
 .../version-2.7.2/transaction-api.md               |  1 +
 .../version-2.7.3/transaction-api.md               |  1 +
 21 files changed, 226 insertions(+), 64 deletions(-)

[pulsar] 04/07: Fix documentation for transactions to disable send timeout by default (#10901)

Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit c2d37079fa515e0eff60733647fba59ada79fd00
Author: xuesongxs <54...@users.noreply.github.com>
AuthorDate: Wed Jun 16 09:23:43 2021 +0800

    Fix documentation for transactions to disable send timeout by default (#10901)
    
    
    (cherry picked from commit edc03c0bb6d4213beca74b429f2935904442af5e)
---
 site2/docs/transaction-api.md                                 | 1 +
 site2/website/versioned_docs/version-2.7.0/transaction-api.md | 1 +
 site2/website/versioned_docs/version-2.7.1/transaction-api.md | 1 +
 site2/website/versioned_docs/version-2.7.2/transaction-api.md | 1 +
 site2/website/versioned_docs/version-2.7.3/transaction-api.md | 1 +
 5 files changed, 5 insertions(+)

diff --git a/site2/docs/transaction-api.md b/site2/docs/transaction-api.md
index b615038..048eac1 100644
--- a/site2/docs/transaction-api.md
+++ b/site2/docs/transaction-api.md
@@ -106,6 +106,7 @@ Consumer<String> sourceConsumer = pulsarClient
 Producer<String> sinkProducer = pulsarClient
         .newProducer(Schema.STRING)
         .topic(sinkTopic)
+        .sendTimeout(0, TimeUnit.MILLISECONDS)
         .create();
 
 Transaction txn = pulsarClient
diff --git a/site2/website/versioned_docs/version-2.7.0/transaction-api.md b/site2/website/versioned_docs/version-2.7.0/transaction-api.md
index 0eeea49..7d6a897 100644
--- a/site2/website/versioned_docs/version-2.7.0/transaction-api.md
+++ b/site2/website/versioned_docs/version-2.7.0/transaction-api.md
@@ -109,6 +109,7 @@ Consumer<String> sourceConsumer = pulsarClient
 Producer<String> sinkProducer = pulsarClient
         .newProducer(Schema.STRING)
         .topic(sinkTopic)
+        .sendTimeout(0, TimeUnit.MILLISECONDS)
         .create();
 
 Transaction txn = pulsarClient
diff --git a/site2/website/versioned_docs/version-2.7.1/transaction-api.md b/site2/website/versioned_docs/version-2.7.1/transaction-api.md
index 45e46c8..630d3c2 100644
--- a/site2/website/versioned_docs/version-2.7.1/transaction-api.md
+++ b/site2/website/versioned_docs/version-2.7.1/transaction-api.md
@@ -109,6 +109,7 @@ Consumer<String> sourceConsumer = pulsarClient
 Producer<String> sinkProducer = pulsarClient
         .newProducer(Schema.STRING)
         .topic(sinkTopic)
+        .sendTimeout(0, TimeUnit.MILLISECONDS)
         .create();
 
 Transaction txn = pulsarClient
diff --git a/site2/website/versioned_docs/version-2.7.2/transaction-api.md b/site2/website/versioned_docs/version-2.7.2/transaction-api.md
index 2fead1b..baeed78 100644
--- a/site2/website/versioned_docs/version-2.7.2/transaction-api.md
+++ b/site2/website/versioned_docs/version-2.7.2/transaction-api.md
@@ -109,6 +109,7 @@ Consumer<String> sourceConsumer = pulsarClient
 Producer<String> sinkProducer = pulsarClient
         .newProducer(Schema.STRING)
         .topic(sinkTopic)
+        .sendTimeout(0, TimeUnit.MILLISECONDS)
         .create();
 
 Transaction txn = pulsarClient
diff --git a/site2/website/versioned_docs/version-2.7.3/transaction-api.md b/site2/website/versioned_docs/version-2.7.3/transaction-api.md
index 39b46a5..23c4509 100644
--- a/site2/website/versioned_docs/version-2.7.3/transaction-api.md
+++ b/site2/website/versioned_docs/version-2.7.3/transaction-api.md
@@ -109,6 +109,7 @@ Consumer<String> sourceConsumer = pulsarClient
 Producer<String> sinkProducer = pulsarClient
         .newProducer(Schema.STRING)
         .topic(sinkTopic)
+        .sendTimeout(0, TimeUnit.MILLISECONDS)
         .create();
 
 Transaction txn = pulsarClient

[pulsar] 07/07: [Transactions] Prevent NPE in case of closeAsync() without a successful execution of startAsync() (#10948)

Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 0734191f42f7fca638091bb38193d7b7eb31d4b3
Author: Enrico Olivelli <eo...@gmail.com>
AuthorDate: Thu Jun 17 12:47:11 2021 +0200

    [Transactions] Prevent NPE in case of closeAsync() without a successful execution of startAsync() (#10948)
    
    Fixes #10947
    ### Motivation
    
    If the initialisation of TransactionCoordinatorClientImpl is not completed calling closeAsync will result in a NPE.
    
    (cherry picked from commit 3fd1bfda34d3809797d1839b7b8f239426595c7c)
---
 .../impl/transaction/TransactionCoordinatorClientImpl.java   | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java
index f6e6b03..8db8054 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java
@@ -134,11 +134,13 @@ public class TransactionCoordinatorClientImpl implements TransactionCoordinatorC
             LOG.warn("The transaction meta store is closing or closed, doing nothing.");
             result.complete(null);
         } else {
-            for (TransactionMetaStoreHandler handler : handlers) {
-                try {
-                    handler.close();
-                } catch (IOException e) {
-                    LOG.warn("Close transaction meta store handler error", e);
+            if (handlers != null) {
+                for (TransactionMetaStoreHandler handler : handlers) {
+                    try {
+                        handler.close();
+                    } catch (IOException e) {
+                        LOG.warn("Close transaction meta store handler error", e);
+                    }
                 }
             }
             this.handlers = null;

[pulsar] 05/07: When the Replicator is enabled, no managedLedger is created when updating the number of partitions (#10910)

Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit ec86a9de598853fb38c2122730f035e8345b6165
Author: feynmanlin <fe...@tencent.com>
AuthorDate: Thu Jun 17 10:20:45 2021 +0800

    When the Replicator is enabled, no managedLedger is created when updating the number of partitions (#10910)
    
    Fixes #10673
    
    ### Motivation
    When updating the number of partitions, we need to update the data in two places in zk:
    ```
    /admin/partitioned-topics
    /managed-ledgers/
    ```
    
    Now we only update the number of partitions in `/admin/partitioned-topics`, so if we do not create a Producer or Consumer, the data obtained in another cluster will be incorrect
    
    ### Modifications
    1)Try to create managedLedger when updating the number of partitions
    2)Ensure that the number of partitions in `/admin/partitioned-topics` is updated every time
    
    (cherry picked from commit 202da117b529b24bdf9c994750266dac597294a8)
---
 .../broker/admin/impl/PersistentTopicsBase.java    |  3 +-
 .../pulsar/broker/service/ReplicatorTest.java      | 37 ++++++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index 8bf5a7c..97f4a8d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -422,6 +422,7 @@ public class PersistentTopicsBase extends AdminResource {
                 throw new RestException(Status.FORBIDDEN, "Local cluster is not part of replicate cluster list");
             }
             try {
+                tryCreatePartitionsAsync(numPartitions).get(DEFAULT_OPERATION_TIMEOUT_SEC, TimeUnit.SECONDS);
                 createSubscriptions(topicName, numPartitions).get(DEFAULT_OPERATION_TIMEOUT_SEC, TimeUnit.SECONDS);
             } catch (Exception e) {
                 if (e.getCause() instanceof RestException) {
@@ -435,7 +436,7 @@ public class PersistentTopicsBase extends AdminResource {
             if (!updateLocalTopicOnly) {
                 CompletableFuture<Void> updatePartition = new CompletableFuture<>();
                 final String path = ZkAdminPaths.partitionedTopicPath(topicName);
-                updatePartitionInOtherCluster(numPartitions, clusters).thenAccept((res) -> {
+                updatePartitionInOtherCluster(numPartitions, clusters).thenRun(() -> {
                     try {
                         namespaceResources().getPartitionedTopicResources().setAsync(path, (p) -> {
                             return new PartitionedTopicMetadata(numPartitions);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
index ead91a5..e5219d5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
@@ -36,6 +36,7 @@ import java.lang.reflect.Method;
 import java.util.List;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
@@ -803,6 +804,42 @@ public class ReplicatorTest extends ReplicatorTestBase {
         reader2.closeAsync().get();
     }
 
+    @Test
+    public void testReplicatorWithPartitionedTopic() throws Exception {
+        final String namespace = "pulsar/partitionedNs-" + UUID.randomUUID();
+        final String persistentTopicName = "persistent://" + namespace + "/partTopic" + UUID.randomUUID();
+
+        admin1.namespaces().createNamespace(namespace);
+        admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2", "r3"));
+        // Create partitioned-topic from R1
+        admin1.topics().createPartitionedTopic(persistentTopicName, 3);
+        // List partitioned topics from R2
+        Awaitility.await().untilAsserted(() -> assertNotNull(admin2.topics().getPartitionedTopicList(namespace)));
+        Awaitility.await().untilAsserted(() -> assertEquals(
+                admin2.topics().getPartitionedTopicList(namespace).get(0), persistentTopicName));
+        assertEquals(admin1.topics().getList(namespace).size(), 3);
+        // List partitioned topics from R3
+        Awaitility.await().untilAsserted(() -> assertNotNull(admin3.topics().getPartitionedTopicList(namespace)));
+        Awaitility.await().untilAsserted(() -> assertEquals(
+                admin3.topics().getPartitionedTopicList(namespace).get(0), persistentTopicName));
+        // Update partitioned topic from R2
+        admin2.topics().updatePartitionedTopic(persistentTopicName, 5);
+        assertEquals(admin2.topics().getPartitionedTopicMetadata(persistentTopicName).partitions, 5);
+        assertEquals(admin2.topics().getList(namespace).size(), 5);
+        // Update partitioned topic from R3
+        admin3.topics().updatePartitionedTopic(persistentTopicName, 5);
+        assertEquals(admin3.topics().getPartitionedTopicMetadata(persistentTopicName).partitions, 5);
+        assertEquals(admin3.topics().getList(namespace).size(), 5);
+        // Update partitioned topic from R1
+        admin1.topics().updatePartitionedTopic(persistentTopicName, 6);
+        assertEquals(admin1.topics().getPartitionedTopicMetadata(persistentTopicName).partitions, 6);
+        assertEquals(admin2.topics().getPartitionedTopicMetadata(persistentTopicName).partitions, 6);
+        assertEquals(admin3.topics().getPartitionedTopicMetadata(persistentTopicName).partitions, 6);
+        assertEquals(admin1.topics().getList(namespace).size(), 6);
+        assertEquals(admin2.topics().getList(namespace).size(), 6);
+        assertEquals(admin3.topics().getList(namespace).size(), 6);
+    }
+
     /**
      * It verifies that broker should not start replicator for partitioned-topic (topic without -partition postfix)
      *

[pulsar] 03/07: [Fixed] throwable exception not thrown. (#10863)

Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 489d24da6f78756b978e466e5799387781504701
Author: Qiang Zhao <74...@users.noreply.github.com>
AuthorDate: Tue Jun 15 10:44:57 2021 +0800

    [Fixed]  throwable exception not thrown. (#10863)
    
    Fixes # throwable exception not thrown.
    
    ### Motivation
    
    I found some throwable exception that not thrown, maybe it's a bug.
    
    (cherry picked from commit e0988f71630df59557356249fb9b5b46e699461c)
---
 .../main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
index f6d69a0..ec9de23 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
@@ -1377,7 +1377,7 @@ public abstract class NamespacesBase extends AdminResource {
                             "[{}] Failed to update backlog configuration"
                                     + " for namespace {}: conflicts with retention quota",
                             clientAppId(), namespaceName);
-                    new RestException(Status.PRECONDITION_FAILED,
+                    throw new RestException(Status.PRECONDITION_FAILED,
                             "Backlog Quota exceeds configured retention quota for namespace."
                                     + " Please increase retention quota and retry");
                 }
@@ -1427,7 +1427,7 @@ public abstract class NamespacesBase extends AdminResource {
                 log.warn("[{}] Failed to update retention configuration"
                                 + " for namespace {}: conflicts with backlog quota",
                         clientAppId(), namespaceName);
-                new RestException(Status.PRECONDITION_FAILED,
+                throw new RestException(Status.PRECONDITION_FAILED,
                         "Retention Quota must exceed configured backlog quota for namespace.");
             }
             policies.retention_policies = retention;

[pulsar] 02/07: When topic does not exist, optimize the prompt message (#10845)

Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit b20da1c64a3135af98b72ca6dd3367ee24806525
Author: feynmanlin <fe...@tencent.com>
AuthorDate: Tue Jun 8 20:02:07 2021 +0800

    When topic does not exist, optimize the prompt message (#10845)
    
    ### Motivation
    If we set enableRetry=true, the client will subscribe to the retry topic.
    If we set AllowAutoTopicCreation=false, the client will receive `Topic dose not exists`, which means that the retry Topic does not exists.
    This makes users very confused. My topic clearly exists, why is it still prompted like this?
    Therefore, the prompt information is improved to facilitate users to troubleshoot problems.
    
    ### Modifications
    add topic name to the returning message
    
    (cherry picked from commit e72a0d8d63baf336ac4fd4bf045e40de365e58eb)
---
 .../org/apache/pulsar/broker/service/ServerCnx.java |  3 ++-
 .../client/api/SimpleProducerConsumerTest.java      | 21 +++++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 4445f64..c446833 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -986,7 +986,8 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
                                 .thenCompose(optTopic -> {
                                     if (!optTopic.isPresent()) {
                                         return FutureUtil
-                                                .failedFuture(new TopicNotFoundException("Topic does not exist"));
+                                                .failedFuture(new TopicNotFoundException(
+                                                        "Topic " + topicName + " does not exist"));
                                     }
 
                                     Topic topic = optTopic.get();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
index f66f222..5ebb2fb 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
@@ -4051,6 +4051,27 @@ public class SimpleProducerConsumerTest extends ProducerConsumerBase {
         assertEquals(1, res.getFields().size());
     }
 
+    @Test
+    public void testTopicDoesNotExists() throws Exception {
+        cleanup();
+        conf.setAllowAutoTopicCreation(false);
+        setup();
+        String topic = "persistent://my-property/my-ns/none" + UUID.randomUUID();
+        admin.topics().createPartitionedTopic(topic, 3);
+        try {
+            @Cleanup
+            Consumer<byte[]> consumer = pulsarClient.newConsumer()
+                    .enableRetry(true)
+                    .topic(topic).subscriptionName("sub").subscribe();
+            fail("should fail");
+        } catch (Exception e) {
+            String retryTopic = topic + "-sub-RETRY";
+            assertTrue(e.getMessage().contains("Topic " + retryTopic + " does not exist"));
+        } finally {
+            conf.setAllowAutoTopicCreation(true);
+        }
+    }
+
     /**
      * Test validates that consumer of partitioned-topic utilizes threads of all partitioned-consumers and slow-listener
      * of one of the partition doesn't impact listener-processing of other partition.

[pulsar] 06/07: Fix compaction not working for system topic (#10941)

Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit f13d6ad5b22ca3c72e55f7b19425b437ce8025d5
Author: lipenghui <pe...@apache.org>
AuthorDate: Thu Jun 17 15:08:23 2021 +0800

    Fix compaction not working for system topic (#10941)
    
    If a topic only have non-durable subscriptions but not durable subscriptions,
    and the non-durable subscription reach the end of the topic, we will get 0 estimated backlog size
    So that the compaction will never been triggered.
    
    The expected behavior is if we have no durable subscriptions, we should use the total size for triggering
    the compaction.
    
    (cherry picked from commit 797cb128045c1f2bf304c7de7454f1f12f8312e7)
---
 .../broker/service/persistent/PersistentTopic.java |  3 ++-
 .../pulsar/broker/admin/TopicPoliciesTest.java     | 31 +++++++++++++++++++++-
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 644b726..aa73813 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -1374,7 +1374,8 @@ public class PersistentTopic extends AbstractTopic
                 } else {
                     // compaction has never run, so take full backlog size,
                     // or total size if we have no durable subs yet.
-                    backlogEstimate = subscriptions.isEmpty()
+                    backlogEstimate = subscriptions.isEmpty() || subscriptions.values().stream()
+                                .noneMatch(sub -> sub.getCursor().isDurable())
                             ? ledger.getTotalSize()
                             : ledger.getEstimatedBacklogSize();
                 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java
index 276153d..30a48b2 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java
@@ -45,7 +45,6 @@ import org.apache.pulsar.client.api.SubscriptionType;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.BacklogQuota;
 import org.apache.pulsar.common.policies.data.ClusterData;
-import org.apache.pulsar.common.policies.data.ClusterDataImpl;
 import org.apache.pulsar.common.policies.data.DispatchRate;
 import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode;
 import org.apache.pulsar.common.policies.data.InactiveTopicPolicies;
@@ -54,6 +53,7 @@ import org.apache.pulsar.common.policies.data.PublishRate;
 import org.apache.pulsar.common.policies.data.RetentionPolicies;
 import org.apache.pulsar.common.policies.data.SubscribeRate;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
+import org.apache.pulsar.common.policies.data.TopicStats;
 import org.awaitility.Awaitility;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
@@ -91,6 +91,8 @@ public class TopicPoliciesTest extends MockedPulsarServiceBaseTest {
 
     private final String persistenceTopic = "persistent://" + myNamespace + "/test-set-persistence";
 
+    private final String topicPolicyEventsTopic = "persistent://" + myNamespace + "/__change_events";
+
     @BeforeMethod
     @Override
     protected void setup() throws Exception {
@@ -2278,4 +2280,31 @@ public class TopicPoliciesTest extends MockedPulsarServiceBaseTest {
         }
     }
 
+    @Test
+    public void testSystemTopicShouldBeCompacted() throws Exception {
+        BacklogQuota backlogQuota = BacklogQuota.builder()
+                .limitSize(1024)
+                .retentionPolicy(BacklogQuota.RetentionPolicy.consumer_backlog_eviction)
+                .build();
+        log.info("Backlog quota: {} will set to the topic: {}", backlogQuota, testTopic);
+
+        Awaitility.await()
+                .until(() -> pulsar.getTopicPoliciesService().cacheIsInitialized(TopicName.get(testTopic)));
+
+        admin.topics().setBacklogQuota(testTopic, backlogQuota);
+        log.info("Backlog quota set success on topic: {}", testTopic);
+
+        Awaitility.await()
+                .untilAsserted(() -> Assert.assertEquals(admin.topics().getBacklogQuotaMap(testTopic)
+                        .get(BacklogQuota.BacklogQuotaType.destination_storage), backlogQuota));
+
+        pulsar.getBrokerService().checkCompaction();
+
+        Awaitility.await()
+                .untilAsserted(() -> {
+                    TopicStats stats = admin.topics().getStats(topicPolicyEventsTopic);
+                    Assert.assertTrue(stats.getSubscriptions().containsKey("__compaction"));
+                });
+    }
+
 }

[pulsar] 01/07: Load credentials from secrets for Kinesis connectors (#10822)

Posted by pe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

penghui pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 3f000b8f8d7e35c62897e3f78c3fe5769e9b8f5d
Author: Yang Yang <yy...@streamnative.io>
AuthorDate: Tue Jun 8 20:01:30 2021 +0800

    Load credentials from secrets for Kinesis connectors (#10822)
    
    ### Motivation
    
    Support loading `awsCredentialPluginParam` from secrets for kinesis connectors to avoid leaking of sensitive information with best efforts.
    
    ### Modifications
    
    Load kinesis configs with `IOConfigUtils.loadWithSecrets()`.
    
    ### Verifying this change
    
    This change is already covered by existing tests and this PR also adds tests to confirm both loading credentials from plaintext config object and secrets work as expected.
    
    
    (cherry picked from commit cb79032b84228207122bd666b0553cc82e442a4e)
---
 pulsar-io/kinesis/pom.xml                          |  5 ++
 .../pulsar/io/kinesis/BaseKinesisConfig.java       |  1 +
 .../org/apache/pulsar/io/kinesis/KinesisSink.java  | 11 ++--
 .../pulsar/io/kinesis/KinesisSinkConfig.java       | 21 +++----
 .../apache/pulsar/io/kinesis/KinesisSource.java    | 19 +++---
 .../pulsar/io/kinesis/KinesisSourceConfig.java     |  5 --
 .../pulsar/io/kinesis/KinesisSinkConfigTests.java  | 36 +++++++++--
 .../io/kinesis/KinesisSourceConfigTests.java       | 73 +++++++++++++++++-----
 8 files changed, 118 insertions(+), 53 deletions(-)

diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml
index cd1ad28..0271ea9 100644
--- a/pulsar-io/kinesis/pom.xml
+++ b/pulsar-io/kinesis/pom.xml
@@ -31,6 +31,11 @@
   <name>Pulsar IO :: Kinesis</name>
 
   <dependencies>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>pulsar-io-common</artifactId>
+      <version>${project.version}</version>
+    </dependency>
 
     <dependency>
       <groupId>${project.groupId}</groupId>
diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/BaseKinesisConfig.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/BaseKinesisConfig.java
index cad2ac5..92da3e6 100644
--- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/BaseKinesisConfig.java
+++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/BaseKinesisConfig.java
@@ -64,6 +64,7 @@ public abstract class BaseKinesisConfig implements Serializable {
     @FieldDoc(
         required = false,
         defaultValue = "",
+        sensitive = true,
         help = "json-parameters to initialize `AwsCredentialsProviderPlugin`")
     private String awsCredentialPluginParam = "";
 
diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
index ed7c70c..c6fe369 100644
--- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
+++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
@@ -50,6 +50,7 @@ import org.apache.commons.lang3.builder.ToStringStyle;
 import org.apache.pulsar.functions.api.Record;
 import org.apache.pulsar.io.aws.AbstractAwsConnector;
 import org.apache.pulsar.io.aws.AwsCredentialProviderPlugin;
+import org.apache.pulsar.io.common.IOConfigUtils;
 import org.apache.pulsar.io.core.Sink;
 import org.apache.pulsar.io.core.SinkContext;
 import org.apache.pulsar.io.core.annotations.Connector;
@@ -100,7 +101,7 @@ public class KinesisSink extends AbstractAwsConnector implements Sink<byte[]> {
     private static final int maxPartitionedKeyLength = 256;
     private SinkContext sinkContext;
     private ScheduledExecutorService scheduledExecutor;
-    // 
+    //
     private static final int FALSE = 0;
     private static final int TRUE = 1;
     private volatile int previousPublishFailed = FALSE;
@@ -153,12 +154,12 @@ public class KinesisSink extends AbstractAwsConnector implements Sink<byte[]> {
     @Override
     public void open(Map<String, Object> config, SinkContext sinkContext) throws Exception {
         scheduledExecutor = Executors.newSingleThreadScheduledExecutor();
-        kinesisSinkConfig = KinesisSinkConfig.load(config);
+        kinesisSinkConfig = IOConfigUtils.loadWithSecrets(config, KinesisSinkConfig.class, sinkContext);
         this.sinkContext = sinkContext;
 
         checkArgument(isNotBlank(kinesisSinkConfig.getAwsKinesisStreamName()), "empty kinesis-stream name");
-        checkArgument(isNotBlank(kinesisSinkConfig.getAwsEndpoint()) || 
-                      isNotBlank(kinesisSinkConfig.getAwsRegion()), 
+        checkArgument(isNotBlank(kinesisSinkConfig.getAwsEndpoint()) ||
+                      isNotBlank(kinesisSinkConfig.getAwsRegion()),
                       "Either the aws-end-point or aws-region must be set");
         checkArgument(isNotBlank(kinesisSinkConfig.getAwsCredentialPluginParam()), "empty aws-credential param");
 
@@ -287,4 +288,4 @@ public class KinesisSink extends AbstractAwsConnector implements Sink<byte[]> {
         }
     }
 
-}
\ No newline at end of file
+}
diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java
index fa00550..61b0c76 100644
--- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java
+++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java
@@ -34,7 +34,7 @@ import org.apache.pulsar.io.core.annotations.FieldDoc;
 public class KinesisSinkConfig extends BaseKinesisConfig implements Serializable {
 
     private static final long serialVersionUID = 1L;
-    
+
     @FieldDoc(
         required = true,
         defaultValue = "ONLY_RAW_PAYLOAD",
@@ -57,7 +57,7 @@ public class KinesisSinkConfig extends BaseKinesisConfig implements Serializable
             + "  #   properties and encryptionCtx, and publishes flatbuffer payload into the configured kinesis stream."
     )
     private MessageFormat messageFormat = MessageFormat.ONLY_RAW_PAYLOAD; // default : ONLY_RAW_PAYLOAD
-    
+
     @FieldDoc(
         required = false,
         defaultValue = "false",
@@ -81,11 +81,6 @@ public class KinesisSinkConfig extends BaseKinesisConfig implements Serializable
         return mapper.readValue(new File(yamlFile), KinesisSinkConfig.class);
     }
 
-    public static KinesisSinkConfig load(Map<String, Object> map) throws IOException {
-        ObjectMapper mapper = new ObjectMapper();
-        return mapper.readValue(new ObjectMapper().writeValueAsString(map), KinesisSinkConfig.class);
-    }
-    
     public static enum MessageFormat {
         /**
          * Kinesis sink directly publishes pulsar-payload as a message into the kinesis-stream
@@ -94,13 +89,13 @@ public class KinesisSinkConfig extends BaseKinesisConfig implements Serializable
         /**
          * Kinesis sink creates a json payload with message-payload, properties and encryptionCtx and publishes json
          * payload to kinesis stream.
-         * 
-         * schema: 
+         *
+         * schema:
          * {"type":"object","properties":{"encryptionCtx":{"type":"object","properties":{"metadata":{"type":"object","additionalProperties":{"type":"string"}},"uncompressedMessageSize":{"type":"integer"},"keysMetadataMap":{"type":"object","additionalProperties":{"type":"object","additionalProperties":{"type":"string"}}},"keysMapBase64":{"type":"object","additionalProperties":{"type":"string"}},"encParamBase64":{"type":"string"},"compressionType":{"type":"string","enum":["NONE","LZ4","ZLI [...]
          * Example:
          * {"payloadBase64":"cGF5bG9hZA==","properties":{"prop1":"value"},"encryptionCtx":{"keysMapBase64":{"key1":"dGVzdDE=","key2":"dGVzdDI="},"keysMetadataMap":{"key1":{"ckms":"cmks-1","version":"v1"},"key2":{"ckms":"cmks-2","version":"v2"}},"metadata":{"ckms":"cmks-1","version":"v1"},"encParamBase64":"cGFyYW0=","algorithm":"algo","compressionType":"LZ4","uncompressedMessageSize":10,"batchSize":10}}
-         * 
-         * 
+         *
+         *
          */
         FULL_MESSAGE_IN_JSON,
         /**
@@ -108,5 +103,5 @@ public class KinesisSinkConfig extends BaseKinesisConfig implements Serializable
          */
         FULL_MESSAGE_IN_FB;
     }
-    
-}
\ No newline at end of file
+
+}
diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSource.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSource.java
index 598f3af..655c2f7 100644
--- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSource.java
+++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSource.java
@@ -29,6 +29,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.io.aws.AbstractAwsConnector;
 import org.apache.pulsar.io.aws.AwsCredentialProviderPlugin;
+import org.apache.pulsar.io.common.IOConfigUtils;
 import org.apache.pulsar.io.core.Source;
 import org.apache.pulsar.io.core.SourceContext;
 import org.apache.pulsar.io.core.annotations.Connector;
@@ -43,7 +44,7 @@ import software.amazon.kinesis.retrieval.RetrievalConfig;
 import software.amazon.kinesis.retrieval.polling.PollingConfig;
 
 /**
- * 
+ *
  * @see ConfigsBuilder
  */
 @Connector(
@@ -72,23 +73,23 @@ public class KinesisSource extends AbstractAwsConnector implements Source<byte[]
 
     @Override
     public void open(Map<String, Object> config, SourceContext sourceContext) throws Exception {
-        this.kinesisSourceConfig = KinesisSourceConfig.load(config);
-        
+        this.kinesisSourceConfig = IOConfigUtils.loadWithSecrets(config, KinesisSourceConfig.class, sourceContext);
+
         checkArgument(isNotBlank(kinesisSourceConfig.getAwsKinesisStreamName()), "empty kinesis-stream name");
-        checkArgument(isNotBlank(kinesisSourceConfig.getAwsEndpoint()) || 
-                      isNotBlank(kinesisSourceConfig.getAwsRegion()), 
+        checkArgument(isNotBlank(kinesisSourceConfig.getAwsEndpoint()) ||
+                      isNotBlank(kinesisSourceConfig.getAwsRegion()),
                      "Either the aws-end-point or aws-region must be set");
         checkArgument(isNotBlank(kinesisSourceConfig.getAwsCredentialPluginParam()), "empty aws-credential param");
-        
+
         if (kinesisSourceConfig.getInitialPositionInStream() == InitialPositionInStream.AT_TIMESTAMP) {
             checkArgument((kinesisSourceConfig.getStartAtTime() != null),"Timestamp must be specified");
         }
-        
+
         queue = new LinkedBlockingQueue<KinesisRecord> (kinesisSourceConfig.getReceiveQueueSize());
         workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID();
-        
+
         AwsCredentialProviderPlugin credentialsProvider = createCredentialProvider(
-                kinesisSourceConfig.getAwsCredentialPluginName(), 
+                kinesisSourceConfig.getAwsCredentialPluginName(),
                 kinesisSourceConfig.getAwsCredentialPluginParam());
 
         KinesisAsyncClient kClient = kinesisSourceConfig.buildKinesisAsyncClient(credentialsProvider);
diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSourceConfig.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSourceConfig.java
index d75b521..cd9e313 100644
--- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSourceConfig.java
+++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSourceConfig.java
@@ -146,11 +146,6 @@ public class KinesisSourceConfig extends BaseKinesisConfig implements Serializab
         return mapper.readValue(new File(yamlFile), KinesisSourceConfig.class);
     }
 
-    public static KinesisSourceConfig load(Map<String, Object> map) throws IOException {
-        ObjectMapper mapper = new ObjectMapper();
-        return mapper.readValue(new ObjectMapper().writeValueAsString(map), KinesisSourceConfig.class);
-    }
-
     public KinesisAsyncClient buildKinesisAsyncClient(AwsCredentialProviderPlugin credPlugin) {
         KinesisAsyncClientBuilder builder = KinesisAsyncClient.builder();
 
diff --git a/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTests.java b/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTests.java
index 642bd9f..dd799e0 100644
--- a/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTests.java
+++ b/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTests.java
@@ -26,7 +26,10 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.pulsar.io.common.IOConfigUtils;
+import org.apache.pulsar.io.core.SinkContext;
 import org.apache.pulsar.io.kinesis.KinesisSinkConfig.MessageFormat;
+import org.mockito.Mockito;
 import org.testng.annotations.Test;
 
 public class KinesisSinkConfigTests {
@@ -35,7 +38,7 @@ public class KinesisSinkConfigTests {
     public final void loadFromYamlFileTest() throws IOException {
         File yamlFile = getFile("sinkConfig.yaml");
         KinesisSinkConfig config = KinesisSinkConfig.load(yamlFile.getAbsolutePath());
-        
+
         assertNotNull(config);
         assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws");
         assertEquals(config.getAwsRegion(), "us-east-1");
@@ -45,7 +48,7 @@ public class KinesisSinkConfigTests {
         assertEquals(config.getMessageFormat(), MessageFormat.ONLY_RAW_PAYLOAD);
         assertEquals(true, config.isRetainOrdering());
     }
-    
+
     @Test
     public final void loadFromMapTest() throws IOException {
         Map<String, Object> map = new HashMap<String, Object> ();
@@ -53,9 +56,30 @@ public class KinesisSinkConfigTests {
         map.put("awsRegion", "us-east-1");
         map.put("awsKinesisStreamName", "my-stream");
         map.put("awsCredentialPluginParam", "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}");
-        
-        KinesisSinkConfig config = KinesisSinkConfig.load(map);
-        
+
+        SinkContext sinkContext = Mockito.mock(SinkContext.class);
+        KinesisSinkConfig config = IOConfigUtils.loadWithSecrets(map, KinesisSinkConfig.class, sinkContext);
+
+        assertNotNull(config);
+        assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws");
+        assertEquals(config.getAwsRegion(), "us-east-1");
+        assertEquals(config.getAwsKinesisStreamName(), "my-stream");
+        assertEquals(config.getAwsCredentialPluginParam(),
+                "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}");
+    }
+
+    @Test
+    public final void loadFromMapCredentialFromSecretTest() throws IOException {
+        Map<String, Object> map = new HashMap<String, Object> ();
+        map.put("awsEndpoint", "https://some.endpoint.aws");
+        map.put("awsRegion", "us-east-1");
+        map.put("awsKinesisStreamName", "my-stream");
+
+        SinkContext sinkContext = Mockito.mock(SinkContext.class);
+        Mockito.when(sinkContext.getSecret("awsCredentialPluginParam"))
+                .thenReturn("{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}");
+        KinesisSinkConfig config = IOConfigUtils.loadWithSecrets(map, KinesisSinkConfig.class, sinkContext);
+
         assertNotNull(config);
         assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws");
         assertEquals(config.getAwsRegion(), "us-east-1");
@@ -63,7 +87,7 @@ public class KinesisSinkConfigTests {
         assertEquals(config.getAwsCredentialPluginParam(),
                 "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}");
     }
-    
+
     private File getFile(String name) {
         ClassLoader classLoader = getClass().getClassLoader();
         return new File(classLoader.getResource(name).getFile());
diff --git a/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSourceConfigTests.java b/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSourceConfigTests.java
index b79e126..4bca8e7 100644
--- a/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSourceConfigTests.java
+++ b/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSourceConfigTests.java
@@ -30,6 +30,9 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.pulsar.io.common.IOConfigUtils;
+import org.apache.pulsar.io.core.SourceContext;
+import org.mockito.Mockito;
 import org.testng.annotations.Test;
 import software.amazon.kinesis.common.InitialPositionInStream;
 
@@ -37,7 +40,7 @@ import software.amazon.kinesis.common.InitialPositionInStream;
 public class KinesisSourceConfigTests {
 
     private static final Date DAY;
-    
+
     static {
         Calendar then = Calendar.getInstance();
         then.set(Calendar.YEAR, 2019);
@@ -67,14 +70,14 @@ public class KinesisSourceConfigTests {
         assertEquals(config.getNumRetries(), 3);
         assertEquals(config.getReceiveQueueSize(), 2000);
         assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON);
-        
+
         Calendar cal = Calendar.getInstance();
         cal.setTime(config.getStartAtTime());
         ZonedDateTime actual = ZonedDateTime.ofInstant(cal.toInstant(), ZoneOffset.UTC);
         ZonedDateTime expected = ZonedDateTime.ofInstant(DAY.toInstant(), ZoneOffset.UTC);
         assertEquals(actual, expected);
     }
-    
+
     @Test
     public final void loadFromMapTest() throws IOException {
         Map<String, Object> map = new HashMap<String, Object> ();
@@ -89,9 +92,10 @@ public class KinesisSourceConfigTests {
         map.put("applicationName", "My test application");
         map.put("initialPositionInStream", InitialPositionInStream.TRIM_HORIZON);
         map.put("startAtTime", DAY);
-        
-        KinesisSourceConfig config = KinesisSourceConfig.load(map);
-        
+
+        SourceContext sourceContext = Mockito.mock(SourceContext.class);
+        KinesisSourceConfig config = IOConfigUtils.loadWithSecrets(map, KinesisSourceConfig.class, sourceContext);
+
         assertNotNull(config);
         assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws");
         assertEquals(config.getAwsRegion(), "us-east-1");
@@ -104,41 +108,80 @@ public class KinesisSourceConfigTests {
         assertEquals(config.getNumRetries(), 3);
         assertEquals(config.getReceiveQueueSize(), 2000);
         assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON);
-        
+
         Calendar cal = Calendar.getInstance();
         cal.setTime(config.getStartAtTime());
         ZonedDateTime actual = ZonedDateTime.ofInstant(cal.toInstant(), ZoneOffset.UTC);
         ZonedDateTime expected = ZonedDateTime.ofInstant(DAY.toInstant(), ZoneOffset.UTC);
         assertEquals(actual, expected);
     }
-    
-    @Test(expectedExceptions = IllegalArgumentException.class, 
+
+    @Test
+    public final void loadFromMapCredentialFromSecretTest() throws IOException {
+        Map<String, Object> map = new HashMap<String, Object> ();
+        map.put("awsEndpoint", "https://some.endpoint.aws");
+        map.put("awsRegion", "us-east-1");
+        map.put("awsKinesisStreamName", "my-stream");
+        map.put("checkpointInterval", "30000");
+        map.put("backoffTime", "4000");
+        map.put("numRetries", "3");
+        map.put("receiveQueueSize", 2000);
+        map.put("applicationName", "My test application");
+        map.put("initialPositionInStream", InitialPositionInStream.TRIM_HORIZON);
+        map.put("startAtTime", DAY);
+
+        SourceContext sourceContext = Mockito.mock(SourceContext.class);
+        Mockito.when(sourceContext.getSecret("awsCredentialPluginParam"))
+                .thenReturn("{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}");
+        KinesisSourceConfig config = IOConfigUtils.loadWithSecrets(map, KinesisSourceConfig.class, sourceContext);
+
+        assertNotNull(config);
+        assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws");
+        assertEquals(config.getAwsRegion(), "us-east-1");
+        assertEquals(config.getAwsKinesisStreamName(), "my-stream");
+        assertEquals(config.getAwsCredentialPluginParam(),
+                "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}");
+        assertEquals(config.getApplicationName(), "My test application");
+        assertEquals(config.getCheckpointInterval(), 30000);
+        assertEquals(config.getBackoffTime(), 4000);
+        assertEquals(config.getNumRetries(), 3);
+        assertEquals(config.getReceiveQueueSize(), 2000);
+        assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON);
+
+        Calendar cal = Calendar.getInstance();
+        cal.setTime(config.getStartAtTime());
+        ZonedDateTime actual = ZonedDateTime.ofInstant(cal.toInstant(), ZoneOffset.UTC);
+        ZonedDateTime expected = ZonedDateTime.ofInstant(DAY.toInstant(), ZoneOffset.UTC);
+        assertEquals(actual, expected);
+    }
+
+    @Test(expectedExceptions = IllegalArgumentException.class,
             expectedExceptionsMessageRegExp = "empty aws-credential param")
     public final void missingCredentialsTest() throws Exception {
         Map<String, Object> map = new HashMap<String, Object> ();
         map.put("awsEndpoint", "https://some.endpoint.aws");
         map.put("awsRegion", "us-east-1");
         map.put("awsKinesisStreamName", "my-stream");
-     
+
         KinesisSource source = new KinesisSource();
         source.open(map, null);
     }
-    
-    @Test(expectedExceptions = IllegalArgumentException.class, 
+
+    @Test(expectedExceptions = IllegalArgumentException.class,
             expectedExceptionsMessageRegExp = "Timestamp must be specified")
     public final void missingStartTimeTest() throws Exception {
         Map<String, Object> map = new HashMap<String, Object> ();
         map.put("awsEndpoint", "https://some.endpoint.aws");
         map.put("awsRegion", "us-east-1");
         map.put("awsKinesisStreamName", "my-stream");
-        map.put("awsCredentialPluginParam", 
+        map.put("awsCredentialPluginParam",
                 "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}");
         map.put("initialPositionInStream", InitialPositionInStream.AT_TIMESTAMP);
-     
+
         KinesisSource source = new KinesisSource();
         source.open(map, null);
     }
-    
+
     private File getFile(String name) {
         ClassLoader classLoader = getClass().getClassLoader();
         return new File(classLoader.getResource(name).getFile());