[hudi] branch master updated: [HUDI-1751] DeltaStreamer print many unnecessary warn log (#2754)

[na...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

nagarwal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new dadd081  [HUDI-1751] DeltaStreamer print many unnecessary warn log (#2754)
dadd081 is described below

commit dadd081d454a25cb57668ca89a8338a9cb9493ad
Author: li36909 <li...@gmail.com>
AuthorDate: Wed Apr 7 15:47:03 2021 +0800

    [HUDI-1751] DeltaStreamer print many unnecessary warn log (#2754)
---
 .../org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java
index 789daf1..bc29be9 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java
@@ -173,9 +173,13 @@ public class KafkaOffsetGen {
     this.props = props;
 
     kafkaParams = new HashMap<>();
-    for (Object prop : props.keySet()) {
+    props.keySet().stream().filter(prop -> {
+      // In order to prevent printing unnecessary warn logs, here filter out the hoodie
+      // configuration items before passing to kafkaParams
+      return !prop.toString().startsWith("hoodie.");
+    }).forEach(prop -> {
       kafkaParams.put(prop.toString(), props.get(prop.toString()));
-    }
+    });
     DataSourceUtils.checkRequiredProperties(props, Collections.singletonList(Config.KAFKA_TOPIC_NAME));
     topicName = props.getString(Config.KAFKA_TOPIC_NAME);
     String kafkaAutoResetOffsetsStr = props.getString(Config.KAFKA_AUTO_RESET_OFFSETS, Config.DEFAULT_KAFKA_AUTO_RESET_OFFSETS.name().toLowerCase());