You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2025/03/26 15:50:10 UTC

(pulsar) branch branch-3.3 updated: [improve][io] Enhance Kafka connector logging with focused bootstrap server information (#24128)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
     new 5b1a6430e67 [improve][io] Enhance Kafka connector logging with focused bootstrap server information (#24128)
5b1a6430e67 is described below

commit 5b1a6430e67ad8ae99ee3b45aaeb2b31ffe14296
Author: Rui Fu <fr...@users.noreply.github.com>
AuthorDate: Wed Mar 26 22:48:49 2025 +0800

    [improve][io] Enhance Kafka connector logging with focused bootstrap server information (#24128)
    
    (cherry picked from commit d5f126f0c1c41c00ef5df762ea5b20985587400b)
---
 .../pulsar/io/kafka/connect/KafkaConnectSink.java  | 11 +++---
 .../apache/pulsar/io/kafka/KafkaAbstractSink.java  |  6 ++--
 .../org/apache/pulsar/io/kafka/KafkaBytesSink.java | 10 +++---
 .../apache/pulsar/io/kafka/KafkaBytesSource.java   | 41 +++++++++++-----------
 4 files changed, 35 insertions(+), 33 deletions(-)

diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
index ff0bfd391e8..69f1959537b 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
@@ -44,6 +44,7 @@ import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.producer.ProducerConfig;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.record.TimestampType;
 import org.apache.kafka.connect.connector.Task;
@@ -160,7 +161,7 @@ public class KafkaConnectSink implements Sink<GenericObject> {
         kafkaSinkConfig = PulsarKafkaConnectSinkConfig.load(config);
         Objects.requireNonNull(kafkaSinkConfig.getTopic(), "Kafka topic is not set");
         Preconditions.checkArgument(ctx.getSubscriptionType() == SubscriptionType.Failover
-                || ctx.getSubscriptionType() == SubscriptionType.Exclusive,
+                        || ctx.getSubscriptionType() == SubscriptionType.Exclusive,
                 "Source must run with Exclusive or Failover subscription type");
         topicName = kafkaSinkConfig.getTopic();
         unwrapKeyValueIfAvailable = kafkaSinkConfig.isUnwrapKeyValueIfAvailable();
@@ -220,7 +221,7 @@ public class KafkaConnectSink implements Sink<GenericObject> {
         scheduledExecutor.scheduleWithFixedDelay(() ->
                 this.flushIfNeeded(true), lingerMs, lingerMs, TimeUnit.MILLISECONDS);
 
-        log.info("Kafka sink started : {}.", props);
+        log.info("Kafka sink started : {}.", props.getOrDefault(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, ""));
     }
 
     private void flushIfNeeded(boolean force) {
@@ -282,15 +283,15 @@ public class KafkaConnectSink implements Sink<GenericObject> {
 
     @VisibleForTesting
     protected void ackUntil(Record<GenericObject> lastNotFlushed,
-                          Map<TopicPartition, OffsetAndMetadata> committedOffsets,
-                          java.util.function.Consumer<Record<GenericObject>> cb) {
+                            Map<TopicPartition, OffsetAndMetadata> committedOffsets,
+                            java.util.function.Consumer<Record<GenericObject>> cb) {
         // lastNotFlushed is needed in case of default preCommit() implementation
         // which calls flush() and returns currentOffsets passed to it.
         // We don't want to ack messages added to pendingFlushQueue after the preCommit/flush call
 
         // to avoid creation of new TopicPartition for each record in pendingFlushQueue
         Map<String, Map<Integer, Long>> topicOffsets = new HashMap<>();
-        for (Map.Entry<TopicPartition, OffsetAndMetadata> e: committedOffsets.entrySet()) {
+        for (Map.Entry<TopicPartition, OffsetAndMetadata> e : committedOffsets.entrySet()) {
             TopicPartition tp = e.getKey();
             if (!topicOffsets.containsKey(tp.topic())) {
                 topicOffsets.put(tp.topic(), new HashMap<>());
diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSink.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSink.java
index 2bedba928b7..232fd7fff7c 100644
--- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSink.java
+++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSink.java
@@ -81,11 +81,11 @@ public abstract class KafkaAbstractSink<K, V> implements Sink<byte[]> {
         kafkaSinkConfig = KafkaSinkConfig.load(config, sinkContext);
         if (kafkaSinkConfig.getBatchSize() <= 0) {
             throw new IllegalArgumentException("Invalid Kafka Producer batchSize : "
-                + kafkaSinkConfig.getBatchSize());
+                    + kafkaSinkConfig.getBatchSize());
         }
         if (kafkaSinkConfig.getMaxRequestSize() <= 0) {
             throw new IllegalArgumentException("Invalid Kafka Producer maxRequestSize : "
-                + kafkaSinkConfig.getMaxRequestSize());
+                    + kafkaSinkConfig.getMaxRequestSize());
         }
         if (kafkaSinkConfig.getProducerConfigProperties() != null) {
             props.putAll(kafkaSinkConfig.getProducerConfigProperties());
@@ -122,7 +122,7 @@ public abstract class KafkaAbstractSink<K, V> implements Sink<byte[]> {
 
         producer = new KafkaProducer<>(beforeCreateProducer(props));
 
-        log.info("Kafka sink started : {}.", props);
+        log.info("Kafka sink started : {}.", props.getOrDefault(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, ""));
     }
 
     public abstract KeyValue<K, V> extractKeyValue(Record<byte[]> message);
diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSink.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSink.java
index 0ed23856995..e65dd76064d 100644
--- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSink.java
+++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSink.java
@@ -33,10 +33,10 @@ import org.apache.pulsar.io.core.annotations.IOType;
  * apply schema into it.
  */
 @Connector(
-    name = "kafka",
-    type = IOType.SINK,
-    help = "The KafkaBytesSink is used for moving messages from Pulsar to Kafka.",
-    configClass = KafkaSinkConfig.class
+        name = "kafka",
+        type = IOType.SINK,
+        help = "The KafkaBytesSink is used for moving messages from Pulsar to Kafka.",
+        configClass = KafkaSinkConfig.class
 )
 @Slf4j
 public class KafkaBytesSink extends KafkaAbstractSink<String, byte[]> {
@@ -45,7 +45,7 @@ public class KafkaBytesSink extends KafkaAbstractSink<String, byte[]> {
     protected Properties beforeCreateProducer(Properties props) {
         props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
         props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
-        log.info("Created kafka producer config : {}", props);
+        log.info("Created kafka producer on : {}", props.getOrDefault(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, ""));
         return props;
     }
 
diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSource.java
index 51408f04519..39bcf14ccd2 100644
--- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSource.java
+++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSource.java
@@ -49,22 +49,22 @@ import org.apache.pulsar.io.core.annotations.Connector;
 import org.apache.pulsar.io.core.annotations.IOType;
 
 /**
- *  Kafka Source that transfers the data from Kafka to Pulsar and sets the Schema type properly.
- *  We use the key and the value deserializer in order to decide the type of Schema to be set on the topic on Pulsar.
- *  In case of KafkaAvroDeserializer we use the Schema Registry to download the schema and apply it to the topic.
- *  Please refer to {@link #getSchemaFromDeserializerAndAdaptConfiguration(String, Properties, boolean)} for the list
- *  of supported Deserializers.
- *  If you set StringDeserializer for the key then we use the raw key as key for the Pulsar message.
- *  If you set another Deserializer for the key we use the KeyValue schema type in Pulsar with the SEPARATED encoding.
- *  This way the Key is stored in the Pulsar key, encoded as base64 string and with a Schema, the Value of the message
- *  is stored in the Pulsar value with a Schema.
- *  This way there is a one-to-one mapping between Kafka key/value pair and the Pulsar data model.
+ * Kafka Source that transfers the data from Kafka to Pulsar and sets the Schema type properly.
+ * We use the key and the value deserializer in order to decide the type of Schema to be set on the topic on Pulsar.
+ * In case of KafkaAvroDeserializer we use the Schema Registry to download the schema and apply it to the topic.
+ * Please refer to {@link #getSchemaFromDeserializerAndAdaptConfiguration(String, Properties, boolean)} for the list
+ * of supported Deserializers.
+ * If you set StringDeserializer for the key then we use the raw key as key for the Pulsar message.
+ * If you set another Deserializer for the key we use the KeyValue schema type in Pulsar with the SEPARATED encoding.
+ * This way the Key is stored in the Pulsar key, encoded as base64 string and with a Schema, the Value of the message
+ * is stored in the Pulsar value with a Schema.
+ * This way there is a one-to-one mapping between Kafka key/value pair and the Pulsar data model.
  */
 @Connector(
-    name = "kafka",
-    type = IOType.SOURCE,
-    help = "Transfer data from Kafka to Pulsar.",
-    configClass = KafkaSourceConfig.class
+        name = "kafka",
+        type = IOType.SOURCE,
+        help = "Transfer data from Kafka to Pulsar.",
+        configClass = KafkaSourceConfig.class
 )
 @Slf4j
 public class KafkaBytesSource extends KafkaAbstractSource<ByteBuffer> {
@@ -78,7 +78,7 @@ public class KafkaBytesSource extends KafkaAbstractSource<ByteBuffer> {
     protected Properties beforeCreateConsumer(Properties props) {
         props.putIfAbsent(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
         props.putIfAbsent(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
-        log.info("Created kafka consumer config : {}", props);
+        log.info("Created kafka consumer on : {}", props.getOrDefault(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, ""));
 
         keySchema = getSchemaFromDeserializerAndAdaptConfiguration(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
                 props, true);
@@ -86,7 +86,7 @@ public class KafkaBytesSource extends KafkaAbstractSource<ByteBuffer> {
                 props, false);
 
         boolean needsSchemaCache = keySchema == DeferredSchemaPlaceholder.INSTANCE
-                                    || valueSchema == DeferredSchemaPlaceholder.INSTANCE;
+                || valueSchema == DeferredSchemaPlaceholder.INSTANCE;
 
         if (needsSchemaCache) {
             initSchemaCache(props);
@@ -174,8 +174,8 @@ public class KafkaBytesSource extends KafkaAbstractSource<ByteBuffer> {
 
         Schema<?> result;
         if (ByteArrayDeserializer.class.getName().equals(kafkaDeserializerClass)
-            || ByteBufferDeserializer.class.getName().equals(kafkaDeserializerClass)
-            || BytesDeserializer.class.getName().equals(kafkaDeserializerClass)) {
+                || ByteBufferDeserializer.class.getName().equals(kafkaDeserializerClass)
+                || BytesDeserializer.class.getName().equals(kafkaDeserializerClass)) {
             result = Schema.BYTEBUFFER;
         } else if (StringDeserializer.class.getName().equals(kafkaDeserializerClass)) {
             if (isKey) {
@@ -193,7 +193,7 @@ public class KafkaBytesSource extends KafkaAbstractSource<ByteBuffer> {
             result = Schema.INT64;
         } else if (ShortDeserializer.class.getName().equals(kafkaDeserializerClass)) {
             result = Schema.INT16;
-        } else if (KafkaAvroDeserializer.class.getName().equals(kafkaDeserializerClass)){
+        } else if (KafkaAvroDeserializer.class.getName().equals(kafkaDeserializerClass)) {
             // in this case we have to inject our custom deserializer
             // that extracts Avro schema information
             props.put(key, ExtractKafkaAvroSchemaDeserializer.class.getName());
@@ -238,7 +238,7 @@ public class KafkaBytesSource extends KafkaAbstractSource<ByteBuffer> {
         }
     }
 
-     static final class DeferredSchemaPlaceholder extends ByteBufferSchemaWrapper {
+    static final class DeferredSchemaPlaceholder extends ByteBufferSchemaWrapper {
         DeferredSchemaPlaceholder() {
             super(SchemaInfoImpl
                     .builder()
@@ -247,6 +247,7 @@ public class KafkaBytesSource extends KafkaAbstractSource<ByteBuffer> {
                     .schema(new byte[0])
                     .build());
         }
+
         static final DeferredSchemaPlaceholder INSTANCE = new DeferredSchemaPlaceholder();
     }