You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@kafka.apache.org by "Juhi Bhadviya (Jira)" <ji...@apache.org> on 2022/03/15 06:06:00 UTC
[jira] [Updated] (KAFKA-13738) Mirror Maker2 properties are not being picked up from properties file
[ https://issues.apache.org/jira/browse/KAFKA-13738?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Juhi Bhadviya updated KAFKA-13738:
----------------------------------
Description:
{quote}We have created a custom docker image using `confluentinc/cp-kafka:7.0.1` :
FROM confluentinc/cp-kafka:7.0.1 as Builder
FROM openjdk11:alpine-jre
RUN apk update && apk upgrade && apk --no-cache add bash
COPY --from=Builder /usr/share/java/kafka/kafka_*-ccs.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/scala*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/metrics-core*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/jopt-simple*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/argparse*[.]jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/slf4j-api*[.]jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/slf4j-log4j12*[.]jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/confluent-log4j*[.]jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/connect*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/reflections*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/javassist*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/maven*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/commons*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/jackson*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/cp-base-new/kafka-clients*-ccs.jar /usr/share/java/kafka/
COPY --from=Builder /etc/kafka/connect-log4j.properties /etc/kafka/connect-log4j.properties
ENV CLASSPATH='/usr/share/java/kafka/{*}:/usr/share/java/kafka/{*}'
ENV KAFKA_LOG4J_OPTS='-Dlog4j.configuration=[file:/etc/kafka/connect-log4j.properties|file://etc/kafka/connect-log4j.properties]'
COPY --from=0 /bin/connect-mirror-maker /bin/kafka-run-class /bin/
RUN cp /opt/java/openjdk/lib/security/cacerts /tmp/kafka.client.truststore.jks
RUN mkdir -p /tmp/config
CMD ["sh", "-c", "/bin/connect-mirror-maker /tmp/config/mm2.properties"]
```
{quote}
The mm2.properties file is as below:
clusters = source, target
source.bootstrap.servers = ""
target.bootstrap.servers = ""
#security.protocol=SSL
source.ssl.truststore.location=/tmp/kafka.client.truststore.jks
source.security.protocol=SASL_SSL
source.sasl.mechanism=SCRAM-SHA-512
source.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password="";
target.ssl.truststore.location=/tmp/kafka.client.truststore.jks
target.security.protocol=SASL_SSL
target.sasl.mechanism=SCRAM-SHA-512
target.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password="";
source->target.enabled = true
target->source.enabled = false
topics = .*
groups = .*
tasks.max = 1
replication.factor = 3
refresh.topics.enabled = true
sync.topic.configs.enabled = true
refresh.topics.interval.seconds = 5
topics.blacklist = random
groups.blacklist = random
Enable heartbeats and checkpoints.
source->target.emit.heartbeats.enabled = true
source->target.emit.checkpoints.enabled = true
source->target.sync.group.offsets.enabled = true
However in MM2 logs, the properties are not set as per the property file provided, instead uses default values:
```
sync.group.offsets.enabled = false
```
Due this consumer offset sync is not working.
was:
We have created a custom docker image using `confluentinc/cp-kafka:7.0.1` :
```
FROM confluentinc/cp-kafka:7.0.1 as Builder
FROM openjdk11:alpine-jre
RUN apk update && apk upgrade && apk --no-cache add bash
COPY --from=Builder /usr/share/java/kafka/kafka_*-ccs.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/scala-*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/metrics-core-*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/jopt-simple-*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/argparse*[.]jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/slf4j-api-*[.]jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/slf4j-log4j12-*[.]jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/confluent-log4j-*[.]jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/connect-*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/reflections-*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/javassist-*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/maven-*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/commons-*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/kafka/jackson-*.jar /usr/share/java/kafka/
COPY --from=Builder /usr/share/java/cp-base-new/kafka-clients-*-ccs.jar /usr/share/java/kafka/
COPY --from=Builder /etc/kafka/connect-log4j.properties /etc/kafka/connect-log4j.properties
ENV CLASSPATH='/usr/share/java/kafka/*:/usr/share/java/kafka/*'
ENV KAFKA_LOG4J_OPTS='-Dlog4j.configuration=file:/etc/kafka/connect-log4j.properties'
COPY --from=0 /bin/connect-mirror-maker /bin/kafka-run-class /bin/
RUN cp /opt/java/openjdk/lib/security/cacerts /tmp/kafka.client.truststore.jks
RUN mkdir -p /tmp/config
CMD ["sh", "-c", "/bin/connect-mirror-maker /tmp/config/mm2.properties"]
```
The mm2.properties file is as below:
```
clusters = source, target
source.bootstrap.servers = ""
target.bootstrap.servers = ""
#security.protocol=SSL
source.ssl.truststore.location=/tmp/kafka.client.truststore.jks
source.security.protocol=SASL_SSL
source.sasl.mechanism=SCRAM-SHA-512
source.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password="";
target.ssl.truststore.location=/tmp/kafka.client.truststore.jks
target.security.protocol=SASL_SSL
target.sasl.mechanism=SCRAM-SHA-512
target.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password="";
source->target.enabled = true
target->source.enabled = false
topics = .*
groups = .*
tasks.max = 1
replication.factor = 3
refresh.topics.enabled = true
sync.topic.configs.enabled = true
refresh.topics.interval.seconds = 5
topics.blacklist = random
groups.blacklist = random
# Enable heartbeats and checkpoints.
source->target.emit.heartbeats.enabled = true
source->target.emit.checkpoints.enabled = true
source->target.sync.group.offsets.enabled = true
```
However in MM2 logs, the properties are not set as per the property file provided, instead uses default values:
```
sync.group.offsets.enabled = false
```
Due this consumer offset sync is not working.
> Mirror Maker2 properties are not being picked up from properties file
> ---------------------------------------------------------------------
>
> Key: KAFKA-13738
> URL: https://issues.apache.org/jira/browse/KAFKA-13738
> Project: Kafka
> Issue Type: Bug
> Components: mirrormaker
> Affects Versions: 2.8.0
> Reporter: Juhi Bhadviya
> Priority: Major
> Labels: mirror-maker, mirrormaker
>
> {quote}We have created a custom docker image using `confluentinc/cp-kafka:7.0.1` :
> FROM confluentinc/cp-kafka:7.0.1 as Builder
> FROM openjdk11:alpine-jre
> RUN apk update && apk upgrade && apk --no-cache add bash
> COPY --from=Builder /usr/share/java/kafka/kafka_*-ccs.jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/kafka/scala*.jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/kafka/metrics-core*.jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/kafka/jopt-simple*.jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/kafka/argparse*[.]jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/kafka/slf4j-api*[.]jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/kafka/slf4j-log4j12*[.]jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/kafka/confluent-log4j*[.]jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/kafka/connect*.jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/kafka/reflections*.jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/kafka/javassist*.jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/kafka/maven*.jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/kafka/commons*.jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/kafka/jackson*.jar /usr/share/java/kafka/
> COPY --from=Builder /usr/share/java/cp-base-new/kafka-clients*-ccs.jar /usr/share/java/kafka/
> COPY --from=Builder /etc/kafka/connect-log4j.properties /etc/kafka/connect-log4j.properties
> ENV CLASSPATH='/usr/share/java/kafka/{*}:/usr/share/java/kafka/{*}'
> ENV KAFKA_LOG4J_OPTS='-Dlog4j.configuration=[file:/etc/kafka/connect-log4j.properties|file://etc/kafka/connect-log4j.properties]'
> COPY --from=0 /bin/connect-mirror-maker /bin/kafka-run-class /bin/
> RUN cp /opt/java/openjdk/lib/security/cacerts /tmp/kafka.client.truststore.jks
> RUN mkdir -p /tmp/config
> CMD ["sh", "-c", "/bin/connect-mirror-maker /tmp/config/mm2.properties"]
> ```
> {quote}
> The mm2.properties file is as below:
> clusters = source, target
> source.bootstrap.servers = ""
> target.bootstrap.servers = ""
> #security.protocol=SSL
> source.ssl.truststore.location=/tmp/kafka.client.truststore.jks
> source.security.protocol=SASL_SSL
> source.sasl.mechanism=SCRAM-SHA-512
> source.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password="";
> target.ssl.truststore.location=/tmp/kafka.client.truststore.jks
> target.security.protocol=SASL_SSL
> target.sasl.mechanism=SCRAM-SHA-512
> target.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password="";
> source->target.enabled = true
> target->source.enabled = false
> topics = .*
> groups = .*
> tasks.max = 1
> replication.factor = 3
> refresh.topics.enabled = true
> sync.topic.configs.enabled = true
> refresh.topics.interval.seconds = 5
> topics.blacklist = random
> groups.blacklist = random
> Enable heartbeats and checkpoints.
> source->target.emit.heartbeats.enabled = true
> source->target.emit.checkpoints.enabled = true
> source->target.sync.group.offsets.enabled = true
>
> However in MM2 logs, the properties are not set as per the property file provided, instead uses default values:
> ```
> sync.group.offsets.enabled = false
> ```
> Due this consumer offset sync is not working.
--
This message was sent by Atlassian Jira
(v8.20.1#820001)