You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@kafka.apache.org by Rafael Telles <ra...@simbioseventures.com> on 2017/03/29 13:53:27 UTC
Kafka running but not listening to port 9092
Hello there!
I have two clusters of Kafka brokers, one of them (with 15 brokers + 3
Zookeeper servers) became sick (a lot of under-replicated partitions,
throwing a lot of NotEnoughReplicasExceptions). I logged in some of the
brokers that other couldn't connect to, and I found out that they were all
running their Kafka process, but they were not listening to the default TCP
port (9092) as expected:
root@dc3-kafka-02:/home/kafka/kafka_2.11-0.10.1.1# ps aux | grep kafka
root 14055 21.6 33.6 23001236 5513176 ? Sl Mar23 1866:20
/usr/lib/jvm/java-8-oracle/bin/java -Xms2G -Xmx6G -server -XX:+UseG1GC
-XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35
-XX:+DisableExplicitGC -Djava.awt.headless=true
-Xloggc:/home/kafka/kafka_2.11-0.10.1.1/bin/../logs/kafkaServer-gc.log
-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps
-XX:+PrintGCTimeStamps -Dcom.sun.management.jmxremote
-Dcom.sun.management.jmxremote.authenticate=false
-Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.port=17264
-Dkafka.logs.dir=/home/kafka/kafka_2.11-0.10.1.1/bin/../logs
-Dlog4j.configuration=
file:/home/kafka/kafka_2.11-0.10.1.1/bin/../config/log4j.properties -cp
:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/aopalliance-repackaged-2.4.0-b34.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/argparse4j-0.5.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/connect-api-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/connect-file-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/connect-json-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/connect-runtime-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/guava-18.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/hk2-api-2.4.0-b34.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/hk2-locator-2.4.0-b34.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/hk2-utils-2.4.0-b34.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jackson-annotations-2.6.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jackson-core-2.6.3.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jackson-databind-2.6.3.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jackson-jaxrs-base-2.6.3.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jackson-jaxrs-json-provider-2.6.3.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jackson-module-jaxb-annotations-2.6.3.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/javassist-3.18.2-GA.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/javax.annotation-api-1.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/javax.inject-1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/javax.inject-2.4.0-b34.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/javax.servlet-api-3.1.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/javax.ws.rs-api-2.0.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-client-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-common-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-container-servlet-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-container-servlet-core-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-guava-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-media-jaxb-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-server-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-continuation-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-http-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-io-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-security-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-server-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-servlet-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-servlets-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-util-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jopt-simple-4.9.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka_2.11-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka_2.11-0.10.1.1-sources.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka_2.11-0.10.1.1-test-sources.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka-clients-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka-log4j-appender-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka-streams-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka-streams-examples-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka-tools-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/log4j-1.2.17.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/lz4-1.3.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/metrics-core-2.2.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/osgi-resource-locator-1.0.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/raven-7.8.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/raven-log4j-7.8.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/reflections-0.9.10.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/rocksdbjni-4.9.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/scala-library-2.11.8.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/scala-parser-combinators_2.11-1.0.4.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/slf4j-api-1.7.21.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/slf4j-log4j12-1.7.21.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/snappy-java-1.1.2.6.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/validation-api-1.1.0.Final.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/zkclient-0.9.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/zookeeper-3.4.8.jar
kafka.Kafka /home/kafka/kafka_2.11-0.10.1.1/config/server.properties
root 28615 0.0 0.0 14180 1024 pts/0 S+ 13:35 0:00 grep --color=auto kafka
root@dc3-kafka-02:/home/kafka/kafka_2.11-0.10.1.1# netstat -tulpn | grep
9092
...returns empty
If I restart Kafka in these brokers, they start listening to 9092 again.
Thanks so much for your help, best regards.
Rafael
Re: Kafka running but not listening to port 9092
Posted by Neil Moore <ne...@adobe.com>.
I don't know what the problem is but have you looked through the logs? If that doesn't suggest the resolution try posting them here.
________________________________
From: Rafael Telles <ra...@simbioseventures.com>
Sent: 29 March 2017 14:53:27
To: users@kafka.apache.org
Subject: Kafka running but not listening to port 9092
Hello there!
I have two clusters of Kafka brokers, one of them (with 15 brokers + 3
Zookeeper servers) became sick (a lot of under-replicated partitions,
throwing a lot of NotEnoughReplicasExceptions). I logged in some of the
brokers that other couldn't connect to, and I found out that they were all
running their Kafka process, but they were not listening to the default TCP
port (9092) as expected:
root@dc3-kafka-02:/home/kafka/kafka_2.11-0.10.1.1# ps aux | grep kafka
root 14055 21.6 33.6 23001236 5513176 ? Sl Mar23 1866:20
/usr/lib/jvm/java-8-oracle/bin/java -Xms2G -Xmx6G -server -XX:+UseG1GC
-XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35
-XX:+DisableExplicitGC -Djava.awt.headless=true
-Xloggc:/home/kafka/kafka_2.11-0.10.1.1/bin/../logs/kafkaServer-gc.log
-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps
-XX:+PrintGCTimeStamps -Dcom.sun.management.jmxremote
-Dcom.sun.management.jmxremote.authenticate=false
-Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.port=17264
-Dkafka.logs.dir=/home/kafka/kafka_2.11-0.10.1.1/bin/../logs
-Dlog4j.configuration=
file:/home/kafka/kafka_2.11-0.10.1.1/bin/../config/log4j.properties -cp
:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/aopalliance-repackaged-2.4.0-b34.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/argparse4j-0.5.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/connect-api-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/connect-file-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/connect-json-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/connect-runtime-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/guava-18.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/hk2-api-2.4.0-b34.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/hk2-locator-2.4.0-b34.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/hk2-utils-2.4.0-b34.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jackson-annotations-2.6.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jackson-core-2.6.3.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jackson-databind-2.6.3.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jackson-jaxrs-base-2.6.3.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jackson-jaxrs-json-provider-2.6.3.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jackson-module-jaxb-annotations-2.6.3.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/javassist-3.18.2-GA.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/javax.annotation-api-1.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/javax.inject-1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/javax.inject-2.4.0-b34.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/javax.servlet-api-3.1.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/javax.ws.rs-api-2.0.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-client-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-common-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-container-servlet-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-container-servlet-core-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-guava-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-media-jaxb-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jersey-server-2.22.2.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-continuation-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-http-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-io-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-security-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-server-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-servlet-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-servlets-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jetty-util-9.2.15.v20160210.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/jopt-simple-4.9.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka_2.11-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka_2.11-0.10.1.1-sources.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka_2.11-0.10.1.1-test-sources.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka-clients-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka-log4j-appender-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka-streams-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka-streams-examples-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/kafka-tools-0.10.1.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/log4j-1.2.17.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/lz4-1.3.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/metrics-core-2.2.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/osgi-resource-locator-1.0.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/raven-7.8.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/raven-log4j-7.8.1.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/reflections-0.9.10.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/rocksdbjni-4.9.0.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/scala-library-2.11.8.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/scala-parser-combinators_2.11-1.0.4.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/slf4j-api-1.7.21.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/slf4j-log4j12-1.7.21.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/snappy-java-1.1.2.6.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/validation-api-1.1.0.Final.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/zkclient-0.9.jar:/home/kafka/kafka_2.11-0.10.1.1/bin/../libs/zookeeper-3.4.8.jar
kafka.Kafka /home/kafka/kafka_2.11-0.10.1.1/config/server.properties
root 28615 0.0 0.0 14180 1024 pts/0 S+ 13:35 0:00 grep --color=auto kafka
root@dc3-kafka-02:/home/kafka/kafka_2.11-0.10.1.1# netstat -tulpn | grep
9092
...returns empty
If I restart Kafka in these brokers, they start listening to 9092 again.
Thanks so much for your help, best regards.
Rafael