You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by rs...@apache.org on 2018/08/14 17:32:12 UTC
[kafka] branch 1.0 updated: KAFKA-7119: Handle transient Kerberos
errors as non-fatal exceptions (#5487)
This is an automated email from the ASF dual-hosted git repository.
rsivaram pushed a commit to branch 1.0
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/1.0 by this push:
new d3f536c KAFKA-7119: Handle transient Kerberos errors as non-fatal exceptions (#5487)
d3f536c is described below
commit d3f536c05231f2e933933220b23d734e8ea8740f
Author: Rajini Sivaram <ra...@googlemail.com>
AuthorDate: Tue Aug 14 18:00:54 2018 +0100
KAFKA-7119: Handle transient Kerberos errors as non-fatal exceptions (#5487)
Reviewers: Jun Rao <ju...@gmail.com>, Ismael Juma <is...@juma.me.uk>
---
.../authenticator/SaslClientAuthenticator.java | 86 +++++++++++-
.../kafka/server/GssapiAuthenticationTest.scala | 155 +++++++++++++++++++++
.../kafka/server/ScramServerStartupTest.scala | 11 +-
.../test/scala/unit/kafka/utils/TestUtils.scala | 6 +
4 files changed, 244 insertions(+), 14 deletions(-)
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
index b01ae4c..01650cf 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
@@ -42,6 +42,7 @@ import org.apache.kafka.common.requests.SaslAuthenticateResponse;
import org.apache.kafka.common.requests.SaslHandshakeRequest;
import org.apache.kafka.common.requests.SaslHandshakeResponse;
import org.apache.kafka.common.security.auth.KafkaPrincipal;
+import org.apache.kafka.common.utils.Java;
import org.apache.kafka.common.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,6 +52,7 @@ import javax.security.sasl.Sasl;
import javax.security.sasl.SaslClient;
import javax.security.sasl.SaslException;
import java.io.IOException;
+import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.channels.SelectionKey;
import java.security.Principal;
@@ -363,11 +365,9 @@ public class SaslClientAuthenticator implements Authenticator {
});
} catch (PrivilegedActionException e) {
String error = "An error: (" + e + ") occurred when evaluating SASL token received from the Kafka Broker.";
+ KerberosError kerberosError = KerberosError.fromException(e);
// Try to provide hints to use about what went wrong so they can fix their configuration.
- // TODO: introspect about e: look for GSS information.
- final String unknownServerErrorText =
- "(Mechanism level: Server not found in Kerberos database (7) - UNKNOWN_SERVER)";
- if (e.toString().contains(unknownServerErrorText)) {
+ if (kerberosError == KerberosError.SERVER_NOT_FOUND) {
error += " This may be caused by Java's being unable to resolve the Kafka Broker's" +
" hostname correctly. You may want to try to adding" +
" '-Dsun.net.spi.nameservice.provider.1=dns,sun' to your client's JVMFLAGS environment." +
@@ -376,7 +376,13 @@ public class SaslClientAuthenticator implements Authenticator {
}
error += " Kafka Client will go to AUTHENTICATION_FAILED state.";
//Unwrap the SaslException inside `PrivilegedActionException`
- throw new SaslAuthenticationException(error, e.getCause());
+ Throwable cause = e.getCause();
+ // Treat transient Kerberos errors as non-fatal SaslExceptions that are processed as I/O exceptions
+ // and all other failures as fatal SaslAuthenticationException.
+ if (kerberosError != null && kerberosError.retriable)
+ throw new SaslException(error, cause);
+ else
+ throw new SaslAuthenticationException(error, cause);
}
}
@@ -439,4 +445,74 @@ public class SaslClientAuthenticator implements Authenticator {
throw new KafkaException("Principal could not be determined from Subject, this may be a transient failure due to Kerberos re-login");
}
}
+
+ /**
+ * Kerberos exceptions that may require special handling. The standard Kerberos error codes
+ * for these errors are retrieved using KrbException#errorCode() from the underlying Kerberos
+ * exception thrown during {@link SaslClient#evaluateChallenge(byte[])}.
+ */
+ private enum KerberosError {
+ // (Mechanism level: Server not found in Kerberos database (7) - UNKNOWN_SERVER)
+ // This is retriable, but included here to add extra logging for this case.
+ SERVER_NOT_FOUND(7, false),
+ // (Mechanism level: Client not yet valid - try again later (21))
+ CLIENT_NOT_YET_VALID(21, true),
+ // (Mechanism level: Ticket not yet valid (33) - Ticket not yet valid)])
+ // This could be a small timing window.
+ TICKET_NOT_YET_VALID(33, true),
+ // (Mechanism level: Request is a replay (34) - Request is a replay)
+ // Replay detection used to prevent DoS attacks can result in false positives, so retry on error.
+ REPLAY(34, true);
+
+
+ private static final Class<?> KRB_EXCEPTION_CLASS;
+ private static final Method KRB_EXCEPTION_RETURN_CODE_METHOD;
+
+ static {
+ try {
+ if (Java.isIbmJdk()) {
+ KRB_EXCEPTION_CLASS = Class.forName("com.ibm.security.krb5.internal.KrbException");
+ } else {
+ KRB_EXCEPTION_CLASS = Class.forName("sun.security.krb5.KrbException");
+ }
+ KRB_EXCEPTION_RETURN_CODE_METHOD = KRB_EXCEPTION_CLASS.getMethod("returnCode");
+ } catch (Exception e) {
+ throw new KafkaException("Kerberos exceptions could not be initialized", e);
+ }
+ }
+
+ private final int errorCode;
+ private final boolean retriable;
+
+ KerberosError(int errorCode, boolean retriable) {
+ this.errorCode = errorCode;
+ this.retriable = retriable;
+ }
+
+ private static KerberosError fromException(Exception exception) {
+ Throwable cause = exception.getCause();
+ while (cause != null && !KRB_EXCEPTION_CLASS.isInstance(cause)) {
+ cause = cause.getCause();
+ }
+ if (cause == null)
+ return null;
+ else {
+ try {
+ Integer errorCode = (Integer) KRB_EXCEPTION_RETURN_CODE_METHOD.invoke(cause);
+ return fromErrorCode(errorCode);
+ } catch (Exception e) {
+ LOG.trace("Kerberos return code could not be determined from {} due to {}", exception, e);
+ return null;
+ }
+ }
+ }
+
+ private static KerberosError fromErrorCode(int errorCode) {
+ for (KerberosError error : values()) {
+ if (error.errorCode == errorCode)
+ return error;
+ }
+ return null;
+ }
+ }
}
diff --git a/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala
new file mode 100644
index 0000000..037f614
--- /dev/null
+++ b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.server
+
+import java.net.InetSocketAddress
+import java.util.Properties
+import java.util.concurrent.{Executors, TimeUnit}
+
+import kafka.api.{Both, IntegrationTestHarness, SaslSetup}
+import kafka.utils.TestUtils
+import org.apache.kafka.clients.CommonClientConfigs
+import org.apache.kafka.common.config.SaslConfigs
+import org.apache.kafka.common.network._
+import org.apache.kafka.common.security.{JaasContext, TestSecurityConfig}
+import org.apache.kafka.common.security.auth.SecurityProtocol
+import org.junit.Assert._
+import org.junit.{After, Before, Test}
+
+class GssapiAuthenticationTest extends IntegrationTestHarness with SaslSetup {
+
+ override val producerCount = 0
+ override val consumerCount = 0
+ override val serverCount = 1
+ override protected def securityProtocol = SecurityProtocol.SASL_PLAINTEXT
+
+ private val kafkaClientSaslMechanism = "GSSAPI"
+ private val kafkaServerSaslMechanisms = List("GSSAPI")
+
+ private val numThreads = 10
+ private val executor = Executors.newFixedThreadPool(numThreads)
+ private val clientConfig: Properties = new Properties
+ private var serverAddr: InetSocketAddress = _
+
+ @Before
+ override def setUp() {
+ startSasl(jaasSections(kafkaServerSaslMechanisms, Option(kafkaClientSaslMechanism), Both))
+ super.setUp()
+ serverAddr = new InetSocketAddress("localhost",
+ servers.head.boundPort(ListenerName.forSecurityProtocol(SecurityProtocol.SASL_PLAINTEXT)))
+
+ clientConfig.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_PLAINTEXT.name)
+ clientConfig.put(SaslConfigs.SASL_MECHANISM, kafkaClientSaslMechanism)
+ clientConfig.put(SaslConfigs.SASL_JAAS_CONFIG, jaasClientLoginModule(kafkaClientSaslMechanism))
+ clientConfig.put(CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_CONFIG, "5000")
+ }
+
+ @After
+ override def tearDown(): Unit = {
+ executor.shutdownNow()
+ super.tearDown()
+ closeSasl()
+ }
+
+ /**
+ * Tests that Kerberos replay error `Request is a replay (34)` is not handled as an authentication exception
+ * since replay detection used to detect DoS attacks may occasionally reject valid concurrent requests.
+ */
+ @Test
+ def testRequestIsAReplay(): Unit = {
+ val successfulAuthsPerThread = 10
+ val futures = (0 until numThreads).map(_ => executor.submit(new Runnable {
+ override def run(): Unit = verifyRetriableFailuresDuringAuthentication(successfulAuthsPerThread)
+ }))
+ futures.foreach(_.get(60, TimeUnit.SECONDS))
+ assertEquals(0, TestUtils.totalMetricValue(servers.head, "failed-authentication-total"))
+ val successfulAuths = TestUtils.totalMetricValue(servers.head, "successful-authentication-total")
+ assertTrue("Too few authentications: " + successfulAuths, successfulAuths > successfulAuthsPerThread * numThreads)
+ }
+
+ /**
+ * Tests that Kerberos error `Server not found in Kerberos database (7)` is handled
+ * as a fatal authentication failure.
+ */
+ @Test
+ def testServerNotFoundInKerberosDatabase(): Unit = {
+ val jaasConfig = clientConfig.getProperty(SaslConfigs.SASL_JAAS_CONFIG)
+ val invalidServiceConfig = jaasConfig.replace("serviceName=\"kafka\"", "serviceName=\"invalid-service\"")
+ clientConfig.put(SaslConfigs.SASL_JAAS_CONFIG, invalidServiceConfig)
+ clientConfig.put(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "invalid-service")
+ verifyNonRetriableAuthenticationFailure()
+ }
+
+ /**
+ * Verifies that any exceptions during authentication with the current `clientConfig` are
+ * notified with disconnect state `AUTHENTICATE` (and not `AUTHENTICATION_FAILED`). This
+ * is to ensure that NetworkClient doesn't handle this as a fatal authentication failure,
+ * but as a transient I/O exception. So Producer/Consumer/AdminClient will retry
+ * any operation based on their configuration until timeout and will not propagate
+ * the exception to the application.
+ */
+ private def verifyRetriableFailuresDuringAuthentication(numSuccessfulAuths: Int): Unit = {
+ val selector = createSelector()
+ try {
+ var actualSuccessfulAuths = 0
+ while (actualSuccessfulAuths < numSuccessfulAuths) {
+ val nodeId = actualSuccessfulAuths.toString
+ selector.connect(nodeId, serverAddr, 1024, 1024)
+ TestUtils.waitUntilTrue(() => {
+ selector.poll(100)
+ val disconnectState = selector.disconnected().get(nodeId)
+ // Verify that disconnect state is not AUTHENTICATION_FAILED
+ if (disconnectState != null)
+ assertEquals(ChannelState.State.AUTHENTICATE, disconnectState.state())
+ selector.isChannelReady(nodeId) || disconnectState != null
+ }, "Client not ready or disconnected within timeout")
+ if (selector.isChannelReady(nodeId))
+ actualSuccessfulAuths += 1
+ selector.close(nodeId)
+ }
+ } finally {
+ selector.close()
+ }
+ }
+
+ /**
+ * Verifies that authentication with the current `clientConfig` results in disconnection and that
+ * the disconnection is notified with disconnect state `AUTHENTICATION_FAILED`. This is to ensure
+ * that NetworkClient handles this as a fatal authentication failure that is propagated to
+ * applications by Producer/Consumer/AdminClient without retrying and waiting for timeout.
+ */
+ private def verifyNonRetriableAuthenticationFailure(): Unit = {
+ val selector = createSelector()
+ val nodeId = "1"
+ selector.connect(nodeId, serverAddr, 1024, 1024)
+ TestUtils.waitUntilTrue(() => {
+ selector.poll(100)
+ val disconnectState = selector.disconnected().get(nodeId)
+ if (disconnectState != null)
+ assertEquals(ChannelState.State.AUTHENTICATION_FAILED, disconnectState.state())
+ disconnectState != null
+ }, "Client not disconnected within timeout")
+ }
+
+ private def createSelector(): Selector = {
+ val channelBuilder = ChannelBuilders.clientChannelBuilder(securityProtocol,
+ JaasContext.Type.CLIENT, new TestSecurityConfig(clientConfig), null, kafkaClientSaslMechanism, true)
+ NetworkTestUtils.createSelector(channelBuilder)
+ }
+}
diff --git a/core/src/test/scala/integration/kafka/server/ScramServerStartupTest.scala b/core/src/test/scala/integration/kafka/server/ScramServerStartupTest.scala
index 317fb7f..6d18825 100644
--- a/core/src/test/scala/integration/kafka/server/ScramServerStartupTest.scala
+++ b/core/src/test/scala/integration/kafka/server/ScramServerStartupTest.scala
@@ -57,16 +57,9 @@ class ScramServerStartupTest extends IntegrationTestHarness with SaslSetup {
@Test
def testAuthentications(): Unit = {
- val successfulAuths = totalAuthentications("successful-authentication-total")
+ val successfulAuths = TestUtils.totalMetricValue(servers.head, "successful-authentication-total")
assertTrue("No successful authentications", successfulAuths > 0)
- val failedAuths = totalAuthentications("failed-authentication-total")
+ val failedAuths = TestUtils.totalMetricValue(servers.head, "failed-authentication-total")
assertEquals(0, failedAuths)
}
-
- private def totalAuthentications(metricName: String): Int = {
- val allMetrics = servers.head.metrics.metrics
- val totalAuthCount = allMetrics.values().asScala.filter(_.metricName().name() == metricName)
- .foldLeft(0.0)((total, metric) => total + metric.metricValue.asInstanceOf[Double])
- totalAuthCount.toInt
- }
}
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index b45a841..2aa0731 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -1496,6 +1496,12 @@ object TestUtils extends Logging {
out.toString
}
+ def totalMetricValue(server: KafkaServer, metricName: String): Long = {
+ val allMetrics = server.metrics.metrics
+ val total = allMetrics.values().asScala.filter(_.metricName().name() == metricName)
+ .foldLeft(0.0)((total, metric) => total + metric.metricValue.asInstanceOf[Double])
+ total.toLong
+ }
}
class IntEncoder(props: VerifiableProperties = null) extends Encoder[Int] {