You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by rs...@apache.org on 2018/08/14 17:32:12 UTC

[kafka] branch 1.0 updated: KAFKA-7119: Handle transient Kerberos errors as non-fatal exceptions (#5487)

This is an automated email from the ASF dual-hosted git repository.

rsivaram pushed a commit to branch 1.0
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/1.0 by this push:
     new d3f536c  KAFKA-7119: Handle transient Kerberos errors as non-fatal exceptions (#5487)
d3f536c is described below

commit d3f536c05231f2e933933220b23d734e8ea8740f
Author: Rajini Sivaram <ra...@googlemail.com>
AuthorDate: Tue Aug 14 18:00:54 2018 +0100

    KAFKA-7119: Handle transient Kerberos errors as non-fatal exceptions (#5487)
    
    Reviewers: Jun Rao <ju...@gmail.com>, Ismael Juma <is...@juma.me.uk>
---
 .../authenticator/SaslClientAuthenticator.java     |  86 +++++++++++-
 .../kafka/server/GssapiAuthenticationTest.scala    | 155 +++++++++++++++++++++
 .../kafka/server/ScramServerStartupTest.scala      |  11 +-
 .../test/scala/unit/kafka/utils/TestUtils.scala    |   6 +
 4 files changed, 244 insertions(+), 14 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
index b01ae4c..01650cf 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
@@ -42,6 +42,7 @@ import org.apache.kafka.common.requests.SaslAuthenticateResponse;
 import org.apache.kafka.common.requests.SaslHandshakeRequest;
 import org.apache.kafka.common.requests.SaslHandshakeResponse;
 import org.apache.kafka.common.security.auth.KafkaPrincipal;
+import org.apache.kafka.common.utils.Java;
 import org.apache.kafka.common.utils.Utils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -51,6 +52,7 @@ import javax.security.sasl.Sasl;
 import javax.security.sasl.SaslClient;
 import javax.security.sasl.SaslException;
 import java.io.IOException;
+import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
 import java.nio.channels.SelectionKey;
 import java.security.Principal;
@@ -363,11 +365,9 @@ public class SaslClientAuthenticator implements Authenticator {
                 });
         } catch (PrivilegedActionException e) {
             String error = "An error: (" + e + ") occurred when evaluating SASL token received from the Kafka Broker.";
+            KerberosError kerberosError = KerberosError.fromException(e);
             // Try to provide hints to use about what went wrong so they can fix their configuration.
-            // TODO: introspect about e: look for GSS information.
-            final String unknownServerErrorText =
-                "(Mechanism level: Server not found in Kerberos database (7) - UNKNOWN_SERVER)";
-            if (e.toString().contains(unknownServerErrorText)) {
+            if (kerberosError == KerberosError.SERVER_NOT_FOUND) {
                 error += " This may be caused by Java's being unable to resolve the Kafka Broker's" +
                     " hostname correctly. You may want to try to adding" +
                     " '-Dsun.net.spi.nameservice.provider.1=dns,sun' to your client's JVMFLAGS environment." +
@@ -376,7 +376,13 @@ public class SaslClientAuthenticator implements Authenticator {
             }
             error += " Kafka Client will go to AUTHENTICATION_FAILED state.";
             //Unwrap the SaslException inside `PrivilegedActionException`
-            throw new SaslAuthenticationException(error, e.getCause());
+            Throwable cause = e.getCause();
+            // Treat transient Kerberos errors as non-fatal SaslExceptions that are processed as I/O exceptions
+            // and all other failures as fatal SaslAuthenticationException.
+            if (kerberosError != null && kerberosError.retriable)
+                throw new SaslException(error, cause);
+            else
+                throw new SaslAuthenticationException(error, cause);
         }
     }
 
@@ -439,4 +445,74 @@ public class SaslClientAuthenticator implements Authenticator {
                 throw new KafkaException("Principal could not be determined from Subject, this may be a transient failure due to Kerberos re-login");
         }
     }
+
+    /**
+     * Kerberos exceptions that may require special handling. The standard Kerberos error codes
+     * for these errors are retrieved using KrbException#errorCode() from the underlying Kerberos
+     * exception thrown during {@link SaslClient#evaluateChallenge(byte[])}.
+     */
+    private enum KerberosError {
+        // (Mechanism level: Server not found in Kerberos database (7) - UNKNOWN_SERVER)
+        // This is retriable, but included here to add extra logging for this case.
+        SERVER_NOT_FOUND(7, false),
+        // (Mechanism level: Client not yet valid - try again later (21))
+        CLIENT_NOT_YET_VALID(21, true),
+        // (Mechanism level: Ticket not yet valid (33) - Ticket not yet valid)])
+        // This could be a small timing window.
+        TICKET_NOT_YET_VALID(33, true),
+        // (Mechanism level: Request is a replay (34) - Request is a replay)
+        // Replay detection used to prevent DoS attacks can result in false positives, so retry on error.
+        REPLAY(34, true);
+
+
+        private static final Class<?> KRB_EXCEPTION_CLASS;
+        private static final Method KRB_EXCEPTION_RETURN_CODE_METHOD;
+
+        static {
+            try {
+                if (Java.isIbmJdk()) {
+                    KRB_EXCEPTION_CLASS = Class.forName("com.ibm.security.krb5.internal.KrbException");
+                } else {
+                    KRB_EXCEPTION_CLASS = Class.forName("sun.security.krb5.KrbException");
+                }
+                KRB_EXCEPTION_RETURN_CODE_METHOD = KRB_EXCEPTION_CLASS.getMethod("returnCode");
+            } catch (Exception e) {
+                throw new KafkaException("Kerberos exceptions could not be initialized", e);
+            }
+        }
+
+        private final int errorCode;
+        private final boolean retriable;
+
+        KerberosError(int errorCode, boolean retriable) {
+            this.errorCode = errorCode;
+            this.retriable = retriable;
+        }
+
+        private static KerberosError fromException(Exception exception) {
+            Throwable cause = exception.getCause();
+            while (cause != null && !KRB_EXCEPTION_CLASS.isInstance(cause)) {
+                cause = cause.getCause();
+            }
+            if (cause == null)
+                return null;
+            else {
+                try {
+                    Integer errorCode = (Integer) KRB_EXCEPTION_RETURN_CODE_METHOD.invoke(cause);
+                    return fromErrorCode(errorCode);
+                } catch (Exception e) {
+                    LOG.trace("Kerberos return code could not be determined from {} due to {}", exception, e);
+                    return null;
+                }
+            }
+        }
+
+        private static KerberosError fromErrorCode(int errorCode) {
+            for (KerberosError error : values()) {
+                if (error.errorCode == errorCode)
+                    return error;
+            }
+            return null;
+        }
+    }
 }
diff --git a/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala
new file mode 100644
index 0000000..037f614
--- /dev/null
+++ b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala
@@ -0,0 +1,155 @@
+/**
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+
+package kafka.server
+
+import java.net.InetSocketAddress
+import java.util.Properties
+import java.util.concurrent.{Executors, TimeUnit}
+
+import kafka.api.{Both, IntegrationTestHarness, SaslSetup}
+import kafka.utils.TestUtils
+import org.apache.kafka.clients.CommonClientConfigs
+import org.apache.kafka.common.config.SaslConfigs
+import org.apache.kafka.common.network._
+import org.apache.kafka.common.security.{JaasContext, TestSecurityConfig}
+import org.apache.kafka.common.security.auth.SecurityProtocol
+import org.junit.Assert._
+import org.junit.{After, Before, Test}
+
+class GssapiAuthenticationTest extends IntegrationTestHarness with SaslSetup {
+
+  override val producerCount = 0
+  override val consumerCount = 0
+  override val serverCount = 1
+  override protected def securityProtocol = SecurityProtocol.SASL_PLAINTEXT
+
+  private val kafkaClientSaslMechanism = "GSSAPI"
+  private val kafkaServerSaslMechanisms = List("GSSAPI")
+
+  private val numThreads = 10
+  private val executor = Executors.newFixedThreadPool(numThreads)
+  private val clientConfig: Properties = new Properties
+  private var serverAddr: InetSocketAddress = _
+
+  @Before
+  override def setUp() {
+    startSasl(jaasSections(kafkaServerSaslMechanisms, Option(kafkaClientSaslMechanism), Both))
+    super.setUp()
+    serverAddr = new InetSocketAddress("localhost",
+      servers.head.boundPort(ListenerName.forSecurityProtocol(SecurityProtocol.SASL_PLAINTEXT)))
+
+    clientConfig.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_PLAINTEXT.name)
+    clientConfig.put(SaslConfigs.SASL_MECHANISM, kafkaClientSaslMechanism)
+    clientConfig.put(SaslConfigs.SASL_JAAS_CONFIG, jaasClientLoginModule(kafkaClientSaslMechanism))
+    clientConfig.put(CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_CONFIG, "5000")
+  }
+
+  @After
+  override def tearDown(): Unit = {
+    executor.shutdownNow()
+    super.tearDown()
+    closeSasl()
+  }
+
+  /**
+   * Tests that Kerberos replay error `Request is a replay (34)` is not handled as an authentication exception
+   * since replay detection used to detect DoS attacks may occasionally reject valid concurrent requests.
+   */
+  @Test
+  def testRequestIsAReplay(): Unit = {
+    val successfulAuthsPerThread = 10
+    val futures = (0 until numThreads).map(_ => executor.submit(new Runnable {
+      override def run(): Unit = verifyRetriableFailuresDuringAuthentication(successfulAuthsPerThread)
+    }))
+    futures.foreach(_.get(60, TimeUnit.SECONDS))
+    assertEquals(0, TestUtils.totalMetricValue(servers.head, "failed-authentication-total"))
+    val successfulAuths = TestUtils.totalMetricValue(servers.head, "successful-authentication-total")
+    assertTrue("Too few authentications: " + successfulAuths, successfulAuths > successfulAuthsPerThread * numThreads)
+  }
+
+  /**
+   * Tests that Kerberos error `Server not found in Kerberos database (7)` is handled
+   * as a fatal authentication failure.
+   */
+  @Test
+  def testServerNotFoundInKerberosDatabase(): Unit = {
+    val jaasConfig = clientConfig.getProperty(SaslConfigs.SASL_JAAS_CONFIG)
+    val invalidServiceConfig = jaasConfig.replace("serviceName=\"kafka\"", "serviceName=\"invalid-service\"")
+    clientConfig.put(SaslConfigs.SASL_JAAS_CONFIG, invalidServiceConfig)
+    clientConfig.put(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "invalid-service")
+    verifyNonRetriableAuthenticationFailure()
+  }
+
+  /**
+   * Verifies that any exceptions during authentication with the current `clientConfig` are
+   * notified with disconnect state `AUTHENTICATE` (and not `AUTHENTICATION_FAILED`). This
+   * is to ensure that NetworkClient doesn't handle this as a fatal authentication failure,
+   * but as a transient I/O exception. So Producer/Consumer/AdminClient will retry
+   * any operation based on their configuration until timeout and will not propagate
+   * the exception to the application.
+   */
+  private def verifyRetriableFailuresDuringAuthentication(numSuccessfulAuths: Int): Unit = {
+    val selector = createSelector()
+    try {
+      var actualSuccessfulAuths = 0
+      while (actualSuccessfulAuths < numSuccessfulAuths) {
+        val nodeId = actualSuccessfulAuths.toString
+        selector.connect(nodeId, serverAddr, 1024, 1024)
+        TestUtils.waitUntilTrue(() => {
+          selector.poll(100)
+          val disconnectState = selector.disconnected().get(nodeId)
+          // Verify that disconnect state is not AUTHENTICATION_FAILED
+          if (disconnectState != null)
+            assertEquals(ChannelState.State.AUTHENTICATE, disconnectState.state())
+          selector.isChannelReady(nodeId) || disconnectState != null
+        }, "Client not ready or disconnected within timeout")
+        if (selector.isChannelReady(nodeId))
+          actualSuccessfulAuths += 1
+        selector.close(nodeId)
+      }
+    } finally {
+      selector.close()
+    }
+  }
+
+  /**
+   * Verifies that authentication with the current `clientConfig` results in disconnection and that
+   * the disconnection is notified with disconnect state `AUTHENTICATION_FAILED`. This is to ensure
+   * that NetworkClient handles this as a fatal authentication failure that is propagated to
+   * applications by Producer/Consumer/AdminClient without retrying and waiting for timeout.
+   */
+  private def verifyNonRetriableAuthenticationFailure(): Unit = {
+    val selector = createSelector()
+    val nodeId = "1"
+    selector.connect(nodeId, serverAddr, 1024, 1024)
+    TestUtils.waitUntilTrue(() => {
+      selector.poll(100)
+      val disconnectState = selector.disconnected().get(nodeId)
+      if (disconnectState != null)
+        assertEquals(ChannelState.State.AUTHENTICATION_FAILED, disconnectState.state())
+      disconnectState != null
+    }, "Client not disconnected within timeout")
+  }
+
+  private def createSelector(): Selector = {
+    val channelBuilder = ChannelBuilders.clientChannelBuilder(securityProtocol,
+      JaasContext.Type.CLIENT, new TestSecurityConfig(clientConfig), null, kafkaClientSaslMechanism, true)
+    NetworkTestUtils.createSelector(channelBuilder)
+  }
+}
diff --git a/core/src/test/scala/integration/kafka/server/ScramServerStartupTest.scala b/core/src/test/scala/integration/kafka/server/ScramServerStartupTest.scala
index 317fb7f..6d18825 100644
--- a/core/src/test/scala/integration/kafka/server/ScramServerStartupTest.scala
+++ b/core/src/test/scala/integration/kafka/server/ScramServerStartupTest.scala
@@ -57,16 +57,9 @@ class ScramServerStartupTest extends IntegrationTestHarness with SaslSetup {
 
   @Test
   def testAuthentications(): Unit = {
-    val successfulAuths = totalAuthentications("successful-authentication-total")
+    val successfulAuths = TestUtils.totalMetricValue(servers.head, "successful-authentication-total")
     assertTrue("No successful authentications", successfulAuths > 0)
-    val failedAuths = totalAuthentications("failed-authentication-total")
+    val failedAuths = TestUtils.totalMetricValue(servers.head, "failed-authentication-total")
     assertEquals(0, failedAuths)
   }
-
-  private def totalAuthentications(metricName: String): Int = {
-    val allMetrics = servers.head.metrics.metrics
-    val totalAuthCount = allMetrics.values().asScala.filter(_.metricName().name() == metricName)
-      .foldLeft(0.0)((total, metric) => total + metric.metricValue.asInstanceOf[Double])
-    totalAuthCount.toInt
-  }
 }
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index b45a841..2aa0731 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -1496,6 +1496,12 @@ object TestUtils extends Logging {
     out.toString
   }
 
+  def totalMetricValue(server: KafkaServer, metricName: String): Long = {
+    val allMetrics = server.metrics.metrics
+    val total = allMetrics.values().asScala.filter(_.metricName().name() == metricName)
+      .foldLeft(0.0)((total, metric) => total + metric.metricValue.asInstanceOf[Double])
+    total.toLong
+  }
 }
 
 class IntEncoder(props: VerifiableProperties = null) extends Encoder[Int] {