You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by su...@apache.org on 2023/04/19 17:26:54 UTC
[spark] branch master updated: [SPARK-43187][TEST] Remove workaround for MiniKdc's BindException
This is an automated email from the ASF dual-hosted git repository.
sunchao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 77b72fc5535 [SPARK-43187][TEST] Remove workaround for MiniKdc's BindException
77b72fc5535 is described below
commit 77b72fc553569008d1000e628cdd26e07d325b72
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Wed Apr 19 10:26:40 2023 -0700
[SPARK-43187][TEST] Remove workaround for MiniKdc's BindException
### What changes were proposed in this pull request?
This PR basically reverts the SPARK-31631, which was aimed to address [HADOOP-12656](https://issues.apache.org/jira/browse/HADOOP-12656)
### Why are the changes needed?
Since [HADOOP-12656](https://issues.apache.org/jira/browse/HADOOP-12656) got fixed in Hadoop 2.8.0/3.0.0, and SPARK-42452 removed support for Hadoop2, we can remove this workaround now.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass GA.
Closes #40849 from pan3793/SPARK-43187.
Authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: Chao Sun <su...@apple.com>
---
.../apache/spark/sql/kafka010/KafkaTestUtils.scala | 27 ++-----------------
.../HadoopDelegationTokenManagerSuite.scala | 30 ++--------------------
2 files changed, 4 insertions(+), 53 deletions(-)
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
index 7c9c40883a5..4e11a66bc2e 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
@@ -26,7 +26,6 @@ import javax.security.auth.login.Configuration
import scala.collection.JavaConverters._
import scala.io.Source
-import scala.util.control.NonFatal
import com.google.common.io.Files
import kafka.api.Request
@@ -139,30 +138,8 @@ class KafkaTestUtils(
val kdcDir = Utils.createTempDir()
val kdcConf = MiniKdc.createConf()
kdcConf.setProperty(MiniKdc.DEBUG, "true")
- // The port for MiniKdc service gets selected in the constructor, but will be bound
- // to it later in MiniKdc.start() -> MiniKdc.initKDCServer() -> KdcServer.start().
- // In meantime, when some other service might capture the port during this progress, and
- // cause BindException.
- // This makes our tests which have dedicated JVMs and rely on MiniKDC being flaky
- //
- // https://issues.apache.org/jira/browse/HADOOP-12656 get fixed in Hadoop 2.8.0.
- //
- // The workaround here is to periodically repeat this process with a timeout , since we are
- // using Hadoop 2.7.4 as default.
- // https://issues.apache.org/jira/browse/SPARK-31631
- eventually(timeout(60.seconds), interval(1.second)) {
- try {
- kdc = new MiniKdc(kdcConf, kdcDir)
- kdc.start()
- } catch {
- case NonFatal(e) =>
- if (kdc != null) {
- kdc.stop()
- kdc = null
- }
- throw e
- }
- }
+ kdc = new MiniKdc(kdcConf, kdcDir)
+ kdc.start()
// TODO https://issues.apache.org/jira/browse/SPARK-30037
// Need to build spark's own MiniKDC and customize krb5.conf like Kafka
rewriteKrb5Conf()
diff --git a/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala
index d9d559509f4..275bca34598 100644
--- a/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala
@@ -19,14 +19,10 @@ package org.apache.spark.deploy.security
import java.security.PrivilegedExceptionAction
-import scala.util.control.NonFatal
-
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION
import org.apache.hadoop.minikdc.MiniKdc
import org.apache.hadoop.security.{Credentials, UserGroupInformation}
-import org.scalatest.concurrent.Eventually._
-import org.scalatest.time.SpanSugar._
import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.deploy.SparkHadoopUtil
@@ -92,30 +88,8 @@ class HadoopDelegationTokenManagerSuite extends SparkFunSuite {
// krb5.conf. MiniKdc sets "java.security.krb5.conf" in start and removes it when stop called.
val kdcDir = Utils.createTempDir()
val kdcConf = MiniKdc.createConf()
- // The port for MiniKdc service gets selected in the constructor, but will be bound
- // to it later in MiniKdc.start() -> MiniKdc.initKDCServer() -> KdcServer.start().
- // In meantime, when some other service might capture the port during this progress, and
- // cause BindException.
- // This makes our tests which have dedicated JVMs and rely on MiniKDC being flaky
- //
- // https://issues.apache.org/jira/browse/HADOOP-12656 get fixed in Hadoop 2.8.0.
- //
- // The workaround here is to periodically repeat this process with a timeout , since we are
- // using Hadoop 2.7.4 as default.
- // https://issues.apache.org/jira/browse/SPARK-31631
- eventually(timeout(60.seconds), interval(1.second)) {
- try {
- kdc = new MiniKdc(kdcConf, kdcDir)
- kdc.start()
- } catch {
- case NonFatal(e) =>
- if (kdc != null) {
- kdc.stop()
- kdc = null
- }
- throw e
- }
- }
+ kdc = new MiniKdc(kdcConf, kdcDir)
+ kdc.start()
val krbConf = new Configuration()
krbConf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org