You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2023/04/13 05:06:40 UTC
[kyuubi] branch master updated: [KYUUBI #4652] Upgrade Hadoop from 3.3.4 to 3.3.5
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new a9d3e1192 [KYUUBI #4652] Upgrade Hadoop from 3.3.4 to 3.3.5
a9d3e1192 is described below
commit a9d3e11926ec828f8439ce5017e1aa8b62183b3e
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Thu Apr 13 13:06:29 2023 +0800
[KYUUBI #4652] Upgrade Hadoop from 3.3.4 to 3.3.5
### _Why are the changes needed?_
Upgrade Hadoop from 3.3.4 to 3.3.5, close #4652
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4689 from huapan123456/master.
Closes #4652
810af9848 [Cheng Pan] fix
380786e9f [Cheng Pan] nit
7c5f18ef9 [Cheng Pan] nit
2d308943b [Cheng Pan] nit
76869d822 [Cheng Pan] handle HDFS-16591
36142c787 [huapan] fix(KYUUBI #4652): the position of JaasConfiguration has changed on hadoop-clint-api-3.3.5
1b5de6dfe [huapan] feat(KYUUBI #4652): upgrade hadoop-version from 3.3.4 to 3.3.5
f9d34c716 [huapan] feat(KYUUBI #4652): upgrade hadoop-client-api from 3.3.4 to 3.3.5
Lead-authored-by: Cheng Pan <ch...@apache.org>
Co-authored-by: huapan <hu...@52tt.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
dev/dependencyList | 4 +-
.../client/zookeeper/ZookeeperClientProvider.scala | 66 +++++++++++-----------
.../zookeeper/ZookeeperDiscoveryClientSuite.scala | 9 +--
pom.xml | 2 +-
4 files changed, 41 insertions(+), 40 deletions(-)
diff --git a/dev/dependencyList b/dev/dependencyList
index ab7697d35..fc9913604 100644
--- a/dev/dependencyList
+++ b/dev/dependencyList
@@ -51,8 +51,8 @@ grpc-protobuf/1.48.0//grpc-protobuf-1.48.0.jar
grpc-stub/1.48.0//grpc-stub-1.48.0.jar
gson/2.9.0//gson-2.9.0.jar
guava/31.1-jre//guava-31.1-jre.jar
-hadoop-client-api/3.3.4//hadoop-client-api-3.3.4.jar
-hadoop-client-runtime/3.3.4//hadoop-client-runtime-3.3.4.jar
+hadoop-client-api/3.3.5//hadoop-client-api-3.3.5.jar
+hadoop-client-runtime/3.3.5//hadoop-client-runtime-3.3.5.jar
hive-common/3.1.3//hive-common-3.1.3.jar
hive-metastore/3.1.3//hive-metastore-3.1.3.jar
hive-serde/3.1.3//hive-serde-3.1.3.jar
diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala
index 8dd32d6b6..b7297d969 100644
--- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala
+++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala
@@ -18,6 +18,7 @@
package org.apache.kyuubi.ha.client.zookeeper
import java.io.{File, IOException}
+import java.nio.charset.StandardCharsets
import javax.security.auth.login.Configuration
import scala.util.Random
@@ -26,13 +27,13 @@ import com.google.common.annotations.VisibleForTesting
import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
import org.apache.curator.retry._
import org.apache.hadoop.security.UserGroupInformation
-import org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.JaasConfiguration
import org.apache.kyuubi.Logging
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.ha.HighAvailabilityConf._
import org.apache.kyuubi.ha.client.{AuthTypes, RetryPolicies}
import org.apache.kyuubi.ha.client.RetryPolicies._
+import org.apache.kyuubi.reflection.DynConstructors
import org.apache.kyuubi.util.KyuubiHadoopUtils
object ZookeeperClientProvider extends Logging {
@@ -65,10 +66,8 @@ object ZookeeperClientProvider extends Logging {
.aclProvider(new ZookeeperACLProvider(conf))
.retryPolicy(retryPolicy)
- conf.get(HA_ZK_AUTH_DIGEST) match {
- case Some(anthString) =>
- builder.authorization("digest", anthString.getBytes("UTF-8"))
- case _ =>
+ conf.get(HA_ZK_AUTH_DIGEST).foreach { authString =>
+ builder.authorization("digest", authString.getBytes(StandardCharsets.UTF_8))
}
builder.build()
@@ -103,46 +102,47 @@ object ZookeeperClientProvider extends Logging {
*/
@throws[Exception]
def setUpZooKeeperAuth(conf: KyuubiConf): Unit = {
- def setupZkAuth(): Unit = {
- val keyTabFile = getKeyTabFile(conf)
- val maybePrincipal = conf.get(HA_ZK_AUTH_PRINCIPAL)
- val kerberized = maybePrincipal.isDefined && keyTabFile.isDefined
- if (UserGroupInformation.isSecurityEnabled && kerberized) {
- if (!new File(keyTabFile.get).exists()) {
- throw new IOException(s"${HA_ZK_AUTH_KEYTAB.key}: $keyTabFile does not exists")
+ def setupZkAuth(): Unit = (conf.get(HA_ZK_AUTH_PRINCIPAL), getKeyTabFile(conf)) match {
+ case (Some(principal), Some(keytab)) if UserGroupInformation.isSecurityEnabled =>
+ if (!new File(keytab).exists()) {
+ throw new IOException(s"${HA_ZK_AUTH_KEYTAB.key}: $keytab does not exists")
}
System.setProperty("zookeeper.sasl.clientconfig", "KyuubiZooKeeperClient")
- var principal = maybePrincipal.get
- principal = KyuubiHadoopUtils.getServerPrincipal(principal)
- val jaasConf = new JaasConfiguration("KyuubiZooKeeperClient", principal, keyTabFile.get)
+ val serverPrincipal = KyuubiHadoopUtils.getServerPrincipal(principal)
+ // HDFS-16591 makes breaking change on JaasConfiguration
+ val jaasConf = DynConstructors.builder()
+ .impl( // Hadoop 3.3.5 and above
+ "org.apache.hadoop.security.authentication.util.JaasConfiguration",
+ classOf[String],
+ classOf[String],
+ classOf[String])
+ .impl( // Hadoop 3.3.4 and previous
+ // scalastyle:off
+ "org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.JaasConfiguration",
+ // scalastyle:on
+ classOf[String],
+ classOf[String],
+ classOf[String])
+ .build[Configuration]()
+ .newInstance("KyuubiZooKeeperClient", serverPrincipal, keytab)
Configuration.setConfiguration(jaasConf)
- }
+ case _ =>
}
- if (conf.get(HA_ENGINE_REF_ID).isEmpty
- && AuthTypes.withName(conf.get(HA_ZK_AUTH_TYPE)) == AuthTypes.KERBEROS) {
+ if (conf.get(HA_ENGINE_REF_ID).isEmpty &&
+ AuthTypes.withName(conf.get(HA_ZK_AUTH_TYPE)) == AuthTypes.KERBEROS) {
setupZkAuth()
- } else if (conf.get(HA_ENGINE_REF_ID).nonEmpty && AuthTypes
- .withName(conf.get(HA_ZK_ENGINE_AUTH_TYPE)) == AuthTypes.KERBEROS) {
+ } else if (conf.get(HA_ENGINE_REF_ID).nonEmpty &&
+ AuthTypes.withName(conf.get(HA_ZK_ENGINE_AUTH_TYPE)) == AuthTypes.KERBEROS) {
setupZkAuth()
}
-
}
@VisibleForTesting
def getKeyTabFile(conf: KyuubiConf): Option[String] = {
- val zkAuthKeytab = conf.get(HA_ZK_AUTH_KEYTAB)
- if (zkAuthKeytab.isDefined) {
- val zkAuthKeytabPath = zkAuthKeytab.get
- val relativeFileName = new File(zkAuthKeytabPath).getName
- if (new File(relativeFileName).exists()) {
- Some(relativeFileName)
- } else {
- Some(zkAuthKeytabPath)
- }
- } else {
- None
+ conf.get(HA_ZK_AUTH_KEYTAB).map { fullPath =>
+ val filename = new File(fullPath).getName
+ if (new File(filename).exists()) filename else fullPath
}
}
-
}
diff --git a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClientSuite.scala b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClientSuite.scala
index bbd8b94ac..e3f5546a9 100644
--- a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClientSuite.scala
+++ b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClientSuite.scala
@@ -37,6 +37,7 @@ import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.ha.HighAvailabilityConf._
import org.apache.kyuubi.ha.client._
import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient
+import org.apache.kyuubi.ha.client.zookeeper.ZookeeperClientProvider._
import org.apache.kyuubi.service._
import org.apache.kyuubi.zookeeper.EmbeddedZookeeper
import org.apache.kyuubi.zookeeper.ZookeeperConf.ZK_CLIENT_PORT
@@ -117,7 +118,7 @@ abstract class ZookeeperDiscoveryClientSuite extends DiscoveryClientTests
conf.set(HA_ZK_AUTH_PRINCIPAL.key, principal)
conf.set(HA_ZK_AUTH_TYPE.key, AuthTypes.KERBEROS.toString)
- ZookeeperClientProvider.setUpZooKeeperAuth(conf)
+ setUpZooKeeperAuth(conf)
val configuration = Configuration.getConfiguration
val entries = configuration.getAppConfigurationEntry("KyuubiZooKeeperClient")
@@ -129,9 +130,9 @@ abstract class ZookeeperDiscoveryClientSuite extends DiscoveryClientTests
assert(options("useKeyTab").toString.toBoolean)
conf.set(HA_ZK_AUTH_KEYTAB.key, s"${keytab.getName}")
- val e = intercept[IOException](ZookeeperClientProvider.setUpZooKeeperAuth(conf))
- assert(e.getMessage ===
- s"${HA_ZK_AUTH_KEYTAB.key}: ${ZookeeperClientProvider.getKeyTabFile(conf)} does not exists")
+ val e = intercept[IOException](setUpZooKeeperAuth(conf))
+ assert(
+ e.getMessage === s"${HA_ZK_AUTH_KEYTAB.key}: ${getKeyTabFile(conf).get} does not exists")
}
}
diff --git a/pom.xml b/pom.xml
index aa4a83a56..1c702091c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -143,7 +143,7 @@
<grpc.version>1.48.0</grpc.version>
<guava.version>31.1-jre</guava.version>
<guava.failureaccess.version>1.0.1</guava.failureaccess.version>
- <hadoop.version>3.3.4</hadoop.version>
+ <hadoop.version>3.3.5</hadoop.version>
<hikaricp.version>4.0.3</hikaricp.version>
<derby.version>10.14.2.0</derby.version>
<fliptables.verion>1.0.2</fliptables.verion>