You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2023/05/21 12:49:13 UTC
[kyuubi] branch master updated: [KYUUBI #4852] Switch to Kyuubi Shaded Zookeeper
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 208ab3af6 [KYUUBI #4852] Switch to Kyuubi Shaded Zookeeper
208ab3af6 is described below
commit 208ab3af6215881c550067fa738e77a7b6162806
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Sun May 21 20:49:00 2023 +0800
[KYUUBI #4852] Switch to Kyuubi Shaded Zookeeper
### _Why are the changes needed?_
This PR aims to migrate the vanilla Zookeeper and Curator to the Kyuubi Shaded Zookeeper. It's the first step to adapting JDK 17.
There is a known issue [ZOOKEEPER-3779](https://issues.apache.org/jira/browse/ZOOKEEPER-3779) that Zookeeper 3.4 client can not run on JDK 14 and above, in https://github.com/apache/kyuubi-shaded/pull/5, we fixed this issue by a surgical.
With the above fixing, zk-3.4 and zk-3.6 clients both work well on JDK 17, we just randomly pick some cases to make sure zk-3.6 is tested
zk-3.4 client supports zk-3.4+ server, but zk-3.6 client only supports zk-3.5+ server; in the meanwhile, zk-3.4 is adopted widely, (CDH 5/6, HDP, EMR created before 2023).
We are sticky to zk-3.4 to ensure that Kyuubi can be out-of-box in the most existing Hadoop cluster but also provide zk-3.6 as an alternative(simply replace the kyuubi-shaded-zk-3.4 jar w/ kyuubi-shaded-zk-3.6, or build w/ -Pzookeeper-3.6) for users who concerns that zk-3.4 is EOL.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4852 from pan3793/shaded-zk.
Closes #4852
d960cc945 [Cheng Pan] remove staging repo
1b3622080 [Cheng Pan] Switch to Kyuubi Shaded Zookeeper
Authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.github/workflows/master.yml | 6 +-
LICENSE-binary | 4 --
NOTICE-binary | 9 ---
bin/kyuubi-zk-cli | 2 +-
dev/dependencyList | 4 --
externals/kyuubi-flink-sql-engine/pom.xml | 28 +-------
externals/kyuubi-hive-sql-engine/pom.xml | 16 +----
externals/kyuubi-spark-sql-engine/pom.xml | 29 +--------
kyuubi-assembly/pom.xml | 20 ++----
kyuubi-ctl/pom.xml | 20 ++----
kyuubi-ha/pom.xml | 14 +---
.../apache/kyuubi/ha/client/DiscoveryPaths.scala | 2 +-
.../ha/client/zookeeper/ZookeeperACLProvider.scala | 7 +-
.../client/zookeeper/ZookeeperClientProvider.scala | 4 +-
.../zookeeper/ZookeeperDiscoveryClient.scala | 34 +++++-----
.../zookeeper/ZookeeperDiscoveryClientSuite.scala | 8 +--
kyuubi-hive-jdbc-shaded/pom.xml | 12 ----
kyuubi-hive-jdbc/pom.xml | 14 +---
.../jdbc/hive/ZooKeeperHiveClientHelper.java | 6 +-
.../kyuubi/server/api/v1/AdminResource.scala | 2 +-
kyuubi-zookeeper/pom.xml | 22 +------
.../kyuubi/zookeeper/EmbeddedZookeeper.scala | 3 +-
.../kyuubi/zookeeper/EmbeddedZookeeperSuite.scala | 7 +-
pom.xml | 76 ++++------------------
scalastyle-config.xml | 8 +++
25 files changed, 80 insertions(+), 277 deletions(-)
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index aeb0a14bb..7edc4f662 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -57,17 +57,17 @@ jobs:
include:
- java: 8
spark: '3.3'
- spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.3 -Dspark.archive.name=spark-3.1.3-bin-hadoop3.2.tgz'
+ spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.3 -Dspark.archive.name=spark-3.1.3-bin-hadoop3.2.tgz -Pzookeeper-3.6'
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest'
comment: 'verify-on-spark-3.1-binary'
- java: 8
spark: '3.3'
- spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz'
+ spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6'
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest'
comment: 'verify-on-spark-3.2-binary'
- java: 8
spark: '3.3'
- spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.0 -Dspark.archive.name=spark-3.4.0-bin-hadoop3.tgz'
+ spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.0 -Dspark.archive.name=spark-3.4.0-bin-hadoop3.tgz -Pzookeeper-3.6'
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest'
comment: 'verify-on-spark-3.4-binary'
env:
diff --git a/LICENSE-binary b/LICENSE-binary
index 542259658..7322dec51 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -212,9 +212,6 @@ com.google.android:annotations
commons-lang:commons-lang
commons-logging:commons-logging
org.apache.commons:commons-lang3
-org.apache.curator:curator-client
-org.apache.curator:curator-framework
-org.apache.curator:curator-recipes
org.apache.derby:derby
com.google.errorprone:error_prone_annotations
net.jodah:failsafe
@@ -318,7 +315,6 @@ io.swagger.core.v3:swagger-jaxrs2
io.swagger.core.v3:swagger-models
io.vertx:vertx-core
io.vertx:vertx-grpc
-org.apache.zookeeper:zookeeper
com.squareup.retrofit2:retrofit
com.squareup.okhttp3:okhttp
org.apache.kafka:kafka-clients
diff --git a/NOTICE-binary b/NOTICE-binary
index 16281d0d8..40ec15010 100644
--- a/NOTICE-binary
+++ b/NOTICE-binary
@@ -92,15 +92,6 @@ Copyright 2001-2020 The Apache Software Foundation
Apache Commons Logging
Copyright 2003-2013 The Apache Software Foundation
-Curator Client
-Copyright 2011-2017 The Apache Software Foundation
-
-Curator Framework
-Copyright 2011-2017 The Apache Software Foundation
-
-Curator Recipes
-Copyright 2011-2017 The Apache Software Foundation
-
=========================================================================
== NOTICE file corresponding to section 4(d) of the Apache License,
== Version 2.0, in this case for the Apache Derby distribution.
diff --git a/bin/kyuubi-zk-cli b/bin/kyuubi-zk-cli
index 089b7ad18..f503c3e5a 100755
--- a/bin/kyuubi-zk-cli
+++ b/bin/kyuubi-zk-cli
@@ -17,7 +17,7 @@
#
## Zookeeper Shell Client Entrance
-CLASS="org.apache.zookeeper.ZooKeeperMain"
+CLASS="org.apache.kyuubi.shaded.zookeeper.ZooKeeperMain"
export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)"
diff --git a/dev/dependencyList b/dev/dependencyList
index 6e5673f9d..ab80bcfab 100644
--- a/dev/dependencyList
+++ b/dev/dependencyList
@@ -32,9 +32,6 @@ commons-collections/3.2.2//commons-collections-3.2.2.jar
commons-lang/2.6//commons-lang-2.6.jar
commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
-curator-client/2.12.0//curator-client-2.12.0.jar
-curator-framework/2.12.0//curator-framework-2.12.0.jar
-curator-recipes/2.12.0//curator-recipes-2.12.0.jar
derby/10.14.2.0//derby-10.14.2.0.jar
error_prone_annotations/2.14.0//error_prone_annotations-2.14.0.jar
failsafe/2.4.4//failsafe-2.4.4.jar
@@ -195,5 +192,4 @@ units/1.6//units-1.6.jar
vertx-core/4.3.2//vertx-core-4.3.2.jar
vertx-grpc/4.3.2//vertx-grpc-4.3.2.jar
zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
-zookeeper/3.4.14//zookeeper-3.4.14.jar
zstd-jni/1.5.2-1//zstd-jni-1.5.2-1.jar
diff --git a/externals/kyuubi-flink-sql-engine/pom.xml b/externals/kyuubi-flink-sql-engine/pom.xml
index c73310a64..8c62521b7 100644
--- a/externals/kyuubi-flink-sql-engine/pom.xml
+++ b/externals/kyuubi-flink-sql-engine/pom.xml
@@ -186,20 +186,15 @@
<shadedArtifactAttached>false</shadedArtifactAttached>
<artifactSet>
<includes>
- <include>org.apache.kyuubi:kyuubi-common_${scala.binary.version}</include>
- <include>org.apache.kyuubi:kyuubi-ha_${scala.binary.version}</include>
<include>com.fasterxml.jackson.core:*</include>
<include>com.fasterxml.jackson.module:*</include>
<include>com.google.guava:failureaccess</include>
<include>com.google.guava:guava</include>
<include>commons-codec:commons-codec</include>
<include>org.apache.commons:commons-lang3</include>
- <include>org.apache.curator:curator-client</include>
- <include>org.apache.curator:curator-framework</include>
- <include>org.apache.curator:curator-recipes</include>
<include>org.apache.hive:hive-service-rpc</include>
<include>org.apache.thrift:*</include>
- <include>org.apache.zookeeper:*</include>
+ <include>org.apache.kyuubi:*</include>
</includes>
</artifactSet>
<filters>
@@ -228,13 +223,6 @@
<include>com.fasterxml.jackson.**</include>
</includes>
</relocation>
- <relocation>
- <pattern>org.apache.curator</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.curator</shadedPattern>
- <includes>
- <include>org.apache.curator.**</include>
- </includes>
- </relocation>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.common</shadedPattern>
@@ -278,20 +266,6 @@
<include>org.apache.thrift.**</include>
</includes>
</relocation>
- <relocation>
- <pattern>org.apache.jute</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.jute</shadedPattern>
- <includes>
- <include>org.apache.jute.**</include>
- </includes>
- </relocation>
- <relocation>
- <pattern>org.apache.zookeeper</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.zookeeper</shadedPattern>
- <includes>
- <include>org.apache.zookeeper.**</include>
- </includes>
- </relocation>
</relocations>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"></transformer>
diff --git a/externals/kyuubi-hive-sql-engine/pom.xml b/externals/kyuubi-hive-sql-engine/pom.xml
index 0319d3dd2..6af94abe5 100644
--- a/externals/kyuubi-hive-sql-engine/pom.xml
+++ b/externals/kyuubi-hive-sql-engine/pom.xml
@@ -179,12 +179,7 @@
<include>com.fasterxml.jackson.core:jackson-core</include>
<include>com.fasterxml.jackson.core:jackson-databind</include>
<include>com.fasterxml.jackson.module:jackson-module-scala_${scala.binary.version}</include>
- <include>org.apache.kyuubi:kyuubi-common_${scala.binary.version}</include>
- <include>org.apache.kyuubi:kyuubi-events_${scala.binary.version}</include>
- <include>org.apache.kyuubi:kyuubi-ha_${scala.binary.version}</include>
- <include>org.apache.curator:curator-client</include>
- <include>org.apache.curator:curator-framework</include>
- <include>org.apache.curator:curator-recipes</include>
+ <include>org.apache.kyuubi:*</include>
</includes>
</artifactSet>
<filters>
@@ -205,15 +200,6 @@
</excludes>
</filter>
</filters>
- <relocations>
- <relocation>
- <pattern>org.apache.curator</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.curator</shadedPattern>
- <includes>
- <include>org.apache.curator.**</include>
- </includes>
- </relocation>
- </relocations>
</configuration>
<executions>
<execution>
diff --git a/externals/kyuubi-spark-sql-engine/pom.xml b/externals/kyuubi-spark-sql-engine/pom.xml
index 62fe39c00..67674e8c9 100644
--- a/externals/kyuubi-spark-sql-engine/pom.xml
+++ b/externals/kyuubi-spark-sql-engine/pom.xml
@@ -194,15 +194,9 @@
<include>io.perfmark:perfmark-api</include>
<include>io.vertx:*</include>
<include>net.jodah:failsafe</include>
- <include>org.apache.curator:curator-client</include>
- <include>org.apache.curator:curator-framework</include>
- <include>org.apache.curator:curator-recipes</include>
<include>org.apache.hive:hive-service-rpc</include>
- <include>org.apache.kyuubi:kyuubi-common_${scala.binary.version}</include>
- <include>org.apache.kyuubi:kyuubi-events_${scala.binary.version}</include>
- <include>org.apache.kyuubi:kyuubi-ha_${scala.binary.version}</include>
+ <include>org.apache.kyuubi:*</include>
<include>org.apache.thrift:*</include>
- <include>org.apache.zookeeper:zookeeper</include>
<include>org.checkerframework:checker-qual</include>
<include>org.codehaus.mojo:animal-sniffer-annotations</include>
</includes>
@@ -227,27 +221,6 @@
</filter>
</filters>
<relocations>
- <relocation>
- <pattern>org.apache.curator</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.curator</shadedPattern>
- <includes>
- <include>org.apache.curator.**</include>
- </includes>
- </relocation>
- <relocation>
- <pattern>org.apache.zookeeper</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.zookeeper</shadedPattern>
- <includes>
- <include>org.apache.zookeeper.**</include>
- </includes>
- </relocation>
- <relocation>
- <pattern>org.apache.jute</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.jute</shadedPattern>
- <includes>
- <include>org.apache.jute.**</include>
- </includes>
- </relocation>
<relocation>
<pattern>org.apache.hive.service.rpc.thrift</pattern>
<shadedPattern>${kyuubi.shade.packageName}.org.apache.hive.service.rpc.thrift</shadedPattern>
diff --git a/kyuubi-assembly/pom.xml b/kyuubi-assembly/pom.xml
index 0524470a2..1087201b1 100644
--- a/kyuubi-assembly/pom.xml
+++ b/kyuubi-assembly/pom.xml
@@ -69,28 +69,18 @@
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client-api</artifactId>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client-runtime</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-client</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
</dependency>
<dependency>
diff --git a/kyuubi-ctl/pom.xml b/kyuubi-ctl/pom.xml
index eb4060ffd..da1a5777a 100644
--- a/kyuubi-ctl/pom.xml
+++ b/kyuubi-ctl/pom.xml
@@ -48,6 +48,11 @@
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
@@ -60,16 +65,6 @@
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- </dependency>
-
<dependency>
<groupId>com.github.scopt</groupId>
<artifactId>scopt_${scala.binary.version}</artifactId>
@@ -86,11 +81,6 @@
<version>${snakeyaml.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-common_${scala.binary.version}</artifactId>
diff --git a/kyuubi-ha/pom.xml b/kyuubi-ha/pom.xml
index b4605b6a1..cf605b37f 100644
--- a/kyuubi-ha/pom.xml
+++ b/kyuubi-ha/pom.xml
@@ -38,18 +38,8 @@
</dependency>
<dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
</dependency>
<dependency>
diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/DiscoveryPaths.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/DiscoveryPaths.scala
index 987a88dda..fe7ebe2ab 100644
--- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/DiscoveryPaths.scala
+++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/DiscoveryPaths.scala
@@ -17,7 +17,7 @@
package org.apache.kyuubi.ha.client
-import org.apache.curator.utils.ZKPaths
+import org.apache.kyuubi.shaded.curator.utils.ZKPaths
object DiscoveryPaths {
def makePath(parent: String, firstChild: String, restChildren: String*): String = {
diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperACLProvider.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperACLProvider.scala
index 467c323b7..87ea65c17 100644
--- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperACLProvider.scala
+++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperACLProvider.scala
@@ -17,13 +17,12 @@
package org.apache.kyuubi.ha.client.zookeeper
-import org.apache.curator.framework.api.ACLProvider
-import org.apache.zookeeper.ZooDefs
-import org.apache.zookeeper.data.ACL
-
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.ha.HighAvailabilityConf
import org.apache.kyuubi.ha.client.AuthTypes
+import org.apache.kyuubi.shaded.curator.framework.api.ACLProvider
+import org.apache.kyuubi.shaded.zookeeper.ZooDefs
+import org.apache.kyuubi.shaded.zookeeper.data.ACL
class ZookeeperACLProvider(conf: KyuubiConf) extends ACLProvider {
diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala
index 806aff95f..d9161f3cd 100644
--- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala
+++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala
@@ -24,8 +24,6 @@ import javax.security.auth.login.Configuration
import scala.util.Random
import com.google.common.annotations.VisibleForTesting
-import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
-import org.apache.curator.retry._
import org.apache.hadoop.security.UserGroupInformation
import org.apache.kyuubi.Logging
@@ -34,6 +32,8 @@ import org.apache.kyuubi.ha.HighAvailabilityConf._
import org.apache.kyuubi.ha.client.{AuthTypes, RetryPolicies}
import org.apache.kyuubi.ha.client.RetryPolicies._
import org.apache.kyuubi.reflection.DynConstructors
+import org.apache.kyuubi.shaded.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
+import org.apache.kyuubi.shaded.curator.retry._
import org.apache.kyuubi.util.KyuubiHadoopUtils
object ZookeeperClientProvider extends Logging {
diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClient.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClient.scala
index daa27047e..33aa5149d 100644
--- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClient.scala
+++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClient.scala
@@ -25,23 +25,6 @@ import java.util.concurrent.atomic.AtomicBoolean
import scala.collection.JavaConverters._
import com.google.common.annotations.VisibleForTesting
-import org.apache.curator.framework.CuratorFramework
-import org.apache.curator.framework.recipes.atomic.{AtomicValue, DistributedAtomicInteger}
-import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex
-import org.apache.curator.framework.recipes.nodes.PersistentNode
-import org.apache.curator.framework.state.ConnectionState
-import org.apache.curator.framework.state.ConnectionState.CONNECTED
-import org.apache.curator.framework.state.ConnectionState.LOST
-import org.apache.curator.framework.state.ConnectionState.RECONNECTED
-import org.apache.curator.framework.state.ConnectionStateListener
-import org.apache.curator.retry.RetryForever
-import org.apache.curator.utils.ZKPaths
-import org.apache.zookeeper.CreateMode
-import org.apache.zookeeper.CreateMode.PERSISTENT
-import org.apache.zookeeper.KeeperException
-import org.apache.zookeeper.KeeperException.NodeExistsException
-import org.apache.zookeeper.WatchedEvent
-import org.apache.zookeeper.Watcher
import org.apache.kyuubi.KYUUBI_VERSION
import org.apache.kyuubi.KyuubiException
@@ -58,6 +41,23 @@ import org.apache.kyuubi.ha.client.ServiceNodeInfo
import org.apache.kyuubi.ha.client.zookeeper.ZookeeperClientProvider.buildZookeeperClient
import org.apache.kyuubi.ha.client.zookeeper.ZookeeperClientProvider.getGracefulStopThreadDelay
import org.apache.kyuubi.ha.client.zookeeper.ZookeeperDiscoveryClient.connectionChecker
+import org.apache.kyuubi.shaded.curator.framework.CuratorFramework
+import org.apache.kyuubi.shaded.curator.framework.recipes.atomic.{AtomicValue, DistributedAtomicInteger}
+import org.apache.kyuubi.shaded.curator.framework.recipes.locks.InterProcessSemaphoreMutex
+import org.apache.kyuubi.shaded.curator.framework.recipes.nodes.PersistentNode
+import org.apache.kyuubi.shaded.curator.framework.state.ConnectionState
+import org.apache.kyuubi.shaded.curator.framework.state.ConnectionState.CONNECTED
+import org.apache.kyuubi.shaded.curator.framework.state.ConnectionState.LOST
+import org.apache.kyuubi.shaded.curator.framework.state.ConnectionState.RECONNECTED
+import org.apache.kyuubi.shaded.curator.framework.state.ConnectionStateListener
+import org.apache.kyuubi.shaded.curator.retry.RetryForever
+import org.apache.kyuubi.shaded.curator.utils.ZKPaths
+import org.apache.kyuubi.shaded.zookeeper.CreateMode
+import org.apache.kyuubi.shaded.zookeeper.CreateMode.PERSISTENT
+import org.apache.kyuubi.shaded.zookeeper.KeeperException
+import org.apache.kyuubi.shaded.zookeeper.KeeperException.NodeExistsException
+import org.apache.kyuubi.shaded.zookeeper.WatchedEvent
+import org.apache.kyuubi.shaded.zookeeper.Watcher
import org.apache.kyuubi.util.ThreadUtils
class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient {
diff --git a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClientSuite.scala b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClientSuite.scala
index e3f5546a9..de849f83e 100644
--- a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClientSuite.scala
+++ b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClientSuite.scala
@@ -25,11 +25,7 @@ import javax.security.auth.login.Configuration
import scala.collection.JavaConverters._
-import org.apache.curator.framework.CuratorFrameworkFactory
-import org.apache.curator.retry.ExponentialBackoffRetry
import org.apache.hadoop.util.StringUtils
-import org.apache.zookeeper.ZooDefs
-import org.apache.zookeeper.data.ACL
import org.scalatest.time.SpanSugar._
import org.apache.kyuubi.{KerberizedTestHelper, KYUUBI_VERSION}
@@ -39,6 +35,10 @@ import org.apache.kyuubi.ha.client._
import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient
import org.apache.kyuubi.ha.client.zookeeper.ZookeeperClientProvider._
import org.apache.kyuubi.service._
+import org.apache.kyuubi.shaded.curator.framework.CuratorFrameworkFactory
+import org.apache.kyuubi.shaded.curator.retry.ExponentialBackoffRetry
+import org.apache.kyuubi.shaded.zookeeper.ZooDefs
+import org.apache.kyuubi.shaded.zookeeper.data.ACL
import org.apache.kyuubi.zookeeper.EmbeddedZookeeper
import org.apache.kyuubi.zookeeper.ZookeeperConf.ZK_CLIENT_PORT
diff --git a/kyuubi-hive-jdbc-shaded/pom.xml b/kyuubi-hive-jdbc-shaded/pom.xml
index 1a6f258b0..e0e390b75 100644
--- a/kyuubi-hive-jdbc-shaded/pom.xml
+++ b/kyuubi-hive-jdbc-shaded/pom.xml
@@ -108,10 +108,6 @@
<pattern>org.apache.commons</pattern>
<shadedPattern>${kyuubi.shade.packageName}.org.apache.commons</shadedPattern>
</relocation>
- <relocation>
- <pattern>org.apache.curator</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.curator</shadedPattern>
- </relocation>
<relocation>
<pattern>org.apache.hive</pattern>
<shadedPattern>${kyuubi.shade.packageName}.org.apache.hive</shadedPattern>
@@ -120,18 +116,10 @@
<pattern>org.apache.http</pattern>
<shadedPattern>${kyuubi.shade.packageName}.org.apache.http</shadedPattern>
</relocation>
- <relocation>
- <pattern>org.apache.jute</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.jute</shadedPattern>
- </relocation>
<relocation>
<pattern>org.apache.thrift</pattern>
<shadedPattern>${kyuubi.shade.packageName}.org.apache.thrift</shadedPattern>
</relocation>
- <relocation>
- <pattern>org.apache.zookeeper</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.zookeeper</shadedPattern>
- </relocation>
</relocations>
</configuration>
<executions>
diff --git a/kyuubi-hive-jdbc/pom.xml b/kyuubi-hive-jdbc/pom.xml
index 36ea7acc2..d6889e365 100644
--- a/kyuubi-hive-jdbc/pom.xml
+++ b/kyuubi-hive-jdbc/pom.xml
@@ -102,24 +102,14 @@
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-client</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
</dependency>
<dependency>
diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/ZooKeeperHiveClientHelper.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/ZooKeeperHiveClientHelper.java
index 41fadfa2f..bfa5e632e 100644
--- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/ZooKeeperHiveClientHelper.java
+++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/ZooKeeperHiveClientHelper.java
@@ -25,9 +25,9 @@ import java.util.Map;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.kyuubi.shaded.curator.framework.CuratorFramework;
+import org.apache.kyuubi.shaded.curator.framework.CuratorFrameworkFactory;
+import org.apache.kyuubi.shaded.curator.retry.ExponentialBackoffRetry;
class ZooKeeperHiveClientHelper {
// Pattern for key1=value1;key2=value2
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/AdminResource.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/AdminResource.scala
index cf8478021..fc271b058 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/AdminResource.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/AdminResource.scala
@@ -28,7 +28,6 @@ import io.swagger.v3.oas.annotations.media.{ArraySchema, Content, Schema}
import io.swagger.v3.oas.annotations.responses.ApiResponse
import io.swagger.v3.oas.annotations.tags.Tag
import org.apache.commons.lang3.StringUtils
-import org.apache.zookeeper.KeeperException.NoNodeException
import org.apache.kyuubi.{KYUUBI_VERSION, Logging, Utils}
import org.apache.kyuubi.client.api.v1.dto.{Engine, OperationData, ServerData, SessionData}
@@ -41,6 +40,7 @@ import org.apache.kyuubi.operation.{KyuubiOperation, OperationHandle}
import org.apache.kyuubi.server.KyuubiServer
import org.apache.kyuubi.server.api.{ApiRequestContext, ApiUtils}
import org.apache.kyuubi.session.{KyuubiSession, SessionHandle}
+import org.apache.kyuubi.shaded.zookeeper.KeeperException.NoNodeException
@Tag(name = "Admin")
@Produces(Array(MediaType.APPLICATION_JSON))
diff --git a/kyuubi-zookeeper/pom.xml b/kyuubi-zookeeper/pom.xml
index cd3e017cd..e10ac2d9d 100644
--- a/kyuubi-zookeeper/pom.xml
+++ b/kyuubi-zookeeper/pom.xml
@@ -37,8 +37,8 @@
</dependency>
<dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
</dependency>
<dependency>
@@ -48,24 +48,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-client</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <scope>test</scope>
- </dependency>
</dependencies>
<build>
diff --git a/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala b/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala
index 0e980132a..04457f3ce 100644
--- a/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala
+++ b/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala
@@ -20,11 +20,10 @@ package org.apache.kyuubi.zookeeper
import java.io.File
import java.net.InetSocketAddress
-import org.apache.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer}
-
import org.apache.kyuubi.Utils._
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.service.{AbstractService, ServiceState}
+import org.apache.kyuubi.shaded.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer}
import org.apache.kyuubi.zookeeper.ZookeeperConf._
class EmbeddedZookeeper extends AbstractService("EmbeddedZookeeper") {
diff --git a/kyuubi-zookeeper/src/test/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeperSuite.scala b/kyuubi-zookeeper/src/test/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeperSuite.scala
index 518aa48b9..69e798ac5 100644
--- a/kyuubi-zookeeper/src/test/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeperSuite.scala
+++ b/kyuubi-zookeeper/src/test/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeperSuite.scala
@@ -17,12 +17,11 @@
package org.apache.kyuubi.zookeeper
-import org.apache.curator.framework.CuratorFrameworkFactory
-import org.apache.curator.framework.imps.CuratorFrameworkState
-import org.apache.curator.retry.ExponentialBackoffRetry
-
import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.shaded.curator.framework.CuratorFrameworkFactory
+import org.apache.kyuubi.shaded.curator.framework.imps.CuratorFrameworkState
+import org.apache.kyuubi.shaded.curator.retry.ExponentialBackoffRetry
import org.apache.kyuubi.zookeeper.ZookeeperConf.{ZK_CLIENT_PORT, ZK_CLIENT_PORT_ADDRESS}
class EmbeddedZookeeperSuite extends KyuubiFunSuite {
diff --git a/pom.xml b/pom.xml
index 62866fe57..7996fa7a9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -129,7 +129,6 @@
<commons-io.version>2.11.0</commons-io.version>
<commons-lang.version>2.6</commons-lang.version>
<commons-lang3.version>3.12.0</commons-lang3.version>
- <curator.version>2.12.0</curator.version>
<etcd.version>0.7.3</etcd.version>
<delta.version>2.3.0</delta.version>
<failsafe.verion>2.4.4</failsafe.verion>
@@ -168,6 +167,8 @@
<kafka.version>3.4.0</kafka.version>
<kubernetes-client.version>6.4.1</kubernetes-client.version>
<kudu.version>1.15.0</kudu.version>
+ <kyuubi-shaded-zookeeper.artifacts>kyuubi-shaded-zookeeper-34</kyuubi-shaded-zookeeper.artifacts>
+ <kyuubi-shaded-zookeeper.version>0.1.0</kyuubi-shaded-zookeeper.version>
<ldapsdk.version>6.0.5</ldapsdk.version>
<log4j.version>2.20.0</log4j.version>
<mysql.jdbc.version>8.0.32</mysql.jdbc.version>
@@ -203,7 +204,6 @@
<trino.client.version>363</trino.client.version>
<trino.tpcds.version>1.4</trino.tpcds.version>
<trino.tpch.version>1.1</trino.tpch.version>
- <zookeeper.version>3.4.14</zookeeper.version>
<!-- webui -->
<webui.skip>true</webui.skip>
@@ -260,6 +260,11 @@
<dependencyManagement>
<dependencies>
+ <dependency>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>${kyuubi-shaded-zookeeper.artifacts}</artifactId>
+ <version>${kyuubi-shaded-zookeeper.version}</version>
+ </dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
@@ -609,54 +614,6 @@
</exclusions>
</dependency>
- <!--
- Do not add hive dependencies here if they are dependencies of
- org.apache.spark:spark-hive_${scala.binary.version}.
- We should leave these dependencies' versions as they are defined in
- spark-parent_${scala.binary.version}-${spark.version}.pom.
- Otherwise, kyuubi-spark-sql-engine tests may encounter hive version compatibility issue.
- -->
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <version>${curator.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <version>${curator.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-client</artifactId>
- <version>${curator.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
@@ -1116,18 +1073,6 @@
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- <version>${zookeeper.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<dependency>
<groupId>io.etcd</groupId>
<artifactId>jetcd-core</artifactId>
@@ -2232,6 +2177,13 @@
</properties>
</profile>
+ <profile>
+ <id>zookeeper-3.6</id>
+ <properties>
+ <kyuubi-shaded-zookeeper.artifacts>kyuubi-shaded-zookeeper-36</kyuubi-shaded-zookeeper.artifacts>
+ </properties>
+ </profile>
+
<profile>
<id>spark-provided</id>
<properties>
diff --git a/scalastyle-config.xml b/scalastyle-config.xml
index f7d0d2449..5eaa9cef5 100644
--- a/scalastyle-config.xml
+++ b/scalastyle-config.xml
@@ -195,6 +195,14 @@ This file is divided into 3 sections:
of Commons Lang 2 (package org.apache.commons.lang.*)</customMessage>
</check>
+ <check customId="kyuubishaded" level="error" class="org.scalastyle.scalariform.TokenChecker" enabled="true">
+ <parameters>
+ <parameter name="regex">org\.apache\.curator\.</parameter>
+ <parameter name="regex">org\.apache\.zookeeper\.</parameter>
+ </parameters>
+ <customMessage>Use Kyuubi shaded classes (package org.apache.kyuubi.shaded.*) instead classes</customMessage>
+ </check>
+
<check customId="extractopt" level="error" class="org.scalastyle.scalariform.TokenChecker" enabled="true">
<parameters><parameter name="regex">extractOpt</parameter></parameters>
<customMessage>Use Utils.jsonOption(x).map(.extract[T]) instead of .extractOpt[T], as the latter