You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2021/10/15 07:27:35 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #1236] Add module
kyuubi-hive-jdbc-shaded to resolve IDEA-93855
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 86b8e1c [KYUUBI #1236] Add module kyuubi-hive-jdbc-shaded to resolve IDEA-93855
86b8e1c is described below
commit 86b8e1ce62b4c5a9f0f6f4bf291e17e9bfde2c10
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Fri Oct 15 15:27:26 2021 +0800
[KYUUBI #1236] Add module kyuubi-hive-jdbc-shaded to resolve IDEA-93855
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
The idea based on the analysis of https://github.com/apache/incubator-kyuubi/issues/1131#issuecomment-943163597
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [x] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #1236 from pan3793/shade.
Closes #1236
7a4e3ccc [Cheng Pan] Add module kyuubi-hive-jdbc-shaded to resolve IDEA-93855
Authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
externals/kyuubi-spark-sql-engine/pom.xml | 2 +-
.../spark/operation/SparkOperationSuite.scala | 2 +-
kubernetes/integration-tests/pom.xml | 2 +-
.../pom.xml | 25 +-
kyuubi-hive-jdbc/pom.xml | 271 +--------------------
kyuubi-server/pom.xml | 2 +-
pom.xml | 1 +
7 files changed, 28 insertions(+), 277 deletions(-)
diff --git a/externals/kyuubi-spark-sql-engine/pom.xml b/externals/kyuubi-spark-sql-engine/pom.xml
index b2aec51..a82b09c 100644
--- a/externals/kyuubi-spark-sql-engine/pom.xml
+++ b/externals/kyuubi-spark-sql-engine/pom.xml
@@ -101,7 +101,7 @@
<dependency>
<groupId>org.apache.kyuubi</groupId>
- <artifactId>kyuubi-hive-jdbc</artifactId>
+ <artifactId>kyuubi-hive-jdbc-shaded</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala
index a31e4f6..38ec6b0 100644
--- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala
+++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala
@@ -404,7 +404,7 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveJDBCTests {
assert(metaData.allTablesAreSelectable)
assert(metaData.getDatabaseProductName === "Spark SQL")
assert(metaData.getDatabaseProductVersion === KYUUBI_VERSION)
- assert(metaData.getDriverName === "Kyuubi Project Hive JDBC Client")
+ assert(metaData.getDriverName === "Kyuubi Project Hive JDBC Shaded Client")
assert(metaData.getDriverVersion === KYUUBI_VERSION)
assert(metaData.getDatabaseMajorVersion === Utils.majorVersion(KYUUBI_VERSION))
assert(metaData.getDatabaseMinorVersion === Utils.minorVersion(KYUUBI_VERSION))
diff --git a/kubernetes/integration-tests/pom.xml b/kubernetes/integration-tests/pom.xml
index 10a7f52..7f2d348 100644
--- a/kubernetes/integration-tests/pom.xml
+++ b/kubernetes/integration-tests/pom.xml
@@ -67,7 +67,7 @@
<!-- for hive driver related dependency -->
<dependency>
<groupId>org.apache.kyuubi</groupId>
- <artifactId>kyuubi-hive-jdbc</artifactId>
+ <artifactId>kyuubi-hive-jdbc-shaded</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
diff --git a/kyuubi-hive-jdbc/pom.xml b/kyuubi-hive-jdbc-shaded/pom.xml
similarity index 95%
copy from kyuubi-hive-jdbc/pom.xml
copy to kyuubi-hive-jdbc-shaded/pom.xml
index 36bd556..ae1bacb 100644
--- a/kyuubi-hive-jdbc/pom.xml
+++ b/kyuubi-hive-jdbc-shaded/pom.xml
@@ -29,12 +29,19 @@
<!--
A scala-free module in runtime scope.
-->
- <artifactId>kyuubi-hive-jdbc</artifactId>
- <name>Kyuubi Project Hive JDBC Client</name>
+ <artifactId>kyuubi-hive-jdbc-shaded</artifactId>
+ <name>Kyuubi Project Hive JDBC Shaded Client</name>
<packaging>jar</packaging>
<dependencies>
<dependency>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>kyuubi-hive-jdbc</artifactId>
+ <version>${project.version}</version>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
<version>${hive.jdbc.storage-api.version}</version>
@@ -262,6 +269,9 @@
</dependencies>
<build>
+ <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
+ <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -279,6 +289,15 @@
</artifactSet>
<filters>
<filter>
+ <artifact>org.apache.kyuubi:kyuubi-hive-jdbc</artifact>
+ <includes>
+ <include>**</include>
+ </includes>
+ <excludes>
+ <exclude>META-INF/MANIFEST.MF</exclude>
+ </excludes>
+ </filter>
+ <filter>
<artifact>commons-codec:commons-codec</artifact>
<excludes>
<exclude>META-INF/MANIFEST.MF</exclude>
@@ -469,8 +488,6 @@
</relocations>
<transformers>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- <transformer
implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
diff --git a/kyuubi-hive-jdbc/pom.xml b/kyuubi-hive-jdbc/pom.xml
index 36bd556..8ab3f6f 100644
--- a/kyuubi-hive-jdbc/pom.xml
+++ b/kyuubi-hive-jdbc/pom.xml
@@ -38,7 +38,6 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
<version>${hive.jdbc.storage-api.version}</version>
- <optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -51,7 +50,6 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-service-rpc</artifactId>
<version>${hive.jdbc.version}</version>
- <optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -64,21 +62,18 @@
<groupId>org.apache.hive.shims</groupId>
<artifactId>hive-shims-common</artifactId>
<version>${hive.jdbc.version}</version>
- <optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.hive.shims</groupId>
<artifactId>hive-shims-0.23</artifactId>
<version>${hive.jdbc.version}</version>
- <optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-common</artifactId>
<version>${hive.jdbc.version}</version>
- <optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -91,7 +86,6 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
<version>${hive.jdbc.version}</version>
- <optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -104,7 +98,6 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-serde</artifactId>
<version>${hive.jdbc.version}</version>
- <optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -117,7 +110,6 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-service</artifactId>
<version>${hive.jdbc.version}</version>
- <optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -130,7 +122,6 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-jdbc</artifactId>
<version>${hive.jdbc.version}</version>
- <optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -143,28 +134,24 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>${hive.jdbc.commons-codec.version}</version>
- <optional>true</optional>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>${hive.jdbc.commons-lang.version}</version>
- <optional>true</optional>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${hive.jdbc.guava.version}</version>
- <optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<version>${hive.jdbc.curator.version}</version>
- <optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -177,7 +164,6 @@
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
<version>${hive.jdbc.curator.version}</version>
- <optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -190,7 +176,6 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${hive.jdbc.httpcomponents-client.version}</version>
- <optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -203,14 +188,12 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>${hive.jdbc.httpcomponents-core.version}</version>
- <optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
<version>${hive.jdbc.thrift.version}</version>
- <optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -223,7 +206,6 @@
<groupId>org.apache.thrift</groupId>
<artifactId>libfb303</artifactId>
<version>${hive.jdbc.fb303.version}</version>
- <optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -236,7 +218,6 @@
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>${hive.jdbc.zookeeper.version}</version>
- <optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -262,255 +243,7 @@
</dependencies>
<build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <configuration>
- <minimizeJar>true</minimizeJar>
- <createSourcesJar>true</createSourcesJar>
- <shadeSourcesContent>true</shadeSourcesContent>
- <shadedArtifactAttached>false</shadedArtifactAttached>
- <artifactSet>
- <excludes>
- <exclude>org.apache.hadoop:*</exclude>
- <exclude>org.slf4j:*</exclude>
- </excludes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>commons-codec:commons-codec</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>commons-lang:commons-lang</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>com.google.guava:guava</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.hive:hive-metastore</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- <exclude>package.jdo</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.hive:hive-storage-api</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.hive:hive-service-rpc</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.hive:hive-shims</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.hive.shims:hive-shims-common</artifact>
- <includes>
- <include>**</include>
- </includes>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.hive.shims:hive-shims-0.23</artifact>
- <includes>
- <include>**</include>
- </includes>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.hive:hive-common</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.hive:hive-serde</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.hive:hive-service</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- <exclude>hive-webapps/**</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.hive:hive-jdbc</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- <exclude>META-INF/services/*</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.curator:curator-client</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.curator:curator-framework</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.httpcomponents:httpclient</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.httpcomponents:httpcore</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.thrift:libfb303</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.thrift:libthrift</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.zookeeper:zookeeper</artifact>
- <excludes>
- <exclude>META-INF/MANIFEST.MF</exclude>
- <exclude>LICENSE.txt</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- <exclude>META-INF/DEPENDENCIES</exclude>
- <exclude>META-INF/maven/**</exclude>
- <exclude>mozilla/**</exclude>
- <exclude>*.properties</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>com.facebook</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.com.facebook</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.com.google.common</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.commons.codec</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.commons.codec</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.commons.lang</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.commons.lang</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.curator</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.curator</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.hadoop.hive</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.hadoop.hive</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.hive</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.hive</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.http</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.http</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.jute</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.jute</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.thrift</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.thrift</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.zookeeper</pattern>
- <shadedPattern>${kyuubi.shade.packageName}.org.apache.zookeeper</shadedPattern>
- </relocation>
- </relocations>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
- </transformers>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <!--
- This is to ensure references to shaded Hive classes can be resolved in IDEs such as Intellij.
- For reference: https://youtrack.jetbrains.com/issue/IDEA-126596
- -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>compile</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>${basedir}/target/${project.artifactId}-${project.version}.jar</file>
- <type>jar</type>
- <classifier>optional</classifier>
- </artifact>
- </artifacts>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
+ <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
+ <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
</build>
</project>
diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml
index 8c12c2e..4ef323c 100644
--- a/kyuubi-server/pom.xml
+++ b/kyuubi-server/pom.xml
@@ -172,7 +172,7 @@
<dependency>
<groupId>org.apache.kyuubi</groupId>
- <artifactId>kyuubi-hive-jdbc</artifactId>
+ <artifactId>kyuubi-hive-jdbc-shaded</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
diff --git a/pom.xml b/pom.xml
index df9f65a..bd7f8eb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -73,6 +73,7 @@
<module>kyuubi-ctl</module>
<module>kyuubi-ha</module>
<module>kyuubi-hive-jdbc</module>
+ <module>kyuubi-hive-jdbc-shaded</module>
<module>kyuubi-metrics</module>
<module>kyuubi-server</module>
<module>kyuubi-zookeeper</module>