You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2022/07/02 14:34:17 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #2987] Remove Hive shims-common and shims-0.23 dependencies from Kyuubi Hive JDBC
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new d8d6903f5 [KYUUBI #2987] Remove Hive shims-common and shims-0.23 dependencies from Kyuubi Hive JDBC
d8d6903f5 is described below
commit d8d6903f5dd7d170d6fc4ae0c16f01288fa4d98b
Author: jiaoqingbo <11...@qq.com>
AuthorDate: Sat Jul 2 22:34:09 2022 +0800
[KYUUBI #2987] Remove Hive shims-common and shims-0.23 dependencies from Kyuubi Hive JDBC
### _Why are the changes needed?_
fix #2987
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #2992 from jiaoqingbo/kyuubi-2987.
Closes #2987
97e25292 [jiaoqingbo] [KYUUBI #2987] Remove Hive shims-common and shims-0.23 dependencies from Kyuubi Hive JDBC
Authored-by: jiaoqingbo <11...@qq.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
kyuubi-hive-jdbc-shaded/pom.xml | 38 ---------
kyuubi-hive-jdbc/pom.xml | 24 ------
.../jdbc/hive/auth/HadoopThriftAuthBridge.java | 1 -
.../kyuubi/jdbc/hive/auth/TFilterTransport.java | 98 ++++++++++++++++++++++
.../jdbc/hive/auth/TSubjectAssumingTransport.java | 1 -
...ngTransport.java => TUGIAssumingTransport.java} | 32 +++----
6 files changed, 115 insertions(+), 79 deletions(-)
diff --git a/kyuubi-hive-jdbc-shaded/pom.xml b/kyuubi-hive-jdbc-shaded/pom.xml
index b7eebfac2..843a12fbe 100644
--- a/kyuubi-hive-jdbc-shaded/pom.xml
+++ b/kyuubi-hive-jdbc-shaded/pom.xml
@@ -60,32 +60,6 @@
<artifactId>hive-service-rpc</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.hive.shims</groupId>
- <artifactId>hive-shims-common</artifactId>
- <version>${hive.version}</version>
- <optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hive.shims</groupId>
- <artifactId>hive-shims-0.23</artifactId>
- <version>${hive.version}</version>
- <optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-common</artifactId>
@@ -186,18 +160,6 @@
<include>**</include>
</includes>
</filter>
- <filter>
- <artifact>org.apache.hive.shims:hive-shims-common</artifact>
- <includes>
- <include>**</include>
- </includes>
- </filter>
- <filter>
- <artifact>org.apache.hive.shims:hive-shims-0.23</artifact>
- <includes>
- <include>**</include>
- </includes>
- </filter>
<filter>
<artifact>org.apache.hive:hive-common</artifact>
<includes>
diff --git a/kyuubi-hive-jdbc/pom.xml b/kyuubi-hive-jdbc/pom.xml
index d345272e8..8fd938d25 100644
--- a/kyuubi-hive-jdbc/pom.xml
+++ b/kyuubi-hive-jdbc/pom.xml
@@ -64,30 +64,6 @@
<artifactId>hive-service-rpc</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.hive.shims</groupId>
- <artifactId>hive-shims-common</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hive.shims</groupId>
- <artifactId>hive-shims-0.23</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-common</artifactId>
diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/HadoopThriftAuthBridge.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/HadoopThriftAuthBridge.java
index 2f5068793..097aaaefa 100644
--- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/HadoopThriftAuthBridge.java
+++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/HadoopThriftAuthBridge.java
@@ -29,7 +29,6 @@ import javax.security.sasl.RealmChoiceCallback;
import javax.security.sasl.SaslException;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport;
import org.apache.hadoop.security.SaslRpcServer;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.SecurityUtil;
diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/TFilterTransport.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/TFilterTransport.java
new file mode 100644
index 000000000..1c7da82fe
--- /dev/null
+++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/TFilterTransport.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.jdbc.hive.auth;
+
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * Transport that simply wraps another transport. This is the equivalent of FilterInputStream for
+ * Thrift transports.
+ */
+public class TFilterTransport extends TTransport {
+ protected final TTransport wrapped;
+
+ public TFilterTransport(TTransport wrapped) {
+ this.wrapped = wrapped;
+ }
+
+ @Override
+ public void open() throws TTransportException {
+ wrapped.open();
+ }
+
+ @Override
+ public boolean isOpen() {
+ return wrapped.isOpen();
+ }
+
+ @Override
+ public boolean peek() {
+ return wrapped.peek();
+ }
+
+ @Override
+ public void close() {
+ wrapped.close();
+ }
+
+ @Override
+ public int read(byte[] buf, int off, int len) throws TTransportException {
+ return wrapped.read(buf, off, len);
+ }
+
+ @Override
+ public int readAll(byte[] buf, int off, int len) throws TTransportException {
+ return wrapped.readAll(buf, off, len);
+ }
+
+ @Override
+ public void write(byte[] buf) throws TTransportException {
+ wrapped.write(buf);
+ }
+
+ @Override
+ public void write(byte[] buf, int off, int len) throws TTransportException {
+ wrapped.write(buf, off, len);
+ }
+
+ @Override
+ public void flush() throws TTransportException {
+ wrapped.flush();
+ }
+
+ @Override
+ public byte[] getBuffer() {
+ return wrapped.getBuffer();
+ }
+
+ @Override
+ public int getBufferPosition() {
+ return wrapped.getBufferPosition();
+ }
+
+ @Override
+ public int getBytesRemainingInBuffer() {
+ return wrapped.getBytesRemainingInBuffer();
+ }
+
+ @Override
+ public void consumeBuffer(int len) {
+ wrapped.consumeBuffer(len);
+ }
+}
diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/TSubjectAssumingTransport.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/TSubjectAssumingTransport.java
index 96781cb12..1c0f54201 100644
--- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/TSubjectAssumingTransport.java
+++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/TSubjectAssumingTransport.java
@@ -22,7 +22,6 @@ import java.security.AccessController;
import java.security.PrivilegedActionException;
import java.security.PrivilegedExceptionAction;
import javax.security.auth.Subject;
-import org.apache.hadoop.hive.thrift.TFilterTransport;
import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportException;
diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/TSubjectAssumingTransport.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/TUGIAssumingTransport.java
similarity index 68%
copy from kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/TSubjectAssumingTransport.java
copy to kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/TUGIAssumingTransport.java
index 96781cb12..50929edf0 100644
--- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/TSubjectAssumingTransport.java
+++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/TUGIAssumingTransport.java
@@ -17,38 +17,38 @@
package org.apache.kyuubi.jdbc.hive.auth;
-import java.security.AccessControlContext;
-import java.security.AccessController;
-import java.security.PrivilegedActionException;
+import java.io.IOException;
import java.security.PrivilegedExceptionAction;
-import javax.security.auth.Subject;
-import org.apache.hadoop.hive.thrift.TFilterTransport;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportException;
/**
- * This is used on the client side, where the API explicitly opens a transport to the server using
- * the Subject.doAs().
+ * The Thrift SASL transports call Sasl.createSaslServer and Sasl.createSaslClient inside open().
+ * So, we need to assume the correct UGI when the transport is opened so that the SASL mechanisms
+ * have access to the right principal. This transport wraps the Sasl transports to set up the right
+ * UGI context for open().
+ *
+ * <p>This is used on the client side, where the API explicitly opens a transport to the server.
*/
-public class TSubjectAssumingTransport extends TFilterTransport {
+public class TUGIAssumingTransport extends TFilterTransport {
+ protected UserGroupInformation ugi;
- public TSubjectAssumingTransport(TTransport wrapped) {
+ public TUGIAssumingTransport(TTransport wrapped, UserGroupInformation ugi) {
super(wrapped);
+ this.ugi = ugi;
}
@Override
public void open() throws TTransportException {
try {
- AccessControlContext context = AccessController.getContext();
- Subject subject = Subject.getSubject(context);
- Subject.doAs(
- subject,
+ ugi.doAs(
new PrivilegedExceptionAction<Void>() {
public Void run() {
try {
wrapped.open();
} catch (TTransportException tte) {
- // Wrap the transport exception in an RTE, since Subject.doAs() then goes
+ // Wrap the transport exception in an RTE, since UGI.doAs() then goes
// and unwraps this for us out of the doAs block. We then unwrap one
// more time in our catch clause to get back the TTE. (ugh)
throw new RuntimeException(tte);
@@ -56,8 +56,10 @@ public class TSubjectAssumingTransport extends TFilterTransport {
return null;
}
});
- } catch (PrivilegedActionException ioe) {
+ } catch (IOException ioe) {
throw new RuntimeException("Received an ioe we never threw!", ioe);
+ } catch (InterruptedException ie) {
+ throw new RuntimeException("Received an ie we never threw!", ie);
} catch (RuntimeException rte) {
if (rte.getCause() instanceof TTransportException) {
throw (TTransportException) rte.getCause();