You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2024/02/29 05:50:20 UTC
(kyuubi-shaded) branch master updated: [KYUUBI-SHADED #35] Create a light shaded HiveMetaStoreClient for getting DelegationToken
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi-shaded.git
The following commit(s) were added to refs/heads/master by this push:
new d31a830 [KYUUBI-SHADED #35] Create a light shaded HiveMetaStoreClient for getting DelegationToken
d31a830 is described below
commit d31a83059082d100a01659337934450fc91970d4
Author: zhouyifan279 <zh...@gmail.com>
AuthorDate: Thu Feb 29 13:50:09 2024 +0800
[KYUUBI-SHADED #35] Create a light shaded HiveMetaStoreClient for getting DelegationToken
### _Why are the changes needed?_
org.apache.thrift:libthrift:0.9.3 has serveral CVEs:
- CVE-2020-13949 - THRIFT-5237(fixed in 0.14.0) - https://github.com/apache/thrift/pull/2191
- CVE-2019-0205 - THRIFT-4053(fixed in 0.11.0) - https://github.com/apache/thrift/pull/1371
- CVE-2018-1320 - THRIFT-4506(fixed in 0.9.3.1)
HiveMetaStoreClient of Hive 2.3.9 and 3.1.3 depends on libthrift:0.9.3 and Kyuubi currently uses it to get HMS delegation token.
As Kyuubi only use HiveMetaStoreClient to get delegation token, we think it is better to create a lightweight HiveMetaStoreClient with only the necessary api so that we can:
- Decouple Kyuubi's libthrift version from Hive
- Remove unnessary dependencies introduced by vanilla HiveMetaStoreClient
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [x] Add screenshots for manual tests if appropriate
<img width="1483" alt="image" src="https://github.com/apache/kyuubi-shaded/assets/88070094/e3198035-6db4-46b1-a47c-db66cb9a9acb">
- [ ] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #35 from zhouyifan279/hive-metastore-client.
b1654cb [Cheng Pan] nit
b8ec695 [Cheng Pan] fix IDE
2034d62 [Cheng Pan] migrate to kyuubi shaded zk
1cf969e [zhouyifan279] Rename package
170f2e3 [zhouyifan279] Rename package
fe442fc [zhouyifan279] Rename package
fe6f97e [zhouyifan279] Remove unused MetastoreConf.ConfVars
620e834 [zhouyifan279] draft
993218e [zhouyifan279] draft
Lead-authored-by: zhouyifan279 <zh...@gmail.com>
Co-authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.rat-excludes | 1 +
kyuubi-relocated-hive-metastore-client/pom.xml | 179 +++
.../shaded/hive/metastore/api/MetaException.java | 372 ++++++
.../hive/metastore/api/ThriftHiveMetastore.java | 1192 ++++++++++++++++++
.../shaded/hive/common/SSLZookeeperFactory.java | 84 ++
.../shaded/hive/common/ZKDeRegisterWatcher.java | 38 +
.../shaded/hive/common/ZooKeeperHiveHelper.java | 450 +++++++
.../hive/common/classification/RetrySemantics.java | 61 +
.../shaded/hive/metastore/HiveMetaException.java | 41 +
.../shaded/hive/metastore/HiveMetaStoreClient.java | 721 +++++++++++
.../shaded/hive/metastore/IMetaStoreClient.java | 58 +
.../hive/metastore/MetaStorePlainSaslHelper.java | 78 ++
.../hive/metastore/RetryingMetaStoreClient.java | 293 +++++
.../hive/metastore/annotation/NoReconnect.java | 28 +
.../shaded/hive/metastore/conf/MetastoreConf.java | 1263 ++++++++++++++++++++
.../shaded/hive/metastore/conf/SizeValidator.java | 116 ++
.../hive/metastore/conf/StringSetValidator.java | 50 +
.../shaded/hive/metastore/conf/TimeValidator.java | 73 ++
.../shaded/hive/metastore/conf/Validator.java | 85 ++
.../hive/metastore/hooks/URIResolverHook.java | 35 +
.../security/DelegationTokenIdentifier.java | 46 +
.../security/DelegationTokenSelector.java | 30 +
.../metastore/security/HadoopThriftAuthBridge.java | 205 ++++
.../security/HadoopThriftAuthBridge23.java | 112 ++
.../hive/metastore/security/TFilterTransport.java | 115 ++
.../metastore/security/TUGIAssumingTransport.java | 72 ++
.../shaded/hive/metastore/utils/JavaUtils.java | 99 ++
.../hive/metastore/utils/MetaStoreUtils.java | 101 ++
.../kyuubi/shaded/hive/metastore/utils/Retry.java | 69 ++
.../shaded/hive/metastore/utils/SecurityUtils.java | 369 ++++++
.../shaded/hive/metastore/utils/StringUtils.java | 138 +++
.../src/main/thrift/fb303.thrift | 113 ++
.../src/main/thrift/hive_metastore.thrift | 45 +
.../kyuubi-relocated-zookeeper-34/pom.xml | 29 +
.../kyuubi-relocated-zookeeper-36/pom.xml | 29 +
pom.xml | 1 +
36 files changed, 6791 insertions(+)
diff --git a/.rat-excludes b/.rat-excludes
index 74cc616..4eae213 100644
--- a/.rat-excludes
+++ b/.rat-excludes
@@ -30,3 +30,4 @@
**/dependency-reduced-pom.xml
NOTICE*
build/apache-maven-*/**
+**/src/gen/**
diff --git a/kyuubi-relocated-hive-metastore-client/pom.xml b/kyuubi-relocated-hive-metastore-client/pom.xml
new file mode 100644
index 0000000..8951bc2
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/pom.xml
@@ -0,0 +1,179 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>kyuubi-relocated-parent</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>kyuubi-relocated-hive-metastore-client</artifactId>
+ <description>Relocated Hive MetaStore Client classes used by Kyuubi internally.</description>
+
+ <properties>
+ <!-- Dependency versions -->
+ <commons-lang3.version>3.12.0</commons-lang3.version>
+ <hadoop.version>3.3.6</hadoop.version>
+ <libfb303.version>0.9.3</libfb303.version>
+ <libthrift.version>0.16.0</libthrift.version>
+ <slf4j.version>1.7.36</slf4j.version>
+ <!-- Thrift properties -->
+ <thrift.home>you-must-set-this-to-run-thrift</thrift.home>
+ <thrift.file>src/main/thrift/hive_metastore.thrift</thrift.file>
+ <thrift.gen.dir>${basedir}/src/gen/thrift</thrift.gen.dir>
+ <thrift.args>-I ${thrift.home} -strict --gen java:beans,generated_annotations=undated</thrift.args>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.kyuubi</groupId>
+ <!-- TODO: supports kyuubi-relocated-zookeeper-34 -->
+ <artifactId>kyuubi-relocated-zookeeper-36</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <!-- TODO: remove this dependency -->
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons-lang3.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libfb303</artifactId>
+ <version>${libfb303.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ <version>${libthrift.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <phase>generate-sources</phase>
+ <configuration>
+ <sources>
+ <source>src/gen/thrift/gen-javabean</source>
+ <source>${project.build.directory}/generated-sources</source>
+ <source>src/gen/version</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>thriftif</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>3.1.0</version>
+ <executions>
+ <execution>
+ <id>generate-thrift-sources</id>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <phase>generate-sources</phase>
+ <configuration>
+ <target>
+ <delete dir="${thrift.gen.dir}"></delete>
+ <mkdir dir="${thrift.gen.dir}"></mkdir>
+ <echo message="Generating Thrift code for ${thrift.file}"></echo>
+ <exec dir="." executable="${thrift.home}/bin/thrift" failonerror="true">
+ <arg line="${thrift.args} -I ${basedir}/include -I ${basedir}/.. -o ${thrift.gen.dir} ${thrift.file} "></arg>
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>com.google.code.maven-replacer-plugin</groupId>
+ <artifactId>replacer</artifactId>
+ <version>1.5.3</version>
+ <executions>
+ <execution>
+ <id>process-thrift-sources-interface-annotations</id>
+ <goals>
+ <goal>replace</goal>
+ </goals>
+ <phase>process-sources</phase>
+ <configuration>
+ <basedir>${basedir}/src/gen/thrift/gen-javabean/org/apache/kyuubi/shaded/hive/metastore/api/</basedir>
+ <filesToInclude>*.java</filesToInclude>
+ <replacements>
+ <replacement>
+ <token>public class</token>
+ <value>@org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public class</value>
+ <unescape>true</unescape>
+ </replacement>
+ <replacement>
+ <token>public static class</token>
+ <value>@org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class</value>
+ <unescape>true</unescape>
+ </replacement>
+ <replacement>
+ <token>public interface</token>
+ <value>@org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public interface</value>
+ <unescape>true</unescape>
+ </replacement>
+ </replacements>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
diff --git a/kyuubi-relocated-hive-metastore-client/src/gen/thrift/gen-javabean/org/apache/kyuubi/shaded/hive/metastore/api/MetaException.java b/kyuubi-relocated-hive-metastore-client/src/gen/thrift/gen-javabean/org/apache/kyuubi/shaded/hive/metastore/api/MetaException.java
new file mode 100644
index 0000000..3fd073a
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/gen/thrift/gen-javabean/org/apache/kyuubi/shaded/hive/metastore/api/MetaException.java
@@ -0,0 +1,372 @@
+/**
+ * Autogenerated by Thrift Compiler (0.16.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.kyuubi.shaded.hive.metastore.api;
+
+@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"})
+@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.16.0)")
+@org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public class MetaException extends org.apache.thrift.TException implements org.apache.thrift.TBase<MetaException, MetaException._Fields>, java.io.Serializable, Cloneable, Comparable<MetaException> {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("MetaException");
+
+ private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1);
+
+ private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new MetaExceptionStandardSchemeFactory();
+ private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new MetaExceptionTupleSchemeFactory();
+
+ private @org.apache.thrift.annotation.Nullable java.lang.String message; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ MESSAGE((short)1, "message");
+
+ private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>();
+
+ static {
+ for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ @org.apache.thrift.annotation.Nullable
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // MESSAGE
+ return MESSAGE;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ @org.apache.thrift.annotation.Nullable
+ public static _Fields findByName(java.lang.String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final java.lang.String _fieldName;
+
+ _Fields(short thriftId, java.lang.String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public java.lang.String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(MetaException.class, metaDataMap);
+ }
+
+ public MetaException() {
+ }
+
+ public MetaException(
+ java.lang.String message)
+ {
+ this();
+ this.message = message;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public MetaException(MetaException other) {
+ if (other.isSetMessage()) {
+ this.message = other.message;
+ }
+ }
+
+ public MetaException deepCopy() {
+ return new MetaException(this);
+ }
+
+ @Override
+ public void clear() {
+ this.message = null;
+ }
+
+ @org.apache.thrift.annotation.Nullable
+ public java.lang.String getMessage() {
+ return this.message;
+ }
+
+ public void setMessage(@org.apache.thrift.annotation.Nullable java.lang.String message) {
+ this.message = message;
+ }
+
+ public void unsetMessage() {
+ this.message = null;
+ }
+
+ /** Returns true if field message is set (has been assigned a value) and false otherwise */
+ public boolean isSetMessage() {
+ return this.message != null;
+ }
+
+ public void setMessageIsSet(boolean value) {
+ if (!value) {
+ this.message = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) {
+ switch (field) {
+ case MESSAGE:
+ if (value == null) {
+ unsetMessage();
+ } else {
+ setMessage((java.lang.String)value);
+ }
+ break;
+
+ }
+ }
+
+ @org.apache.thrift.annotation.Nullable
+ public java.lang.Object getFieldValue(_Fields field) {
+ switch (field) {
+ case MESSAGE:
+ return getMessage();
+
+ }
+ throw new java.lang.IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new java.lang.IllegalArgumentException();
+ }
+
+ switch (field) {
+ case MESSAGE:
+ return isSetMessage();
+ }
+ throw new java.lang.IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(java.lang.Object that) {
+ if (that instanceof MetaException)
+ return this.equals((MetaException)that);
+ return false;
+ }
+
+ public boolean equals(MetaException that) {
+ if (that == null)
+ return false;
+ if (this == that)
+ return true;
+
+ boolean this_present_message = true && this.isSetMessage();
+ boolean that_present_message = true && that.isSetMessage();
+ if (this_present_message || that_present_message) {
+ if (!(this_present_message && that_present_message))
+ return false;
+ if (!this.message.equals(that.message))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int hashCode = 1;
+
+ hashCode = hashCode * 8191 + ((isSetMessage()) ? 131071 : 524287);
+ if (isSetMessage())
+ hashCode = hashCode * 8191 + message.hashCode();
+
+ return hashCode;
+ }
+
+ @Override
+ public int compareTo(MetaException other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+
+ lastComparison = java.lang.Boolean.compare(isSetMessage(), other.isSetMessage());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetMessage()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.message, other.message);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ @org.apache.thrift.annotation.Nullable
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ scheme(iprot).read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ scheme(oprot).write(oprot, this);
+ }
+
+ @Override
+ public java.lang.String toString() {
+ java.lang.StringBuilder sb = new java.lang.StringBuilder("MetaException(");
+ boolean first = true;
+
+ sb.append("message:");
+ if (this.message == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.message);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class MetaExceptionStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory {
+ public MetaExceptionStandardScheme getScheme() {
+ return new MetaExceptionStandardScheme();
+ }
+ }
+
+ private static class MetaExceptionStandardScheme extends org.apache.thrift.scheme.StandardScheme<MetaException> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, MetaException struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // MESSAGE
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.message = iprot.readString();
+ struct.setMessageIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, MetaException struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.message != null) {
+ oprot.writeFieldBegin(MESSAGE_FIELD_DESC);
+ oprot.writeString(struct.message);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class MetaExceptionTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory {
+ public MetaExceptionTupleScheme getScheme() {
+ return new MetaExceptionTupleScheme();
+ }
+ }
+
+ private static class MetaExceptionTupleScheme extends org.apache.thrift.scheme.TupleScheme<MetaException> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, MetaException struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot;
+ java.util.BitSet optionals = new java.util.BitSet();
+ if (struct.isSetMessage()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetMessage()) {
+ oprot.writeString(struct.message);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, MetaException struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot;
+ java.util.BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.message = iprot.readString();
+ struct.setMessageIsSet(true);
+ }
+ }
+ }
+
+ private static <S extends org.apache.thrift.scheme.IScheme> S scheme(org.apache.thrift.protocol.TProtocol proto) {
+ return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme();
+ }
+}
+
diff --git a/kyuubi-relocated-hive-metastore-client/src/gen/thrift/gen-javabean/org/apache/kyuubi/shaded/hive/metastore/api/ThriftHiveMetastore.java b/kyuubi-relocated-hive-metastore-client/src/gen/thrift/gen-javabean/org/apache/kyuubi/shaded/hive/metastore/api/ThriftHiveMetastore.java
new file mode 100644
index 0000000..7d65933
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/gen/thrift/gen-javabean/org/apache/kyuubi/shaded/hive/metastore/api/ThriftHiveMetastore.java
@@ -0,0 +1,1192 @@
+/**
+ * Autogenerated by Thrift Compiler (0.16.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.kyuubi.shaded.hive.metastore.api;
+
+@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"})
+@javax.annotation.Generated(value = "Autogenerated by Thrift Compiler (0.16.0)")
+@org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public class ThriftHiveMetastore {
+
+ /**
+ * This interface is live.
+ */
+ @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public interface Iface extends com.facebook.fb303.FacebookService.Iface {
+
+ public java.lang.String get_delegation_token(java.lang.String token_owner, java.lang.String renewer_kerberos_principal_name) throws MetaException, org.apache.thrift.TException;
+
+ }
+
+ @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public interface AsyncIface extends com.facebook.fb303.FacebookService .AsyncIface {
+
+ public void get_delegation_token(java.lang.String token_owner, java.lang.String renewer_kerberos_principal_name, org.apache.thrift.async.AsyncMethodCallback<java.lang.String> resultHandler) throws org.apache.thrift.TException;
+
+ }
+
+ @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class Client extends com.facebook.fb303.FacebookService.Client implements Iface {
+ @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
+ public Factory() {}
+ public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
+ return new Client(prot);
+ }
+ public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
+ return new Client(iprot, oprot);
+ }
+ }
+
+ public Client(org.apache.thrift.protocol.TProtocol prot)
+ {
+ super(prot, prot);
+ }
+
+ public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
+ super(iprot, oprot);
+ }
+
+ public java.lang.String get_delegation_token(java.lang.String token_owner, java.lang.String renewer_kerberos_principal_name) throws MetaException, org.apache.thrift.TException
+ {
+ send_get_delegation_token(token_owner, renewer_kerberos_principal_name);
+ return recv_get_delegation_token();
+ }
+
+ public void send_get_delegation_token(java.lang.String token_owner, java.lang.String renewer_kerberos_principal_name) throws org.apache.thrift.TException
+ {
+ get_delegation_token_args args = new get_delegation_token_args();
+ args.setToken_owner(token_owner);
+ args.setRenewer_kerberos_principal_name(renewer_kerberos_principal_name);
+ sendBase("get_delegation_token", args);
+ }
+
+ public java.lang.String recv_get_delegation_token() throws MetaException, org.apache.thrift.TException
+ {
+ get_delegation_token_result result = new get_delegation_token_result();
+ receiveBase(result, "get_delegation_token");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ if (result.o1 != null) {
+ throw result.o1;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get_delegation_token failed: unknown result");
+ }
+
+ }
+ @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class AsyncClient extends com.facebook.fb303.FacebookService.AsyncClient implements AsyncIface {
+ @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
+ private org.apache.thrift.async.TAsyncClientManager clientManager;
+ private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
+ public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
+ this.clientManager = clientManager;
+ this.protocolFactory = protocolFactory;
+ }
+ public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
+ return new AsyncClient(protocolFactory, clientManager, transport);
+ }
+ }
+
+ public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
+ super(protocolFactory, clientManager, transport);
+ }
+
+ public void get_delegation_token(java.lang.String token_owner, java.lang.String renewer_kerberos_principal_name, org.apache.thrift.async.AsyncMethodCallback<java.lang.String> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ get_delegation_token_call method_call = new get_delegation_token_call(token_owner, renewer_kerberos_principal_name, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class get_delegation_token_call extends org.apache.thrift.async.TAsyncMethodCall<java.lang.String> {
+ private java.lang.String token_owner;
+ private java.lang.String renewer_kerberos_principal_name;
+ public get_delegation_token_call(java.lang.String token_owner, java.lang.String renewer_kerberos_principal_name, org.apache.thrift.async.AsyncMethodCallback<java.lang.String> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.token_owner = token_owner;
+ this.renewer_kerberos_principal_name = renewer_kerberos_principal_name;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("get_delegation_token", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ get_delegation_token_args args = new get_delegation_token_args();
+ args.setToken_owner(token_owner);
+ args.setRenewer_kerberos_principal_name(renewer_kerberos_principal_name);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public java.lang.String getResult() throws MetaException, org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new java.lang.IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_get_delegation_token();
+ }
+ }
+
+ }
+
+ @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class Processor<I extends Iface> extends com.facebook.fb303.FacebookService.Processor<I> implements org.apache.thrift.TProcessor {
+ private static final org.slf4j.Logger _LOGGER = org.slf4j.LoggerFactory.getLogger(Processor.class.getName());
+ public Processor(I iface) {
+ super(iface, getProcessMap(new java.util.HashMap<java.lang.String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
+ }
+
+ protected Processor(I iface, java.util.Map<java.lang.String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
+ super(iface, getProcessMap(processMap));
+ }
+
+ private static <I extends Iface> java.util.Map<java.lang.String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> getProcessMap(java.util.Map<java.lang.String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
+ processMap.put("get_delegation_token", new get_delegation_token());
+ return processMap;
+ }
+
+ @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class get_delegation_token<I extends Iface> extends org.apache.thrift.ProcessFunction<I, get_delegation_token_args> {
+ public get_delegation_token() {
+ super("get_delegation_token");
+ }
+
+ public get_delegation_token_args getEmptyArgsInstance() {
+ return new get_delegation_token_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ @Override
+ protected boolean rethrowUnhandledExceptions() {
+ return false;
+ }
+
+ public get_delegation_token_result getResult(I iface, get_delegation_token_args args) throws org.apache.thrift.TException {
+ get_delegation_token_result result = new get_delegation_token_result();
+ try {
+ result.success = iface.get_delegation_token(args.token_owner, args.renewer_kerberos_principal_name);
+ } catch (MetaException o1) {
+ result.o1 = o1;
+ }
+ return result;
+ }
+ }
+
+ }
+
+ @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class AsyncProcessor<I extends AsyncIface> extends com.facebook.fb303.FacebookService.AsyncProcessor<I> {
+ private static final org.slf4j.Logger _LOGGER = org.slf4j.LoggerFactory.getLogger(AsyncProcessor.class.getName());
+ public AsyncProcessor(I iface) {
+ super(iface, getProcessMap(new java.util.HashMap<java.lang.String, org.apache.thrift.AsyncProcessFunction<I, ? extends org.apache.thrift.TBase, ?>>()));
+ }
+
+ protected AsyncProcessor(I iface, java.util.Map<java.lang.String, org.apache.thrift.AsyncProcessFunction<I, ? extends org.apache.thrift.TBase, ?>> processMap) {
+ super(iface, getProcessMap(processMap));
+ }
+
+ private static <I extends AsyncIface> java.util.Map<java.lang.String, org.apache.thrift.AsyncProcessFunction<I, ? extends org.apache.thrift.TBase,?>> getProcessMap(java.util.Map<java.lang.String, org.apache.thrift.AsyncProcessFunction<I, ? extends org.apache.thrift.TBase, ?>> processMap) {
+ processMap.put("get_delegation_token", new get_delegation_token());
+ return processMap;
+ }
+
+ @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class get_delegation_token<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, get_delegation_token_args, java.lang.String> {
+ public get_delegation_token() {
+ super("get_delegation_token");
+ }
+
+ public get_delegation_token_args getEmptyArgsInstance() {
+ return new get_delegation_token_args();
+ }
+
+ public org.apache.thrift.async.AsyncMethodCallback<java.lang.String> getResultHandler(final org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) {
+ final org.apache.thrift.AsyncProcessFunction fcall = this;
+ return new org.apache.thrift.async.AsyncMethodCallback<java.lang.String>() {
+ public void onComplete(java.lang.String o) {
+ get_delegation_token_result result = new get_delegation_token_result();
+ result.success = o;
+ try {
+ fcall.sendResponse(fb, result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
+ } catch (org.apache.thrift.transport.TTransportException e) {
+ _LOGGER.error("TTransportException writing to internal frame buffer", e);
+ fb.close();
+ } catch (java.lang.Exception e) {
+ _LOGGER.error("Exception writing to internal frame buffer", e);
+ onError(e);
+ }
+ }
+ public void onError(java.lang.Exception e) {
+ byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
+ org.apache.thrift.TSerializable msg;
+ get_delegation_token_result result = new get_delegation_token_result();
+ if (e instanceof MetaException) {
+ result.o1 = (MetaException) e;
+ result.setO1IsSet(true);
+ msg = result;
+ } else if (e instanceof org.apache.thrift.transport.TTransportException) {
+ _LOGGER.error("TTransportException inside handler", e);
+ fb.close();
+ return;
+ } else if (e instanceof org.apache.thrift.TApplicationException) {
+ _LOGGER.error("TApplicationException inside handler", e);
+ msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
+ msg = (org.apache.thrift.TApplicationException)e;
+ } else {
+ _LOGGER.error("Exception inside handler", e);
+ msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
+ msg = new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
+ }
+ try {
+ fcall.sendResponse(fb,msg,msgType,seqid);
+ } catch (java.lang.Exception ex) {
+ _LOGGER.error("Exception writing to internal frame buffer", ex);
+ fb.close();
+ }
+ }
+ };
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public void start(I iface, get_delegation_token_args args, org.apache.thrift.async.AsyncMethodCallback<java.lang.String> resultHandler) throws org.apache.thrift.TException {
+ iface.get_delegation_token(args.token_owner, args.renewer_kerberos_principal_name,resultHandler);
+ }
+ }
+
+ }
+
+ @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class get_delegation_token_args implements org.apache.thrift.TBase<get_delegation_token_args, get_delegation_token_args._Fields>, java.io.Serializable, Cloneable, Comparable<get_delegation_token_args> {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_delegation_token_args");
+
+ private static final org.apache.thrift.protocol.TField TOKEN_OWNER_FIELD_DESC = new org.apache.thrift.protocol.TField("token_owner", org.apache.thrift.protocol.TType.STRING, (short)1);
+ private static final org.apache.thrift.protocol.TField RENEWER_KERBEROS_PRINCIPAL_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("renewer_kerberos_principal_name", org.apache.thrift.protocol.TType.STRING, (short)2);
+
+ private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new get_delegation_token_argsStandardSchemeFactory();
+ private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new get_delegation_token_argsTupleSchemeFactory();
+
+ private @org.apache.thrift.annotation.Nullable java.lang.String token_owner; // required
+ private @org.apache.thrift.annotation.Nullable java.lang.String renewer_kerberos_principal_name; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ TOKEN_OWNER((short)1, "token_owner"),
+ RENEWER_KERBEROS_PRINCIPAL_NAME((short)2, "renewer_kerberos_principal_name");
+
+ private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>();
+
+ static {
+ for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ @org.apache.thrift.annotation.Nullable
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // TOKEN_OWNER
+ return TOKEN_OWNER;
+ case 2: // RENEWER_KERBEROS_PRINCIPAL_NAME
+ return RENEWER_KERBEROS_PRINCIPAL_NAME;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ @org.apache.thrift.annotation.Nullable
+ public static _Fields findByName(java.lang.String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final java.lang.String _fieldName;
+
+ _Fields(short thriftId, java.lang.String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public java.lang.String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.TOKEN_OWNER, new org.apache.thrift.meta_data.FieldMetaData("token_owner", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ tmpMap.put(_Fields.RENEWER_KERBEROS_PRINCIPAL_NAME, new org.apache.thrift.meta_data.FieldMetaData("renewer_kerberos_principal_name", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_delegation_token_args.class, metaDataMap);
+ }
+
+ public get_delegation_token_args() {
+ }
+
+ public get_delegation_token_args(
+ java.lang.String token_owner,
+ java.lang.String renewer_kerberos_principal_name)
+ {
+ this();
+ this.token_owner = token_owner;
+ this.renewer_kerberos_principal_name = renewer_kerberos_principal_name;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public get_delegation_token_args(get_delegation_token_args other) {
+ if (other.isSetToken_owner()) {
+ this.token_owner = other.token_owner;
+ }
+ if (other.isSetRenewer_kerberos_principal_name()) {
+ this.renewer_kerberos_principal_name = other.renewer_kerberos_principal_name;
+ }
+ }
+
+ public get_delegation_token_args deepCopy() {
+ return new get_delegation_token_args(this);
+ }
+
+ @Override
+ public void clear() {
+ this.token_owner = null;
+ this.renewer_kerberos_principal_name = null;
+ }
+
+ @org.apache.thrift.annotation.Nullable
+ public java.lang.String getToken_owner() {
+ return this.token_owner;
+ }
+
+ public void setToken_owner(@org.apache.thrift.annotation.Nullable java.lang.String token_owner) {
+ this.token_owner = token_owner;
+ }
+
+ public void unsetToken_owner() {
+ this.token_owner = null;
+ }
+
+ /** Returns true if field token_owner is set (has been assigned a value) and false otherwise */
+ public boolean isSetToken_owner() {
+ return this.token_owner != null;
+ }
+
+ public void setToken_ownerIsSet(boolean value) {
+ if (!value) {
+ this.token_owner = null;
+ }
+ }
+
+ @org.apache.thrift.annotation.Nullable
+ public java.lang.String getRenewer_kerberos_principal_name() {
+ return this.renewer_kerberos_principal_name;
+ }
+
+ public void setRenewer_kerberos_principal_name(@org.apache.thrift.annotation.Nullable java.lang.String renewer_kerberos_principal_name) {
+ this.renewer_kerberos_principal_name = renewer_kerberos_principal_name;
+ }
+
+ public void unsetRenewer_kerberos_principal_name() {
+ this.renewer_kerberos_principal_name = null;
+ }
+
+ /** Returns true if field renewer_kerberos_principal_name is set (has been assigned a value) and false otherwise */
+ public boolean isSetRenewer_kerberos_principal_name() {
+ return this.renewer_kerberos_principal_name != null;
+ }
+
+ public void setRenewer_kerberos_principal_nameIsSet(boolean value) {
+ if (!value) {
+ this.renewer_kerberos_principal_name = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) {
+ switch (field) {
+ case TOKEN_OWNER:
+ if (value == null) {
+ unsetToken_owner();
+ } else {
+ setToken_owner((java.lang.String)value);
+ }
+ break;
+
+ case RENEWER_KERBEROS_PRINCIPAL_NAME:
+ if (value == null) {
+ unsetRenewer_kerberos_principal_name();
+ } else {
+ setRenewer_kerberos_principal_name((java.lang.String)value);
+ }
+ break;
+
+ }
+ }
+
+ @org.apache.thrift.annotation.Nullable
+ public java.lang.Object getFieldValue(_Fields field) {
+ switch (field) {
+ case TOKEN_OWNER:
+ return getToken_owner();
+
+ case RENEWER_KERBEROS_PRINCIPAL_NAME:
+ return getRenewer_kerberos_principal_name();
+
+ }
+ throw new java.lang.IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new java.lang.IllegalArgumentException();
+ }
+
+ switch (field) {
+ case TOKEN_OWNER:
+ return isSetToken_owner();
+ case RENEWER_KERBEROS_PRINCIPAL_NAME:
+ return isSetRenewer_kerberos_principal_name();
+ }
+ throw new java.lang.IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(java.lang.Object that) {
+ if (that instanceof get_delegation_token_args)
+ return this.equals((get_delegation_token_args)that);
+ return false;
+ }
+
+ public boolean equals(get_delegation_token_args that) {
+ if (that == null)
+ return false;
+ if (this == that)
+ return true;
+
+ boolean this_present_token_owner = true && this.isSetToken_owner();
+ boolean that_present_token_owner = true && that.isSetToken_owner();
+ if (this_present_token_owner || that_present_token_owner) {
+ if (!(this_present_token_owner && that_present_token_owner))
+ return false;
+ if (!this.token_owner.equals(that.token_owner))
+ return false;
+ }
+
+ boolean this_present_renewer_kerberos_principal_name = true && this.isSetRenewer_kerberos_principal_name();
+ boolean that_present_renewer_kerberos_principal_name = true && that.isSetRenewer_kerberos_principal_name();
+ if (this_present_renewer_kerberos_principal_name || that_present_renewer_kerberos_principal_name) {
+ if (!(this_present_renewer_kerberos_principal_name && that_present_renewer_kerberos_principal_name))
+ return false;
+ if (!this.renewer_kerberos_principal_name.equals(that.renewer_kerberos_principal_name))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int hashCode = 1;
+
+ hashCode = hashCode * 8191 + ((isSetToken_owner()) ? 131071 : 524287);
+ if (isSetToken_owner())
+ hashCode = hashCode * 8191 + token_owner.hashCode();
+
+ hashCode = hashCode * 8191 + ((isSetRenewer_kerberos_principal_name()) ? 131071 : 524287);
+ if (isSetRenewer_kerberos_principal_name())
+ hashCode = hashCode * 8191 + renewer_kerberos_principal_name.hashCode();
+
+ return hashCode;
+ }
+
+ @Override
+ public int compareTo(get_delegation_token_args other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+
+ lastComparison = java.lang.Boolean.compare(isSetToken_owner(), other.isSetToken_owner());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetToken_owner()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.token_owner, other.token_owner);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = java.lang.Boolean.compare(isSetRenewer_kerberos_principal_name(), other.isSetRenewer_kerberos_principal_name());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetRenewer_kerberos_principal_name()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.renewer_kerberos_principal_name, other.renewer_kerberos_principal_name);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ @org.apache.thrift.annotation.Nullable
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ scheme(iprot).read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ scheme(oprot).write(oprot, this);
+ }
+
+ @Override
+ public java.lang.String toString() {
+ java.lang.StringBuilder sb = new java.lang.StringBuilder("get_delegation_token_args(");
+ boolean first = true;
+
+ sb.append("token_owner:");
+ if (this.token_owner == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.token_owner);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("renewer_kerberos_principal_name:");
+ if (this.renewer_kerberos_principal_name == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.renewer_kerberos_principal_name);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class get_delegation_token_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory {
+ public get_delegation_token_argsStandardScheme getScheme() {
+ return new get_delegation_token_argsStandardScheme();
+ }
+ }
+
+ private static class get_delegation_token_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme<get_delegation_token_args> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, get_delegation_token_args struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // TOKEN_OWNER
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.token_owner = iprot.readString();
+ struct.setToken_ownerIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 2: // RENEWER_KERBEROS_PRINCIPAL_NAME
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.renewer_kerberos_principal_name = iprot.readString();
+ struct.setRenewer_kerberos_principal_nameIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, get_delegation_token_args struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.token_owner != null) {
+ oprot.writeFieldBegin(TOKEN_OWNER_FIELD_DESC);
+ oprot.writeString(struct.token_owner);
+ oprot.writeFieldEnd();
+ }
+ if (struct.renewer_kerberos_principal_name != null) {
+ oprot.writeFieldBegin(RENEWER_KERBEROS_PRINCIPAL_NAME_FIELD_DESC);
+ oprot.writeString(struct.renewer_kerberos_principal_name);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class get_delegation_token_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory {
+ public get_delegation_token_argsTupleScheme getScheme() {
+ return new get_delegation_token_argsTupleScheme();
+ }
+ }
+
+ private static class get_delegation_token_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme<get_delegation_token_args> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, get_delegation_token_args struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot;
+ java.util.BitSet optionals = new java.util.BitSet();
+ if (struct.isSetToken_owner()) {
+ optionals.set(0);
+ }
+ if (struct.isSetRenewer_kerberos_principal_name()) {
+ optionals.set(1);
+ }
+ oprot.writeBitSet(optionals, 2);
+ if (struct.isSetToken_owner()) {
+ oprot.writeString(struct.token_owner);
+ }
+ if (struct.isSetRenewer_kerberos_principal_name()) {
+ oprot.writeString(struct.renewer_kerberos_principal_name);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, get_delegation_token_args struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot;
+ java.util.BitSet incoming = iprot.readBitSet(2);
+ if (incoming.get(0)) {
+ struct.token_owner = iprot.readString();
+ struct.setToken_ownerIsSet(true);
+ }
+ if (incoming.get(1)) {
+ struct.renewer_kerberos_principal_name = iprot.readString();
+ struct.setRenewer_kerberos_principal_nameIsSet(true);
+ }
+ }
+ }
+
+ private static <S extends org.apache.thrift.scheme.IScheme> S scheme(org.apache.thrift.protocol.TProtocol proto) {
+ return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme();
+ }
+ }
+
+ @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class get_delegation_token_result implements org.apache.thrift.TBase<get_delegation_token_result, get_delegation_token_result._Fields>, java.io.Serializable, Cloneable, Comparable<get_delegation_token_result> {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_delegation_token_result");
+
+ private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
+ private static final org.apache.thrift.protocol.TField O1_FIELD_DESC = new org.apache.thrift.protocol.TField("o1", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+
+ private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new get_delegation_token_resultStandardSchemeFactory();
+ private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new get_delegation_token_resultTupleSchemeFactory();
+
+ private @org.apache.thrift.annotation.Nullable java.lang.String success; // required
+ private @org.apache.thrift.annotation.Nullable MetaException o1; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ SUCCESS((short)0, "success"),
+ O1((short)1, "o1");
+
+ private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>();
+
+ static {
+ for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ @org.apache.thrift.annotation.Nullable
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 0: // SUCCESS
+ return SUCCESS;
+ case 1: // O1
+ return O1;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ @org.apache.thrift.annotation.Nullable
+ public static _Fields findByName(java.lang.String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final java.lang.String _fieldName;
+
+ _Fields(short thriftId, java.lang.String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public java.lang.String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ tmpMap.put(_Fields.O1, new org.apache.thrift.meta_data.FieldMetaData("o1", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, MetaException.class)));
+ metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_delegation_token_result.class, metaDataMap);
+ }
+
+ public get_delegation_token_result() {
+ }
+
+ public get_delegation_token_result(
+ java.lang.String success,
+ MetaException o1)
+ {
+ this();
+ this.success = success;
+ this.o1 = o1;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public get_delegation_token_result(get_delegation_token_result other) {
+ if (other.isSetSuccess()) {
+ this.success = other.success;
+ }
+ if (other.isSetO1()) {
+ this.o1 = new MetaException(other.o1);
+ }
+ }
+
+ public get_delegation_token_result deepCopy() {
+ return new get_delegation_token_result(this);
+ }
+
+ @Override
+ public void clear() {
+ this.success = null;
+ this.o1 = null;
+ }
+
+ @org.apache.thrift.annotation.Nullable
+ public java.lang.String getSuccess() {
+ return this.success;
+ }
+
+ public void setSuccess(@org.apache.thrift.annotation.Nullable java.lang.String success) {
+ this.success = success;
+ }
+
+ public void unsetSuccess() {
+ this.success = null;
+ }
+
+ /** Returns true if field success is set (has been assigned a value) and false otherwise */
+ public boolean isSetSuccess() {
+ return this.success != null;
+ }
+
+ public void setSuccessIsSet(boolean value) {
+ if (!value) {
+ this.success = null;
+ }
+ }
+
+ @org.apache.thrift.annotation.Nullable
+ public MetaException getO1() {
+ return this.o1;
+ }
+
+ public void setO1(@org.apache.thrift.annotation.Nullable MetaException o1) {
+ this.o1 = o1;
+ }
+
+ public void unsetO1() {
+ this.o1 = null;
+ }
+
+ /** Returns true if field o1 is set (has been assigned a value) and false otherwise */
+ public boolean isSetO1() {
+ return this.o1 != null;
+ }
+
+ public void setO1IsSet(boolean value) {
+ if (!value) {
+ this.o1 = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) {
+ switch (field) {
+ case SUCCESS:
+ if (value == null) {
+ unsetSuccess();
+ } else {
+ setSuccess((java.lang.String)value);
+ }
+ break;
+
+ case O1:
+ if (value == null) {
+ unsetO1();
+ } else {
+ setO1((MetaException)value);
+ }
+ break;
+
+ }
+ }
+
+ @org.apache.thrift.annotation.Nullable
+ public java.lang.Object getFieldValue(_Fields field) {
+ switch (field) {
+ case SUCCESS:
+ return getSuccess();
+
+ case O1:
+ return getO1();
+
+ }
+ throw new java.lang.IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new java.lang.IllegalArgumentException();
+ }
+
+ switch (field) {
+ case SUCCESS:
+ return isSetSuccess();
+ case O1:
+ return isSetO1();
+ }
+ throw new java.lang.IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(java.lang.Object that) {
+ if (that instanceof get_delegation_token_result)
+ return this.equals((get_delegation_token_result)that);
+ return false;
+ }
+
+ public boolean equals(get_delegation_token_result that) {
+ if (that == null)
+ return false;
+ if (this == that)
+ return true;
+
+ boolean this_present_success = true && this.isSetSuccess();
+ boolean that_present_success = true && that.isSetSuccess();
+ if (this_present_success || that_present_success) {
+ if (!(this_present_success && that_present_success))
+ return false;
+ if (!this.success.equals(that.success))
+ return false;
+ }
+
+ boolean this_present_o1 = true && this.isSetO1();
+ boolean that_present_o1 = true && that.isSetO1();
+ if (this_present_o1 || that_present_o1) {
+ if (!(this_present_o1 && that_present_o1))
+ return false;
+ if (!this.o1.equals(that.o1))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int hashCode = 1;
+
+ hashCode = hashCode * 8191 + ((isSetSuccess()) ? 131071 : 524287);
+ if (isSetSuccess())
+ hashCode = hashCode * 8191 + success.hashCode();
+
+ hashCode = hashCode * 8191 + ((isSetO1()) ? 131071 : 524287);
+ if (isSetO1())
+ hashCode = hashCode * 8191 + o1.hashCode();
+
+ return hashCode;
+ }
+
+ @Override
+ public int compareTo(get_delegation_token_result other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+
+ lastComparison = java.lang.Boolean.compare(isSetSuccess(), other.isSetSuccess());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetSuccess()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = java.lang.Boolean.compare(isSetO1(), other.isSetO1());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetO1()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.o1, other.o1);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ @org.apache.thrift.annotation.Nullable
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ scheme(iprot).read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ scheme(oprot).write(oprot, this);
+ }
+
+ @Override
+ public java.lang.String toString() {
+ java.lang.StringBuilder sb = new java.lang.StringBuilder("get_delegation_token_result(");
+ boolean first = true;
+
+ sb.append("success:");
+ if (this.success == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.success);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("o1:");
+ if (this.o1 == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.o1);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class get_delegation_token_resultStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory {
+ public get_delegation_token_resultStandardScheme getScheme() {
+ return new get_delegation_token_resultStandardScheme();
+ }
+ }
+
+ private static class get_delegation_token_resultStandardScheme extends org.apache.thrift.scheme.StandardScheme<get_delegation_token_result> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, get_delegation_token_result struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 0: // SUCCESS
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.success = iprot.readString();
+ struct.setSuccessIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 1: // O1
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.o1 = new MetaException();
+ struct.o1.read(iprot);
+ struct.setO1IsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, get_delegation_token_result struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.success != null) {
+ oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
+ oprot.writeString(struct.success);
+ oprot.writeFieldEnd();
+ }
+ if (struct.o1 != null) {
+ oprot.writeFieldBegin(O1_FIELD_DESC);
+ struct.o1.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class get_delegation_token_resultTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory {
+ public get_delegation_token_resultTupleScheme getScheme() {
+ return new get_delegation_token_resultTupleScheme();
+ }
+ }
+
+ private static class get_delegation_token_resultTupleScheme extends org.apache.thrift.scheme.TupleScheme<get_delegation_token_result> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, get_delegation_token_result struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot;
+ java.util.BitSet optionals = new java.util.BitSet();
+ if (struct.isSetSuccess()) {
+ optionals.set(0);
+ }
+ if (struct.isSetO1()) {
+ optionals.set(1);
+ }
+ oprot.writeBitSet(optionals, 2);
+ if (struct.isSetSuccess()) {
+ oprot.writeString(struct.success);
+ }
+ if (struct.isSetO1()) {
+ struct.o1.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, get_delegation_token_result struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot;
+ java.util.BitSet incoming = iprot.readBitSet(2);
+ if (incoming.get(0)) {
+ struct.success = iprot.readString();
+ struct.setSuccessIsSet(true);
+ }
+ if (incoming.get(1)) {
+ struct.o1 = new MetaException();
+ struct.o1.read(iprot);
+ struct.setO1IsSet(true);
+ }
+ }
+ }
+
+ private static <S extends org.apache.thrift.scheme.IScheme> S scheme(org.apache.thrift.protocol.TProtocol proto) {
+ return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme();
+ }
+ }
+
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/SSLZookeeperFactory.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/SSLZookeeperFactory.java
new file mode 100644
index 0000000..9437859
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/SSLZookeeperFactory.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.shaded.hive.common;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kyuubi.shaded.curator.utils.ZookeeperFactory;
+import org.apache.kyuubi.shaded.zookeeper.Watcher;
+import org.apache.kyuubi.shaded.zookeeper.ZooKeeper;
+import org.apache.kyuubi.shaded.zookeeper.client.ZKClientConfig;
+import org.apache.kyuubi.shaded.zookeeper.common.ClientX509Util;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Factory to create Zookeeper clients with the zookeeper.client.secure enabled, allowing SSL
+ * communication with the Zookeeper server.
+ */
+public class SSLZookeeperFactory implements ZookeeperFactory {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SSLZookeeperFactory.class);
+
+ private boolean sslEnabled;
+ private String keyStoreLocation;
+ private String keyStorePassword;
+ private String trustStoreLocation;
+ private String trustStorePassword;
+
+ public SSLZookeeperFactory(
+ boolean sslEnabled,
+ String keyStoreLocation,
+ String keyStorePassword,
+ String trustStoreLocation,
+ String trustStorePassword) {
+
+ this.sslEnabled = sslEnabled;
+ this.keyStoreLocation = keyStoreLocation;
+ this.keyStorePassword = keyStorePassword;
+ this.trustStoreLocation = trustStoreLocation;
+ this.trustStorePassword = trustStorePassword;
+ if (sslEnabled) {
+ if (StringUtils.isEmpty(keyStoreLocation)) {
+ LOG.warn("Missing keystoreLocation parameter");
+ }
+ if (StringUtils.isEmpty(trustStoreLocation)) {
+ LOG.warn("Missing trustStoreLocation parameter");
+ }
+ }
+ }
+
+ @Override
+ public ZooKeeper newZooKeeper(
+ String connectString, int sessionTimeout, Watcher watcher, boolean canBeReadOnly)
+ throws Exception {
+ if (!this.sslEnabled) {
+ return new ZooKeeper(connectString, sessionTimeout, watcher, canBeReadOnly);
+ }
+ ZKClientConfig clientConfig = new ZKClientConfig();
+ clientConfig.setProperty(ZKClientConfig.SECURE_CLIENT, "true");
+ clientConfig.setProperty(
+ ZKClientConfig.ZOOKEEPER_CLIENT_CNXN_SOCKET, "org.apache.zookeeper.ClientCnxnSocketNetty");
+ ClientX509Util x509Util = new ClientX509Util();
+ clientConfig.setProperty(x509Util.getSslKeystoreLocationProperty(), this.keyStoreLocation);
+ clientConfig.setProperty(x509Util.getSslKeystorePasswdProperty(), this.keyStorePassword);
+ clientConfig.setProperty(x509Util.getSslTruststoreLocationProperty(), this.trustStoreLocation);
+ clientConfig.setProperty(x509Util.getSslTruststorePasswdProperty(), this.trustStorePassword);
+ return new ZooKeeper(connectString, sessionTimeout, watcher, canBeReadOnly, clientConfig);
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/ZKDeRegisterWatcher.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/ZKDeRegisterWatcher.java
new file mode 100644
index 0000000..e273a2a
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/ZKDeRegisterWatcher.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.shaded.hive.common;
+
+import org.apache.kyuubi.shaded.zookeeper.WatchedEvent;
+import org.apache.kyuubi.shaded.zookeeper.Watcher;
+
+/** The watcher class which sets the de-register flag when the given znode is deleted. */
+public class ZKDeRegisterWatcher implements Watcher {
+ private ZooKeeperHiveHelper zooKeeperHiveHelper;
+
+ public ZKDeRegisterWatcher(ZooKeeperHiveHelper zooKeeperHiveHelper) {
+ this.zooKeeperHiveHelper = zooKeeperHiveHelper;
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ if (event.getType().equals(Watcher.Event.EventType.NodeDeleted)) {
+ zooKeeperHiveHelper.deregisterZnode();
+ }
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/ZooKeeperHiveHelper.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/ZooKeeperHiveHelper.java
new file mode 100644
index 0000000..fda077b
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/ZooKeeperHiveHelper.java
@@ -0,0 +1,450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.shaded.hive.common;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.kyuubi.shaded.curator.framework.CuratorFramework;
+import org.apache.kyuubi.shaded.curator.framework.CuratorFrameworkFactory;
+import org.apache.kyuubi.shaded.curator.framework.api.ACLProvider;
+import org.apache.kyuubi.shaded.curator.framework.recipes.nodes.PersistentNode;
+import org.apache.kyuubi.shaded.curator.retry.ExponentialBackoffRetry;
+import org.apache.kyuubi.shaded.curator.retry.RetryOneTime;
+import org.apache.kyuubi.shaded.zookeeper.CreateMode;
+import org.apache.kyuubi.shaded.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+// The class serves three purposes (for HiveServer2 and HiveMetaStore)
+// 1. An instance of this class holds ZooKeeper related configuration parameter values from Hive
+// configuration and metastore configuration.
+// 2. For a server which is added to ZooKeeper specified by the configuration, an instance of
+// this class holds the znode corresponding to that server, zookeeper client used to watch the
+// znode.
+// 3. For a metastore client it provides API to find server URIs from specified ZooKeeper.
+//
+// We could have differentiated these three functionality into three different classes by
+// including an instance of first class in the second and the third, but there's isn't much stuff
+// in the first and the third.. Also note that the third functionality overlaps with
+// ZooKeeperHiveClientHelper class, but that overlap is very small. So for now all the three
+// functionality are bundled in a single class.
+
+/**
+ * ZooKeeperHiveHelper. A helper class to hold ZooKeeper related configuration, to register and
+ * deregister ZooKeeper node for a given server and to fetch registered server URIs for clients.
+ */
+public class ZooKeeperHiveHelper {
+ public static final Logger LOG = LoggerFactory.getLogger(ZooKeeperHiveHelper.class.getName());
+ public static final String ZOOKEEPER_PATH_SEPARATOR = "/";
+
+ /** ZooKeeperHiveHelperBuilder. A builder class to initialize ZooKeeperHiveHelper. */
+ public static class ZooKeeperHiveHelperBuilder {
+ private String quorum = null;
+ private String clientPort = null;
+ private String serverRegistryNameSpace = null;
+ private int connectionTimeout;
+ private int sessionTimeout;
+ private int baseSleepTime;
+ private int maxRetries;
+ private boolean sslEnabled = false;
+ private String keyStoreLocation = null;
+ private String keyStorePassword = null;
+ private String trustStoreLocation = null;
+ private String trustStorePassword = null;
+
+ public ZooKeeperHiveHelper build() {
+ return new ZooKeeperHiveHelper(this);
+ }
+
+ public ZooKeeperHiveHelperBuilder quorum(String quorum) {
+ this.quorum = quorum;
+ return this;
+ }
+
+ public ZooKeeperHiveHelperBuilder clientPort(String clientPort) {
+ this.clientPort = clientPort;
+ return this;
+ }
+
+ public ZooKeeperHiveHelperBuilder serverRegistryNameSpace(String serverRegistryNameSpace) {
+ this.serverRegistryNameSpace = serverRegistryNameSpace;
+ return this;
+ }
+
+ public ZooKeeperHiveHelperBuilder connectionTimeout(int connectionTimeout) {
+ this.connectionTimeout = connectionTimeout;
+ return this;
+ }
+
+ public ZooKeeperHiveHelperBuilder sessionTimeout(int sessionTimeout) {
+ this.sessionTimeout = sessionTimeout;
+ return this;
+ }
+
+ public ZooKeeperHiveHelperBuilder baseSleepTime(int baseSleepTime) {
+ this.baseSleepTime = baseSleepTime;
+ return this;
+ }
+
+ public ZooKeeperHiveHelperBuilder maxRetries(int maxRetries) {
+ this.maxRetries = maxRetries;
+ return this;
+ }
+
+ public ZooKeeperHiveHelperBuilder sslEnabled(boolean sslEnabled) {
+ this.sslEnabled = sslEnabled;
+ return this;
+ }
+
+ public ZooKeeperHiveHelperBuilder keyStoreLocation(String keyStoreLocation) {
+ this.keyStoreLocation = keyStoreLocation;
+ return this;
+ }
+
+ public ZooKeeperHiveHelperBuilder keyStorePassword(String keyStorePassword) {
+ this.keyStorePassword = keyStorePassword;
+ return this;
+ }
+
+ public ZooKeeperHiveHelperBuilder trustStoreLocation(String trustStoreLocation) {
+ this.trustStoreLocation = trustStoreLocation;
+ return this;
+ }
+
+ public ZooKeeperHiveHelperBuilder trustStorePassword(String trustStorePassword) {
+ this.trustStorePassword = trustStorePassword;
+ return this;
+ }
+
+ public String getQuorum() {
+ return quorum;
+ }
+
+ public String getClientPort() {
+ return clientPort;
+ }
+
+ public String getServerRegistryNameSpace() {
+ return serverRegistryNameSpace;
+ }
+
+ public int getConnectionTimeout() {
+ return connectionTimeout;
+ }
+
+ public int getSessionTimeout() {
+ return sessionTimeout;
+ }
+
+ public int getBaseSleepTime() {
+ return baseSleepTime;
+ }
+
+ public int getMaxRetries() {
+ return maxRetries;
+ }
+
+ public boolean isSslEnabled() {
+ return sslEnabled;
+ }
+
+ public String getKeyStoreLocation() {
+ return keyStoreLocation;
+ }
+
+ public String getKeyStorePassword() {
+ return keyStorePassword;
+ }
+
+ public String getTrustStoreLocation() {
+ return trustStoreLocation;
+ }
+
+ public String getTrustStorePassword() {
+ return trustStorePassword;
+ }
+ }
+
+ public static ZooKeeperHiveHelper.ZooKeeperHiveHelperBuilder builder() {
+ return new ZooKeeperHiveHelper.ZooKeeperHiveHelperBuilder();
+ }
+
+ private String quorum;
+ private String rootNamespace;
+ private int connectionTimeout;
+ private int sessionTimeout;
+ private int baseSleepTime;
+ private int maxRetries;
+ private boolean sslEnabled;
+
+ private SSLZookeeperFactory sslZookeeperFactory;
+ private CuratorFramework zooKeeperClient;
+ private boolean deregisteredWithZooKeeper = false; // Set to true only when deregistration happens
+ private PersistentNode znode;
+
+ public ZooKeeperHiveHelper(ZooKeeperHiveHelperBuilder builder) {
+ // Get the ensemble server addresses in the format host1:port1, host2:port2, ... . Append
+ // the configured port to hostname if the hostname doesn't contain a port.
+ String[] hosts = builder.getQuorum().split(",");
+ StringBuilder quorumServers = new StringBuilder();
+ for (int i = 0; i < hosts.length; i++) {
+ quorumServers.append(hosts[i].trim());
+ if (!hosts[i].contains(":")) {
+ quorumServers.append(":");
+ quorumServers.append(builder.getClientPort());
+ }
+
+ if (i != hosts.length - 1) {
+ quorumServers.append(",");
+ }
+ }
+
+ this.quorum = quorumServers.toString();
+ this.rootNamespace = builder.getServerRegistryNameSpace();
+ this.connectionTimeout = builder.getConnectionTimeout();
+ this.sessionTimeout = builder.getSessionTimeout();
+ this.baseSleepTime = builder.getBaseSleepTime();
+ this.maxRetries = builder.getMaxRetries();
+ this.sslEnabled = builder.isSslEnabled();
+ this.sslZookeeperFactory =
+ new SSLZookeeperFactory(
+ sslEnabled,
+ builder.getKeyStoreLocation(),
+ builder.getKeyStorePassword(),
+ builder.getTrustStoreLocation(),
+ builder.getTrustStorePassword());
+ }
+
+ /** Get the ensemble server addresses. The format is: host1:port, host2:port.. */
+ public String getQuorumServers() {
+ return quorum;
+ }
+
+ /**
+ * Adds a server instance to ZooKeeper as a znode.
+ *
+ * @throws Exception
+ */
+ public void addServerInstanceToZooKeeper(
+ String znodePathPrefix,
+ String znodeData,
+ ACLProvider zooKeeperAclProvider,
+ ZKDeRegisterWatcher watcher)
+ throws Exception {
+ // This might be the first server getting added to the ZooKeeper, so the parent node may need
+ // to be created.
+ zooKeeperClient = startZookeeperClient(zooKeeperAclProvider, true);
+
+ // Create a znode under the rootNamespace parent for the given path prefix for a server. Also
+ // add a watcher to watch the znode.
+ try {
+ String pathPrefix =
+ ZOOKEEPER_PATH_SEPARATOR + rootNamespace + ZOOKEEPER_PATH_SEPARATOR + znodePathPrefix;
+ byte[] znodeDataUTF8 = znodeData.getBytes(StandardCharsets.UTF_8);
+ znode =
+ new PersistentNode(
+ zooKeeperClient, CreateMode.EPHEMERAL_SEQUENTIAL, false, pathPrefix, znodeDataUTF8);
+ znode.start();
+ // We'll wait for 120s for node creation
+ long znodeCreationTimeout = 120;
+ if (!znode.waitForInitialCreate(znodeCreationTimeout, TimeUnit.SECONDS)) {
+ throw new Exception(
+ "Max znode creation wait time: " + znodeCreationTimeout + "s exhausted");
+ }
+ setDeregisteredWithZooKeeper(false);
+ final String znodePath = znode.getActualPath();
+ if (zooKeeperClient.checkExists().usingWatcher(watcher).forPath(znodePath) == null) {
+ // No node exists, throw exception
+ throw new Exception(
+ "Unable to create znode with path prefix "
+ + znodePathPrefix
+ + " and data "
+ + znodeData
+ + " on ZooKeeper.");
+ }
+ LOG.info(
+ "Created a znode (actual path "
+ + znodePath
+ + ") on ZooKeeper with path prefix "
+ + znodePathPrefix
+ + " and data "
+ + znodeData);
+ } catch (Exception e) {
+ LOG.error(
+ "Unable to create znode with path prefix "
+ + znodePathPrefix
+ + " and data "
+ + znodeData
+ + " on ZooKeeper.",
+ e);
+ if (znode != null) {
+ znode.close();
+ }
+ throw (e);
+ }
+ }
+
+ public CuratorFramework startZookeeperClient(
+ ACLProvider zooKeeperAclProvider, boolean addParentNode) throws Exception {
+ CuratorFramework zkClient = getNewZookeeperClient(zooKeeperAclProvider);
+ zkClient.start();
+
+ // Create the parent znodes recursively; ignore if the parent already exists.
+ if (addParentNode) {
+ try {
+ zkClient
+ .create()
+ .creatingParentsIfNeeded()
+ .withMode(CreateMode.PERSISTENT)
+ .forPath(ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + rootNamespace);
+ LOG.info("Created the root name space: " + rootNamespace + " on ZooKeeper");
+ } catch (KeeperException e) {
+ if (e.code() != KeeperException.Code.NODEEXISTS) {
+ LOG.error("Unable to create namespace: " + rootNamespace + " on ZooKeeper", e);
+ throw e;
+ }
+ }
+ }
+ return zkClient;
+ }
+
+ public CuratorFramework getNewZookeeperClient() {
+ return getNewZookeeperClient(null, null);
+ }
+
+ public CuratorFramework getNewZookeeperClient(ACLProvider zooKeeperAclProvider) {
+ return getNewZookeeperClient(zooKeeperAclProvider, null);
+ }
+
+ public CuratorFramework getNewZookeeperClient(
+ ACLProvider zooKeeperAclProvider, String nameSpace) {
+ LOG.info(
+ "Creating curator client with connectString: {} namespace: {} sessionTimeoutMs: {}"
+ + " connectionTimeoutMs: {} exponentialBackoff - sleepTime: {} maxRetries: {} sslEnabled: {}",
+ quorum,
+ nameSpace,
+ sessionTimeout,
+ connectionTimeout,
+ baseSleepTime,
+ maxRetries,
+ sslEnabled);
+ // Create a CuratorFramework instance to be used as the ZooKeeper client.
+ // Use the zooKeeperAclProvider, when specified, to create appropriate ACLs.
+ CuratorFrameworkFactory.Builder builder =
+ CuratorFrameworkFactory.builder()
+ .connectString(quorum)
+ .namespace(nameSpace)
+ .zookeeperFactory(this.sslZookeeperFactory);
+ if (connectionTimeout > 0) {
+ builder = builder.connectionTimeoutMs(connectionTimeout);
+ }
+ if (sessionTimeout > 0) {
+ builder = builder.sessionTimeoutMs(sessionTimeout);
+ }
+ if (maxRetries > 0) {
+ builder = builder.retryPolicy(new ExponentialBackoffRetry(baseSleepTime, maxRetries));
+ } else {
+ // Retry policy is mandatory
+ builder = builder.retryPolicy(new RetryOneTime(1000));
+ }
+ if (zooKeeperAclProvider != null) {
+ builder = builder.aclProvider(zooKeeperAclProvider);
+ }
+
+ return builder.build();
+ }
+
+ public void removeServerInstanceFromZooKeeper() throws Exception {
+ setDeregisteredWithZooKeeper(true);
+
+ if (znode != null) {
+ znode.close();
+ znode = null;
+ }
+ if (zooKeeperClient != null) {
+ zooKeeperClient.close();
+ }
+ LOG.info("Server instance removed from ZooKeeper.");
+ }
+
+ public void deregisterZnode() {
+ if (znode != null) {
+ try {
+ znode.close();
+ LOG.warn(
+ "This server instance with path "
+ + znode.getActualPath()
+ + " is now de-registered from ZooKeeper. ");
+ } catch (IOException e) {
+ LOG.error("Failed to close the persistent ephemeral znode", e);
+ } finally {
+ setDeregisteredWithZooKeeper(true);
+ znode = null;
+ }
+ }
+ }
+
+ public synchronized boolean isDeregisteredWithZooKeeper() {
+ return deregisteredWithZooKeeper;
+ }
+
+ private synchronized void setDeregisteredWithZooKeeper(boolean deregisteredWithZooKeeper) {
+ this.deregisteredWithZooKeeper = deregisteredWithZooKeeper;
+ }
+
+ /**
+ * This method is supposed to be called from client code connecting to one of the servers managed
+ * by the configured ZooKeeper. It starts and closes its own ZooKeeper client instead of using the
+ * class member.
+ *
+ * @return list of server URIs stored under the configured zookeeper namespace
+ * @throws Exception
+ */
+ public List<String> getServerUris() throws Exception {
+ CuratorFramework zkClient = null;
+ List<String> serverUris;
+ try {
+ zkClient = startZookeeperClient(null, false);
+ List<String> serverNodes =
+ zkClient.getChildren().forPath(ZOOKEEPER_PATH_SEPARATOR + rootNamespace);
+ serverUris = new ArrayList<String>(serverNodes.size());
+ for (String serverNode : serverNodes) {
+ byte[] serverUriBytes =
+ zkClient
+ .getData()
+ .forPath(
+ ZOOKEEPER_PATH_SEPARATOR
+ + rootNamespace
+ + ZOOKEEPER_PATH_SEPARATOR
+ + serverNode);
+ serverUris.add(new String(serverUriBytes, StandardCharsets.UTF_8));
+ }
+ zkClient.close();
+ return serverUris;
+ } catch (Exception e) {
+ if (zkClient != null) {
+ zkClient.close();
+ }
+ throw e;
+ }
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/classification/RetrySemantics.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/classification/RetrySemantics.java
new file mode 100644
index 0000000..f17ff72
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/common/classification/RetrySemantics.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.shaded.hive.common.classification;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * These annotations are meant to indicate how to handle retry logic. Initially meant for Metastore
+ * API when made across a network, i.e. asynchronously where the response may not reach the caller
+ * and thus it cannot know if the operation was actually performed on the server.
+ */
+@InterfaceStability.Evolving
+@InterfaceAudience.LimitedPrivate("Hive developer")
+public class RetrySemantics {
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target(ElementType.METHOD)
+ public @interface Idempotent {
+ String[] value() default "";
+
+ int maxRetryCount() default Integer.MAX_VALUE;
+
+ int delayMs() default 100;
+ }
+
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target(ElementType.METHOD)
+ public @interface ReadOnly {
+ /*trivially retry-able*/
+ }
+
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target(ElementType.METHOD)
+ public @interface CannotRetry {}
+
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target(ElementType.METHOD)
+ public @interface SafeToRetry {
+ /*may not be Idempotent but is safe to retry*/
+ String[] value() default "";
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/HiveMetaException.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/HiveMetaException.java
new file mode 100644
index 0000000..8b62f85
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/HiveMetaException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.shaded.hive.metastore;
+
+public class HiveMetaException extends Exception {
+
+ public HiveMetaException() {
+ // TODO Auto-generated constructor stub
+ }
+
+ public HiveMetaException(String message) {
+ super(message);
+ // TODO Auto-generated constructor stub
+ }
+
+ public HiveMetaException(Throwable cause) {
+ super(cause);
+ // TODO Auto-generated constructor stub
+ }
+
+ public HiveMetaException(String message, Throwable cause) {
+ super(message, cause);
+ // TODO Auto-generated constructor stub
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/HiveMetaStoreClient.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/HiveMetaStoreClient.java
new file mode 100644
index 0000000..c9e61fc
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/HiveMetaStoreClient.java
@@ -0,0 +1,721 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.shaded.hive.metastore;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.http.HttpException;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpRequestInterceptor;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.protocol.HttpContext;
+import org.apache.kyuubi.shaded.hive.metastore.api.MetaException;
+import org.apache.kyuubi.shaded.hive.metastore.api.ThriftHiveMetastore;
+import org.apache.kyuubi.shaded.hive.metastore.conf.MetastoreConf;
+import org.apache.kyuubi.shaded.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.kyuubi.shaded.hive.metastore.hooks.URIResolverHook;
+import org.apache.kyuubi.shaded.hive.metastore.security.HadoopThriftAuthBridge;
+import org.apache.kyuubi.shaded.hive.metastore.utils.JavaUtils;
+import org.apache.kyuubi.shaded.hive.metastore.utils.MetaStoreUtils;
+import org.apache.kyuubi.shaded.hive.metastore.utils.SecurityUtils;
+import org.apache.thrift.TConfiguration;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.transport.THttpClient;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.thrift.transport.layered.TFramedTransport;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Hive Metastore Client. The public implementation of IMetaStoreClient. Methods not inherited from
+ * IMetaStoreClient are not public and can change. Hence this is marked as unstable. For users who
+ * require retry mechanism when the connection between metastore and client is broken,
+ * RetryingMetaStoreClient class should be used.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
+
+ ThriftHiveMetastore.Iface client = null;
+ private TTransport transport = null;
+ private boolean isConnected = false;
+ private URI metastoreUris[];
+ protected final Configuration
+ conf; // Keep a copy of HiveConf so if Session conf changes, we may need to get a new HMS
+ // client.
+ private String tokenStrForm;
+ private final boolean localMetaStore;
+ private final URIResolverHook uriResolverHook;
+
+ private Map<String, String> currentMetaVars;
+
+ private static final AtomicInteger connCount = new AtomicInteger(0);
+
+ // for thrift connects
+ private int retries = 5;
+ private long retryDelaySeconds = 0;
+
+ protected static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreClient.class);
+
+ public HiveMetaStoreClient(Configuration conf) throws MetaException {
+
+ if (conf == null) {
+ conf = MetastoreConf.newMetastoreConf();
+ this.conf = conf;
+ } else {
+ this.conf = new Configuration(conf);
+ }
+ uriResolverHook = loadUriResolverHook();
+
+ String msUri = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS);
+ localMetaStore = MetastoreConf.isEmbeddedMetaStore(msUri);
+ if (localMetaStore) {
+ // instantiate the metastore server handler directly instead of connecting
+ // through the network
+ isConnected = true;
+ return;
+ }
+
+ // get the number retries
+ retries = MetastoreConf.getIntVar(conf, ConfVars.THRIFT_CONNECTION_RETRIES);
+ retryDelaySeconds =
+ MetastoreConf.getTimeVar(conf, ConfVars.CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS);
+
+ // user wants file store based configuration
+ if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS) != null) {
+ resolveUris();
+ } else {
+ LOG.error("NOT getting uris from conf");
+ throw new MetaException("MetaStoreURIs not found in conf file");
+ }
+
+ // finally open the store
+ open();
+ }
+
+ private void resolveUris() throws MetaException {
+ String thriftUris = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS);
+ String serviceDiscoveryMode =
+ MetastoreConf.getVar(conf, ConfVars.THRIFT_SERVICE_DISCOVERY_MODE);
+ List<String> metastoreUrisString = null;
+
+ // The metastore URIs can come from THRIFT_URIS directly or need to be fetched from the
+ // Zookeeper
+ try {
+ if (serviceDiscoveryMode == null || serviceDiscoveryMode.trim().isEmpty()) {
+ metastoreUrisString = Arrays.asList(thriftUris.split(","));
+ } else if (serviceDiscoveryMode.equalsIgnoreCase("zookeeper")) {
+ metastoreUrisString = new ArrayList<String>();
+ // Add scheme to the bare URI we get.
+ for (String s : MetastoreConf.getZKConfig(conf).getServerUris()) {
+ metastoreUrisString.add("thrift://" + s);
+ }
+ } else {
+ throw new IllegalArgumentException(
+ "Invalid metastore dynamic service discovery mode " + serviceDiscoveryMode);
+ }
+ } catch (Exception e) {
+ MetaStoreUtils.throwMetaException(e);
+ }
+
+ if (metastoreUrisString.isEmpty() && "zookeeper".equalsIgnoreCase(serviceDiscoveryMode)) {
+ throw new MetaException(
+ "No metastore service discovered in ZooKeeper. "
+ + "Please ensure that at least one metastore server is online");
+ }
+
+ LOG.info("Resolved metastore uris: {}", metastoreUrisString);
+
+ List<URI> metastoreURIArray = new ArrayList<URI>();
+ try {
+ for (String s : metastoreUrisString) {
+ URI tmpUri = new URI(s);
+ if (tmpUri.getScheme() == null) {
+ throw new IllegalArgumentException("URI: " + s + " does not have a scheme");
+ }
+ if (uriResolverHook != null) {
+ metastoreURIArray.addAll(uriResolverHook.resolveURI(tmpUri));
+ } else {
+ metastoreURIArray.add(tmpUri);
+ }
+ }
+ metastoreUris = new URI[metastoreURIArray.size()];
+ for (int j = 0; j < metastoreURIArray.size(); j++) {
+ metastoreUris[j] = metastoreURIArray.get(j);
+ }
+
+ if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URI_SELECTION).equalsIgnoreCase("RANDOM")) {
+ List<URI> uriList = Arrays.asList(metastoreUris);
+ Collections.shuffle(uriList);
+ metastoreUris = uriList.toArray(new URI[uriList.size()]);
+ }
+ } catch (IllegalArgumentException e) {
+ throw (e);
+ } catch (Exception e) {
+ MetaStoreUtils.throwMetaException(e);
+ }
+ }
+
+ // multiple clients may initialize the hook at the same time
+ private synchronized URIResolverHook loadUriResolverHook() throws IllegalStateException {
+
+ String uriResolverClassName = MetastoreConf.getAsString(conf, ConfVars.URI_RESOLVER);
+ if (uriResolverClassName.equals("")) {
+ return null;
+ } else {
+ LOG.info("Loading uri resolver : " + uriResolverClassName);
+ try {
+ Class<?> uriResolverClass =
+ Class.forName(uriResolverClassName, true, JavaUtils.getClassLoader());
+ return (URIResolverHook) ReflectionUtils.newInstance(uriResolverClass, null);
+ } catch (Exception e) {
+ LOG.error("Exception loading uri resolver hook", e);
+ return null;
+ }
+ }
+ }
+
+ /**
+ * Swaps the first element of the metastoreUris array with a random element from the remainder of
+ * the array.
+ */
+ private void promoteRandomMetaStoreURI() {
+ if (metastoreUris.length <= 1) {
+ return;
+ }
+ Random rng = new Random();
+ int index = rng.nextInt(metastoreUris.length - 1) + 1;
+ URI tmp = metastoreUris[0];
+ metastoreUris[0] = metastoreUris[index];
+ metastoreUris[index] = tmp;
+ }
+
+ public TTransport getTTransport() {
+ return transport;
+ }
+
+ @Override
+ public boolean isLocalMetaStore() {
+ return localMetaStore;
+ }
+
+ @Override
+ public void reconnect() throws MetaException {
+ if (localMetaStore) {
+ // For direct DB connections we don't yet support reestablishing connections.
+ throw new MetaException(
+ "Retries for direct MetaStore DB connections " + "are not supported by this client");
+ } else {
+ close();
+
+ if (uriResolverHook != null) {
+ // for dynamic uris, re-lookup if there are new metastore locations
+ resolveUris();
+ }
+
+ if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URI_SELECTION).equalsIgnoreCase("RANDOM")) {
+ // Swap the first element of the metastoreUris[] with a random element from the rest
+ // of the array. Rationale being that this method will generally be called when the default
+ // connection has died and the default connection is likely to be the first array element.
+ promoteRandomMetaStoreURI();
+ }
+ open();
+ }
+ }
+
+ private <T extends TTransport> T configureThriftMaxMessageSize(T transport) {
+ int maxThriftMessageSize =
+ (int) MetastoreConf.getSizeVar(conf, ConfVars.THRIFT_METASTORE_CLIENT_MAX_MESSAGE_SIZE);
+ if (maxThriftMessageSize > 0) {
+ if (transport.getConfiguration() == null) {
+ LOG.warn(
+ "TTransport {} is returning a null Configuration, Thrift max message size is not getting configured",
+ transport.getClass().getName());
+ return transport;
+ }
+ transport.getConfiguration().setMaxMessageSize(maxThriftMessageSize);
+ }
+ return transport;
+ }
+
+ private Map<String, String> getAdditionalHeaders() {
+ Map<String, String> headers = new HashMap<>();
+ String keyValuePairs = MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_ADDITIONAL_HEADERS);
+ try {
+ String[] headerKeyValues = keyValuePairs.split(",");
+ for (String header : headerKeyValues) {
+ String[] parts = header.split("=");
+ headers.put(parts[0].trim(), parts[1].trim());
+ }
+ } catch (Exception ex) {
+ LOG.warn(
+ "Could not parse the headers provided in " + ConfVars.METASTORE_CLIENT_ADDITIONAL_HEADERS,
+ ex);
+ }
+ return headers;
+ }
+
+ /*
+ Creates a THttpClient if HTTP mode is enabled. If Client auth mode is set to JWT,
+ then the method fetches JWT from environment variable: HMS_JWT and sets in auth
+ header in http request
+ */
+ private THttpClient createHttpClient(URI store, boolean useSSL)
+ throws MetaException, TTransportException {
+ String path = MetaStoreUtils.getHttpPath(MetastoreConf.getVar(conf, ConfVars.THRIFT_HTTP_PATH));
+ String urlScheme;
+ if (useSSL || Objects.equals(store.getScheme(), "https")) {
+ urlScheme = "https://";
+ } else {
+ urlScheme = "http://";
+ }
+ String httpUrl = urlScheme + store.getHost() + ":" + store.getPort() + path;
+
+ HttpClientBuilder httpClientBuilder = createHttpClientBuilder();
+ THttpClient tHttpClient;
+ try {
+ if (useSSL) {
+ String trustStorePath = MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTSTORE_PATH).trim();
+ if (trustStorePath.isEmpty()) {
+ throw new IllegalArgumentException(
+ ConfVars.SSL_TRUSTSTORE_PATH + " Not configured for SSL connection");
+ }
+ String trustStorePassword =
+ MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_TRUSTSTORE_PASSWORD);
+ String trustStoreType = MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTSTORE_TYPE).trim();
+ String trustStoreAlgorithm =
+ MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTMANAGERFACTORY_ALGORITHM).trim();
+ tHttpClient =
+ SecurityUtils.getThriftHttpsClient(
+ httpUrl,
+ trustStorePath,
+ trustStorePassword,
+ trustStoreAlgorithm,
+ trustStoreType,
+ httpClientBuilder);
+ } else {
+ tHttpClient = new THttpClient(httpUrl, httpClientBuilder.build());
+ }
+ } catch (Exception e) {
+ if (e instanceof TTransportException) {
+ throw (TTransportException) e;
+ } else {
+ throw new MetaException(
+ "Failed to create http transport client to url: " + httpUrl + ". Error:" + e);
+ }
+ }
+ LOG.debug("Created thrift http client for URL: " + httpUrl);
+ return configureThriftMaxMessageSize(tHttpClient);
+ }
+
+ protected HttpClientBuilder createHttpClientBuilder() throws MetaException {
+ HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
+ String authType = MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_AUTH_MODE);
+ Map<String, String> additionalHeaders = getAdditionalHeaders();
+ if (authType.equalsIgnoreCase("jwt")) {
+ // fetch JWT token from environment and set it in Auth Header in HTTP request
+ String jwtToken = System.getenv("HMS_JWT");
+ if (jwtToken == null || jwtToken.isEmpty()) {
+ LOG.debug("No jwt token set in environment variable: HMS_JWT");
+ throw new MetaException(
+ "For auth mode JWT, valid signed jwt token must be provided in the "
+ + "environment variable HMS_JWT");
+ }
+ httpClientBuilder.addInterceptorFirst(
+ new HttpRequestInterceptor() {
+ @Override
+ public void process(HttpRequest httpRequest, HttpContext httpContext)
+ throws HttpException, IOException {
+ httpRequest.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + jwtToken);
+ for (Map.Entry<String, String> entry : additionalHeaders.entrySet()) {
+ httpRequest.addHeader(entry.getKey(), entry.getValue());
+ }
+ }
+ });
+ } else {
+ String user = MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_PLAIN_USERNAME);
+ if (user == null || user.equals("")) {
+ try {
+ user = UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ throw new MetaException("Failed to get client username from UGI");
+ }
+ }
+ final String httpUser = user;
+ httpClientBuilder.addInterceptorFirst(
+ new HttpRequestInterceptor() {
+ @Override
+ public void process(HttpRequest httpRequest, HttpContext httpContext)
+ throws HttpException, IOException {
+ httpRequest.addHeader(MetaStoreUtils.USER_NAME_HTTP_HEADER, httpUser);
+ for (Map.Entry<String, String> entry : additionalHeaders.entrySet()) {
+ httpRequest.addHeader(entry.getKey(), entry.getValue());
+ }
+ }
+ });
+ }
+ return httpClientBuilder;
+ }
+
+ private TTransport createBinaryClient(URI store, boolean useSSL)
+ throws TTransportException, MetaException {
+ TTransport binaryTransport = null;
+ try {
+ int clientSocketTimeout =
+ (int)
+ MetastoreConf.getTimeVar(conf, ConfVars.CLIENT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS);
+ int connectionTimeout =
+ (int)
+ MetastoreConf.getTimeVar(
+ conf, ConfVars.CLIENT_CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS);
+ if (useSSL) {
+ String trustStorePath = MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTSTORE_PATH).trim();
+ if (trustStorePath.isEmpty()) {
+ throw new IllegalArgumentException(
+ ConfVars.SSL_TRUSTSTORE_PATH + " Not configured for SSL connection");
+ }
+ String trustStorePassword =
+ MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_TRUSTSTORE_PASSWORD);
+ String trustStoreType = MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTSTORE_TYPE).trim();
+ String trustStoreAlgorithm =
+ MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTMANAGERFACTORY_ALGORITHM).trim();
+ binaryTransport =
+ SecurityUtils.getSSLSocket(
+ store.getHost(),
+ store.getPort(),
+ clientSocketTimeout,
+ connectionTimeout,
+ trustStorePath,
+ trustStorePassword,
+ trustStoreType,
+ trustStoreAlgorithm);
+ } else {
+ binaryTransport =
+ new TSocket(
+ new TConfiguration(),
+ store.getHost(),
+ store.getPort(),
+ clientSocketTimeout,
+ connectionTimeout);
+ }
+ binaryTransport = createAuthBinaryTransport(store, binaryTransport);
+ } catch (Exception e) {
+ if (e instanceof TTransportException) {
+ throw (TTransportException) e;
+ } else {
+ throw new MetaException(
+ "Failed to create binary transport client to url: " + store + ". Error: " + e);
+ }
+ }
+ LOG.debug("Created thrift binary client for URI: " + store);
+ return configureThriftMaxMessageSize(binaryTransport);
+ }
+
+ private void open() throws MetaException {
+ isConnected = false;
+ TTransportException tte = null;
+ MetaException recentME = null;
+ boolean useSSL = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL);
+ boolean useCompactProtocol =
+ MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_COMPACT_PROTOCOL);
+ String transportMode =
+ MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_THRIFT_TRANSPORT_MODE);
+ boolean isHttpTransportMode = transportMode.equalsIgnoreCase("http");
+
+ for (int attempt = 0; !isConnected && attempt < retries; ++attempt) {
+ for (URI store : metastoreUris) {
+ LOG.info(
+ "Trying to connect to metastore with URI ({}) in {} transport mode",
+ store,
+ transportMode);
+ try {
+ try {
+ if (isHttpTransportMode) {
+ transport = createHttpClient(store, useSSL);
+ } else {
+ transport = createBinaryClient(store, useSSL);
+ }
+ } catch (TTransportException te) {
+ tte = te;
+ throw new MetaException(te.toString());
+ }
+
+ final TProtocol protocol;
+ if (useCompactProtocol) {
+ protocol = new TCompactProtocol(transport);
+ } else {
+ protocol = new TBinaryProtocol(transport);
+ }
+ client = new ThriftHiveMetastore.Client(protocol);
+ try {
+ if (!transport.isOpen()) {
+ transport.open();
+ final int newCount = connCount.incrementAndGet();
+ if (useSSL) {
+ LOG.info(
+ "Opened an SSL connection to metastore, current connections: {}", newCount);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(
+ "METASTORE SSL CONNECTION TRACE - open [{}]",
+ System.identityHashCode(this),
+ new Exception());
+ }
+ } else {
+ LOG.info(
+ "Opened a connection to metastore, URI ({}) " + "current connections: {}",
+ store,
+ newCount);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(
+ "METASTORE CONNECTION TRACE - open [{}]",
+ System.identityHashCode(this),
+ new Exception());
+ }
+ }
+ }
+ isConnected = true;
+ } catch (TTransportException e) {
+ tte = e;
+ String errMsg =
+ String.format(
+ "Failed to connect to the MetaStore Server URI (%s) in %s " + "transport mode",
+ store, transportMode);
+ LOG.warn(errMsg);
+ LOG.debug(errMsg, e);
+ }
+ } catch (MetaException e) {
+ recentME = e;
+ String errMsg =
+ "Failed to connect to metastore with URI ("
+ + store
+ + ") transport mode:"
+ + transportMode
+ + " in attempt "
+ + attempt;
+ LOG.error(errMsg, e);
+ }
+ if (isConnected) {
+ break;
+ }
+ }
+ // Wait before launching the next round of connection retries.
+ if (!isConnected && retryDelaySeconds > 0) {
+ try {
+ LOG.info("Waiting " + retryDelaySeconds + " seconds before next connection attempt.");
+ Thread.sleep(retryDelaySeconds * 1000);
+ } catch (InterruptedException ignore) {
+ }
+ }
+ }
+
+ if (!isConnected) {
+ // Either tte or recentME should be set but protect from a bug which causes both of them to
+ // be null. When MetaException wraps TTransportException, tte will be set so stringify that
+ // directly.
+ String exceptionString = "Unknown exception";
+ if (tte != null) {
+ exceptionString = StringUtils.stringifyException(tte);
+ } else if (recentME != null) {
+ exceptionString = StringUtils.stringifyException(recentME);
+ }
+ throw new MetaException(
+ "Could not connect to meta store using any of the URIs provided."
+ + " Most recent failure: "
+ + exceptionString);
+ }
+
+ snapshotActiveConf();
+ }
+
+ // wraps the underlyingTransport in the appropriate transport based on mode of authentication
+ private TTransport createAuthBinaryTransport(URI store, TTransport underlyingTransport)
+ throws MetaException {
+ boolean isHttpTransportMode =
+ MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_THRIFT_TRANSPORT_MODE)
+ .equalsIgnoreCase("http");
+ if (!isHttpTransportMode) {
+ throw new IllegalArgumentException("HTTP mode is not supported");
+ }
+ Objects.requireNonNull(underlyingTransport, "Underlying transport should not be null");
+ // default transport is the underlying one
+ TTransport transport = underlyingTransport;
+ boolean useFramedTransport =
+ MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_FRAMED_TRANSPORT);
+ boolean useSSL = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL);
+ boolean useSasl = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_SASL);
+ String clientAuthMode = MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_AUTH_MODE);
+ boolean usePasswordAuth = false;
+
+ if (clientAuthMode != null) {
+ usePasswordAuth = "PLAIN".equalsIgnoreCase(clientAuthMode);
+ }
+ if (usePasswordAuth) {
+ // we are using PLAIN Sasl connection with user/password
+ LOG.debug("HMSC::open(): Creating plain authentication thrift connection.");
+ String userName = MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_PLAIN_USERNAME);
+
+ if (null == userName || userName.isEmpty()) {
+ throw new MetaException("No user specified for plain transport.");
+ }
+
+ // The password is not directly provided. It should be obtained from a keystore pointed
+ // by configuration "hadoop.security.credential.provider.path".
+ try {
+ String passwd = null;
+ char[] pwdCharArray = conf.getPassword(userName);
+ if (null != pwdCharArray) {
+ passwd = new String(pwdCharArray);
+ }
+ if (null == passwd) {
+ throw new MetaException("No password found for user " + userName);
+ }
+ // Overlay the SASL transport on top of the base socket transport (SSL or non-SSL)
+ transport =
+ MetaStorePlainSaslHelper.getPlainTransport(userName, passwd, underlyingTransport);
+ } catch (IOException | TTransportException sasle) {
+ // IOException covers SaslException
+ LOG.error("Could not create client transport", sasle);
+ throw new MetaException(sasle.toString());
+ }
+ } else if (useSasl) {
+ // Wrap thrift connection with SASL for secure connection.
+ try {
+ HadoopThriftAuthBridge.Client authBridge =
+ HadoopThriftAuthBridge.getBridge().createClient();
+
+ // check if we should use delegation tokens to authenticate
+ // the call below gets hold of the tokens if they are set up by hadoop
+ // this should happen on the map/reduce tasks if the client added the
+ // tokens into hadoop's credential store in the front end during job
+ // submission.
+ String tokenSig = MetastoreConf.getVar(conf, ConfVars.TOKEN_SIGNATURE);
+ // tokenSig could be null
+ tokenStrForm = SecurityUtils.getTokenStrForm(tokenSig);
+
+ if (tokenStrForm != null) {
+ LOG.debug(
+ "HMSC::open(): Found delegation token. Creating DIGEST-based thrift connection.");
+ // authenticate using delegation tokens via the "DIGEST" mechanism
+ transport =
+ authBridge.createClientTransport(
+ null,
+ store.getHost(),
+ "DIGEST",
+ tokenStrForm,
+ underlyingTransport,
+ MetaStoreUtils.getMetaStoreSaslProperties(conf, useSSL));
+ } else {
+ LOG.debug(
+ "HMSC::open(): Could not find delegation token. Creating KERBEROS-based thrift connection.");
+ String principalConfig = MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL);
+ transport =
+ authBridge.createClientTransport(
+ principalConfig,
+ store.getHost(),
+ "KERBEROS",
+ null,
+ underlyingTransport,
+ MetaStoreUtils.getMetaStoreSaslProperties(conf, useSSL));
+ }
+ } catch (IOException ioe) {
+ LOG.error("Failed to create client transport", ioe);
+ throw new MetaException(ioe.toString());
+ }
+ } else {
+ if (useFramedTransport) {
+ try {
+ transport = new TFramedTransport(transport);
+ } catch (TTransportException e) {
+ LOG.error("Failed to create client transport", e);
+ throw new MetaException(e.toString());
+ }
+ }
+ }
+ return transport;
+ }
+
+ private void snapshotActiveConf() {
+ currentMetaVars = new HashMap<>(MetastoreConf.metaVars.length);
+ for (ConfVars oneVar : MetastoreConf.metaVars) {
+ currentMetaVars.put(oneVar.getVarname(), MetastoreConf.getAsString(conf, oneVar));
+ }
+ }
+
+ @Override
+ public void close() {
+ isConnected = false;
+ currentMetaVars = null;
+ try {
+ if (null != client) {
+ client.shutdown();
+ if ((transport == null) || !transport.isOpen()) {
+ final int newCount = connCount.decrementAndGet();
+ LOG.info("Closed a connection to metastore, current connections: {}", newCount);
+ }
+ }
+ } catch (TException e) {
+ LOG.debug("Unable to shutdown metastore client. Will try closing transport directly.", e);
+ }
+ // Transport would have got closed via client.shutdown(), so we dont need this, but
+ // just in case, we make this call.
+ if ((transport != null) && transport.isOpen()) {
+ transport.close();
+ final int newCount = connCount.decrementAndGet();
+ LOG.info("Closed a connection to metastore, current connections: {}", newCount);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(
+ "METASTORE CONNECTION TRACE - close [{}]",
+ System.identityHashCode(this),
+ new Exception());
+ }
+ }
+ }
+
+ @Override
+ public String getDelegationToken(String owner, String renewerKerberosPrincipalName)
+ throws MetaException, TException {
+ // This is expected to be a no-op, so we will return null when we use local metastore.
+ if (localMetaStore) {
+ return null;
+ }
+ return client.get_delegation_token(owner, renewerKerberosPrincipalName);
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/IMetaStoreClient.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/IMetaStoreClient.java
new file mode 100644
index 0000000..fd054a8
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/IMetaStoreClient.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.shaded.hive.metastore;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.kyuubi.shaded.hive.metastore.annotation.NoReconnect;
+import org.apache.kyuubi.shaded.hive.metastore.api.MetaException;
+import org.apache.thrift.TException;
+
+/** Wrapper around hive metastore thrift api */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface IMetaStoreClient extends AutoCloseable {
+
+ /**
+ * Returns true if the current client is using an in process metastore (local metastore).
+ *
+ * @return
+ */
+ boolean isLocalMetaStore();
+
+ /** Tries to reconnect this MetaStoreClient to the MetaStore. */
+ void reconnect() throws MetaException;
+
+ /** close connection to meta store */
+ @NoReconnect
+ void close();
+
+ /**
+ * This is expected to be a no-op when in local mode, which means that the implementation will
+ * return null.
+ *
+ * @param owner the intended owner for the token
+ * @param renewerKerberosPrincipalName
+ * @return the string of the token
+ * @throws MetaException
+ * @throws TException
+ */
+ String getDelegationToken(String owner, String renewerKerberosPrincipalName)
+ throws MetaException, TException;
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/MetaStorePlainSaslHelper.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/MetaStorePlainSaslHelper.java
new file mode 100644
index 0000000..9828fe5
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/MetaStorePlainSaslHelper.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.shaded.hive.metastore;
+
+import java.io.IOException;
+import java.util.HashMap;
+import javax.security.auth.callback.Callback;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.callback.NameCallback;
+import javax.security.auth.callback.PasswordCallback;
+import javax.security.auth.callback.UnsupportedCallbackException;
+import javax.security.sasl.SaslException;
+import org.apache.thrift.transport.TSaslClientTransport;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+// This is a copy of org.apache.hive.service.auth.PlainSaslHelper modified for HMS. See whether
+// we can deduplicate the code.
+public final class MetaStorePlainSaslHelper {
+
+ public static TTransport getPlainTransport(
+ String username, String password, TTransport underlyingTransport)
+ throws SaslException, TTransportException {
+ return new TSaslClientTransport(
+ "PLAIN",
+ null,
+ null,
+ null,
+ new HashMap<String, String>(),
+ new PlainCallbackHandler(username, password),
+ underlyingTransport);
+ }
+
+ private MetaStorePlainSaslHelper() {
+ throw new UnsupportedOperationException("Can't initialize class");
+ }
+
+ public static class PlainCallbackHandler implements CallbackHandler {
+
+ private final String username;
+ private final String password;
+
+ public PlainCallbackHandler(String username, String password) {
+ this.username = username;
+ this.password = password;
+ }
+
+ @Override
+ public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException {
+ for (Callback callback : callbacks) {
+ if (callback instanceof NameCallback) {
+ NameCallback nameCallback = (NameCallback) callback;
+ nameCallback.setName(username);
+ } else if (callback instanceof PasswordCallback) {
+ PasswordCallback passCallback = (PasswordCallback) callback;
+ passCallback.setPassword(password.toCharArray());
+ } else {
+ throw new UnsupportedCallbackException(callback);
+ }
+ }
+ }
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/RetryingMetaStoreClient.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/RetryingMetaStoreClient.java
new file mode 100644
index 0000000..bc432db
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/RetryingMetaStoreClient.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.shaded.hive.metastore;
+
+import java.io.IOException;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.kyuubi.shaded.hive.common.classification.RetrySemantics;
+import org.apache.kyuubi.shaded.hive.metastore.annotation.NoReconnect;
+import org.apache.kyuubi.shaded.hive.metastore.api.MetaException;
+import org.apache.kyuubi.shaded.hive.metastore.conf.MetastoreConf;
+import org.apache.kyuubi.shaded.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.kyuubi.shaded.hive.metastore.utils.JavaUtils;
+import org.apache.kyuubi.shaded.hive.metastore.utils.SecurityUtils;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * RetryingMetaStoreClient. Creates a proxy for a IMetaStoreClient implementation and retries calls
+ * to it on failure. If the login user is authenticated using keytab, it relogins user before each
+ * call.
+ */
+@InterfaceAudience.Public
+public class RetryingMetaStoreClient implements InvocationHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RetryingMetaStoreClient.class.getName());
+
+ private final IMetaStoreClient base;
+ private final UserGroupInformation ugi;
+ private final int retryLimit;
+ private final long retryDelaySeconds;
+ private final ConcurrentHashMap<String, Long> metaCallTimeMap;
+ private final long connectionLifeTimeInMillis;
+ private long lastConnectionTime;
+ private boolean localMetaStore;
+
+ protected RetryingMetaStoreClient(
+ Configuration conf,
+ Class<?>[] constructorArgTypes,
+ Object[] constructorArgs,
+ ConcurrentHashMap<String, Long> metaCallTimeMap,
+ Class<? extends IMetaStoreClient> msClientClass)
+ throws MetaException {
+
+ this.ugi = getUGI();
+
+ if (this.ugi == null) {
+ LOG.warn("RetryingMetaStoreClient unable to determine current user UGI.");
+ }
+
+ this.retryLimit = MetastoreConf.getIntVar(conf, ConfVars.THRIFT_FAILURE_RETRIES);
+ this.retryDelaySeconds =
+ MetastoreConf.getTimeVar(conf, ConfVars.CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS);
+ this.metaCallTimeMap = metaCallTimeMap;
+ this.connectionLifeTimeInMillis =
+ MetastoreConf.getTimeVar(conf, ConfVars.CLIENT_SOCKET_LIFETIME, TimeUnit.MILLISECONDS);
+ this.lastConnectionTime = System.currentTimeMillis();
+ String msUri = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS);
+ localMetaStore = (msUri == null) || msUri.trim().isEmpty();
+
+ SecurityUtils.reloginExpiringKeytabUser();
+
+ this.base = JavaUtils.newInstance(msClientClass, constructorArgTypes, constructorArgs);
+
+ LOG.info(
+ "RetryingMetaStoreClient proxy="
+ + msClientClass
+ + " ugi="
+ + this.ugi
+ + " retries="
+ + this.retryLimit
+ + " delay="
+ + this.retryDelaySeconds
+ + " lifetime="
+ + this.connectionLifeTimeInMillis);
+ }
+
+ public static IMetaStoreClient getProxy(Configuration hiveConf) throws MetaException {
+ return getProxy(
+ hiveConf,
+ new Class[] {Configuration.class},
+ new Object[] {hiveConf},
+ null,
+ HiveMetaStoreClient.class.getName());
+ }
+
+ /**
+ * This constructor is meant for Hive internal use only. Please use getProxy(HiveConf conf,
+ * HiveMetaHookLoader hookLoader) for external purpose.
+ */
+ public static IMetaStoreClient getProxy(
+ Configuration hiveConf,
+ Class<?>[] constructorArgTypes,
+ Object[] constructorArgs,
+ ConcurrentHashMap<String, Long> metaCallTimeMap,
+ String mscClassName)
+ throws MetaException {
+
+ @SuppressWarnings("unchecked")
+ Class<? extends IMetaStoreClient> baseClass =
+ JavaUtils.getClass(mscClassName, IMetaStoreClient.class);
+
+ RetryingMetaStoreClient handler =
+ new RetryingMetaStoreClient(
+ hiveConf, constructorArgTypes, constructorArgs, metaCallTimeMap, baseClass);
+ return (IMetaStoreClient)
+ Proxy.newProxyInstance(
+ RetryingMetaStoreClient.class.getClassLoader(), baseClass.getInterfaces(), handler);
+ }
+
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+ Object ret;
+ int retriesMade = 0;
+ TException caughtException;
+
+ boolean allowReconnect = !method.isAnnotationPresent(NoReconnect.class);
+ boolean allowRetry = true;
+ Annotation[] directives = method.getDeclaredAnnotations();
+ if (directives != null) {
+ for (Annotation a : directives) {
+ if (a instanceof RetrySemantics.CannotRetry) {
+ allowRetry = false;
+ }
+ }
+ }
+
+ while (true) {
+ try {
+ SecurityUtils.reloginExpiringKeytabUser();
+
+ if (allowReconnect) {
+ if (retriesMade > 0 || hasConnectionLifeTimeReached(method)) {
+ if (this.ugi != null) {
+ // Perform reconnect with the proper user context
+ try {
+ LOG.info("RetryingMetaStoreClient trying reconnect as " + this.ugi);
+
+ this.ugi.doAs(
+ new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws MetaException {
+ base.reconnect();
+ return null;
+ }
+ });
+ } catch (UndeclaredThrowableException e) {
+ Throwable te = e.getCause();
+ if (te instanceof PrivilegedActionException) {
+ throw te.getCause();
+ } else {
+ throw te;
+ }
+ }
+ lastConnectionTime = System.currentTimeMillis();
+ } else {
+ LOG.warn("RetryingMetaStoreClient unable to reconnect. No UGI information.");
+ throw new MetaException("UGI information unavailable. Will not attempt a reconnect.");
+ }
+ }
+ }
+
+ if (metaCallTimeMap == null) {
+ ret = method.invoke(base, args);
+ } else {
+ // need to capture the timing
+ long startTime = System.currentTimeMillis();
+ ret = method.invoke(base, args);
+ long timeTaken = System.currentTimeMillis() - startTime;
+ addMethodTime(method, timeTaken);
+ }
+ break;
+ } catch (UndeclaredThrowableException e) {
+ throw e.getCause();
+ } catch (InvocationTargetException e) {
+ Throwable t = e.getCause();
+ // Metastore client needs retry for only TTransportException.
+ if (TTransportException.class.isAssignableFrom(t.getClass())) {
+ caughtException = (TTransportException) t;
+ } else {
+ throw t;
+ }
+ }
+
+ if (retriesMade >= retryLimit || base.isLocalMetaStore() || !allowRetry) {
+ throw caughtException;
+ }
+ retriesMade++;
+ LOG.warn(
+ "MetaStoreClient lost connection. Attempting to reconnect ("
+ + retriesMade
+ + " of "
+ + retryLimit
+ + ") after "
+ + retryDelaySeconds
+ + "s. "
+ + method.getName(),
+ caughtException);
+ Thread.sleep(retryDelaySeconds * 1000);
+ }
+ return ret;
+ }
+
+ /**
+ * Returns the UGI for the current user.
+ *
+ * @return the UGI for the current user.
+ */
+ private UserGroupInformation getUGI() {
+ UserGroupInformation ugi = null;
+
+ try {
+ ugi = UserGroupInformation.getCurrentUser();
+ } catch (IOException e) {
+ // Swallow the exception and let the call determine what to do.
+ }
+
+ return ugi;
+ }
+
+ private void addMethodTime(Method method, long timeTaken) {
+ String methodStr = getMethodString(method);
+ while (true) {
+ Long curTime = metaCallTimeMap.get(methodStr), newTime = timeTaken;
+ if (curTime != null && metaCallTimeMap.replace(methodStr, curTime, newTime + curTime)) break;
+ if (curTime == null && (null == metaCallTimeMap.putIfAbsent(methodStr, newTime))) break;
+ }
+ }
+
+ /**
+ * @param method
+ * @return String representation with arg types. eg getDatabase_(String, )
+ */
+ private String getMethodString(Method method) {
+ StringBuilder methodSb = new StringBuilder(method.getName());
+ methodSb.append("_(");
+ if (method.getParameterTypes().length != 0) {
+ Iterator<Class<?>> it = Arrays.stream(method.getParameterTypes()).iterator();
+ methodSb.append(it.next().getSimpleName());
+ while (it.hasNext()) {
+ methodSb.append(", ");
+ methodSb.append(it.next().getSimpleName());
+ }
+ }
+ methodSb.append(")");
+ return methodSb.toString();
+ }
+
+ private boolean hasConnectionLifeTimeReached(Method method) {
+ if (connectionLifeTimeInMillis <= 0 || localMetaStore) {
+ return false;
+ }
+
+ boolean shouldReconnect =
+ (System.currentTimeMillis() - lastConnectionTime) >= connectionLifeTimeInMillis;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reconnection status for Method: " + method.getName() + " is " + shouldReconnect);
+ }
+ return shouldReconnect;
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/annotation/NoReconnect.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/annotation/NoReconnect.java
new file mode 100644
index 0000000..dcc701f
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/annotation/NoReconnect.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.kyuubi.shaded.hive.metastore.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface NoReconnect {}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/MetastoreConf.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/MetastoreConf.java
new file mode 100644
index 0000000..c803ec4
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/MetastoreConf.java
@@ -0,0 +1,1263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.shaded.hive.metastore.conf;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.kyuubi.shaded.hive.common.ZooKeeperHiveHelper;
+import org.apache.kyuubi.shaded.hive.metastore.utils.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A set of definitions of config values used by the Metastore. One of the key aims of this class is
+ * to provide backwards compatibility with existing Hive configuration keys while allowing the
+ * metastore to have its own, Hive independent keys. For this reason access to the underlying
+ * Configuration object should always be done via the static methods provided here rather than
+ * directly via {@link Configuration#get(String)} and {@link Configuration#set(String, String)}. All
+ * the methods of this class will handle checking both the MetastoreConf key and the Hive key. The
+ * algorithm is, on reads, to check first the MetastoreConf key, then the Hive key, then return the
+ * default if neither are set. On write the Metastore key only is set.
+ *
+ * <p>This class does not extend Configuration. Rather it provides static methods for operating on a
+ * Configuration object. This allows it to work on HiveConf objects, which otherwise would not be
+ * the case.
+ */
+public class MetastoreConf {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MetastoreConf.class);
+ private static final Pattern TIME_UNIT_SUFFIX = Pattern.compile("([0-9]+)([a-zA-Z]+)");
+
+ private static final Map<String, ConfVars> metaConfs = new HashMap<>();
+ private static volatile URL hiveSiteURL = null;
+ private static URL hiveDefaultURL = null;
+ private static URL hiveMetastoreSiteURL = null;
+ private static URL metastoreSiteURL = null;
+ private static AtomicBoolean beenDumped = new AtomicBoolean();
+
+ private static Map<String, ConfVars> keyToVars;
+
+ static {
+ keyToVars = new HashMap<>(ConfVars.values().length * 2);
+ for (ConfVars var : ConfVars.values()) {
+ keyToVars.put(var.varname, var);
+ keyToVars.put(var.hiveName, var);
+ }
+ }
+
+ static final String TEST_ENV_WORKAROUND = "metastore.testing.env.workaround.dont.ever.set.this.";
+
+ private static class TimeValue {
+ final long val;
+ final TimeUnit unit;
+
+ private TimeValue(long val, TimeUnit unit) {
+ this.val = val;
+ this.unit = unit;
+ }
+
+ @Override
+ public String toString() {
+ switch (unit) {
+ case NANOSECONDS:
+ return Long.toString(val) + "ns";
+ case MICROSECONDS:
+ return Long.toString(val) + "us";
+ case MILLISECONDS:
+ return Long.toString(val) + "ms";
+ case SECONDS:
+ return Long.toString(val) + "s";
+ case MINUTES:
+ return Long.toString(val) + "m";
+ case HOURS:
+ return Long.toString(val) + "h";
+ case DAYS:
+ return Long.toString(val) + "d";
+ }
+ throw new RuntimeException("Unknown time unit " + unit);
+ }
+ }
+
+ /**
+ * Metastore related options that the db is initialized against. When a conf var in this is list
+ * is changed, the metastore instance for the CLI will be recreated so that the change will take
+ * effect. TODO - I suspect the vast majority of these don't need to be here. But it requires
+ * testing before just pulling them out.
+ */
+ public static final MetastoreConf.ConfVars[] metaVars = {
+ ConfVars.THRIFT_URIS,
+ ConfVars.THRIFT_ZOOKEEPER_CLIENT_PORT,
+ ConfVars.THRIFT_ZOOKEEPER_NAMESPACE,
+ ConfVars.THRIFT_CONNECTION_RETRIES,
+ ConfVars.THRIFT_FAILURE_RETRIES,
+ ConfVars.CLIENT_CONNECT_RETRY_DELAY,
+ ConfVars.CLIENT_SOCKET_TIMEOUT,
+ ConfVars.CLIENT_SOCKET_LIFETIME,
+ ConfVars.KERBEROS_PRINCIPAL,
+ ConfVars.USE_THRIFT_SASL,
+ ConfVars.METASTORE_CLIENT_AUTH_MODE,
+ ConfVars.METASTORE_CLIENT_PLAIN_USERNAME
+ };
+
+ /** User configurable Metastore vars */
+ private static final MetastoreConf.ConfVars[] metaConfVars = {ConfVars.CLIENT_SOCKET_TIMEOUT};
+
+ static {
+ for (ConfVars confVar : metaConfVars) {
+ metaConfs.put(confVar.varname, confVar);
+ metaConfs.put(confVar.hiveName, confVar);
+ }
+ }
+
+ /** Variables that we should never print the value of for security reasons. */
+ private static final Set<String> unprintables =
+ StringUtils.asSet(
+ ConfVars.SSL_KEYSTORE_PASSWORD.varname,
+ ConfVars.SSL_KEYSTORE_PASSWORD.hiveName,
+ ConfVars.SSL_TRUSTSTORE_PASSWORD.varname,
+ ConfVars.SSL_TRUSTSTORE_PASSWORD.hiveName,
+ ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_PASSWORD.varname,
+ ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_PASSWORD.hiveName,
+ ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD.varname,
+ ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD.hiveName);
+
+ public static ConfVars getMetaConf(String name) {
+ return metaConfs.get(name);
+ }
+
+ public enum ConfVars {
+ // alpha order, PLEASE!
+ CLIENT_CONNECT_RETRY_DELAY(
+ "metastore.client.connect.retry.delay",
+ "hive.metastore.client.connect.retry.delay",
+ 1,
+ TimeUnit.SECONDS,
+ "Number of seconds for the client to wait between consecutive connection attempts"),
+ CLIENT_KERBEROS_PRINCIPAL(
+ "metastore.client.kerberos.principal",
+ "hive.metastore.client.kerberos.principal",
+ "", // E.g. "hive-metastore/_HOST@EXAMPLE.COM".
+ "The Kerberos principal associated with the HA cluster of hcat_servers."),
+ CLIENT_SOCKET_LIFETIME(
+ "metastore.client.socket.lifetime",
+ "hive.metastore.client.socket.lifetime",
+ 0,
+ TimeUnit.SECONDS,
+ "MetaStore Client socket lifetime in seconds. After this time is exceeded, client\n"
+ + "reconnects on the next MetaStore operation. A value of 0s means the connection\n"
+ + "has an infinite lifetime."),
+ CLIENT_SOCKET_TIMEOUT(
+ "metastore.client.socket.timeout",
+ "hive.metastore.client.socket.timeout",
+ 600,
+ TimeUnit.SECONDS,
+ "MetaStore Client socket timeout in seconds"),
+ CLIENT_CONNECTION_TIMEOUT(
+ "metastore.client.connection.timeout",
+ "hive.metastore.client.connection.timeout",
+ 600,
+ TimeUnit.SECONDS,
+ "MetaStore Client connection timeout in seconds"),
+ DUMP_CONFIG_ON_CREATION(
+ "metastore.dump.config.on.creation",
+ "metastore.dump.config.on.creation",
+ true,
+ "If true, a printout of the config file (minus sensitive values) will be dumped to the "
+ + "log whenever newMetastoreConf() is called. Can produce a lot of logs"),
+ KERBEROS_PRINCIPAL(
+ "metastore.kerberos.principal",
+ "hive.metastore.kerberos.principal",
+ "hive-metastore/_HOST@EXAMPLE.COM",
+ "The service principal for the metastore Thrift server. \n"
+ + "The special string _HOST will be replaced automatically with the correct host name."),
+ SSL_KEYSTORE_PASSWORD(
+ "metastore.keystore.password",
+ "hive.metastore.keystore.password",
+ "",
+ "Metastore SSL certificate keystore password."),
+ SSL_TRUSTSTORE_PATH(
+ "metastore.truststore.path",
+ "hive.metastore.truststore.path",
+ "",
+ "Metastore SSL certificate truststore location."),
+ SSL_TRUSTSTORE_PASSWORD(
+ "metastore.truststore.password",
+ "hive.metastore.truststore.password",
+ "",
+ "Metastore SSL certificate truststore password."),
+ SSL_TRUSTSTORE_TYPE(
+ "metastore.truststore.type",
+ "hive.metastore.truststore.type",
+ "",
+ "Metastore SSL certificate truststore type."),
+ SSL_TRUSTMANAGERFACTORY_ALGORITHM(
+ "metastore.trustmanagerfactory.algorithm",
+ "hive.metastore.trustmanagerfactory.algorithm",
+ "",
+ "Metastore SSL certificate truststore algorithm."),
+ THRIFT_TRANSPORT_MODE(
+ "metastore.server.thrift.transport.mode",
+ "hive.metastore.server.thrift.transport.mode",
+ "binary",
+ "Transport mode for thrift server in Metastore. Can be binary or http"),
+ THRIFT_HTTP_PATH(
+ "metastore.server.thrift.http.path",
+ "hive.metastore.server.thrift.http.path",
+ "metastore",
+ "Path component of URL endpoint when in HTTP mode"),
+ THRIFT_CONNECTION_RETRIES(
+ "metastore.connect.retries",
+ "hive.metastore.connect.retries",
+ 3,
+ "Number of retries while opening a connection to metastore"),
+ THRIFT_FAILURE_RETRIES(
+ "metastore.failure.retries",
+ "hive.metastore.failure.retries",
+ 1,
+ "Number of retries upon failure of Thrift metastore calls"),
+ THRIFT_URIS(
+ "metastore.thrift.uris",
+ "hive.metastore.uris",
+ "",
+ "URIs Used by metastore client to connect to remotemetastore\n."
+ + "If dynamic service discovery mode is set, the URIs are used to connect to the"
+ + " corresponding service discovery servers e.g. a zookeeper. Otherwise they are "
+ + "used as URIs for remote metastore."),
+ THRIFT_METASTORE_CLIENT_MAX_MESSAGE_SIZE(
+ "metastore.thrift.client.max.message.size",
+ "hive.thrift.client.max.message.size",
+ "1gb",
+ new SizeValidator(-1L, true, (long) Integer.MAX_VALUE, true),
+ "Thrift client configuration for max message size. 0 or -1 will use the default defined in the Thrift "
+ + "library. The upper limit is 2147483648 bytes (or 2gb)."),
+ THRIFT_SERVICE_DISCOVERY_MODE(
+ "metastore.service.discovery.mode",
+ "hive.metastore.service.discovery.mode",
+ "",
+ "Specifies which dynamic service discovery method to use. Currently we support only "
+ + "\"zookeeper\" to specify ZooKeeper based service discovery."),
+ THRIFT_ZOOKEEPER_CLIENT_PORT(
+ "metastore.zookeeper.client.port",
+ "hive.zookeeper.client.port",
+ "2181",
+ "The port of ZooKeeper servers to talk to.\n"
+ + "If the list of Zookeeper servers specified in hive.metastore.thrift.uris"
+ + " does not contain port numbers, this value is used."),
+ THRIFT_ZOOKEEPER_SESSION_TIMEOUT(
+ "metastore.zookeeper.session.timeout",
+ "hive.zookeeper.session.timeout",
+ 120000L,
+ TimeUnit.MILLISECONDS,
+ new TimeValidator(TimeUnit.MILLISECONDS),
+ "ZooKeeper client's session timeout (in milliseconds). The client is disconnected\n"
+ + "if a heartbeat is not sent in the timeout."),
+ THRIFT_ZOOKEEPER_CONNECTION_TIMEOUT(
+ "metastore.zookeeper.connection.timeout",
+ "hive.zookeeper.connection.timeout",
+ 15L,
+ TimeUnit.SECONDS,
+ new TimeValidator(TimeUnit.SECONDS),
+ "ZooKeeper client's connection timeout in seconds. "
+ + "Connection timeout * hive.metastore.zookeeper.connection.max.retries\n"
+ + "with exponential backoff is when curator client deems connection is lost to zookeeper."),
+ THRIFT_ZOOKEEPER_NAMESPACE(
+ "metastore.zookeeper.namespace",
+ "hive.zookeeper.namespace",
+ "hive_metastore",
+ "The parent node under which all ZooKeeper nodes for metastores are created."),
+ THRIFT_ZOOKEEPER_CONNECTION_MAX_RETRIES(
+ "metastore.zookeeper.connection.max.retries",
+ "hive.zookeeper.connection.max.retries",
+ 3,
+ "Max number of times to retry when connecting to the ZooKeeper server."),
+ THRIFT_ZOOKEEPER_CONNECTION_BASESLEEPTIME(
+ "metastore.zookeeper.connection.basesleeptime",
+ "hive.zookeeper.connection.basesleeptime",
+ 1000L,
+ TimeUnit.MILLISECONDS,
+ new TimeValidator(TimeUnit.MILLISECONDS),
+ "Initial amount of time (in milliseconds) to wait between retries\n"
+ + "when connecting to the ZooKeeper server when using ExponentialBackoffRetry policy."),
+ THRIFT_ZOOKEEPER_SSL_ENABLE(
+ "metastore.zookeeper.ssl.client.enable",
+ "hive.zookeeper.ssl.client.enable",
+ false,
+ "Set client to use TLS when connecting to ZooKeeper. An explicit value overrides any value set via the "
+ + "zookeeper.client.secure system property (note the different name). Defaults to false if neither is set."),
+ THRIFT_ZOOKEEPER_SSL_KEYSTORE_LOCATION(
+ "metastore.zookeeper.ssl.keystore.location",
+ "hive.zookeeper.ssl.keystore.location",
+ "",
+ "Keystore location when using a client-side certificate with TLS connectivity to ZooKeeper. "
+ + "Overrides any explicit value set via the zookeeper.ssl.keyStore.location "
+ + "system property (note the camelCase)."),
+ THRIFT_ZOOKEEPER_SSL_KEYSTORE_PASSWORD(
+ "metastore.zookeeper.ssl.keystore.password",
+ "hive.zookeeper.ssl.keystore.password",
+ "",
+ "Keystore password when using a client-side certificate with TLS connectivity to ZooKeeper."
+ + "Overrides any explicit value set via the zookeeper.ssl.keyStore.password"
+ + "system property (note the camelCase)."),
+ THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION(
+ "metastore.zookeeper.ssl.truststore.location",
+ "hive.zookeeper.ssl.truststore.location",
+ "",
+ "Truststore location when using a client-side certificate with TLS connectivity to ZooKeeper. "
+ + "Overrides any explicit value set via the zookeeper.ssl.trustStore.location "
+ + "system property (note the camelCase)."),
+ THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD(
+ "metastore.zookeeper.ssl.truststore.password",
+ "hive.zookeeper.ssl.truststore.password",
+ "",
+ "Truststore password when using a client-side certificate with TLS connectivity to ZooKeeper."
+ + "Overrides any explicit value set via the zookeeper.ssl.trustStore.password "
+ + "system property (note the camelCase)."),
+ THRIFT_URI_SELECTION(
+ "metastore.thrift.uri.selection",
+ "hive.metastore.uri.selection",
+ "RANDOM",
+ new StringSetValidator("RANDOM", "SEQUENTIAL"),
+ "Determines the selection mechanism used by metastore client to connect to remote "
+ + "metastore. SEQUENTIAL implies that the first valid metastore from the URIs specified "
+ + "through hive.metastore.uris will be picked. RANDOM implies that the metastore "
+ + "will be picked randomly"),
+ TOKEN_SIGNATURE(
+ "metastore.token.signature",
+ "hive.metastore.token.signature",
+ "",
+ "The delegation token service name to match when selecting a token from the current user's tokens."),
+ URI_RESOLVER(
+ "metastore.uri.resolver",
+ "hive.metastore.uri.resolver",
+ "",
+ "If set, fully qualified class name of resolver for hive metastore uri's"),
+ // TODO: Should we have a separate config for the metastoreclient or THRIFT_TRANSPORT_MODE
+ // would suffice ?
+ METASTORE_CLIENT_THRIFT_TRANSPORT_MODE(
+ "metastore.client.thrift.transport.mode",
+ "hive.metastore.client.thrift.transport.mode",
+ "binary",
+ "Transport mode to be used by the metastore client. It should be the same as "
+ + THRIFT_TRANSPORT_MODE),
+ USE_SSL(
+ "metastore.use.SSL",
+ "hive.metastore.use.SSL",
+ false,
+ "Set this to true for using SSL encryption in HMS server."),
+ // We should somehow unify next two options.
+ USE_THRIFT_SASL(
+ "metastore.sasl.enabled",
+ "hive.metastore.sasl.enabled",
+ false,
+ "If true, the metastore Thrift interface will be secured with SASL. Clients must authenticate with Kerberos."),
+ METASTORE_CLIENT_AUTH_MODE(
+ "metastore.client.auth.mode",
+ "hive.metastore.client.auth.mode",
+ "NOSASL",
+ new StringSetValidator("NOSASL", "PLAIN", "KERBEROS", "JWT"),
+ "If PLAIN, clients will authenticate using plain authentication, by providing username"
+ + " and password. Any other value is ignored right now but may be used later."
+ + "If JWT- Supported only in HTTP transport mode. If set, HMS Client will pick the value of JWT from "
+ + "environment variable HMS_JWT and set it in Authorization header in http request"),
+ METASTORE_CLIENT_ADDITIONAL_HEADERS(
+ "metastore.client.http.additional.headers",
+ "hive.metastore.client.http.additional.headers",
+ "",
+ "Comma separated list of headers which are passed to the metastore service in the http headers"),
+ METASTORE_CLIENT_PLAIN_USERNAME(
+ "metastore.client.plain.username",
+ "hive.metastore.client.plain.username",
+ "",
+ "The username used by the metastore client when "
+ + METASTORE_CLIENT_AUTH_MODE
+ + " is true. The password is obtained from "
+ + CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH
+ + " using username as the "
+ + "alias."),
+ USE_THRIFT_FRAMED_TRANSPORT(
+ "metastore.thrift.framed.transport.enabled",
+ "hive.metastore.thrift.framed.transport.enabled",
+ false,
+ "If true, the metastore Thrift interface will use TFramedTransport. When false (default) a standard TTransport is used."),
+ USE_THRIFT_COMPACT_PROTOCOL(
+ "metastore.thrift.compact.protocol.enabled",
+ "hive.metastore.thrift.compact.protocol.enabled",
+ false,
+ "If true, the metastore Thrift interface will use TCompactProtocol. When false (default) TBinaryProtocol will be used.\n"
+ + "Setting it to true will break compatibility with older clients running TBinaryProtocol.");
+
+ private final String varname;
+ private final String hiveName;
+ private final Object defaultVal;
+ private final Validator validator;
+ private final boolean caseSensitive;
+ private final String description;
+ private String deprecatedName = null;
+ private String hiveDeprecatedName = null;
+
+ ConfVars(String varname, String hiveName, String defaultVal, String description) {
+ this.varname = varname;
+ this.hiveName = hiveName;
+ this.defaultVal = defaultVal;
+ validator = null;
+ caseSensitive = false;
+ this.description = description;
+ }
+
+ ConfVars(
+ String varname,
+ String hiveName,
+ String defaultVal,
+ Validator validator,
+ String description) {
+ this.varname = varname;
+ this.hiveName = hiveName;
+ this.defaultVal = defaultVal;
+ this.validator = validator;
+ caseSensitive = false;
+ this.description = description;
+ }
+
+ ConfVars(
+ String varname,
+ String hiveName,
+ String defaultVal,
+ boolean caseSensitive,
+ String description) {
+ this.varname = varname;
+ this.hiveName = hiveName;
+ this.defaultVal = defaultVal;
+ validator = null;
+ this.caseSensitive = caseSensitive;
+ this.description = description;
+ }
+
+ ConfVars(String varname, String hiveName, long defaultVal, String description) {
+ this.varname = varname;
+ this.hiveName = hiveName;
+ this.defaultVal = defaultVal;
+ validator = null;
+ caseSensitive = false;
+ this.description = description;
+ }
+
+ ConfVars(
+ String varname, String hiveName, long defaultVal, Validator validator, String description) {
+ this.varname = varname;
+ this.hiveName = hiveName;
+ this.defaultVal = defaultVal;
+ this.validator = validator;
+ caseSensitive = false;
+ this.description = description;
+ }
+
+ ConfVars(
+ String varname,
+ String hiveName,
+ long defaultVal,
+ Validator validator,
+ String description,
+ String deprecatedName,
+ String hiveDeprecatedName) {
+ this.varname = varname;
+ this.hiveName = hiveName;
+ this.defaultVal = defaultVal;
+ this.validator = validator;
+ caseSensitive = false;
+ this.description = description;
+ this.deprecatedName = deprecatedName;
+ this.hiveDeprecatedName = hiveDeprecatedName;
+ }
+
+ ConfVars(String varname, String hiveName, boolean defaultVal, String description) {
+ this.varname = varname;
+ this.hiveName = hiveName;
+ this.defaultVal = defaultVal;
+ validator = null;
+ caseSensitive = false;
+ this.description = description;
+ }
+
+ ConfVars(String varname, String hiveName, double defaultVal, String description) {
+ this.varname = varname;
+ this.hiveName = hiveName;
+ this.defaultVal = defaultVal;
+ validator = null;
+ caseSensitive = false;
+ this.description = description;
+ }
+
+ ConfVars(String varname, String hiveName, long defaultVal, TimeUnit unit, String description) {
+ this.varname = varname;
+ this.hiveName = hiveName;
+ this.defaultVal = new TimeValue(defaultVal, unit);
+ validator = new TimeValidator(unit);
+ caseSensitive = false;
+ this.description = description;
+ }
+
+ ConfVars(
+ String varname,
+ String hiveName,
+ long defaultVal,
+ TimeUnit unit,
+ Validator validator,
+ String description) {
+ this.varname = varname;
+ this.hiveName = hiveName;
+ this.defaultVal = new TimeValue(defaultVal, unit);
+ this.validator = validator;
+ caseSensitive = false;
+ this.description = description;
+ }
+
+ public void validate(String value) throws IllegalArgumentException {
+ if (validator != null) {
+ validator.validate(value);
+ }
+ }
+
+ public boolean isCaseSensitive() {
+ return caseSensitive;
+ }
+
+ /**
+ * If you are calling this, you're probably doing it wrong. You shouldn't need to use the
+ * underlying variable name. Use one of the getVar methods instead. Only use this if you are
+ * 100% sure you know you're doing. The reason for this is that MetastoreConf goes to a lot of
+ * trouble to make sure it checks both Hive and Metastore values for config keys. If you call
+ * {@link Configuration#get(String)} you are undermining that.
+ *
+ * @return variable name
+ */
+ public String getVarname() {
+ return varname;
+ }
+
+ /**
+ * Use this method if you need to set a system property and are going to instantiate the
+ * configuration file via HiveConf. This is because HiveConf only looks for values it knows, so
+ * it will miss all of the metastore.* ones. Do not use this to explicitly set or get the
+ * underlying config value unless you are 100% sure you know what you're doing. The reason for
+ * this is that MetastoreConf goes to a lot of trouble to make sure it checks both Hive and
+ * Metastore values for config keys. If you call {@link Configuration#get(String)} you are
+ * undermining that.
+ *
+ * @return hive.* configuration key
+ */
+ public String getHiveName() {
+ return hiveName;
+ }
+
+ public Object getDefaultVal() {
+ return defaultVal;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ /**
+ * This is useful if you need the variable name for a LOG message or {@link
+ * System#setProperty(String, String)}, beware however that you should only use this with
+ * setProperty if you're going to create a configuration via {@link
+ * MetastoreConf#newMetastoreConf()}. If you are going to create it with HiveConf, then use
+ * {@link #getHiveName()}.
+ *
+ * @return metastore.* configuration key
+ */
+ @Override
+ public String toString() {
+ return varname;
+ }
+ }
+
+ // Make sure no one calls this
+ private MetastoreConf() {
+ throw new RuntimeException("You should never be creating one of these!");
+ }
+
+ public static Configuration newMetastoreConf() {
+ return newMetastoreConf(new Configuration());
+ }
+
+ public static Configuration newMetastoreConf(Configuration conf) {
+
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = MetastoreConf.class.getClassLoader();
+ }
+ // We don't add this to the resources because we don't want to read config values from it.
+ // But we do find it because we want to remember where it is for later in case anyone calls
+ // getHiveDefaultLocation().
+ hiveDefaultURL = classLoader.getResource("hive-default.xml");
+
+ // Add in hive-site.xml. We add this first so that it gets overridden by the new metastore
+ // specific files if they exist.
+ if (hiveSiteURL == null) {
+ /*
+ * this 'if' is pretty lame - QTestUtil.QTestUtil() uses hiveSiteURL to load a specific
+ * hive-site.xml from data/conf/<subdir> so this makes it follow the same logic - otherwise
+ * HiveConf and MetastoreConf may load different hive-site.xml ( For example,
+ * HiveConf uses data/conf/tez/hive-site.xml and MetastoreConf data/conf/hive-site.xml)
+ */
+ hiveSiteURL = findConfigFile(classLoader, "hive-site.xml");
+ }
+ if (hiveSiteURL != null) {
+ conf.addResource(hiveSiteURL);
+ }
+
+ // Now add hivemetastore-site.xml. Again we add this before our own config files so that the
+ // newer overrides the older.
+ hiveMetastoreSiteURL = findConfigFile(classLoader, "hivemetastore-site.xml");
+ if (hiveMetastoreSiteURL != null) {
+ conf.addResource(hiveMetastoreSiteURL);
+ }
+
+ // Add in our conf file
+ metastoreSiteURL = findConfigFile(classLoader, "metastore-site.xml");
+ if (metastoreSiteURL != null) {
+ conf.addResource(metastoreSiteURL);
+ }
+
+ // If a system property that matches one of our conf value names is set then use the value
+ // it's set to to set our own conf value.
+ for (ConfVars var : ConfVars.values()) {
+ if (System.getProperty(var.varname) != null) {
+ LOG.debug(
+ "Setting conf value "
+ + var.varname
+ + " using value "
+ + System.getProperty(var.varname));
+ conf.set(var.varname, System.getProperty(var.varname));
+ }
+ }
+
+ // Pick up any system properties that start with "hive." and set them in our config. This
+ // way we can properly pull any Hive values from the environment without needing to know all
+ // of the Hive config values.
+ System.getProperties().stringPropertyNames().stream()
+ .filter(s -> s.startsWith("hive."))
+ .forEach(
+ s -> {
+ String v = System.getProperty(s);
+ LOG.debug("Picking up system property " + s + " with value " + v);
+ conf.set(s, v);
+ });
+
+ if (!beenDumped.getAndSet(true)
+ && getBoolVar(conf, ConfVars.DUMP_CONFIG_ON_CREATION)
+ && LOG.isDebugEnabled()) {
+ LOG.debug(dumpConfig(conf));
+ }
+
+ /*
+ Add deprecated config names to configuration.
+ The parameters for Configuration.addDeprecation are (oldKey, newKey) and it is assumed that the config is set via
+ newKey and the value is retrieved via oldKey.
+ However in this case we assume the value is set with the deprecated key (oldKey) in some config file and we
+ retrieve it in the code via the new key. So the parameter order we use here is: (newKey, deprecatedKey).
+ We do this with the HiveConf configs as well.
+ */
+ for (ConfVars var : ConfVars.values()) {
+ if (var.deprecatedName != null) {
+ Configuration.addDeprecation(var.getVarname(), var.deprecatedName);
+ }
+ if (var.hiveDeprecatedName != null) {
+ Configuration.addDeprecation(var.getHiveName(), var.hiveDeprecatedName);
+ }
+ }
+
+ return conf;
+ }
+
+ private static URL findConfigFile(ClassLoader classLoader, String name) {
+ // First, look in the classpath
+ URL result = classLoader.getResource(name);
+ if (result == null) {
+ // Nope, so look to see if our conf dir has been explicitly set
+ result = seeIfConfAtThisLocation("METASTORE_CONF_DIR", name, false);
+ }
+ if (result == null) {
+ // Nope, so look to see if our home dir has been explicitly set
+ result = seeIfConfAtThisLocation("METASTORE_HOME", name, true);
+ }
+ if (result == null) {
+ // Nope, so look to see if Hive's conf dir has been explicitly set
+ result = seeIfConfAtThisLocation("HIVE_CONF_DIR", name, false);
+ }
+ if (result == null) {
+ // Nope, so look to see if Hive's home dir has been explicitly set
+ result = seeIfConfAtThisLocation("HIVE_HOME", name, true);
+ }
+ if (result == null) {
+ // Nope, so look to see if we can find a conf file by finding our jar, going up one
+ // directory, and looking for a conf directory.
+ URI jarUri = null;
+ try {
+ jarUri = MetastoreConf.class.getProtectionDomain().getCodeSource().getLocation().toURI();
+ } catch (Throwable e) {
+ LOG.warn("Cannot get jar URI", e);
+ }
+ if (jarUri != null) {
+ result = seeIfConfAtThisLocation(new File(jarUri).getParent(), name, true);
+ }
+ }
+
+ if (result == null) {
+ LOG.info("Unable to find config file: " + name);
+ } else {
+ LOG.info("Found configuration file: " + result);
+ }
+
+ return result;
+ }
+
+ private static URL seeIfConfAtThisLocation(String envVar, String name, boolean inConfDir) {
+ String path = System.getenv(envVar);
+ if (path == null) {
+ // Workaround for testing since tests can't set the env vars.
+ path = System.getProperty(TEST_ENV_WORKAROUND + envVar);
+ }
+ if (path != null) {
+ String suffix = inConfDir ? "conf" + File.separatorChar + name : name;
+ return checkConfigFile(new File(path, suffix));
+ }
+ return null;
+ }
+
+ private static URL checkConfigFile(File f) {
+ try {
+ return (f.exists() && f.isFile()) ? f.toURI().toURL() : null;
+ } catch (Throwable e) {
+ LOG.warn("Error looking for config " + f, e);
+ return null;
+ }
+ }
+
+ // In all of the getters, we try the metastore value name first. If it is not set we try the
+ // Hive value name.
+
+ /**
+ * Get the variable as a string
+ *
+ * @param conf configuration to retrieve it from
+ * @param var variable to retrieve
+ * @return value, or default value if value not in config file
+ */
+ public static String getVar(Configuration conf, ConfVars var) {
+ assert var.defaultVal.getClass() == String.class;
+ String val = conf.get(var.varname);
+ return val == null ? conf.get(var.hiveName, (String) var.defaultVal) : val;
+ }
+
+ /**
+ * Get the variable as a string
+ *
+ * @param conf configuration to retrieve it from
+ * @param var variable to retrieve
+ * @param defaultVal default to return if the variable is unset
+ * @return value, or default value passed in if the value is not in the config file
+ */
+ public static String getVar(Configuration conf, ConfVars var, String defaultVal) {
+ assert var.defaultVal.getClass() == String.class;
+ String val = conf.get(var.varname);
+ return val == null ? conf.get(var.hiveName, defaultVal) : val;
+ }
+
+ /**
+ * Treat a configuration value as a comma separated list.
+ *
+ * @param conf configuration to retrieve it from
+ * @param var variable to retrieve
+ * @return collection of strings. If the value is unset it will return an empty collection.
+ */
+ public static Collection<String> getStringCollection(Configuration conf, ConfVars var) {
+ assert var.defaultVal.getClass() == String.class;
+ String val = conf.get(var.varname);
+ if (val == null) {
+ val = conf.get(var.hiveName, (String) var.defaultVal);
+ }
+ if (val == null) {
+ return Collections.emptySet();
+ }
+ return StringUtils.asSet(val.split(","));
+ }
+
+ /**
+ * Set the variable as a string
+ *
+ * @param conf configuration file to set it in
+ * @param var variable to set
+ * @param val value to set it to
+ */
+ public static void setVar(Configuration conf, ConfVars var, String val) {
+ assert var.defaultVal.getClass() == String.class;
+ conf.set(var.varname, val);
+ }
+
+ /**
+ * Get the variable as a int. Note that all integer valued variables are stored as longs, thus
+ * this downcasts from a long to an in.
+ *
+ * @param conf configuration to retrieve it from
+ * @param var variable to retrieve
+ * @return value, or default value if value not in config file
+ */
+ public static int getIntVar(Configuration conf, ConfVars var) {
+ long val = getLongVar(conf, var);
+ assert val <= Integer.MAX_VALUE;
+ return (int) val;
+ }
+
+ /**
+ * Get the variable as a long
+ *
+ * @param conf configuration to retrieve it from
+ * @param var variable to retrieve
+ * @return value, or default value if value not in config file
+ */
+ public static long getLongVar(Configuration conf, ConfVars var) {
+ assert var.defaultVal.getClass() == Long.class;
+ String val = conf.get(var.varname);
+ return val == null ? conf.getLong(var.hiveName, (Long) var.defaultVal) : Long.parseLong(val);
+ }
+
+ /**
+ * Set the variable as a long
+ *
+ * @param conf configuration file to set it in
+ * @param var variable to set
+ * @param val value to set it to
+ */
+ public static void setLongVar(Configuration conf, ConfVars var, long val) {
+ assert var.defaultVal.getClass() == Long.class;
+ conf.setLong(var.varname, val);
+ }
+
+ /**
+ * Get the variable as a boolean
+ *
+ * @param conf configuration to retrieve it from
+ * @param var variable to retrieve
+ * @return value, or default value if value not in config file
+ */
+ public static boolean getBoolVar(Configuration conf, ConfVars var) {
+ assert var.defaultVal.getClass() == Boolean.class;
+ String val = conf.get(var.varname);
+ return val == null
+ ? conf.getBoolean(var.hiveName, (Boolean) var.defaultVal)
+ : Boolean.valueOf(val);
+ }
+
+ /**
+ * Get values from comma-separated config, to an array after extracting individual values.
+ *
+ * @param conf Configuration to retrieve it from
+ * @param var variable to retrieve
+ * @return Array of String, containing each value from the comma-separated config, or default
+ * value if value not in config file
+ */
+ public static String[] getTrimmedStringsVar(Configuration conf, ConfVars var) {
+ assert var.defaultVal.getClass() == String.class;
+ String[] result = conf.getTrimmedStrings(var.varname, (String[]) null);
+ if (result != null) {
+ return result;
+ }
+ if (var.hiveName != null) {
+ result = conf.getTrimmedStrings(var.hiveName, (String[]) null);
+ if (result != null) {
+ return result;
+ }
+ }
+ return org.apache.hadoop.util.StringUtils.getTrimmedStrings((String) var.getDefaultVal());
+ }
+
+ /**
+ * Set the variable as a boolean
+ *
+ * @param conf configuration file to set it in
+ * @param var variable to set
+ * @param val value to set it to
+ */
+ public static void setBoolVar(Configuration conf, ConfVars var, boolean val) {
+ assert var.defaultVal.getClass() == Boolean.class;
+ conf.setBoolean(var.varname, val);
+ }
+
+ /**
+ * Get the variable as a double
+ *
+ * @param conf configuration to retrieve it from
+ * @param var variable to retrieve
+ * @return value, or default value if value not in config file
+ */
+ public static double getDoubleVar(Configuration conf, ConfVars var) {
+ assert var.defaultVal.getClass() == Double.class;
+ String val = conf.get(var.varname);
+ return val == null
+ ? conf.getDouble(var.hiveName, (Double) var.defaultVal)
+ : Double.valueOf(val);
+ }
+
+ /**
+ * Set the variable as a double
+ *
+ * @param conf configuration file to set it in
+ * @param var variable to set
+ * @param val value to set it to
+ */
+ public static void setDoubleVar(Configuration conf, ConfVars var, double val) {
+ assert var.defaultVal.getClass() == Double.class;
+ conf.setDouble(var.varname, val);
+ }
+
+ public static long getSizeVar(Configuration conf, ConfVars var) {
+ return SizeValidator.toSizeBytes(getVar(conf, var));
+ }
+
+ /**
+ * Get a class instance based on a configuration value
+ *
+ * @param conf configuration file to retrieve it from
+ * @param var variable to retrieve
+ * @param defaultValue default class to return if the value isn't set
+ * @param xface interface that class must implement
+ * @param <I> interface that class implements
+ * @return instance of the class
+ */
+ public static <I> Class<? extends I> getClass(
+ Configuration conf, ConfVars var, Class<? extends I> defaultValue, Class<I> xface) {
+ assert var.defaultVal.getClass() == String.class;
+ String val = conf.get(var.varname);
+ return val == null
+ ? conf.getClass(var.hiveName, defaultValue, xface)
+ : conf.getClass(var.varname, defaultValue, xface);
+ }
+
+ /**
+ * Set the class name in the configuration file
+ *
+ * @param conf configuration file to set it in
+ * @param var variable to set
+ * @param theClass the class to set it to
+ * @param xface interface that the class implements. I don't know why this is required, but the
+ * underlying {@link Configuration#setClass(String, Class, Class)} requires it.
+ * @param <I> interface the class implements.
+ */
+ public static <I> void setClass(
+ Configuration conf, ConfVars var, Class<? extends I> theClass, Class<I> xface) {
+ assert var.defaultVal.getClass() == String.class;
+ conf.setClass(var.varname, theClass, xface);
+ }
+
+ /**
+ * Get the variable as a long indicating a period of time
+ *
+ * @param conf configuration to retrieve it from
+ * @param var variable to retrieve
+ * @param outUnit Timeout to return value in
+ * @return value, or default value if value not in config file
+ */
+ public static long getTimeVar(Configuration conf, ConfVars var, TimeUnit outUnit) {
+ assert var.defaultVal.getClass() == TimeValue.class;
+ String val = conf.get(var.varname);
+
+ if (val == null) {
+ // Look for it under the old Hive name
+ val = conf.get(var.hiveName);
+ }
+
+ if (val != null) {
+ return convertTimeStr(val, ((TimeValue) var.defaultVal).unit, outUnit);
+ } else {
+ return outUnit.convert(((TimeValue) var.defaultVal).val, ((TimeValue) var.defaultVal).unit);
+ }
+ }
+
+ /**
+ * Set the variable as a string
+ *
+ * @param conf configuration file to set it in
+ * @param var variable to set
+ * @param duration value to set it to
+ * @param unit time unit that duration is expressed in
+ */
+ public static void setTimeVar(Configuration conf, ConfVars var, long duration, TimeUnit unit) {
+ assert var.defaultVal.getClass() == TimeValue.class;
+ conf.setTimeDuration(var.varname, duration, unit);
+ }
+
+ public static long convertTimeStr(String val, TimeUnit defaultUnit, TimeUnit outUnit) {
+ if (val.charAt(val.length() - 1) >= 'A') {
+ // It ends in a character, this means they appended a time indicator (e.g. 600s)
+ Matcher m = TIME_UNIT_SUFFIX.matcher(val);
+ if (m.matches()) {
+ long duration = Long.parseLong(m.group(1));
+ String unit = m.group(2).toLowerCase();
+
+ // If/else chain arranged in likely order of frequency for performance
+ if (unit.equals("s") || unit.startsWith("sec")) {
+ return outUnit.convert(duration, TimeUnit.SECONDS);
+ } else if (unit.equals("ms") || unit.startsWith("msec")) {
+ return outUnit.convert(duration, TimeUnit.MILLISECONDS);
+ } else if (unit.equals("m") || unit.startsWith("min")) {
+ return outUnit.convert(duration, TimeUnit.MINUTES);
+ } else if (unit.equals("us") || unit.startsWith("usec")) {
+ return outUnit.convert(duration, TimeUnit.MICROSECONDS);
+ } else if (unit.equals("ns") || unit.startsWith("nsec")) {
+ return outUnit.convert(duration, TimeUnit.NANOSECONDS);
+ } else if (unit.equals("h") || unit.startsWith("hour")) {
+ return outUnit.convert(duration, TimeUnit.HOURS);
+ } else if (unit.equals("d") || unit.startsWith("day")) {
+ return outUnit.convert(duration, TimeUnit.DAYS);
+ } else {
+ throw new IllegalArgumentException("Invalid time unit " + unit);
+ }
+ } else {
+ throw new IllegalArgumentException("Invalid time unit " + val);
+ }
+ }
+
+ // If they gave a value but not a time unit assume the default time unit.
+ return outUnit.convert(Long.parseLong(val), defaultUnit);
+ }
+
+ static String timeAbbreviationFor(TimeUnit timeunit) {
+ switch (timeunit) {
+ case DAYS:
+ return "d";
+ case HOURS:
+ return "h";
+ case MINUTES:
+ return "m";
+ case SECONDS:
+ return "s";
+ case MILLISECONDS:
+ return "ms";
+ case MICROSECONDS:
+ return "us";
+ case NANOSECONDS:
+ return "ns";
+ }
+ throw new IllegalArgumentException("Invalid timeunit " + timeunit);
+ }
+
+ /**
+ * Get a password from the configuration file. This uses Hadoop's {@link
+ * Configuration#getPassword(String)} to handle getting secure passwords.
+ *
+ * @param conf configuration file to read from
+ * @param var configuration value to read
+ * @return the password as a string, or the default value.
+ * @throws IOException if thrown by Configuration.getPassword
+ */
+ public static String getPassword(Configuration conf, ConfVars var) throws IOException {
+ assert var.defaultVal.getClass() == String.class;
+ char[] pw = conf.getPassword(var.varname);
+ if (pw == null) {
+ // Might be under the hive name
+ pw = conf.getPassword(var.hiveName);
+ }
+ return pw == null ? var.defaultVal.toString() : new String(pw);
+ }
+
+ /**
+ * Get the configuration value based on a string rather than a ConfVar. This will do the mapping
+ * between metastore keys and Hive keys. That is, if there's a ConfVar with a metastore key of
+ * "metastore.a" and a hive key of "hive.a", the value for that variable will be returned if
+ * either of those keys is present in the config. If neither are present than the default value
+ * will be returned.
+ *
+ * @param conf configuration to read.
+ * @param key metastore or hive key to read.
+ * @return the value set
+ */
+ public static String get(Configuration conf, String key) {
+ ConfVars var = keyToVars.get(key);
+ if (var == null) {
+ // Ok, this isn't one we track. Just return whatever matches the string
+ return conf.get(key);
+ }
+ // Check if the metastore key is set first
+ String val = conf.get(var.varname);
+ return val == null ? conf.get(var.hiveName, var.defaultVal.toString()) : val;
+ }
+
+ public static boolean isPrintable(String key) {
+ return !unprintables.contains(key);
+ }
+
+ /**
+ * Return the configuration value as a String. For time based values it will be returned in the
+ * default time unit appended with an appropriate abbreviation (eg s for seconds, ...)
+ *
+ * @param conf configuration to read
+ * @param var variable to read
+ * @return value as a String
+ */
+ public static String getAsString(Configuration conf, ConfVars var) {
+ if (var.defaultVal.getClass() == String.class) {
+ return getVar(conf, var);
+ } else if (var.defaultVal.getClass() == Boolean.class) {
+ return Boolean.toString(getBoolVar(conf, var));
+ } else if (var.defaultVal.getClass() == Long.class) {
+ return Long.toString(getLongVar(conf, var));
+ } else if (var.defaultVal.getClass() == Double.class) {
+ return Double.toString(getDoubleVar(conf, var));
+ } else if (var.defaultVal.getClass() == TimeValue.class) {
+ TimeUnit timeUnit = ((TimeValue) var.defaultVal).unit;
+ return getTimeVar(conf, var, timeUnit) + timeAbbreviationFor(timeUnit);
+ } else {
+ throw new RuntimeException(
+ "Unknown type for getObject " + var.defaultVal.getClass().getName());
+ }
+ }
+
+ public static URL getHiveDefaultLocation() {
+ return hiveDefaultURL;
+ }
+
+ public static URL getHiveSiteLocation() {
+ return hiveSiteURL;
+ }
+
+ public static URL getHiveMetastoreSiteURL() {
+ return hiveMetastoreSiteURL;
+ }
+
+ public static URL getMetastoreSiteURL() {
+ return metastoreSiteURL;
+ }
+
+ public List<URL> getResourceFileLocations() {
+ return Arrays.asList(hiveSiteURL, hiveMetastoreSiteURL, metastoreSiteURL);
+ }
+
+ /**
+ * Check if metastore is being used in embedded mode. This utility function exists so that the
+ * logic for determining the mode is same in HiveConf and HiveMetaStoreClient
+ *
+ * @param msUri - metastore server uri
+ * @return true if the metastore is embedded
+ */
+ public static boolean isEmbeddedMetaStore(String msUri) {
+ return (msUri == null) || msUri.trim().isEmpty();
+ }
+
+ public static ZooKeeperHiveHelper getZKConfig(Configuration conf) {
+ String keyStorePassword = "";
+ String trustStorePassword = "";
+ if (MetastoreConf.getBoolVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_ENABLE)) {
+ try {
+ keyStorePassword =
+ MetastoreConf.getPassword(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_PASSWORD);
+ trustStorePassword =
+ MetastoreConf.getPassword(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to read zookeeper configuration passwords", e);
+ }
+ }
+ return ZooKeeperHiveHelper.builder()
+ .quorum(MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS))
+ .clientPort(MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_CLIENT_PORT))
+ .serverRegistryNameSpace(MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_NAMESPACE))
+ .connectionTimeout(
+ (int)
+ getTimeVar(
+ conf, ConfVars.THRIFT_ZOOKEEPER_CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS))
+ .sessionTimeout(
+ (int)
+ MetastoreConf.getTimeVar(
+ conf, ConfVars.THRIFT_ZOOKEEPER_SESSION_TIMEOUT, TimeUnit.MILLISECONDS))
+ .baseSleepTime(
+ (int)
+ MetastoreConf.getTimeVar(
+ conf,
+ ConfVars.THRIFT_ZOOKEEPER_CONNECTION_BASESLEEPTIME,
+ TimeUnit.MILLISECONDS))
+ .maxRetries(MetastoreConf.getIntVar(conf, ConfVars.THRIFT_ZOOKEEPER_CONNECTION_MAX_RETRIES))
+ .sslEnabled(MetastoreConf.getBoolVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_ENABLE))
+ .keyStoreLocation(
+ MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_LOCATION))
+ .keyStorePassword(keyStorePassword)
+ .trustStoreLocation(
+ MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION))
+ .trustStorePassword(trustStorePassword)
+ .build();
+ }
+
+ /**
+ * Dump the configuration file to the log. It will be dumped at an INFO level. This can
+ * potentially produce a lot of logs, so you might want to be careful when and where you do it. It
+ * takes care not to dump hidden keys.
+ *
+ * @param conf Configuration file to dump
+ * @return String containing dumped config file.
+ */
+ static String dumpConfig(Configuration conf) {
+ StringBuilder buf = new StringBuilder("MetastoreConf object:\n");
+ if (hiveSiteURL != null) {
+ buf.append("Used hive-site file: ").append(hiveSiteURL).append('\n');
+ }
+ if (hiveMetastoreSiteURL != null) {
+ buf.append("Used hivemetastore-site file: ").append(hiveMetastoreSiteURL).append('\n');
+ }
+ if (metastoreSiteURL != null) {
+ buf.append("Used metastore-site file: ").append(metastoreSiteURL).append('\n');
+ }
+ for (ConfVars var : ConfVars.values()) {
+ if (!unprintables.contains(var.varname)) {
+ buf.append("Key: <")
+ .append(var.varname)
+ .append("> old hive key: <")
+ .append(var.hiveName)
+ .append("> value: <")
+ .append(getAsString(conf, var))
+ .append(">\n");
+ }
+ }
+ buf.append("Finished MetastoreConf object.\n");
+ return buf.toString();
+ }
+
+ public static char[] getValueFromKeystore(String keystorePath, String key) throws IOException {
+ char[] valueCharArray = null;
+ if (keystorePath != null && key != null) {
+ Configuration conf = new Configuration();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, keystorePath);
+ valueCharArray = conf.getPassword(key);
+ }
+ return valueCharArray;
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/SizeValidator.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/SizeValidator.java
new file mode 100644
index 0000000..41e1c65
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/SizeValidator.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.shaded.hive.metastore.conf;
+
+public class SizeValidator implements Validator {
+ private final Long min;
+ private final boolean minInclusive;
+
+ private final Long max;
+ private final boolean maxInclusive;
+
+ public SizeValidator() {
+ this(null, false, null, false);
+ }
+
+ public SizeValidator(Long min, boolean minInclusive, Long max, boolean maxInclusive) {
+ this.min = min;
+ this.minInclusive = minInclusive;
+ this.max = max;
+ this.maxInclusive = maxInclusive;
+ }
+
+ @Override
+ public void validate(String value) {
+ long size = toSizeBytes(value);
+ if (min != null && (minInclusive ? size < min : size <= min)) {
+ throw new IllegalArgumentException(
+ value + " is smaller than minimum " + min + sizeString(min));
+ }
+ if (max != null && (maxInclusive ? size > max : size >= max)) {
+ throw new IllegalArgumentException(
+ value + " is larger than maximum " + max + sizeString(max));
+ }
+ }
+
+ public String toDescription() {
+ String description =
+ "Expects a byte size value with unit (blank for bytes, kb, mb, gb, tb, pb)";
+ if (min != null && max != null) {
+ description +=
+ ".\nThe size should be in between "
+ + sizeString(min)
+ + (minInclusive ? " (inclusive)" : " (exclusive)")
+ + " and "
+ + sizeString(max)
+ + (maxInclusive ? " (inclusive)" : " (exclusive)");
+ } else if (min != null) {
+ description +=
+ ".\nThe time should be bigger than "
+ + (minInclusive ? "or equal to " : "")
+ + sizeString(min);
+ } else if (max != null) {
+ description +=
+ ".\nThe size should be smaller than "
+ + (maxInclusive ? "or equal to " : "")
+ + sizeString(max);
+ }
+ return description;
+ }
+
+ private String sizeString(long size) {
+ final String[] units = {" bytes", "Kb", "Mb", "Gb", "Tb"};
+ long current = 1;
+ for (int i = 0; i < units.length && current > 0; ++i) {
+ long next = current << 10;
+ if ((size & (next - 1)) != 0) return (long) (size / current) + units[i];
+ current = next;
+ }
+ return current > 0 ? ((long) (size / current) + "Pb") : (size + units[0]);
+ }
+
+ public static long toSizeBytes(String value) {
+ String[] parsed = parseNumberFollowedByUnit(value.trim());
+ return Long.parseLong(parsed[0].trim()) * multiplierFor(parsed[1].trim());
+ }
+
+ private static String[] parseNumberFollowedByUnit(String value) {
+ char[] chars = value.toCharArray();
+ int i = 0;
+ for (; i < chars.length && (chars[i] == '-' || Character.isDigit(chars[i])); i++) {}
+ return new String[] {value.substring(0, i), value.substring(i)};
+ }
+
+ private static long multiplierFor(String unit) {
+ unit = unit.trim().toLowerCase();
+ if (unit.isEmpty() || unit.equals("b") || unit.equals("bytes")) {
+ return 1;
+ } else if (unit.equals("kb")) {
+ return 1024;
+ } else if (unit.equals("mb")) {
+ return 1024 * 1024;
+ } else if (unit.equals("gb")) {
+ return 1024 * 1024 * 1024;
+ } else if (unit.equals("tb")) {
+ return 1024L * 1024 * 1024 * 1024;
+ } else if (unit.equals("pb")) {
+ return 1024L * 1024 * 1024 * 1024 * 1024;
+ }
+ throw new IllegalArgumentException("Invalid size unit " + unit);
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/StringSetValidator.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/StringSetValidator.java
new file mode 100644
index 0000000..93da891
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/StringSetValidator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.shaded.hive.metastore.conf;
+
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+public class StringSetValidator implements Validator {
+
+ private final boolean caseSensitive;
+ private final Set<String> expected = new LinkedHashSet<String>();
+
+ public StringSetValidator(String... values) {
+ this(false, values);
+ }
+
+ public StringSetValidator(boolean caseSensitive, String... values) {
+ this.caseSensitive = caseSensitive;
+ for (String value : values) {
+ expected.add(caseSensitive ? value : value.toLowerCase());
+ }
+ }
+
+ public Set<String> getExpected() {
+ return new HashSet<String>(expected);
+ }
+
+ @Override
+ public void validate(String value) {
+ if (value == null || !expected.contains(caseSensitive ? value : value.toLowerCase())) {
+ throw new IllegalArgumentException("Invalid value.. expects one of " + expected);
+ }
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/TimeValidator.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/TimeValidator.java
new file mode 100644
index 0000000..d3e110d
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/TimeValidator.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.shaded.hive.metastore.conf;
+
+import java.util.concurrent.TimeUnit;
+
+public class TimeValidator implements Validator {
+
+ private final TimeUnit unit;
+ private final Long min;
+ private final boolean minInclusive;
+
+ private final Long max;
+ private final boolean maxInclusive;
+
+ public TimeValidator(TimeUnit unit) {
+ this(unit, null, false, null, false);
+ }
+
+ public TimeValidator(
+ TimeUnit unit, Long min, boolean minInclusive, Long max, boolean maxInclusive) {
+ this.unit = unit;
+ this.min = min;
+ this.minInclusive = minInclusive;
+ this.max = max;
+ this.maxInclusive = maxInclusive;
+ }
+
+ @Override
+ public void validate(String value) {
+ // First just check that this translates
+ TimeUnit defaultUnit = unit;
+ long time = MetastoreConf.convertTimeStr(value, defaultUnit, defaultUnit);
+ if (min != null) {
+ if (minInclusive ? time < min : time <= min) {
+ throw new IllegalArgumentException(
+ value
+ + " is smaller than minimum "
+ + min
+ + MetastoreConf.timeAbbreviationFor(defaultUnit));
+ }
+ }
+
+ if (max != null) {
+ if (maxInclusive ? time > max : time >= max) {
+ throw new IllegalArgumentException(
+ value
+ + " is larger than maximum "
+ + max
+ + MetastoreConf.timeAbbreviationFor(defaultUnit));
+ }
+ }
+ }
+
+ private String timeString(long time, TimeUnit timeUnit) {
+ return time + " " + MetastoreConf.timeAbbreviationFor(timeUnit);
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/Validator.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/Validator.java
new file mode 100644
index 0000000..a4a9676
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/conf/Validator.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.shaded.hive.metastore.conf;
+
+/** validate value for a ConfVar, return non-null string for fail message */
+public interface Validator {
+
+ /**
+ * Validate if the given value is acceptable.
+ *
+ * @param value value to test
+ * @throws IllegalArgumentException if the value is invalid
+ */
+ void validate(String value) throws IllegalArgumentException;
+
+ enum TYPE {
+ INT {
+ @Override
+ protected boolean inRange(String value, Object lower, Object upper) {
+ int ivalue = Integer.parseInt(value);
+ if (lower != null && ivalue < (Integer) lower) {
+ return false;
+ }
+ if (upper != null && ivalue > (Integer) upper) {
+ return false;
+ }
+ return true;
+ }
+ },
+ LONG {
+ @Override
+ protected boolean inRange(String value, Object lower, Object upper) {
+ long lvalue = Long.parseLong(value);
+ if (lower != null && lvalue < (Long) lower) {
+ return false;
+ }
+ if (upper != null && lvalue > (Long) upper) {
+ return false;
+ }
+ return true;
+ }
+ },
+ FLOAT {
+ @Override
+ protected boolean inRange(String value, Object lower, Object upper) {
+ float fvalue = Float.parseFloat(value);
+ if (lower != null && fvalue < (Float) lower) {
+ return false;
+ }
+ if (upper != null && fvalue > (Float) upper) {
+ return false;
+ }
+ return true;
+ }
+ };
+
+ public static TYPE valueOf(Object lower, Object upper) {
+ if (lower instanceof Integer || upper instanceof Integer) {
+ return INT;
+ } else if (lower instanceof Long || upper instanceof Long) {
+ return LONG;
+ } else if (lower instanceof Float || upper instanceof Float) {
+ return FLOAT;
+ }
+ throw new IllegalArgumentException("invalid range from " + lower + " to " + upper);
+ }
+
+ protected abstract boolean inRange(String value, Object lower, Object upper);
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/hooks/URIResolverHook.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/hooks/URIResolverHook.java
new file mode 100644
index 0000000..146870c
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/hooks/URIResolverHook.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.shaded.hive.metastore.hooks;
+
+import java.net.URI;
+import java.util.List;
+import org.apache.kyuubi.shaded.hive.metastore.HiveMetaException;
+
+/** Allows different metastore uris to be resolved. */
+public interface URIResolverHook {
+
+ /**
+ * Resolve to a proper thrift uri, or a list of uris, given uri of another scheme.
+ *
+ * @param uri
+ * @return
+ */
+ public List<URI> resolveURI(URI uri) throws HiveMetaException;
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/DelegationTokenIdentifier.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/DelegationTokenIdentifier.java
new file mode 100644
index 0000000..eaf60af
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/DelegationTokenIdentifier.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.shaded.hive.metastore.security;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+
+/** A delegation token identifier that is specific to Hive. */
+public class DelegationTokenIdentifier extends AbstractDelegationTokenIdentifier {
+ public static final Text HIVE_DELEGATION_KIND = new Text("HIVE_DELEGATION_TOKEN");
+
+ /** Create an empty delegation token identifier for reading into. */
+ public DelegationTokenIdentifier() {}
+
+ /**
+ * Create a new delegation token identifier
+ *
+ * @param owner the effective username of the token owner
+ * @param renewer the username of the renewer
+ * @param realUser the real username of the token owner
+ */
+ public DelegationTokenIdentifier(Text owner, Text renewer, Text realUser) {
+ super(owner, renewer, realUser);
+ }
+
+ @Override
+ public Text getKind() {
+ return HIVE_DELEGATION_KIND;
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/DelegationTokenSelector.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/DelegationTokenSelector.java
new file mode 100644
index 0000000..76068de
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/DelegationTokenSelector.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.shaded.hive.metastore.security;
+
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
+
+/** A delegation token that is specialized for Hive */
+public class DelegationTokenSelector
+ extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> {
+
+ public DelegationTokenSelector() {
+ super(DelegationTokenIdentifier.HIVE_DELEGATION_KIND);
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/HadoopThriftAuthBridge.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/HadoopThriftAuthBridge.java
new file mode 100644
index 0000000..decb2cf
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/HadoopThriftAuthBridge.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.shaded.hive.metastore.security;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.PrivilegedExceptionAction;
+import java.util.Base64;
+import java.util.Map;
+import javax.security.auth.callback.Callback;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.callback.NameCallback;
+import javax.security.auth.callback.PasswordCallback;
+import javax.security.auth.callback.UnsupportedCallbackException;
+import javax.security.sasl.RealmCallback;
+import javax.security.sasl.RealmChoiceCallback;
+import javax.security.sasl.SaslException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.thrift.transport.TSaslClientTransport;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Functions that bridge Thrift's SASL transports to Hadoop's SASL callback handlers and
+ * authentication classes. HIVE-11378 This class is not directly used anymore. It now exists only as
+ * a shell to be extended by HadoopThriftAuthBridge23 in 0.23 shims. I have made it abstract to
+ * avoid maintenance errors.
+ */
+public abstract class HadoopThriftAuthBridge {
+ private static final Logger LOG = LoggerFactory.getLogger(HadoopThriftAuthBridge.class);
+
+ // We want to have only one auth bridge. In the past this was handled by ShimLoader, but since
+ // we're no longer using that we'll do it here.
+ private static volatile HadoopThriftAuthBridge self = null;
+
+ public static HadoopThriftAuthBridge getBridge() {
+ if (self == null) {
+ synchronized (HadoopThriftAuthBridge.class) {
+ if (self == null) self = new HadoopThriftAuthBridge23();
+ }
+ }
+ return self;
+ }
+
+ public Client createClient() {
+ return new Client();
+ }
+
+ /**
+ * Read and return Hadoop SASL configuration which can be configured using "hadoop.rpc.protection"
+ *
+ * @param conf
+ * @return Hadoop SASL configuration
+ */
+ public abstract Map<String, String> getHadoopSaslProperties(Configuration conf);
+
+ public static class Client {
+ /**
+ * Create a client-side SASL transport that wraps an underlying transport.
+ *
+ * @param methodStr The authentication method to use. Currently only KERBEROS is supported.
+ * @param principalConfig The Kerberos principal of the target server.
+ * @param underlyingTransport The underlying transport mechanism, usually a TSocket.
+ * @param saslProps the sasl properties to create the client with
+ */
+ public TTransport createClientTransport(
+ String principalConfig,
+ String host,
+ String methodStr,
+ String tokenStrForm,
+ final TTransport underlyingTransport,
+ final Map<String, String> saslProps)
+ throws IOException {
+ final AuthMethod method = AuthMethod.valueOf(AuthMethod.class, methodStr);
+
+ TTransport saslTransport = null;
+ switch (method) {
+ case DIGEST:
+ Token<DelegationTokenIdentifier> t = new Token<>();
+ t.decodeFromUrlString(tokenStrForm);
+ try {
+ saslTransport =
+ new TSaslClientTransport(
+ method.getMechanismName(),
+ null,
+ null,
+ SaslRpcServer.SASL_DEFAULT_REALM,
+ saslProps,
+ new SaslClientCallbackHandler(t),
+ underlyingTransport);
+ } catch (TTransportException e) {
+ e.printStackTrace();
+ }
+ return new TUGIAssumingTransport(saslTransport, UserGroupInformation.getCurrentUser());
+
+ case KERBEROS:
+ String serverPrincipal = SecurityUtil.getServerPrincipal(principalConfig, host);
+ final String names[] = SaslRpcServer.splitKerberosName(serverPrincipal);
+ if (names.length != 3) {
+ throw new IOException(
+ "Kerberos principal name does NOT have the expected hostname part: "
+ + serverPrincipal);
+ }
+ try {
+ return UserGroupInformation.getCurrentUser()
+ .doAs(
+ new PrivilegedExceptionAction<TUGIAssumingTransport>() {
+ @Override
+ public TUGIAssumingTransport run() throws IOException, TTransportException {
+ TTransport saslTransport =
+ new TSaslClientTransport(
+ method.getMechanismName(),
+ null,
+ names[0],
+ names[1],
+ saslProps,
+ null,
+ underlyingTransport);
+ return new TUGIAssumingTransport(
+ saslTransport, UserGroupInformation.getCurrentUser());
+ }
+ });
+ } catch (InterruptedException | SaslException se) {
+ throw new IOException("Could not instantiate SASL transport", se);
+ }
+
+ default:
+ throw new IOException("Unsupported authentication method: " + method);
+ }
+ }
+
+ private static class SaslClientCallbackHandler implements CallbackHandler {
+ private final String userName;
+ private final char[] userPassword;
+
+ public SaslClientCallbackHandler(Token<? extends TokenIdentifier> token) {
+ this.userName = encodeIdentifier(token.getIdentifier());
+ this.userPassword = encodePassword(token.getPassword());
+ }
+
+ @Override
+ public void handle(Callback[] callbacks) throws UnsupportedCallbackException {
+ NameCallback nc = null;
+ PasswordCallback pc = null;
+ RealmCallback rc = null;
+ for (Callback callback : callbacks) {
+ if (callback instanceof RealmChoiceCallback) {
+ continue;
+ } else if (callback instanceof NameCallback) {
+ nc = (NameCallback) callback;
+ } else if (callback instanceof PasswordCallback) {
+ pc = (PasswordCallback) callback;
+ } else if (callback instanceof RealmCallback) {
+ rc = (RealmCallback) callback;
+ } else {
+ throw new UnsupportedCallbackException(callback, "Unrecognized SASL client callback");
+ }
+ }
+ if (nc != null) {
+ LOG.debug("SASL client callback: setting username: {}", userName);
+ nc.setName(userName);
+ }
+ if (pc != null) {
+ LOG.debug("SASL client callback: setting userPassword");
+ pc.setPassword(userPassword);
+ }
+ if (rc != null) {
+ LOG.debug("SASL client callback: setting realm: {}", rc.getDefaultText());
+ rc.setText(rc.getDefaultText());
+ }
+ }
+
+ static String encodeIdentifier(byte[] identifier) {
+ return new String(Base64.getEncoder().encode(identifier), StandardCharsets.UTF_8);
+ }
+
+ static char[] encodePassword(byte[] password) {
+ return Base64.getEncoder().encodeToString(password).toCharArray();
+ }
+ }
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/HadoopThriftAuthBridge23.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/HadoopThriftAuthBridge23.java
new file mode 100644
index 0000000..28882c3
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/HadoopThriftAuthBridge23.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.shaded.hive.metastore.security;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.Map;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SaslRpcServer;
+
+/**
+ * Functions that bridge Thrift's SASL transports to Hadoop's SASL callback handlers and
+ * authentication classes.
+ *
+ * <p>This is a 0.23/2.x specific implementation
+ */
+public class HadoopThriftAuthBridge23 extends HadoopThriftAuthBridge {
+
+ private static Field SASL_PROPS_FIELD;
+ private static Class<?> SASL_PROPERTIES_RESOLVER_CLASS;
+ private static Method RES_GET_INSTANCE_METHOD;
+ private static Method GET_DEFAULT_PROP_METHOD;
+
+ static {
+ SASL_PROPERTIES_RESOLVER_CLASS = null;
+ SASL_PROPS_FIELD = null;
+ final String SASL_PROP_RES_CLASSNAME = "org.apache.hadoop.security.SaslPropertiesResolver";
+ try {
+ SASL_PROPERTIES_RESOLVER_CLASS = Class.forName(SASL_PROP_RES_CLASSNAME);
+
+ } catch (ClassNotFoundException e) {
+ }
+
+ if (SASL_PROPERTIES_RESOLVER_CLASS != null) {
+ // found the class, so this would be hadoop version 2.4 or newer (See
+ // HADOOP-10221, HADOOP-10451)
+ try {
+ RES_GET_INSTANCE_METHOD =
+ SASL_PROPERTIES_RESOLVER_CLASS.getMethod("getInstance", Configuration.class);
+ GET_DEFAULT_PROP_METHOD = SASL_PROPERTIES_RESOLVER_CLASS.getMethod("getDefaultProperties");
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ // this must be hadoop 2.4 , where getDefaultProperties was protected
+ }
+ }
+
+ if (SASL_PROPERTIES_RESOLVER_CLASS == null || GET_DEFAULT_PROP_METHOD == null) {
+ // this must be a hadoop 2.4 version or earlier.
+ // Resorting to the earlier method of getting the properties, which uses SASL_PROPS field
+ try {
+ SASL_PROPS_FIELD = SaslRpcServer.class.getField("SASL_PROPS");
+ } catch (NoSuchFieldException e) {
+ // Older version of hadoop should have had this field
+ throw new IllegalStateException(
+ "Error finding hadoop SASL_PROPS field in " + SaslRpcServer.class.getSimpleName(), e);
+ }
+ }
+ }
+
+ // TODO RIVEN switch this back to package level when we can move TestHadoopAuthBridge23 into
+ // riven.
+ // Package permission so that HadoopThriftAuthBridge can construct it but others cannot.
+ protected HadoopThriftAuthBridge23() {}
+
+ /**
+ * Read and return Hadoop SASL configuration which can be configured using "hadoop.rpc.protection"
+ *
+ * @param conf
+ * @return Hadoop SASL configuration
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ public Map<String, String> getHadoopSaslProperties(Configuration conf) {
+ if (SASL_PROPS_FIELD != null) {
+ // hadoop 2.4 and earlier way of finding the sasl property settings
+ // Initialize the SaslRpcServer to ensure QOP parameters are read from
+ // conf
+ SaslRpcServer.init(conf);
+ try {
+ return (Map<String, String>) SASL_PROPS_FIELD.get(null);
+ } catch (Exception e) {
+ throw new IllegalStateException("Error finding hadoop SASL properties", e);
+ }
+ }
+ // 2.5 and later way of finding sasl property
+ try {
+ Configurable saslPropertiesResolver =
+ (Configurable) RES_GET_INSTANCE_METHOD.invoke(null, conf);
+ saslPropertiesResolver.setConf(conf);
+ return (Map<String, String>) GET_DEFAULT_PROP_METHOD.invoke(saslPropertiesResolver);
+ } catch (Exception e) {
+ throw new IllegalStateException("Error finding hadoop SASL properties", e);
+ }
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/TFilterTransport.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/TFilterTransport.java
new file mode 100644
index 0000000..915d855
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/TFilterTransport.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.shaded.hive.metastore.security;
+
+import org.apache.thrift.TConfiguration;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * Transport that simply wraps another transport. This is the equivalent of FilterInputStream for
+ * Thrift transports.
+ */
+public class TFilterTransport extends TTransport {
+ protected final TTransport wrapped;
+
+ public TFilterTransport(TTransport wrapped) {
+ this.wrapped = wrapped;
+ }
+
+ @Override
+ public void open() throws TTransportException {
+ wrapped.open();
+ }
+
+ @Override
+ public boolean isOpen() {
+ return wrapped.isOpen();
+ }
+
+ @Override
+ public boolean peek() {
+ return wrapped.peek();
+ }
+
+ @Override
+ public void close() {
+ wrapped.close();
+ }
+
+ @Override
+ public int read(byte[] buf, int off, int len) throws TTransportException {
+ return wrapped.read(buf, off, len);
+ }
+
+ @Override
+ public int readAll(byte[] buf, int off, int len) throws TTransportException {
+ return wrapped.readAll(buf, off, len);
+ }
+
+ @Override
+ public void write(byte[] buf) throws TTransportException {
+ wrapped.write(buf);
+ }
+
+ @Override
+ public void write(byte[] buf, int off, int len) throws TTransportException {
+ wrapped.write(buf, off, len);
+ }
+
+ @Override
+ public void flush() throws TTransportException {
+ wrapped.flush();
+ }
+
+ @Override
+ public byte[] getBuffer() {
+ return wrapped.getBuffer();
+ }
+
+ @Override
+ public int getBufferPosition() {
+ return wrapped.getBufferPosition();
+ }
+
+ @Override
+ public int getBytesRemainingInBuffer() {
+ return wrapped.getBytesRemainingInBuffer();
+ }
+
+ @Override
+ public void consumeBuffer(int len) {
+ wrapped.consumeBuffer(len);
+ }
+
+ @Override
+ public TConfiguration getConfiguration() {
+ return wrapped.getConfiguration();
+ }
+
+ @Override
+ public void updateKnownMessageSize(long l) throws TTransportException {
+ wrapped.updateKnownMessageSize(l);
+ }
+
+ @Override
+ public void checkReadBytesAvailable(long l) throws TTransportException {
+ wrapped.checkReadBytesAvailable(l);
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/TUGIAssumingTransport.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/TUGIAssumingTransport.java
new file mode 100644
index 0000000..5689d2c
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/security/TUGIAssumingTransport.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.shaded.hive.metastore.security;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * The Thrift SASL transports call Sasl.createSaslServer and Sasl.createSaslClient inside open().
+ * So, we need to assume the correct UGI when the transport is opened so that the SASL mechanisms
+ * have access to the right principal. This transport wraps the Sasl transports to set up the right
+ * UGI context for open().
+ *
+ * <p>This is used on the client side, where the API explicitly opens a transport to the server.
+ */
+public class TUGIAssumingTransport extends TFilterTransport {
+ protected UserGroupInformation ugi;
+
+ public TUGIAssumingTransport(TTransport wrapped, UserGroupInformation ugi) {
+ super(wrapped);
+ this.ugi = ugi;
+ }
+
+ @Override
+ public void open() throws TTransportException {
+ try {
+ ugi.doAs(
+ new PrivilegedExceptionAction<Void>() {
+ public Void run() {
+ try {
+ wrapped.open();
+ } catch (TTransportException tte) {
+ // Wrap the transport exception in an RTE, since UGI.doAs() then goes
+ // and unwraps this for us out of the doAs block. We then unwrap one
+ // more time in our catch clause to get back the TTE. (ugh)
+ throw new RuntimeException(tte);
+ }
+ return null;
+ }
+ });
+ } catch (IOException ioe) {
+ throw new RuntimeException("Received an ioe we never threw!", ioe);
+ } catch (InterruptedException ie) {
+ throw new RuntimeException("Received an ie we never threw!", ie);
+ } catch (RuntimeException rte) {
+ if (rte.getCause() instanceof TTransportException) {
+ throw (TTransportException) rte.getCause();
+ } else {
+ throw rte;
+ }
+ }
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/JavaUtils.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/JavaUtils.java
new file mode 100644
index 0000000..f51bcd8
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/JavaUtils.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.shaded.hive.metastore.utils;
+
+import java.lang.reflect.Constructor;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import org.apache.commons.lang3.ClassUtils;
+import org.apache.kyuubi.shaded.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class JavaUtils {
+ public static final Logger LOG = LoggerFactory.getLogger(JavaUtils.class);
+
+ /**
+ * Standard way of getting classloader in Hive code (outside of Hadoop).
+ *
+ * <p>Uses the context loader to get access to classpaths to auxiliary and jars added with 'add
+ * jar' command. Falls back to current classloader.
+ *
+ * <p>In Hadoop-related code, we use Configuration.getClassLoader().
+ *
+ * @return the class loader
+ */
+ public static ClassLoader getClassLoader() {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = JavaUtils.class.getClassLoader();
+ }
+ return classLoader;
+ }
+
+ @SuppressWarnings(value = "unchecked")
+ public static <T> Class<? extends T> getClass(String className, Class<T> clazz)
+ throws MetaException {
+ try {
+ return (Class<? extends T>) Class.forName(className, true, getClassLoader());
+ } catch (ClassNotFoundException e) {
+ throw new MetaException(className + " class not found");
+ }
+ }
+
+ /**
+ * Create an object of the given class.
+ *
+ * @param theClass
+ * @param parameterTypes an array of parameterTypes for the constructor
+ * @param initargs the list of arguments for the constructor
+ */
+ public static <T> T newInstance(Class<T> theClass, Class<?>[] parameterTypes, Object[] initargs) {
+ // Perform some sanity checks on the arguments.
+ if (parameterTypes.length != initargs.length) {
+ throw new IllegalArgumentException(
+ "Number of constructor parameter types doesn't match number of arguments");
+ }
+ for (int i = 0; i < parameterTypes.length; i++) {
+ // initargs are boxed to Object, so we need to wrapper primitive types here.
+ Class<?> clazz = ClassUtils.primitiveToWrapper(parameterTypes[i]);
+ if (initargs[i] != null && !(clazz.isInstance(initargs[i]))) {
+ throw new IllegalArgumentException(
+ "Object : " + initargs[i] + " is not an instance of " + clazz);
+ }
+ }
+
+ try {
+ Constructor<T> meth = theClass.getDeclaredConstructor(parameterTypes);
+ meth.setAccessible(true);
+ return meth.newInstance(initargs);
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to instantiate " + theClass.getName(), e);
+ }
+ }
+
+ /** @return name of current host */
+ public static String hostname() {
+ try {
+ return InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ LOG.error("Unable to resolve my host name " + e.getMessage());
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/MetaStoreUtils.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/MetaStoreUtils.java
new file mode 100644
index 0000000..275e30b
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/MetaStoreUtils.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.shaded.hive.metastore.utils;
+
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.kyuubi.shaded.hive.metastore.api.MetaException;
+import org.apache.kyuubi.shaded.hive.metastore.security.HadoopThriftAuthBridge;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MetaStoreUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MetaStoreUtils.class);
+
+ public static final String USER_NAME_HTTP_HEADER = "x-actor-username";
+
+ /**
+ * Catches exceptions that cannot be handled and wraps them in MetaException.
+ *
+ * @param e exception to wrap.
+ * @throws MetaException wrapper for the exception
+ */
+ public static void throwMetaException(Exception e) throws MetaException {
+ throw new MetaException("Got exception: " + e.getClass().getName() + " " + e.getMessage());
+ }
+
+ /**
+ * Read and return the meta store Sasl configuration. Currently it uses the default Hadoop SASL
+ * configuration and can be configured using "hadoop.rpc.protection" HADOOP-10211, made a backward
+ * incompatible change due to which this call doesn't work with Hadoop 2.4.0 and later.
+ *
+ * @param conf
+ * @return The SASL configuration
+ */
+ public static Map<String, String> getMetaStoreSaslProperties(Configuration conf, boolean useSSL) {
+ // As of now Hive Meta Store uses the same configuration as Hadoop SASL configuration
+
+ // If SSL is enabled, override the given value of "hadoop.rpc.protection" and set it to
+ // "authentication"
+ // This disables any encryption provided by SASL, since SSL already provides it
+ String hadoopRpcProtectionVal = conf.get(CommonConfigurationKeysPublic.HADOOP_RPC_PROTECTION);
+ String hadoopRpcProtectionAuth = SaslRpcServer.QualityOfProtection.AUTHENTICATION.toString();
+
+ if (useSSL
+ && hadoopRpcProtectionVal != null
+ && !hadoopRpcProtectionVal.equals(hadoopRpcProtectionAuth)) {
+ LOG.warn(
+ "Overriding value of "
+ + CommonConfigurationKeysPublic.HADOOP_RPC_PROTECTION
+ + " setting it from "
+ + hadoopRpcProtectionVal
+ + " to "
+ + hadoopRpcProtectionAuth
+ + " because SSL is enabled");
+ conf.set(CommonConfigurationKeysPublic.HADOOP_RPC_PROTECTION, hadoopRpcProtectionAuth);
+ }
+ return HadoopThriftAuthBridge.getBridge().getHadoopSaslProperties(conf);
+ }
+
+ /**
+ * The config parameter can be like "path", "/path", "/path/", "path/*", "/path1/path2/*" and so
+ * on. httpPath should end up as "/*", "/path/*" or "/path1/../pathN/*"
+ *
+ * @param httpPath
+ * @return
+ */
+ public static String getHttpPath(String httpPath) {
+ if (httpPath == null || httpPath.equals("")) {
+ httpPath = "/*";
+ } else {
+ if (!httpPath.startsWith("/")) {
+ httpPath = "/" + httpPath;
+ }
+ if (httpPath.endsWith("/")) {
+ httpPath = httpPath + "*";
+ }
+ if (!httpPath.endsWith("/*")) {
+ httpPath = httpPath + "/*";
+ }
+ }
+ return httpPath;
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/Retry.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/Retry.java
new file mode 100644
index 0000000..e8778d1
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/Retry.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.shaded.hive.metastore.utils;
+
+/** Class to implement any retry logic in case of exceptions. */
+public abstract class Retry<T> {
+
+ public static final int MAX_RETRIES = 4;
+ public static final int DELAY = 30 * 1000;
+ private int tries = 0;
+ private Class retryExceptionType;
+
+ public Retry(Class exceptionClassType) {
+ this.retryExceptionType = exceptionClassType;
+ }
+
+ public abstract T execute() throws Exception;
+
+ public T run() throws Exception {
+ try {
+ return execute();
+ } catch (Exception e) {
+ if (e.getClass().equals(retryExceptionType)) {
+ tries++;
+ if (MAX_RETRIES == tries) {
+ throw e;
+ } else {
+ return run();
+ }
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ public T runWithDelay() throws Exception {
+ try {
+ return execute();
+ } catch (Exception e) {
+ if (e.getClass().equals(retryExceptionType)) {
+ tries++;
+ if (MAX_RETRIES == tries) {
+ throw e;
+ } else {
+ Thread.sleep((long) DELAY * tries);
+ return runWithDelay();
+ }
+ } else {
+ throw e;
+ }
+ }
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/SecurityUtils.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/SecurityUtils.java
new file mode 100644
index 0000000..40fd46e
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/SecurityUtils.java
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.shaded.hive.metastore.utils;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.util.*;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLParameters;
+import javax.net.ssl.SSLServerSocket;
+import javax.net.ssl.SSLSocket;
+import javax.net.ssl.TrustManagerFactory;
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
+import javax.security.auth.login.LoginException;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
+import org.apache.http.ssl.SSLContexts;
+import org.apache.kyuubi.shaded.hive.metastore.api.MetaException;
+import org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier;
+import org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenSelector;
+import org.apache.kyuubi.shaded.zookeeper.client.ZooKeeperSaslClient;
+import org.apache.thrift.transport.THttpClient;
+import org.apache.thrift.transport.TSSLTransportFactory;
+import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SecurityUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(SecurityUtils.class);
+
+ public static UserGroupInformation getUGI() throws LoginException, IOException {
+ String doAs = System.getenv("HADOOP_USER_NAME");
+ if (doAs != null && doAs.length() > 0) {
+ /*
+ * this allows doAs (proxy user) to be passed along across process boundary where
+ * delegation tokens are not supported. For example, a DDL stmt via WebHCat with
+ * a doAs parameter, forks to 'hcat' which needs to start a Session that
+ * proxies the end user
+ */
+ return UserGroupInformation.createProxyUser(doAs, UserGroupInformation.getLoginUser());
+ }
+ return UserGroupInformation.getCurrentUser();
+ }
+ /**
+ * Dynamically sets up the JAAS configuration that uses kerberos
+ *
+ * @param principal
+ * @param keyTabFile
+ * @throws IOException
+ */
+ public static void setZookeeperClientKerberosJaasConfig(String principal, String keyTabFile)
+ throws IOException {
+ // ZooKeeper property name to pick the correct JAAS conf section
+ final String SASL_LOGIN_CONTEXT_NAME = "HiveZooKeeperClient";
+ System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY, SASL_LOGIN_CONTEXT_NAME);
+
+ principal = SecurityUtil.getServerPrincipal(principal, "0.0.0.0");
+ JaasConfiguration jaasConf =
+ new JaasConfiguration(SASL_LOGIN_CONTEXT_NAME, principal, keyTabFile);
+
+ // Install the Configuration in the runtime.
+ javax.security.auth.login.Configuration.setConfiguration(jaasConf);
+ }
+
+ /** A JAAS configuration for ZooKeeper clients intended to use for SASL Kerberos. */
+ private static class JaasConfiguration extends javax.security.auth.login.Configuration {
+ // Current installed Configuration
+ private static final boolean IBM_JAVA = System.getProperty("java.vendor").contains("IBM");
+ private final javax.security.auth.login.Configuration baseConfig =
+ javax.security.auth.login.Configuration.getConfiguration();
+ private final String loginContextName;
+ private final String principal;
+ private final String keyTabFile;
+
+ public JaasConfiguration(String hiveLoginContextName, String principal, String keyTabFile) {
+ this.loginContextName = hiveLoginContextName;
+ this.principal = principal;
+ this.keyTabFile = keyTabFile;
+ }
+
+ @Override
+ public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
+ if (loginContextName.equals(appName)) {
+ Map<String, String> krbOptions = new HashMap<String, String>();
+ if (IBM_JAVA) {
+ krbOptions.put("credsType", "both");
+ krbOptions.put("useKeytab", keyTabFile);
+ } else {
+ krbOptions.put("doNotPrompt", "true");
+ krbOptions.put("storeKey", "true");
+ krbOptions.put("useKeyTab", "true");
+ krbOptions.put("keyTab", keyTabFile);
+ }
+ krbOptions.put("principal", principal);
+ krbOptions.put("refreshKrb5Config", "true");
+ AppConfigurationEntry hiveZooKeeperClientEntry =
+ new AppConfigurationEntry(
+ KerberosUtil.getKrb5LoginModuleName(), LoginModuleControlFlag.REQUIRED, krbOptions);
+ return new AppConfigurationEntry[] {hiveZooKeeperClientEntry};
+ }
+ // Try the base config
+ if (baseConfig != null) {
+ return baseConfig.getAppConfigurationEntry(appName);
+ }
+ return null;
+ }
+ }
+
+ /**
+ * Get the string form of the token given a token signature. The signature is used as the value of
+ * the "service" field in the token for lookup. Ref: AbstractDelegationTokenSelector in Hadoop. If
+ * there exists such a token in the token cache (credential store) of the job, the lookup returns
+ * that. This is relevant only when running against a "secure" hadoop release The method gets hold
+ * of the tokens if they are set up by hadoop - this should happen on the map/reduce tasks if the
+ * client added the tokens into hadoop's credential store in the front end during job submission.
+ * The method will select the hive delegation token among the set of tokens and return the string
+ * form of it
+ *
+ * @param tokenSignature
+ * @return the string form of the token found
+ * @throws IOException
+ */
+ public static String getTokenStrForm(String tokenSignature) throws IOException {
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ TokenSelector<? extends TokenIdentifier> tokenSelector = new DelegationTokenSelector();
+
+ Token<? extends TokenIdentifier> token =
+ tokenSelector.selectToken(
+ tokenSignature == null ? new Text() : new Text(tokenSignature), ugi.getTokens());
+ return token != null ? token.encodeToUrlString() : null;
+ }
+
+ /**
+ * Create a delegation token object for the given token string and service. Add the token to given
+ * UGI
+ *
+ * @param ugi
+ * @param tokenStr
+ * @param tokenService
+ * @throws IOException
+ */
+ public static void setTokenStr(UserGroupInformation ugi, String tokenStr, String tokenService)
+ throws IOException {
+ Token<DelegationTokenIdentifier> delegationToken = createToken(tokenStr, tokenService);
+ ugi.addToken(delegationToken);
+ }
+
+ /**
+ * Create a new token using the given string and service
+ *
+ * @param tokenStr
+ * @param tokenService
+ * @return
+ * @throws IOException
+ */
+ private static Token<DelegationTokenIdentifier> createToken(String tokenStr, String tokenService)
+ throws IOException {
+ Token<DelegationTokenIdentifier> delegationToken = new Token<DelegationTokenIdentifier>();
+ delegationToken.decodeFromUrlString(tokenStr);
+ delegationToken.setService(new Text(tokenService));
+ return delegationToken;
+ }
+
+ /**
+ * @return the user name set in hadoop.job.ugi param or the current user from System
+ * @throws IOException if underlying Hadoop call throws LoginException
+ */
+ public static String getUser() throws IOException {
+ try {
+ UserGroupInformation ugi = getUGI();
+ return ugi.getUserName();
+ } catch (LoginException le) {
+ throw new IOException(le);
+ }
+ }
+
+ public static TServerSocket getServerSocket(String hiveHost, int portNum)
+ throws TTransportException {
+ InetSocketAddress serverAddress;
+ if (hiveHost == null || hiveHost.isEmpty()) {
+ // Wildcard bind
+ serverAddress = new InetSocketAddress(portNum);
+ } else {
+ serverAddress = new InetSocketAddress(hiveHost, portNum);
+ }
+ return new TServerSocket(serverAddress);
+ }
+
+ public static TServerSocket getServerSSLSocket(
+ String hiveHost,
+ int portNum,
+ String keyStorePath,
+ String keyStorePassWord,
+ String keyStoreType,
+ String keyStoreAlgorithm,
+ List<String> sslVersionBlacklist)
+ throws TTransportException, UnknownHostException {
+ TSSLTransportFactory.TSSLTransportParameters params =
+ new TSSLTransportFactory.TSSLTransportParameters();
+ String kStoreType = keyStoreType.isEmpty() ? KeyStore.getDefaultType() : keyStoreType;
+ String kStoreAlgorithm =
+ keyStoreAlgorithm.isEmpty() ? KeyManagerFactory.getDefaultAlgorithm() : keyStoreAlgorithm;
+ params.setKeyStore(keyStorePath, keyStorePassWord, kStoreAlgorithm, kStoreType);
+ InetSocketAddress serverAddress;
+ if (hiveHost == null || hiveHost.isEmpty()) {
+ // Wildcard bind
+ serverAddress = new InetSocketAddress(portNum);
+ } else {
+ serverAddress = new InetSocketAddress(hiveHost, portNum);
+ }
+ TServerSocket thriftServerSocket =
+ TSSLTransportFactory.getServerSocket(portNum, 0, serverAddress.getAddress(), params);
+ if (thriftServerSocket.getServerSocket() instanceof SSLServerSocket) {
+ List<String> sslVersionBlacklistLocal = new ArrayList<>();
+ for (String sslVersion : sslVersionBlacklist) {
+ sslVersionBlacklistLocal.add(sslVersion.trim().toLowerCase());
+ }
+ SSLServerSocket sslServerSocket = (SSLServerSocket) thriftServerSocket.getServerSocket();
+ List<String> enabledProtocols = new ArrayList<>();
+ for (String protocol : sslServerSocket.getEnabledProtocols()) {
+ if (sslVersionBlacklistLocal.contains(protocol.toLowerCase())) {
+ LOG.debug("Disabling SSL Protocol: " + protocol);
+ } else {
+ enabledProtocols.add(protocol);
+ }
+ }
+ sslServerSocket.setEnabledProtocols(enabledProtocols.toArray(new String[0]));
+ LOG.info(
+ "SSL Server Socket Enabled Protocols: "
+ + Arrays.toString(sslServerSocket.getEnabledProtocols()));
+ }
+ return thriftServerSocket;
+ }
+
+ public static TTransport getSSLSocket(
+ String host,
+ int port,
+ int socketTimeout,
+ int connectionTimeout,
+ String trustStorePath,
+ String trustStorePassWord,
+ String trustStoreType,
+ String trustStoreAlgorithm)
+ throws TTransportException {
+ TSSLTransportFactory.TSSLTransportParameters params =
+ new TSSLTransportFactory.TSSLTransportParameters();
+ String tStoreType = trustStoreType.isEmpty() ? KeyStore.getDefaultType() : trustStoreType;
+ String tStoreAlgorithm =
+ trustStoreAlgorithm.isEmpty()
+ ? TrustManagerFactory.getDefaultAlgorithm()
+ : trustStoreAlgorithm;
+ params.setTrustStore(trustStorePath, trustStorePassWord, tStoreAlgorithm, tStoreType);
+ params.requireClientAuth(true);
+ // The underlying SSLSocket object is bound to host:port with the given SO_TIMEOUT and
+ // connection timeout and SSLContext created with the given params
+ TSocket tSSLSocket = TSSLTransportFactory.getClientSocket(host, port, socketTimeout, params);
+ tSSLSocket.setConnectTimeout(connectionTimeout);
+ return getSSLSocketWithHttps(tSSLSocket);
+ }
+
+ /*
+ Sets the ssl related configs in the underlying http client builder and wrap it up
+ in a THttpClient
+ */
+ public static THttpClient getThriftHttpsClient(
+ String httpsUrl,
+ String trustStorePath,
+ String trustStorePasswd,
+ String trustStoreAlgorithm,
+ String trustStoreType,
+ HttpClientBuilder underlyingHttpClientBuilder)
+ throws TTransportException, IOException, KeyStoreException, NoSuchAlgorithmException,
+ CertificateException, KeyManagementException {
+ Objects.requireNonNull(underlyingHttpClientBuilder, "httpClientBuilder should not be null");
+ if (trustStoreType == null || trustStoreType.isEmpty()) {
+ trustStoreType = KeyStore.getDefaultType();
+ }
+ KeyStore sslTrustStore = KeyStore.getInstance(trustStoreType);
+ try (FileInputStream fis = new FileInputStream(trustStorePath)) {
+ sslTrustStore.load(fis, trustStorePasswd.toCharArray());
+ }
+
+ SSLContext sslContext =
+ SSLContexts.custom()
+ .setTrustManagerFactoryAlgorithm(trustStoreAlgorithm)
+ .loadTrustMaterial(sslTrustStore, null)
+ .build();
+ SSLConnectionSocketFactory socketFactory =
+ new SSLConnectionSocketFactory(sslContext, new DefaultHostnameVerifier(null));
+ final Registry<ConnectionSocketFactory> registry =
+ RegistryBuilder.<ConnectionSocketFactory>create().register("https", socketFactory).build();
+ underlyingHttpClientBuilder.setConnectionManager(
+ new BasicHttpClientConnectionManager(registry));
+ return new THttpClient(httpsUrl, underlyingHttpClientBuilder.build());
+ }
+
+ // Using endpoint identification algorithm as HTTPS enables us to do
+ // CNAMEs/subjectAltName verification
+ private static TSocket getSSLSocketWithHttps(TSocket tSSLSocket) throws TTransportException {
+ SSLSocket sslSocket = (SSLSocket) tSSLSocket.getSocket();
+ SSLParameters sslParams = sslSocket.getSSLParameters();
+ sslParams.setEndpointIdentificationAlgorithm("HTTPS");
+ sslSocket.setSSLParameters(sslParams);
+ return new TSocket(sslSocket);
+ }
+
+ /**
+ * Relogin if login user is logged in using keytab Relogin is actually done by ugi code only if
+ * sufficient time has passed A no-op if kerberos security is not enabled
+ *
+ * @throws MetaException
+ */
+ public static void reloginExpiringKeytabUser() throws MetaException {
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ return;
+ }
+ try {
+ UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+ // checkTGT calls ugi.relogin only after checking if it is close to tgt expiry
+ // hadoop relogin is actually done only every x minutes (x=10 in hadoop 1.x)
+ if (ugi.isFromKeytab()) {
+ ugi.checkTGTAndReloginFromKeytab();
+ }
+ } catch (IOException e) {
+ String msg = "Error doing relogin using keytab " + e.getMessage();
+ LOG.error(msg, e);
+ throw new MetaException(msg);
+ }
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/StringUtils.java b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/StringUtils.java
new file mode 100644
index 0000000..a9ed9ff
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/java/org/apache/kyuubi/shaded/hive/metastore/utils/StringUtils.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.shaded.hive.metastore.utils;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class StringUtils {
+
+ /**
+ * Return the internalized string, or null if the given string is null.
+ *
+ * @param str The string to intern
+ * @return The identical string cached in the string pool.
+ */
+ public static String intern(String str) {
+ if (str == null) {
+ return null;
+ }
+ return str.intern();
+ }
+
+ /**
+ * Return an interned list with identical contents as the given list.
+ *
+ * @param list The list whose strings will be interned
+ * @return An identical list with its strings interned.
+ */
+ public static List<String> intern(List<String> list) {
+ if (list == null) {
+ return null;
+ }
+ List<String> newList = new ArrayList<>(list.size());
+ for (String str : list) {
+ newList.add(intern(str));
+ }
+ return newList;
+ }
+
+ /**
+ * Return an interned map with identical contents as the given map.
+ *
+ * @param map The map whose strings will be interned
+ * @return An identical map with its strings interned.
+ */
+ public static Map<String, String> intern(Map<String, String> map) {
+ if (map == null) {
+ return null;
+ }
+
+ if (map.isEmpty()) {
+ // nothing to intern
+ return map;
+ }
+ Map<String, String> newMap = new HashMap<>(map.size());
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ newMap.put(intern(entry.getKey()), intern(entry.getValue()));
+ }
+ return newMap;
+ }
+
+ public static Set<String> asSet(String... elements) {
+ if (elements == null) return new HashSet<>();
+ Set<String> set = new HashSet<>(elements.length);
+ Collections.addAll(set, elements);
+ return set;
+ }
+
+ /**
+ * Normalize all identifiers to make equality comparisons easier.
+ *
+ * @param identifier identifier
+ * @return normalized version, with white space removed and all lower case.
+ */
+ public static String normalizeIdentifier(String identifier) {
+ return identifier.trim().toLowerCase();
+ }
+
+ /**
+ * Make a string representation of the exception.
+ *
+ * @param e The exception to stringify
+ * @return A string with exception name and call stack.
+ * @deprecated
+ */
+ @Deprecated
+ public static String stringifyException(Throwable e) {
+ StringWriter stm = new StringWriter();
+ PrintWriter wrt = new PrintWriter(stm);
+ e.printStackTrace(wrt);
+ wrt.close();
+ return stm.toString();
+ }
+
+ /**
+ * Given an array of bytes it will convert the bytes to a hex string representation of the bytes.
+ *
+ * @param bytes Input bytes
+ * @param start start index, inclusively
+ * @param end end index, exclusively
+ * @return hex string representation of the byte array
+ */
+ public static String byteToHexString(byte[] bytes, int start, int end) {
+ return org.apache.hadoop.util.StringUtils.byteToHexString(bytes, start, end);
+ }
+
+ /**
+ * Checks if the input string/char sequence is empty
+ *
+ * @param cs Input char sequence
+ * @return true if empty and false if not
+ */
+ public static boolean isEmpty(CharSequence cs) {
+ return cs == null || cs.length() == 0;
+ }
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/thrift/fb303.thrift b/kyuubi-relocated-hive-metastore-client/src/main/thrift/fb303.thrift
new file mode 100644
index 0000000..f1ca982
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/thrift/fb303.thrift
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * fb303.thrift
+ */
+
+namespace java com.facebook.fb303
+namespace cpp facebook.fb303
+namespace perl Facebook.FB303
+namespace netstd Facebook.FB303.Test
+
+/**
+ * Common status reporting mechanism across all services
+ */
+enum fb_status {
+ DEAD = 0,
+ STARTING = 1,
+ ALIVE = 2,
+ STOPPING = 3,
+ STOPPED = 4,
+ WARNING = 5,
+}
+
+/**
+ * Standard base service
+ */
+service FacebookService {
+
+ /**
+ * Returns a descriptive name of the service
+ */
+ string getName(),
+
+ /**
+ * Returns the version of the service
+ */
+ string getVersion(),
+
+ /**
+ * Gets the status of this service
+ */
+ fb_status getStatus(),
+
+ /**
+ * User friendly description of status, such as why the service is in
+ * the dead or warning state, or what is being started or stopped.
+ */
+ string getStatusDetails(),
+
+ /**
+ * Gets the counters for this service
+ */
+ map<string, i64> getCounters(),
+
+ /**
+ * Gets the value of a single counter
+ */
+ i64 getCounter(1: string key),
+
+ /**
+ * Sets an option
+ */
+ void setOption(1: string key, 2: string value),
+
+ /**
+ * Gets an option
+ */
+ string getOption(1: string key),
+
+ /**
+ * Gets all options
+ */
+ map<string, string> getOptions(),
+
+ /**
+ * Returns a CPU profile over the given time interval (client and server
+ * must agree on the profile format).
+ */
+ string getCpuProfile(1: i32 profileDurationInSec),
+
+ /**
+ * Returns the unix time that the server has been running since
+ */
+ i64 aliveSince(),
+
+ /**
+ * Tell the server to reload its configuration, reopen log files, etc
+ */
+ oneway void reinitialize(),
+
+ /**
+ * Suggest a shutdown to the server
+ */
+ oneway void shutdown(),
+
+}
diff --git a/kyuubi-relocated-hive-metastore-client/src/main/thrift/hive_metastore.thrift b/kyuubi-relocated-hive-metastore-client/src/main/thrift/hive_metastore.thrift
new file mode 100644
index 0000000..04347b5
--- /dev/null
+++ b/kyuubi-relocated-hive-metastore-client/src/main/thrift/hive_metastore.thrift
@@ -0,0 +1,45 @@
+#!/usr/local/bin/thrift -java
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#
+# Thrift Service that the MetaStore is built on
+#
+
+include "fb303.thrift"
+
+namespace java org.apache.kyuubi.shaded.hive.metastore.api
+
+// Exceptions.
+
+exception MetaException {
+ 1: string message
+}
+
+/**
+* This interface is live.
+*/
+service ThriftHiveMetastore extends fb303.FacebookService
+{
+ // get metastore server delegation token for use from the map/reduce tasks to authenticate
+ // to metastore server
+ string get_delegation_token(1:string token_owner, 2:string renewer_kerberos_principal_name)
+ throws (1:MetaException o1)
+}
+
diff --git a/kyuubi-relocated-zookeeper-parent/kyuubi-relocated-zookeeper-34/pom.xml b/kyuubi-relocated-zookeeper-parent/kyuubi-relocated-zookeeper-34/pom.xml
index 03d3cc7..fed1c9e 100644
--- a/kyuubi-relocated-zookeeper-parent/kyuubi-relocated-zookeeper-34/pom.xml
+++ b/kyuubi-relocated-zookeeper-parent/kyuubi-relocated-zookeeper-34/pom.xml
@@ -152,4 +152,33 @@ under the License.
</exclusions>
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <!-- This is to ensure references to shaded classes can be resolved in IDEs such as
+ Intellij. For reference: https://youtrack.jetbrains.com/issue/IDEA-126596 -->
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>compile</id>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>${basedir}/target/${project.artifactId}-${project.version}.jar</file>
+ <type>jar</type>
+ <classifier>optional</classifier>
+ </artifact>
+ </artifacts>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/kyuubi-relocated-zookeeper-parent/kyuubi-relocated-zookeeper-36/pom.xml b/kyuubi-relocated-zookeeper-parent/kyuubi-relocated-zookeeper-36/pom.xml
index e8a697c..bbe663a 100644
--- a/kyuubi-relocated-zookeeper-parent/kyuubi-relocated-zookeeper-36/pom.xml
+++ b/kyuubi-relocated-zookeeper-parent/kyuubi-relocated-zookeeper-36/pom.xml
@@ -161,4 +161,33 @@ under the License.
</exclusions>
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <!-- This is to ensure references to shaded classes can be resolved in IDEs such as
+ Intellij. For reference: https://youtrack.jetbrains.com/issue/IDEA-126596 -->
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>compile</id>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>${basedir}/target/${project.artifactId}-${project.version}.jar</file>
+ <type>jar</type>
+ <classifier>optional</classifier>
+ </artifact>
+ </artifacts>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/pom.xml b/pom.xml
index a6f729f..a4d1320 100644
--- a/pom.xml
+++ b/pom.xml
@@ -52,6 +52,7 @@
<modules>
<module>kyuubi-relocated-force-shading</module>
+ <module>kyuubi-relocated-hive-metastore-client</module>
<module>kyuubi-relocated-hive-service-rpc</module>
<module>kyuubi-relocated-zookeeper-parent</module>
</modules>