You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by fe...@apache.org on 2023/05/15 09:45:35 UTC
[kyuubi] branch branch-1.7 updated: [KYUUBI #4835] [K8S] Using hive conf to check whether to apply HIVE_DELEGATION_TOKEN
This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a commit to branch branch-1.7
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/branch-1.7 by this push:
new 787d42be3 [KYUUBI #4835] [K8S] Using hive conf to check whether to apply HIVE_DELEGATION_TOKEN
787d42be3 is described below
commit 787d42be3a1b356f1c0e96267901520ce6aa0153
Author: fwang12 <fw...@ebay.com>
AuthorDate: Mon May 15 17:45:19 2023 +0800
[KYUUBI #4835] [K8S] Using hive conf to check whether to apply HIVE_DELEGATION_TOKEN
### _Why are the changes needed?_
Now we check the sparkContext.hadoopConfiguration to determine whether to apply HIVE_DELEGATION_TOKEN
Here is the method to create sparkContext hadoopConguration.
And it will add `__spark_hadoop_conf__.xml` to hadoop configuration resource.
```
/**
* Return an appropriate (subclass) of Configuration. Creating config can initialize some Hadoop
* subsystems.
*/
def newConfiguration(conf: SparkConf): Configuration = {
val hadoopConf = SparkHadoopUtil.newConfiguration(conf)
hadoopConf.addResource(SparkHadoopUtil.SPARK_HADOOP_CONF_FILE)
hadoopConf
}
```
```
/**
* Name of the file containing the gateway's Hadoop configuration, to be overlayed on top of the
* cluster's Hadoop config. It is up to the Spark code launching the application to create
* this file if it's desired. If the file doesn't exist, it will just be ignored.
*/
private[spark] val SPARK_HADOOP_CONF_FILE = "__spark_hadoop_conf__.xml"
```
<img width="1091" alt="image" src="https://github.com/apache/kyuubi/assets/6757692/f2a87a23-4565-4164-9eaa-5f7e166519de">
Per the code, this file is only created in yarn module.
#### Spark on yarn
after unzip `__spark_conf__.zip` in spark staging dir, there is a file named `__spark_hadoop_conf__.xml`.
```
grep hive.metastore.uris __spark_hadoop_conf__.xml
<property><name>hive.metastore.uris</name><value>thrift://*******:9083</value><source>programatically</source></property>
```
#### Spark on K8S
Seems for spark on k8s, there is no file named `__spark_hadoop_conf__.xml`
<img width="1580" alt="image" src="https://github.com/apache/kyuubi/assets/6757692/99de73d0-3519-4af3-8f0a-90967949ec0e">
<img width="875" alt="image" src="https://github.com/apache/kyuubi/assets/6757692/f7c477a5-23ca-4b25-8638-4b040b78899d">
We need to check the `hiveConf` instead of `hadoopConf`.
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4835 from turboFei/hive_token.
Closes #4835
7657cbb11 [fwang12] hive conf
7c0af6789 [fwang12] save
Authored-by: fwang12 <fw...@ebay.com>
Signed-off-by: fwang12 <fw...@ebay.com>
(cherry picked from commit 6b5c138651b46a3828f5184b8cd3218ce932a923)
Signed-off-by: fwang12 <fw...@ebay.com>
---
.../engine/spark/SparkTBinaryFrontendService.scala | 27 +++++++++++++++++++-
.../spark/SparkTBinaryFrontendServiceSuite.scala | 29 ++++++++++++++++++++++
2 files changed, 55 insertions(+), 1 deletion(-)
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendService.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendService.scala
index d4eaf3454..2a93537f5 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendService.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendService.scala
@@ -19,6 +19,7 @@ package org.apache.kyuubi.engine.spark
import scala.collection.JavaConverters._
+import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io.Text
import org.apache.hadoop.security.{Credentials, UserGroupInformation}
import org.apache.hadoop.security.token.{Token, TokenIdentifier}
@@ -29,6 +30,7 @@ import org.apache.spark.kyuubi.SparkContextHelper
import org.apache.kyuubi.{KyuubiSQLException, Logging}
import org.apache.kyuubi.config.KyuubiReservedKeys._
import org.apache.kyuubi.ha.client.{EngineServiceDiscovery, ServiceDiscovery}
+import org.apache.kyuubi.reflection.DynConstructors
import org.apache.kyuubi.service.{Serverable, Service, TBinaryFrontendService}
import org.apache.kyuubi.service.TFrontendService._
import org.apache.kyuubi.util.KyuubiHadoopUtils
@@ -101,6 +103,8 @@ class SparkTBinaryFrontendService(
object SparkTBinaryFrontendService extends Logging {
val HIVE_DELEGATION_TOKEN = new Text("HIVE_DELEGATION_TOKEN")
+ val HIVE_CONF_CLASSNAME = "org.apache.hadoop.hive.conf.HiveConf"
+ @volatile private var _hiveConf: Configuration = _
private[spark] def renewDelegationToken(sc: SparkContext, delegationToken: String): Unit = {
val newCreds = KyuubiHadoopUtils.decodeCredentials(delegationToken)
@@ -124,7 +128,7 @@ object SparkTBinaryFrontendService extends Logging {
newTokens: Map[Text, Token[_ <: TokenIdentifier]],
oldCreds: Credentials,
updateCreds: Credentials): Unit = {
- val metastoreUris = sc.hadoopConfiguration.getTrimmed("hive.metastore.uris", "")
+ val metastoreUris = hiveConf(sc.hadoopConfiguration).getTrimmed("hive.metastore.uris", "")
// `HiveMetaStoreClient` selects the first token whose service is "" and kind is
// "HIVE_DELEGATION_TOKEN" to authenticate.
@@ -195,4 +199,25 @@ object SparkTBinaryFrontendService extends Logging {
1
}
}
+
+ private[kyuubi] def hiveConf(hadoopConf: Configuration): Configuration = {
+ if (_hiveConf == null) {
+ synchronized {
+ if (_hiveConf == null) {
+ _hiveConf =
+ try {
+ DynConstructors.builder()
+ .impl(HIVE_CONF_CLASSNAME, classOf[Configuration], classOf[Class[_]])
+ .build[Configuration]()
+ .newInstance(hadoopConf, Class.forName(HIVE_CONF_CLASSNAME))
+ } catch {
+ case e: Throwable =>
+ warn("Fail to create Hive Configuration", e)
+ hadoopConf
+ }
+ }
+ }
+ }
+ _hiveConf
+ }
}
diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendServiceSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendServiceSuite.scala
new file mode 100644
index 000000000..5f81e51f8
--- /dev/null
+++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendServiceSuite.scala
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.engine.spark
+
+import org.apache.hadoop.conf.Configuration
+
+import org.apache.kyuubi.KyuubiFunSuite
+
+class SparkTBinaryFrontendServiceSuite extends KyuubiFunSuite {
+ test("new hive conf") {
+ val hiveConf = SparkTBinaryFrontendService.hiveConf(new Configuration())
+ assert(hiveConf.getClass().getName == SparkTBinaryFrontendService.HIVE_CONF_CLASSNAME)
+ }
+}