You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by fe...@apache.org on 2022/11/23 01:54:33 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #3835] Allow to use spark-internal as resource for batch job
This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new b225a4293 [KYUUBI #3835] Allow to use spark-internal as resource for batch job
b225a4293 is described below
commit b225a4293608c128c8d3b3bb0f0961944ade7bd1
Author: fwang12 <fw...@ebay.com>
AuthorDate: Wed Nov 23 09:54:25 2022 +0800
[KYUUBI #3835] Allow to use spark-internal as resource for batch job
### _Why are the changes needed?_
Allow to use spark-internal as resource for batch job.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [x] Add screenshots for manual tests if appropriate
<img width="654" alt="image" src="https://user-images.githubusercontent.com/6757692/203333994-4dee486b-1b3f-4f46-83f9-39fe02f66f0c.png">
- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #3835 from turboFei/spark_internal.
Closes #3835
552c0e5e [fwang12] allow to use internal resource
Authored-by: fwang12 <fw...@ebay.com>
Signed-off-by: fwang12 <fw...@ebay.com>
---
.../org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala | 1 +
.../scala/org/apache/kyuubi/session/KyuubiBatchSessionImpl.scala | 9 ++++++---
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
index cd12dbd36..410f141e0 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
@@ -190,6 +190,7 @@ object SparkProcessBuilder {
final val APP_KEY = "spark.app.name"
final val TAG_KEY = "spark.yarn.tags"
final val MASTER_KEY = "spark.master"
+ final val INTERNAL_RESOURCE = "spark-internal"
/**
* The path configs from Spark project that might upload local files:
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiBatchSessionImpl.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiBatchSessionImpl.scala
index 24dcc0477..5a17915cb 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiBatchSessionImpl.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiBatchSessionImpl.scala
@@ -27,6 +27,7 @@ import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.kyuubi.client.api.v1.dto.BatchRequest
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.engine.KyuubiApplicationManager
+import org.apache.kyuubi.engine.spark.SparkProcessBuilder
import org.apache.kyuubi.events.{EventBus, KyuubiSessionEvent}
import org.apache.kyuubi.metrics.MetricsConstants.{CONN_OPEN, CONN_TOTAL}
import org.apache.kyuubi.metrics.MetricsSystem
@@ -99,9 +100,11 @@ class KyuubiBatchSessionImpl(
batchRequest.getBatchType,
normalizedConf,
sessionManager.getConf)
- KyuubiApplicationManager.checkApplicationAccessPath(
- batchRequest.getResource,
- sessionManager.getConf)
+ if (batchRequest.getResource != SparkProcessBuilder.INTERNAL_RESOURCE) {
+ KyuubiApplicationManager.checkApplicationAccessPath(
+ batchRequest.getResource,
+ sessionManager.getConf)
+ }
}
override def open(): Unit = {