You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@livy.apache.org by js...@apache.org on 2018/01/04 02:22:47 UTC
incubator-livy git commit: [LIVY-429] preserve url fragment in
spark.yarn.dist.archives
Repository: incubator-livy
Updated Branches:
refs/heads/master e54318314 -> 85e8fba3c
[LIVY-429] preserve url fragment in spark.yarn.dist.archives
## What changes were proposed in this pull request?
Preserve url fragment identifier in spark.yarn.dist.archives setting when there is no schema specified. Current code truncates url fragment, as a result Spark just copies archives to job local dir without extracting it.
## How was this patch tested?
Added case to existing unit test
Author: Artem Plotnikov <ap...@joom.com>
Closes #71 from broartem/LIVY-429.
Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/85e8fba3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/85e8fba3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/85e8fba3
Branch: refs/heads/master
Commit: 85e8fba3ce5d0e180655fd9ee239cb320d550780
Parents: e543183
Author: Artem Plotnikov <ap...@joom.com>
Authored: Thu Jan 4 10:22:40 2018 +0800
Committer: jerryshao <ss...@hortonworks.com>
Committed: Thu Jan 4 10:22:40 2018 +0800
----------------------------------------------------------------------
server/src/main/scala/org/apache/livy/sessions/Session.scala | 7 +++++--
.../src/test/scala/org/apache/livy/sessions/SessionSpec.scala | 4 ++--
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/85e8fba3/server/src/main/scala/org/apache/livy/sessions/Session.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/sessions/Session.scala b/server/src/main/scala/org/apache/livy/sessions/Session.scala
index d467076..0b54779 100644
--- a/server/src/main/scala/org/apache/livy/sessions/Session.scala
+++ b/server/src/main/scala/org/apache/livy/sessions/Session.scala
@@ -116,8 +116,11 @@ object Session {
val defaultFS = livyConf.hadoopConf.get("fs.defaultFS").stripSuffix("/")
val resolved =
if (uri.getScheme() == null) {
- require(uri.getPath().startsWith("/"), s"Path '${uri.getPath()}' is not absolute.")
- new URI(defaultFS + uri.getPath())
+ val pathWithSegment =
+ if (uri.getFragment() != null) uri.getPath() + '#' + uri.getFragment() else uri.getPath()
+
+ require(pathWithSegment.startsWith("/"), s"Path '${uri.getPath()}' is not absolute.")
+ new URI(defaultFS + pathWithSegment)
} else {
uri
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/85e8fba3/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
----------------------------------------------------------------------
diff --git a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
index a7bfaaa..05b41bb 100644
--- a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
+++ b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
@@ -29,8 +29,8 @@ class SessionSpec extends FunSuite with LivyBaseUnitTestSuite {
val conf = new LivyConf(false)
conf.hadoopConf.set("fs.defaultFS", "dummy:///")
- val uris = Seq("http://example.com/foo", "hdfs:/bar", "/baz")
- val expected = Seq(uris(0), uris(1), "dummy:///baz")
+ val uris = Seq("http://example.com/foo", "hdfs:/bar", "/baz", "/foo#bar")
+ val expected = Seq(uris(0), uris(1), "dummy:///baz", "dummy:///foo#bar")
assert(Session.resolveURIs(uris, conf) === expected)
intercept[IllegalArgumentException] {