You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by mr...@apache.org on 2014/04/11 09:50:05 UTC
git commit: SPARK-1417: Spark on Yarn - spark UI link from
resourcemanager is broken
Repository: spark
Updated Branches:
refs/heads/master 44f654eec -> 446bb3417
SPARK-1417: Spark on Yarn - spark UI link from resourcemanager is broken
Author: Thomas Graves <tg...@apache.org>
Closes #344 from tgravescs/SPARK-1417 and squashes the following commits:
c450b5f [Thomas Graves] fix test
e1c1d7e [Thomas Graves] add missing $ to appUIAddress
e982ddb [Thomas Graves] use appUIHostPort in appUIAddress
0803ec2 [Thomas Graves] Review comment updates - remove extra newline, simplify assert in test
658a8ec [Thomas Graves] Add a appUIHostPort routine
0614208 [Thomas Graves] Fix test
2a6b1b7 [Thomas Graves] SPARK-1417: Spark on Yarn - spark UI link from resourcemanager is broken
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/446bb341
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/446bb341
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/446bb341
Branch: refs/heads/master
Commit: 446bb3417a2855a194d49acc0ac316a021eced9d
Parents: 44f654e
Author: Thomas Graves <tg...@apache.org>
Authored: Fri Apr 11 13:17:48 2014 +0530
Committer: Mridul Muralidharan <mr...@apache.org>
Committed: Fri Apr 11 13:17:48 2014 +0530
----------------------------------------------------------------------
.../scala/org/apache/spark/ui/SparkUI.scala | 7 +++-
.../scala/org/apache/spark/SparkUISuite.scala | 35 ++++++++++++++++++++
.../spark/deploy/yarn/ApplicationMaster.scala | 2 +-
.../spark/deploy/yarn/ApplicationMaster.scala | 2 +-
4 files changed, 43 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/446bb341/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index 4c891d7..7fa4fd3 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -113,7 +113,12 @@ private[spark] class SparkUI(
logInfo("Stopped Spark Web UI at %s".format(appUIAddress))
}
- private[spark] def appUIAddress = "http://" + publicHost + ":" + boundPort
+ /**
+ * Return the application UI host:port. This does not include the scheme (http://).
+ */
+ private[spark] def appUIHostPort = publicHost + ":" + boundPort
+
+ private[spark] def appUIAddress = s"http://$appUIHostPort"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/446bb341/core/src/test/scala/org/apache/spark/SparkUISuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/SparkUISuite.scala b/core/src/test/scala/org/apache/spark/SparkUISuite.scala
new file mode 100644
index 0000000..d0d119c
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/SparkUISuite.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import java.net.URI
+
+import org.scalatest.FunSuite
+
+class SparkUISuite extends FunSuite with SharedSparkContext {
+
+ test("verify appUIAddress contains the scheme") {
+ val uiAddress = sc.ui.appUIAddress
+ assert(uiAddress.equals("http://" + sc.ui.appUIHostPort))
+ }
+
+ test("verify appUIAddress contains the port") {
+ val splitUIAddress = sc.ui.appUIAddress.split(':')
+ assert(splitUIAddress(2).toInt == sc.ui.boundPort)
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/446bb341/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
----------------------------------------------------------------------
diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 910484e..67ec95c 100644
--- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -234,7 +234,7 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration,
assert(sparkContext != null || count >= numTries)
if (null != sparkContext) {
- uiAddress = sparkContext.ui.appUIAddress
+ uiAddress = sparkContext.ui.appUIHostPort
this.yarnAllocator = YarnAllocationHandler.newAllocator(
yarnConf,
resourceManager,
http://git-wip-us.apache.org/repos/asf/spark/blob/446bb341/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
----------------------------------------------------------------------
diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index c8a4d2e..61af0f9 100644
--- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -220,7 +220,7 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration,
assert(sparkContext != null || numTries >= maxNumTries)
if (sparkContext != null) {
- uiAddress = sparkContext.ui.appUIAddress
+ uiAddress = sparkContext.ui.appUIHostPort
this.yarnAllocator = YarnAllocationHandler.newAllocator(
yarnConf,
amClient,