You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2022/06/14 15:58:40 UTC
[spark] branch master updated: [SPARK-39464][CORE][TESTS] Use `Utils.localCanonicalHostName` instead of `localhost` in tests
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7346fb184e2 [SPARK-39464][CORE][TESTS] Use `Utils.localCanonicalHostName` instead of `localhost` in tests
7346fb184e2 is described below
commit 7346fb184e2d0d27f9720b44cfdd5fb7fe5a10f5
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Tue Jun 14 08:58:23 2022 -0700
[SPARK-39464][CORE][TESTS] Use `Utils.localCanonicalHostName` instead of `localhost` in tests
### What changes were proposed in this pull request?
This PR aims to use `Utils.localCanonicalHostName` instead of a constant `localhost` in the following suites.
- `MasterSuite`
- `MasterWebUISuite`
- `RocksDBBackendHistoryServerSuite`
### Why are the changes needed?
These test cases fails when we run with `SPARK_LOCAL_IP` on `IPv6`-only environment.
### Does this PR introduce _any_ user-facing change?
No. This is a test-only change.
### How was this patch tested?
Pass the CIs first and manually test on `IPv6`-only environment.
Closes #36866 from dongjoon-hyun/SPARK-39464.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../spark/deploy/history/HistoryServerSuite.scala | 35 +++++++++++-----------
.../apache/spark/deploy/master/MasterSuite.scala | 12 ++++----
.../spark/deploy/master/ui/MasterWebUISuite.scala | 9 +++---
3 files changed, 29 insertions(+), 27 deletions(-)
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index 02a32a80ddd..1aa846b3ac4 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -74,6 +74,7 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with
private var provider: FsHistoryProvider = null
private var server: HistoryServer = null
+ private val localhost: String = Utils.localCanonicalHostName()
private var port: Int = -1
protected def diskBackend: HybridStoreDiskBackend.Value
@@ -428,12 +429,12 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with
// build a URL for an app or app/attempt plus a page underneath
def buildURL(appId: String, suffix: String): URL = {
- new URL(s"http://localhost:$port/history/$appId$suffix")
+ new URL(s"http://$localhost:$port/history/$appId$suffix")
}
// build a rest URL for the application and suffix.
def applications(appId: String, suffix: String): URL = {
- new URL(s"http://localhost:$port/api/v1/applications/$appId$suffix")
+ new URL(s"http://$localhost:$port/api/v1/applications/$appId$suffix")
}
// start initial job
@@ -586,11 +587,11 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with
val port = server.boundPort
val testUrls = Seq(
- s"http://localhost:$port/api/v1/applications/$appId/1/jobs",
- s"http://localhost:$port/history/$appId/1/jobs/",
- s"http://localhost:$port/api/v1/applications/$appId/logs",
- s"http://localhost:$port/api/v1/applications/$appId/1/logs",
- s"http://localhost:$port/api/v1/applications/$appId/2/logs")
+ s"http://$localhost:$port/api/v1/applications/$appId/1/jobs",
+ s"http://$localhost:$port/history/$appId/1/jobs/",
+ s"http://$localhost:$port/api/v1/applications/$appId/logs",
+ s"http://$localhost:$port/api/v1/applications/$appId/1/logs",
+ s"http://$localhost:$port/api/v1/applications/$appId/2/logs")
tests.foreach { case (user, expectedCode) =>
testUrls.foreach { url =>
@@ -609,9 +610,9 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with
val port = server.boundPort
val testUrls = Seq(
- s"http://localhost:$port/api/v1/applications/$appId/logs",
- s"http://localhost:$port/api/v1/applications/$appId/1/logs",
- s"http://localhost:$port/api/v1/applications/$appId/2/logs")
+ s"http://$localhost:$port/api/v1/applications/$appId/logs",
+ s"http://$localhost:$port/api/v1/applications/$appId/1/logs",
+ s"http://$localhost:$port/api/v1/applications/$appId/2/logs")
testUrls.foreach { url =>
TestUtils.httpResponseCode(new URL(url))
@@ -636,9 +637,9 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with
def buildPageAttemptUrl(appId: String, attemptId: Option[Int]): URL = {
attemptId match {
case Some(id) =>
- new URL(s"http://localhost:$port/history/$appId/$id")
+ new URL(s"http://$localhost:$port/history/$appId/$id")
case None =>
- new URL(s"http://localhost:$port/history/$appId")
+ new URL(s"http://$localhost:$port/history/$appId")
}
}
@@ -663,7 +664,7 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with
}
def getContentAndCode(path: String, port: Int = port): (Int, Option[String], Option[String]) = {
- HistoryServerSuite.getContentAndCode(new URL(s"http://localhost:$port/api/v1/$path"))
+ HistoryServerSuite.getContentAndCode(new URL(s"http://$localhost:$port/api/v1/$path"))
}
def getUrl(path: String): String = {
@@ -671,7 +672,7 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with
}
def generateURL(path: String): URL = {
- new URL(s"http://localhost:$port/api/v1/$path")
+ new URL(s"http://$localhost:$port/api/v1/$path")
}
def generateExpectation(name: String, path: String): Unit = {
@@ -686,7 +687,7 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with
test("SPARK-31697: HistoryServer should set Content-Type") {
val port = server.boundPort
val nonExistenceAppId = "local-non-existence"
- val url = new URL(s"http://localhost:$port/history/$nonExistenceAppId")
+ val url = new URL(s"http://$localhost:$port/history/$nonExistenceAppId")
val conn = url.openConnection().asInstanceOf[HttpURLConnection]
conn.setRequestMethod("GET")
conn.connect()
@@ -697,7 +698,7 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with
test("Redirect to the root page when accessed to /history/") {
val port = server.boundPort
- val url = new URL(s"http://localhost:$port/history/")
+ val url = new URL(s"http://$localhost:$port/history/")
val conn = url.openConnection().asInstanceOf[HttpURLConnection]
conn.setRequestMethod("GET")
conn.setUseCaches(false)
@@ -705,7 +706,7 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with
conn.setInstanceFollowRedirects(false)
conn.connect()
assert(conn.getResponseCode === 302)
- assert(conn.getHeaderField("Location") === s"http://localhost:$port/")
+ assert(conn.getHeaderField("Location") === s"http://$localhost:$port/")
}
}
diff --git a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
index 3832e382a70..1fac3522aed 100644
--- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
@@ -325,9 +325,9 @@ class MasterSuite extends SparkFunSuite
val conf = new SparkConf()
val localCluster = LocalSparkCluster(2, 2, 512, conf)
localCluster.start()
- val masterUrl = s"http://localhost:${localCluster.masterWebUIPort}"
+ val masterUrl = s"http://${Utils.localCanonicalHostName()}:${localCluster.masterWebUIPort}"
try {
- eventually(timeout(5.seconds), interval(100.milliseconds)) {
+ eventually(timeout(50.seconds), interval(100.milliseconds)) {
val json = Utils
.tryWithResource(Source.fromURL(s"$masterUrl/json"))(_.getLines().mkString("\n"))
val JArray(workers) = (parse(json) \ "workers")
@@ -362,9 +362,9 @@ class MasterSuite extends SparkFunSuite
conf.set(UI_REVERSE_PROXY, true)
val localCluster = LocalSparkCluster(2, 2, 512, conf)
localCluster.start()
- val masterUrl = s"http://localhost:${localCluster.masterWebUIPort}"
+ val masterUrl = s"http://${Utils.localCanonicalHostName()}:${localCluster.masterWebUIPort}"
try {
- eventually(timeout(5.seconds), interval(100.milliseconds)) {
+ eventually(timeout(50.seconds), interval(100.milliseconds)) {
val json = Utils
.tryWithResource(Source.fromURL(s"$masterUrl/json"))(_.getLines().mkString("\n"))
val JArray(workers) = (parse(json) \ "workers")
@@ -400,9 +400,9 @@ class MasterSuite extends SparkFunSuite
conf.set(UI_REVERSE_PROXY_URL, reverseProxyUrl)
val localCluster = LocalSparkCluster(2, 2, 512, conf)
localCluster.start()
- val masterUrl = s"http://localhost:${localCluster.masterWebUIPort}"
+ val masterUrl = s"http://${Utils.localCanonicalHostName()}:${localCluster.masterWebUIPort}"
try {
- eventually(timeout(5.seconds), interval(100.milliseconds)) {
+ eventually(timeout(50.seconds), interval(100.milliseconds)) {
val json = Utils
.tryWithResource(Source.fromURL(s"$masterUrl/json"))(_.getLines().mkString("\n"))
val JArray(workers) = (parse(json) \ "workers")
diff --git a/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala
index be83ec12f92..253517fdcf9 100644
--- a/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala
@@ -32,6 +32,7 @@ import org.apache.spark.deploy.DeployMessages.{DecommissionWorkersOnHosts, KillD
import org.apache.spark.deploy.DeployTestUtils._
import org.apache.spark.deploy.master._
import org.apache.spark.rpc.{RpcEndpointRef, RpcEnv}
+import org.apache.spark.util.Utils
class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll {
@@ -68,7 +69,7 @@ class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll {
when(master.idToApp).thenReturn(HashMap[String, ApplicationInfo]((activeApp.id, activeApp)))
- val url = s"http://localhost:${masterWebUI.boundPort}/app/kill/"
+ val url = s"http://${Utils.localCanonicalHostName()}:${masterWebUI.boundPort}/app/kill/"
val body = convPostDataToString(Map(("id", activeApp.id), ("terminate", "true")))
val conn = sendHttpRequest(url, "POST", body)
conn.getResponseCode
@@ -79,7 +80,7 @@ class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll {
test("kill driver") {
val activeDriverId = "driver-0"
- val url = s"http://localhost:${masterWebUI.boundPort}/driver/kill/"
+ val url = s"http://${Utils.localCanonicalHostName()}:${masterWebUI.boundPort}/driver/kill/"
val body = convPostDataToString(Map(("id", activeDriverId), ("terminate", "true")))
val conn = sendHttpRequest(url, "POST", body)
conn.getResponseCode
@@ -89,7 +90,7 @@ class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll {
}
private def testKillWorkers(hostnames: Seq[String]): Unit = {
- val url = s"http://localhost:${masterWebUI.boundPort}/workers/kill/"
+ val url = s"http://${Utils.localCanonicalHostName()}:${masterWebUI.boundPort}/workers/kill/"
val body = convPostDataToString(hostnames.map(("host", _)))
val conn = sendHttpRequest(url, "POST", body)
// The master is mocked here, so cannot assert on the response code
@@ -99,7 +100,7 @@ class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll {
}
test("Kill one host") {
- testKillWorkers(Seq("localhost"))
+ testKillWorkers(Seq("${Utils.localCanonicalHostName()}"))
}
test("Kill multiple hosts") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org