You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2022/06/23 00:52:59 UTC
[spark] branch master updated: [SPARK-39562][SQL][TESTS] Make `hive-thriftserver` module unit tests pass in IPv6 env
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e6196bc7321 [SPARK-39562][SQL][TESTS] Make `hive-thriftserver` module unit tests pass in IPv6 env
e6196bc7321 is described below
commit e6196bc7321fdf26644908ecba080f50a2424573
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Wed Jun 22 17:52:46 2022 -0700
[SPARK-39562][SQL][TESTS] Make `hive-thriftserver` module unit tests pass in IPv6 env
### What changes were proposed in this pull request?
This PR aims to make `hive-thrift server` module unit tests pass in IPv6 environment.
### Why are the changes needed?
To have IPv6 test coverage of Spark Thrift Server.
### Does this PR introduce _any_ user-facing change?
No. This is a test-only change.
### How was this patch tested?
Pass the CIs.
Closes #36961 from dongjoon-hyun/SPARK-39562.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../sql/hive/thriftserver/HiveThriftServer2Suites.scala | 9 +++++----
.../SparkThriftServerProtocolVersionsSuite.scala | 12 ++++++------
.../apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala | 8 ++++----
3 files changed, 15 insertions(+), 14 deletions(-)
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index 7c93fa5ab2e..ecbe87b163d 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -67,7 +67,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftServer2Test {
private def withCLIServiceClient(f: ThriftCLIServiceClient => Unit): Unit = {
// Transport creation logic below mimics HiveConnection.createBinaryTransport
- val rawTransport = new TSocket("localhost", serverPort)
+ val rawTransport = new TSocket(localhost, serverPort)
val user = System.getProperty("user.name")
val transport = PlainSaslHelper.getPlainTransport(user, "anonymous", rawTransport)
val protocol = new TBinaryProtocol(transport)
@@ -1189,6 +1189,7 @@ abstract class HiveThriftServer2TestBase extends SparkFunSuite with BeforeAndAft
protected val startScript = "../../sbin/start-thriftserver.sh".split("/").mkString(File.separator)
protected val stopScript = "../../sbin/stop-thriftserver.sh".split("/").mkString(File.separator)
+ val localhost = Utils.localHostNameForURI()
private var listeningPort: Int = _
protected def serverPort: Int = listeningPort
@@ -1240,7 +1241,7 @@ abstract class HiveThriftServer2TestBase extends SparkFunSuite with BeforeAndAft
| --master local
| --hiveconf ${ConfVars.METASTORECONNECTURLKEY}=$metastoreJdbcUri
| --hiveconf ${ConfVars.METASTOREWAREHOUSE}=$warehousePath
- | --hiveconf ${ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST}=localhost
+ | --hiveconf ${ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST}=$localhost
| --hiveconf ${ConfVars.HIVE_SERVER2_TRANSPORT_MODE}=$mode
| --hiveconf ${ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION}=$operationLogPath
| --hiveconf ${ConfVars.LOCALSCRATCHDIR}=$lScratchDir
@@ -1423,14 +1424,14 @@ abstract class HiveThriftServer2TestBase extends SparkFunSuite with BeforeAndAft
Utils.classForName(classOf[HiveDriver].getCanonicalName)
protected def jdbcUri(database: String = "default"): String = if (mode == ServerMode.http) {
- s"""jdbc:hive2://localhost:$serverPort/
+ s"""jdbc:hive2://$localhost:$serverPort/
|$database?
|hive.server2.transport.mode=http;
|hive.server2.thrift.http.path=cliservice;
|${hiveConfList}#${hiveVarList}
""".stripMargin.split("\n").mkString.trim
} else {
- s"jdbc:hive2://localhost:$serverPort/$database?${hiveConfList}#${hiveVarList}"
+ s"jdbc:hive2://$localhost:$serverPort/$database?${hiveConfList}#${hiveVarList}"
}
private def tryCaptureSysLog(f: => Unit): Unit = {
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
index daf410556f5..60afcf81536 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
@@ -40,8 +40,8 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftServer2TestBase {
def testExecuteStatementWithProtocolVersion(
version: TProtocolVersion,
sql: String)(f: HiveQueryResultSet => Unit): Unit = {
- val rawTransport = new TSocket("localhost", serverPort)
- val connection = new HiveConnection(s"jdbc:hive2://localhost:$serverPort", new Properties)
+ val rawTransport = new TSocket(localhost, serverPort)
+ val connection = new HiveConnection(s"jdbc:hive2://$localhost:$serverPort", new Properties)
val user = System.getProperty("user.name")
val transport = PlainSaslHelper.getPlainTransport(user, "anonymous", rawTransport)
val client = new Client(new TBinaryProtocol(transport))
@@ -77,8 +77,8 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftServer2TestBase {
}
def testGetInfoWithProtocolVersion(version: TProtocolVersion): Unit = {
- val rawTransport = new TSocket("localhost", serverPort)
- val connection = new HiveConnection(s"jdbc:hive2://localhost:$serverPort", new Properties)
+ val rawTransport = new TSocket(localhost, serverPort)
+ val connection = new HiveConnection(s"jdbc:hive2://$localhost:$serverPort", new Properties)
val transport = PlainSaslHelper.getPlainTransport(user, "anonymous", rawTransport)
val client = new Client(new TBinaryProtocol(transport))
transport.open()
@@ -107,8 +107,8 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftServer2TestBase {
schema: String,
tableNamePattern: String,
tableTypes: JList[String])(f: HiveQueryResultSet => Unit): Unit = {
- val rawTransport = new TSocket("localhost", serverPort)
- val connection = new HiveConnection(s"jdbc:hive2://localhost:$serverPort", new Properties)
+ val rawTransport = new TSocket(localhost, serverPort)
+ val connection = new HiveConnection(s"jdbc:hive2://$localhost:$serverPort", new Properties)
val transport = PlainSaslHelper.getPlainTransport(user, "anonymous", rawTransport)
val client = new Client(new TBinaryProtocol(transport))
transport.open()
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
index cd5bb1f6283..02ee8dd273d 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
@@ -92,7 +92,7 @@ class UISeleniumSuite
| --master local
| --hiveconf ${ConfVars.METASTORECONNECTURLKEY}=$metastoreJdbcUri
| --hiveconf ${ConfVars.METASTOREWAREHOUSE}=$warehousePath
- | --hiveconf ${ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST}=localhost
+ | --hiveconf ${ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST}=$localhost
| --hiveconf ${ConfVars.HIVE_SERVER2_TRANSPORT_MODE}=$mode
| --hiveconf $portConf=0
| --driver-class-path $driverClassPath
@@ -103,7 +103,7 @@ class UISeleniumSuite
test("thrift server ui test") {
withJdbcStatement("test_map") { statement =>
- val baseURL = s"http://localhost:$uiPort"
+ val baseURL = s"http://$localhost:$uiPort"
val queries = Seq(
"CREATE TABLE test_map(key INT, value STRING)",
@@ -131,11 +131,11 @@ class UISeleniumSuite
test("SPARK-36400: Redact sensitive information in UI by config") {
withJdbcStatement("test_tbl1", "test_tbl2") { statement =>
- val baseURL = s"http://localhost:$uiPort"
+ val baseURL = s"http://$localhost:$uiPort"
val Seq(nonMaskedQuery, maskedQuery) = Seq("test_tbl1", "test_tbl2").map (tblName =>
s"CREATE TABLE $tblName(a int) " +
- s"OPTIONS(url='jdbc:postgresql://localhost:5432/$tblName', " +
+ s"OPTIONS(url='jdbc:postgresql://$localhost:5432/$tblName', " +
"user='test_user', password='abcde')")
statement.execute(nonMaskedQuery)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org