You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2024/02/12 17:12:25 UTC
(spark) branch master updated: [SPARK-44445][BUILD][TESTS] Use `org.seleniumhq.selenium.htmlunit3-driver` instead of `net.sourceforge.htmlunit`
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f5b1e37c9e6a [SPARK-44445][BUILD][TESTS] Use `org.seleniumhq.selenium.htmlunit3-driver` instead of `net.sourceforge.htmlunit`
f5b1e37c9e6a is described below
commit f5b1e37c9e6a4ec2fd897f97cd4526415e6c0e49
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Mon Feb 12 09:12:10 2024 -0800
[SPARK-44445][BUILD][TESTS] Use `org.seleniumhq.selenium.htmlunit3-driver` instead of `net.sourceforge.htmlunit`
### What changes were proposed in this pull request?
This PR aims to use `org.seleniumhq.selenium.htmlunit3-driver` 4.17.0 instead of old `net.sourceforge.htmlunit` test dependencies and `org.seleniumhq.selenium.htmlunit-driver` test dependency;
- Remove `net.sourceforge.htmlunit.htmlunit` `2.70.0`.
- Remove `net.sourceforge.htmlunit.htmlunit-core-js` `2.70.0`.
- Remove `org.seleniumhq.selenium.htmlunit-driver` `4.12.0`.
- Remove `xml-apis:xml-apis` `1.4.01`.
### Why are the changes needed?
To help browser-based test suites.
### Does this PR introduce _any_ user-facing change?
No. This is a test-only dependency and code change.
### How was this patch tested?
Manual tests.
```
build/sbt -Dguava.version=32.1.2-jre -Dspark.test.webdriver.chrome.driver=/opt/homebrew/bin/chromedriver -Dtest.default.exclude.tags="" -Phive -Phive-thriftserver \
"core/testOnly *HistoryServerSuite"
```
```
build/sbt -Dguava.version=32.1.2-jre -Dspark.test.webdriver.chrome.driver=/opt/homebrew/bin/chromedriver -Dtest.default.exclude.tags="" -Phive -Phive-thriftserver \
"core/testOnly *UISeleniumSuite"
```
```
build/sbt -Dguava.version=32.1.2-jre -Dspark.test.webdriver.chrome.driver=/opt/homebrew/bin/chromedriver -Dtest.default.exclude.tags="" -Phive -Phive-thriftserver \
"sql/testOnly *UISeleniumSuite"
```
```
build/sbt -Dguava.version=32.1.2-jre -Dspark.test.webdriver.chrome.driver=/opt/homebrew/bin/chromedriver -Dtest.default.exclude.tags="" -Phive -Phive-thriftserver \
"streaming/testOnly *UISeleniumSuite"
```
```
build/sbt -Dguava.version=32.1.2-jre -Dspark.test.webdriver.chrome.driver=/opt/homebrew/bin/chromedriver -Dtest.default.exclude.tags="" -Phive -Phive-thriftserver \
"hive-thriftserver/testOnly *UISeleniumSuite"
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #45079 from dongjoon-hyun/SPARK-44445.
Authored-by: Dongjoon Hyun <dh...@apple.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
core/pom.xml | 19 +---------------
.../spark/deploy/history/HistoryServerSuite.scala | 4 +++-
.../history/RealBrowserUIHistoryServerSuite.scala | 6 ++++-
.../org/apache/spark/ui/UISeleniumSuite.scala | 4 ++--
pom.xml | 26 +++-------------------
sql/core/pom.xml | 2 +-
.../spark/sql/execution/ui/UISeleniumSuite.scala | 8 ++++---
sql/hive-thriftserver/pom.xml | 2 +-
streaming/pom.xml | 2 +-
9 files changed, 22 insertions(+), 51 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index f780551fb555..9b5297cb8543 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -354,18 +354,7 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>htmlunit-driver</artifactId>
- <scope>test</scope>
- </dependency>
- <!-- Coerce sbt into honoring these dependency updates: -->
- <dependency>
- <groupId>net.sourceforge.htmlunit</groupId>
- <artifactId>htmlunit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>net.sourceforge.htmlunit</groupId>
- <artifactId>htmlunit-core-js</artifactId>
+ <artifactId>htmlunit3-driver</artifactId>
<scope>test</scope>
</dependency>
<!-- at least just for tests, coerce SBT to use the updated httpcore/client version -->
@@ -384,12 +373,6 @@
<artifactId>httpcore</artifactId>
<scope>test</scope>
</dependency>
- <!-- Added for selenium: -->
- <dependency>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index 1ca1e8fefd06..b3d7315e169b 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -561,7 +561,9 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with
// app is no longer incomplete
listApplications(false) should not contain(appId)
- assert(jobcount === getNumJobs("/jobs"))
+ eventually(stdTimeout, stdInterval) {
+ assert(jobcount === getNumJobs("/jobs"))
+ }
// no need to retain the test dir now the tests complete
ShutdownHookManager.registerShutdownDeleteDir(logDir)
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala
index ea3a5ef5ba10..5eeb2c7a8053 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala
@@ -22,8 +22,10 @@ import javax.servlet.http.HttpServletRequest
import org.eclipse.jetty.proxy.ProxyServlet
import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
import org.openqa.selenium.WebDriver
+import org.scalatest.concurrent.Eventually._
import org.scalatest.matchers.must.Matchers
import org.scalatest.matchers.should.Matchers._
+import org.scalatest.time.SpanSugar._
import org.scalatestplus.selenium.WebBrowser
import org.apache.spark._
@@ -147,7 +149,9 @@ abstract class RealBrowserUIHistoryServerSuite(val driverProp: String)
// there are at least some URL links that were generated via javascript,
// and they all contain the spark.ui.proxyBase (uiRoot)
- links.length should be > 4
+ eventually(timeout(10.seconds)) {
+ links.length should be > 4
+ }
for (link <- links) {
link should startWith(url + uiRoot)
}
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index 8689a9c16d69..dbcc95fc350e 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -24,9 +24,9 @@ import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
import scala.io.Source
import scala.xml.Node
-import com.gargoylesoftware.css.parser.CSSParseException
-import com.gargoylesoftware.htmlunit.DefaultCssErrorHandler
import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap
+import org.htmlunit.DefaultCssErrorHandler
+import org.htmlunit.cssparser.parser.CSSParseException
import org.json4s._
import org.json4s.jackson.JsonMethods
import org.openqa.selenium.{By, WebDriver}
diff --git a/pom.xml b/pom.xml
index 79d572f1b8bf..ed6f48262570 100644
--- a/pom.xml
+++ b/pom.xml
@@ -215,8 +215,7 @@
<antlr4.version>4.13.1</antlr4.version>
<jpam.version>1.1</jpam.version>
<selenium.version>4.12.1</selenium.version>
- <htmlunit-driver.version>4.12.0</htmlunit-driver.version>
- <htmlunit.version>2.70.0</htmlunit.version>
+ <htmlunit3-driver.version>4.17.0</htmlunit3-driver.version>
<maven-antrun.version>3.1.0</maven-antrun.version>
<commons-crypto.version>1.1.0</commons-crypto.version>
<commons-cli.version>1.6.0</commons-cli.version>
@@ -710,29 +709,10 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>htmlunit-driver</artifactId>
- <version>${htmlunit-driver.version}</version>
+ <artifactId>htmlunit3-driver</artifactId>
+ <version>${htmlunit3-driver.version}</version>
<scope>test</scope>
</dependency>
- <!-- Update htmlunit dependency that selenium uses for better JS support -->
- <dependency>
- <groupId>net.sourceforge.htmlunit</groupId>
- <artifactId>htmlunit</artifactId>
- <version>${htmlunit.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>net.sourceforge.htmlunit</groupId>
- <artifactId>htmlunit-core-js</artifactId>
- <version>${htmlunit.version}</version>
- <scope>test</scope>
- </dependency>
- <!-- Added for selenium only, and should match its dependent version: -->
- <dependency>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- <version>1.4.01</version>
- </dependency>
<!-- log4j -->
<dependency>
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index f2d888ceafe7..1ef41f9b43da 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -225,7 +225,7 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>htmlunit-driver</artifactId>
+ <artifactId>htmlunit3-driver</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala
index fca2c3b4c7c5..c376d1f4614f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala
@@ -22,6 +22,7 @@ import scala.jdk.CollectionConverters._
import org.apache.commons.text.StringEscapeUtils.escapeJava
import org.apache.commons.text.translate.EntityArrays._
+import org.openqa.selenium.By
import org.openqa.selenium.htmlunit.HtmlUnitDriver
import org.scalatest.concurrent.Eventually.eventually
import org.scalatest.concurrent.Futures.{interval, timeout}
@@ -50,7 +51,8 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser {
val webUrl = spark.sparkContext.uiWebUrl
assert(webUrl.isDefined, "please turn on spark.ui.enabled")
go to s"${webUrl.get}/SQL"
- findAll(cssSelector("""#failed-table td .stacktrace-details""")).map(_.text).toList
+ findAll(cssSelector("""#failed-table td .stacktrace-details"""))
+ .map(_.underlying.findElement(By.xpath("..")).getText).toList
}
private def findErrorSummaryOnSQLUI(): String = {
@@ -102,13 +104,13 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser {
eventually(timeout(10.seconds), interval(100.milliseconds)) {
val sd = findErrorMessageOnSQLUI()
assert(sd.size === 1, "Analyze fail shall show the query in failed table")
- assert(sd.head.startsWith("[TABLE_OR_VIEW_NOT_FOUND]"))
+ assert(sd.head.startsWith("TABLE_OR_VIEW_NOT_FOUND"))
val id = findExecutionIDOnSQLUI()
// check query detail page
go to s"${spark.sparkContext.uiWebUrl.get}/SQL/execution/?id=$id"
val planDot = findAll(cssSelector(""".dot-file""")).map(_.text).toList
- assert(planDot.head.startsWith("digraph G {"))
+ assert(planDot.size === 1)
val planDetails = findAll(cssSelector("""#physical-plan-details""")).map(_.text).toList
assert(planDetails.head.isEmpty)
}
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 920a9b67376f..04d77209bb39 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -103,7 +103,7 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>htmlunit-driver</artifactId>
+ <artifactId>htmlunit3-driver</artifactId>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index f409791f4670..85a4d268d2a2 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -113,7 +113,7 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>htmlunit-driver</artifactId>
+ <artifactId>htmlunit3-driver</artifactId>
<scope>test</scope>
</dependency>
<dependency>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org