You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2020/05/21 18:44:28 UTC
[spark] branch master updated: [SPARK-31765][WEBUI] Upgrade
HtmlUnit >= 2.37.0
This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 92877c4 [SPARK-31765][WEBUI] Upgrade HtmlUnit >= 2.37.0
92877c4 is described below
commit 92877c4ef2ad113c156b7d9c359f396187c78fa3
Author: Kousuke Saruta <sa...@oss.nttdata.com>
AuthorDate: Thu May 21 11:43:25 2020 -0700
[SPARK-31765][WEBUI] Upgrade HtmlUnit >= 2.37.0
### What changes were proposed in this pull request?
This PR upgrades HtmlUnit.
Selenium and Jetty also upgraded because of dependency.
### Why are the changes needed?
Recently, a security issue which affects HtmlUnit is reported.
https://nvd.nist.gov/vuln/detail/CVE-2020-5529
According to the report, arbitrary code can be run by malicious users.
HtmlUnit is used for test so the impact might not be large but it's better to upgrade it just in case.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing testcases.
Closes #28585 from sarutak/upgrade-htmlunit.
Authored-by: Kousuke Saruta <sa...@oss.nttdata.com>
Signed-off-by: Gengliang Wang <ge...@databricks.com>
---
core/pom.xml | 2 +-
core/src/main/scala/org/apache/spark/ui/JettyUtils.scala | 7 ++++++-
core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala | 3 ++-
pom.xml | 10 +++++-----
sql/core/pom.xml | 2 +-
sql/hive-thriftserver/pom.xml | 2 +-
streaming/pom.xml | 2 +-
7 files changed, 17 insertions(+), 11 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index b0f6888..14b217d 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -334,7 +334,7 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>selenium-htmlunit-driver</artifactId>
+ <artifactId>htmlunit-driver</artifactId>
<scope>test</scope>
</dependency>
<!-- Coerce sbt into honoring these dependency updates: -->
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index 4b4788f..f1962ef 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -23,6 +23,7 @@ import javax.servlet.DispatcherType
import javax.servlet.http._
import scala.language.implicitConversions
+import scala.util.Try
import scala.xml.Node
import org.eclipse.jetty.client.HttpClient
@@ -500,7 +501,11 @@ private[spark] case class ServerInfo(
threadPool match {
case pool: QueuedThreadPool =>
// Workaround for SPARK-30385 to avoid Jetty's acceptor thread shrink.
- pool.setIdleTimeout(0)
+ // As of Jetty 9.4.21, the implementation of
+ // QueuedThreadPool#setIdleTimeout is changed and IllegalStateException
+ // will be thrown if we try to set idle timeout after the server has started.
+ // But this workaround works for Jetty 9.4.28 by ignoring the exception.
+ Try(pool.setIdleTimeout(0))
case _ =>
}
server.stop()
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index 3ec9385..e96d82a 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -24,6 +24,7 @@ import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
import scala.io.Source
import scala.xml.Node
+import com.gargoylesoftware.css.parser.CSSParseException
import com.gargoylesoftware.htmlunit.DefaultCssErrorHandler
import org.json4s._
import org.json4s.jackson.JsonMethods
@@ -33,7 +34,6 @@ import org.scalatest._
import org.scalatest.concurrent.Eventually._
import org.scalatest.time.SpanSugar._
import org.scalatestplus.selenium.WebBrowser
-import org.w3c.css.sac.CSSParseException
import org.apache.spark._
import org.apache.spark.LocalSparkContext._
@@ -784,6 +784,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
eventually(timeout(10.seconds), interval(50.milliseconds)) {
goToUi(sc, "/jobs")
+
val jobDesc =
driver.findElement(By.cssSelector("div[class='application-timeline-content']"))
jobDesc.getAttribute("data-title") should include ("collect at <console>:25")
diff --git a/pom.xml b/pom.xml
index fd4cebc..29f7fec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -139,7 +139,7 @@
<orc.classifier></orc.classifier>
<hive.parquet.group>com.twitter</hive.parquet.group>
<hive.parquet.version>1.6.0</hive.parquet.version>
- <jetty.version>9.4.18.v20190429</jetty.version>
+ <jetty.version>9.4.28.v20200408</jetty.version>
<javaxservlet.version>3.1.0</javaxservlet.version>
<chill.version>0.9.5</chill.version>
<ivy.version>2.4.0</ivy.version>
@@ -187,8 +187,8 @@
<libthrift.version>0.12.0</libthrift.version>
<antlr4.version>4.7.1</antlr4.version>
<jpam.version>1.1</jpam.version>
- <selenium.version>2.52.0</selenium.version>
- <htmlunit.version>2.22</htmlunit.version>
+ <selenium.version>3.141.59</selenium.version>
+ <htmlunit.version>2.40.0</htmlunit.version>
<!--
Managed up from older version from Avro; sync with jackson-module-paranamer dependency version
-->
@@ -591,8 +591,8 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>selenium-htmlunit-driver</artifactId>
- <version>${selenium.version}</version>
+ <artifactId>htmlunit-driver</artifactId>
+ <version>${htmlunit.version}</version>
<scope>test</scope>
</dependency>
<!-- Update htmlunit dependency that selenium uses for better JS support -->
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 7c5fcba..e4ef146 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -162,7 +162,7 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>selenium-htmlunit-driver</artifactId>
+ <artifactId>htmlunit-driver</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 1de2677..5bf20b2 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -95,7 +95,7 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>selenium-htmlunit-driver</artifactId>
+ <artifactId>htmlunit-driver</artifactId>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index ea351d4..53b49dd 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -109,7 +109,7 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>selenium-htmlunit-driver</artifactId>
+ <artifactId>htmlunit-driver</artifactId>
<scope>test</scope>
</dependency>
<dependency>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org