You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2018/01/13 15:40:07 UTC
spark git commit: [SPARK-23043][BUILD] Upgrade json4s to 3.5.3
Repository: spark
Updated Branches:
refs/heads/master cd9f49a2a -> 628a1ca5a
[SPARK-23043][BUILD] Upgrade json4s to 3.5.3
## What changes were proposed in this pull request?
Spark still use a few years old version 3.2.11. This change is to upgrade json4s to 3.5.3.
Note that this change does not include the Jackson update because the Jackson version referenced in json4s 3.5.3 is 2.8.4, which has a security vulnerability ([see](https://issues.apache.org/jira/browse/SPARK-20433)).
## How was this patch tested?
Existing unit tests and build.
Author: shimamoto <ch...@gmail.com>
Closes #20233 from shimamoto/upgrade-json4s.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/628a1ca5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/628a1ca5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/628a1ca5
Branch: refs/heads/master
Commit: 628a1ca5a4d14397a90e9e96a7e03e8f63531b20
Parents: cd9f49a
Author: shimamoto <ch...@gmail.com>
Authored: Sat Jan 13 09:40:00 2018 -0600
Committer: Sean Owen <so...@cloudera.com>
Committed: Sat Jan 13 09:40:00 2018 -0600
----------------------------------------------------------------------
.../deploy/history/HistoryServerSuite.scala | 2 +-
.../org/apache/spark/ui/UISeleniumSuite.scala | 19 ++++++++++---------
dev/deps/spark-deps-hadoop-2.6 | 8 ++++----
dev/deps/spark-deps-hadoop-2.7 | 8 ++++----
pom.xml | 13 +++++++------
5 files changed, 26 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/628a1ca5/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index 3738f85..87778dd 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -486,7 +486,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
json match {
case JNothing => Seq()
case apps: JArray =>
- apps.filter(app => {
+ apps.children.filter(app => {
(app \ "attempts") match {
case attempts: JArray =>
val state = (attempts.children.head \ "completed").asInstanceOf[JBool]
http://git-wip-us.apache.org/repos/asf/spark/blob/628a1ca5/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index 3265467..ed51fc4 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -131,7 +131,8 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
val storageJson = getJson(ui, "storage/rdd")
storageJson.children.length should be (1)
- (storageJson \ "storageLevel").extract[String] should be (StorageLevels.DISK_ONLY.description)
+ (storageJson.children.head \ "storageLevel").extract[String] should be (
+ StorageLevels.DISK_ONLY.description)
val rddJson = getJson(ui, "storage/rdd/0")
(rddJson \ "storageLevel").extract[String] should be (StorageLevels.DISK_ONLY.description)
@@ -150,7 +151,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
val updatedStorageJson = getJson(ui, "storage/rdd")
updatedStorageJson.children.length should be (1)
- (updatedStorageJson \ "storageLevel").extract[String] should be (
+ (updatedStorageJson.children.head \ "storageLevel").extract[String] should be (
StorageLevels.MEMORY_ONLY.description)
val updatedRddJson = getJson(ui, "storage/rdd/0")
(updatedRddJson \ "storageLevel").extract[String] should be (
@@ -204,7 +205,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
}
val stageJson = getJson(sc.ui.get, "stages")
stageJson.children.length should be (1)
- (stageJson \ "status").extract[String] should be (StageStatus.FAILED.name())
+ (stageJson.children.head \ "status").extract[String] should be (StageStatus.FAILED.name())
// Regression test for SPARK-2105
class NotSerializable
@@ -325,11 +326,11 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
find(cssSelector(".progress-cell .progress")).get.text should be ("2/2 (1 failed)")
}
val jobJson = getJson(sc.ui.get, "jobs")
- (jobJson \ "numTasks").extract[Int]should be (2)
- (jobJson \ "numCompletedTasks").extract[Int] should be (3)
- (jobJson \ "numFailedTasks").extract[Int] should be (1)
- (jobJson \ "numCompletedStages").extract[Int] should be (2)
- (jobJson \ "numFailedStages").extract[Int] should be (1)
+ (jobJson \\ "numTasks").extract[Int]should be (2)
+ (jobJson \\ "numCompletedTasks").extract[Int] should be (3)
+ (jobJson \\ "numFailedTasks").extract[Int] should be (1)
+ (jobJson \\ "numCompletedStages").extract[Int] should be (2)
+ (jobJson \\ "numFailedStages").extract[Int] should be (1)
val stageJson = getJson(sc.ui.get, "stages")
for {
@@ -656,7 +657,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
sc.ui.get.webUrl + "/api/v1/applications"))
val appListJsonAst = JsonMethods.parse(appListRawJson)
appListJsonAst.children.length should be (1)
- val attempts = (appListJsonAst \ "attempts").children
+ val attempts = (appListJsonAst.children.head \ "attempts").children
attempts.size should be (1)
(attempts(0) \ "completed").extract[Boolean] should be (false)
parseDate(attempts(0) \ "startTime") should be (sc.startTime)
http://git-wip-us.apache.org/repos/asf/spark/blob/628a1ca5/dev/deps/spark-deps-hadoop-2.6
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6
index a7fce2e..2a29876 100644
--- a/dev/deps/spark-deps-hadoop-2.6
+++ b/dev/deps/spark-deps-hadoop-2.6
@@ -122,9 +122,10 @@ jline-2.12.1.jar
joda-time-2.9.3.jar
jodd-core-3.5.2.jar
jpam-1.1.jar
-json4s-ast_2.11-3.2.11.jar
-json4s-core_2.11-3.2.11.jar
-json4s-jackson_2.11-3.2.11.jar
+json4s-ast_2.11-3.5.3.jar
+json4s-core_2.11-3.5.3.jar
+json4s-jackson_2.11-3.5.3.jar
+json4s-scalap_2.11-3.5.3.jar
jsr305-1.3.9.jar
jta-1.1.jar
jtransforms-2.4.0.jar
@@ -167,7 +168,6 @@ scala-library-2.11.8.jar
scala-parser-combinators_2.11-1.0.4.jar
scala-reflect-2.11.8.jar
scala-xml_2.11-1.0.5.jar
-scalap-2.11.8.jar
shapeless_2.11-2.3.2.jar
slf4j-api-1.7.16.jar
slf4j-log4j12-1.7.16.jar
http://git-wip-us.apache.org/repos/asf/spark/blob/628a1ca5/dev/deps/spark-deps-hadoop-2.7
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7
index 94b2e98..abee326 100644
--- a/dev/deps/spark-deps-hadoop-2.7
+++ b/dev/deps/spark-deps-hadoop-2.7
@@ -122,9 +122,10 @@ jline-2.12.1.jar
joda-time-2.9.3.jar
jodd-core-3.5.2.jar
jpam-1.1.jar
-json4s-ast_2.11-3.2.11.jar
-json4s-core_2.11-3.2.11.jar
-json4s-jackson_2.11-3.2.11.jar
+json4s-ast_2.11-3.5.3.jar
+json4s-core_2.11-3.5.3.jar
+json4s-jackson_2.11-3.5.3.jar
+json4s-scalap_2.11-3.5.3.jar
jsp-api-2.1.jar
jsr305-1.3.9.jar
jta-1.1.jar
@@ -168,7 +169,6 @@ scala-library-2.11.8.jar
scala-parser-combinators_2.11-1.0.4.jar
scala-reflect-2.11.8.jar
scala-xml_2.11-1.0.5.jar
-scalap-2.11.8.jar
shapeless_2.11-2.3.2.jar
slf4j-api-1.7.16.jar
slf4j-log4j12-1.7.16.jar
http://git-wip-us.apache.org/repos/asf/spark/blob/628a1ca5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d14594a..666d5d7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -705,7 +705,13 @@
<dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-jackson_${scala.binary.version}</artifactId>
- <version>3.2.11</version>
+ <version>3.5.3</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
@@ -732,11 +738,6 @@
<artifactId>scala-parser-combinators_${scala.binary.version}</artifactId>
<version>1.0.4</version>
</dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scalap</artifactId>
- <version>${scala.version}</version>
- </dependency>
<!-- SPARK-16770 affecting Scala 2.11.x -->
<dependency>
<groupId>jline</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org