You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/09/15 07:47:47 UTC
spark git commit: [SPARK-21987][SQL] fix a compatibility issue of sql
event logs
Repository: spark
Updated Branches:
refs/heads/master 4decedfdb -> 3c6198c86
[SPARK-21987][SQL] fix a compatibility issue of sql event logs
## What changes were proposed in this pull request?
In https://github.com/apache/spark/pull/18600 we removed the `metadata` field from `SparkPlanInfo`. This causes a problem when we replay event logs that are generated by older Spark versions.
## How was this patch tested?
a regression test.
Author: Wenchen Fan <we...@databricks.com>
Closes #19237 from cloud-fan/event.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3c6198c8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3c6198c8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3c6198c8
Branch: refs/heads/master
Commit: 3c6198c86ef36e7e5814d74ede00672d0eeb7f32
Parents: 4decedf
Author: Wenchen Fan <we...@databricks.com>
Authored: Fri Sep 15 00:47:44 2017 -0700
Committer: gatorsmile <ga...@gmail.com>
Committed: Fri Sep 15 00:47:44 2017 -0700
----------------------------------------------------------------------
.../spark/sql/execution/SparkPlanInfo.scala | 3 ++
.../sql/execution/SQLJsonProtocolSuite.scala | 52 ++++++++++++++++++++
2 files changed, 55 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3c6198c8/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
index 2118b91..2a23158 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.execution
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
import org.apache.spark.sql.execution.metric.SQLMetricInfo
@@ -26,6 +28,7 @@ import org.apache.spark.sql.execution.metric.SQLMetricInfo
* Stores information about a SQL SparkPlan.
*/
@DeveloperApi
+@JsonIgnoreProperties(Array("metadata")) // The metadata field was removed in Spark 2.3.
class SparkPlanInfo(
val nodeName: String,
val simpleString: String,
http://git-wip-us.apache.org/repos/asf/spark/blob/3c6198c8/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLJsonProtocolSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLJsonProtocolSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLJsonProtocolSuite.scala
new file mode 100644
index 0000000..c2e62b9
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLJsonProtocolSuite.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.json4s.jackson.JsonMethods.parse
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart
+import org.apache.spark.util.JsonProtocol
+
+class SQLJsonProtocolSuite extends SparkFunSuite {
+
+ test("SparkPlanGraph backward compatibility: metadata") {
+ val SQLExecutionStartJsonString =
+ """
+ |{
+ | "Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart",
+ | "executionId":0,
+ | "description":"test desc",
+ | "details":"test detail",
+ | "physicalPlanDescription":"test plan",
+ | "sparkPlanInfo": {
+ | "nodeName":"TestNode",
+ | "simpleString":"test string",
+ | "children":[],
+ | "metadata":{},
+ | "metrics":[]
+ | },
+ | "time":0
+ |}
+ """.stripMargin
+ val reconstructedEvent = JsonProtocol.sparkEventFromJson(parse(SQLExecutionStartJsonString))
+ val expectedEvent = SparkListenerSQLExecutionStart(0, "test desc", "test detail", "test plan",
+ new SparkPlanInfo("TestNode", "test string", Nil, Nil), 0)
+ assert(reconstructedEvent == expectedEvent)
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org