You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2019/03/14 20:09:13 UTC
[spark] branch master updated: [SPARK-27145][MINOR] Close store in
the SQLAppStatusListenerSuite after test
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8b52240 [SPARK-27145][MINOR] Close store in the SQLAppStatusListenerSuite after test
8b52240 is described below
commit 8b5224097bf73b0b655ef2e8d08b245a5851d7f9
Author: Shahid <sh...@gmail.com>
AuthorDate: Thu Mar 14 13:08:41 2019 -0700
[SPARK-27145][MINOR] Close store in the SQLAppStatusListenerSuite after test
## What changes were proposed in this pull request?
We create many stores in the SQLAppStatusListenerSuite, but we need to the close store after test.
## How was this patch tested?
Existing tests
Closes #24079 from shahidki31/SPARK-27145.
Authored-by: Shahid <sh...@gmail.com>
Signed-off-by: Marcelo Vanzin <va...@cloudera.com>
---
.../sql/execution/ui/AllExecutionsPageSuite.scala | 18 +++++++++++----
.../execution/ui/SQLAppStatusListenerSuite.scala | 26 ++++++++++++++++------
2 files changed, 33 insertions(+), 11 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
index 5a3a923..95a6af3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
@@ -24,6 +24,7 @@ import javax.servlet.http.HttpServletRequest
import scala.xml.Node
import org.mockito.Mockito.{mock, when, RETURNS_SMART_NULLS}
+import org.scalatest.BeforeAndAfter
import org.apache.spark.scheduler.{JobFailed, SparkListenerJobEnd, SparkListenerJobStart}
import org.apache.spark.sql.DataFrame
@@ -32,10 +33,19 @@ import org.apache.spark.sql.test.SharedSQLContext
import org.apache.spark.status.ElementTrackingStore
import org.apache.spark.util.kvstore.InMemoryStore
-class AllExecutionsPageSuite extends SharedSQLContext {
+class AllExecutionsPageSuite extends SharedSQLContext with BeforeAndAfter {
import testImplicits._
+ var kvstore: ElementTrackingStore = _
+
+ after {
+ if (kvstore != null) {
+ kvstore.close()
+ kvstore = null
+ }
+ }
+
test("SPARK-27019: correctly display SQL page when event reordering happens") {
val statusStore = createStatusStore
val tab = mock(classOf[SQLTab], RETURNS_SMART_NULLS)
@@ -70,9 +80,9 @@ class AllExecutionsPageSuite extends SharedSQLContext {
private def createStatusStore: SQLAppStatusStore = {
val conf = sparkContext.conf
- val store = new ElementTrackingStore(new InMemoryStore, conf)
- val listener = new SQLAppStatusListener(conf, store, live = true)
- new SQLAppStatusStore(store, Some(listener))
+ kvstore = new ElementTrackingStore(new InMemoryStore, conf)
+ val listener = new SQLAppStatusListener(conf, kvstore, live = true)
+ new SQLAppStatusStore(kvstore, Some(listener))
}
private def createTestDataFrame: DataFrame = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
index f19bf5f..d845117 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
@@ -22,6 +22,7 @@ import java.util.Properties
import scala.collection.mutable.ListBuffer
import org.json4s.jackson.JsonMethods._
+import org.scalatest.BeforeAndAfter
import org.apache.spark._
import org.apache.spark.LocalSparkContext._
@@ -43,13 +44,24 @@ import org.apache.spark.util.{AccumulatorMetadata, JsonProtocol, LongAccumulator
import org.apache.spark.util.kvstore.InMemoryStore
-class SQLAppStatusListenerSuite extends SparkFunSuite with SharedSQLContext with JsonTestUtils {
+class SQLAppStatusListenerSuite extends SparkFunSuite with SharedSQLContext with JsonTestUtils
+ with BeforeAndAfter {
+
import testImplicits._
override protected def sparkConf = {
super.sparkConf.set(LIVE_ENTITY_UPDATE_PERIOD, 0L).set(ASYNC_TRACKING_ENABLED, false)
}
+ private var kvstore: ElementTrackingStore = _
+
+ after {
+ if (kvstore != null) {
+ kvstore.close()
+ kvstore = null
+ }
+ }
+
private def createTestDataFrame: DataFrame = {
Seq(
(1, 1),
@@ -126,9 +138,9 @@ class SQLAppStatusListenerSuite extends SparkFunSuite with SharedSQLContext with
private def createStatusStore(): SQLAppStatusStore = {
val conf = sparkContext.conf
- val store = new ElementTrackingStore(new InMemoryStore, conf)
- val listener = new SQLAppStatusListener(conf, store, live = true)
- new SQLAppStatusStore(store, Some(listener))
+ kvstore = new ElementTrackingStore(new InMemoryStore, conf)
+ val listener = new SQLAppStatusListener(conf, kvstore, live = true)
+ new SQLAppStatusStore(kvstore, Some(listener))
}
test("basic") {
@@ -548,9 +560,9 @@ class SQLAppStatusListenerSuite extends SparkFunSuite with SharedSQLContext with
test("eviction should respect execution completion time") {
val conf = sparkContext.conf.clone().set(UI_RETAINED_EXECUTIONS.key, "2")
- val store = new ElementTrackingStore(new InMemoryStore, conf)
- val listener = new SQLAppStatusListener(conf, store, live = true)
- val statusStore = new SQLAppStatusStore(store, Some(listener))
+ kvstore = new ElementTrackingStore(new InMemoryStore, conf)
+ val listener = new SQLAppStatusListener(conf, kvstore, live = true)
+ val statusStore = new SQLAppStatusStore(kvstore, Some(listener))
var time = 0
val df = createTestDataFrame
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org