You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2016/06/01 17:14:45 UTC
[1/2] spark git commit: [SPARK-15686][SQL] Move user-facing streaming
classes into sql.streaming
Repository: spark
Updated Branches:
refs/heads/master d5012c274 -> a71d1364a
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/util/ContinuousQueryListener.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/util/ContinuousQueryListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/util/ContinuousQueryListener.scala
deleted file mode 100644
index ba1facf..0000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/util/ContinuousQueryListener.scala
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.util
-
-import org.apache.spark.annotation.Experimental
-import org.apache.spark.sql.ContinuousQuery
-import org.apache.spark.sql.util.ContinuousQueryListener._
-
-/**
- * :: Experimental ::
- * Interface for listening to events related to [[ContinuousQuery ContinuousQueries]].
- * @note The methods are not thread-safe as they may be called from different threads.
- */
-@Experimental
-abstract class ContinuousQueryListener {
-
- /**
- * Called when a query is started.
- * @note This is called synchronously with
- * [[org.apache.spark.sql.DataFrameWriter `DataFrameWriter.startStream()`]],
- * that is, `onQueryStart` will be called on all listeners before
- * `DataFrameWriter.startStream()` returns the corresponding [[ContinuousQuery]]. Please
- * don't block this method as it will block your query.
- */
- def onQueryStarted(queryStarted: QueryStarted): Unit
-
- /**
- * Called when there is some status update (ingestion rate updated, etc.)
- *
- * @note This method is asynchronous. The status in [[ContinuousQuery]] will always be
- * latest no matter when this method is called. Therefore, the status of [[ContinuousQuery]]
- * may be changed before/when you process the event. E.g., you may find [[ContinuousQuery]]
- * is terminated when you are processing [[QueryProgress]].
- */
- def onQueryProgress(queryProgress: QueryProgress): Unit
-
- /** Called when a query is stopped, with or without error */
- def onQueryTerminated(queryTerminated: QueryTerminated): Unit
-}
-
-
-/**
- * :: Experimental ::
- * Companion object of [[ContinuousQueryListener]] that defines the listener events.
- */
-@Experimental
-object ContinuousQueryListener {
-
- /** Base type of [[ContinuousQueryListener]] events */
- trait Event
-
- /** Event representing the start of a query */
- class QueryStarted private[sql](val query: ContinuousQuery) extends Event
-
- /** Event representing any progress updates in a query */
- class QueryProgress private[sql](val query: ContinuousQuery) extends Event
-
- /** Event representing that termination of a query */
- class QueryTerminated private[sql](val query: ContinuousQuery) extends Event
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/ProcessingTimeSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ProcessingTimeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ProcessingTimeSuite.scala
index 0d18a64..52c2007 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ProcessingTimeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ProcessingTimeSuite.scala
@@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit
import scala.concurrent.duration._
import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.streaming.ProcessingTime
class ProcessingTimeSuite extends SparkFunSuite {
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/StreamTest.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StreamTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/StreamTest.scala
deleted file mode 100644
index b033725..0000000
--- a/sql/core/src/test/scala/org/apache/spark/sql/StreamTest.scala
+++ /dev/null
@@ -1,565 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import java.lang.Thread.UncaughtExceptionHandler
-
-import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
-import scala.language.experimental.macros
-import scala.reflect.ClassTag
-import scala.util.Random
-import scala.util.control.NonFatal
-
-import org.scalatest.Assertions
-import org.scalatest.concurrent.{Eventually, Timeouts}
-import org.scalatest.concurrent.PatienceConfiguration.Timeout
-import org.scalatest.exceptions.TestFailedDueToTimeoutException
-import org.scalatest.time.Span
-import org.scalatest.time.SpanSugar._
-
-import org.apache.spark.sql.catalyst.encoders.{encoderFor, ExpressionEncoder, RowEncoder}
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.util._
-import org.apache.spark.sql.execution.streaming._
-import org.apache.spark.util.{Clock, ManualClock, SystemClock, Utils}
-
-/**
- * A framework for implementing tests for streaming queries and sources.
- *
- * A test consists of a set of steps (expressed as a `StreamAction`) that are executed in order,
- * blocking as necessary to let the stream catch up. For example, the following adds some data to
- * a stream, blocking until it can verify that the correct values are eventually produced.
- *
- * {{{
- * val inputData = MemoryStream[Int]
- val mapped = inputData.toDS().map(_ + 1)
-
- testStream(mapped)(
- AddData(inputData, 1, 2, 3),
- CheckAnswer(2, 3, 4))
- * }}}
- *
- * Note that while we do sleep to allow the other thread to progress without spinning,
- * `StreamAction` checks should not depend on the amount of time spent sleeping. Instead they
- * should check the actual progress of the stream before verifying the required test condition.
- *
- * Currently it is assumed that all streaming queries will eventually complete in 10 seconds to
- * avoid hanging forever in the case of failures. However, individual suites can change this
- * by overriding `streamingTimeout`.
- */
-trait StreamTest extends QueryTest with Timeouts {
-
- /** How long to wait for an active stream to catch up when checking a result. */
- val streamingTimeout = 10.seconds
-
- /** A trait for actions that can be performed while testing a streaming DataFrame. */
- trait StreamAction
-
- /** A trait to mark actions that require the stream to be actively running. */
- trait StreamMustBeRunning
-
- /**
- * Adds the given data to the stream. Subsequent check answers will block until this data has
- * been processed.
- */
- object AddData {
- def apply[A](source: MemoryStream[A], data: A*): AddDataMemory[A] =
- AddDataMemory(source, data)
- }
-
- /** A trait that can be extended when testing a source. */
- trait AddData extends StreamAction {
- /**
- * Called to adding the data to a source. It should find the source to add data to from
- * the active query, and then return the source object the data was added, as well as the
- * offset of added data.
- */
- def addData(query: Option[StreamExecution]): (Source, Offset)
- }
-
- case class AddDataMemory[A](source: MemoryStream[A], data: Seq[A]) extends AddData {
- override def toString: String = s"AddData to $source: ${data.mkString(",")}"
-
- override def addData(query: Option[StreamExecution]): (Source, Offset) = {
- (source, source.addData(data))
- }
- }
-
- /**
- * Checks to make sure that the current data stored in the sink matches the `expectedAnswer`.
- * This operation automatically blocks until all added data has been processed.
- */
- object CheckAnswer {
- def apply[A : Encoder](data: A*): CheckAnswerRows = {
- val encoder = encoderFor[A]
- val toExternalRow = RowEncoder(encoder.schema)
- CheckAnswerRows(data.map(d => toExternalRow.fromRow(encoder.toRow(d))), false)
- }
-
- def apply(rows: Row*): CheckAnswerRows = CheckAnswerRows(rows, false)
- }
-
- /**
- * Checks to make sure that the current data stored in the sink matches the `expectedAnswer`.
- * This operation automatically blocks until all added data has been processed.
- */
- object CheckLastBatch {
- def apply[A : Encoder](data: A*): CheckAnswerRows = {
- val encoder = encoderFor[A]
- val toExternalRow = RowEncoder(encoder.schema)
- CheckAnswerRows(data.map(d => toExternalRow.fromRow(encoder.toRow(d))), true)
- }
-
- def apply(rows: Row*): CheckAnswerRows = CheckAnswerRows(rows, true)
- }
-
- case class CheckAnswerRows(expectedAnswer: Seq[Row], lastOnly: Boolean)
- extends StreamAction with StreamMustBeRunning {
- override def toString: String = s"$operatorName: ${expectedAnswer.mkString(",")}"
- private def operatorName = if (lastOnly) "CheckLastBatch" else "CheckAnswer"
- }
-
- /** Stops the stream. It must currently be running. */
- case object StopStream extends StreamAction with StreamMustBeRunning
-
- /** Starts the stream, resuming if data has already been processed. It must not be running. */
- case class StartStream(
- trigger: Trigger = ProcessingTime(0),
- triggerClock: Clock = new SystemClock)
- extends StreamAction
-
- /** Advance the trigger clock's time manually. */
- case class AdvanceManualClock(timeToAdd: Long) extends StreamAction
-
- /** Signals that a failure is expected and should not kill the test. */
- case class ExpectFailure[T <: Throwable : ClassTag]() extends StreamAction {
- val causeClass: Class[T] = implicitly[ClassTag[T]].runtimeClass.asInstanceOf[Class[T]]
- override def toString(): String = s"ExpectFailure[${causeClass.getCanonicalName}]"
- }
-
- /** Assert that a body is true */
- class Assert(condition: => Boolean, val message: String = "") extends StreamAction {
- def run(): Unit = { Assertions.assert(condition) }
- override def toString: String = s"Assert(<condition>, $message)"
- }
-
- object Assert {
- def apply(condition: => Boolean, message: String = ""): Assert = new Assert(condition, message)
- def apply(message: String)(body: => Unit): Assert = new Assert( { body; true }, message)
- def apply(body: => Unit): Assert = new Assert( { body; true }, "")
- }
-
- /** Assert that a condition on the active query is true */
- class AssertOnQuery(val condition: StreamExecution => Boolean, val message: String)
- extends StreamAction {
- override def toString: String = s"AssertOnQuery(<condition>, $message)"
- }
-
- object AssertOnQuery {
- def apply(condition: StreamExecution => Boolean, message: String = ""): AssertOnQuery = {
- new AssertOnQuery(condition, message)
- }
-
- def apply(message: String)(condition: StreamExecution => Boolean): AssertOnQuery = {
- new AssertOnQuery(condition, message)
- }
- }
-
- /**
- * Executes the specified actions on the given streaming DataFrame and provides helpful
- * error messages in the case of failures or incorrect answers.
- *
- * Note that if the stream is not explicitly started before an action that requires it to be
- * running then it will be automatically started before performing any other actions.
- */
- def testStream(
- _stream: Dataset[_],
- outputMode: OutputMode = OutputMode.Append)(actions: StreamAction*): Unit = {
-
- val stream = _stream.toDF()
- var pos = 0
- var currentPlan: LogicalPlan = stream.logicalPlan
- var currentStream: StreamExecution = null
- var lastStream: StreamExecution = null
- val awaiting = new mutable.HashMap[Int, Offset]() // source index -> offset to wait for
- val sink = new MemorySink(stream.schema, outputMode)
-
- @volatile
- var streamDeathCause: Throwable = null
-
- // If the test doesn't manually start the stream, we do it automatically at the beginning.
- val startedManually =
- actions.takeWhile(!_.isInstanceOf[StreamMustBeRunning]).exists(_.isInstanceOf[StartStream])
- val startedTest = if (startedManually) actions else StartStream() +: actions
-
- def testActions = actions.zipWithIndex.map {
- case (a, i) =>
- if ((pos == i && startedManually) || (pos == (i + 1) && !startedManually)) {
- "=> " + a.toString
- } else {
- " " + a.toString
- }
- }.mkString("\n")
-
- def currentOffsets =
- if (currentStream != null) currentStream.committedOffsets.toString else "not started"
-
- def threadState =
- if (currentStream != null && currentStream.microBatchThread.isAlive) "alive" else "dead"
-
- def testState =
- s"""
- |== Progress ==
- |$testActions
- |
- |== Stream ==
- |Output Mode: $outputMode
- |Stream state: $currentOffsets
- |Thread state: $threadState
- |${if (streamDeathCause != null) stackTraceToString(streamDeathCause) else ""}
- |
- |== Sink ==
- |${sink.toDebugString}
- |
- |
- |== Plan ==
- |${if (currentStream != null) currentStream.lastExecution else ""}
- """.stripMargin
-
- def verify(condition: => Boolean, message: String): Unit = {
- if (!condition) {
- failTest(message)
- }
- }
-
- def eventually[T](message: String)(func: => T): T = {
- try {
- Eventually.eventually(Timeout(streamingTimeout)) {
- func
- }
- } catch {
- case NonFatal(e) =>
- failTest(message, e)
- }
- }
-
- def failTest(message: String, cause: Throwable = null) = {
-
- // Recursively pretty print a exception with truncated stacktrace and internal cause
- def exceptionToString(e: Throwable, prefix: String = ""): String = {
- val base = s"$prefix${e.getMessage}" +
- e.getStackTrace.take(10).mkString(s"\n$prefix", s"\n$prefix\t", "\n")
- if (e.getCause != null) {
- base + s"\n$prefix\tCaused by: " + exceptionToString(e.getCause, s"$prefix\t")
- } else {
- base
- }
- }
- val c = Option(cause).map(exceptionToString(_))
- val m = if (message != null && message.size > 0) Some(message) else None
- fail(
- s"""
- |${(m ++ c).mkString(": ")}
- |$testState
- """.stripMargin)
- }
-
- val testThread = Thread.currentThread()
- val metadataRoot = Utils.createTempDir(namePrefix = "streaming.metadata").getCanonicalPath
-
- try {
- startedTest.foreach { action =>
- action match {
- case StartStream(trigger, triggerClock) =>
- verify(currentStream == null, "stream already running")
- lastStream = currentStream
- currentStream =
- spark
- .streams
- .startQuery(
- StreamExecution.nextName,
- metadataRoot,
- stream,
- sink,
- outputMode,
- trigger,
- triggerClock)
- .asInstanceOf[StreamExecution]
- currentStream.microBatchThread.setUncaughtExceptionHandler(
- new UncaughtExceptionHandler {
- override def uncaughtException(t: Thread, e: Throwable): Unit = {
- streamDeathCause = e
- testThread.interrupt()
- }
- })
-
- case AdvanceManualClock(timeToAdd) =>
- verify(currentStream != null,
- "can not advance manual clock when a stream is not running")
- verify(currentStream.triggerClock.isInstanceOf[ManualClock],
- s"can not advance clock of type ${currentStream.triggerClock.getClass}")
- currentStream.triggerClock.asInstanceOf[ManualClock].advance(timeToAdd)
-
- case StopStream =>
- verify(currentStream != null, "can not stop a stream that is not running")
- try failAfter(streamingTimeout) {
- currentStream.stop()
- verify(!currentStream.microBatchThread.isAlive,
- s"microbatch thread not stopped")
- verify(!currentStream.isActive,
- "query.isActive() is false even after stopping")
- verify(currentStream.exception.isEmpty,
- s"query.exception() is not empty after clean stop: " +
- currentStream.exception.map(_.toString()).getOrElse(""))
- } catch {
- case _: InterruptedException =>
- case _: org.scalatest.exceptions.TestFailedDueToTimeoutException =>
- failTest("Timed out while stopping and waiting for microbatchthread to terminate.")
- case t: Throwable =>
- failTest("Error while stopping stream", t)
- } finally {
- lastStream = currentStream
- currentStream = null
- }
-
- case ef: ExpectFailure[_] =>
- verify(currentStream != null, "can not expect failure when stream is not running")
- try failAfter(streamingTimeout) {
- val thrownException = intercept[ContinuousQueryException] {
- currentStream.awaitTermination()
- }
- eventually("microbatch thread not stopped after termination with failure") {
- assert(!currentStream.microBatchThread.isAlive)
- }
- verify(thrownException.query.eq(currentStream),
- s"incorrect query reference in exception")
- verify(currentStream.exception === Some(thrownException),
- s"incorrect exception returned by query.exception()")
-
- val exception = currentStream.exception.get
- verify(exception.cause.getClass === ef.causeClass,
- "incorrect cause in exception returned by query.exception()\n" +
- s"\tExpected: ${ef.causeClass}\n\tReturned: ${exception.cause.getClass}")
- } catch {
- case _: InterruptedException =>
- case _: org.scalatest.exceptions.TestFailedDueToTimeoutException =>
- failTest("Timed out while waiting for failure")
- case t: Throwable =>
- failTest("Error while checking stream failure", t)
- } finally {
- lastStream = currentStream
- currentStream = null
- streamDeathCause = null
- }
-
- case a: AssertOnQuery =>
- verify(currentStream != null || lastStream != null,
- "cannot assert when not stream has been started")
- val streamToAssert = Option(currentStream).getOrElse(lastStream)
- verify(a.condition(streamToAssert), s"Assert on query failed: ${a.message}")
-
- case a: Assert =>
- val streamToAssert = Option(currentStream).getOrElse(lastStream)
- verify({ a.run(); true }, s"Assert failed: ${a.message}")
-
- case a: AddData =>
- try {
- // Add data and get the source where it was added, and the expected offset of the
- // added data.
- val queryToUse = Option(currentStream).orElse(Option(lastStream))
- val (source, offset) = a.addData(queryToUse)
-
- def findSourceIndex(plan: LogicalPlan): Option[Int] = {
- plan
- .collect { case StreamingExecutionRelation(s, _) => s }
- .zipWithIndex
- .find(_._1 == source)
- .map(_._2)
- }
-
- // Try to find the index of the source to which data was added. Either get the index
- // from the current active query or the original input logical plan.
- val sourceIndex =
- queryToUse.flatMap { query =>
- findSourceIndex(query.logicalPlan)
- }.orElse {
- findSourceIndex(stream.logicalPlan)
- }.getOrElse {
- throw new IllegalArgumentException(
- "Could find index of the source to which data was added")
- }
-
- // Store the expected offset of added data to wait for it later
- awaiting.put(sourceIndex, offset)
- } catch {
- case NonFatal(e) =>
- failTest("Error adding data", e)
- }
-
- case CheckAnswerRows(expectedAnswer, lastOnly) =>
- verify(currentStream != null, "stream not running")
- // Get the map of source index to the current source objects
- val indexToSource = currentStream
- .logicalPlan
- .collect { case StreamingExecutionRelation(s, _) => s }
- .zipWithIndex
- .map(_.swap)
- .toMap
-
- // Block until all data added has been processed for all the source
- awaiting.foreach { case (sourceIndex, offset) =>
- failAfter(streamingTimeout) {
- currentStream.awaitOffset(indexToSource(sourceIndex), offset)
- }
- }
-
- val sparkAnswer = try if (lastOnly) sink.latestBatchData else sink.allData catch {
- case e: Exception =>
- failTest("Exception while getting data from sink", e)
- }
-
- QueryTest.sameRows(expectedAnswer, sparkAnswer).foreach {
- error => failTest(error)
- }
- }
- pos += 1
- }
- } catch {
- case _: InterruptedException if streamDeathCause != null =>
- failTest("Stream Thread Died")
- case _: org.scalatest.exceptions.TestFailedDueToTimeoutException =>
- failTest("Timed out waiting for stream")
- } finally {
- if (currentStream != null && currentStream.microBatchThread.isAlive) {
- currentStream.stop()
- }
- }
- }
-
- /**
- * Creates a stress test that randomly starts/stops/adds data/checks the result.
- *
- * @param ds a dataframe that executes + 1 on a stream of integers, returning the result.
- * @param addData and add data action that adds the given numbers to the stream, encoding them
- * as needed
- */
- def runStressTest(
- ds: Dataset[Int],
- addData: Seq[Int] => StreamAction,
- iterations: Int = 100): Unit = {
- implicit val intEncoder = ExpressionEncoder[Int]()
- var dataPos = 0
- var running = true
- val actions = new ArrayBuffer[StreamAction]()
-
- def addCheck() = { actions += CheckAnswer(1 to dataPos: _*) }
-
- def addRandomData() = {
- val numItems = Random.nextInt(10)
- val data = dataPos until (dataPos + numItems)
- dataPos += numItems
- actions += addData(data)
- }
-
- (1 to iterations).foreach { i =>
- val rand = Random.nextDouble()
- if(!running) {
- rand match {
- case r if r < 0.7 => // AddData
- addRandomData()
-
- case _ => // StartStream
- actions += StartStream()
- running = true
- }
- } else {
- rand match {
- case r if r < 0.1 =>
- addCheck()
-
- case r if r < 0.7 => // AddData
- addRandomData()
-
- case _ => // StopStream
- addCheck()
- actions += StopStream
- running = false
- }
- }
- }
- if(!running) { actions += StartStream() }
- addCheck()
- testStream(ds)(actions: _*)
- }
-
-
- object AwaitTerminationTester {
-
- trait ExpectedBehavior
-
- /** Expect awaitTermination to not be blocked */
- case object ExpectNotBlocked extends ExpectedBehavior
-
- /** Expect awaitTermination to get blocked */
- case object ExpectBlocked extends ExpectedBehavior
-
- /** Expect awaitTermination to throw an exception */
- case class ExpectException[E <: Exception]()(implicit val t: ClassTag[E])
- extends ExpectedBehavior
-
- private val DEFAULT_TEST_TIMEOUT = 1 second
-
- def test(
- expectedBehavior: ExpectedBehavior,
- awaitTermFunc: () => Unit,
- testTimeout: Span = DEFAULT_TEST_TIMEOUT
- ): Unit = {
-
- expectedBehavior match {
- case ExpectNotBlocked =>
- withClue("Got blocked when expected non-blocking.") {
- failAfter(testTimeout) {
- awaitTermFunc()
- }
- }
-
- case ExpectBlocked =>
- withClue("Was not blocked when expected.") {
- intercept[TestFailedDueToTimeoutException] {
- failAfter(testTimeout) {
- awaitTermFunc()
- }
- }
- }
-
- case e: ExpectException[_] =>
- val thrownException =
- withClue(s"Did not throw ${e.t.runtimeClass.getSimpleName} when expected.") {
- intercept[ContinuousQueryException] {
- failAfter(testTimeout) {
- awaitTermFunc()
- }
- }
- }
- assert(thrownException.cause.getClass === e.t.runtimeClass,
- "exception of incorrect type was throw")
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/ProcessingTimeExecutorSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/ProcessingTimeExecutorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/ProcessingTimeExecutorSuite.scala
index 7f99d30..00d5e05 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/ProcessingTimeExecutorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/ProcessingTimeExecutorSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.streaming
import java.util.concurrent.{CountDownLatch, TimeUnit}
import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.ProcessingTime
+import org.apache.spark.sql.streaming.ProcessingTime
import org.apache.spark.util.{Clock, ManualClock, SystemClock}
class ProcessingTimeExecutorSuite extends SparkFunSuite {
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQueryListenerSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQueryListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQueryListenerSuite.scala
new file mode 100644
index 0000000..cdd97da
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQueryListenerSuite.scala
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming
+
+import java.util.concurrent.ConcurrentLinkedQueue
+
+import org.scalatest.BeforeAndAfter
+import org.scalatest.PrivateMethodTester._
+import org.scalatest.concurrent.AsyncAssertions.Waiter
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.concurrent.PatienceConfiguration.Timeout
+import org.scalatest.time.SpanSugar._
+
+import org.apache.spark.sql.execution.streaming._
+
+
+class ContinuousQueryListenerSuite extends StreamTest with BeforeAndAfter {
+
+ import testImplicits._
+ import ContinuousQueryListener._
+
+ after {
+ spark.streams.active.foreach(_.stop())
+ assert(spark.streams.active.isEmpty)
+ assert(addedListeners.isEmpty)
+ // Make sure we don't leak any events to the next test
+ spark.sparkContext.listenerBus.waitUntilEmpty(10000)
+ }
+
+ test("single listener") {
+ val listener = new QueryStatusCollector
+ val input = MemoryStream[Int]
+ withListenerAdded(listener) {
+ testStream(input.toDS)(
+ StartStream(),
+ Assert("Incorrect query status in onQueryStarted") {
+ val status = listener.startStatus
+ assert(status != null)
+ assert(status.active == true)
+ assert(status.sourceStatuses.size === 1)
+ assert(status.sourceStatuses(0).description.contains("Memory"))
+
+ // The source and sink offsets must be None as this must be called before the
+ // batches have started
+ assert(status.sourceStatuses(0).offset === None)
+ assert(status.sinkStatus.offset === CompositeOffset(None :: Nil))
+
+ // No progress events or termination events
+ assert(listener.progressStatuses.isEmpty)
+ assert(listener.terminationStatus === null)
+ },
+ AddDataMemory(input, Seq(1, 2, 3)),
+ CheckAnswer(1, 2, 3),
+ Assert("Incorrect query status in onQueryProgress") {
+ eventually(Timeout(streamingTimeout)) {
+
+ // There should be only on progress event as batch has been processed
+ assert(listener.progressStatuses.size === 1)
+ val status = listener.progressStatuses.peek()
+ assert(status != null)
+ assert(status.active == true)
+ assert(status.sourceStatuses(0).offset === Some(LongOffset(0)))
+ assert(status.sinkStatus.offset === CompositeOffset.fill(LongOffset(0)))
+
+ // No termination events
+ assert(listener.terminationStatus === null)
+ }
+ },
+ StopStream,
+ Assert("Incorrect query status in onQueryTerminated") {
+ eventually(Timeout(streamingTimeout)) {
+ val status = listener.terminationStatus
+ assert(status != null)
+
+ assert(status.active === false) // must be inactive by the time onQueryTerm is called
+ assert(status.sourceStatuses(0).offset === Some(LongOffset(0)))
+ assert(status.sinkStatus.offset === CompositeOffset.fill(LongOffset(0)))
+ }
+ listener.checkAsyncErrors()
+ }
+ )
+ }
+ }
+
+ test("adding and removing listener") {
+ def isListenerActive(listener: QueryStatusCollector): Boolean = {
+ listener.reset()
+ testStream(MemoryStream[Int].toDS)(
+ StartStream(),
+ StopStream
+ )
+ listener.startStatus != null
+ }
+
+ try {
+ val listener1 = new QueryStatusCollector
+ val listener2 = new QueryStatusCollector
+
+ spark.streams.addListener(listener1)
+ assert(isListenerActive(listener1) === true)
+ assert(isListenerActive(listener2) === false)
+ spark.streams.addListener(listener2)
+ assert(isListenerActive(listener1) === true)
+ assert(isListenerActive(listener2) === true)
+ spark.streams.removeListener(listener1)
+ assert(isListenerActive(listener1) === false)
+ assert(isListenerActive(listener2) === true)
+ } finally {
+ addedListeners.foreach(spark.streams.removeListener)
+ }
+ }
+
+ test("event ordering") {
+ val listener = new QueryStatusCollector
+ withListenerAdded(listener) {
+ for (i <- 1 to 100) {
+ listener.reset()
+ require(listener.startStatus === null)
+ testStream(MemoryStream[Int].toDS)(
+ StartStream(),
+ Assert(listener.startStatus !== null, "onQueryStarted not called before query returned"),
+ StopStream,
+ Assert { listener.checkAsyncErrors() }
+ )
+ }
+ }
+ }
+
+
+ private def withListenerAdded(listener: ContinuousQueryListener)(body: => Unit): Unit = {
+ try {
+ failAfter(1 minute) {
+ spark.streams.addListener(listener)
+ body
+ }
+ } finally {
+ spark.streams.removeListener(listener)
+ }
+ }
+
+ private def addedListeners(): Array[ContinuousQueryListener] = {
+ val listenerBusMethod =
+ PrivateMethod[ContinuousQueryListenerBus]('listenerBus)
+ val listenerBus = spark.streams invokePrivate listenerBusMethod()
+ listenerBus.listeners.toArray.map(_.asInstanceOf[ContinuousQueryListener])
+ }
+
+ class QueryStatusCollector extends ContinuousQueryListener {
+ // to catch errors in the async listener events
+ @volatile private var asyncTestWaiter = new Waiter
+
+ @volatile var startStatus: QueryStatus = null
+ @volatile var terminationStatus: QueryStatus = null
+ val progressStatuses = new ConcurrentLinkedQueue[QueryStatus]
+
+ def reset(): Unit = {
+ startStatus = null
+ terminationStatus = null
+ progressStatuses.clear()
+ asyncTestWaiter = new Waiter
+ }
+
+ def checkAsyncErrors(): Unit = {
+ asyncTestWaiter.await(timeout(streamingTimeout))
+ }
+
+
+ override def onQueryStarted(queryStarted: QueryStarted): Unit = {
+ asyncTestWaiter {
+ startStatus = QueryStatus(queryStarted.query)
+ }
+ }
+
+ override def onQueryProgress(queryProgress: QueryProgress): Unit = {
+ asyncTestWaiter {
+ assert(startStatus != null, "onQueryProgress called before onQueryStarted")
+ progressStatuses.add(QueryStatus(queryProgress.query))
+ }
+ }
+
+ override def onQueryTerminated(queryTerminated: QueryTerminated): Unit = {
+ asyncTestWaiter {
+ assert(startStatus != null, "onQueryTerminated called before onQueryStarted")
+ terminationStatus = QueryStatus(queryTerminated.query)
+ }
+ asyncTestWaiter.dismiss()
+ }
+ }
+
+ case class QueryStatus(
+ active: Boolean,
+ exception: Option[Exception],
+ sourceStatuses: Array[SourceStatus],
+ sinkStatus: SinkStatus)
+
+ object QueryStatus {
+ def apply(query: ContinuousQuery): QueryStatus = {
+ QueryStatus(query.isActive, query.exception, query.sourceStatuses, query.sinkStatus)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQueryManagerSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQueryManagerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQueryManagerSuite.scala
index b75c3ea..c1e4970 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQueryManagerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQueryManagerSuite.scala
@@ -28,12 +28,11 @@ import org.scalatest.time.Span
import org.scalatest.time.SpanSugar._
import org.apache.spark.SparkException
-import org.apache.spark.sql.{ContinuousQuery, Dataset, OutputMode, StreamTest}
+import org.apache.spark.sql.Dataset
import org.apache.spark.sql.execution.streaming._
-import org.apache.spark.sql.test.SharedSQLContext
import org.apache.spark.util.Utils
-class ContinuousQueryManagerSuite extends StreamTest with SharedSQLContext with BeforeAndAfter {
+class ContinuousQueryManagerSuite extends StreamTest with BeforeAndAfter {
import AwaitTerminationTester._
import testImplicits._
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQuerySuite.scala
index f469cde..e4ca86d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ContinuousQuerySuite.scala
@@ -18,11 +18,10 @@
package org.apache.spark.sql.streaming
import org.apache.spark.SparkException
-import org.apache.spark.sql.StreamTest
import org.apache.spark.sql.execution.streaming.{CompositeOffset, LongOffset, MemoryStream, StreamExecution}
-import org.apache.spark.sql.test.SharedSQLContext
-class ContinuousQuerySuite extends StreamTest with SharedSQLContext {
+
+class ContinuousQuerySuite extends StreamTest {
import AwaitTerminationTester._
import testImplicits._
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
index 3d8dcaf..1c73208 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
@@ -32,7 +32,7 @@ import org.apache.spark.sql.test.SharedSQLContext
import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
import org.apache.spark.util.Utils
-class FileStreamSinkSuite extends StreamTest with SharedSQLContext {
+class FileStreamSinkSuite extends StreamTest {
import testImplicits._
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
index 1d784f1..f681b88 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
@@ -137,7 +137,7 @@ class FileStreamSourceTest extends StreamTest with SharedSQLContext {
val valueSchema = new StructType().add("value", StringType)
}
-class FileStreamSourceSuite extends FileStreamSourceTest with SharedSQLContext {
+class FileStreamSourceSuite extends FileStreamSourceTest {
import testImplicits._
@@ -594,7 +594,7 @@ class FileStreamSourceSuite extends FileStreamSourceTest with SharedSQLContext {
}
}
-class FileStreamSourceStressTestSuite extends FileStreamSourceTest with SharedSQLContext {
+class FileStreamSourceStressTestSuite extends FileStreamSourceTest {
import testImplicits._
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStressSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStressSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStressSuite.scala
index 4efb7cf..1c0fb34 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStressSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStressSuite.scala
@@ -23,9 +23,7 @@ import java.util.UUID
import scala.util.Random
import scala.util.control.NonFatal
-import org.apache.spark.sql.{ContinuousQuery, ContinuousQueryException, StreamTest}
import org.apache.spark.sql.catalyst.util._
-import org.apache.spark.sql.test.SharedSQLContext
import org.apache.spark.util.Utils
/**
@@ -38,7 +36,7 @@ import org.apache.spark.util.Utils
*
* At the end, the resulting files are loaded and the answer is checked.
*/
-class FileStressSuite extends StreamTest with SharedSQLContext {
+class FileStressSuite extends StreamTest {
import testImplicits._
testQuietly("fault tolerance stress test - unpartitioned output") {
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/streaming/MemorySinkSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/MemorySinkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/MemorySinkSuite.scala
index e5bd0b4..df76499 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/MemorySinkSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/MemorySinkSuite.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.test.SharedSQLContext
import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
import org.apache.spark.util.Utils
-class MemorySinkSuite extends StreamTest with SharedSQLContext with BeforeAndAfter {
+class MemorySinkSuite extends StreamTest with BeforeAndAfter {
import testImplicits._
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/streaming/MemorySourceStressSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/MemorySourceStressSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/MemorySourceStressSuite.scala
index 81760d2..7f2972e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/MemorySourceStressSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/MemorySourceStressSuite.scala
@@ -17,11 +17,9 @@
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.StreamTest
import org.apache.spark.sql.execution.streaming._
-import org.apache.spark.sql.test.SharedSQLContext
-class MemorySourceStressSuite extends StreamTest with SharedSQLContext {
+class MemorySourceStressSuite extends StreamTest {
import testImplicits._
test("memory stress test") {
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
index c17cb1d..9414b1c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.test.SharedSQLContext
import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
import org.apache.spark.util.ManualClock
-class StreamSuite extends StreamTest with SharedSQLContext {
+class StreamSuite extends StreamTest {
import testImplicits._
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
new file mode 100644
index 0000000..dd8672a
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
@@ -0,0 +1,567 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming
+
+import java.lang.Thread.UncaughtExceptionHandler
+
+import scala.collection.mutable
+import scala.collection.mutable.ArrayBuffer
+import scala.language.experimental.macros
+import scala.reflect.ClassTag
+import scala.util.Random
+import scala.util.control.NonFatal
+
+import org.scalatest.Assertions
+import org.scalatest.concurrent.{Eventually, Timeouts}
+import org.scalatest.concurrent.PatienceConfiguration.Timeout
+import org.scalatest.exceptions.TestFailedDueToTimeoutException
+import org.scalatest.time.Span
+import org.scalatest.time.SpanSugar._
+
+import org.apache.spark.sql.{Dataset, Encoder, QueryTest, Row}
+import org.apache.spark.sql.catalyst.encoders.{encoderFor, ExpressionEncoder, RowEncoder}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.execution.streaming._
+import org.apache.spark.sql.test.SharedSQLContext
+import org.apache.spark.util.{Clock, ManualClock, SystemClock, Utils}
+
+/**
+ * A framework for implementing tests for streaming queries and sources.
+ *
+ * A test consists of a set of steps (expressed as a `StreamAction`) that are executed in order,
+ * blocking as necessary to let the stream catch up. For example, the following adds some data to
+ * a stream, blocking until it can verify that the correct values are eventually produced.
+ *
+ * {{{
+ * val inputData = MemoryStream[Int]
+ val mapped = inputData.toDS().map(_ + 1)
+
+ testStream(mapped)(
+ AddData(inputData, 1, 2, 3),
+ CheckAnswer(2, 3, 4))
+ * }}}
+ *
+ * Note that while we do sleep to allow the other thread to progress without spinning,
+ * `StreamAction` checks should not depend on the amount of time spent sleeping. Instead they
+ * should check the actual progress of the stream before verifying the required test condition.
+ *
+ * Currently it is assumed that all streaming queries will eventually complete in 10 seconds to
+ * avoid hanging forever in the case of failures. However, individual suites can change this
+ * by overriding `streamingTimeout`.
+ */
+trait StreamTest extends QueryTest with SharedSQLContext with Timeouts {
+
+ /** How long to wait for an active stream to catch up when checking a result. */
+ val streamingTimeout = 10.seconds
+
+ /** A trait for actions that can be performed while testing a streaming DataFrame. */
+ trait StreamAction
+
+ /** A trait to mark actions that require the stream to be actively running. */
+ trait StreamMustBeRunning
+
+ /**
+ * Adds the given data to the stream. Subsequent check answers will block until this data has
+ * been processed.
+ */
+ object AddData {
+ def apply[A](source: MemoryStream[A], data: A*): AddDataMemory[A] =
+ AddDataMemory(source, data)
+ }
+
+ /** A trait that can be extended when testing a source. */
+ trait AddData extends StreamAction {
+ /**
+ * Called to adding the data to a source. It should find the source to add data to from
+ * the active query, and then return the source object the data was added, as well as the
+ * offset of added data.
+ */
+ def addData(query: Option[StreamExecution]): (Source, Offset)
+ }
+
+ case class AddDataMemory[A](source: MemoryStream[A], data: Seq[A]) extends AddData {
+ override def toString: String = s"AddData to $source: ${data.mkString(",")}"
+
+ override def addData(query: Option[StreamExecution]): (Source, Offset) = {
+ (source, source.addData(data))
+ }
+ }
+
+ /**
+ * Checks to make sure that the current data stored in the sink matches the `expectedAnswer`.
+ * This operation automatically blocks until all added data has been processed.
+ */
+ object CheckAnswer {
+ def apply[A : Encoder](data: A*): CheckAnswerRows = {
+ val encoder = encoderFor[A]
+ val toExternalRow = RowEncoder(encoder.schema)
+ CheckAnswerRows(data.map(d => toExternalRow.fromRow(encoder.toRow(d))), false)
+ }
+
+ def apply(rows: Row*): CheckAnswerRows = CheckAnswerRows(rows, false)
+ }
+
+ /**
+ * Checks to make sure that the current data stored in the sink matches the `expectedAnswer`.
+ * This operation automatically blocks until all added data has been processed.
+ */
+ object CheckLastBatch {
+ def apply[A : Encoder](data: A*): CheckAnswerRows = {
+ val encoder = encoderFor[A]
+ val toExternalRow = RowEncoder(encoder.schema)
+ CheckAnswerRows(data.map(d => toExternalRow.fromRow(encoder.toRow(d))), true)
+ }
+
+ def apply(rows: Row*): CheckAnswerRows = CheckAnswerRows(rows, true)
+ }
+
+ case class CheckAnswerRows(expectedAnswer: Seq[Row], lastOnly: Boolean)
+ extends StreamAction with StreamMustBeRunning {
+ override def toString: String = s"$operatorName: ${expectedAnswer.mkString(",")}"
+ private def operatorName = if (lastOnly) "CheckLastBatch" else "CheckAnswer"
+ }
+
+ /** Stops the stream. It must currently be running. */
+ case object StopStream extends StreamAction with StreamMustBeRunning
+
+ /** Starts the stream, resuming if data has already been processed. It must not be running. */
+ case class StartStream(
+ trigger: Trigger = ProcessingTime(0),
+ triggerClock: Clock = new SystemClock)
+ extends StreamAction
+
+ /** Advance the trigger clock's time manually. */
+ case class AdvanceManualClock(timeToAdd: Long) extends StreamAction
+
+ /** Signals that a failure is expected and should not kill the test. */
+ case class ExpectFailure[T <: Throwable : ClassTag]() extends StreamAction {
+ val causeClass: Class[T] = implicitly[ClassTag[T]].runtimeClass.asInstanceOf[Class[T]]
+ override def toString(): String = s"ExpectFailure[${causeClass.getCanonicalName}]"
+ }
+
+ /** Assert that a body is true */
+ class Assert(condition: => Boolean, val message: String = "") extends StreamAction {
+ def run(): Unit = { Assertions.assert(condition) }
+ override def toString: String = s"Assert(<condition>, $message)"
+ }
+
+ object Assert {
+ def apply(condition: => Boolean, message: String = ""): Assert = new Assert(condition, message)
+ def apply(message: String)(body: => Unit): Assert = new Assert( { body; true }, message)
+ def apply(body: => Unit): Assert = new Assert( { body; true }, "")
+ }
+
+ /** Assert that a condition on the active query is true */
+ class AssertOnQuery(val condition: StreamExecution => Boolean, val message: String)
+ extends StreamAction {
+ override def toString: String = s"AssertOnQuery(<condition>, $message)"
+ }
+
+ object AssertOnQuery {
+ def apply(condition: StreamExecution => Boolean, message: String = ""): AssertOnQuery = {
+ new AssertOnQuery(condition, message)
+ }
+
+ def apply(message: String)(condition: StreamExecution => Boolean): AssertOnQuery = {
+ new AssertOnQuery(condition, message)
+ }
+ }
+
+ /**
+ * Executes the specified actions on the given streaming DataFrame and provides helpful
+ * error messages in the case of failures or incorrect answers.
+ *
+ * Note that if the stream is not explicitly started before an action that requires it to be
+ * running then it will be automatically started before performing any other actions.
+ */
+ def testStream(
+ _stream: Dataset[_],
+ outputMode: OutputMode = OutputMode.Append)(actions: StreamAction*): Unit = {
+
+ val stream = _stream.toDF()
+ var pos = 0
+ var currentPlan: LogicalPlan = stream.logicalPlan
+ var currentStream: StreamExecution = null
+ var lastStream: StreamExecution = null
+ val awaiting = new mutable.HashMap[Int, Offset]() // source index -> offset to wait for
+ val sink = new MemorySink(stream.schema, outputMode)
+
+ @volatile
+ var streamDeathCause: Throwable = null
+
+ // If the test doesn't manually start the stream, we do it automatically at the beginning.
+ val startedManually =
+ actions.takeWhile(!_.isInstanceOf[StreamMustBeRunning]).exists(_.isInstanceOf[StartStream])
+ val startedTest = if (startedManually) actions else StartStream() +: actions
+
+ def testActions = actions.zipWithIndex.map {
+ case (a, i) =>
+ if ((pos == i && startedManually) || (pos == (i + 1) && !startedManually)) {
+ "=> " + a.toString
+ } else {
+ " " + a.toString
+ }
+ }.mkString("\n")
+
+ def currentOffsets =
+ if (currentStream != null) currentStream.committedOffsets.toString else "not started"
+
+ def threadState =
+ if (currentStream != null && currentStream.microBatchThread.isAlive) "alive" else "dead"
+
+ def testState =
+ s"""
+ |== Progress ==
+ |$testActions
+ |
+ |== Stream ==
+ |Output Mode: $outputMode
+ |Stream state: $currentOffsets
+ |Thread state: $threadState
+ |${if (streamDeathCause != null) stackTraceToString(streamDeathCause) else ""}
+ |
+ |== Sink ==
+ |${sink.toDebugString}
+ |
+ |
+ |== Plan ==
+ |${if (currentStream != null) currentStream.lastExecution else ""}
+ """.stripMargin
+
+ def verify(condition: => Boolean, message: String): Unit = {
+ if (!condition) {
+ failTest(message)
+ }
+ }
+
+ def eventually[T](message: String)(func: => T): T = {
+ try {
+ Eventually.eventually(Timeout(streamingTimeout)) {
+ func
+ }
+ } catch {
+ case NonFatal(e) =>
+ failTest(message, e)
+ }
+ }
+
+ def failTest(message: String, cause: Throwable = null) = {
+
+ // Recursively pretty print a exception with truncated stacktrace and internal cause
+ def exceptionToString(e: Throwable, prefix: String = ""): String = {
+ val base = s"$prefix${e.getMessage}" +
+ e.getStackTrace.take(10).mkString(s"\n$prefix", s"\n$prefix\t", "\n")
+ if (e.getCause != null) {
+ base + s"\n$prefix\tCaused by: " + exceptionToString(e.getCause, s"$prefix\t")
+ } else {
+ base
+ }
+ }
+ val c = Option(cause).map(exceptionToString(_))
+ val m = if (message != null && message.size > 0) Some(message) else None
+ fail(
+ s"""
+ |${(m ++ c).mkString(": ")}
+ |$testState
+ """.stripMargin)
+ }
+
+ val testThread = Thread.currentThread()
+ val metadataRoot = Utils.createTempDir(namePrefix = "streaming.metadata").getCanonicalPath
+
+ try {
+ startedTest.foreach { action =>
+ action match {
+ case StartStream(trigger, triggerClock) =>
+ verify(currentStream == null, "stream already running")
+ lastStream = currentStream
+ currentStream =
+ spark
+ .streams
+ .startQuery(
+ StreamExecution.nextName,
+ metadataRoot,
+ stream,
+ sink,
+ outputMode,
+ trigger,
+ triggerClock)
+ .asInstanceOf[StreamExecution]
+ currentStream.microBatchThread.setUncaughtExceptionHandler(
+ new UncaughtExceptionHandler {
+ override def uncaughtException(t: Thread, e: Throwable): Unit = {
+ streamDeathCause = e
+ testThread.interrupt()
+ }
+ })
+
+ case AdvanceManualClock(timeToAdd) =>
+ verify(currentStream != null,
+ "can not advance manual clock when a stream is not running")
+ verify(currentStream.triggerClock.isInstanceOf[ManualClock],
+ s"can not advance clock of type ${currentStream.triggerClock.getClass}")
+ currentStream.triggerClock.asInstanceOf[ManualClock].advance(timeToAdd)
+
+ case StopStream =>
+ verify(currentStream != null, "can not stop a stream that is not running")
+ try failAfter(streamingTimeout) {
+ currentStream.stop()
+ verify(!currentStream.microBatchThread.isAlive,
+ s"microbatch thread not stopped")
+ verify(!currentStream.isActive,
+ "query.isActive() is false even after stopping")
+ verify(currentStream.exception.isEmpty,
+ s"query.exception() is not empty after clean stop: " +
+ currentStream.exception.map(_.toString()).getOrElse(""))
+ } catch {
+ case _: InterruptedException =>
+ case _: org.scalatest.exceptions.TestFailedDueToTimeoutException =>
+ failTest("Timed out while stopping and waiting for microbatchthread to terminate.")
+ case t: Throwable =>
+ failTest("Error while stopping stream", t)
+ } finally {
+ lastStream = currentStream
+ currentStream = null
+ }
+
+ case ef: ExpectFailure[_] =>
+ verify(currentStream != null, "can not expect failure when stream is not running")
+ try failAfter(streamingTimeout) {
+ val thrownException = intercept[ContinuousQueryException] {
+ currentStream.awaitTermination()
+ }
+ eventually("microbatch thread not stopped after termination with failure") {
+ assert(!currentStream.microBatchThread.isAlive)
+ }
+ verify(thrownException.query.eq(currentStream),
+ s"incorrect query reference in exception")
+ verify(currentStream.exception === Some(thrownException),
+ s"incorrect exception returned by query.exception()")
+
+ val exception = currentStream.exception.get
+ verify(exception.cause.getClass === ef.causeClass,
+ "incorrect cause in exception returned by query.exception()\n" +
+ s"\tExpected: ${ef.causeClass}\n\tReturned: ${exception.cause.getClass}")
+ } catch {
+ case _: InterruptedException =>
+ case _: org.scalatest.exceptions.TestFailedDueToTimeoutException =>
+ failTest("Timed out while waiting for failure")
+ case t: Throwable =>
+ failTest("Error while checking stream failure", t)
+ } finally {
+ lastStream = currentStream
+ currentStream = null
+ streamDeathCause = null
+ }
+
+ case a: AssertOnQuery =>
+ verify(currentStream != null || lastStream != null,
+ "cannot assert when not stream has been started")
+ val streamToAssert = Option(currentStream).getOrElse(lastStream)
+ verify(a.condition(streamToAssert), s"Assert on query failed: ${a.message}")
+
+ case a: Assert =>
+ val streamToAssert = Option(currentStream).getOrElse(lastStream)
+ verify({ a.run(); true }, s"Assert failed: ${a.message}")
+
+ case a: AddData =>
+ try {
+ // Add data and get the source where it was added, and the expected offset of the
+ // added data.
+ val queryToUse = Option(currentStream).orElse(Option(lastStream))
+ val (source, offset) = a.addData(queryToUse)
+
+ def findSourceIndex(plan: LogicalPlan): Option[Int] = {
+ plan
+ .collect { case StreamingExecutionRelation(s, _) => s }
+ .zipWithIndex
+ .find(_._1 == source)
+ .map(_._2)
+ }
+
+ // Try to find the index of the source to which data was added. Either get the index
+ // from the current active query or the original input logical plan.
+ val sourceIndex =
+ queryToUse.flatMap { query =>
+ findSourceIndex(query.logicalPlan)
+ }.orElse {
+ findSourceIndex(stream.logicalPlan)
+ }.getOrElse {
+ throw new IllegalArgumentException(
+ "Could find index of the source to which data was added")
+ }
+
+ // Store the expected offset of added data to wait for it later
+ awaiting.put(sourceIndex, offset)
+ } catch {
+ case NonFatal(e) =>
+ failTest("Error adding data", e)
+ }
+
+ case CheckAnswerRows(expectedAnswer, lastOnly) =>
+ verify(currentStream != null, "stream not running")
+ // Get the map of source index to the current source objects
+ val indexToSource = currentStream
+ .logicalPlan
+ .collect { case StreamingExecutionRelation(s, _) => s }
+ .zipWithIndex
+ .map(_.swap)
+ .toMap
+
+ // Block until all data added has been processed for all the source
+ awaiting.foreach { case (sourceIndex, offset) =>
+ failAfter(streamingTimeout) {
+ currentStream.awaitOffset(indexToSource(sourceIndex), offset)
+ }
+ }
+
+ val sparkAnswer = try if (lastOnly) sink.latestBatchData else sink.allData catch {
+ case e: Exception =>
+ failTest("Exception while getting data from sink", e)
+ }
+
+ QueryTest.sameRows(expectedAnswer, sparkAnswer).foreach {
+ error => failTest(error)
+ }
+ }
+ pos += 1
+ }
+ } catch {
+ case _: InterruptedException if streamDeathCause != null =>
+ failTest("Stream Thread Died")
+ case _: org.scalatest.exceptions.TestFailedDueToTimeoutException =>
+ failTest("Timed out waiting for stream")
+ } finally {
+ if (currentStream != null && currentStream.microBatchThread.isAlive) {
+ currentStream.stop()
+ }
+ }
+ }
+
+ /**
+ * Creates a stress test that randomly starts/stops/adds data/checks the result.
+ *
+ * @param ds a dataframe that executes + 1 on a stream of integers, returning the result.
+ * @param addData and add data action that adds the given numbers to the stream, encoding them
+ * as needed
+ */
+ def runStressTest(
+ ds: Dataset[Int],
+ addData: Seq[Int] => StreamAction,
+ iterations: Int = 100): Unit = {
+ implicit val intEncoder = ExpressionEncoder[Int]()
+ var dataPos = 0
+ var running = true
+ val actions = new ArrayBuffer[StreamAction]()
+
+ def addCheck() = { actions += CheckAnswer(1 to dataPos: _*) }
+
+ def addRandomData() = {
+ val numItems = Random.nextInt(10)
+ val data = dataPos until (dataPos + numItems)
+ dataPos += numItems
+ actions += addData(data)
+ }
+
+ (1 to iterations).foreach { i =>
+ val rand = Random.nextDouble()
+ if(!running) {
+ rand match {
+ case r if r < 0.7 => // AddData
+ addRandomData()
+
+ case _ => // StartStream
+ actions += StartStream()
+ running = true
+ }
+ } else {
+ rand match {
+ case r if r < 0.1 =>
+ addCheck()
+
+ case r if r < 0.7 => // AddData
+ addRandomData()
+
+ case _ => // StopStream
+ addCheck()
+ actions += StopStream
+ running = false
+ }
+ }
+ }
+ if(!running) { actions += StartStream() }
+ addCheck()
+ testStream(ds)(actions: _*)
+ }
+
+
+ object AwaitTerminationTester {
+
+ trait ExpectedBehavior
+
+ /** Expect awaitTermination to not be blocked */
+ case object ExpectNotBlocked extends ExpectedBehavior
+
+ /** Expect awaitTermination to get blocked */
+ case object ExpectBlocked extends ExpectedBehavior
+
+ /** Expect awaitTermination to throw an exception */
+ case class ExpectException[E <: Exception]()(implicit val t: ClassTag[E])
+ extends ExpectedBehavior
+
+ private val DEFAULT_TEST_TIMEOUT = 1.second
+
+ def test(
+ expectedBehavior: ExpectedBehavior,
+ awaitTermFunc: () => Unit,
+ testTimeout: Span = DEFAULT_TEST_TIMEOUT
+ ): Unit = {
+
+ expectedBehavior match {
+ case ExpectNotBlocked =>
+ withClue("Got blocked when expected non-blocking.") {
+ failAfter(testTimeout) {
+ awaitTermFunc()
+ }
+ }
+
+ case ExpectBlocked =>
+ withClue("Was not blocked when expected.") {
+ intercept[TestFailedDueToTimeoutException] {
+ failAfter(testTimeout) {
+ awaitTermFunc()
+ }
+ }
+ }
+
+ case e: ExpectException[_] =>
+ val thrownException =
+ withClue(s"Did not throw ${e.t.runtimeClass.getSimpleName} when expected.") {
+ intercept[ContinuousQueryException] {
+ failAfter(testTimeout) {
+ awaitTermFunc()
+ }
+ }
+ }
+ assert(thrownException.cause.getClass === e.t.runtimeClass,
+ "exception of incorrect type was throw")
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
index 322bbb9..1f174ae 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
@@ -20,19 +20,18 @@ package org.apache.spark.sql.streaming
import org.scalatest.BeforeAndAfterAll
import org.apache.spark.SparkException
-import org.apache.spark.sql.{AnalysisException, StreamTest}
+import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.InternalOutputModes._
import org.apache.spark.sql.execution.streaming._
import org.apache.spark.sql.execution.streaming.state.StateStore
import org.apache.spark.sql.expressions.scalalang.typed
import org.apache.spark.sql.functions._
-import org.apache.spark.sql.test.SharedSQLContext
object FailureSinglton {
var firstTime = true
}
-class StreamingAggregationSuite extends StreamTest with SharedSQLContext with BeforeAndAfterAll {
+class StreamingAggregationSuite extends StreamTest with BeforeAndAfterAll {
override def afterAll(): Unit = {
super.afterAll()
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataFrameReaderWriterSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataFrameReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataFrameReaderWriterSuite.scala
index 38a0534..a2aac69 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataFrameReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataFrameReaderWriterSuite.scala
@@ -27,7 +27,7 @@ import org.scalatest.BeforeAndAfter
import org.apache.spark.sql._
import org.apache.spark.sql.execution.streaming._
import org.apache.spark.sql.sources.{StreamSinkProvider, StreamSourceProvider}
-import org.apache.spark.sql.test.SharedSQLContext
+import org.apache.spark.sql.streaming.{ContinuousQuery, OutputMode, ProcessingTime, StreamTest}
import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
import org.apache.spark.util.Utils
@@ -101,7 +101,7 @@ class DefaultSource extends StreamSourceProvider with StreamSinkProvider {
}
}
-class DataFrameReaderWriterSuite extends StreamTest with SharedSQLContext with BeforeAndAfter {
+class DataFrameReaderWriterSuite extends StreamTest with BeforeAndAfter {
private def newMetadataDir =
Utils.createTempDir(namePrefix = "streaming.metadata").getCanonicalPath
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/test/scala/org/apache/spark/sql/util/ContinuousQueryListenerSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/util/ContinuousQueryListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/util/ContinuousQueryListenerSuite.scala
deleted file mode 100644
index 8788898..0000000
--- a/sql/core/src/test/scala/org/apache/spark/sql/util/ContinuousQueryListenerSuite.scala
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.util
-
-import java.util.concurrent.ConcurrentLinkedQueue
-
-import org.scalatest.BeforeAndAfter
-import org.scalatest.PrivateMethodTester._
-import org.scalatest.concurrent.AsyncAssertions.Waiter
-import org.scalatest.concurrent.Eventually._
-import org.scalatest.concurrent.PatienceConfiguration.Timeout
-import org.scalatest.time.SpanSugar._
-
-import org.apache.spark.sql._
-import org.apache.spark.sql.execution.streaming._
-import org.apache.spark.sql.test.SharedSQLContext
-import org.apache.spark.sql.util.ContinuousQueryListener.{QueryProgress, QueryStarted, QueryTerminated}
-
-class ContinuousQueryListenerSuite extends StreamTest with SharedSQLContext with BeforeAndAfter {
-
- import testImplicits._
-
- after {
- spark.streams.active.foreach(_.stop())
- assert(spark.streams.active.isEmpty)
- assert(addedListeners.isEmpty)
- // Make sure we don't leak any events to the next test
- spark.sparkContext.listenerBus.waitUntilEmpty(10000)
- }
-
- test("single listener") {
- val listener = new QueryStatusCollector
- val input = MemoryStream[Int]
- withListenerAdded(listener) {
- testStream(input.toDS)(
- StartStream(),
- Assert("Incorrect query status in onQueryStarted") {
- val status = listener.startStatus
- assert(status != null)
- assert(status.active == true)
- assert(status.sourceStatuses.size === 1)
- assert(status.sourceStatuses(0).description.contains("Memory"))
-
- // The source and sink offsets must be None as this must be called before the
- // batches have started
- assert(status.sourceStatuses(0).offset === None)
- assert(status.sinkStatus.offset === CompositeOffset(None :: Nil))
-
- // No progress events or termination events
- assert(listener.progressStatuses.isEmpty)
- assert(listener.terminationStatus === null)
- },
- AddDataMemory(input, Seq(1, 2, 3)),
- CheckAnswer(1, 2, 3),
- Assert("Incorrect query status in onQueryProgress") {
- eventually(Timeout(streamingTimeout)) {
-
- // There should be only on progress event as batch has been processed
- assert(listener.progressStatuses.size === 1)
- val status = listener.progressStatuses.peek()
- assert(status != null)
- assert(status.active == true)
- assert(status.sourceStatuses(0).offset === Some(LongOffset(0)))
- assert(status.sinkStatus.offset === CompositeOffset.fill(LongOffset(0)))
-
- // No termination events
- assert(listener.terminationStatus === null)
- }
- },
- StopStream,
- Assert("Incorrect query status in onQueryTerminated") {
- eventually(Timeout(streamingTimeout)) {
- val status = listener.terminationStatus
- assert(status != null)
-
- assert(status.active === false) // must be inactive by the time onQueryTerm is called
- assert(status.sourceStatuses(0).offset === Some(LongOffset(0)))
- assert(status.sinkStatus.offset === CompositeOffset.fill(LongOffset(0)))
- }
- listener.checkAsyncErrors()
- }
- )
- }
- }
-
- test("adding and removing listener") {
- def isListenerActive(listener: QueryStatusCollector): Boolean = {
- listener.reset()
- testStream(MemoryStream[Int].toDS)(
- StartStream(),
- StopStream
- )
- listener.startStatus != null
- }
-
- try {
- val listener1 = new QueryStatusCollector
- val listener2 = new QueryStatusCollector
-
- spark.streams.addListener(listener1)
- assert(isListenerActive(listener1) === true)
- assert(isListenerActive(listener2) === false)
- spark.streams.addListener(listener2)
- assert(isListenerActive(listener1) === true)
- assert(isListenerActive(listener2) === true)
- spark.streams.removeListener(listener1)
- assert(isListenerActive(listener1) === false)
- assert(isListenerActive(listener2) === true)
- } finally {
- addedListeners.foreach(spark.streams.removeListener)
- }
- }
-
- test("event ordering") {
- val listener = new QueryStatusCollector
- withListenerAdded(listener) {
- for (i <- 1 to 100) {
- listener.reset()
- require(listener.startStatus === null)
- testStream(MemoryStream[Int].toDS)(
- StartStream(),
- Assert(listener.startStatus !== null, "onQueryStarted not called before query returned"),
- StopStream,
- Assert { listener.checkAsyncErrors() }
- )
- }
- }
- }
-
-
- private def withListenerAdded(listener: ContinuousQueryListener)(body: => Unit): Unit = {
- try {
- failAfter(1 minute) {
- spark.streams.addListener(listener)
- body
- }
- } finally {
- spark.streams.removeListener(listener)
- }
- }
-
- private def addedListeners(): Array[ContinuousQueryListener] = {
- val listenerBusMethod =
- PrivateMethod[ContinuousQueryListenerBus]('listenerBus)
- val listenerBus = spark.streams invokePrivate listenerBusMethod()
- listenerBus.listeners.toArray.map(_.asInstanceOf[ContinuousQueryListener])
- }
-
- class QueryStatusCollector extends ContinuousQueryListener {
- // to catch errors in the async listener events
- @volatile private var asyncTestWaiter = new Waiter
-
- @volatile var startStatus: QueryStatus = null
- @volatile var terminationStatus: QueryStatus = null
- val progressStatuses = new ConcurrentLinkedQueue[QueryStatus]
-
- def reset(): Unit = {
- startStatus = null
- terminationStatus = null
- progressStatuses.clear()
- asyncTestWaiter = new Waiter
- }
-
- def checkAsyncErrors(): Unit = {
- asyncTestWaiter.await(timeout(streamingTimeout))
- }
-
-
- override def onQueryStarted(queryStarted: QueryStarted): Unit = {
- asyncTestWaiter {
- startStatus = QueryStatus(queryStarted.query)
- }
- }
-
- override def onQueryProgress(queryProgress: QueryProgress): Unit = {
- asyncTestWaiter {
- assert(startStatus != null, "onQueryProgress called before onQueryStarted")
- progressStatuses.add(QueryStatus(queryProgress.query))
- }
- }
-
- override def onQueryTerminated(queryTerminated: QueryTerminated): Unit = {
- asyncTestWaiter {
- assert(startStatus != null, "onQueryTerminated called before onQueryStarted")
- terminationStatus = QueryStatus(queryTerminated.query)
- }
- asyncTestWaiter.dismiss()
- }
- }
-
- case class QueryStatus(
- active: Boolean,
- exception: Option[Exception],
- sourceStatuses: Array[SourceStatus],
- sinkStatus: SinkStatus)
-
- object QueryStatus {
- def apply(query: ContinuousQuery): QueryStatus = {
- QueryStatus(query.isActive, query.exception, query.sourceStatuses, query.sinkStatus)
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[2/2] spark git commit: [SPARK-15686][SQL] Move user-facing streaming
classes into sql.streaming
Posted by ma...@apache.org.
[SPARK-15686][SQL] Move user-facing streaming classes into sql.streaming
## What changes were proposed in this pull request?
This patch moves all user-facing structured streaming classes into sql.streaming. As part of this, I also added some since version annotation to methods and classes that don't have them.
## How was this patch tested?
Updated tests to reflect the moves.
Author: Reynold Xin <rx...@databricks.com>
Closes #13429 from rxin/SPARK-15686.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a71d1364
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a71d1364
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a71d1364
Branch: refs/heads/master
Commit: a71d1364ae87aa388128da34dd0b9b02ff85e458
Parents: d5012c2
Author: Reynold Xin <rx...@databricks.com>
Authored: Wed Jun 1 10:14:40 2016 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Wed Jun 1 10:14:40 2016 -0700
----------------------------------------------------------------------
python/pyspark/sql/streaming.py | 3 +-
python/pyspark/sql/utils.py | 2 +-
.../java/org/apache/spark/sql/OutputMode.java | 54 --
.../apache/spark/sql/streaming/OutputMode.java | 55 ++
.../apache/spark/sql/InternalOutputModes.scala | 2 +
.../analysis/UnsupportedOperationChecker.scala | 3 +-
.../apache/spark/sql/JavaOutputModeSuite.java | 31 -
.../sql/streaming/JavaOutputModeSuite.java | 31 +
.../analysis/UnsupportedOperationsSuite.scala | 3 +-
.../org/apache/spark/sql/ContinuousQuery.scala | 109 ----
.../spark/sql/ContinuousQueryException.scala | 54 --
.../spark/sql/ContinuousQueryManager.scala | 232 --------
.../org/apache/spark/sql/DataFrameWriter.scala | 1 +
.../scala/org/apache/spark/sql/Dataset.scala | 1 +
.../scala/org/apache/spark/sql/SQLContext.scala | 6 +-
.../scala/org/apache/spark/sql/SinkStatus.scala | 34 --
.../org/apache/spark/sql/SourceStatus.scala | 34 --
.../org/apache/spark/sql/SparkSession.scala | 7 +-
.../scala/org/apache/spark/sql/Trigger.scala | 133 -----
.../spark/sql/execution/SparkStrategies.scala | 3 +-
.../sql/execution/datasources/DataSource.scala | 1 +
.../streaming/ContinuousQueryListenerBus.scala | 11 +-
.../streaming/IncrementalExecution.scala | 3 +-
.../execution/streaming/StreamExecution.scala | 5 +-
.../execution/streaming/TriggerExecutor.scala | 2 +-
.../spark/sql/execution/streaming/console.scala | 3 +-
.../spark/sql/execution/streaming/memory.scala | 1 +
.../spark/sql/internal/SessionState.scala | 3 +-
.../apache/spark/sql/sources/interfaces.scala | 1 +
.../spark/sql/streaming/ContinuousQuery.scala | 110 ++++
.../streaming/ContinuousQueryException.scala | 54 ++
.../sql/streaming/ContinuousQueryListener.scala | 95 ++++
.../sql/streaming/ContinuousQueryManager.scala | 231 ++++++++
.../apache/spark/sql/streaming/SinkStatus.scala | 34 ++
.../spark/sql/streaming/SourceStatus.scala | 34 ++
.../apache/spark/sql/streaming/Trigger.scala | 147 +++++
.../sql/util/ContinuousQueryListener.scala | 75 ---
.../apache/spark/sql/ProcessingTimeSuite.scala | 1 +
.../scala/org/apache/spark/sql/StreamTest.scala | 565 ------------------
.../streaming/ProcessingTimeExecutorSuite.scala | 2 +-
.../ContinuousQueryListenerSuite.scala | 216 +++++++
.../streaming/ContinuousQueryManagerSuite.scala | 5 +-
.../sql/streaming/ContinuousQuerySuite.scala | 5 +-
.../sql/streaming/FileStreamSinkSuite.scala | 2 +-
.../sql/streaming/FileStreamSourceSuite.scala | 4 +-
.../spark/sql/streaming/FileStressSuite.scala | 4 +-
.../spark/sql/streaming/MemorySinkSuite.scala | 2 +-
.../sql/streaming/MemorySourceStressSuite.scala | 4 +-
.../spark/sql/streaming/StreamSuite.scala | 2 +-
.../apache/spark/sql/streaming/StreamTest.scala | 567 +++++++++++++++++++
.../streaming/StreamingAggregationSuite.scala | 5 +-
.../test/DataFrameReaderWriterSuite.scala | 4 +-
.../sql/util/ContinuousQueryListenerSuite.scala | 217 -------
53 files changed, 1630 insertions(+), 1583 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/python/pyspark/sql/streaming.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index 8238b8e..cd75622 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -201,7 +201,8 @@ class ProcessingTime(Trigger):
self.interval = interval
def _to_java_trigger(self, sqlContext):
- return sqlContext._sc._jvm.org.apache.spark.sql.ProcessingTime.create(self.interval)
+ return sqlContext._sc._jvm.org.apache.spark.sql.streaming.ProcessingTime.create(
+ self.interval)
def _test():
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/python/pyspark/sql/utils.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index 8c8768f..9ddaf78 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -71,7 +71,7 @@ def capture_sql_exception(f):
raise AnalysisException(s.split(': ', 1)[1], stackTrace)
if s.startswith('org.apache.spark.sql.catalyst.parser.ParseException: '):
raise ParseException(s.split(': ', 1)[1], stackTrace)
- if s.startswith('org.apache.spark.sql.ContinuousQueryException: '):
+ if s.startswith('org.apache.spark.sql.streaming.ContinuousQueryException: '):
raise ContinuousQueryException(s.split(': ', 1)[1], stackTrace)
if s.startswith('org.apache.spark.sql.execution.QueryExecutionException: '):
raise QueryExecutionException(s.split(': ', 1)[1], stackTrace)
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/catalyst/src/main/java/org/apache/spark/sql/OutputMode.java
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/OutputMode.java b/sql/catalyst/src/main/java/org/apache/spark/sql/OutputMode.java
deleted file mode 100644
index 1936d53..0000000
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/OutputMode.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql;
-
-import org.apache.spark.annotation.Experimental;
-
-/**
- * :: Experimental ::
- *
- * OutputMode is used to what data will be written to a streaming sink when there is
- * new data available in a streaming DataFrame/Dataset.
- *
- * @since 2.0.0
- */
-@Experimental
-public class OutputMode {
-
- /**
- * OutputMode in which only the new rows in the streaming DataFrame/Dataset will be
- * written to the sink. This output mode can be only be used in queries that do not
- * contain any aggregation.
- *
- * @since 2.0.0
- */
- public static OutputMode Append() {
- return InternalOutputModes.Append$.MODULE$;
- }
-
- /**
- * OutputMode in which all the rows in the streaming DataFrame/Dataset will be written
- * to the sink every time these is some updates. This output mode can only be used in queries
- * that contain aggregations.
- *
- * @since 2.0.0
- */
- public static OutputMode Complete() {
- return InternalOutputModes.Complete$.MODULE$;
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/catalyst/src/main/java/org/apache/spark/sql/streaming/OutputMode.java
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/streaming/OutputMode.java b/sql/catalyst/src/main/java/org/apache/spark/sql/streaming/OutputMode.java
new file mode 100644
index 0000000..41e2582
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/streaming/OutputMode.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming;
+
+import org.apache.spark.annotation.Experimental;
+import org.apache.spark.sql.InternalOutputModes;
+
+/**
+ * :: Experimental ::
+ *
+ * OutputMode is used to what data will be written to a streaming sink when there is
+ * new data available in a streaming DataFrame/Dataset.
+ *
+ * @since 2.0.0
+ */
+@Experimental
+public class OutputMode {
+
+ /**
+ * OutputMode in which only the new rows in the streaming DataFrame/Dataset will be
+ * written to the sink. This output mode can be only be used in queries that do not
+ * contain any aggregation.
+ *
+ * @since 2.0.0
+ */
+ public static OutputMode Append() {
+ return InternalOutputModes.Append$.MODULE$;
+ }
+
+ /**
+ * OutputMode in which all the rows in the streaming DataFrame/Dataset will be written
+ * to the sink every time these is some updates. This output mode can only be used in queries
+ * that contain aggregations.
+ *
+ * @since 2.0.0
+ */
+ public static OutputMode Complete() {
+ return InternalOutputModes.Complete$.MODULE$;
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/catalyst/src/main/scala/org/apache/spark/sql/InternalOutputModes.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/InternalOutputModes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/InternalOutputModes.scala
index 8ef5d9a..153f9f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/InternalOutputModes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/InternalOutputModes.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql
+import org.apache.spark.sql.streaming.OutputMode
+
/**
* Internal helper class to generate objects representing various [[OutputMode]]s,
*/
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
index f4c0347..8373fa3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
@@ -17,9 +17,10 @@
package org.apache.spark.sql.catalyst.analysis
-import org.apache.spark.sql.{AnalysisException, InternalOutputModes, OutputMode}
+import org.apache.spark.sql.{AnalysisException, InternalOutputModes}
import org.apache.spark.sql.catalyst.plans._
import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.streaming.OutputMode
/**
* Analyzes the presence of unsupported operations in a logical plan.
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/catalyst/src/test/java/org/apache/spark/sql/JavaOutputModeSuite.java
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/java/org/apache/spark/sql/JavaOutputModeSuite.java b/sql/catalyst/src/test/java/org/apache/spark/sql/JavaOutputModeSuite.java
deleted file mode 100644
index 1764f33..0000000
--- a/sql/catalyst/src/test/java/org/apache/spark/sql/JavaOutputModeSuite.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql;
-
-import org.junit.Test;
-
-public class JavaOutputModeSuite {
-
- @Test
- public void testOutputModes() {
- OutputMode o1 = OutputMode.Append();
- assert(o1.toString().toLowerCase().contains("append"));
- OutputMode o2 = OutputMode.Complete();
- assert (o2.toString().toLowerCase().contains("complete"));
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/catalyst/src/test/java/org/apache/spark/sql/streaming/JavaOutputModeSuite.java
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/java/org/apache/spark/sql/streaming/JavaOutputModeSuite.java b/sql/catalyst/src/test/java/org/apache/spark/sql/streaming/JavaOutputModeSuite.java
new file mode 100644
index 0000000..e0a54fe
--- /dev/null
+++ b/sql/catalyst/src/test/java/org/apache/spark/sql/streaming/JavaOutputModeSuite.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming;
+
+import org.junit.Test;
+
+public class JavaOutputModeSuite {
+
+ @Test
+ public void testOutputModes() {
+ OutputMode o1 = OutputMode.Append();
+ assert(o1.toString().toLowerCase().contains("append"));
+ OutputMode o2 = OutputMode.Complete();
+ assert (o2.toString().toLowerCase().contains("complete"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
index c2e3d47..378cca3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
@@ -18,7 +18,7 @@
package org.apache.spark.sql.catalyst.analysis
import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.{AnalysisException, OutputMode}
+import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.InternalOutputModes._
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
@@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference,
import org.apache.spark.sql.catalyst.expressions.aggregate.Count
import org.apache.spark.sql.catalyst.plans._
import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.streaming.OutputMode
import org.apache.spark.sql.types.IntegerType
/** A dummy command for testing unsupported operations. */
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/ContinuousQuery.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/ContinuousQuery.scala b/sql/core/src/main/scala/org/apache/spark/sql/ContinuousQuery.scala
deleted file mode 100644
index 4d5afe2..0000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/ContinuousQuery.scala
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.annotation.Experimental
-
-/**
- * :: Experimental ::
- * A handle to a query that is executing continuously in the background as new data arrives.
- * All these methods are thread-safe.
- * @since 2.0.0
- */
-@Experimental
-trait ContinuousQuery {
-
- /**
- * Returns the name of the query.
- * @since 2.0.0
- */
- def name: String
-
- /**
- * Returns the [[SparkSession]] associated with `this`.
- * @since 2.0.0
- */
- def sparkSession: SparkSession
-
- /**
- * Whether the query is currently active or not
- * @since 2.0.0
- */
- def isActive: Boolean
-
- /**
- * Returns the [[ContinuousQueryException]] if the query was terminated by an exception.
- * @since 2.0.0
- */
- def exception: Option[ContinuousQueryException]
-
- /**
- * Returns current status of all the sources.
- * @since 2.0.0
- */
- def sourceStatuses: Array[SourceStatus]
-
- /** Returns current status of the sink. */
- def sinkStatus: SinkStatus
-
- /**
- * Waits for the termination of `this` query, either by `query.stop()` or by an exception.
- * If the query has terminated with an exception, then the exception will be thrown.
- *
- * If the query has terminated, then all subsequent calls to this method will either return
- * immediately (if the query was terminated by `stop()`), or throw the exception
- * immediately (if the query has terminated with exception).
- *
- * @throws ContinuousQueryException, if `this` query has terminated with an exception.
- *
- * @since 2.0.0
- */
- def awaitTermination(): Unit
-
- /**
- * Waits for the termination of `this` query, either by `query.stop()` or by an exception.
- * If the query has terminated with an exception, then the exception will be thrown.
- * Otherwise, it returns whether the query has terminated or not within the `timeoutMs`
- * milliseconds.
- *
- * If the query has terminated, then all subsequent calls to this method will either return
- * `true` immediately (if the query was terminated by `stop()`), or throw the exception
- * immediately (if the query has terminated with exception).
- *
- * @throws ContinuousQueryException, if `this` query has terminated with an exception
- *
- * @since 2.0.0
- */
- def awaitTermination(timeoutMs: Long): Boolean
-
- /**
- * Blocks until all available data in the source has been processed an committed to the sink.
- * This method is intended for testing. Note that in the case of continually arriving data, this
- * method may block forever. Additionally, this method is only guaranteed to block until data that
- * has been synchronously appended data to a [[org.apache.spark.sql.execution.streaming.Source]]
- * prior to invocation. (i.e. `getOffset` must immediately reflect the addition).
- */
- def processAllAvailable(): Unit
-
- /**
- * Stops the execution of this query if it is running. This method blocks until the threads
- * performing execution has stopped.
- * @since 2.0.0
- */
- def stop(): Unit
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/ContinuousQueryException.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/ContinuousQueryException.scala b/sql/core/src/main/scala/org/apache/spark/sql/ContinuousQueryException.scala
deleted file mode 100644
index fec3862..0000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/ContinuousQueryException.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.annotation.Experimental
-import org.apache.spark.sql.execution.streaming.{Offset, StreamExecution}
-
-/**
- * :: Experimental ::
- * Exception that stopped a [[ContinuousQuery]].
- * @param query Query that caused the exception
- * @param message Message of this exception
- * @param cause Internal cause of this exception
- * @param startOffset Starting offset (if known) of the range of data in which exception occurred
- * @param endOffset Ending offset (if known) of the range of data in exception occurred
- * @since 2.0.0
- */
-@Experimental
-class ContinuousQueryException private[sql](
- @transient val query: ContinuousQuery,
- val message: String,
- val cause: Throwable,
- val startOffset: Option[Offset] = None,
- val endOffset: Option[Offset] = None)
- extends Exception(message, cause) {
-
- /** Time when the exception occurred */
- val time: Long = System.currentTimeMillis
-
- override def toString(): String = {
- val causeStr =
- s"${cause.getMessage} ${cause.getStackTrace.take(10).mkString("", "\n|\t", "\n")}"
- s"""
- |$causeStr
- |
- |${query.asInstanceOf[StreamExecution].toDebugString}
- """.stripMargin
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/ContinuousQueryManager.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/ContinuousQueryManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/ContinuousQueryManager.scala
deleted file mode 100644
index c686400..0000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/ContinuousQueryManager.scala
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.collection.mutable
-
-import org.apache.spark.annotation.Experimental
-import org.apache.spark.sql.catalyst.analysis.UnsupportedOperationChecker
-import org.apache.spark.sql.execution.streaming._
-import org.apache.spark.sql.execution.streaming.state.StateStoreCoordinatorRef
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.util.ContinuousQueryListener
-import org.apache.spark.util.{Clock, SystemClock}
-
-/**
- * :: Experimental ::
- * A class to manage all the [[org.apache.spark.sql.ContinuousQuery ContinuousQueries]] active
- * on a [[SparkSession]].
- *
- * @since 2.0.0
- */
-@Experimental
-class ContinuousQueryManager private[sql] (sparkSession: SparkSession) {
-
- private[sql] val stateStoreCoordinator =
- StateStoreCoordinatorRef.forDriver(sparkSession.sparkContext.env)
- private val listenerBus = new ContinuousQueryListenerBus(sparkSession.sparkContext.listenerBus)
- private val activeQueries = new mutable.HashMap[String, ContinuousQuery]
- private val activeQueriesLock = new Object
- private val awaitTerminationLock = new Object
-
- private var lastTerminatedQuery: ContinuousQuery = null
-
- /**
- * Returns a list of active queries associated with this SQLContext
- *
- * @since 2.0.0
- */
- def active: Array[ContinuousQuery] = activeQueriesLock.synchronized {
- activeQueries.values.toArray
- }
-
- /**
- * Returns an active query from this SQLContext or throws exception if bad name
- *
- * @since 2.0.0
- */
- def get(name: String): ContinuousQuery = activeQueriesLock.synchronized {
- activeQueries.getOrElse(name,
- throw new IllegalArgumentException(s"There is no active query with name $name"))
- }
-
- /**
- * Wait until any of the queries on the associated SQLContext has terminated since the
- * creation of the context, or since `resetTerminated()` was called. If any query was terminated
- * with an exception, then the exception will be thrown.
- *
- * If a query has terminated, then subsequent calls to `awaitAnyTermination()` will either
- * return immediately (if the query was terminated by `query.stop()`),
- * or throw the exception immediately (if the query was terminated with exception). Use
- * `resetTerminated()` to clear past terminations and wait for new terminations.
- *
- * In the case where multiple queries have terminated since `resetTermination()` was called,
- * if any query has terminated with exception, then `awaitAnyTermination()` will
- * throw any of the exception. For correctly documenting exceptions across multiple queries,
- * users need to stop all of them after any of them terminates with exception, and then check the
- * `query.exception()` for each query.
- *
- * @throws ContinuousQueryException, if any query has terminated with an exception
- *
- * @since 2.0.0
- */
- def awaitAnyTermination(): Unit = {
- awaitTerminationLock.synchronized {
- while (lastTerminatedQuery == null) {
- awaitTerminationLock.wait(10)
- }
- if (lastTerminatedQuery != null && lastTerminatedQuery.exception.nonEmpty) {
- throw lastTerminatedQuery.exception.get
- }
- }
- }
-
- /**
- * Wait until any of the queries on the associated SQLContext has terminated since the
- * creation of the context, or since `resetTerminated()` was called. Returns whether any query
- * has terminated or not (multiple may have terminated). If any query has terminated with an
- * exception, then the exception will be thrown.
- *
- * If a query has terminated, then subsequent calls to `awaitAnyTermination()` will either
- * return `true` immediately (if the query was terminated by `query.stop()`),
- * or throw the exception immediately (if the query was terminated with exception). Use
- * `resetTerminated()` to clear past terminations and wait for new terminations.
- *
- * In the case where multiple queries have terminated since `resetTermination()` was called,
- * if any query has terminated with exception, then `awaitAnyTermination()` will
- * throw any of the exception. For correctly documenting exceptions across multiple queries,
- * users need to stop all of them after any of them terminates with exception, and then check the
- * `query.exception()` for each query.
- *
- * @throws ContinuousQueryException, if any query has terminated with an exception
- *
- * @since 2.0.0
- */
- def awaitAnyTermination(timeoutMs: Long): Boolean = {
-
- val startTime = System.currentTimeMillis
- def isTimedout = System.currentTimeMillis - startTime >= timeoutMs
-
- awaitTerminationLock.synchronized {
- while (!isTimedout && lastTerminatedQuery == null) {
- awaitTerminationLock.wait(10)
- }
- if (lastTerminatedQuery != null && lastTerminatedQuery.exception.nonEmpty) {
- throw lastTerminatedQuery.exception.get
- }
- lastTerminatedQuery != null
- }
- }
-
- /**
- * Forget about past terminated queries so that `awaitAnyTermination()` can be used again to
- * wait for new terminations.
- *
- * @since 2.0.0
- */
- def resetTerminated(): Unit = {
- awaitTerminationLock.synchronized {
- lastTerminatedQuery = null
- }
- }
-
- /**
- * Register a [[ContinuousQueryListener]] to receive up-calls for life cycle events of
- * [[org.apache.spark.sql.ContinuousQuery ContinuousQueries]].
- *
- * @since 2.0.0
- */
- def addListener(listener: ContinuousQueryListener): Unit = {
- listenerBus.addListener(listener)
- }
-
- /**
- * Deregister a [[ContinuousQueryListener]].
- *
- * @since 2.0.0
- */
- def removeListener(listener: ContinuousQueryListener): Unit = {
- listenerBus.removeListener(listener)
- }
-
- /** Post a listener event */
- private[sql] def postListenerEvent(event: ContinuousQueryListener.Event): Unit = {
- listenerBus.post(event)
- }
-
- /** Start a query */
- private[sql] def startQuery(
- name: String,
- checkpointLocation: String,
- df: DataFrame,
- sink: Sink,
- outputMode: OutputMode,
- trigger: Trigger = ProcessingTime(0),
- triggerClock: Clock = new SystemClock()): ContinuousQuery = {
- activeQueriesLock.synchronized {
- if (activeQueries.contains(name)) {
- throw new IllegalArgumentException(
- s"Cannot start query with name $name as a query with that name is already active")
- }
- val analyzedPlan = df.queryExecution.analyzed
- df.queryExecution.assertAnalyzed()
-
- if (sparkSession.conf.get(SQLConf.UNSUPPORTED_OPERATION_CHECK_ENABLED)) {
- UnsupportedOperationChecker.checkForStreaming(analyzedPlan, outputMode)
- }
-
- var nextSourceId = 0L
-
- val logicalPlan = analyzedPlan.transform {
- case StreamingRelation(dataSource, _, output) =>
- // Materialize source to avoid creating it in every batch
- val metadataPath = s"$checkpointLocation/sources/$nextSourceId"
- val source = dataSource.createSource(metadataPath)
- nextSourceId += 1
- // We still need to use the previous `output` instead of `source.schema` as attributes in
- // "df.logicalPlan" has already used attributes of the previous `output`.
- StreamingExecutionRelation(source, output)
- }
- val query = new StreamExecution(
- sparkSession,
- name,
- checkpointLocation,
- logicalPlan,
- sink,
- trigger,
- triggerClock,
- outputMode)
- query.start()
- activeQueries.put(name, query)
- query
- }
- }
-
- /** Notify (by the ContinuousQuery) that the query has been terminated */
- private[sql] def notifyQueryTermination(terminatedQuery: ContinuousQuery): Unit = {
- activeQueriesLock.synchronized {
- activeQueries -= terminatedQuery.name
- }
- awaitTerminationLock.synchronized {
- if (lastTerminatedQuery == null || terminatedQuery.exception.nonEmpty) {
- lastTerminatedQuery = terminatedQuery
- }
- awaitTerminationLock.notifyAll()
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 291b825..25678e9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.execution.datasources.{BucketSpec, CreateTableUsingA
import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils
import org.apache.spark.sql.execution.streaming.{MemoryPlan, MemorySink, StreamExecution}
import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.streaming.{ContinuousQuery, OutputMode, ProcessingTime, Trigger}
import org.apache.spark.util.Utils
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 7be49b1..3a6ec45 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -49,6 +49,7 @@ import org.apache.spark.sql.execution.command.{CreateViewCommand, ExplainCommand
import org.apache.spark.sql.execution.datasources.{CreateTableUsingAsSelect, LogicalRelation}
import org.apache.spark.sql.execution.datasources.json.JacksonGenerator
import org.apache.spark.sql.execution.python.EvaluatePython
+import org.apache.spark.sql.streaming.ContinuousQuery
import org.apache.spark.sql.types._
import org.apache.spark.storage.StorageLevel
import org.apache.spark.util.Utils
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 0dc70c0..2e14c5d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -30,13 +30,11 @@ import org.apache.spark.internal.Logging
import org.apache.spark.internal.config.ConfigEntry
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst._
-import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.command.ShowTablesCommand
import org.apache.spark.sql.internal.{SessionState, SharedState, SQLConf}
import org.apache.spark.sql.sources.BaseRelation
+import org.apache.spark.sql.streaming.ContinuousQueryManager
import org.apache.spark.sql.types._
import org.apache.spark.sql.util.ExecutionListenerManager
@@ -645,7 +643,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
/**
* Returns a [[ContinuousQueryManager]] that allows managing all the
- * [[org.apache.spark.sql.ContinuousQuery ContinuousQueries]] active on `this` context.
+ * [[org.apache.spark.sql.streaming.ContinuousQuery ContinuousQueries]] active on `this` context.
*
* @since 2.0.0
*/
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/SinkStatus.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SinkStatus.scala b/sql/core/src/main/scala/org/apache/spark/sql/SinkStatus.scala
deleted file mode 100644
index 5a98528..0000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/SinkStatus.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.annotation.Experimental
-import org.apache.spark.sql.execution.streaming.{Offset, Sink}
-
-/**
- * :: Experimental ::
- * Status and metrics of a streaming [[Sink]].
- *
- * @param description Description of the source corresponding to this status
- * @param offset Current offset up to which data has been written by the sink
- * @since 2.0.0
- */
-@Experimental
-class SinkStatus private[sql](
- val description: String,
- val offset: Offset)
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/SourceStatus.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SourceStatus.scala b/sql/core/src/main/scala/org/apache/spark/sql/SourceStatus.scala
deleted file mode 100644
index 2479e67..0000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/SourceStatus.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.annotation.Experimental
-import org.apache.spark.sql.execution.streaming.{Offset, Source}
-
-/**
- * :: Experimental ::
- * Status and metrics of a streaming [[Source]].
- *
- * @param description Description of the source corresponding to this status
- * @param offset Current offset of the source, if known
- * @since 2.0.0
- */
-@Experimental
-class SourceStatus private[sql] (
- val description: String,
- val offset: Option[Offset])
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index dc4b72a..52bedf9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -25,7 +25,7 @@ import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
import scala.util.control.NonFatal
-import org.apache.spark.{SparkConf, SparkContext, SparkException}
+import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.annotation.{DeveloperApi, Experimental}
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.internal.Logging
@@ -40,8 +40,9 @@ import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Range}
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.execution.ui.SQLListener
-import org.apache.spark.sql.internal.{CatalogImpl, SessionState, SharedState, SQLConf}
+import org.apache.spark.sql.internal.{CatalogImpl, SessionState, SharedState}
import org.apache.spark.sql.sources.BaseRelation
+import org.apache.spark.sql.streaming._
import org.apache.spark.sql.types.{DataType, LongType, StructType}
import org.apache.spark.sql.util.ExecutionListenerManager
import org.apache.spark.util.Utils
@@ -182,7 +183,7 @@ class SparkSession private(
/**
* :: Experimental ::
* Returns a [[ContinuousQueryManager]] that allows managing all the
- * [[org.apache.spark.sql.ContinuousQuery ContinuousQueries]] active on `this`.
+ * [[ContinuousQuery ContinuousQueries]] active on `this`.
*
* @group basic
* @since 2.0.0
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/Trigger.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Trigger.scala b/sql/core/src/main/scala/org/apache/spark/sql/Trigger.scala
deleted file mode 100644
index 256e8a4..0000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/Trigger.scala
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import java.util.concurrent.TimeUnit
-
-import scala.concurrent.duration.Duration
-
-import org.apache.commons.lang3.StringUtils
-
-import org.apache.spark.annotation.Experimental
-import org.apache.spark.unsafe.types.CalendarInterval
-
-/**
- * :: Experimental ::
- * Used to indicate how often results should be produced by a [[ContinuousQuery]].
- */
-@Experimental
-sealed trait Trigger {}
-
-/**
- * :: Experimental ::
- * A trigger that runs a query periodically based on the processing time. If `interval` is 0,
- * the query will run as fast as possible.
- *
- * Scala Example:
- * {{{
- * df.write.trigger(ProcessingTime("10 seconds"))
- *
- * import scala.concurrent.duration._
- * df.write.trigger(ProcessingTime(10.seconds))
- * }}}
- *
- * Java Example:
- * {{{
- * df.write.trigger(ProcessingTime.create("10 seconds"))
- *
- * import java.util.concurrent.TimeUnit
- * df.write.trigger(ProcessingTime.create(10, TimeUnit.SECONDS))
- * }}}
- */
-@Experimental
-case class ProcessingTime(intervalMs: Long) extends Trigger {
- require(intervalMs >= 0, "the interval of trigger should not be negative")
-}
-
-/**
- * :: Experimental ::
- * Used to create [[ProcessingTime]] triggers for [[ContinuousQuery]]s.
- */
-@Experimental
-object ProcessingTime {
-
- /**
- * Create a [[ProcessingTime]]. If `interval` is 0, the query will run as fast as possible.
- *
- * Example:
- * {{{
- * df.write.trigger(ProcessingTime("10 seconds"))
- * }}}
- */
- def apply(interval: String): ProcessingTime = {
- if (StringUtils.isBlank(interval)) {
- throw new IllegalArgumentException(
- "interval cannot be null or blank.")
- }
- val cal = if (interval.startsWith("interval")) {
- CalendarInterval.fromString(interval)
- } else {
- CalendarInterval.fromString("interval " + interval)
- }
- if (cal == null) {
- throw new IllegalArgumentException(s"Invalid interval: $interval")
- }
- if (cal.months > 0) {
- throw new IllegalArgumentException(s"Doesn't support month or year interval: $interval")
- }
- new ProcessingTime(cal.microseconds / 1000)
- }
-
- /**
- * Create a [[ProcessingTime]]. If `interval` is 0, the query will run as fast as possible.
- *
- * Example:
- * {{{
- * import scala.concurrent.duration._
- * df.write.trigger(ProcessingTime(10.seconds))
- * }}}
- */
- def apply(interval: Duration): ProcessingTime = {
- new ProcessingTime(interval.toMillis)
- }
-
- /**
- * Create a [[ProcessingTime]]. If `interval` is 0, the query will run as fast as possible.
- *
- * Example:
- * {{{
- * df.write.trigger(ProcessingTime.create("10 seconds"))
- * }}}
- */
- def create(interval: String): ProcessingTime = {
- apply(interval)
- }
-
- /**
- * Create a [[ProcessingTime]]. If `interval` is 0, the query will run as fast as possible.
- *
- * Example:
- * {{{
- * import java.util.concurrent.TimeUnit
- * df.write.trigger(ProcessingTime.create(10, TimeUnit.SECONDS))
- * }}}
- */
- def create(interval: Long, unit: TimeUnit): ProcessingTime = {
- new ProcessingTime(unit.toMillis(interval))
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index e405252..7e3e45e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.execution.exchange.ShuffleExchange
import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight}
import org.apache.spark.sql.execution.streaming.MemoryPlan
import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.streaming.ContinuousQuery
private[sql] abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
self: SparkPlanner =>
@@ -201,7 +202,7 @@ private[sql] abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
/**
* Used to plan aggregation queries that are computed incrementally as part of a
- * [[org.apache.spark.sql.ContinuousQuery]]. Currently this rule is injected into the planner
+ * [[ContinuousQuery]]. Currently this rule is injected into the planner
* on-demand, only when planning in a [[org.apache.spark.sql.execution.streaming.StreamExecution]]
*/
object StatefulAggregationStrategy extends Strategy {
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
index 814880b..93f1ad0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
@@ -37,6 +37,7 @@ import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
import org.apache.spark.sql.execution.streaming._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources._
+import org.apache.spark.sql.streaming.OutputMode
import org.apache.spark.sql.types.{CalendarIntervalType, StructType}
import org.apache.spark.util.Utils
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ContinuousQueryListenerBus.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ContinuousQueryListenerBus.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ContinuousQueryListenerBus.scala
index b1d24b6..2a1be09 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ContinuousQueryListenerBus.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ContinuousQueryListenerBus.scala
@@ -18,8 +18,7 @@
package org.apache.spark.sql.execution.streaming
import org.apache.spark.scheduler.{LiveListenerBus, SparkListener, SparkListenerEvent}
-import org.apache.spark.sql.util.ContinuousQueryListener
-import org.apache.spark.sql.util.ContinuousQueryListener._
+import org.apache.spark.sql.streaming.ContinuousQueryListener
import org.apache.spark.util.ListenerBus
/**
@@ -30,7 +29,10 @@ import org.apache.spark.util.ListenerBus
* dispatch them to ContinuousQueryListener.
*/
class ContinuousQueryListenerBus(sparkListenerBus: LiveListenerBus)
- extends SparkListener with ListenerBus[ContinuousQueryListener, ContinuousQueryListener.Event] {
+ extends SparkListener
+ with ListenerBus[ContinuousQueryListener, ContinuousQueryListener.Event] {
+
+ import ContinuousQueryListener._
sparkListenerBus.addListener(this)
@@ -74,7 +76,8 @@ class ContinuousQueryListenerBus(sparkListenerBus: LiveListenerBus)
* listener bus.
*/
private case class WrappedContinuousQueryListenerEvent(
- streamingListenerEvent: ContinuousQueryListener.Event) extends SparkListenerEvent {
+ streamingListenerEvent: ContinuousQueryListener.Event)
+ extends SparkListenerEvent {
// Do not log streaming events in event log as history server does not support these events.
protected[spark] override def logEvent: Boolean = false
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
index 5c86049..bc0e443 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
@@ -17,10 +17,11 @@
package org.apache.spark.sql.execution.streaming
-import org.apache.spark.sql.{InternalOutputModes, OutputMode, SparkSession}
+import org.apache.spark.sql.{InternalOutputModes, SparkSession}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.execution.{QueryExecution, SparkPlan, SparkPlanner, UnaryExecNode}
+import org.apache.spark.sql.streaming.OutputMode
/**
* A variant of [[QueryExecution]] that allows the execution of the given [[LogicalPlan]]
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
index ab0900d..16d38a2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
@@ -33,8 +33,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap}
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.execution.QueryExecution
-import org.apache.spark.sql.util.ContinuousQueryListener
-import org.apache.spark.sql.util.ContinuousQueryListener._
+import org.apache.spark.sql.streaming._
import org.apache.spark.util.{Clock, UninterruptibleThread, Utils}
/**
@@ -54,6 +53,8 @@ class StreamExecution(
val outputMode: OutputMode)
extends ContinuousQuery with Logging {
+ import org.apache.spark.sql.streaming.ContinuousQueryListener._
+
/**
* A lock used to wait/notify when batches complete. Use a fair lock to avoid thread starvation.
*/
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/TriggerExecutor.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/TriggerExecutor.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/TriggerExecutor.scala
index 569907b..ac510df 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/TriggerExecutor.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/TriggerExecutor.scala
@@ -18,7 +18,7 @@
package org.apache.spark.sql.execution.streaming
import org.apache.spark.internal.Logging
-import org.apache.spark.sql.ProcessingTime
+import org.apache.spark.sql.streaming.ProcessingTime
import org.apache.spark.util.{Clock, SystemClock}
trait TriggerExecutor {
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
index 391f1e5..2ec2a3c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
@@ -18,8 +18,9 @@
package org.apache.spark.sql.execution.streaming
import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{DataFrame, OutputMode, SQLContext}
+import org.apache.spark.sql.{DataFrame, SQLContext}
import org.apache.spark.sql.sources.{DataSourceRegister, StreamSinkProvider}
+import org.apache.spark.sql.streaming.OutputMode
class ConsoleSink(options: Map[String, String]) extends Sink with Logging {
// Number of rows to display, by default 20 rows
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
index e4a95e7..4496f41 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.encoders.encoderFor
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.LeafNode
+import org.apache.spark.sql.streaming.OutputMode
import org.apache.spark.sql.types.StructType
object MemoryStream {
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala
index 4c7bbf0..b2db377 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.command.AnalyzeTableCommand
import org.apache.spark.sql.execution.datasources.{DataSourceAnalysis, FindDataSourceTable, PreInsertCastAndRename, ResolveDataSource}
+import org.apache.spark.sql.streaming.{ContinuousQuery, ContinuousQueryManager}
import org.apache.spark.sql.util.ExecutionListenerManager
@@ -142,7 +143,7 @@ private[sql] class SessionState(sparkSession: SparkSession) {
lazy val listenerManager: ExecutionListenerManager = new ExecutionListenerManager
/**
- * Interface to start and stop [[org.apache.spark.sql.ContinuousQuery]]s.
+ * Interface to start and stop [[ContinuousQuery]]s.
*/
lazy val continuousQueryManager: ContinuousQueryManager = {
new ContinuousQueryManager(sparkSession)
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala
index 3d4edbb..d2077a0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.execution.streaming.{Sink, Source}
+import org.apache.spark.sql.streaming.OutputMode
import org.apache.spark.sql.types.StructType
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQuery.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQuery.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQuery.scala
new file mode 100644
index 0000000..451cfd8
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQuery.scala
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming
+
+import org.apache.spark.annotation.Experimental
+import org.apache.spark.sql.SparkSession
+
+/**
+ * :: Experimental ::
+ * A handle to a query that is executing continuously in the background as new data arrives.
+ * All these methods are thread-safe.
+ * @since 2.0.0
+ */
+@Experimental
+trait ContinuousQuery {
+
+ /**
+ * Returns the name of the query.
+ * @since 2.0.0
+ */
+ def name: String
+
+ /**
+ * Returns the [[SparkSession]] associated with `this`.
+ * @since 2.0.0
+ */
+ def sparkSession: SparkSession
+
+ /**
+ * Whether the query is currently active or not
+ * @since 2.0.0
+ */
+ def isActive: Boolean
+
+ /**
+ * Returns the [[ContinuousQueryException]] if the query was terminated by an exception.
+ * @since 2.0.0
+ */
+ def exception: Option[ContinuousQueryException]
+
+ /**
+ * Returns current status of all the sources.
+ * @since 2.0.0
+ */
+ def sourceStatuses: Array[SourceStatus]
+
+ /** Returns current status of the sink. */
+ def sinkStatus: SinkStatus
+
+ /**
+ * Waits for the termination of `this` query, either by `query.stop()` or by an exception.
+ * If the query has terminated with an exception, then the exception will be thrown.
+ *
+ * If the query has terminated, then all subsequent calls to this method will either return
+ * immediately (if the query was terminated by `stop()`), or throw the exception
+ * immediately (if the query has terminated with exception).
+ *
+ * @throws ContinuousQueryException, if `this` query has terminated with an exception.
+ *
+ * @since 2.0.0
+ */
+ def awaitTermination(): Unit
+
+ /**
+ * Waits for the termination of `this` query, either by `query.stop()` or by an exception.
+ * If the query has terminated with an exception, then the exception will be thrown.
+ * Otherwise, it returns whether the query has terminated or not within the `timeoutMs`
+ * milliseconds.
+ *
+ * If the query has terminated, then all subsequent calls to this method will either return
+ * `true` immediately (if the query was terminated by `stop()`), or throw the exception
+ * immediately (if the query has terminated with exception).
+ *
+ * @throws ContinuousQueryException, if `this` query has terminated with an exception
+ *
+ * @since 2.0.0
+ */
+ def awaitTermination(timeoutMs: Long): Boolean
+
+ /**
+ * Blocks until all available data in the source has been processed an committed to the sink.
+ * This method is intended for testing. Note that in the case of continually arriving data, this
+ * method may block forever. Additionally, this method is only guaranteed to block until data that
+ * has been synchronously appended data to a [[org.apache.spark.sql.execution.streaming.Source]]
+ * prior to invocation. (i.e. `getOffset` must immediately reflect the addition).
+ */
+ def processAllAvailable(): Unit
+
+ /**
+ * Stops the execution of this query if it is running. This method blocks until the threads
+ * performing execution has stopped.
+ * @since 2.0.0
+ */
+ def stop(): Unit
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQueryException.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQueryException.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQueryException.scala
new file mode 100644
index 0000000..5196c5a
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQueryException.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming
+
+import org.apache.spark.annotation.Experimental
+import org.apache.spark.sql.execution.streaming.{Offset, StreamExecution}
+
+/**
+ * :: Experimental ::
+ * Exception that stopped a [[ContinuousQuery]].
+ * @param query Query that caused the exception
+ * @param message Message of this exception
+ * @param cause Internal cause of this exception
+ * @param startOffset Starting offset (if known) of the range of data in which exception occurred
+ * @param endOffset Ending offset (if known) of the range of data in exception occurred
+ * @since 2.0.0
+ */
+@Experimental
+class ContinuousQueryException private[sql](
+ @transient val query: ContinuousQuery,
+ val message: String,
+ val cause: Throwable,
+ val startOffset: Option[Offset] = None,
+ val endOffset: Option[Offset] = None)
+ extends Exception(message, cause) {
+
+ /** Time when the exception occurred */
+ val time: Long = System.currentTimeMillis
+
+ override def toString(): String = {
+ val causeStr =
+ s"${cause.getMessage} ${cause.getStackTrace.take(10).mkString("", "\n|\t", "\n")}"
+ s"""
+ |$causeStr
+ |
+ |${query.asInstanceOf[StreamExecution].toDebugString}
+ """.stripMargin
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQueryListener.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQueryListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQueryListener.scala
new file mode 100644
index 0000000..6bdd513
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQueryListener.scala
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming
+
+import org.apache.spark.annotation.Experimental
+
+/**
+ * :: Experimental ::
+ * Interface for listening to events related to [[ContinuousQuery ContinuousQueries]].
+ * @note The methods are not thread-safe as they may be called from different threads.
+ *
+ * @since 2.0.0
+ */
+@Experimental
+abstract class ContinuousQueryListener {
+
+ import ContinuousQueryListener._
+
+ /**
+ * Called when a query is started.
+ * @note This is called synchronously with
+ * [[org.apache.spark.sql.DataFrameWriter `DataFrameWriter.startStream()`]],
+ * that is, `onQueryStart` will be called on all listeners before
+ * `DataFrameWriter.startStream()` returns the corresponding [[ContinuousQuery]]. Please
+ * don't block this method as it will block your query.
+ * @since 2.0.0
+ */
+ def onQueryStarted(queryStarted: QueryStarted): Unit
+
+ /**
+ * Called when there is some status update (ingestion rate updated, etc.)
+ *
+ * @note This method is asynchronous. The status in [[ContinuousQuery]] will always be
+ * latest no matter when this method is called. Therefore, the status of [[ContinuousQuery]]
+ * may be changed before/when you process the event. E.g., you may find [[ContinuousQuery]]
+ * is terminated when you are processing [[QueryProgress]].
+ * @since 2.0.0
+ */
+ def onQueryProgress(queryProgress: QueryProgress): Unit
+
+ /**
+ * Called when a query is stopped, with or without error.
+ * @since 2.0.0
+ */
+ def onQueryTerminated(queryTerminated: QueryTerminated): Unit
+}
+
+
+/**
+ * :: Experimental ::
+ * Companion object of [[ContinuousQueryListener]] that defines the listener events.
+ * @since 2.0.0
+ */
+@Experimental
+object ContinuousQueryListener {
+
+ /**
+ * Base type of [[ContinuousQueryListener]] events.
+ * @since 2.0.0
+ */
+ trait Event
+
+ /**
+ * Event representing the start of a query.
+ * @since 2.0.0
+ */
+ class QueryStarted private[sql](val query: ContinuousQuery) extends Event
+
+ /**
+ * Event representing any progress updates in a query.
+ * @since 2.0.0
+ */
+ class QueryProgress private[sql](val query: ContinuousQuery) extends Event
+
+ /**
+ * Event representing that termination of a query.
+ * @since 2.0.0
+ */
+ class QueryTerminated private[sql](val query: ContinuousQuery) extends Event
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQueryManager.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQueryManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQueryManager.scala
new file mode 100644
index 0000000..1bfdd2d
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ContinuousQueryManager.scala
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming
+
+import scala.collection.mutable
+
+import org.apache.spark.annotation.Experimental
+import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.catalyst.analysis.UnsupportedOperationChecker
+import org.apache.spark.sql.execution.streaming._
+import org.apache.spark.sql.execution.streaming.state.StateStoreCoordinatorRef
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.util.{Clock, SystemClock}
+
+/**
+ * :: Experimental ::
+ * A class to manage all the [[ContinuousQuery]] active on a [[SparkSession]].
+ *
+ * @since 2.0.0
+ */
+@Experimental
+class ContinuousQueryManager private[sql] (sparkSession: SparkSession) {
+
+ private[sql] val stateStoreCoordinator =
+ StateStoreCoordinatorRef.forDriver(sparkSession.sparkContext.env)
+ private val listenerBus = new ContinuousQueryListenerBus(sparkSession.sparkContext.listenerBus)
+ private val activeQueries = new mutable.HashMap[String, ContinuousQuery]
+ private val activeQueriesLock = new Object
+ private val awaitTerminationLock = new Object
+
+ private var lastTerminatedQuery: ContinuousQuery = null
+
+ /**
+ * Returns a list of active queries associated with this SQLContext
+ *
+ * @since 2.0.0
+ */
+ def active: Array[ContinuousQuery] = activeQueriesLock.synchronized {
+ activeQueries.values.toArray
+ }
+
+ /**
+ * Returns an active query from this SQLContext or throws exception if bad name
+ *
+ * @since 2.0.0
+ */
+ def get(name: String): ContinuousQuery = activeQueriesLock.synchronized {
+ activeQueries.getOrElse(name,
+ throw new IllegalArgumentException(s"There is no active query with name $name"))
+ }
+
+ /**
+ * Wait until any of the queries on the associated SQLContext has terminated since the
+ * creation of the context, or since `resetTerminated()` was called. If any query was terminated
+ * with an exception, then the exception will be thrown.
+ *
+ * If a query has terminated, then subsequent calls to `awaitAnyTermination()` will either
+ * return immediately (if the query was terminated by `query.stop()`),
+ * or throw the exception immediately (if the query was terminated with exception). Use
+ * `resetTerminated()` to clear past terminations and wait for new terminations.
+ *
+ * In the case where multiple queries have terminated since `resetTermination()` was called,
+ * if any query has terminated with exception, then `awaitAnyTermination()` will
+ * throw any of the exception. For correctly documenting exceptions across multiple queries,
+ * users need to stop all of them after any of them terminates with exception, and then check the
+ * `query.exception()` for each query.
+ *
+ * @throws ContinuousQueryException, if any query has terminated with an exception
+ *
+ * @since 2.0.0
+ */
+ def awaitAnyTermination(): Unit = {
+ awaitTerminationLock.synchronized {
+ while (lastTerminatedQuery == null) {
+ awaitTerminationLock.wait(10)
+ }
+ if (lastTerminatedQuery != null && lastTerminatedQuery.exception.nonEmpty) {
+ throw lastTerminatedQuery.exception.get
+ }
+ }
+ }
+
+ /**
+ * Wait until any of the queries on the associated SQLContext has terminated since the
+ * creation of the context, or since `resetTerminated()` was called. Returns whether any query
+ * has terminated or not (multiple may have terminated). If any query has terminated with an
+ * exception, then the exception will be thrown.
+ *
+ * If a query has terminated, then subsequent calls to `awaitAnyTermination()` will either
+ * return `true` immediately (if the query was terminated by `query.stop()`),
+ * or throw the exception immediately (if the query was terminated with exception). Use
+ * `resetTerminated()` to clear past terminations and wait for new terminations.
+ *
+ * In the case where multiple queries have terminated since `resetTermination()` was called,
+ * if any query has terminated with exception, then `awaitAnyTermination()` will
+ * throw any of the exception. For correctly documenting exceptions across multiple queries,
+ * users need to stop all of them after any of them terminates with exception, and then check the
+ * `query.exception()` for each query.
+ *
+ * @throws ContinuousQueryException, if any query has terminated with an exception
+ *
+ * @since 2.0.0
+ */
+ def awaitAnyTermination(timeoutMs: Long): Boolean = {
+
+ val startTime = System.currentTimeMillis
+ def isTimedout = System.currentTimeMillis - startTime >= timeoutMs
+
+ awaitTerminationLock.synchronized {
+ while (!isTimedout && lastTerminatedQuery == null) {
+ awaitTerminationLock.wait(10)
+ }
+ if (lastTerminatedQuery != null && lastTerminatedQuery.exception.nonEmpty) {
+ throw lastTerminatedQuery.exception.get
+ }
+ lastTerminatedQuery != null
+ }
+ }
+
+ /**
+ * Forget about past terminated queries so that `awaitAnyTermination()` can be used again to
+ * wait for new terminations.
+ *
+ * @since 2.0.0
+ */
+ def resetTerminated(): Unit = {
+ awaitTerminationLock.synchronized {
+ lastTerminatedQuery = null
+ }
+ }
+
+ /**
+ * Register a [[ContinuousQueryListener]] to receive up-calls for life cycle events of
+ * [[ContinuousQuery]].
+ *
+ * @since 2.0.0
+ */
+ def addListener(listener: ContinuousQueryListener): Unit = {
+ listenerBus.addListener(listener)
+ }
+
+ /**
+ * Deregister a [[ContinuousQueryListener]].
+ *
+ * @since 2.0.0
+ */
+ def removeListener(listener: ContinuousQueryListener): Unit = {
+ listenerBus.removeListener(listener)
+ }
+
+ /** Post a listener event */
+ private[sql] def postListenerEvent(event: ContinuousQueryListener.Event): Unit = {
+ listenerBus.post(event)
+ }
+
+ /** Start a query */
+ private[sql] def startQuery(
+ name: String,
+ checkpointLocation: String,
+ df: DataFrame,
+ sink: Sink,
+ outputMode: OutputMode,
+ trigger: Trigger = ProcessingTime(0),
+ triggerClock: Clock = new SystemClock()): ContinuousQuery = {
+ activeQueriesLock.synchronized {
+ if (activeQueries.contains(name)) {
+ throw new IllegalArgumentException(
+ s"Cannot start query with name $name as a query with that name is already active")
+ }
+ val analyzedPlan = df.queryExecution.analyzed
+ df.queryExecution.assertAnalyzed()
+
+ if (sparkSession.conf.get(SQLConf.UNSUPPORTED_OPERATION_CHECK_ENABLED)) {
+ UnsupportedOperationChecker.checkForStreaming(analyzedPlan, outputMode)
+ }
+
+ var nextSourceId = 0L
+
+ val logicalPlan = analyzedPlan.transform {
+ case StreamingRelation(dataSource, _, output) =>
+ // Materialize source to avoid creating it in every batch
+ val metadataPath = s"$checkpointLocation/sources/$nextSourceId"
+ val source = dataSource.createSource(metadataPath)
+ nextSourceId += 1
+ // We still need to use the previous `output` instead of `source.schema` as attributes in
+ // "df.logicalPlan" has already used attributes of the previous `output`.
+ StreamingExecutionRelation(source, output)
+ }
+ val query = new StreamExecution(
+ sparkSession,
+ name,
+ checkpointLocation,
+ logicalPlan,
+ sink,
+ trigger,
+ triggerClock,
+ outputMode)
+ query.start()
+ activeQueries.put(name, query)
+ query
+ }
+ }
+
+ /** Notify (by the ContinuousQuery) that the query has been terminated */
+ private[sql] def notifyQueryTermination(terminatedQuery: ContinuousQuery): Unit = {
+ activeQueriesLock.synchronized {
+ activeQueries -= terminatedQuery.name
+ }
+ awaitTerminationLock.synchronized {
+ if (lastTerminatedQuery == null || terminatedQuery.exception.nonEmpty) {
+ lastTerminatedQuery = terminatedQuery
+ }
+ awaitTerminationLock.notifyAll()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/streaming/SinkStatus.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/SinkStatus.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/SinkStatus.scala
new file mode 100644
index 0000000..79ddf01
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/SinkStatus.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming
+
+import org.apache.spark.annotation.Experimental
+import org.apache.spark.sql.execution.streaming.{Offset, Sink}
+
+/**
+ * :: Experimental ::
+ * Status and metrics of a streaming [[Sink]].
+ *
+ * @param description Description of the source corresponding to this status
+ * @param offset Current offset up to which data has been written by the sink
+ * @since 2.0.0
+ */
+@Experimental
+class SinkStatus private[sql](
+ val description: String,
+ val offset: Offset)
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/streaming/SourceStatus.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/SourceStatus.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/SourceStatus.scala
new file mode 100644
index 0000000..8fccd5b
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/SourceStatus.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming
+
+import org.apache.spark.annotation.Experimental
+import org.apache.spark.sql.execution.streaming.{Offset, Source}
+
+/**
+ * :: Experimental ::
+ * Status and metrics of a streaming [[Source]].
+ *
+ * @param description Description of the source corresponding to this status
+ * @param offset Current offset of the source, if known
+ * @since 2.0.0
+ */
+@Experimental
+class SourceStatus private[sql] (
+ val description: String,
+ val offset: Option[Offset])
http://git-wip-us.apache.org/repos/asf/spark/blob/a71d1364/sql/core/src/main/scala/org/apache/spark/sql/streaming/Trigger.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/Trigger.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/Trigger.scala
new file mode 100644
index 0000000..d3fdbac
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/Trigger.scala
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming
+
+import java.util.concurrent.TimeUnit
+
+import scala.concurrent.duration.Duration
+
+import org.apache.commons.lang3.StringUtils
+
+import org.apache.spark.annotation.Experimental
+import org.apache.spark.unsafe.types.CalendarInterval
+
+/**
+ * :: Experimental ::
+ * Used to indicate how often results should be produced by a [[ContinuousQuery]].
+ *
+ * @since 2.0.0
+ */
+@Experimental
+sealed trait Trigger
+
+/**
+ * :: Experimental ::
+ * A trigger that runs a query periodically based on the processing time. If `interval` is 0,
+ * the query will run as fast as possible.
+ *
+ * Scala Example:
+ * {{{
+ * df.write.trigger(ProcessingTime("10 seconds"))
+ *
+ * import scala.concurrent.duration._
+ * df.write.trigger(ProcessingTime(10.seconds))
+ * }}}
+ *
+ * Java Example:
+ * {{{
+ * df.write.trigger(ProcessingTime.create("10 seconds"))
+ *
+ * import java.util.concurrent.TimeUnit
+ * df.write.trigger(ProcessingTime.create(10, TimeUnit.SECONDS))
+ * }}}
+ *
+ * @since 2.0.0
+ */
+@Experimental
+case class ProcessingTime(intervalMs: Long) extends Trigger {
+ require(intervalMs >= 0, "the interval of trigger should not be negative")
+}
+
+/**
+ * :: Experimental ::
+ * Used to create [[ProcessingTime]] triggers for [[ContinuousQuery]]s.
+ *
+ * @since 2.0.0
+ */
+@Experimental
+object ProcessingTime {
+
+ /**
+ * Create a [[ProcessingTime]]. If `interval` is 0, the query will run as fast as possible.
+ *
+ * Example:
+ * {{{
+ * df.write.trigger(ProcessingTime("10 seconds"))
+ * }}}
+ *
+ * @since 2.0.0
+ */
+ def apply(interval: String): ProcessingTime = {
+ if (StringUtils.isBlank(interval)) {
+ throw new IllegalArgumentException(
+ "interval cannot be null or blank.")
+ }
+ val cal = if (interval.startsWith("interval")) {
+ CalendarInterval.fromString(interval)
+ } else {
+ CalendarInterval.fromString("interval " + interval)
+ }
+ if (cal == null) {
+ throw new IllegalArgumentException(s"Invalid interval: $interval")
+ }
+ if (cal.months > 0) {
+ throw new IllegalArgumentException(s"Doesn't support month or year interval: $interval")
+ }
+ new ProcessingTime(cal.microseconds / 1000)
+ }
+
+ /**
+ * Create a [[ProcessingTime]]. If `interval` is 0, the query will run as fast as possible.
+ *
+ * Example:
+ * {{{
+ * import scala.concurrent.duration._
+ * df.write.trigger(ProcessingTime(10.seconds))
+ * }}}
+ *
+ * @since 2.0.0
+ */
+ def apply(interval: Duration): ProcessingTime = {
+ new ProcessingTime(interval.toMillis)
+ }
+
+ /**
+ * Create a [[ProcessingTime]]. If `interval` is 0, the query will run as fast as possible.
+ *
+ * Example:
+ * {{{
+ * df.write.trigger(ProcessingTime.create("10 seconds"))
+ * }}}
+ *
+ * @since 2.0.0
+ */
+ def create(interval: String): ProcessingTime = {
+ apply(interval)
+ }
+
+ /**
+ * Create a [[ProcessingTime]]. If `interval` is 0, the query will run as fast as possible.
+ *
+ * Example:
+ * {{{
+ * import java.util.concurrent.TimeUnit
+ * df.write.trigger(ProcessingTime.create(10, TimeUnit.SECONDS))
+ * }}}
+ *
+ * @since 2.0.0
+ */
+ def create(interval: Long, unit: TimeUnit): ProcessingTime = {
+ new ProcessingTime(unit.toMillis(interval))
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org