You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2016/10/10 18:29:13 UTC
spark git commit: [SPARK-17830] Annotate spark.sql package with
InterfaceStability
Repository: spark
Updated Branches:
refs/heads/master 4bafacaa5 -> 689de9200
[SPARK-17830] Annotate spark.sql package with InterfaceStability
## What changes were proposed in this pull request?
This patch annotates the InterfaceStability level for top level classes in o.a.spark.sql and o.a.spark.sql.util packages, to experiment with this new annotation.
## How was this patch tested?
N/A
Author: Reynold Xin <rx...@databricks.com>
Closes #15392 from rxin/SPARK-17830.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/689de920
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/689de920
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/689de920
Branch: refs/heads/master
Commit: 689de920056ae20fe203c2b6faf5b1462e8ea73c
Parents: 4bafaca
Author: Reynold Xin <rx...@databricks.com>
Authored: Mon Oct 10 11:29:09 2016 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Mon Oct 10 11:29:09 2016 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/Column.scala | 5 +++-
.../apache/spark/sql/DataFrameNaFunctions.scala | 3 +-
.../org/apache/spark/sql/DataFrameReader.scala | 2 ++
.../spark/sql/DataFrameStatFunctions.scala | 3 +-
.../org/apache/spark/sql/DataFrameWriter.scala | 3 +-
.../scala/org/apache/spark/sql/Dataset.scala | 29 +++++++++++++++++---
.../org/apache/spark/sql/DatasetHolder.scala | 3 ++
.../apache/spark/sql/ExperimentalMethods.scala | 5 ++--
.../org/apache/spark/sql/ForeachWriter.scala | 5 +++-
.../spark/sql/KeyValueGroupedDataset.scala | 3 +-
.../spark/sql/RelationalGroupedDataset.scala | 4 +--
.../org/apache/spark/sql/RuntimeConfig.scala | 2 ++
.../scala/org/apache/spark/sql/SQLContext.scala | 18 +++++++++++-
.../org/apache/spark/sql/SQLImplicits.scala | 2 ++
.../org/apache/spark/sql/SparkSession.scala | 23 +++++++++++++++-
.../org/apache/spark/sql/UDFRegistration.scala | 2 ++
.../scala/org/apache/spark/sql/functions.scala | 8 ++++--
.../scala/org/apache/spark/sql/package.scala | 5 ++--
.../spark/sql/util/QueryExecutionListener.scala | 4 ++-
19 files changed, 107 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 63da501..d22bb17 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql
import scala.language.implicitConversions
-import org.apache.spark.annotation.Experimental
+import org.apache.spark.annotation.{Experimental, InterfaceStability}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.encoders.{encoderFor, ExpressionEncoder}
@@ -59,6 +59,7 @@ private[sql] object Column {
*
* @since 1.6.0
*/
+@InterfaceStability.Stable
class TypedColumn[-T, U](
expr: Expression,
private[sql] val encoder: ExpressionEncoder[U])
@@ -124,6 +125,7 @@ class TypedColumn[-T, U](
*
* @since 1.3.0
*/
+@InterfaceStability.Stable
class Column(protected[sql] val expr: Expression) extends Logging {
def this(name: String) = this(name match {
@@ -1185,6 +1187,7 @@ class Column(protected[sql] val expr: Expression) extends Logging {
* @since 1.3.0
*/
@Experimental
+@InterfaceStability.Evolving
class ColumnName(name: String) extends Column(name) {
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
index ad00966..65a9c00 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
@@ -21,7 +21,7 @@ import java.{lang => jl}
import scala.collection.JavaConverters._
-import org.apache.spark.annotation.Experimental
+import org.apache.spark.annotation.{Experimental, InterfaceStability}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.functions._
import org.apache.spark.sql.types._
@@ -34,6 +34,7 @@ import org.apache.spark.sql.types._
* @since 1.3.1
*/
@Experimental
+@InterfaceStability.Evolving
final class DataFrameNaFunctions private[sql](df: DataFrame) {
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index b84fb2f..b54e695 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -24,6 +24,7 @@ import scala.collection.JavaConverters._
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.internal.Logging
import org.apache.spark.Partition
+import org.apache.spark.annotation.InterfaceStability
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.json.{JacksonParser, JSONOptions}
import org.apache.spark.sql.execution.LogicalRDD
@@ -38,6 +39,7 @@ import org.apache.spark.sql.types.StructType
*
* @since 1.4.0
*/
+@InterfaceStability.Stable
class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/DataFrameStatFunctions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameStatFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameStatFunctions.scala
index d69be36..a212bb6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameStatFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameStatFunctions.scala
@@ -21,7 +21,7 @@ import java.{lang => jl, util => ju}
import scala.collection.JavaConverters._
-import org.apache.spark.annotation.Experimental
+import org.apache.spark.annotation.{Experimental, InterfaceStability}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.execution.stat._
import org.apache.spark.sql.types._
@@ -34,6 +34,7 @@ import org.apache.spark.util.sketch.{BloomFilter, CountMinSketch}
* @since 1.4.0
*/
@Experimental
+@InterfaceStability.Evolving
final class DataFrameStatFunctions private[sql](df: DataFrame) {
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 7374a8e..35ef050 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -21,12 +21,12 @@ import java.util.Properties
import scala.collection.JavaConverters._
+import org.apache.spark.annotation.InterfaceStability
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType}
import org.apache.spark.sql.catalyst.plans.logical.InsertIntoTable
import org.apache.spark.sql.execution.datasources.{CaseInsensitiveMap, CreateTable, DataSource, HadoopFsRelation}
-import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcUtils}
import org.apache.spark.sql.types.StructType
/**
@@ -35,6 +35,7 @@ import org.apache.spark.sql.types.StructType
*
* @since 1.4.0
*/
+@InterfaceStability.Stable
final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
private val df = ds.toDF()
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 4b52508..30349ba 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -26,7 +26,7 @@ import scala.util.control.NonFatal
import org.apache.commons.lang3.StringUtils
-import org.apache.spark.annotation.{DeveloperApi, Experimental}
+import org.apache.spark.annotation.{DeveloperApi, Experimental, InterfaceStability}
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.api.java.function._
import org.apache.spark.api.python.{PythonRDD, SerDeUtil}
@@ -149,9 +149,10 @@ private[sql] object Dataset {
*
* @since 1.6.0
*/
+@InterfaceStability.Stable
class Dataset[T] private[sql](
@transient val sparkSession: SparkSession,
- @DeveloperApi @transient val queryExecution: QueryExecution,
+ @DeveloperApi @InterfaceStability.Unstable @transient val queryExecution: QueryExecution,
encoder: Encoder[T])
extends Serializable {
@@ -369,6 +370,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def as[U : Encoder]: Dataset[U] = Dataset[U](sparkSession, logicalPlan)
/**
@@ -477,6 +479,7 @@ class Dataset[T] private[sql](
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def isStreaming: Boolean = logicalPlan.isStreaming
/**
@@ -798,6 +801,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def joinWith[U](other: Dataset[U], condition: Column, joinType: String): Dataset[(T, U)] = {
// Creates a Join node and resolve it first, to get join condition resolved, self-join resolved,
// etc.
@@ -869,6 +873,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def joinWith[U](other: Dataset[U], condition: Column): Dataset[(T, U)] = {
joinWith(other, condition, "inner")
}
@@ -1071,6 +1076,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def select[U1](c1: TypedColumn[T, U1]): Dataset[U1] = {
implicit val encoder = c1.encoder
val project = Project(c1.withInputType(exprEnc, logicalPlan.output).named :: Nil,
@@ -1105,6 +1111,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def select[U1, U2](c1: TypedColumn[T, U1], c2: TypedColumn[T, U2]): Dataset[(U1, U2)] =
selectUntyped(c1, c2).asInstanceOf[Dataset[(U1, U2)]]
@@ -1116,6 +1123,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def select[U1, U2, U3](
c1: TypedColumn[T, U1],
c2: TypedColumn[T, U2],
@@ -1130,6 +1138,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def select[U1, U2, U3, U4](
c1: TypedColumn[T, U1],
c2: TypedColumn[T, U2],
@@ -1145,6 +1154,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def select[U1, U2, U3, U4, U5](
c1: TypedColumn[T, U1],
c2: TypedColumn[T, U2],
@@ -1315,6 +1325,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def reduce(func: (T, T) => T): T = rdd.reduce(func)
/**
@@ -1327,6 +1338,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def reduce(func: ReduceFunction[T]): T = reduce(func.call(_, _))
/**
@@ -1338,6 +1350,7 @@ class Dataset[T] private[sql](
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def groupByKey[K: Encoder](func: T => K): KeyValueGroupedDataset[K, T] = {
val inputPlan = logicalPlan
val withGroupingKey = AppendColumns(func, inputPlan)
@@ -1360,6 +1373,7 @@ class Dataset[T] private[sql](
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def groupByKey[K](func: MapFunction[T, K], encoder: Encoder[K]): KeyValueGroupedDataset[K, T] =
groupByKey(func.call(_))(encoder)
@@ -2028,6 +2042,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def filter(func: T => Boolean): Dataset[T] = {
withTypedPlan(TypedFilter(func, logicalPlan))
}
@@ -2041,6 +2056,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def filter(func: FilterFunction[T]): Dataset[T] = {
withTypedPlan(TypedFilter(func, logicalPlan))
}
@@ -2054,6 +2070,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def map[U : Encoder](func: T => U): Dataset[U] = withTypedPlan {
MapElements[T, U](func, logicalPlan)
}
@@ -2067,6 +2084,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def map[U](func: MapFunction[T, U], encoder: Encoder[U]): Dataset[U] = {
implicit val uEnc = encoder
withTypedPlan(MapElements[T, U](func, logicalPlan))
@@ -2081,6 +2099,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def mapPartitions[U : Encoder](func: Iterator[T] => Iterator[U]): Dataset[U] = {
new Dataset[U](
sparkSession,
@@ -2097,6 +2116,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def mapPartitions[U](f: MapPartitionsFunction[T, U], encoder: Encoder[U]): Dataset[U] = {
val func: (Iterator[T]) => Iterator[U] = x => f.call(x.asJava).asScala
mapPartitions(func)(encoder)
@@ -2127,6 +2147,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def flatMap[U : Encoder](func: T => TraversableOnce[U]): Dataset[U] =
mapPartitions(_.flatMap(func))
@@ -2140,6 +2161,7 @@ class Dataset[T] private[sql](
* @since 1.6.0
*/
@Experimental
+ @InterfaceStability.Evolving
def flatMap[U](f: FlatMapFunction[T, U], encoder: Encoder[U]): Dataset[U] = {
val func: (T) => Iterator[U] = x => f.call(x).asScala
flatMap(func)(encoder)
@@ -2505,13 +2527,11 @@ class Dataset[T] private[sql](
}
/**
- * :: Experimental ::
* Interface for saving the content of the non-streaming Dataset out into external storage.
*
* @group basic
* @since 1.6.0
*/
- @Experimental
def write: DataFrameWriter[T] = {
if (isStreaming) {
logicalPlan.failAnalysis(
@@ -2528,6 +2548,7 @@ class Dataset[T] private[sql](
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def writeStream: DataStreamWriter[T] = {
if (!isStreaming) {
logicalPlan.failAnalysis(
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/DatasetHolder.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DatasetHolder.scala b/sql/core/src/main/scala/org/apache/spark/sql/DatasetHolder.scala
index 47b81c1..18bccee 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DatasetHolder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DatasetHolder.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql
+import org.apache.spark.annotation.InterfaceStability
+
/**
* A container for a [[Dataset]], used for implicit conversions in Scala.
*
@@ -27,6 +29,7 @@ package org.apache.spark.sql
*
* @since 1.6.0
*/
+@InterfaceStability.Stable
case class DatasetHolder[T] private[sql](private val ds: Dataset[T]) {
// This is declared with parentheses to prevent the Scala compiler from treating
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/ExperimentalMethods.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/ExperimentalMethods.scala b/sql/core/src/main/scala/org/apache/spark/sql/ExperimentalMethods.scala
index a435734..1e8ba51 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/ExperimentalMethods.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/ExperimentalMethods.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql
-import org.apache.spark.annotation.Experimental
+import org.apache.spark.annotation.{Experimental, InterfaceStability}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.rules.Rule
@@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.rules.Rule
* @since 1.3.0
*/
@Experimental
+@InterfaceStability.Unstable
class ExperimentalMethods private[sql]() {
/**
@@ -41,10 +42,8 @@ class ExperimentalMethods private[sql]() {
*
* @since 1.3.0
*/
- @Experimental
@volatile var extraStrategies: Seq[Strategy] = Nil
- @Experimental
@volatile var extraOptimizations: Seq[Rule[LogicalPlan]] = Nil
}
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/ForeachWriter.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/ForeachWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/ForeachWriter.scala
index f56b25b..1163035 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/ForeachWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/ForeachWriter.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql
-import org.apache.spark.annotation.Experimental
+import org.apache.spark.annotation.{Experimental, InterfaceStability}
import org.apache.spark.sql.streaming.StreamingQuery
/**
@@ -68,8 +68,11 @@ import org.apache.spark.sql.streaming.StreamingQuery
* @since 2.0.0
*/
@Experimental
+@InterfaceStability.Evolving
abstract class ForeachWriter[T] extends Serializable {
+ // TODO: Move this to org.apache.spark.sql.util or consolidate this with batch API.
+
/**
* Called when starting to process one partition of new data in the executor. The `version` is
* for data deduplication when there are failures. When recovering from a failure, some data may
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
index cea16fb..828eb94 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql
import scala.collection.JavaConverters._
-import org.apache.spark.annotation.Experimental
+import org.apache.spark.annotation.{Experimental, InterfaceStability}
import org.apache.spark.api.java.function._
import org.apache.spark.sql.catalyst.encoders.{encoderFor, ExpressionEncoder}
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, CreateStruct}
@@ -36,6 +36,7 @@ import org.apache.spark.sql.expressions.ReduceAggregator
* @since 2.0.0
*/
@Experimental
+@InterfaceStability.Evolving
class KeyValueGroupedDataset[K, V] private[sql](
kEncoder: Encoder[K],
vEncoder: Encoder[V],
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
index 6c3fe07..f019d1e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
@@ -20,10 +20,9 @@ package org.apache.spark.sql
import scala.collection.JavaConverters._
import scala.language.implicitConversions
+import org.apache.spark.annotation.InterfaceStability
import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.sql.api.r.SQLUtils._
import org.apache.spark.sql.catalyst.analysis.{Star, UnresolvedAlias, UnresolvedAttribute, UnresolvedFunction}
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate._
import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, FlatMapGroupsInR, Pivot}
@@ -43,6 +42,7 @@ import org.apache.spark.sql.types.StructType
*
* @since 2.0.0
*/
+@InterfaceStability.Stable
class RelationalGroupedDataset protected[sql](
df: DataFrame,
groupingExprs: Seq[Expression],
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
index 7e07e0c..c2baa74 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql
+import org.apache.spark.annotation.InterfaceStability
import org.apache.spark.internal.config.{ConfigEntry, OptionalConfigEntry}
import org.apache.spark.sql.internal.SQLConf
@@ -28,6 +29,7 @@ import org.apache.spark.sql.internal.SQLConf
*
* @since 2.0.0
*/
+@InterfaceStability.Stable
class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) {
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 2edf2e1..3c5cf03 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -24,7 +24,7 @@ import scala.collection.immutable
import scala.reflect.runtime.universe.TypeTag
import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.annotation.{DeveloperApi, Experimental}
+import org.apache.spark.annotation.{DeveloperApi, Experimental, InterfaceStability}
import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
import org.apache.spark.internal.Logging
import org.apache.spark.internal.config.ConfigEntry
@@ -55,6 +55,7 @@ import org.apache.spark.sql.util.ExecutionListenerManager
* @groupname Ungrouped Support functions for language integrated queries
* @since 1.0.0
*/
+@InterfaceStability.Stable
class SQLContext private[sql](val sparkSession: SparkSession)
extends Logging with Serializable {
@@ -95,6 +96,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* that listen for execution metrics.
*/
@Experimental
+ @InterfaceStability.Evolving
def listenerManager: ExecutionListenerManager = sparkSession.listenerManager
/**
@@ -166,6 +168,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
*/
@Experimental
@transient
+ @InterfaceStability.Unstable
def experimental: ExperimentalMethods = sparkSession.experimental
/**
@@ -261,6 +264,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @since 1.3.0
*/
@Experimental
+ @InterfaceStability.Evolving
object implicits extends SQLImplicits with Serializable {
protected override def _sqlContext: SQLContext = self
}
@@ -274,6 +278,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @since 1.3.0
*/
@Experimental
+ @InterfaceStability.Evolving
def createDataFrame[A <: Product : TypeTag](rdd: RDD[A]): DataFrame = {
sparkSession.createDataFrame(rdd)
}
@@ -286,6 +291,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @since 1.3.0
*/
@Experimental
+ @InterfaceStability.Evolving
def createDataFrame[A <: Product : TypeTag](data: Seq[A]): DataFrame = {
sparkSession.createDataFrame(data)
}
@@ -333,6 +339,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @since 1.3.0
*/
@DeveloperApi
+ @InterfaceStability.Evolving
def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame = {
sparkSession.createDataFrame(rowRDD, schema)
}
@@ -376,6 +383,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @group dataset
*/
@Experimental
+ @InterfaceStability.Evolving
def createDataset[T : Encoder](data: Seq[T]): Dataset[T] = {
sparkSession.createDataset(data)
}
@@ -413,6 +421,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @group dataset
*/
@Experimental
+ @InterfaceStability.Evolving
def createDataset[T : Encoder](data: java.util.List[T]): Dataset[T] = {
sparkSession.createDataset(data)
}
@@ -436,6 +445,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @since 1.3.0
*/
@DeveloperApi
+ @InterfaceStability.Evolving
def createDataFrame(rowRDD: JavaRDD[Row], schema: StructType): DataFrame = {
sparkSession.createDataFrame(rowRDD, schema)
}
@@ -450,6 +460,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @since 1.6.0
*/
@DeveloperApi
+ @InterfaceStability.Evolving
def createDataFrame(rows: java.util.List[Row], schema: StructType): DataFrame = {
sparkSession.createDataFrame(rows, schema)
}
@@ -515,6 +526,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def readStream: DataStreamReader = sparkSession.readStream
@@ -632,6 +644,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @group dataframe
*/
@Experimental
+ @InterfaceStability.Evolving
def range(end: Long): DataFrame = sparkSession.range(end).toDF()
/**
@@ -643,6 +656,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @group dataframe
*/
@Experimental
+ @InterfaceStability.Evolving
def range(start: Long, end: Long): DataFrame = sparkSession.range(start, end).toDF()
/**
@@ -654,6 +668,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @group dataframe
*/
@Experimental
+ @InterfaceStability.Evolving
def range(start: Long, end: Long, step: Long): DataFrame = {
sparkSession.range(start, end, step).toDF()
}
@@ -668,6 +683,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @group dataframe
*/
@Experimental
+ @InterfaceStability.Evolving
def range(start: Long, end: Long, step: Long, numPartitions: Int): DataFrame = {
sparkSession.range(start, end, step, numPartitions).toDF()
}
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
index 4409525..73d16d8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql
import scala.language.implicitConversions
import scala.reflect.runtime.universe.TypeTag
+import org.apache.spark.annotation.InterfaceStability
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
@@ -28,6 +29,7 @@ import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
*
* @since 1.6.0
*/
+@InterfaceStability.Evolving
abstract class SQLImplicits {
protected def _sqlContext: SQLContext
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 6d7ac0f..d26eea5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -26,7 +26,7 @@ import scala.reflect.runtime.universe.TypeTag
import scala.util.control.NonFatal
import org.apache.spark.{SPARK_VERSION, SparkConf, SparkContext}
-import org.apache.spark.annotation.{DeveloperApi, Experimental}
+import org.apache.spark.annotation.{DeveloperApi, Experimental, InterfaceStability}
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.internal.Logging
import org.apache.spark.internal.config.CATALOG_IMPLEMENTATION
@@ -68,6 +68,7 @@ import org.apache.spark.util.Utils
* .getOrCreate()
* }}}
*/
+@InterfaceStability.Stable
class SparkSession private(
@transient val sparkContext: SparkContext,
@transient private val existingSharedState: Option[SharedState])
@@ -137,6 +138,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def listenerManager: ExecutionListenerManager = sessionState.listenerManager
/**
@@ -147,6 +149,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Unstable
def experimental: ExperimentalMethods = sessionState.experimentalMethods
/**
@@ -190,6 +193,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Unstable
def streams: StreamingQueryManager = sessionState.streamingQueryManager
/**
@@ -229,6 +233,7 @@ class SparkSession private(
* @return 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def emptyDataset[T: Encoder]: Dataset[T] = {
val encoder = implicitly[Encoder[T]]
new Dataset(self, LocalRelation(encoder.schema.toAttributes), encoder)
@@ -241,6 +246,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def createDataFrame[A <: Product : TypeTag](rdd: RDD[A]): DataFrame = {
SparkSession.setActiveSession(this)
val encoder = Encoders.product[A]
@@ -254,6 +260,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def createDataFrame[A <: Product : TypeTag](data: Seq[A]): DataFrame = {
SparkSession.setActiveSession(this)
val schema = ScalaReflection.schemaFor[A].dataType.asInstanceOf[StructType]
@@ -293,6 +300,7 @@ class SparkSession private(
* @since 2.0.0
*/
@DeveloperApi
+ @InterfaceStability.Evolving
def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame = {
createDataFrame(rowRDD, schema, needsConversion = true)
}
@@ -306,6 +314,7 @@ class SparkSession private(
* @since 2.0.0
*/
@DeveloperApi
+ @InterfaceStability.Evolving
def createDataFrame(rowRDD: JavaRDD[Row], schema: StructType): DataFrame = {
createDataFrame(rowRDD.rdd, schema)
}
@@ -319,6 +328,7 @@ class SparkSession private(
* @since 2.0.0
*/
@DeveloperApi
+ @InterfaceStability.Evolving
def createDataFrame(rows: java.util.List[Row], schema: StructType): DataFrame = {
Dataset.ofRows(self, LocalRelation.fromExternalRows(schema.toAttributes, rows.asScala))
}
@@ -410,6 +420,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def createDataset[T : Encoder](data: Seq[T]): Dataset[T] = {
val enc = encoderFor[T]
val attributes = enc.schema.toAttributes
@@ -428,6 +439,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def createDataset[T : Encoder](data: RDD[T]): Dataset[T] = {
Dataset[T](self, ExternalRDD(data, self))
}
@@ -449,6 +461,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def createDataset[T : Encoder](data: java.util.List[T]): Dataset[T] = {
createDataset(data.asScala)
}
@@ -461,6 +474,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def range(end: Long): Dataset[java.lang.Long] = range(0, end)
/**
@@ -471,6 +485,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def range(start: Long, end: Long): Dataset[java.lang.Long] = {
range(start, end, step = 1, numPartitions = sparkContext.defaultParallelism)
}
@@ -483,6 +498,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def range(start: Long, end: Long, step: Long): Dataset[java.lang.Long] = {
range(start, end, step, numPartitions = sparkContext.defaultParallelism)
}
@@ -496,6 +512,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def range(start: Long, end: Long, step: Long, numPartitions: Int): Dataset[java.lang.Long] = {
new Dataset(self, Range(start, end, step, numPartitions), Encoders.LONG)
}
@@ -596,6 +613,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def readStream: DataStreamReader = new DataStreamReader(self)
@@ -614,6 +632,7 @@ class SparkSession private(
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
object implicits extends SQLImplicits with Serializable {
protected override def _sqlContext: SQLContext = SparkSession.this.sqlContext
}
@@ -670,11 +689,13 @@ class SparkSession private(
}
+@InterfaceStability.Stable
object SparkSession {
/**
* Builder for [[SparkSession]].
*/
+ @InterfaceStability.Stable
class Builder extends Logging {
private[this] val options = new scala.collection.mutable.HashMap[String, String]
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
index b006236..617a147 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql
import scala.reflect.runtime.universe.TypeTag
import scala.util.Try
+import org.apache.spark.annotation.InterfaceStability
import org.apache.spark.internal.Logging
import org.apache.spark.sql.api.java._
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
@@ -36,6 +37,7 @@ import org.apache.spark.sql.types.DataType
*
* @since 1.3.0
*/
+@InterfaceStability.Stable
class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends Logging {
protected[sql] def registerPython(name: String, udf: UserDefinedPythonFunction): Unit = {
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 40f82d8..de49431 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -22,7 +22,7 @@ import scala.language.implicitConversions
import scala.reflect.runtime.universe.{typeTag, TypeTag}
import scala.util.Try
-import org.apache.spark.annotation.Experimental
+import org.apache.spark.annotation.{Experimental, InterfaceStability}
import org.apache.spark.sql.catalyst.ScalaReflection
import org.apache.spark.sql.catalyst.analysis.{Star, UnresolvedFunction}
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
@@ -38,7 +38,7 @@ import org.apache.spark.util.Utils
/**
* :: Experimental ::
- * Functions available for [[DataFrame]].
+ * Functions available for DataFrame operations.
*
* @groupname udf_funcs UDF functions
* @groupname agg_funcs Aggregate functions
@@ -54,6 +54,7 @@ import org.apache.spark.util.Utils
* @since 1.3.0
*/
@Experimental
+@InterfaceStability.Evolving
// scalastyle:off
object functions {
// scalastyle:on
@@ -2730,6 +2731,7 @@ object functions {
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def window(
timeColumn: Column,
windowDuration: String,
@@ -2783,6 +2785,7 @@ object functions {
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def window(timeColumn: Column, windowDuration: String, slideDuration: String): Column = {
window(timeColumn, windowDuration, slideDuration, "0 second")
}
@@ -2821,6 +2824,7 @@ object functions {
* @since 2.0.0
*/
@Experimental
+ @InterfaceStability.Evolving
def window(timeColumn: Column, windowDuration: String): Column = {
window(timeColumn, windowDuration, windowDuration, "0 second")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/package.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/package.scala
index 28d8bc3..161e010 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/package.scala
@@ -17,8 +17,8 @@
package org.apache.spark
-import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.sql.execution.{SparkPlan, SparkStrategy}
+import org.apache.spark.annotation.{DeveloperApi, InterfaceStability}
+import org.apache.spark.sql.execution.SparkStrategy
/**
* Allows the execution of relational queries, including those expressed in SQL using Spark.
@@ -40,6 +40,7 @@ package object sql {
* [[org.apache.spark.sql.sources]]
*/
@DeveloperApi
+ @InterfaceStability.Unstable
type Strategy = SparkStrategy
type DataFrame = Dataset[Row]
http://git-wip-us.apache.org/repos/asf/spark/blob/689de920/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
index 3cae535..5e93fc4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock
import scala.collection.mutable.ListBuffer
import scala.util.control.NonFatal
-import org.apache.spark.annotation.{DeveloperApi, Experimental}
+import org.apache.spark.annotation.{DeveloperApi, Experimental, InterfaceStability}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.execution.QueryExecution
@@ -34,6 +34,7 @@ import org.apache.spark.sql.execution.QueryExecution
* multiple different threads.
*/
@Experimental
+@InterfaceStability.Evolving
trait QueryExecutionListener {
/**
@@ -68,6 +69,7 @@ trait QueryExecutionListener {
* Manager for [[QueryExecutionListener]]. See [[org.apache.spark.sql.SQLContext.listenerManager]].
*/
@Experimental
+@InterfaceStability.Evolving
class ExecutionListenerManager private[sql] () extends Logging {
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org