You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/04/22 20:23:41 UTC
[4/4] git commit: [Fix #274] Document + fix annotation usages
[Fix #274] Document + fix annotation usages
... so that we don't follow an unspoken set of forbidden rules for adding **@AlphaComponent**, **@DeveloperApi**, and **@Experimental** annotations in the code.
In addition, this PR
(1) removes unnecessary `:: * ::` tags,
(2) adds missing `:: * ::` tags, and
(3) removes annotations for internal APIs.
Author: Andrew Or <an...@gmail.com>
Closes #470 from andrewor14/annotations-fix and squashes the following commits:
92a7f42 [Andrew Or] Document + fix annotation usages
(cherry picked from commit b3e5366f696c463f1c2f033b0d5c7365e5d6b0f8)
Signed-off-by: Patrick Wendell <pw...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/898fc348
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/898fc348
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/898fc348
Branch: refs/heads/branch-1.0
Commit: 898fc3480c408a7e88fa95f27ca8f72d660cdff7
Parents: 61d7401
Author: Andrew Or <an...@gmail.com>
Authored: Mon Apr 21 22:24:44 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Tue Apr 22 11:23:18 2014 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/SparkContext.scala | 1 +
.../scala/org/apache/spark/annotation/AlphaComponent.java | 9 ++++++++-
.../scala/org/apache/spark/annotation/DeveloperApi.java | 5 +++++
.../scala/org/apache/spark/annotation/Experimental.java | 5 +++++
core/src/main/scala/org/apache/spark/rdd/RDD.scala | 6 ------
.../main/scala/org/apache/spark/scheduler/SplitInfo.scala | 8 ++++++--
.../scala/org/apache/spark/mllib/clustering/KMeans.scala | 6 +++---
.../org/apache/spark/mllib/regression/RegressionModel.scala | 3 ---
.../src/main/scala/org/apache/spark/sql/SQLContext.scala | 2 --
.../src/main/scala/org/apache/spark/sql/SchemaRDD.scala | 2 +-
.../main/scala/org/apache/spark/sql/hive/HiveContext.scala | 1 -
11 files changed, 29 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/898fc348/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 1db14cb..25ca650 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -1110,6 +1110,7 @@ class SparkContext(config: SparkConf) extends Logging {
}
/**
+ * :: Experimental ::
* Submit a job for execution and return a FutureJob holding the result.
*/
@Experimental
http://git-wip-us.apache.org/repos/asf/spark/blob/898fc348/core/src/main/scala/org/apache/spark/annotation/AlphaComponent.java
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/annotation/AlphaComponent.java b/core/src/main/scala/org/apache/spark/annotation/AlphaComponent.java
index af01fb7..db7b25c 100644
--- a/core/src/main/scala/org/apache/spark/annotation/AlphaComponent.java
+++ b/core/src/main/scala/org/apache/spark/annotation/AlphaComponent.java
@@ -19,7 +19,14 @@ package org.apache.spark.annotation;
import java.lang.annotation.*;
-/** A new component of Spark which may have unstable API's. */
+/**
+ * A new component of Spark which may have unstable API's.
+ *
+ * NOTE: If there exists a Scaladoc comment that immediately precedes this annotation, the first
+ * line of the comment must be ":: AlphaComponent ::" with no trailing blank line. This is because
+ * of the known issue that Scaladoc displays only either the annotation or the comment, whichever
+ * comes first.
+ */
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.TYPE, ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER,
ElementType.CONSTRUCTOR, ElementType.LOCAL_VARIABLE, ElementType.PACKAGE})
http://git-wip-us.apache.org/repos/asf/spark/blob/898fc348/core/src/main/scala/org/apache/spark/annotation/DeveloperApi.java
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/annotation/DeveloperApi.java b/core/src/main/scala/org/apache/spark/annotation/DeveloperApi.java
index 5d546e7..0ecef6d 100644
--- a/core/src/main/scala/org/apache/spark/annotation/DeveloperApi.java
+++ b/core/src/main/scala/org/apache/spark/annotation/DeveloperApi.java
@@ -23,6 +23,11 @@ import java.lang.annotation.*;
* A lower-level, unstable API intended for developers.
*
* Developer API's might change or be removed in minor versions of Spark.
+ *
+ * NOTE: If there exists a Scaladoc comment that immediately precedes this annotation, the first
+ * line of the comment must be ":: DeveloperApi ::" with no trailing blank line. This is because
+ * of the known issue that Scaladoc displays only either the annotation or the comment, whichever
+ * comes first.
*/
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.TYPE, ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER,
http://git-wip-us.apache.org/repos/asf/spark/blob/898fc348/core/src/main/scala/org/apache/spark/annotation/Experimental.java
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/annotation/Experimental.java b/core/src/main/scala/org/apache/spark/annotation/Experimental.java
index 306b141..ff81202 100644
--- a/core/src/main/scala/org/apache/spark/annotation/Experimental.java
+++ b/core/src/main/scala/org/apache/spark/annotation/Experimental.java
@@ -24,6 +24,11 @@ import java.lang.annotation.*;
*
* Experimental API's might change or be removed in minor versions of Spark, or be adopted as
* first-class Spark API's.
+ *
+ * NOTE: If there exists a Scaladoc comment that immediately precedes this annotation, the first
+ * line of the comment must be ":: Experimental ::" with no trailing blank line. This is because
+ * of the known issue that Scaladoc displays only either the annotation or the comment, whichever
+ * comes first.
*/
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.TYPE, ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER,
http://git-wip-us.apache.org/repos/asf/spark/blob/898fc348/core/src/main/scala/org/apache/spark/rdd/RDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index 891efcc..5d2ed2b 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -94,26 +94,20 @@ abstract class RDD[T: ClassTag](
def compute(split: Partition, context: TaskContext): Iterator[T]
/**
- * :: DeveloperApi ::
* Implemented by subclasses to return the set of partitions in this RDD. This method will only
* be called once, so it is safe to implement a time-consuming computation in it.
*/
- @DeveloperApi
protected def getPartitions: Array[Partition]
/**
- * :: DeveloperApi ::
* Implemented by subclasses to return how this RDD depends on parent RDDs. This method will only
* be called once, so it is safe to implement a time-consuming computation in it.
*/
- @DeveloperApi
protected def getDependencies: Seq[Dependency[_]] = deps
/**
- * :: DeveloperApi ::
* Optionally overridden by subclasses to specify placement preferences.
*/
- @DeveloperApi
protected def getPreferredLocations(split: Partition): Seq[String] = Nil
/** Optionally overridden by subclasses to specify how they are partitioned. */
http://git-wip-us.apache.org/repos/asf/spark/blob/898fc348/core/src/main/scala/org/apache/spark/scheduler/SplitInfo.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SplitInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/SplitInfo.scala
index b85eabd..1ce8348 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SplitInfo.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SplitInfo.scala
@@ -24,8 +24,12 @@ import org.apache.spark.annotation.DeveloperApi
// information about a specific split instance : handles both split instances.
// So that we do not need to worry about the differences.
@DeveloperApi
-class SplitInfo(val inputFormatClazz: Class[_], val hostLocation: String, val path: String,
- val length: Long, val underlyingSplit: Any) {
+class SplitInfo(
+ val inputFormatClazz: Class[_],
+ val hostLocation: String,
+ val path: String,
+ val length: Long,
+ val underlyingSplit: Any) {
override def toString(): String = {
"SplitInfo " + super.toString + " .. inputFormatClazz " + inputFormatClazz +
", hostLocation : " + hostLocation + ", path : " + path +
http://git-wip-us.apache.org/repos/asf/spark/blob/898fc348/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
index 90cf852..dee9ef0 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
@@ -391,9 +391,9 @@ object KMeans {
* Returns the squared Euclidean distance between two vectors computed by
* [[org.apache.spark.mllib.util.MLUtils#fastSquaredDistance]].
*/
- private[clustering]
- def fastSquaredDistance(v1: BreezeVectorWithNorm, v2: BreezeVectorWithNorm)
- : Double = {
+ private[clustering] def fastSquaredDistance(
+ v1: BreezeVectorWithNorm,
+ v2: BreezeVectorWithNorm): Double = {
MLUtils.fastSquaredDistance(v1.vector, v1.norm, v2.vector, v2.norm)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/898fc348/mllib/src/main/scala/org/apache/spark/mllib/regression/RegressionModel.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/RegressionModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/RegressionModel.scala
index 027305a..b27e158 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/regression/RegressionModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/RegressionModel.scala
@@ -21,9 +21,6 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.mllib.linalg.Vector
import org.apache.spark.annotation.Experimental
-/**
- * :: Experimental ::
- */
@Experimental
trait RegressionModel extends Serializable {
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/898fc348/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 4d216b5..e25201a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -249,11 +249,9 @@ class SQLContext(@transient val sparkContext: SparkContext)
}
/**
- * :: DeveloperApi ::
* The primary workflow for executing relational queries using Spark. Designed to allow easy
* access to the intermediate phases of query execution for developers.
*/
- @DeveloperApi
protected abstract class QueryExecution {
def logical: LogicalPlan
http://git-wip-us.apache.org/repos/asf/spark/blob/898fc348/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
index f2ae5b0..6cb0e0f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql
import net.razorvine.pickle.Pickler
import org.apache.spark.{Dependency, OneToOneDependency, Partition, TaskContext}
-import org.apache.spark.annotation.{AlphaComponent, Experimental, DeveloperApi}
+import org.apache.spark.annotation.{AlphaComponent, Experimental}
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.expressions._
http://git-wip-us.apache.org/repos/asf/spark/blob/898fc348/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index c0d8adf..b21f24d 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -239,7 +239,6 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) {
sparkContext.parallelize(Seq(new GenericRow(Array[Any]()): Row), 1)
/** Extends QueryExecution with hive specific features. */
- @DeveloperApi
protected[sql] abstract class QueryExecution extends super.QueryExecution {
// TODO: Create mixin for the analyzer instead of overriding things here.
override lazy val optimizedPlan =