You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2016/11/25 11:27:17 UTC
[3/3] spark git commit: [SPARK-3359][BUILD][DOCS] More changes to
resolve javadoc 8 errors that will help unidoc/genjavadoc compatibility
[SPARK-3359][BUILD][DOCS] More changes to resolve javadoc 8 errors that will help unidoc/genjavadoc compatibility
## What changes were proposed in this pull request?
This PR only tries to fix things that looks pretty straightforward and were fixed in other previous PRs before.
This PR roughly fixes several things as below:
- Fix unrecognisable class and method links in javadoc by changing it from `[[..]]` to `` `...` ``
```
[error] .../spark/sql/core/target/java/org/apache/spark/sql/streaming/DataStreamReader.java:226: error: reference not found
[error] * Loads text files and returns a {link DataFrame} whose schema starts with a string column named
```
- Fix an exception annotation and remove code backticks in `throws` annotation
Currently, sbt unidoc with Java 8 complains as below:
```
[error] .../java/org/apache/spark/sql/streaming/StreamingQuery.java:72: error: unexpected text
[error] * throws StreamingQueryException, if <code>this</code> query has terminated with an exception.
```
`throws` should specify the correct class name from `StreamingQueryException,` to `StreamingQueryException` without backticks. (see [JDK-8007644](https://bugs.openjdk.java.net/browse/JDK-8007644)).
- Fix `[[http..]]` to `<a href="http..."></a>`.
```diff
- * [[https://blogs.oracle.com/java-platform-group/entry/diagnosing_tls_ssl_and_https Oracle
- * blog page]].
+ * <a href="https://blogs.oracle.com/java-platform-group/entry/diagnosing_tls_ssl_and_https">
+ * Oracle blog page</a>.
```
`[[http...]]` link markdown in scaladoc is unrecognisable in javadoc.
- It seems class can't have `return` annotation. So, two cases of this were removed.
```
[error] .../java/org/apache/spark/mllib/regression/IsotonicRegression.java:27: error: invalid use of return
[error] * return New instance of IsotonicRegression.
```
- Fix < to `<` and > to `>` according to HTML rules.
- Fix `</p>` complaint
- Exclude unrecognisable in javadoc, `constructor`, `todo` and `groupname`.
## How was this patch tested?
Manually tested by `jekyll build` with Java 7 and 8
```
java version "1.7.0_80"
Java(TM) SE Runtime Environment (build 1.7.0_80-b15)
Java HotSpot(TM) 64-Bit Server VM (build 24.80-b11, mixed mode)
```
```
java version "1.8.0_45"
Java(TM) SE Runtime Environment (build 1.8.0_45-b14)
Java HotSpot(TM) 64-Bit Server VM (build 25.45-b02, mixed mode)
```
Note: this does not yet make sbt unidoc suceed with Java 8 yet but it reduces the number of errors with Java 8.
Author: hyukjinkwon <gu...@gmail.com>
Closes #15999 from HyukjinKwon/SPARK-3359-errors.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/51b1c155
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/51b1c155
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/51b1c155
Branch: refs/heads/master
Commit: 51b1c1551d3a7147403b9e821fcc7c8f57b4824c
Parents: f42db0c
Author: hyukjinkwon <gu...@gmail.com>
Authored: Fri Nov 25 11:27:07 2016 +0000
Committer: Sean Owen <so...@cloudera.com>
Committed: Fri Nov 25 11:27:07 2016 +0000
----------------------------------------------------------------------
.../scala/org/apache/spark/SSLOptions.scala | 4 +-
.../org/apache/spark/api/java/JavaPairRDD.scala | 6 +-
.../org/apache/spark/api/java/JavaRDD.scala | 10 +--
.../spark/api/java/JavaSparkContext.scala | 14 ++--
.../org/apache/spark/io/CompressionCodec.scala | 2 +-
.../main/scala/org/apache/spark/rdd/RDD.scala | 18 ++---
.../spark/security/CryptoStreamUtils.scala | 4 +-
.../spark/serializer/KryoSerializer.scala | 3 +-
.../spark/storage/BlockReplicationPolicy.scala | 7 +-
.../scala/org/apache/spark/ui/UIUtils.scala | 4 +-
.../org/apache/spark/util/AccumulatorV2.scala | 2 +-
.../scala/org/apache/spark/util/RpcUtils.scala | 2 +-
.../org/apache/spark/util/StatCounter.scala | 4 +-
.../org/apache/spark/util/ThreadUtils.scala | 6 +-
.../scala/org/apache/spark/util/Utils.scala | 10 +--
.../spark/util/io/ChunkedByteBuffer.scala | 2 +-
.../scala/org/apache/spark/graphx/Graph.scala | 4 +-
.../org/apache/spark/graphx/GraphLoader.scala | 2 +-
.../apache/spark/graphx/impl/EdgeRDDImpl.scala | 2 +-
.../org/apache/spark/graphx/lib/PageRank.scala | 4 +-
.../apache/spark/graphx/lib/SVDPlusPlus.scala | 3 +-
.../apache/spark/graphx/lib/TriangleCount.scala | 2 +-
.../distribution/MultivariateGaussian.scala | 3 +-
.../scala/org/apache/spark/ml/Predictor.scala | 2 +-
.../spark/ml/attribute/AttributeGroup.scala | 2 +-
.../apache/spark/ml/attribute/attributes.scala | 4 +-
.../ml/classification/LogisticRegression.scala | 74 ++++++++++----------
.../MultilayerPerceptronClassifier.scala | 1 -
.../spark/ml/classification/NaiveBayes.scala | 8 ++-
.../classification/RandomForestClassifier.scala | 6 +-
.../spark/ml/clustering/BisectingKMeans.scala | 14 ++--
.../spark/ml/clustering/ClusteringSummary.scala | 2 +-
.../spark/ml/clustering/GaussianMixture.scala | 6 +-
.../org/apache/spark/ml/clustering/KMeans.scala | 8 +--
.../org/apache/spark/ml/clustering/LDA.scala | 42 +++++------
.../scala/org/apache/spark/ml/feature/DCT.scala | 3 +-
.../org/apache/spark/ml/feature/MinHash.scala | 5 +-
.../apache/spark/ml/feature/MinMaxScaler.scala | 4 +-
.../spark/ml/feature/PolynomialExpansion.scala | 14 ++--
.../spark/ml/feature/RandomProjection.scala | 4 +-
.../spark/ml/feature/StandardScaler.scala | 4 +-
.../spark/ml/feature/StopWordsRemover.scala | 5 +-
.../org/apache/spark/ml/feature/package.scala | 3 +-
.../IterativelyReweightedLeastSquares.scala | 7 +-
.../spark/ml/param/shared/sharedParams.scala | 12 ++--
.../ml/regression/AFTSurvivalRegression.scala | 27 +++----
.../ml/regression/DecisionTreeRegressor.scala | 4 +-
.../spark/ml/regression/GBTRegressor.scala | 4 +-
.../GeneralizedLinearRegression.scala | 12 ++--
.../spark/ml/regression/LinearRegression.scala | 38 +++++-----
.../ml/regression/RandomForestRegressor.scala | 5 +-
.../ml/source/libsvm/LibSVMDataSource.scala | 13 ++--
.../ml/tree/impl/GradientBoostedTrees.scala | 10 +--
.../spark/ml/tree/impl/RandomForest.scala | 2 +-
.../org/apache/spark/ml/tree/treeParams.scala | 6 +-
.../apache/spark/ml/tuning/CrossValidator.scala | 4 +-
.../org/apache/spark/ml/util/ReadWrite.scala | 10 +--
.../spark/mllib/classification/NaiveBayes.scala | 28 ++++----
.../mllib/clustering/BisectingKMeans.scala | 21 +++---
.../mllib/clustering/BisectingKMeansModel.scala | 4 +-
.../mllib/clustering/GaussianMixture.scala | 6 +-
.../mllib/clustering/GaussianMixtureModel.scala | 2 +-
.../org/apache/spark/mllib/clustering/LDA.scala | 24 +++----
.../spark/mllib/clustering/LDAModel.scala | 2 +-
.../spark/mllib/clustering/LDAOptimizer.scala | 2 +-
.../clustering/PowerIterationClustering.scala | 13 ++--
.../mllib/clustering/StreamingKMeans.scala | 4 +-
.../mllib/evaluation/RegressionMetrics.scala | 10 +--
.../org/apache/spark/mllib/fpm/FPGrowth.scala | 12 ++--
.../org/apache/spark/mllib/fpm/PrefixSpan.scala | 7 +-
.../mllib/linalg/distributed/BlockMatrix.scala | 20 +++---
.../linalg/distributed/CoordinateMatrix.scala | 4 +-
.../linalg/distributed/IndexedRowMatrix.scala | 4 +-
.../mllib/linalg/distributed/RowMatrix.scala | 2 +-
.../spark/mllib/optimization/Gradient.scala | 24 +++----
.../mllib/optimization/GradientDescent.scala | 4 +-
.../apache/spark/mllib/optimization/LBFGS.scala | 7 +-
.../apache/spark/mllib/optimization/NNLS.scala | 2 +-
.../spark/mllib/optimization/Updater.scala | 6 +-
.../scala/org/apache/spark/mllib/package.scala | 4 +-
.../apache/spark/mllib/rdd/RDDFunctions.scala | 2 +-
.../apache/spark/mllib/recommendation/ALS.scala | 7 +-
.../MatrixFactorizationModel.scala | 6 +-
.../mllib/regression/IsotonicRegression.scala | 9 ++-
.../stat/MultivariateOnlineSummarizer.scala | 7 +-
.../apache/spark/mllib/stat/Statistics.scala | 11 +--
.../distribution/MultivariateGaussian.scala | 3 +-
.../spark/mllib/tree/GradientBoostedTrees.scala | 2 +-
.../apache/spark/mllib/tree/RandomForest.scala | 8 +--
.../apache/spark/mllib/tree/model/Split.scala | 2 +-
.../org/apache/spark/mllib/util/MLUtils.scala | 10 +--
.../apache/spark/mllib/util/modelSaveLoad.scala | 2 +-
pom.xml | 12 ++++
project/SparkBuild.scala | 5 +-
.../main/scala/org/apache/spark/sql/Row.scala | 2 +-
.../aggregate/CentralMomentAgg.scala | 4 +-
.../org/apache/spark/sql/types/BinaryType.scala | 2 +-
.../apache/spark/sql/types/BooleanType.scala | 2 +-
.../org/apache/spark/sql/types/ByteType.scala | 2 +-
.../spark/sql/types/CalendarIntervalType.scala | 2 +-
.../org/apache/spark/sql/types/DateType.scala | 2 +-
.../apache/spark/sql/types/DecimalType.scala | 4 +-
.../org/apache/spark/sql/types/DoubleType.scala | 2 +-
.../org/apache/spark/sql/types/FloatType.scala | 2 +-
.../apache/spark/sql/types/IntegerType.scala | 2 +-
.../org/apache/spark/sql/types/LongType.scala | 2 +-
.../org/apache/spark/sql/types/MapType.scala | 2 +-
.../org/apache/spark/sql/types/NullType.scala | 2 +-
.../org/apache/spark/sql/types/ShortType.scala | 2 +-
.../org/apache/spark/sql/types/StringType.scala | 2 +-
.../apache/spark/sql/types/TimestampType.scala | 2 +-
.../org/apache/spark/sql/DataFrameReader.scala | 17 ++---
.../spark/sql/DataFrameStatFunctions.scala | 16 +++--
.../org/apache/spark/sql/DataFrameWriter.scala | 4 +-
.../scala/org/apache/spark/sql/SQLContext.scala | 62 ++++++++--------
.../sql/execution/stat/FrequentItems.scala | 3 +-
.../sql/execution/stat/StatFunctions.scala | 4 +-
.../spark/sql/expressions/Aggregator.scala | 8 +--
.../sql/expressions/UserDefinedFunction.scala | 2 +-
.../apache/spark/sql/expressions/Window.scala | 16 ++---
.../spark/sql/expressions/WindowSpec.scala | 16 ++---
.../spark/sql/expressions/scalalang/typed.scala | 2 +-
.../org/apache/spark/sql/expressions/udaf.scala | 24 +++----
.../apache/spark/sql/jdbc/JdbcDialects.scala | 6 +-
.../spark/sql/streaming/DataStreamReader.scala | 20 +++---
.../spark/sql/streaming/DataStreamWriter.scala | 8 +--
.../spark/sql/streaming/StreamingQuery.scala | 10 +--
.../sql/streaming/StreamingQueryManager.scala | 8 ++-
.../spark/sql/util/QueryExecutionListener.scala | 2 +-
.../hive/execution/InsertIntoHiveTable.scala | 4 +-
.../spark/sql/hive/orc/OrcFileFormat.scala | 4 +-
.../spark/sql/hive/orc/OrcFileOperator.scala | 2 +-
132 files changed, 558 insertions(+), 499 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/SSLOptions.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SSLOptions.scala b/core/src/main/scala/org/apache/spark/SSLOptions.scala
index be19179..5f14102 100644
--- a/core/src/main/scala/org/apache/spark/SSLOptions.scala
+++ b/core/src/main/scala/org/apache/spark/SSLOptions.scala
@@ -150,8 +150,8 @@ private[spark] object SSLOptions extends Logging {
* $ - `[ns].enabledAlgorithms` - a comma separated list of ciphers
*
* For a list of protocols and ciphers supported by particular Java versions, you may go to
- * [[https://blogs.oracle.com/java-platform-group/entry/diagnosing_tls_ssl_and_https Oracle
- * blog page]].
+ * <a href="https://blogs.oracle.com/java-platform-group/entry/diagnosing_tls_ssl_and_https">
+ * Oracle blog page</a>.
*
* You can optionally specify the default configuration. If you do, for each setting which is
* missing in SparkConf, the corresponding setting is used from the default configuration.
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
index bff5a29..d7e3a1b 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
@@ -405,7 +405,7 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])
* partitioning of the resulting key-value pair RDD by passing a Partitioner.
*
* @note If you are grouping in order to perform an aggregation (such as a sum or average) over
- * each key, using [[JavaPairRDD.reduceByKey]] or [[JavaPairRDD.combineByKey]]
+ * each key, using `JavaPairRDD.reduceByKey` or `JavaPairRDD.combineByKey`
* will provide much better performance.
*/
def groupByKey(partitioner: Partitioner): JavaPairRDD[K, JIterable[V]] =
@@ -416,7 +416,7 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])
* resulting RDD with into `numPartitions` partitions.
*
* @note If you are grouping in order to perform an aggregation (such as a sum or average) over
- * each key, using [[JavaPairRDD.reduceByKey]] or [[JavaPairRDD.combineByKey]]
+ * each key, using `JavaPairRDD.reduceByKey` or `JavaPairRDD.combineByKey`
* will provide much better performance.
*/
def groupByKey(numPartitions: Int): JavaPairRDD[K, JIterable[V]] =
@@ -546,7 +546,7 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])
* resulting RDD with the existing partitioner/parallelism level.
*
* @note If you are grouping in order to perform an aggregation (such as a sum or average) over
- * each key, using [[JavaPairRDD.reduceByKey]] or [[JavaPairRDD.combineByKey]]
+ * each key, using `JavaPairRDD.reduceByKey` or `JavaPairRDD.combineByKey`
* will provide much better performance.
*/
def groupByKey(): JavaPairRDD[K, JIterable[V]] =
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
index ccd94f8..a20d264 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
@@ -103,10 +103,10 @@ class JavaRDD[T](val rdd: RDD[T])(implicit val classTag: ClassTag[T])
* @param withReplacement can elements be sampled multiple times (replaced when sampled out)
* @param fraction expected size of the sample as a fraction of this RDD's size
* without replacement: probability that each element is chosen; fraction must be [0, 1]
- * with replacement: expected number of times each element is chosen; fraction must be >= 0
+ * with replacement: expected number of times each element is chosen; fraction must be >= 0
*
* @note This is NOT guaranteed to provide exactly the fraction of the count
- * of the given [[RDD]].
+ * of the given `RDD`.
*/
def sample(withReplacement: Boolean, fraction: Double): JavaRDD[T] =
sample(withReplacement, fraction, Utils.random.nextLong)
@@ -117,11 +117,11 @@ class JavaRDD[T](val rdd: RDD[T])(implicit val classTag: ClassTag[T])
* @param withReplacement can elements be sampled multiple times (replaced when sampled out)
* @param fraction expected size of the sample as a fraction of this RDD's size
* without replacement: probability that each element is chosen; fraction must be [0, 1]
- * with replacement: expected number of times each element is chosen; fraction must be >= 0
+ * with replacement: expected number of times each element is chosen; fraction must be >= 0
* @param seed seed for the random number generator
*
* @note This is NOT guaranteed to provide exactly the fraction of the count
- * of the given [[RDD]].
+ * of the given `RDD`.
*/
def sample(withReplacement: Boolean, fraction: Double, seed: Long): JavaRDD[T] =
wrapRDD(rdd.sample(withReplacement, fraction, seed))
@@ -167,7 +167,7 @@ class JavaRDD[T](val rdd: RDD[T])(implicit val classTag: ClassTag[T])
* Return an RDD with the elements from `this` that are not in `other`.
*
* Uses `this` partitioner/partition size, because even if `other` is huge, the resulting
- * RDD will be <= us.
+ * RDD will be <= us.
*/
def subtract(other: JavaRDD[T]): JavaRDD[T] = wrapRDD(rdd.subtract(other))
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
index 38d347a..9481156 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
@@ -238,7 +238,9 @@ class JavaSparkContext(val sc: SparkContext)
* }}}
*
* Do
- * `JavaPairRDD<String, byte[]> rdd = sparkContext.dataStreamFiles("hdfs://a-hdfs-path")`,
+ * {{{
+ * JavaPairRDD<String, byte[]> rdd = sparkContext.dataStreamFiles("hdfs://a-hdfs-path")
+ * }}}
*
* then `rdd` contains
* {{{
@@ -270,7 +272,9 @@ class JavaSparkContext(val sc: SparkContext)
* }}}
*
* Do
- * `JavaPairRDD<String, byte[]> rdd = sparkContext.dataStreamFiles("hdfs://a-hdfs-path")`,
+ * {{{
+ * JavaPairRDD<String, byte[]> rdd = sparkContext.dataStreamFiles("hdfs://a-hdfs-path")
+ * }}},
*
* then `rdd` contains
* {{{
@@ -749,7 +753,7 @@ class JavaSparkContext(val sc: SparkContext)
/**
* Get a local property set in this thread, or null if it is missing. See
- * [[org.apache.spark.api.java.JavaSparkContext.setLocalProperty]].
+ * `org.apache.spark.api.java.JavaSparkContext.setLocalProperty`.
*/
def getLocalProperty(key: String): String = sc.getLocalProperty(key)
@@ -769,7 +773,7 @@ class JavaSparkContext(val sc: SparkContext)
* Application programmers can use this method to group all those jobs together and give a
* group description. Once set, the Spark web UI will associate such jobs with this group.
*
- * The application can also use [[org.apache.spark.api.java.JavaSparkContext.cancelJobGroup]]
+ * The application can also use `org.apache.spark.api.java.JavaSparkContext.cancelJobGroup`
* to cancel all running jobs in this group. For example,
* {{{
* // In the main thread:
@@ -802,7 +806,7 @@ class JavaSparkContext(val sc: SparkContext)
/**
* Cancel active jobs for the specified group. See
- * [[org.apache.spark.api.java.JavaSparkContext.setJobGroup]] for more information.
+ * `org.apache.spark.api.java.JavaSparkContext.setJobGroup` for more information.
*/
def cancelJobGroup(groupId: String): Unit = sc.cancelJobGroup(groupId)
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
index 6ba79e5..2e991ce 100644
--- a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
+++ b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
@@ -172,7 +172,7 @@ private final object SnappyCompressionCodec {
}
/**
- * Wrapper over [[SnappyOutputStream]] which guards against write-after-close and double-close
+ * Wrapper over `SnappyOutputStream` which guards against write-after-close and double-close
* issues. See SPARK-7660 for more details. This wrapping can be removed if we upgrade to a version
* of snappy-java that contains the fix for https://github.com/xerial/snappy-java/issues/107.
*/
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/rdd/RDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index bff2b8f..8e67344 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -70,8 +70,8 @@ import org.apache.spark.util.random.{BernoulliCellSampler, BernoulliSampler, Poi
* All of the scheduling and execution in Spark is done based on these methods, allowing each RDD
* to implement its own way of computing itself. Indeed, users can implement custom RDDs (e.g. for
* reading data from a new storage system) by overriding these functions. Please refer to the
- * [[http://people.csail.mit.edu/matei/papers/2012/nsdi_spark.pdf Spark paper]] for more details
- * on RDD internals.
+ * <a href="http://people.csail.mit.edu/matei/papers/2012/nsdi_spark.pdf">Spark paper</a>
+ * for more details on RDD internals.
*/
abstract class RDD[T: ClassTag](
@transient private var _sc: SparkContext,
@@ -469,7 +469,7 @@ abstract class RDD[T: ClassTag](
* @param withReplacement can elements be sampled multiple times (replaced when sampled out)
* @param fraction expected size of the sample as a fraction of this RDD's size
* without replacement: probability that each element is chosen; fraction must be [0, 1]
- * with replacement: expected number of times each element is chosen; fraction must be >= 0
+ * with replacement: expected number of times each element is chosen; fraction must be >= 0
* @param seed seed for the random number generator
*
* @note This is NOT guaranteed to provide exactly the fraction of the count
@@ -675,8 +675,8 @@ abstract class RDD[T: ClassTag](
* may even differ each time the resulting RDD is evaluated.
*
* @note This operation may be very expensive. If you are grouping in order to perform an
- * aggregation (such as a sum or average) over each key, using [[PairRDDFunctions.aggregateByKey]]
- * or [[PairRDDFunctions.reduceByKey]] will provide much better performance.
+ * aggregation (such as a sum or average) over each key, using `PairRDDFunctions.aggregateByKey`
+ * or `PairRDDFunctions.reduceByKey` will provide much better performance.
*/
def groupBy[K](f: T => K)(implicit kt: ClassTag[K]): RDD[(K, Iterable[T])] = withScope {
groupBy[K](f, defaultPartitioner(this))
@@ -688,8 +688,8 @@ abstract class RDD[T: ClassTag](
* may even differ each time the resulting RDD is evaluated.
*
* @note This operation may be very expensive. If you are grouping in order to perform an
- * aggregation (such as a sum or average) over each key, using [[PairRDDFunctions.aggregateByKey]]
- * or [[PairRDDFunctions.reduceByKey]] will provide much better performance.
+ * aggregation (such as a sum or average) over each key, using `PairRDDFunctions.aggregateByKey`
+ * or `PairRDDFunctions.reduceByKey` will provide much better performance.
*/
def groupBy[K](
f: T => K,
@@ -703,8 +703,8 @@ abstract class RDD[T: ClassTag](
* may even differ each time the resulting RDD is evaluated.
*
* @note This operation may be very expensive. If you are grouping in order to perform an
- * aggregation (such as a sum or average) over each key, using [[PairRDDFunctions.aggregateByKey]]
- * or [[PairRDDFunctions.reduceByKey]] will provide much better performance.
+ * aggregation (such as a sum or average) over each key, using `PairRDDFunctions.aggregateByKey`
+ * or `PairRDDFunctions.reduceByKey` will provide much better performance.
*/
def groupBy[K](f: T => K, p: Partitioner)(implicit kt: ClassTag[K], ord: Ordering[K] = null)
: RDD[(K, Iterable[T])] = withScope {
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala b/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala
index 8f15f50..f41fc38 100644
--- a/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala
+++ b/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala
@@ -46,7 +46,7 @@ private[spark] object CryptoStreamUtils extends Logging {
val COMMONS_CRYPTO_CONF_PREFIX = "commons.crypto."
/**
- * Helper method to wrap [[OutputStream]] with [[CryptoOutputStream]] for encryption.
+ * Helper method to wrap `OutputStream` with `CryptoOutputStream` for encryption.
*/
def createCryptoOutputStream(
os: OutputStream,
@@ -62,7 +62,7 @@ private[spark] object CryptoStreamUtils extends Logging {
}
/**
- * Helper method to wrap [[InputStream]] with [[CryptoInputStream]] for decryption.
+ * Helper method to wrap `InputStream` with `CryptoInputStream` for decryption.
*/
def createCryptoInputStream(
is: InputStream,
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index 19e020c..7eb2da1 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -43,7 +43,8 @@ import org.apache.spark.util.{BoundedPriorityQueue, SerializableConfiguration, S
import org.apache.spark.util.collection.CompactBuffer
/**
- * A Spark serializer that uses the [[https://code.google.com/p/kryo/ Kryo serialization library]].
+ * A Spark serializer that uses the <a href="https://code.google.com/p/kryo/">
+ * Kryo serialization library</a>.
*
* @note This serializer is not guaranteed to be wire-compatible across different versions of
* Spark. It is intended to be used to serialize/de-serialize data within a single
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/storage/BlockReplicationPolicy.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockReplicationPolicy.scala b/core/src/main/scala/org/apache/spark/storage/BlockReplicationPolicy.scala
index bf087af..bb8a684 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockReplicationPolicy.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockReplicationPolicy.scala
@@ -89,17 +89,18 @@ class RandomBlockReplicationPolicy
prioritizedPeers
}
+ // scalastyle:off line.size.limit
/**
* Uses sampling algorithm by Robert Floyd. Finds a random sample in O(n) while
- * minimizing space usage
- * [[http://math.stackexchange.com/questions/178690/
- * whats-the-proof-of-correctness-for-robert-floyds-algorithm-for-selecting-a-sin]]
+ * minimizing space usage. Please see <a href="http://math.stackexchange.com/questions/178690/whats-the-proof-of-correctness-for-robert-floyds-algorithm-for-selecting-a-sin">
+ * here</a>.
*
* @param n total number of indices
* @param m number of samples needed
* @param r random number generator
* @return list of m random unique indices
*/
+ // scalastyle:on line.size.limit
private def getSampleIds(n: Int, m: Int, r: Random): List[Int] = {
val indices = (n - m + 1 to n).foldLeft(Set.empty[Int]) {case (set, i) =>
val t = r.nextInt(i) + 1
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index 57f6f2f..dbeb970 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -422,8 +422,8 @@ private[spark] object UIUtils extends Logging {
* the whole string will rendered as a simple escaped text.
*
* Note: In terms of security, only anchor tags with root relative links are supported. So any
- * attempts to embed links outside Spark UI, or other tags like <script> will cause in the whole
- * description to be treated as plain text.
+ * attempts to embed links outside Spark UI, or other tags like <script> will cause in
+ * the whole description to be treated as plain text.
*
* @param desc the original job or stage description string, which may contain html tags.
* @param basePathUri with which to prepend the relative links; this is used when plainText is
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala b/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala
index 1326f09..00e0cf2 100644
--- a/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala
+++ b/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala
@@ -224,7 +224,7 @@ private[spark] object AccumulatorContext {
* Registers an [[AccumulatorV2]] created on the driver such that it can be used on the executors.
*
* All accumulators registered here can later be used as a container for accumulating partial
- * values across multiple tasks. This is what [[org.apache.spark.scheduler.DAGScheduler]] does.
+ * values across multiple tasks. This is what `org.apache.spark.scheduler.DAGScheduler` does.
* Note: if an accumulator is registered here, it should also be registered with the active
* context cleaner for cleanup so as to avoid memory leaks.
*
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/util/RpcUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/RpcUtils.scala b/core/src/main/scala/org/apache/spark/util/RpcUtils.scala
index e3b5883..46a5cb2 100644
--- a/core/src/main/scala/org/apache/spark/util/RpcUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/RpcUtils.scala
@@ -23,7 +23,7 @@ import org.apache.spark.rpc.{RpcAddress, RpcEndpointRef, RpcEnv, RpcTimeout}
private[spark] object RpcUtils {
/**
- * Retrieve a [[RpcEndpointRef]] which is located in the driver via its name.
+ * Retrieve a `RpcEndpointRef` which is located in the driver via its name.
*/
def makeDriverRef(name: String, conf: SparkConf, rpcEnv: RpcEnv): RpcEndpointRef = {
val driverHost: String = conf.get("spark.driver.host", "localhost")
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/util/StatCounter.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/StatCounter.scala b/core/src/main/scala/org/apache/spark/util/StatCounter.scala
index 4538136..1e02638 100644
--- a/core/src/main/scala/org/apache/spark/util/StatCounter.scala
+++ b/core/src/main/scala/org/apache/spark/util/StatCounter.scala
@@ -22,8 +22,8 @@ import org.apache.spark.annotation.Since
/**
* A class for tracking the statistics of a set of numbers (count, mean and variance) in a
* numerically robust way. Includes support for merging two StatCounters. Based on Welford
- * and Chan's [[http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance algorithms]]
- * for running variance.
+ * and Chan's <a href="http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance">
+ * algorithms</a> for running variance.
*
* @constructor Initialize the StatCounter with the given values.
*/
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala b/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
index d093e7b..60a6e82 100644
--- a/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
@@ -180,8 +180,8 @@ private[spark] object ThreadUtils {
// scalastyle:off awaitresult
/**
- * Preferred alternative to [[Await.result()]]. This method wraps and re-throws any exceptions
- * thrown by the underlying [[Await]] call, ensuring that this thread's stack trace appears in
+ * Preferred alternative to `Await.result()`. This method wraps and re-throws any exceptions
+ * thrown by the underlying `Await` call, ensuring that this thread's stack trace appears in
* logs.
*/
@throws(classOf[SparkException])
@@ -196,7 +196,7 @@ private[spark] object ThreadUtils {
}
/**
- * Calls [[Awaitable.result]] directly to avoid using `ForkJoinPool`'s `BlockingContext`, wraps
+ * Calls `Awaitable.result` directly to avoid using `ForkJoinPool`'s `BlockingContext`, wraps
* and re-throws any exceptions with nice stack track.
*
* Codes running in the user's thread may be in a thread of Scala ForkJoinPool. As concurrent
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/util/Utils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 748d729..f051860 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -1673,8 +1673,8 @@ private[spark] object Utils extends Logging {
}
/**
- * NaN-safe version of [[java.lang.Double.compare()]] which allows NaN values to be compared
- * according to semantics where NaN == NaN and NaN > any non-NaN double.
+ * NaN-safe version of `java.lang.Double.compare()` which allows NaN values to be compared
+ * according to semantics where NaN == NaN and NaN > any non-NaN double.
*/
def nanSafeCompareDoubles(x: Double, y: Double): Int = {
val xIsNan: Boolean = java.lang.Double.isNaN(x)
@@ -1687,8 +1687,8 @@ private[spark] object Utils extends Logging {
}
/**
- * NaN-safe version of [[java.lang.Float.compare()]] which allows NaN values to be compared
- * according to semantics where NaN == NaN and NaN > any non-NaN float.
+ * NaN-safe version of `java.lang.Float.compare()` which allows NaN values to be compared
+ * according to semantics where NaN == NaN and NaN > any non-NaN float.
*/
def nanSafeCompareFloats(x: Float, y: Float): Int = {
val xIsNan: Boolean = java.lang.Float.isNaN(x)
@@ -2354,7 +2354,7 @@ private[spark] object Utils extends Logging {
* A spark url (`spark://host:port`) is a special URI that its scheme is `spark` and only contains
* host and port.
*
- * @throws SparkException if `sparkUrl` is invalid.
+ * @note Throws `SparkException` if sparkUrl is invalid.
*/
def extractHostPortFromSparkUrl(sparkUrl: String): (String, Int) = {
try {
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala b/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
index 89b0874..da08661 100644
--- a/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
+++ b/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
@@ -148,7 +148,7 @@ private[spark] class ChunkedByteBuffer(var chunks: Array[ByteBuffer]) {
/**
* Reads data from a ChunkedByteBuffer.
*
- * @param dispose if true, [[ChunkedByteBuffer.dispose()]] will be called at the end of the stream
+ * @param dispose if true, `ChunkedByteBuffer.dispose()` will be called at the end of the stream
* in order to close any memory-mapped files which back the buffer.
*/
private class ChunkedByteBufferInputStream(
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/graphx/src/main/scala/org/apache/spark/graphx/Graph.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/Graph.scala b/graphx/src/main/scala/org/apache/spark/graphx/Graph.scala
index 922ec79..c55a588 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/Graph.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/Graph.scala
@@ -54,8 +54,8 @@ abstract class Graph[VD: ClassTag, ED: ClassTag] protected () extends Serializab
*
* @return an RDD containing the edges in this graph
*
- * @see [[Edge]] for the edge type.
- * @see [[Graph#triplets]] to get an RDD which contains all the edges
+ * @see `Edge` for the edge type.
+ * @see `Graph#triplets` to get an RDD which contains all the edges
* along with their vertex data.
*
*/
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/graphx/src/main/scala/org/apache/spark/graphx/GraphLoader.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/GraphLoader.scala b/graphx/src/main/scala/org/apache/spark/graphx/GraphLoader.scala
index f678e5f..add21f4 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/GraphLoader.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/GraphLoader.scala
@@ -32,7 +32,7 @@ object GraphLoader extends Logging {
* id and a target id. Skips lines that begin with `#`.
*
* If desired the edges can be automatically oriented in the positive
- * direction (source Id < target Id) by setting `canonicalOrientation` to
+ * direction (source Id < target Id) by setting `canonicalOrientation` to
* true.
*
* @example Loads a file in the following format:
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgeRDDImpl.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgeRDDImpl.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgeRDDImpl.scala
index 98e082c..faa9855 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgeRDDImpl.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgeRDDImpl.scala
@@ -41,7 +41,7 @@ class EdgeRDDImpl[ED: ClassTag, VD: ClassTag] private[graphx] (
/**
* If `partitionsRDD` already has a partitioner, use it. Otherwise assume that the
- * [[PartitionID]]s in `partitionsRDD` correspond to the actual partitions and create a new
+ * `PartitionID`s in `partitionsRDD` correspond to the actual partitions and create a new
* partitioner that allows co-partitioning with `partitionsRDD`.
*/
override val partitioner =
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
index f926984..feb3f47 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
@@ -28,7 +28,7 @@ import org.apache.spark.ml.linalg.{Vector, Vectors}
/**
* PageRank algorithm implementation. There are two implementations of PageRank implemented.
*
- * The first implementation uses the standalone [[Graph]] interface and runs PageRank
+ * The first implementation uses the standalone `Graph` interface and runs PageRank
* for a fixed number of iterations:
* {{{
* var PR = Array.fill(n)( 1.0 )
@@ -41,7 +41,7 @@ import org.apache.spark.ml.linalg.{Vector, Vectors}
* }
* }}}
*
- * The second implementation uses the [[Pregel]] interface and runs PageRank until
+ * The second implementation uses the `Pregel` interface and runs PageRank until
* convergence:
*
* {{{
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/graphx/src/main/scala/org/apache/spark/graphx/lib/SVDPlusPlus.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/SVDPlusPlus.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/SVDPlusPlus.scala
index bb2ffab..59fdd85 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/SVDPlusPlus.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/SVDPlusPlus.scala
@@ -42,7 +42,8 @@ object SVDPlusPlus {
/**
* Implement SVD++ based on "Factorization Meets the Neighborhood:
* a Multifaceted Collaborative Filtering Model",
- * available at [[http://public.research.att.com/~volinsky/netflix/kdd08koren.pdf]].
+ * available at <a href="http://public.research.att.com/~volinsky/netflix/kdd08koren.pdf">
+ * here</a>.
*
* The prediction rule is rui = u + bu + bi + qi*(pu + |N(u)|^^-0.5^^*sum(y)),
* see the details on page 6.
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/graphx/src/main/scala/org/apache/spark/graphx/lib/TriangleCount.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/TriangleCount.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/TriangleCount.scala
index 34e9e22..21b2296 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/TriangleCount.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/TriangleCount.scala
@@ -36,7 +36,7 @@ import org.apache.spark.graphx._
* self cycles and canonicalizes the graph to ensure that the following conditions hold:
* <ul>
* <li> There are no self edges</li>
- * <li> All edges are oriented src > dst</li>
+ * <li> All edges are oriented src > dst</li>
* <li> There are no duplicate edges</li>
* </ul>
* However, the canonicalization procedure is costly as it requires repartitioning the graph.
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib-local/src/main/scala/org/apache/spark/ml/stat/distribution/MultivariateGaussian.scala
----------------------------------------------------------------------
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/stat/distribution/MultivariateGaussian.scala b/mllib-local/src/main/scala/org/apache/spark/ml/stat/distribution/MultivariateGaussian.scala
index 0be2867..3167e0c 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/stat/distribution/MultivariateGaussian.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/stat/distribution/MultivariateGaussian.scala
@@ -28,7 +28,8 @@ import org.apache.spark.ml.linalg.{Matrices, Matrix, Vector, Vectors}
* This class provides basic functionality for a Multivariate Gaussian (Normal) Distribution. In
* the event that the covariance matrix is singular, the density will be computed in a
* reduced dimensional subspace under which the distribution is supported.
- * (see [[http://en.wikipedia.org/wiki/Multivariate_normal_distribution#Degenerate_case]])
+ * (see <a href="http://en.wikipedia.org/wiki/Multivariate_normal_distribution#Degenerate_case">
+ * here</a>)
*
* @param mean The mean vector of the distribution
* @param cov The covariance matrix of the distribution
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala b/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
index aa92edd..4b43a3a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
@@ -59,7 +59,7 @@ private[ml] trait PredictorParams extends Params
/**
* :: DeveloperApi ::
* Abstraction for prediction problems (regression and classification). It accepts all NumericType
- * labels and will automatically cast it to DoubleType in [[fit()]].
+ * labels and will automatically cast it to DoubleType in `fit()`.
*
* @tparam FeaturesType Type of features.
* E.g., [[org.apache.spark.mllib.linalg.VectorUDT]] for vector features.
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala
index 12b9732..527cb2d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala
@@ -239,7 +239,7 @@ object AttributeGroup {
}
}
- /** Creates an attribute group from a [[StructField]] instance. */
+ /** Creates an attribute group from a `StructField` instance. */
def fromStructField(field: StructField): AttributeGroup = {
require(field.dataType == new VectorUDT)
if (field.metadata.contains(ML_ATTR)) {
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala
index 27554ac..cc7e8bc 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala
@@ -98,7 +98,7 @@ sealed abstract class Attribute extends Serializable {
def toMetadata(): Metadata = toMetadata(Metadata.empty)
/**
- * Converts to a [[StructField]] with some existing metadata.
+ * Converts to a `StructField` with some existing metadata.
* @param existingMetadata existing metadata to carry over
*/
def toStructField(existingMetadata: Metadata): StructField = {
@@ -109,7 +109,7 @@ sealed abstract class Attribute extends Serializable {
StructField(name.get, DoubleType, nullable = false, newMetadata)
}
- /** Converts to a [[StructField]]. */
+ /** Converts to a `StructField`. */
def toStructField(): StructField = toStructField(Metadata.empty)
override def toString: String = toMetadataImpl(withType = true).toString
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index d07b4ad..fe29926 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -56,13 +56,13 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
/**
* Set threshold in binary classification, in range [0, 1].
*
- * If the estimated probability of class label 1 is > threshold, then predict 1, else 0.
+ * If the estimated probability of class label 1 is > threshold, then predict 1, else 0.
* A high threshold encourages the model to predict 0 more often;
* a low threshold encourages the model to predict 1 more often.
*
* Note: Calling this with threshold p is equivalent to calling `setThresholds(Array(1-p, p))`.
- * When [[setThreshold()]] is called, any user-set value for [[thresholds]] will be cleared.
- * If both [[threshold]] and [[thresholds]] are set in a ParamMap, then they must be
+ * When `setThreshold()` is called, any user-set value for `thresholds` will be cleared.
+ * If both `threshold` and `thresholds` are set in a ParamMap, then they must be
* equivalent.
*
* Default is 0.5.
@@ -101,12 +101,12 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
/**
* Get threshold for binary classification.
*
- * If [[thresholds]] is set with length 2 (i.e., binary classification),
+ * If `thresholds` is set with length 2 (i.e., binary classification),
* this returns the equivalent threshold: {{{1 / (1 + thresholds(0) / thresholds(1))}}}.
- * Otherwise, returns [[threshold]] if set, or its default value if unset.
+ * Otherwise, returns `threshold` if set, or its default value if unset.
*
* @group getParam
- * @throws IllegalArgumentException if [[thresholds]] is set to an array of length other than 2.
+ * @throws IllegalArgumentException if `thresholds` is set to an array of length other than 2.
*/
override def getThreshold: Double = {
checkThresholdConsistency()
@@ -122,13 +122,13 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
/**
* Set thresholds in multiclass (or binary) classification to adjust the probability of
- * predicting each class. Array must have length equal to the number of classes, with values > 0,
- * excepting that at most one value may be 0.
+ * predicting each class. Array must have length equal to the number of classes,
+ * with values > 0, excepting that at most one value may be 0.
* The class with largest value p/t is predicted, where p is the original probability of that
* class and t is the class's threshold.
*
- * Note: When [[setThresholds()]] is called, any user-set value for [[threshold]] will be cleared.
- * If both [[threshold]] and [[thresholds]] are set in a ParamMap, then they must be
+ * Note: When `setThresholds()` is called, any user-set value for `threshold` will be cleared.
+ * If both `threshold` and `thresholds` are set in a ParamMap, then they must be
* equivalent.
*
* @group setParam
@@ -141,8 +141,8 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
/**
* Get thresholds for binary or multiclass classification.
*
- * If [[thresholds]] is set, return its value.
- * Otherwise, if [[threshold]] is set, return the equivalent thresholds for binary
+ * If `thresholds` is set, return its value.
+ * Otherwise, if `threshold` is set, return the equivalent thresholds for binary
* classification: (1-threshold, threshold).
* If neither are set, throw an exception.
*
@@ -159,9 +159,9 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
}
/**
- * If [[threshold]] and [[thresholds]] are both set, ensures they are consistent.
+ * If `threshold` and `thresholds` are both set, ensures they are consistent.
*
- * @throws IllegalArgumentException if [[threshold]] and [[thresholds]] are not equivalent
+ * @throws IllegalArgumentException if `threshold` and `thresholds` are not equivalent
*/
protected def checkThresholdConsistency(): Unit = {
if (isSet(threshold) && isSet(thresholds)) {
@@ -207,7 +207,7 @@ class LogisticRegression @Since("1.2.0") (
/**
* Set the ElasticNet mixing parameter.
* For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.
- * For 0 < alpha < 1, the penalty is a combination of L1 and L2.
+ * For 0 < alpha < 1, the penalty is a combination of L1 and L2.
* Default is 0.0 which is an L2 penalty.
*
* @group setParam
@@ -294,7 +294,7 @@ class LogisticRegression @Since("1.2.0") (
override def getThresholds: Array[Double] = super.getThresholds
/**
- * Suggested depth for treeAggregate (>= 2).
+ * Suggested depth for treeAggregate (>= 2).
* If the dimensions of features or the number of partitions are large,
* this param could be adjusted to a larger size.
* Default is 2.
@@ -815,7 +815,7 @@ class LogisticRegressionModel private[spark] (
/**
* Predict label for the given feature vector.
- * The behavior of this can be adjusted using [[thresholds]].
+ * The behavior of this can be adjusted using `thresholds`.
*/
override protected def predict(features: Vector): Double = if (isMultinomial) {
super.predict(features)
@@ -1274,7 +1274,7 @@ class BinaryLogisticRegressionSummary private[classification] (
*
* The probability of the multinomial outcome $y$ taking on any of the K possible outcomes is:
*
- * <p><blockquote>
+ * <blockquote>
* $$
* P(y_i=0|\vec{x}_i, \beta) = \frac{e^{\vec{x}_i^T \vec{\beta}_0}}{\sum_{k=0}^{K-1}
* e^{\vec{x}_i^T \vec{\beta}_k}} \\
@@ -1283,7 +1283,7 @@ class BinaryLogisticRegressionSummary private[classification] (
* P(y_i=K-1|\vec{x}_i, \beta) = \frac{e^{\vec{x}_i^T \vec{\beta}_{K-1}}\,}{\sum_{k=0}^{K-1}
* e^{\vec{x}_i^T \vec{\beta}_k}}
* $$
- * </blockquote></p>
+ * </blockquote>
*
* The model coefficients $\beta = (\beta_0, \beta_1, \beta_2, ..., \beta_{K-1})$ become a matrix
* which has dimension of $K \times (N+1)$ if the intercepts are added. If the intercepts are not
@@ -1292,7 +1292,7 @@ class BinaryLogisticRegressionSummary private[classification] (
* Note that the coefficients in the model above lack identifiability. That is, any constant scalar
* can be added to all of the coefficients and the probabilities remain the same.
*
- * <p><blockquote>
+ * <blockquote>
* $$
* \begin{align}
* \frac{e^{\vec{x}_i^T \left(\vec{\beta}_0 + \vec{c}\right)}}{\sum_{k=0}^{K-1}
@@ -1302,7 +1302,7 @@ class BinaryLogisticRegressionSummary private[classification] (
* = \frac{e^{\vec{x}_i^T \vec{\beta}_0}}{\sum_{k=0}^{K-1} e^{\vec{x}_i^T \vec{\beta}_k}}
* \end{align}
* $$
- * </blockquote></p>
+ * </blockquote>
*
* However, when regularization is added to the loss function, the coefficients are indeed
* identifiable because there is only one set of coefficients which minimizes the regularization
@@ -1314,7 +1314,7 @@ class BinaryLogisticRegressionSummary private[classification] (
* The loss of objective function for a single instance of data (we do not include the
* regularization term here for simplicity) can be written as
*
- * <p><blockquote>
+ * <blockquote>
* $$
* \begin{align}
* \ell\left(\beta, x_i\right) &= -log{P\left(y_i \middle| \vec{x}_i, \beta\right)} \\
@@ -1322,14 +1322,14 @@ class BinaryLogisticRegressionSummary private[classification] (
* &= log\left(\sum_{k=0}^{K-1} e^{margins_k}\right) - margins_y
* \end{align}
* $$
- * </blockquote></p>
+ * </blockquote>
*
* where ${margins}_k = \vec{x}_i^T \vec{\beta}_k$.
*
* For optimization, we have to calculate the first derivative of the loss function, and a simple
* calculation shows that
*
- * <p><blockquote>
+ * <blockquote>
* $$
* \begin{align}
* \frac{\partial \ell(\beta, \vec{x}_i, w_i)}{\partial \beta_{j, k}}
@@ -1338,54 +1338,54 @@ class BinaryLogisticRegressionSummary private[classification] (
* &= x_{i, j} \cdot w_i \cdot multiplier_k
* \end{align}
* $$
- * </blockquote></p>
+ * </blockquote>
*
* where $w_i$ is the sample weight, $I_{y=k}$ is an indicator function
*
- * <p><blockquote>
+ * <blockquote>
* $$
* I_{y=k} = \begin{cases}
* 1 & y = k \\
* 0 & else
* \end{cases}
* $$
- * </blockquote></p>
+ * </blockquote>
*
* and
*
- * <p><blockquote>
+ * <blockquote>
* $$
* multiplier_k = \left(\frac{e^{\vec{x}_i \cdot \vec{\beta}_k}}{\sum_{k=0}^{K-1}
* e^{\vec{x}_i \cdot \vec{\beta}_k}} - I_{y=k}\right)
* $$
- * </blockquote></p>
+ * </blockquote>
*
* If any of margins is larger than 709.78, the numerical computation of multiplier and loss
* function will suffer from arithmetic overflow. This issue occurs when there are outliers in
* data which are far away from the hyperplane, and this will cause the failing of training once
- * infinity is introduced. Note that this is only a concern when max(margins) > 0.
+ * infinity is introduced. Note that this is only a concern when max(margins) > 0.
*
- * Fortunately, when max(margins) = maxMargin > 0, the loss function and the multiplier can easily
- * be rewritten into the following equivalent numerically stable formula.
+ * Fortunately, when max(margins) = maxMargin > 0, the loss function and the multiplier can
+ * easily be rewritten into the following equivalent numerically stable formula.
*
- * <p><blockquote>
+ * <blockquote>
* $$
* \ell\left(\beta, x\right) = log\left(\sum_{k=0}^{K-1} e^{margins_k - maxMargin}\right) -
* margins_{y} + maxMargin
* $$
- * </blockquote></p>
+ * </blockquote>
*
* Note that each term, $(margins_k - maxMargin)$ in the exponential is no greater than zero; as a
* result, overflow will not happen with this formula.
*
* For $multiplier$, a similar trick can be applied as the following,
*
- * <p><blockquote>
+ * <blockquote>
* $$
* multiplier_k = \left(\frac{e^{\vec{x}_i \cdot \vec{\beta}_k - maxMargin}}{\sum_{k'=0}^{K-1}
* e^{\vec{x}_i \cdot \vec{\beta}_{k'} - maxMargin}} - I_{y=k}\right)
* $$
- * </blockquote></p>
+ * </blockquote>
*
* @param bcCoefficients The broadcast coefficients corresponding to the features.
* @param bcFeaturesStd The broadcast standard deviation values of the features.
@@ -1513,7 +1513,7 @@ private class LogisticAggregator(
}
/**
- * When maxMargin > 0, the original formula could cause overflow.
+ * When maxMargin > 0, the original formula could cause overflow.
* We address this by subtracting maxMargin from all the margins, so it's guaranteed
* that all of the new margins will be smaller than zero to prevent arithmetic overflow.
*/
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
index 88fe7cb..1b45eaf 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
@@ -289,7 +289,6 @@ object MultilayerPerceptronClassifier
* @param uid uid
* @param layers array of layer sizes including input and output layers
* @param weights the weights of layers
- * @return prediction model
*/
@Since("1.5.0")
@Experimental
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
index f1a7676..a2ac700 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
@@ -60,16 +60,20 @@ private[ml] trait NaiveBayesParams extends PredictorParams with HasWeightCol {
final def getModelType: String = $(modelType)
}
+// scalastyle:off line.size.limit
/**
* Naive Bayes Classifiers.
* It supports Multinomial NB
- * ([[http://nlp.stanford.edu/IR-book/html/htmledition/naive-bayes-text-classification-1.html]])
+ * (see <a href="http://nlp.stanford.edu/IR-book/html/htmledition/naive-bayes-text-classification-1.html">
+ * here</a>)
* which can handle finitely supported discrete data. For example, by converting documents into
* TF-IDF vectors, it can be used for document classification. By making every vector a
* binary (0/1) data, it can also be used as Bernoulli NB
- * ([[http://nlp.stanford.edu/IR-book/html/htmledition/the-bernoulli-model-1.html]]).
+ * (see <a href="http://nlp.stanford.edu/IR-book/html/htmledition/the-bernoulli-model-1.html">
+ * here</a>).
* The input feature values must be nonnegative.
*/
+// scalastyle:on line.size.limit
@Since("1.5.0")
class NaiveBayes @Since("1.5.0") (
@Since("1.5.0") override val uid: String)
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
index 52345b0..907c73e 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
@@ -36,7 +36,7 @@ import org.apache.spark.sql.functions._
/**
- * [[http://en.wikipedia.org/wiki/Random_forest Random Forest]] learning algorithm for
+ * <a href="http://en.wikipedia.org/wiki/Random_forest">Random Forest</a> learning algorithm for
* classification.
* It supports both binary and multiclass labels, as well as both continuous and categorical
* features.
@@ -144,7 +144,7 @@ object RandomForestClassifier extends DefaultParamsReadable[RandomForestClassifi
}
/**
- * [[http://en.wikipedia.org/wiki/Random_forest Random Forest]] model for classification.
+ * <a href="http://en.wikipedia.org/wiki/Random_forest">Random Forest</a> model for classification.
* It supports both binary and multiclass labels, as well as both continuous and categorical
* features.
*
@@ -249,7 +249,7 @@ class RandomForestClassificationModel private[ml] (
* (Hastie, Tibshirani, Friedman. "The Elements of Statistical Learning, 2nd Edition." 2001.)
* and follows the implementation from scikit-learn.
*
- * @see [[DecisionTreeClassificationModel.featureImportances]]
+ * @see `DecisionTreeClassificationModel.featureImportances`
*/
@Since("1.5.0")
lazy val featureImportances: Vector = TreeEnsembleModel.featureImportances(trees, numFeatures)
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
index cf11ba3..c7a170d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
@@ -42,7 +42,7 @@ private[clustering] trait BisectingKMeansParams extends Params
with HasMaxIter with HasFeaturesCol with HasSeed with HasPredictionCol {
/**
- * The desired number of leaf clusters. Must be > 1. Default: 4.
+ * The desired number of leaf clusters. Must be > 1. Default: 4.
* The actual number could be smaller if there are no divisible leaf clusters.
* @group param
*/
@@ -55,8 +55,8 @@ private[clustering] trait BisectingKMeansParams extends Params
def getK: Int = $(k)
/**
- * The minimum number of points (if >= 1.0) or the minimum proportion
- * of points (if < 1.0) of a divisible cluster (default: 1.0).
+ * The minimum number of points (if >= 1.0) or the minimum proportion
+ * of points (if < 1.0) of a divisible cluster (default: 1.0).
* @group expertParam
*/
@Since("2.0.0")
@@ -208,9 +208,9 @@ object BisectingKMeansModel extends MLReadable[BisectingKMeansModel] {
* If bisecting all divisible clusters on the bottom level would result more than `k` leaf clusters,
* larger clusters get higher priority.
*
- * @see [[http://glaros.dtc.umn.edu/gkhome/fetch/papers/docclusterKDDTMW00.pdf
- * Steinbach, Karypis, and Kumar, A comparison of document clustering techniques,
- * KDD Workshop on Text Mining, 2000.]]
+ * @see <a href="http://glaros.dtc.umn.edu/gkhome/fetch/papers/docclusterKDDTMW00.pdf">
+ * Steinbach, Karypis, and Kumar, A comparison of document clustering techniques,
+ * KDD Workshop on Text Mining, 2000.</a>
*/
@Since("2.0.0")
@Experimental
@@ -296,7 +296,7 @@ object BisectingKMeans extends DefaultParamsReadable[BisectingKMeans] {
* :: Experimental ::
* Summary of BisectingKMeans.
*
- * @param predictions [[DataFrame]] produced by [[BisectingKMeansModel.transform()]].
+ * @param predictions `DataFrame` produced by `BisectingKMeansModel.transform()`.
* @param predictionCol Name for column of predicted clusters in `predictions`.
* @param featuresCol Name for column of features in `predictions`.
* @param k Number of clusters.
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/clustering/ClusteringSummary.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/ClusteringSummary.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/ClusteringSummary.scala
index 8b5f525..44e832b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/ClusteringSummary.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/ClusteringSummary.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.{DataFrame, Row}
* :: Experimental ::
* Summary of clustering algorithms.
*
- * @param predictions [[DataFrame]] produced by model.transform().
+ * @param predictions `DataFrame` produced by model.transform().
* @param predictionCol Name for column of predicted clusters in `predictions`.
* @param featuresCol Name for column of features in `predictions`.
* @param k Number of clusters.
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index 19998ca..7410934 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -44,7 +44,7 @@ private[clustering] trait GaussianMixtureParams extends Params with HasMaxIter w
with HasSeed with HasPredictionCol with HasProbabilityCol with HasTol {
/**
- * Number of independent Gaussians in the mixture model. Must be > 1. Default: 2.
+ * Number of independent Gaussians in the mixture model. Must be > 1. Default: 2.
* @group param
*/
@Since("2.0.0")
@@ -76,7 +76,7 @@ private[clustering] trait GaussianMixtureParams extends Params with HasMaxIter w
* @param weights Weight for each Gaussian distribution in the mixture.
* This is a multinomial probability distribution over the k Gaussians,
* where weights(i) is the weight for Gaussian i, and weights sum to 1.
- * @param gaussians Array of [[MultivariateGaussian]] where gaussians(i) represents
+ * @param gaussians Array of `MultivariateGaussian` where gaussians(i) represents
* the Multivariate Gaussian (Normal) Distribution for Gaussian i
*/
@Since("2.0.0")
@@ -374,7 +374,7 @@ object GaussianMixture extends DefaultParamsReadable[GaussianMixture] {
* :: Experimental ::
* Summary of GaussianMixture.
*
- * @param predictions [[DataFrame]] produced by [[GaussianMixtureModel.transform()]].
+ * @param predictions `DataFrame` produced by `GaussianMixtureModel.transform()`.
* @param predictionCol Name for column of predicted clusters in `predictions`.
* @param probabilityCol Name for column of predicted probability of each cluster
* in `predictions`.
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
index 152bd13..6e124eb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
@@ -42,7 +42,7 @@ private[clustering] trait KMeansParams extends Params with HasMaxIter with HasFe
with HasSeed with HasPredictionCol with HasTol {
/**
- * The number of clusters to create (k). Must be > 1. Note that it is possible for fewer than
+ * The number of clusters to create (k). Must be > 1. Note that it is possible for fewer than
* k clusters to be returned, for example, if there are fewer than k distinct points to cluster.
* Default: 2.
* @group param
@@ -72,7 +72,7 @@ private[clustering] trait KMeansParams extends Params with HasMaxIter with HasFe
/**
* Param for the number of steps for the k-means|| initialization mode. This is an advanced
- * setting -- the default of 2 is almost always enough. Must be > 0. Default: 2.
+ * setting -- the default of 2 is almost always enough. Must be > 0. Default: 2.
* @group expertParam
*/
@Since("1.5.0")
@@ -250,7 +250,7 @@ object KMeansModel extends MLReadable[KMeansModel] {
* :: Experimental ::
* K-means clustering with support for k-means|| initialization proposed by Bahmani et al.
*
- * @see [[http://dx.doi.org/10.14778/2180912.2180915 Bahmani et al., Scalable k-means++.]]
+ * @see <a href="http://dx.doi.org/10.14778/2180912.2180915">Bahmani et al., Scalable k-means++.</a>
*/
@Since("1.5.0")
@Experimental
@@ -346,7 +346,7 @@ object KMeans extends DefaultParamsReadable[KMeans] {
* :: Experimental ::
* Summary of KMeans.
*
- * @param predictions [[DataFrame]] produced by [[KMeansModel.transform()]].
+ * @param predictions `DataFrame` produced by `KMeansModel.transform()`.
* @param predictionCol Name for column of predicted clusters in `predictions`.
* @param featuresCol Name for column of features in `predictions`.
* @param k Number of clusters.
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
index 7773802..6032ab3 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
@@ -50,7 +50,7 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
with HasSeed with HasCheckpointInterval {
/**
- * Param for the number of topics (clusters) to infer. Must be > 1. Default: 10.
+ * Param for the number of topics (clusters) to infer. Must be > 1. Default: 10.
*
* @group param
*/
@@ -78,13 +78,13 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
* - EM
* - Currently only supports symmetric distributions, so all values in the vector should be
* the same.
- * - Values should be > 1.0
+ * - Values should be > 1.0
* - default = uniformly (50 / k) + 1, where 50/k is common in LDA libraries and +1 follows
* from Asuncion et al. (2009), who recommend a +1 adjustment for EM.
* - Online
- * - Values should be >= 0
+ * - Values should be >= 0
* - default = uniformly (1.0 / k), following the implementation from
- * [[https://github.com/Blei-Lab/onlineldavb]].
+ * <a href="https://github.com/Blei-Lab/onlineldavb">here</a>.
*
* @group param
*/
@@ -120,13 +120,13 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
*
* Optimizer-specific parameter settings:
* - EM
- * - Value should be > 1.0
+ * - Value should be > 1.0
* - default = 0.1 + 1, where 0.1 gives a small amount of smoothing and +1 follows
* Asuncion et al. (2009), who recommend a +1 adjustment for EM.
* - Online
- * - Value should be >= 0
+ * - Value should be >= 0
* - default = (1.0 / k), following the implementation from
- * [[https://github.com/Blei-Lab/onlineldavb]].
+ * <a href="https://github.com/Blei-Lab/onlineldavb">here</a>.
*
* @group param
*/
@@ -162,11 +162,11 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
* - Online LDA:
* Hoffman, Blei and Bach. "Online Learning for Latent Dirichlet Allocation."
* Neural Information Processing Systems, 2010.
- * [[http://www.cs.columbia.edu/~blei/papers/HoffmanBleiBach2010b.pdf]]
+ * See <a href="http://www.cs.columbia.edu/~blei/papers/HoffmanBleiBach2010b.pdf">here</a>
* - EM:
* Asuncion et al. "On Smoothing and Inference for Topic Models."
* Uncertainty in Artificial Intelligence, 2009.
- * [[http://arxiv.org/pdf/1205.2662.pdf]]
+ * See <a href="http://arxiv.org/pdf/1205.2662.pdf">here</a>
*
* @group param
*/
@@ -245,9 +245,9 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
* Fraction of the corpus to be sampled and used in each iteration of mini-batch gradient descent,
* in range (0, 1].
*
- * Note that this should be adjusted in synch with [[LDA.maxIter]]
+ * Note that this should be adjusted in synch with `LDA.maxIter`
* so the entire corpus is used. Specifically, set both so that
- * maxIterations * miniBatchFraction >= 1.
+ * maxIterations * miniBatchFraction >= 1.
*
* Note: This is the same as the `miniBatchFraction` parameter in
* [[org.apache.spark.mllib.clustering.OnlineLDAOptimizer]].
@@ -293,8 +293,8 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
* cause failures if a data partition is lost, so set this bit with care.
* Note that checkpoints will be cleaned up via reference counting, regardless.
*
- * See [[DistributedLDAModel.getCheckpointFiles]] for getting remaining checkpoints and
- * [[DistributedLDAModel.deleteCheckpointFiles]] for removing remaining checkpoints.
+ * See `DistributedLDAModel.getCheckpointFiles` for getting remaining checkpoints and
+ * `DistributedLDAModel.deleteCheckpointFiles` for removing remaining checkpoints.
*
* Default: true
*
@@ -431,7 +431,7 @@ sealed abstract class LDAModel private[ml] (
private[ml] def getEffectiveTopicConcentration: Double = getModel.topicConcentration
/**
- * The features for LDA should be a [[Vector]] representing the word counts in a document.
+ * The features for LDA should be a `Vector` representing the word counts in a document.
* The vector should be of length vocabSize, with counts for each term (word).
*
* @group setParam
@@ -650,7 +650,7 @@ object LocalLDAModel extends MLReadable[LocalLDAModel] {
* for each training document.
*
* @param oldLocalModelOption Used to implement [[oldLocalModel]] as a lazy val, but keeping
- * [[copy()]] cheap.
+ * `copy()` cheap.
*/
@Since("1.6.0")
@Experimental
@@ -701,7 +701,7 @@ class DistributedLDAModel private[ml] (
* - Even with [[logPrior]], this is NOT the same as the data log likelihood given the
* hyperparameters.
* - This is computed from the topic distributions computed during training. If you call
- * [[logLikelihood()]] on the same training dataset, the topic distributions will be computed
+ * `logLikelihood()` on the same training dataset, the topic distributions will be computed
* again, possibly giving different results.
*/
@Since("1.6.0")
@@ -719,7 +719,7 @@ class DistributedLDAModel private[ml] (
/**
* :: DeveloperApi ::
*
- * If using checkpointing and [[LDA.keepLastCheckpoint]] is set to true, then there may be
+ * If using checkpointing and `LDA.keepLastCheckpoint` is set to true, then there may be
* saved checkpoint files. This method is provided so that users can manage those files.
*
* Note that removing the checkpoints can cause failures if a partition is lost and is needed
@@ -804,13 +804,13 @@ object DistributedLDAModel extends MLReadable[DistributedLDAModel] {
*
* Input data (featuresCol):
* LDA is given a collection of documents as input data, via the featuresCol parameter.
- * Each document is specified as a [[Vector]] of length vocabSize, where each entry is the
+ * Each document is specified as a `Vector` of length vocabSize, where each entry is the
* count for the corresponding term (word) in the document. Feature transformers such as
* [[org.apache.spark.ml.feature.Tokenizer]] and [[org.apache.spark.ml.feature.CountVectorizer]]
* can be useful for converting text to word count vectors.
*
- * @see [[http://en.wikipedia.org/wiki/Latent_Dirichlet_allocation Latent Dirichlet allocation
- * (Wikipedia)]]
+ * @see <a href="http://en.wikipedia.org/wiki/Latent_Dirichlet_allocation">
+ * Latent Dirichlet allocation (Wikipedia)</a>
*/
@Since("1.6.0")
@Experimental
@@ -826,7 +826,7 @@ class LDA @Since("1.6.0") (
optimizeDocConcentration -> true, keepLastCheckpoint -> true)
/**
- * The features for LDA should be a [[Vector]] representing the word counts in a document.
+ * The features for LDA should be a `Vector` representing the word counts in a document.
* The vector should be of length vocabSize, with counts for each term (word).
*
* @group setParam
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/feature/DCT.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/DCT.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/DCT.scala
index 6ff36b3..682787a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/DCT.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/DCT.scala
@@ -32,7 +32,8 @@ import org.apache.spark.sql.types.DataType
* It returns a real vector of the same length representing the DCT. The return vector is scaled
* such that the transform matrix is unitary (aka scaled DCT-II).
*
- * More information on [[https://en.wikipedia.org/wiki/Discrete_cosine_transform#DCT-II Wikipedia]].
+ * More information on <a href="https://en.wikipedia.org/wiki/Discrete_cosine_transform#DCT-II">
+ * DCT-II in Discrete cosine transform (Wikipedia)</a>.
*/
@Since("1.5.0")
class DCT @Since("1.5.0") (@Since("1.5.0") override val uid: String)
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/feature/MinHash.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/MinHash.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/MinHash.scala
index d9d0f32..f37233e 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/MinHash.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/MinHash.scala
@@ -37,7 +37,8 @@ import org.apache.spark.sql.types.StructType
* where `k_i` is the i-th coefficient, and both `x` and `k_i` are from `Z_prime^*`
*
* Reference:
- * [[https://en.wikipedia.org/wiki/Perfect_hash_function Wikipedia on Perfect Hash Function]]
+ * <a href="https://en.wikipedia.org/wiki/Perfect_hash_function">
+ * Wikipedia on Perfect Hash Function</a>
*
* @param numEntries The number of entries of the hash functions.
* @param randCoefficients An array of random coefficients, each used by one hash function.
@@ -98,7 +99,7 @@ class MinHashModel private[ml] (
* as binary "1" values.
*
* References:
- * [[https://en.wikipedia.org/wiki/MinHash Wikipedia on MinHash]]
+ * <a href="https://en.wikipedia.org/wiki/MinHash">Wikipedia on MinHash</a>
*/
@Experimental
@Since("2.1.0")
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
index ccfb0ce..19978c9 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
@@ -78,11 +78,11 @@ private[feature] trait MinMaxScalerParams extends Params with HasInputCol with H
* statistics, which is also known as min-max normalization or Rescaling. The rescaled value for
* feature E is calculated as:
*
- * <p><blockquote>
+ * <blockquote>
* $$
* Rescaled(e_i) = \frac{e_i - E_{min}}{E_{max} - E_{min}} * (max - min) + min
* $$
- * </blockquote></p>
+ * </blockquote>
*
* For the case $E_{max} == E_{min}$, $Rescaled(e_i) = 0.5 * (max + min)$.
*
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala
index 25fb6be..4be17da 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala
@@ -30,10 +30,12 @@ import org.apache.spark.sql.types.DataType
/**
* Perform feature expansion in a polynomial space. As said in wikipedia of Polynomial Expansion,
- * which is available at [[http://en.wikipedia.org/wiki/Polynomial_expansion]], "In mathematics, an
- * expansion of a product of sums expresses it as a sum of products by using the fact that
- * multiplication distributes over addition". Take a 2-variable feature vector as an example:
- * `(x, y)`, if we want to expand it with degree 2, then we get `(x, x * x, y, x * y, y * y)`.
+ * which is available at
+ * <a href="http://en.wikipedia.org/wiki/Polynomial_expansion">Polynomial expansion (Wikipedia)</a>
+ * , "In mathematics, an expansion of a product of sums expresses it as a sum of products by using
+ * the fact that multiplication distributes over addition". Take a 2-variable feature vector
+ * as an example: `(x, y)`, if we want to expand it with degree 2, then we get
+ * `(x, x * x, y, x * y, y * y)`.
*/
@Since("1.4.0")
class PolynomialExpansion @Since("1.4.0") (@Since("1.4.0") override val uid: String)
@@ -76,11 +78,11 @@ class PolynomialExpansion @Since("1.4.0") (@Since("1.4.0") override val uid: Str
* (n + d choose d) (including 1 and first-order values). For example, let f([a, b, c], 3) be the
* function that expands [a, b, c] to their monomials of degree 3. We have the following recursion:
*
- * <p><blockquote>
+ * <blockquote>
* $$
* f([a, b, c], 3) &= f([a, b], 3) ++ f([a, b], 2) * c ++ f([a, b], 1) * c^2 ++ [c^3]
* $$
- * </blockquote></p>
+ * </blockquote>
*
* To handle sparsity, if c is zero, we can skip all monomials that contain it. We remember the
* current index and increment it properly for sparse input.
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/feature/RandomProjection.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RandomProjection.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/RandomProjection.scala
index 1b524c6..2bff59a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RandomProjection.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RandomProjection.scala
@@ -113,8 +113,8 @@ class RandomProjectionModel private[ml] (
*
* References:
*
- * 1. [[https://en.wikipedia.org/wiki/Locality-sensitive_hashing#Stable_distributions
- * Wikipedia on Stable Distributions]]
+ * 1. <a href="https://en.wikipedia.org/wiki/Locality-sensitive_hashing#Stable_distributions">
+ * Wikipedia on Stable Distributions</a>
*
* 2. Wang, Jingdong et al. "Hashing for similarity search: A survey." arXiv preprint
* arXiv:1408.2927 (2014).
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
index d76d556..8f125d8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
@@ -79,8 +79,8 @@ private[feature] trait StandardScalerParams extends Params with HasInputCol with
* statistics on the samples in the training set.
*
* The "unit std" is computed using the
- * [[https://en.wikipedia.org/wiki/Standard_deviation#Corrected_sample_standard_deviation
- * corrected sample standard deviation]],
+ * <a href="https://en.wikipedia.org/wiki/Standard_deviation#Corrected_sample_standard_deviation">
+ * corrected sample standard deviation</a>,
* which is computed as the square root of the unbiased sample variance.
*/
@Since("1.2.0")
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
index 0ced213..a558162 100755
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
@@ -32,7 +32,7 @@ import org.apache.spark.sql.types.{ArrayType, StringType, StructType}
* @note null values from input array are preserved unless adding null to stopWords
* explicitly.
*
- * @see [[http://en.wikipedia.org/wiki/Stop_words]]
+ * @see <a href="http://en.wikipedia.org/wiki/Stop_words">Stop words (Wikipedia)</a>
*/
@Since("1.5.0")
class StopWordsRemover @Since("1.5.0") (@Since("1.5.0") override val uid: String)
@@ -132,7 +132,8 @@ object StopWordsRemover extends DefaultParamsReadable[StopWordsRemover] {
* Loads the default stop words for the given language.
* Supported languages: danish, dutch, english, finnish, french, german, hungarian,
* italian, norwegian, portuguese, russian, spanish, swedish, turkish
- * @see [[http://anoncvs.postgresql.org/cvsweb.cgi/pgsql/src/backend/snowball/stopwords/]]
+ * @see <a href="http://anoncvs.postgresql.org/cvsweb.cgi/pgsql/src/backend/snowball/stopwords/">
+ * here</a>
*/
@Since("2.0.0")
def loadDefaultStopWords(language: String): Array[String] = {
http://git-wip-us.apache.org/repos/asf/spark/blob/51b1c155/mllib/src/main/scala/org/apache/spark/ml/feature/package.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/package.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/package.scala
index b94187a..5dd648a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/package.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/package.scala
@@ -84,6 +84,7 @@ import org.apache.spark.sql.DataFrame
* input dataset, while MLlib's feature transformers operate lazily on individual columns,
* which is more efficient and flexible to handle large and complex datasets.
*
- * @see [[http://scikit-learn.org/stable/modules/preprocessing.html scikit-learn.preprocessing]]
+ * @see <a href="http://scikit-learn.org/stable/modules/preprocessing.html">
+ * scikit-learn.preprocessing</a>
*/
package object feature
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org