You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by fe...@apache.org on 2016/10/26 04:43:03 UTC
spark git commit: [SPARK-18007][SPARKR][ML] update SparkR MLP - add
initalWeights parameter
Repository: spark
Updated Branches:
refs/heads/master c329a568b -> 12b3e8d2e
[SPARK-18007][SPARKR][ML] update SparkR MLP - add initalWeights parameter
## What changes were proposed in this pull request?
update SparkR MLP, add initalWeights parameter.
## How was this patch tested?
test added.
Author: WeichenXu <We...@outlook.com>
Closes #15552 from WeichenXu123/mlp_r_add_initialWeight_param.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/12b3e8d2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/12b3e8d2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/12b3e8d2
Branch: refs/heads/master
Commit: 12b3e8d2e02788c3bebfecdd69755e94d80011c9
Parents: c329a56
Author: WeichenXu <We...@outlook.com>
Authored: Tue Oct 25 21:42:59 2016 -0700
Committer: Felix Cheung <fe...@apache.org>
Committed: Tue Oct 25 21:42:59 2016 -0700
----------------------------------------------------------------------
R/pkg/R/mllib.R | 14 ++++++++++----
R/pkg/inst/tests/testthat/test_mllib.R | 15 +++++++++++++++
.../ml/r/MultilayerPerceptronClassifierWrapper.scala | 9 ++++++++-
3 files changed, 33 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/12b3e8d2/R/pkg/R/mllib.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/mllib.R b/R/pkg/R/mllib.R
index b901307..bf182be 100644
--- a/R/pkg/R/mllib.R
+++ b/R/pkg/R/mllib.R
@@ -665,6 +665,8 @@ setMethod("predict", signature(object = "KMeansModel"),
#' @param tol convergence tolerance of iterations.
#' @param stepSize stepSize parameter.
#' @param seed seed parameter for weights initialization.
+#' @param initialWeights initialWeights parameter for weights initialization, it should be a
+#' numeric vector.
#' @param ... additional arguments passed to the method.
#' @return \code{spark.mlp} returns a fitted Multilayer Perceptron Classification Model.
#' @rdname spark.mlp
@@ -677,8 +679,9 @@ setMethod("predict", signature(object = "KMeansModel"),
#' df <- read.df("data/mllib/sample_multiclass_classification_data.txt", source = "libsvm")
#'
#' # fit a Multilayer Perceptron Classification Model
-#' model <- spark.mlp(df, blockSize = 128, layers = c(4, 5, 4, 3), solver = "l-bfgs",
-#' maxIter = 100, tol = 0.5, stepSize = 1, seed = 1)
+#' model <- spark.mlp(df, blockSize = 128, layers = c(4, 3), solver = "l-bfgs",
+#' maxIter = 100, tol = 0.5, stepSize = 1, seed = 1,
+#' initialWeights = c(0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 9, 9, 9, 9, 9))
#'
#' # get the summary of the model
#' summary(model)
@@ -695,7 +698,7 @@ setMethod("predict", signature(object = "KMeansModel"),
#' @note spark.mlp since 2.1.0
setMethod("spark.mlp", signature(data = "SparkDataFrame"),
function(data, layers, blockSize = 128, solver = "l-bfgs", maxIter = 100,
- tol = 1E-6, stepSize = 0.03, seed = NULL) {
+ tol = 1E-6, stepSize = 0.03, seed = NULL, initialWeights = NULL) {
if (is.null(layers)) {
stop ("layers must be a integer vector with length > 1.")
}
@@ -706,10 +709,13 @@ setMethod("spark.mlp", signature(data = "SparkDataFrame"),
if (!is.null(seed)) {
seed <- as.character(as.integer(seed))
}
+ if (!is.null(initialWeights)) {
+ initialWeights <- as.array(as.numeric(na.omit(initialWeights)))
+ }
jobj <- callJStatic("org.apache.spark.ml.r.MultilayerPerceptronClassifierWrapper",
"fit", data@sdf, as.integer(blockSize), as.array(layers),
as.character(solver), as.integer(maxIter), as.numeric(tol),
- as.numeric(stepSize), seed)
+ as.numeric(stepSize), seed, initialWeights)
new("MultilayerPerceptronClassificationModel", jobj = jobj)
})
http://git-wip-us.apache.org/repos/asf/spark/blob/12b3e8d2/R/pkg/inst/tests/testthat/test_mllib.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_mllib.R b/R/pkg/inst/tests/testthat/test_mllib.R
index c993157..33cc069 100644
--- a/R/pkg/inst/tests/testthat/test_mllib.R
+++ b/R/pkg/inst/tests/testthat/test_mllib.R
@@ -410,6 +410,21 @@ test_that("spark.mlp", {
model <- spark.mlp(df, layers = c(4, 5, 4, 3), maxIter = 10, seed = 10)
mlpPredictions <- collect(select(predict(model, mlpTestDF), "prediction"))
expect_equal(head(mlpPredictions$prediction, 12), c(1, 1, 1, 1, 2, 1, 2, 2, 1, 0, 0, 1))
+
+ # test initialWeights
+ model <- spark.mlp(df, layers = c(4, 3), maxIter = 2, initialWeights =
+ c(0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 9, 9, 9, 9, 9))
+ mlpPredictions <- collect(select(predict(model, mlpTestDF), "prediction"))
+ expect_equal(head(mlpPredictions$prediction, 12), c(1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1))
+
+ model <- spark.mlp(df, layers = c(4, 3), maxIter = 2, initialWeights =
+ c(0.0, 0.0, 0.0, 0.0, 0.0, 5.0, 5.0, 5.0, 5.0, 5.0, 9.0, 9.0, 9.0, 9.0, 9.0))
+ mlpPredictions <- collect(select(predict(model, mlpTestDF), "prediction"))
+ expect_equal(head(mlpPredictions$prediction, 12), c(1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1))
+
+ model <- spark.mlp(df, layers = c(4, 3), maxIter = 2)
+ mlpPredictions <- collect(select(predict(model, mlpTestDF), "prediction"))
+ expect_equal(head(mlpPredictions$prediction, 12), c(1, 1, 1, 1, 0, 1, 0, 2, 1, 0, 0, 1))
})
test_that("spark.naiveBayes", {
http://git-wip-us.apache.org/repos/asf/spark/blob/12b3e8d2/mllib/src/main/scala/org/apache/spark/ml/r/MultilayerPerceptronClassifierWrapper.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/MultilayerPerceptronClassifierWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/MultilayerPerceptronClassifierWrapper.scala
index 1067300..2193eb8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/r/MultilayerPerceptronClassifierWrapper.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/r/MultilayerPerceptronClassifierWrapper.scala
@@ -24,6 +24,7 @@ import org.json4s.jackson.JsonMethods._
import org.apache.spark.ml.{Pipeline, PipelineModel}
import org.apache.spark.ml.classification.{MultilayerPerceptronClassificationModel, MultilayerPerceptronClassifier}
+import org.apache.spark.ml.linalg.Vectors
import org.apache.spark.ml.util.{MLReadable, MLReader, MLWritable, MLWriter}
import org.apache.spark.sql.{DataFrame, Dataset}
@@ -58,7 +59,8 @@ private[r] object MultilayerPerceptronClassifierWrapper
maxIter: Int,
tol: Double,
stepSize: Double,
- seed: String
+ seed: String,
+ initialWeights: Array[Double]
): MultilayerPerceptronClassifierWrapper = {
// get labels and feature names from output schema
val schema = data.schema
@@ -73,6 +75,11 @@ private[r] object MultilayerPerceptronClassifierWrapper
.setStepSize(stepSize)
.setPredictionCol(PREDICTED_LABEL_COL)
if (seed != null && seed.length > 0) mlp.setSeed(seed.toInt)
+ if (initialWeights != null) {
+ require(initialWeights.length > 0)
+ mlp.setInitialWeights(Vectors.dense(initialWeights))
+ }
+
val pipeline = new Pipeline()
.setStages(Array(mlp))
.fit(data)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org