You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by me...@apache.org on 2015/09/11 17:52:32 UTC
spark git commit: [SPARK-9773] [ML] [PySpark] Add Python API for
MultilayerPerceptronClassifier
Repository: spark
Updated Branches:
refs/heads/master b656e6134 -> b01b26260
[SPARK-9773] [ML] [PySpark] Add Python API for MultilayerPerceptronClassifier
Add Python API for ```MultilayerPerceptronClassifier```.
Author: Yanbo Liang <yb...@gmail.com>
Closes #8067 from yanboliang/SPARK-9773.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b01b2626
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b01b2626
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b01b2626
Branch: refs/heads/master
Commit: b01b26260625f0ba14e5f3010207666d62d93864
Parents: b656e61
Author: Yanbo Liang <yb...@gmail.com>
Authored: Fri Sep 11 08:52:28 2015 -0700
Committer: Xiangrui Meng <me...@databricks.com>
Committed: Fri Sep 11 08:52:28 2015 -0700
----------------------------------------------------------------------
.../MultilayerPerceptronClassifier.scala | 9 ++
python/pyspark/ml/classification.py | 132 ++++++++++++++++++-
2 files changed, 140 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b01b2626/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
index 82fc80c..5f60dea 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
@@ -17,6 +17,8 @@
package org.apache.spark.ml.classification
+import scala.collection.JavaConverters._
+
import org.apache.spark.annotation.Experimental
import org.apache.spark.ml.param.shared.{HasTol, HasMaxIter, HasSeed}
import org.apache.spark.ml.{PredictorParams, PredictionModel, Predictor}
@@ -182,6 +184,13 @@ class MultilayerPerceptronClassificationModel private[ml] (
private val mlpModel = FeedForwardTopology.multiLayerPerceptron(layers, true).getInstance(weights)
/**
+ * Returns layers in a Java List.
+ */
+ private[ml] def javaLayers: java.util.List[Int] = {
+ layers.toList.asJava
+ }
+
+ /**
* Predict label for the given features.
* This internal method is used to implement [[transform()]] and output [[predictionCol]].
*/
http://git-wip-us.apache.org/repos/asf/spark/blob/b01b2626/python/pyspark/ml/classification.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index 22bdd1b..88815e5 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -26,7 +26,8 @@ from pyspark.mllib.common import inherit_doc
__all__ = ['LogisticRegression', 'LogisticRegressionModel', 'DecisionTreeClassifier',
'DecisionTreeClassificationModel', 'GBTClassifier', 'GBTClassificationModel',
'RandomForestClassifier', 'RandomForestClassificationModel', 'NaiveBayes',
- 'NaiveBayesModel']
+ 'NaiveBayesModel', 'MultilayerPerceptronClassifier',
+ 'MultilayerPerceptronClassificationModel']
@inherit_doc
@@ -755,6 +756,135 @@ class NaiveBayesModel(JavaModel):
return self._call_java("theta")
+@inherit_doc
+class MultilayerPerceptronClassifier(JavaEstimator, HasFeaturesCol, HasLabelCol, HasPredictionCol,
+ HasMaxIter, HasTol, HasSeed):
+ """
+ Classifier trainer based on the Multilayer Perceptron.
+ Each layer has sigmoid activation function, output layer has softmax.
+ Number of inputs has to be equal to the size of feature vectors.
+ Number of outputs has to be equal to the total number of labels.
+
+ >>> from pyspark.mllib.linalg import Vectors
+ >>> df = sqlContext.createDataFrame([
+ ... (0.0, Vectors.dense([0.0, 0.0])),
+ ... (1.0, Vectors.dense([0.0, 1.0])),
+ ... (1.0, Vectors.dense([1.0, 0.0])),
+ ... (0.0, Vectors.dense([1.0, 1.0]))], ["label", "features"])
+ >>> mlp = MultilayerPerceptronClassifier(maxIter=100, layers=[2, 5, 2], blockSize=1, seed=11)
+ >>> model = mlp.fit(df)
+ >>> model.layers
+ [2, 5, 2]
+ >>> model.weights.size
+ 27
+ >>> testDF = sqlContext.createDataFrame([
+ ... (Vectors.dense([1.0, 0.0]),),
+ ... (Vectors.dense([0.0, 0.0]),)], ["features"])
+ >>> model.transform(testDF).show()
+ +---------+----------+
+ | features|prediction|
+ +---------+----------+
+ |[1.0,0.0]| 1.0|
+ |[0.0,0.0]| 0.0|
+ +---------+----------+
+ ...
+ """
+
+ # a placeholder to make it appear in the generated doc
+ layers = Param(Params._dummy(), "layers", "Sizes of layers from input layer to output layer " +
+ "E.g., Array(780, 100, 10) means 780 inputs, one hidden layer with 100 " +
+ "neurons and output layer of 10 neurons, default is [1, 1].")
+ blockSize = Param(Params._dummy(), "blockSize", "Block size for stacking input data in " +
+ "matrices. Data is stacked within partitions. If block size is more than " +
+ "remaining data in a partition then it is adjusted to the size of this " +
+ "data. Recommended size is between 10 and 1000, default is 128.")
+
+ @keyword_only
+ def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+ maxIter=100, tol=1e-4, seed=None, layers=None, blockSize=128):
+ """
+ __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+ maxIter=100, tol=1e-4, seed=None, layers=[1, 1], blockSize=128)
+ """
+ super(MultilayerPerceptronClassifier, self).__init__()
+ self._java_obj = self._new_java_obj(
+ "org.apache.spark.ml.classification.MultilayerPerceptronClassifier", self.uid)
+ self.layers = Param(self, "layers", "Sizes of layers from input layer to output layer " +
+ "E.g., Array(780, 100, 10) means 780 inputs, one hidden layer with " +
+ "100 neurons and output layer of 10 neurons, default is [1, 1].")
+ self.blockSize = Param(self, "blockSize", "Block size for stacking input data in " +
+ "matrices. Data is stacked within partitions. If block size is " +
+ "more than remaining data in a partition then it is adjusted to " +
+ "the size of this data. Recommended size is between 10 and 1000, " +
+ "default is 128.")
+ self._setDefault(maxIter=100, tol=1E-4, layers=[1, 1], blockSize=128)
+ kwargs = self.__init__._input_kwargs
+ self.setParams(**kwargs)
+
+ @keyword_only
+ def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
+ maxIter=100, tol=1e-4, seed=None, layers=None, blockSize=128):
+ """
+ setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
+ maxIter=100, tol=1e-4, seed=None, layers=[1, 1], blockSize=128)
+ Sets params for MultilayerPerceptronClassifier.
+ """
+ kwargs = self.setParams._input_kwargs
+ if layers is None:
+ return self._set(**kwargs).setLayers([1, 1])
+ else:
+ return self._set(**kwargs)
+
+ def _create_model(self, java_model):
+ return MultilayerPerceptronClassificationModel(java_model)
+
+ def setLayers(self, value):
+ """
+ Sets the value of :py:attr:`layers`.
+ """
+ self._paramMap[self.layers] = value
+ return self
+
+ def getLayers(self):
+ """
+ Gets the value of layers or its default value.
+ """
+ return self.getOrDefault(self.layers)
+
+ def setBlockSize(self, value):
+ """
+ Sets the value of :py:attr:`blockSize`.
+ """
+ self._paramMap[self.blockSize] = value
+ return self
+
+ def getBlockSize(self):
+ """
+ Gets the value of blockSize or its default value.
+ """
+ return self.getOrDefault(self.blockSize)
+
+
+class MultilayerPerceptronClassificationModel(JavaModel):
+ """
+ Model fitted by MultilayerPerceptronClassifier.
+ """
+
+ @property
+ def layers(self):
+ """
+ array of layer sizes including input and output layers.
+ """
+ return self._call_java("javaLayers")
+
+ @property
+ def weights(self):
+ """
+ vector of initial weights for the model that consists of the weights of layers.
+ """
+ return self._call_java("weights")
+
+
if __name__ == "__main__":
import doctest
from pyspark.context import SparkContext
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org